发送的时候使用了多线程连接池管理器
private MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); private HttpClient httpClient = new HttpClient(connectionManager); httpClient.getHttpConnectionManager().getParams() .setDefaultMaxConnectionsPerHost(WORKER_COUNT * 2); httpClient.getHttpConnectionManager().getParams() .setMaxTotalConnections(WORKER_COUNT * 5); httpClient.getHttpConnectionManager().getParams().setSoTimeout(0); httpClient.getHttpConnectionManager().getParams() .setStaleCheckingEnabled(true);
<span style="font-family: Arial, Helvetica, sans-serif;">// 使用DefaultHttpMethodRetryHandler是希望在发送失败后能够自动重新发送</span>
<span style="font-family: Arial, Helvetica, sans-serif;">httpClient.getHttpConnectionManager().getParams()</span>
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
[INFO ] 2014-08-31 13:00:35,804 -xxx.http.HttpMessageManager -430870 [pool-2-thread-126] INFO xxx.http.HttpMessageManager - io exception when send http.
java.net.SocketException: Connection reset
at java.net.SocketInputStream.read(SocketInputStream.java:168)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
at org.apache.commons.httpclient.HttpParser.readRawLine(Unknown Source)
at org.apache.commons.httpclient.HttpParser.readLine(Unknown Source)
at org.apache.commons.httpclient.HttpConnection.readLine(Unknown Source)
at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.readLine(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.readStatusLine(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.readResponse(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.execute(Unknown Source)
at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(Unknown Source)
at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(Unknown Source)
at org.apache.commons.httpclient.HttpClient.executeMethod(Unknown Source)
at org.apache.commons.httpclient.HttpClient.executeMethod(Unknown Source)
at xxx.http.HttpMessageManager$MessageSendWorker.sendHttp(HttpMessageManager.java:308)
at xxx.http.HttpMessageManager$MessageSendWorker.run(HttpMessageManager.java:253)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
[ERROR] 2014-08-31 13:00:35,805 - xxx.http.HttpMessageManager -430871 [pool-2-thread-126] ERROR xxx.http.HttpMessageManager - send http fail after retry 3 times, url=http://10.10.10.10/metrics/put
加粗部分的日志是我自己打印的,因为DefaultHttpMethodRetryHandler在发送失败时会重试3次。
分析发现,通过netstat -an命令查看,客户端会出现几十个CLOSE_WAIT状态的socket连接,server端会出现一些FIN_WAIT2状态的连接,很明显server端主动发起了断连。查看server端的keepalive超时设置为15000毫秒,达到了此阈值时,就会强制关闭连接。
server端断连是一个正常的操作,关键是http client端不是有重发机制吗?所以不应该最后没有发送成功呀
http Client下 多个线程共享connet连接池,每次发送时从池中获取一个连接:
public void executeMethod(final HttpMethod method) throws IOException, HttpException { if (method == null) { throw new IllegalArgumentException("Method may not be null"); } // Link all parameter collections to form the hierarchy: // Global -> HttpClient -> HostConfiguration -> HttpMethod this.hostConfiguration.getParams().setDefaults(this.params); method.getParams().setDefaults(this.hostConfiguration.getParams()); // Generate default request headers Collection defaults = (Collection)this.hostConfiguration.getParams(). getParameter(HostParams.DEFAULT_HEADERS); if (defaults != null) { Iterator i = defaults.iterator(); while (i.hasNext()) { method.addRequestHeader((Header)i.next()); } } try { int maxRedirects = this.params.getIntParameter(HttpClientParams.MAX_REDIRECTS, 100); for (int redirectCount = 0;;) { // make sure the connection we have is appropriate if (this.conn != null && !hostConfiguration.hostEquals(this.conn)) { this.conn.setLocked(false); this.conn.releaseConnection(); this.conn = null; } // get a connection, if we need one if (this.conn == null) { <strong>this.conn = connectionManager.getConnectionWithTimeout( hostConfiguration, this.params.getConnectionManagerTimeout() ); this.conn.setLocked(true); if (this.params.isAuthenticationPreemptive() || this.state.isAuthenticationPreemptive()) { LOG.debug("Preemptively sending default basic credentials"); method.getHostAuthState().setPreemptive(); method.getHostAuthState().setAuthAttempted(true); if (this.conn.isProxied() && !this.conn.isSecure()) { method.getProxyAuthState().setPreemptive(); method.getProxyAuthState().setAuthAttempted(true); } }</strong> } authenticate(method); <strong>executeWithRetry(method);</strong> if (this.connectMethod != null) { fakeResponse(method); break; } 。。。。。 } //end of retry loop } finally { if (this.conn != null) { this.conn.setLocked(false); } // If the response has been fully processed, return the connection // to the pool. Use this flag, rather than other tests (like // responseStream == null), as subclasses, might reset the stream, // for example, reading the entire response into a file and then // setting the file as the stream. if ( (releaseConnection || method.getResponseBodyAsStream() == null) && this.conn != null ) { this.conn.releaseConnection(); } } }
private void executeWithRetry(final HttpMethod method) throws IOException, HttpException { /** How many times did this transparently handle a recoverable exception? */ int execCount = 0; // loop until the method is successfully processed, the retryHandler // returns false or a non-recoverable exception is thrown try { while (true) { execCount++; try { if (LOG.isTraceEnabled()) { LOG.trace("Attempt number " + execCount + " to process request"); } if (this.conn.getParams().isStaleCheckingEnabled()) { this.conn.closeIfStale(); } if (!this.conn.isOpen()) { // this connection must be opened before it can be used // This has nothing to do with opening a secure tunnel this.conn.open(); if (this.conn.isProxied() && this.conn.isSecure() && !(method instanceof ConnectMethod)) { // we need to create a secure tunnel before we can execute the real method if (!executeConnect()) { // abort, the connect method failed return; } } } applyConnectionParams(method); method.execute(state, this.conn); break; } catch (HttpException e) { // filter out protocol exceptions which cannot be recovered from throw e; } catch (IOException e) { LOG.debug("Closing the connection."); this.conn.close(); 。。。。 <strong> // ======================================== HttpMethodRetryHandler handler = (HttpMethodRetryHandler)method.getParams().getParameter( HttpMethodParams.RETRY_HANDLER);</strong> if (handler == null) { handler = new DefaultHttpMethodRetryHandler(); } <strong> if (!handler.retryMethod(method, e, execCount)) {</strong> LOG.debug("Method retry handler returned false. " + "Automatic recovery will not be attempted"); throw e; } 。。。。 } } } catch (IOException e) { if (this.conn.isOpen()) { LOG.debug("Closing the connection."); this.conn.close(); } releaseConnection = true; throw e; } catch (RuntimeException e) { if (this.conn.isOpen()) { LOG.debug("Closing the connection."); this.conn.close(); } releaseConnection = true; throw e; } }
public boolean retryMethod( final HttpMethod method, final IOException exception, int executionCount) { if (method == null) { throw new IllegalArgumentException("HTTP method may not be null"); } if (exception == null) { throw new IllegalArgumentException("Exception parameter may not be null"); } // HttpMethod interface is the WORST thing ever done to HttpClient if (method instanceof HttpMethodBase) {
<strong> // 此处判断如果连接异常,则不重发 if (((HttpMethodBase)method).isAborted()) { return false; }</strong> } 。。。。。 return false; }
解决办法:
不适用httpclient的重发机制,自己搞一个:
// 重试3次 int retry = 0; do { try { int status = httpClient.executeMethod(postMethod); if (HttpStatus.SC_OK == status) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("send http success, url=" + url + ", content=" + message); } }else { String rsp = HttpUtils .getInputStreamAsString(postMethod .getResponseBodyAsStream()); LOGGER.error("send http fail, status is: " + status + ", response is: " + rsp); } return; } catch (HttpException e) { postMethod.abort(); LOGGER.info("http exception when send http.", e); } catch (IOException e) { postMethod.abort(); LOGGER.info("io exception when send http.", e); } finally { postMethod.releaseConnection(); } LOGGER.info("send http fail, retry:" + retry); } while (++retry < 3); // 如果到了第3次还未发送成功,则记录日志 LOGGER.error("send http fail after retry 3 times, url=" + url);