http client遭遇Connection reset问题，以及解决方法
发送的时候使用了多线程连接池管理器
private MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
private HttpClient httpClient = new HttpClient(connectionManager);
httpClient.getHttpConnectionManager().getParams()
.setDefaultMaxConnectionsPerHost(WORKER_COUNT * 2);
httpClient.getHttpConnectionManager().getParams()
.setMaxTotalConnections(WORKER_COUNT * 5);
httpClient.getHttpConnectionManager().getParams().setSoTimeout(0);
httpClient.getHttpConnectionManager().getParams()
.setStaleCheckingEnabled(true);// 使用DefaultHttpMethodRetryHandler是希望在发送失败后能够自动重新发送httpClient.getHttpConnectionManager().getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());结果在实际的运行过程中发现不少的异常，导致数据没有发送成功：
[INFO ] 2014-08-31 13:00:35,804 -xxx.http.HttpMessageManager -430870 [pool-2-thread-126] INFO xxx.http.HttpMessageManager - io exception when send http.
java.net.SocketException: Connection reset
at java.net.SocketInputStream.read(SocketInputStream.java:168)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
at org.apache.commons.httpclient.HttpParser.readRawLine(Unknown Source)
at org.apache.commons.httpclient.HttpParser.readLine(Unknown Source)
at org.apache.commons.httpclient.HttpConnection.readLine(Unknown Source)
at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.readLine(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.readStatusLine(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.readResponse(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.execute(Unknown Source)
at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(Unknown Source)
at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(Unknown Source)
at org.apache.commons.httpclient.HttpClient.executeMethod(Unknown Source)
at org.apache.commons.httpclient.HttpClient.executeMethod(Unknown Source)
at xxx.http.HttpMessageManager$MessageSendWorker.sendHttp(HttpMessageManager.java:308)
at xxx.http.HttpMessageManager$MessageSendWorker.run(HttpMessageManager.java:253)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
[ERROR] 2014-08-31 13:00:35,805 - xxx.http.HttpMessageManager -430871 [pool-2-thread-126] ERROR xxx.http.HttpMessageManager - send http fail after retry 3 times, url=http://10.10.10.10/metrics/put
加粗部分的日志是我自己打印的，因为DefaultHttpMethodRetryHandler在发送失败时会重试3次。
分析发现，通过netstat -an命令查看，客户端会出现几十个CLOSE_WAIT状态的socket连接，server端会出现一些FIN_WAIT2状态的连接，很明显server端主动发起了断连。查看server端的keepalive超时设置为15000毫秒，达到了此阈值时，就会强制关闭连接。
server端断连是一个正常的操作，关键是http client端不是有重发机制吗？所以不应该最后没有发送成功呀
http Client下 多个线程共享connet连接池，每次发送时从池中获取一个连接：
public void executeMethod(final HttpMethod method) throws IOException, HttpException {
if (method == null) {
throw new IllegalArgumentException("Method may not be null");
}
// Link all parameter collections to form the hierarchy:
// Global -> HttpClient -> HostConfiguration -> HttpMethod
this.hostConfiguration.getParams().setDefaults(this.params);
method.getParams().setDefaults(this.hostConfiguration.getParams());
// Generate default request headers
Collection defaults = (Collection)this.hostConfiguration.getParams().
getParameter(HostParams.DEFAULT_HEADERS);
if (defaults != null) {
Iterator i = defaults.iterator();
while (i.hasNext()) {
method.addRequestHeader((Header)i.next());
}
}
try {
int maxRedirects = this.params.getIntParameter(HttpClientParams.MAX_REDIRECTS, 100);
for (int redirectCount = 0;;) {
// make sure the connection we have is appropriate
if (this.conn != null && !hostConfiguration.hostEquals(this.conn)) {
this.conn.setLocked(false);
this.conn.releaseConnection();
this.conn = null;
}
// get a connection, if we need one
if (this.conn == null) {
this.conn = connectionManager.getConnectionWithTimeout(
hostConfiguration,
this.params.getConnectionManagerTimeout()
);
this.conn.setLocked(true);
if (this.params.isAuthenticationPreemptive()
|| this.state.isAuthenticationPreemptive())
{
LOG.debug("Preemptively sending default basic credentials");
method.getHostAuthState().setPreemptive();
method.getHostAuthState().setAuthAttempted(true);
if (this.conn.isProxied() && !this.conn.isSecure()) {
method.getProxyAuthState().setPreemptive();
method.getProxyAuthState().setAuthAttempted(true);
}
}
}
authenticate(method);
executeWithRetry(method);
if (this.connectMethod != null) {
fakeResponse(method);
break;
}
。。。。。
} //end of retry loop
} finally {
if (this.conn != null) {
this.conn.setLocked(false);
}
// If the response has been fully processed, return the connection
// to the pool. Use this flag, rather than other tests (like
// responseStream == null), as subclasses, might reset the stream,
// for example, reading the entire response into a file and then
// setting the file as the stream.
if (
(releaseConnection || method.getResponseBodyAsStream() == null)
&& this.conn != null
) {
this.conn.releaseConnection();
}
}
} private void executeWithRetry(final HttpMethod method)
throws IOException, HttpException {
/** How many times did this transparently handle a recoverable exception? */
int execCount = 0;
// loop until the method is successfully processed, the retryHandler
// returns false or a non-recoverable exception is thrown
try {
while (true) {
execCount++;
try {
if (LOG.isTraceEnabled()) {
LOG.trace("Attempt number " + execCount + " to process request");
}
if (this.conn.getParams().isStaleCheckingEnabled()) {
this.conn.closeIfStale();
}
if (!this.conn.isOpen()) {
// this connection must be opened before it can be used
// This has nothing to do with opening a secure tunnel
this.conn.open();
if (this.conn.isProxied() && this.conn.isSecure()
&& !(method instanceof ConnectMethod)) {
// we need to create a secure tunnel before we can execute the real method
if (!executeConnect()) {
// abort, the connect method failed
return;
}
}
}
applyConnectionParams(method);
method.execute(state, this.conn);
break;
} catch (HttpException e) {
// filter out protocol exceptions which cannot be recovered from
throw e;
} catch (IOException e) {
LOG.debug("Closing the connection.");
this.conn.close();
。。。。
// ========================================
HttpMethodRetryHandler handler =
(HttpMethodRetryHandler)method.getParams().getParameter(
HttpMethodParams.RETRY_HANDLER);
if (handler == null) {
handler = new DefaultHttpMethodRetryHandler();
}
if (!handler.retryMethod(method, e, execCount)) {
LOG.debug("Method retry handler returned false. "
+ "Automatic recovery will not be attempted");
throw e;
}
。。。。
}
}
} catch (IOException e) {
if (this.conn.isOpen()) {
LOG.debug("Closing the connection.");
this.conn.close();
}
releaseConnection = true;
throw e;
} catch (RuntimeException e) {
if (this.conn.isOpen()) {
LOG.debug("Closing the connection.");
this.conn.close();
}
releaseConnection = true;
throw e;
}
} public boolean retryMethod(
final HttpMethod method,
final IOException exception,
int executionCount) {
if (method == null) {
throw new IllegalArgumentException("HTTP method may not be null");
}
if (exception == null) {
throw new IllegalArgumentException("Exception parameter may not be null");
}
// HttpMethod interface is the WORST thing ever done to HttpClient
if (method instanceof HttpMethodBase) { // 此处判断如果连接异常，则不重发
if (((HttpMethodBase)method).isAborted()) {
return false;
}
}
。。。。。
return false;
}一直以为连接异常时也会重发，当server端主动断开时，client会根据StaleCheckingEnabled的值检测连接是否正常，如果正常才发送，发送异常时会重发。今天才发现http client的重发机制也是有限制的，如果网络异常退出，就不会重发，而是直接抛出异常。
解决办法：
不适用httpclient的重发机制，自己搞一个：
// 重试3次
int retry = 0;
do {
try {
int status = httpClient.executeMethod(postMethod);
if (HttpStatus.SC_OK == status) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("send http success, url=" + url
+ ", content=" + message);
}
}else {
String rsp = HttpUtils
.getInputStreamAsString(postMethod
.getResponseBodyAsStream());
LOGGER.error("send http fail, status is: " + status
+ ", response is: " + rsp);
}
return;
} catch (HttpException e) {
postMethod.abort();
LOGGER.info("http exception when send http.", e);
} catch (IOException e) {
postMethod.abort();
LOGGER.info("io exception when send http.", e);
} finally {
postMethod.releaseConnection();
}
LOGGER.info("send http fail, retry:" + retry);
} while (++retry < 3);
// 如果到了第3次还未发送成功，则记录日志
LOGGER.error("send http fail after retry 3 times, url=" + url);