HttpClient 大量连接等待异常的处理
前几天要扫一批连接，因为涉及到ua的适配测试，所以选用了httpclient处理。
httpclient其实很好用，也用过几次，还曾经写过自动刷人人网人气的程序，不过对于它掌握还是处在比较浅显的时代
我这人比较懒，学习动力还是太弱，不逼到份上都懒得学习哎！
这次又用到这东西了就直接看了看api 就开始写了，不过写的时候发现一个问题。
开始跑100多个url的时候是没问题的，不过过了一定量数后，会大量的抛出如下异常
org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection at org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking(ConnPoolByRoute.java:417) at org.apache.http.impl.conn.tsccm.ConnPoolByRoute$1.getPoolEntry(ConnPoolByRoute.java:300) at org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager$1.getConnection(ThreadSafeClientConnManager.java:224) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:401) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:820) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:754) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:732) at com.ua.engine.Engine.httpService(Engine.java:97) at com.ua.engine.Engine.main(Engine.java:27)
对此很不解，还是了解的不够多啊，后来发现问题了，原来之前一直使用连接池中的一条连接进行链接访问，用的多了，连接到期，会造成连接等待的情况，
因为使用的HttpClient4 ,于是查了下资料，发现使用完每个连接后一定要及时的释放掉，不然上面的错会无限的报下去。
而释放连接只用InputStream 就能轻松搞定了
最后整体的代码如下
package com.ua.engine;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
public class Engine {
public static void main(String[] args) {
try {
HashMap<String,List<String>> map = readURL();
BufferedWriter bw = new BufferedWriter(new PrintWriter("D://furl/log.txt"));
List<String> weblist = map.get("web");
int i = 1;
for(String url :weblist){
System.out.println(i);
String temp = Engine.httpService("http://52.test.com"+url,"web")+" "+url;
System.out.println(temp);
bw.write(temp);
bw.write("\r\n");
i++;
}
List<String> touchlist = map.get("touch");
for(String url :touchlist){
System.out.println(i);
String temp = Engine.httpService("http://52.test.com"+url,"touch")+" "+url;
System.out.println(temp);
bw.write(temp);
bw.write("\r\n");
i++;
}
List<String> hdlist = map.get("hd");
for(String url :hdlist){
System.out.println(i);
String temp = Engine.httpService("http://52.test.com"+url,"hd")+" "+url;
System.out.println(temp);
bw.write(temp);
bw.write("\r\n");
i++;
}
List<String> cslist = map.get("cs");
for(String url :cslist){
System.out.println(i);
String temp = Engine.httpService("http://52.test.com"+url,"cs")+" "+url;
System.out.println(temp);
bw.write(temp);
bw.write("\r\n");
i++;
}
bw.flush();
bw.close();
}
catch (Exception e) {
e.printStackTrace();
}
}
private static HttpClient httpclient = null;
static {
HttpClientUtils hcu = new HttpClientUtils();
hcu.setConnTimeout(10000);
hcu.setSoTimeout(10000);
hcu.setBufSize(128 * 1024);
hcu.setAgent("");
hcu.setMaxRedirect(3);
hcu.setMaxTotal(500);
hcu.setMaxPerRoute(100);
httpclient = hcu.newClient();
}
public static String httpService(String url,String version) {
HttpGet get = new HttpGet(url);
if("web".equals(version)){
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.52 Safari/536.5");
}else if("touch".equals(version)){
get.setHeader("User-Agent", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16");
}else if("hd".equals(version)){
get.setHeader("User-Agent", "Mozilla/5.0 (iPad; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16");
}else {
get.setHeader("User-Agent", "lg-kp500 teleca/wap2.0 midp-2.0/cldc-1.1 untrusted/1.0 nokia7610 (3.0417.0ch) symbianos/7.0s series60/2.1 profile/midp-2.0 conf");
}
// HttpEntity entity = response.getEntity();
// System.out.println(EntityUtils.toString(entity));
HttpResponse response = null;
String code = "error "+url;
InputStream in = null;
try {
response = httpclient.execute(get);
HttpEntity entity =response.getEntity();
if( entity != null ){
in = entity.getContent(); //之前没使用这个造成了大量异常抛出，只要是
}
code = response.getStatusLine().getStatusCode()+" "+version+" "+url;
return code;
}
catch (Exception e) {
System.out.println(code);
e.printStackTrace();
}finally {
if(in!=null){
try {
in.close();
}
catch (IOException e) {
e.printStackTrace();
}
}
}
return code;
}
public static HashMap<String,List<String>> readURL() throws Exception{
List<String> weblist = new ArrayList<String>();
List<String> touchlist = new ArrayList<String>();
List<String> hdlist = new ArrayList<String>();
List<String> cslist = new ArrayList<String>();
BufferedReader br = new BufferedReader(new FileReader("D://furl/1.txt"));
while(br.ready()){
String url = br.readLine();
if(url.toLowerCase().indexOf(".html")>-1){
weblist.add(url);
}else if(url.toLowerCase().indexOf("touch")>-1){
touchlist.add(url);
}else if(url.toLowerCase().indexOf("/hd/")>-1){
hdlist.add(url);
}else {
cslist.add(url);
}
}
br = new BufferedReader(new FileReader("D://furl/url1.txt"));
while(br.ready()){
String url = br.readLine();
if(url.toLowerCase().indexOf(".html")>-1){
weblist.add(url);
}else if(url.toLowerCase().indexOf("touch")>-1){
touchlist.add(url);
}else if(url.toLowerCase().indexOf("/hd/")>-1){
hdlist.add(url);
}else {
cslist.add(url);
}
}test
br = new BufferedReader(new FileReader("D://furl/urljian1.txt"));
while(br.ready()){
String url = br.readLine();
if(url.toLowerCase().indexOf(".html")>-1){
weblist.add(url);
}else if(url.toLowerCase().indexOf("touch")>-1){
touchlist.add(url);
}else if(url.toLowerCase().indexOf("/hd/")>-1){
hdlist.add(url);
}else {
cslist.add(url);
}
}
HashMap<String,List<String>> map = new HashMap<String,List<String>>();
System.out.println(weblist.size());
System.out.println(touchlist.size());
System.out.println(hdlist.size());
System.out.println(cslist.size());
map.put("web", weblist);
map.put("touch", touchlist);
map.put("hd", hdlist);
map.put("cs", cslist);
return map;
}
}
package com.ua.engine;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;
public class HttpClientUtils {
private int connTimeout = 5000;
private int soTimeout = 5000;
private int bufSize = 128 * 1024;
private String agent = "";
private int maxRedirect = 3;
private int maxTotal = 500;
private int maxPerRoute = 100;
public HttpClient newClient() {
HttpParams params = new BasicHttpParams();
params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,
this.connTimeout);
params.setParameter(CoreConnectionPNames.SO_TIMEOUT, this.soTimeout);
params.setParameter(CoreConnectionPNames.SO_REUSEADDR, true);
params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
params.setParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE,
this.bufSize);
params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION,
HttpVersion.HTTP_1_1);
params.setParameter(CoreProtocolPNames.USER_AGENT, this.agent);
params.setParameter(ClientPNames.CONNECTION_MANAGER_FACTORY_CLASS_NAME,
"org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager");
params.setParameter(ClientPNames.MAX_REDIRECTS, this.maxRedirect);
ThreadSafeClientConnManager ccm = new ThreadSafeClientConnManager();
ccm.setMaxTotal(this.maxTotal);
ccm.setDefaultMaxPerRoute(this.maxPerRoute);
return new DefaultHttpClient(ccm, params);
}
public long getConnTimeout() {
return connTimeout;
}
public void setConnTimeout(int connTimeout) {
this.connTimeout = connTimeout;
}
public long getSoTimeout() {
return soTimeout;
}
public void setSoTimeout(int soTimeout) {
this.soTimeout = soTimeout;
}
public int getBufSize() {
return bufSize;
}
public void setBufSize(int bufSize) {
this.bufSize = bufSize;
}
public String getAgent() {
return agent;
}
public void setAgent(String agent) {
this.agent = agent;
}
public int getMaxRedirect() {
return maxRedirect;
}
public void setMaxRedirect(int maxRedirect) {
this.maxRedirect = maxRedirect;
}
public int getMaxTotal() {
return maxTotal;
}
public void setMaxTotal(int maxTotal) {
this.maxTotal = maxTotal;
}
public int getMaxPerRoute() {
return maxPerRoute;
}
public void setMaxPerRoute(int maxPerRoute) {
this.maxPerRoute = maxPerRoute;
}
}