httpClinet

httpClient多线程请求






使用httpClient可模拟请求Url获取资源，使用单线程的请求速度上会有一定的限制，参考了Apache给出的例子，自己做了测试实现多线程并发请求，以下代码需要HttpClient 4.2的包，可以在http://hc.apache.org/downloads.cgi下载

1、并发请求



复制代码

  1 package generate.httpclient;
  2
  3 import java.util.List;
  4 import java.util.concurrent.ExecutorService;
  5 import java.util.concurrent.Executors;
  6
  7 import org.apache.http.HttpEntity;
  8 import org.apache.http.HttpResponse;
  9 import org.apache.http.client.HttpClient;
10 import org.apache.http.client.methods.HttpGet;
11 import org.apache.http.conn.ClientConnectionManager;
12 import org.apache.http.conn.params.ConnManagerParams;
13 import org.apache.http.conn.scheme.PlainSocketFactory;
14 import org.apache.http.conn.scheme.Scheme;
15 import org.apache.http.conn.scheme.SchemeRegistry;
16 import org.apache.http.impl.client.DefaultHttpClient;
17 import org.apache.http.impl.conn.PoolingClientConnectionManager;
18 import org.apache.http.params.BasicHttpParams;
19 import org.apache.http.params.HttpConnectionParams;
20 import org.apache.http.params.HttpParams;
21 import org.apache.http.protocol.BasicHttpContext;
22 import org.apache.http.protocol.HttpContext;
23 import org.apache.http.util.EntityUtils;
24
25 public class ThreadPoolHttpClient {
26     // 线程池
27     private ExecutorService exe = null;
28     // 线程池的容量
29     private static final int POOL_SIZE = 20;
30     private HttpClient client = null;
31     String[] urls=null;
32     public ThreadPoolHttpClient(String[] urls){
33         this.urls=urls;
34     }
35     public void test() throws Exception {
36         exe = Executors.newFixedThreadPool(POOL_SIZE);
37         HttpParams params =new BasicHttpParams();
38         /* 从连接池中取连接的超时时间 */
39         ConnManagerParams.setTimeout(params, 1000);
40         /* 连接超时 */
41         HttpConnectionParams.setConnectionTimeout(params, 2000);
42         /* 请求超时 */
43         HttpConnectionParams.setSoTimeout(params, 4000);
44         SchemeRegistry schemeRegistry = new SchemeRegistry();
45         schemeRegistry.register(
46                 new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
47
48         //ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
49         PoolingClientConnectionManager cm=new PoolingClientConnectionManager(schemeRegistry);
50         cm.setMaxTotal(10);
51         final HttpClient httpClient = new DefaultHttpClient(cm,params);
52
53         // URIs to perform GETs on
54         final String[] urisToGet = urls;
55         /* 有多少url创建多少线程，url多时机子撑不住
56         // create a thread for each URI
57         GetThread[] threads = new GetThread[urisToGet.length];
58         for (int i = 0; i < threads.length; i++) {
59             HttpGet httpget = new HttpGet(urisToGet[i]);
60             threads[i] = new GetThread(httpClient, httpget);           
61         }
62         // start the threads
63         for (int j = 0; j < threads.length; j++) {
64             threads[j].start();
65         }
66
67         // join the threads，等待所有请求完成
68         for (int j = 0; j < threads.length; j++) {
69             threads[j].join();
70         }
71         使用线程池*/
72         for (int i = 0; i < urisToGet.length; i++) {
73             final int j=i;
74             System.out.println(j);
75             HttpGet httpget = new HttpGet(urisToGet[i]);
76             exe.execute( new GetThread(httpClient, httpget));
77         }
78        
79        
80         //创建线程池，每次调用POOL_SIZE
81         /*
82         for (int i = 0; i < urisToGet.length; i++) {
83             final int j=i;
84             System.out.println(j);
85             exe.execute(new Thread() {
86                 @Override
87                 public void run() {
88                     this.setName("threadsPoolClient"+j);
89                    
90                         try {
91                             this.sleep(100);
92                             System.out.println(j);
93                         } catch (InterruptedException e) {
94                             // TODO Auto-generated catch block
95                             e.printStackTrace();
96                         }
97                        
98                         HttpGet httpget = new HttpGet(urisToGet[j]);
99                         new GetThread(httpClient, httpget).get();
100                     }
101                    
102                    
103                
104             });
105         }
106        
107         */
108         //exe.shutdown();
109         System.out.println("Done");
110     }
111     static class GetThread extends Thread{
112        
113         private final HttpClient httpClient;
114         private final HttpContext context;
115         private final HttpGet httpget;
116        
117         public GetThread(HttpClient httpClient, HttpGet httpget) {
118             this.httpClient = httpClient;
119             this.context = new BasicHttpContext();
120             this.httpget = httpget;
121         }
122         @Override
123         public void run(){
124             this.setName("threadsPoolClient");
125             try {
126                 Thread.sleep(5000);
127             } catch (InterruptedException e) {
128                 // TODO Auto-generated catch block
129                 e.printStackTrace();
130             }
131             get();
132         }
133        
134         public void get() {
135             try {
136                 HttpResponse response = this.httpClient.execute(this.httpget, this.context);
137                 HttpEntity entity = response.getEntity();
138                 if (entity != null) {
139                     System.out.println(this.httpget.getURI()+": status"+response.getStatusLine().toString());
140                 }
141                 // ensure the connection gets released to the manager
142                 EntityUtils.consume(entity);
143             } catch (Exception ex) {
144                 this.httpget.abort();
145             }finally{
146                 httpget.releaseConnection();
147             }
148         }
149     }
150 }

复制代码



2、多线程异步请求



复制代码

  1 package generate.httpclient;
  2
  3 import java.io.ByteArrayOutputStream;
  4 import java.io.IOException;
  5 import java.io.InputStream;
  6 import java.util.ArrayList;
  7 import java.util.HashMap;
  8 import java.util.List;
  9 import java.util.Map;
10 import java.util.concurrent.CountDownLatch;
11
12 import org.apache.http.HttpResponse;
13 import org.apache.http.client.methods.HttpGet;
14 import org.apache.http.concurrent.FutureCallback;
15 import org.apache.http.impl.nio.client.DefaultHttpAsyncClient;
16 import org.apache.http.nio.client.HttpAsyncClient;
17 import org.apache.http.nio.reactor.IOReactorException;
18
19 public class AsynClient{
20     /**
21      * @param args
22      * @throws IOReactorException
23      * @throws InterruptedException
24      */
25     private List<String> urls;
26     private HandlerFailThread failHandler;
27     public AsynClient(List<String> list){
28         failHandler=new HandlerFailThread();
29         urls=list;
30     }
31     public Map<String,String> asynGet() throws IOReactorException,
32             InterruptedException {
33         final HttpAsyncClient httpclient = new DefaultHttpAsyncClient();
34         httpclient.start();
35         int urlLength=urls.size();
36         HttpGet[] requests = new HttpGet[urlLength];
37         int i=0;
38         for(String url : urls){
39             requests[i]=new HttpGet(url);
40             i++;
41         }
42         final CountDownLatch latch = new CountDownLatch(requests.length);
43         final Map<String, String> responseMap=new HashMap<String, String>();
44         try {
45             for (final HttpGet request : requests) {
46                 httpclient.execute(request, new FutureCallback<HttpResponse>() {
47
48                     public void completed(final HttpResponse response) {
49                         latch.countDown();
50                         responseMap.put(request.getURI().toString(), response.getStatusLine().toString());
51                         try {
52                             System.out.println(request.getRequestLine() + "->"
53                                     + response.getStatusLine()+"->");
54                             //+readInputStream(response.getEntity().getContent())
55                            
56                         } catch (IllegalStateException e) {
57                             failHandler.putFailUrl(request.getURI().toString(),
58                                     response.getStatusLine().toString());
59                             e.printStackTrace();
60                         } catch (Exception e) {
61                             failHandler.putFailUrl(request.getURI().toString(),
62                                     response.getStatusLine().toString());
63                             e.printStackTrace();
64                         }
65                     }
66
67                     public void failed(final Exception ex) {
68                         latch.countDown();
69                         ex.printStackTrace();
70                         failHandler.putFailUrl(request.getURI().toString(),
71                                 ex.getMessage());
72                     }
73
74                     public void cancelled() {
75                         latch.countDown();
76                     }
77
78                 });
79             }
80             System.out.println("Doing...");
81         } finally {
82             latch.await();
83             httpclient.shutdown();
84         }
85         System.out.println("Done");
86         failHandler.printFailUrl();
87         return responseMap;
88     }
89     private String readInputStream(InputStream input) throws IOException{
90         byte[] buffer = new byte[128];
91         int len = 0;
92         ByteArrayOutputStream bytes = new ByteArrayOutputStream();
93         while((len = input.read(buffer)) >= 0) {
94             bytes.write(buffer, 0, len);
95         }
96         return bytes.toString();
97     }
98     /**
99      * Test
100      * @param args
101      */
102     public static void main(String[] args) {
103         List<String> urls=new ArrayList<String>();
104         urls.add("http://127.0.0.1/examples/servlets/");
105         urls.add("http://127.0.0.1/examples/servlets/");
106         urls.add("http://127.0.0.1/examples/servlets/");
107         for(int i=0;i<10;i++){
108             urls.addAll(urls);
109         }
110         System.out.println(urls.size());
111         AsynClient client=new AsynClient(urls);
112         try {
113             client.asynGet();
114         } catch (IOReactorException e) {
115             e.printStackTrace();
116         } catch (InterruptedException e) {
117             e.printStackTrace();
118         }
119         System.out.println("done");
120     }
121 }

复制代码

创建一个线程记录失败的请求





复制代码

1 package generate.httpclient;
2
3 import java.util.HashMap;
4 import java.util.Map;
5
6 public class HandlerFailThread  extends Thread{
7     Map<String, String> failUrl=new HashMap<String, String>();
8     public void putFailUrl(String url,String status){
9         synchronized (failUrl) {
10             failUrl.put(url,status);
11         }
12     }
13     @Override
14     public void run() {
15         while(true){
16            
17         }
18     }
19     public void printFailUrl(){
20         for(Map.Entry<String, String> m: failUrl.entrySet()){
21             System.out.println("****fail:url:"+m.getKey()+ "  code :"+m.getValue());
22         }
23     }
24 }

复制代码

  异步请求，也可通过pool管理，例如

ConnectingIOReactor nio=new DefaultConnectingIOReactor();
  PoolingClientAsyncConnectionManager manager=new PoolingClientAsyncConnectionManager(nio);
  manager.setMaxTotal(1000);
  manager.setDefaultMaxPerRoute(100);
  HttpParams params=new BasicHttpParams();
  /* 连接超时 */
  HttpConnectionParams.setConnectionTimeout(params, 10000);
  /* 请求超时 */
  HttpConnectionParams.setSoTimeout(params, 60*1000);
  DefaultHttpAsyncClient.setDefaultHttpParams(params);
  final HttpAsyncClient httpclient = new DefaultHttpAsyncClient(manager);
  httpclient.start();