多线程-用多线程来并发发起htpp请求，并控制线程数量，并发控制

pom.xml

	4.0.0

	com.wei.plugs
	httpclient
	1.0-SNAPSHOT
	war

	
		org.springframework.boot
		spring-boot-starter-parent
		2.0.6.RELEASE
		 
	

	
		UTF-8
		UTF-8
		1.8
	

	
		
			org.apache.commons
			commons-lang3
		
		
			org.springframework.boot
			spring-boot-starter-web
		
		
			org.springframework.boot
			spring-boot-starter-test
			test
		
		
			org.apache.httpcomponents
			httpclient
		
	

处理类

package com.wei.plugs.processor;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

@Component("processor")
public class Processor {
	private Logger logger = LoggerFactory.getLogger(Processor.class);
	
	public void processOne() {
		logger.info("====processOne任务开始！！====");
		try {
			int concurrent = 50;//线程条数控制
			int pageSize = 1000;//每次获取数据的数量
			int querySpaceTime = 1000;//获取数据间隔时间
			int totalSize = 21235;//总数据量
			String uri = "https://www.baidu.com";
			ExecutorService executor = Executors.newCachedThreadPool();
			
			//总数据量
			List data = new ArrayList();
			for (int i = 0; i < totalSize; i++) {
				data.add(i);
			}
			final Semaphore semaphore = new Semaphore(concurrent);
			List list = null;
			//开始数据下标
			int start = 0;
			int end = 0;
			//循环 直到总数据被取完为止
			do {
				end = start + pageSize;
				if(end > data.size()) {
					end = data.size();
				}
				list = data.subList(start, end);
				final CountDownLatch countDownLatch = new CountDownLatch(list.size());
				if(list == null || list.size() == 0) {
					logger.info("没有查询到processOne需要处理的数据！");
					return;
				}
				
				for (int i = 0; i < list.size(); i++) {
					executor.execute(new ThreadOne(semaphore, countDownLatch,uri));
				}
				countDownLatch.await();//线程阻塞,直到锁为0才释放，继续向下执行
				start += pageSize;
				logger.info("lsit-size:===="+list.size());
				Thread.sleep(querySpaceTime);
			} while (list.size() == pageSize);
			executor.shutdown();
		} catch (Exception e) {
			logger.error("processOne处理数据时出现错误，错误信息：{}", e.getMessage());
		}
	}
}

线程类

package com.wei.plugs.processor;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.wei.plugs.util.HttpClientUtil;

/**
 * 线程
 */
public class ThreadOne implements Runnable {
	private Logger logger = LoggerFactory.getLogger(ThreadOne.class);
	private Semaphore semaphore;
	private CountDownLatch countDownLatch;
	private String uri;
	
	public ThreadOne(Semaphore semaphore,CountDownLatch countDownLatch,String uri) {
		super();
		this.semaphore = semaphore;
		this.countDownLatch =countDownLatch;
		this.uri = uri;
	}

	@Override
	public void run() {
		try {
			semaphore.acquire();
			// 发送请求
			HttpClientUtil.sendGet(uri);
			
			semaphore.release();
		} catch (Exception e) {
			logger.error("processOne线程内出现问题，错误信息：{}", e.getMessage());
		} finally{
			countDownLatch.countDown();
		}
	}
	
}

并发可用httpClient工具类

package com.wei.plugs.util;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * httpclient请求工具类
 * httpclient 连接池
 */
public class HttpClientUtil {
    private static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class);
    
    private static PoolingHttpClientConnectionManager cm = null;

    private static CloseableHttpClient httpClient = null;
    
    //默认content 类型
    private static final String DEFAULT_CONTENT_TYPE = "application/json";
    
    /**
     * 连接池设置
     */
    //长连接时间保持设置
    private static final int HTTP_DEFAULT_KEEP_TIME = 60000;
    
	/*
	 * 1、MaxtTotal是整个池子的大小； 2、DefaultMaxPerRoute是根据连接到的主机对MaxTotal的一个细分；比如：
	 * MaxtTotal=400 DefaultMaxPerRoute=200
	 * 而我只连接到https://www.baidu.com时，到这个主机的并发最多只有200；而不是400； 而我连接到https://www.baidu.com 和
	 * http://qq.com时，到每个主机的并发最多只有200；即加起来是400（但不能超过400）；
	 * 所以起作用的设置是DefaultMaxPerRoute。
	 */    
    private static final int DEFAULT_MAX_PER_ROUTE = 1000;
    //设置连接池的大小
    private static final int MAX_TOTAL = 1000;
    
    /**
     * 请求相关超时时间设置
     */
    //设置链接超时
    private static final int CONNECTION_TIMEOUT = 5000;
    //设置等待数据超时时间
    private static final int SOCKET_TIMEOUT = 15000;
    
    /**
     * 初始化连接池
     */
    private static synchronized void initPools() {
        if (httpClient == null) {
            cm = new PoolingHttpClientConnectionManager();
            cm.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
            cm.setMaxTotal(MAX_TOTAL);
            httpClient = HttpClients.custom().setKeepAliveStrategy(defaultStrategy).setConnectionManager(cm).build();
        }
    }

    /**
     * 长连接保持设置
     * Http connection keepAlive 设置
     */
    private static ConnectionKeepAliveStrategy defaultStrategy = new ConnectionKeepAliveStrategy() {
    	@Override
        public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
            HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
            int keepTime = HTTP_DEFAULT_KEEP_TIME;
            while (it.hasNext()) {
                HeaderElement he = it.nextElement();
                String param = he.getName();
                String value = he.getValue();
                if (value != null && param.equalsIgnoreCase("timeout")) {
                    try {
                        return Long.parseLong(value) * 1000;
                    } catch (Exception e) {
                        e.printStackTrace();
                        logger.error("format KeepAlive timeout exception, exception:" + e.toString());
                    }
                }
            }
            return keepTime * 1000;
        }
    };

    /**
     * 执行http post请求 默认采用Content-Type：application/json，Accept：application/json
     * @param uri 请求地址
     * @param data  请求数据
     * @return 返回是数据
     */
    public static String sendPost(String uri, String data) {
        long startTime = System.currentTimeMillis();
        HttpEntity httpEntity = null;
        HttpEntityEnclosingRequestBase method = null;
        String responseBody = "";
        try {
            if (httpClient == null) {
                initPools();
            }
            method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME, DEFAULT_CONTENT_TYPE);
            method.setEntity(new StringEntity(data));
            HttpContext context = HttpClientContext.create();
            CloseableHttpResponse httpResponse = httpClient.execute(method, context);
            httpEntity = httpResponse.getEntity();
            if (httpEntity != null) {
                responseBody = EntityUtils.toString(httpEntity, "UTF-8");
            }

        } catch (Exception e) {
            if(method != null){
                method.abort();
            }
            e.printStackTrace();
            logger.error(
                    "execute post request exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):"
                            + (System.currentTimeMillis() - startTime));
        } finally {
            if (httpEntity != null) {
                try {
                    EntityUtils.consumeQuietly(httpEntity);
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error(
                            "close response exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):"
                                    + (System.currentTimeMillis() - startTime));
                }
            }
        }
        return responseBody;
    }

    /**
     * 执行GET 请求
     * @param uri 请求链接
     * @return 请求结果
     */
    public static String sendGet(String uri) {
        long startTime = System.currentTimeMillis();
        HttpEntity httpEntity = null;
        HttpRequestBase method = null;
        String responseBody = "";
        try {
            if (httpClient == null) {
                initPools();
            }
            method = getRequest(uri, HttpGet.METHOD_NAME, DEFAULT_CONTENT_TYPE);
            HttpContext context = HttpClientContext.create();
            CloseableHttpResponse httpResponse = httpClient.execute(method, context);
            httpEntity = httpResponse.getEntity();
            if (httpEntity != null) {
                responseBody = EntityUtils.toString(httpEntity, "UTF-8");
                logger.info("请求URL: "+uri+"   返回状态码："+httpResponse.getStatusLine().getStatusCode());
            }
        } catch (Exception e) {
            if(method != null){
                method.abort();
            }
            e.printStackTrace();
            logger.error("execute get request exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):"
                            + (System.currentTimeMillis() - startTime));
        } finally {
            if (httpEntity != null) {
                try {
                    EntityUtils.consumeQuietly(httpEntity);
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("close response exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):"
                                    + (System.currentTimeMillis() - startTime));
                }
            }
        }
        return responseBody;
    }
    
    
    /**
     * 创建请求
     * @param uri 请求url
     * @param methodName 请求的方法类型
     * @param contentType contentType类型
     * @return 
     */
    private static HttpRequestBase getRequest(String uri, String methodName, String contentType) {
        HttpRequestBase method = null;
        
        RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(SOCKET_TIMEOUT).setConnectTimeout(CONNECTION_TIMEOUT)
                .setConnectionRequestTimeout(CONNECTION_TIMEOUT).setExpectContinueEnabled(false).build();

        if (HttpPut.METHOD_NAME.equalsIgnoreCase(methodName)) {
            method = new HttpPut(uri);
        } else if (HttpPost.METHOD_NAME.equalsIgnoreCase(methodName)) {
            method = new HttpPost(uri);
        } else if (HttpGet.METHOD_NAME.equalsIgnoreCase(methodName)) {
            method = new HttpGet(uri);
        } else { 
            method = new HttpPost(uri);
        }
        if (StringUtils.isBlank(contentType)) {
            contentType = DEFAULT_CONTENT_TYPE;
        }
        method.addHeader("Content-Type", contentType);
        method.addHeader("Accept", contentType);
        method.setConfig(requestConfig);
        return method;
    }
    
    public static CloseableHttpClient getHttpClient() {
        return httpClient;
    }

    public static PoolingHttpClientConnectionManager getHttpConnectionManager() {
        return cm;
    }
}