HttpClient高并发-httpClient连接池
我们项目中有一功能是用于加载静态考试问卷文件，以前一直使用的是用ajax加载静态文件方式，此方式使用的好处是比较简单，服务器也没太大的运行压力，但是也有明显的不足，例如存在跨域问题，还有文件的路径会在代码中明文显示，虽然可以加密，但是由于是前端的，对于技术人员破译也没什么难度，安全性很低。
最近就想把问卷文件加载方式给成后台获取，然后返回给前台显示，方式还是通过AJAX异步加载，这样修改后，至少通过页面的源代码没办法查找到静态问卷页面的路径了。由于是考试系统，存在集中考试情况，肯定要使用连接池提高服务器性能。
使用http连接池的大致流程 :
1.创建PoolingHttpClientConnectionManager实例
2.给manager设置参数
3.给manager设置重试策略
4.给manager设置连接管理策略
5.开启监控线程,及时关闭被服务器单向断开的连接
6.构建HttpClient实例
7.创建HttpPost/HttpGet实例,并设置参数
8.获取响应,做适当的处理
9.将用完的连接放回连接池
一、jar
二、代码
public class HmSyncHttpClientUtils {
private static Logger logger = LoggerFactory.getLogger(HmSyncHttpClientUtils.class);
private static final String DEFAULT_ENCODING = "UTF-8";// Charset.defaultCharset().name();
private static int connectPoolTimeout = 2000;// 设定从连接池获取可用连接的时间
private static int connectTimeout = 5000;// 建立连接超时时间
private static int socketTimeout = 5000;// 设置等待数据超时时间5秒钟 根据业务调整
private static int maxTotal = 100;// 连接池最大连接数
private static int maxPerRoute = 10;// 每个主机的并发
private static int maxRoute = 50;// 目标主机的最大连接数
private static CloseableHttpClient httpClient = null;
private final static Object syncLock = new Object();// 相当于线程锁,用于线程安全
private static PoolingHttpClientConnectionManager cm = null;
private static ScheduledExecutorService monitorExecutor;
private static boolean isShowUsePoolLog = true;
/**
* 获取HttpClient对象
*
* @author
* @date 2019年4月11日
* @param url
* @return
*/
public static CloseableHttpClient getHttpClient(final String url) {
String hostname = url.split("/")[2];
int port = 80;
if (hostname.contains(":")) {
final String[] arr = hostname.split(":");
hostname = arr[0];
port = Integer.parseInt(arr[1]);
}
if (HmSyncHttpClientUtils.httpClient == null) {
System.out.println("1****第一次创建httpClient");
// 多线程下多个线程同时调用getHttpClient容易导致重复创建httpClient对象的问题,所以加上了同步锁
synchronized (HmSyncHttpClientUtils.syncLock) {
if (HmSyncHttpClientUtils.httpClient == null) {
Integer maxTotal = HmSettingsResourceBundleUtils.getInteger("httpclient.pool.maxTotal");
if (maxTotal == null) {
maxTotal = HmSyncHttpClientUtils.maxTotal;
}
Integer maxPerRoute = HmSettingsResourceBundleUtils.getInteger("httpclient.pool.maxPerRoute");
if (maxPerRoute == null) {
maxPerRoute = HmSyncHttpClientUtils.maxPerRoute;
}
Integer maxRoute = HmSettingsResourceBundleUtils.getInteger("httpclient.pool.maxRoute");
if (maxRoute == null) {
maxRoute = HmSyncHttpClientUtils.maxRoute;
}
final Integer closeConnTimeout = HmSettingsResourceBundleUtils
.getInteger("httpclient.pool.closeConnTimeout");
final Long timeout = closeConnTimeout != null ? Long.valueOf(closeConnTimeout.toString()) : 5000;
System.out.println("2****第一次创建httpClient -->" + maxTotal);
// 开启监控线程,对异常和空闲线程进行关闭
HmSyncHttpClientUtils.monitorExecutor = Executors.newScheduledThreadPool(1);
HmSyncHttpClientUtils.monitorExecutor.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
// 关闭异常连接
HmSyncHttpClientUtils.cm.closeExpiredConnections();
// 关闭空闲的连接
HmSyncHttpClientUtils.cm.closeIdleConnections(timeout, TimeUnit.MILLISECONDS);
final PoolStats poolStats = HmSyncHttpClientUtils.cm.getTotalStats();
final int usePoolNum = poolStats.getAvailable() + poolStats.getLeased()
+ poolStats.getPending();
if (HmSyncHttpClientUtils.isShowUsePoolLog) {
HmSyncHttpClientUtils.logger.info("***********》关闭异常+空闲连接！ 空闲连接:"
+ poolStats.getAvailable() + " 持久连接:" + poolStats.getLeased() + " 最大连接数:"
+ poolStats.getMax() + " 阻塞连接数:" + poolStats.getPending());
}
if (usePoolNum == 0) {
HmSyncHttpClientUtils.isShowUsePoolLog = false;
} else {
HmSyncHttpClientUtils.isShowUsePoolLog = true;
}
}
}, timeout, timeout, TimeUnit.MILLISECONDS);
HmSyncHttpClientUtils.httpClient = HmSyncHttpClientUtils.createHttpClient(maxTotal, maxPerRoute,
maxRoute, hostname, port);
}
}
} else {
System.out.println("3****获取已有的httpClient");
}
return HmSyncHttpClientUtils.httpClient;
}
/**
* 创建HttpClient对象
*
* @author
* @date 2019年4月11日
* @param maxTotal
* 最大连接数
* @param maxPerRoute
* 每个主机的并发
* @param maxRoute
* 目标主机的最大并发，如果只有一台，可以和maxTotal一样
* @param hostname
* @param port
* @return
*/
private static CloseableHttpClient createHttpClient(
final int maxTotal,
final int maxPerRoute,
final int maxRoute,
final String hostname,
final int port) {
final ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
final LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
final Registry registry = RegistryBuilder. create()
.register("http", plainsf).register("https", sslsf).build();
HmSyncHttpClientUtils.cm = new PoolingHttpClientConnectionManager(registry);
// 将最大连接数增加
HmSyncHttpClientUtils.cm.setMaxTotal(maxTotal);
// 将每个路由基础的连接增加
HmSyncHttpClientUtils.cm.setDefaultMaxPerRoute(maxPerRoute);
final HttpHost httpHost = new HttpHost(hostname, port);
// 将目标主机的最大连接数增加
HmSyncHttpClientUtils.cm.setMaxPerRoute(new HttpRoute(httpHost), maxRoute);
// 请求重试处理
final HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(
final IOException exception,
final int executionCount,
final HttpContext context) {
if (executionCount >= 2) {// 如果已经重试了2次，就放弃
HmSyncHttpClientUtils.logger.info("*******》重试了2次，就放弃");
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
HmSyncHttpClientUtils.logger.info("*******》服务器丢掉连接，重试");
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
HmSyncHttpClientUtils.logger.info("*******》不要重试SSL握手异常");
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
HmSyncHttpClientUtils.logger.info("*******》 中断");
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
HmSyncHttpClientUtils.logger.info("*******》目标服务器不可达");
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
HmSyncHttpClientUtils.logger.info("*******》连接超时被拒绝");
return false;
}
if (exception instanceof SSLException) {// SSL握手异常
HmSyncHttpClientUtils.logger.info("*******》SSL握手异常");
return false;
}
final HttpClientContext clientContext = HttpClientContext.adapt(context);
final HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(HmSyncHttpClientUtils.cm)
.setRetryHandler(httpRequestRetryHandler).build();
return httpClient;
}
private static void setPostParams(final HttpPost httpost, final Map params) {
final List nvps = new ArrayList();
final Set keySet = params.keySet();
for (final String key : keySet) {
nvps.add(new BasicNameValuePair(key, params.get(key).toString()));
}
try {
httpost.setEntity(new UrlEncodedFormEntity(nvps, HmSyncHttpClientUtils.DEFAULT_ENCODING));
} catch (final Exception e) {
e.printStackTrace();
}
}
private static void setParams(final HttpRequestBase httpbase, final Map params) {
try {
if (params != null && params.size() > 0) {
final List nvps = new ArrayList();
final Set keySet = params.keySet();
for (final String key : keySet) {
nvps.add(new BasicNameValuePair(key, params.get(key).toString()));
}
final String param = EntityUtils
.toString(new UrlEncodedFormEntity(nvps, HmSyncHttpClientUtils.DEFAULT_ENCODING));
httpbase.setURI(new URI(httpbase.getURI().toString() + "?" + param));
}
} catch (final Exception e) {
e.printStackTrace();
}
}
private static String httpMethod(
final HttpRequestBase httpBase,
final String url,
final Map params) {
CloseableHttpResponse response = null;
HttpEntity entity = null;
try {
final RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(HmSyncHttpClientUtils.connectPoolTimeout)// 设定从连接池获取可用连接的时间
.setConnectTimeout(HmSyncHttpClientUtils.connectTimeout)// 设定连接服务器超时时间
.setSocketTimeout(HmSyncHttpClientUtils.socketTimeout)// 设定获取数据的超时时间
.build();
httpBase.setConfig(requestConfig);
// httpBase.setHeader("Connection", "close");
HmSyncHttpClientUtils.setParams(httpBase, params);
response = HmSyncHttpClientUtils.getHttpClient(url).execute(httpBase, HttpClientContext.create());
entity = response.getEntity();
return EntityUtils.toString(entity, HmSyncHttpClientUtils.DEFAULT_ENCODING);
} catch (final Exception e) {
e.printStackTrace();
} finally {
try {
// 关闭HttpEntity的流，如果手动关闭了InputStream in = entity.getContent();这个流，也可以不调用这个方法
EntityUtils.consume(entity);
if (response != null) {
response.close();
}
} catch (final Exception e) {
e.printStackTrace();
}
}
return null;
}
/**
* 模拟HTTPPOST提交
*
* @author
* @date 2019年4月8日
* @param url
* @param params
* @return
*/
public static String httpPost(final String url, final Map params) {
final HttpPost httpPost = new HttpPost(url);
return HmSyncHttpClientUtils.httpMethod(httpPost, url, params);
}
/**
* 模拟HTTPGET
*
* @author
* @date 2019年4月8日
* @param url
* @return
*/
public static String httpGet(final String url) {
final HttpGet httpGet = new HttpGet(url);
return HmSyncHttpClientUtils.httpMethod(httpGet, url, null);
}
} 对上面的代码做个简单的说明
1.代码中 HmSettingsResourceBundleUtils.getInteger()方法
此方法就是获取配置文件中相应的参数，此参数在程序中也有默认的设置，都是连接池的相关参数，大家看名称应该都能明白。
2.httpclients4.5.x版本直接调用ClosableHttpResponse.close()就能直接把连接放回连接池,而不是关闭连接,以前的版本貌似要调用其他方法才能把连接放回连接池
3.使用此方式的前提是要访问的地址是很少的，最好就是单一的。例如我这里的所有问卷保存在同一个主机下。
参考：
作者：PigPIgAutumn
链接：https://www.jianshu.com/p/c852cbcf3d68
来源：简书