HttpClient4.3.3 使用样例—获取静态资源

编写不易，转载请注明（http://shihlei.iteye.com/blog/2067688）！

一 概述

 

    HttpClient是最常见的Http请求工具，很好的封装 Http请求，响应，Cookies操作，适合网络爬虫处理静态页面及其他资源。

    HttpCient能自动处理302客户端重定向，这个很喜欢！

    HttpClient4.3.3 API较之前的版本有不少改变，本文描述请求响应，设置cookies，设置代理等方法。

 

二 版本

 

<dependency>
	<groupId>org.apache.httpcomponents</groupId>
	<artifactId>httpclient</artifactId>
	<version>4.3.3</version>
</dependency>

 

 

三 样例

（1）请求响应

 

/**
 * 发送get请求
 * 
 * @param url
 * @throws Exception
 */
public byte[] sendGetRequest(String url) throws Exception {
	HttpClient httpClient = HttpClients.createDefault();
	HttpGet httpGet = new HttpGet(url);
	System.out.println("Send Get Request : " + url);
	return sendRequest(httpClient,httpGet);
}

/**
 * 发送post 请求
 * 
 * @param request
 * @throws Exception
 */
public byte[] sendPostRequest(String url, Map<String, String> params) throws Exception {
	HttpClient httpClient = HttpClients.createDefault();
	HttpPost httpPost = new HttpPost(url);
	// 设置参数
	if (params != null && params.size() > 0) {
		List<NameValuePair> nvps = new ArrayList<NameValuePair>();
		for (Entry<String, String> param : params.entrySet()) {
			nvps.add(new BasicNameValuePair(param.getKey(), param.getValue()));
		}
		httpPost.setEntity(new UrlEncodedFormEntity(nvps));
	}

	System.out.println("Send Post Request : " + url + " , " + params);
	return sendRequest(httpClient,httpPost);
}
// 底层请求获得响应数据
private byte[] sendRequest(HttpClient httpClient ,HttpRequestBase request) throws Exception {
	CloseableHttpResponse response = httpclient.execute(request);

	// 获取返回的状态列表
	StatusLine statusLine = response.getStatusLine();
	System.out.println("StatusLine : " + statusLine);

	if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
		try {
			// 获取response entity
			HttpEntity entity = response.getEntity();
			System.out.println("getContentType ： " + entity.getContentType().getValue());
			System.out.println("getContentEncoding ： " + entity.getContentEncoding().getValue());

			// 读取响应内容
			byte[] responseBody = EntityUtils.toByteArray(entity);

			// 关闭响应流
			EntityUtils.consume(entity);
			return responseBody;
		} finally {
			response.close();
		}
	}
	return null;
}

 

（2）超时和重定向

在execute前针对Request设置超时的Config

 

//设置超时
private void setTimeOut(HttpRequestBase request, int timeout) {
	RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(timeout)
			.setConnectTimeout(timeout).setSocketTimeout(timeout).setRedirectsEnabled(true).build();
	request.setConfig(requestConfig);
}

 

（3）代理

在创建HttpClient实例的时候，设置代理

//获得设置代理的httpClent实例
private CloseableHttpClient getHttpClientWithProxy(HttpProxy proxy){
	HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort());
	CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
	credentialsProvider.setCredentials(new AuthScope(proxy.getHost(), proxy.getPort()),
			new UsernamePasswordCredentials(proxy.getUser(), proxy.getPassword()));

	return HttpClients.custom().setProxy(proxyHost).setDefaultCredentialsProvider(credentialsProvider)
			.build();
}

 

这里有个辅助类：

 

package x.http.simple;


/**
 * Http代理
 * 
 * @author shilei
 * 
 */
public class HttpProxy {
	private String proxy = "http";
	private String host;
	private int port;
	private String user;
	private String password;
	public String getProxy() {
		return proxy;
	}
	public void setProxy(String proxy) {
		this.proxy = proxy;
	}
	public String getHost() {
		return host;
	}
	public void setHost(String host) {
		this.host = host;
	}
	public int getPort() {
		return port;
	}
	public void setPort(int port) {
		this.port = port;
	}
	public String getUser() {
		return user;
	}
	public void setUser(String user) {
		this.user = user;
	}
	public String getPassword() {
		return password;
	}
	public void setPassword(String password) {
		this.password = password;
	}
}

 

（4）Cookies：可以用于认证数据设置

1）全局Cookies设置：在创建HttpClient时

 

/**
 * 设置全局Cookies
 * 
 * @param cookies
 * @return
 */
public HttpClient setGlobeCookieHttpClient(Map<String, String> cookies) {
	HttpClient client = null;
	if (cookies != null && cookies.size() > 0) {
		BasicCookieStore cookieStore = new BasicCookieStore();

		for (Entry<String, String> c : cookies.entrySet()) {
			BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
			cookie.setDomain("sina.com");
			cookie.setPath("/");
			cookieStore.addCookie(cookie);
		}
		client = HttpClientBuilder.create().setDefaultCookieStore(cookieStore).build();
	} else {
		client = HttpClients.createDefault();
	}
	return client;
}

 

2）请求Cookies设置

法一：直接在请求头添加

 

/**
 * 设置Cookie方法1
 * 
 * @param request
 * @param cookies
 */
public void setRequestCookiesStr(HttpUriRequest request, Map<String, String> cookies) {
	StringBuilder cookieStr = new StringBuilder();
	if (cookies != null && cookies.size() > 0) {
		for (Entry<String, String> cookie : cookies.entrySet()) {
			cookieStr.append(cookie.getKey()).append('=').append(cookie.getValue()).append(';');
		}
		cookieStr.deleteCharAt(cookieStr.length() - 1);
		request.addHeader("Cookie", cookieStr.toString());
	}
}

 

法二：请求是通过context设置

 

/**
 * 设置Cookie 方法2
 * 
 * httpclient.execute(request, context) 方法进行添加
 * 
 * @param cookies
 * @return
 */
public HttpContext getetRequestCookieContext(Map<String, String> cookies) {
	HttpContext localContext = null;
	if (cookies != null && cookies.size() > 0) {
		BasicCookieStore cookieStore = new BasicCookieStore();

		for (Entry<String, String> c : cookies.entrySet()) {
			BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
			cookie.setDomain("sina.com");
			cookie.setPath("/");
			cookieStore.addCookie(cookie);
		}
		localContext = new BasicHttpContext();
		localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
	}
	return localContext;
}

 

3）读取响应的Cookies

 

/**
	 * 响应中获取cookie
	 * 
	 * @param response
	 */
	public void getResponseCookies(HttpResponse response) {
		//法1 
		HeaderIterator headerIter = response.headerIterator("Set-Cookie");
		while (headerIter.hasNext()) {
			Object obj = headerIter.next();
			if (obj != null) {
				String cookie = ((Header) headerIter.next()).getValue();

				String cookieRecord = cookie.substring(0, cookie.indexOf(';'));
				System.out.println(cookieRecord);
			}

		}

		//法2
		HeaderElementIterator headerElementIter = new BasicHeaderElementIterator(response.headerIterator("Set-Cookie"));
		while (headerElementIter.hasNext()) {
			HeaderElement elem = headerElementIter.nextElement();
			System.out.println("Element : " + elem.getName() + " = " + elem.getValue());
			NameValuePair[] params = elem.getParameters();
			for (int i = 0; i < params.length; i++) {
				System.out.println(" Params : " + params[i].getName() + " | " + params[i].getValue());
			}
		}
	}

 

 附录：完整代码

 

package x.http.simple.httpclient;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HeaderIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

import x.http.simple.HttpProxy;

public class HttpClientDemo {
	private CloseableHttpClient httpclient;

	public HttpClientDemo() {
		this(null);
	}

	/**
	 * 代理方式
	 * 
	 * @param proxy
	 */
	public HttpClientDemo(HttpProxy proxy) {
		if (proxy != null) {
			httpclient = getHttpClientWithProxy(proxy);
		} else {
			httpclient = HttpClients.createDefault();
		}
	}

	// 获得设置代理的httpClent实例
	private CloseableHttpClient getHttpClientWithProxy(HttpProxy proxy) {
		HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort());
		CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
		credentialsProvider.setCredentials(new AuthScope(proxy.getHost(), proxy.getPort()),
				new UsernamePasswordCredentials(proxy.getUser(), proxy.getPassword()));

		return HttpClients.custom().setProxy(proxyHost).setDefaultCredentialsProvider(credentialsProvider).build();
	}

	/**
	 * 发送get请求
	 * 
	 * @param url
	 * @throws Exception
	 */
	public byte[] sendGetRequest(String url) throws Exception {
		HttpGet httpGet = new HttpGet(url);
		System.out.println("Send Get Request : " + url);
		return sendRequest(httpGet);
	}

	/**
	 * 发送post 请求
	 * 
	 * @param request
	 * @throws Exception
	 */
	public byte[] sendPostRequest(String url, Map<String, String> params) throws Exception {
		HttpPost httpPost = new HttpPost(url);
		// 设置参数
		if (params != null && params.size() > 0) {
			List<NameValuePair> nvps = new ArrayList<NameValuePair>();
			for (Entry<String, String> param : params.entrySet()) {
				nvps.add(new BasicNameValuePair(param.getKey(), param.getValue()));
			}
			httpPost.setEntity(new UrlEncodedFormEntity(nvps));
		}

		System.out.println("Send Post Request : " + url + " , " + params);
		return sendRequest(httpPost);
	}

	// 底层请求打印页面
	private byte[] sendRequest(HttpRequestBase request) throws Exception {
		CloseableHttpResponse response = httpclient.execute(request);

		// 设置超时
		setTimeOut(request, 5000);

		// 获取返回的状态列表
		StatusLine statusLine = response.getStatusLine();
		System.out.println("StatusLine : " + statusLine);

		if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
			try {
				// 获取response entity
				HttpEntity entity = response.getEntity();
				System.out.println("getContentType ： " + entity.getContentType().getValue());
				System.out.println("getContentEncoding ： " + entity.getContentEncoding().getValue());

				// 读取响应内容
				byte[] responseBody = EntityUtils.toByteArray(entity);

				// 关闭响应流
				EntityUtils.consume(entity);
				return responseBody;
			} finally {
				response.close();
			}
		}
		return null;
	}

	// 设置超时重定向
	private void setTimeOut(HttpRequestBase request, int timeout) {
		RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(timeout)
				.setConnectTimeout(timeout).setSocketTimeout(timeout).setRedirectsEnabled(true).build();
		request.setConfig(requestConfig);
	}

	/**
	 * 设置全局Cookies
	 * 
	 * @param cookies
	 * @return
	 */
	public HttpClient setGlobeCookieHttpClient(Map<String, String> cookies) {
		HttpClient client = null;
		if (cookies != null && cookies.size() > 0) {
			BasicCookieStore cookieStore = new BasicCookieStore();

			for (Entry<String, String> c : cookies.entrySet()) {
				BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
				cookie.setDomain("sina.com");
				cookie.setPath("/");
				cookieStore.addCookie(cookie);
			}
			client = HttpClientBuilder.create().setDefaultCookieStore(cookieStore).build();
		} else {
			client = HttpClients.createDefault();
		}
		return client;
	}

	/**
	 * 设置Cookie方法1
	 * 
	 * @param request
	 * @param cookies
	 */
	public void setRequestCookiesStr(HttpUriRequest request, Map<String, String> cookies) {
		StringBuilder cookieStr = new StringBuilder();
		if (cookies != null && cookies.size() > 0) {
			for (Entry<String, String> cookie : cookies.entrySet()) {
				cookieStr.append(cookie.getKey()).append('=').append(cookie.getValue()).append(';');
			}
			cookieStr.deleteCharAt(cookieStr.length() - 1);
			request.addHeader("Cookie", cookieStr.toString());
		}
	}

	/**
	 * 设置Cookie 方法2
	 * 
	 * httpclient.execute(request, context) 方法进行添加
	 * 
	 * @param cookies
	 * @return
	 */
	public HttpContext getetRequestCookieContext(Map<String, String> cookies) {
		HttpContext localContext = null;
		if (cookies != null && cookies.size() > 0) {
			BasicCookieStore cookieStore = new BasicCookieStore();

			for (Entry<String, String> c : cookies.entrySet()) {
				BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
				cookie.setDomain("sina.com");
				cookie.setPath("/");
				cookieStore.addCookie(cookie);
			}
			localContext = new BasicHttpContext();
			localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
		}
		return localContext;
	}

	/**
	 * 响应中获取cookie
	 * 
	 * @param response
	 */
	public void getResponseCookies(HttpResponse response) {
		// 法1
		HeaderIterator headerIter = response.headerIterator("Set-Cookie");
		while (headerIter.hasNext()) {
			Object obj = headerIter.next();
			if (obj != null) {
				String cookie = ((Header) headerIter.next()).getValue();

				String cookieRecord = cookie.substring(0, cookie.indexOf(';'));
				System.out.println(cookieRecord);
			}

		}

		// 法2
		HeaderElementIterator headerElementIter = new BasicHeaderElementIterator(response.headerIterator("Set-Cookie"));
		while (headerElementIter.hasNext()) {
			HeaderElement elem = headerElementIter.nextElement();
			System.out.println("Element : " + elem.getName() + " = " + elem.getValue());
			NameValuePair[] params = elem.getParameters();
			for (int i = 0; i < params.length; i++) {
				System.out.println(" Params : " + params[i].getName() + " | " + params[i].getValue());
			}
		}
	}

	public void shutdown() throws Exception {
		httpclient.close();
	}

	public static void main(String[] args) throws Exception {
		String url = "http://www.google.com.hk";

		HttpClientDemo httpclient = new HttpClientDemo();

		byte[] getResponse = httpclient.sendGetRequest(url);
		System.out.println("Get Body : " + new String(getResponse, "utf-8"));

		byte[] postResponse = httpclient.sendPostRequest(url, null);
		System.out.println("Get Body : " + new String(postResponse, "utf-8"));

		httpclient.shutdown();
	}
}