HttpClient4+使用总结
Get和Post
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.util.zip.GZIPInputStream;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.protocol.HttpContext;
import org.apache.log4j.Logger;
import com.sikaijian.html_utils.encode.EncodeDetector;
/**
* httpclient工具
*
* @author sikaijian
*
*/
public class HttpclientUtil {
/**
* 执行一个请求
*
* @param req
* 请求
* @param httpcontext
* 请求上下文（可以保持会话）
* @return 结果字符串
*/
public String doRequest(HttpUriRequest req, HttpContext httpcontext) {
String resp = null;
HttpClient client = HttpClientFactory.getHttpClient();
try {
HttpResponse response = client.execute(req, httpcontext);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
HttpEntity entity = response.getEntity();
InputStream in = null;
ByteArrayOutputStream bout = new ByteArrayOutputStream();
try {
in = entity.getContent();
in = decompressStream(in);
int readed = -1;
byte[] arr = new byte[1024];
while (-1 != (readed = in.read(arr))) {
bout.write(arr, 0, readed);
}
byte[] data = bout.toByteArray();
String enc = EncodeDetector.getEncoding(data);
resp = new String(data, enc);
} finally {
bout.close();
if (null != in) {
in.close();
}
}
}
} catch (ClientProtocolException e) {
log.error(ExceptionUtil.getDetail(e));
} catch (IOException e) {
log.error(ExceptionUtil.getDetail(e));
} catch (Exception e) {
log.error(ExceptionUtil.getDetail(e));
} finally {
client = null;
}
return resp;
}
/**
* 判断流是否是gzip流，如果是，则解压该gzip流并返回
*
* @param input
* 输入流
* @return a GZIPInputStream
* @throws IOException
*/
public static InputStream decompressStream(InputStream input)
throws IOException {
PushbackInputStream pb = new PushbackInputStream(input, 2);
byte[] signature = new byte[2];
pb.read(signature);
pb.unread(signature);
if (signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b) {
return new GZIPInputStream(pb);
} else {
return pb;
}
}
private static Logger log = Logger.getLogger(HttpclientUtil.class);
}
Form提交
public ResponseState doLogin(String userName, String password){
ResponseState rs = new ResponseState(false, "???xxx");
HttpPost req = new HttpPost(WarnContext.webRoot + "/ads/doLogon.do ");
List <NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("userName", userName));
params.add(new BasicNameValuePair("password", password));
try {
req.setEntity(new UrlEncodedFormEntity(params,HTTP.UTF_8));
rs = netHelper.doRequest(req);
} catch (UnsupportedEncodingException e) {
Log.e("login", "Login Fail: " + e.getMessage());
}
return rs;
}
HttpContext
使用CookieStore存储相关Cookie，包括JSESSIONID（可以做登录）。
HttpClient httpClient = new DefaultHttpClient(); CookieStore cookieStore = new BasicCookieStore(); HttpContext httpContext = new BasicHttpContext(); httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore); // ... // request with httpContext which has sessionId HttpResponse response1 = httpClient.execute(method1, httpContext); // ... HttpResponse response2 = httpClient.execute(method2, httpContext); // ...
HttpPost httppost = new HttpPost(postData);
CookieStore cookieStore = new BasicCookieStore();
BasicClientCookie cookie = new BasicClientCookie("JSESSIONID", getSessionId());
cookieStore.addCookie(cookie);
连接池
import org.apache.http.HttpHost;
import org.apache.http.client.HttpClient;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
/**
* httpClient 工厂类
* @author sikaijian
*
*/
public class HttpClientFactory {
private static PoolingClientConnectionManager cm;
private static HttpParams params;
private static boolean inited = false;
private static void init(){
if(!inited){
// 连接池设置
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
cm = new PoolingClientConnectionManager(schemeRegistry);
cm.setMaxTotal(200); // 连接池里的最大连接数
cm.setDefaultMaxPerRoute(20); // 每个路由的默认最大连接数
HttpHost localhost = new HttpHost("locahost", 80); // 可以针对某特定网站指定最大连接数
cm.setMaxPerRoute(new HttpRoute(localhost), 30);
// 其它设置
params = new BasicHttpParams();
params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY);
inited = true;
}
}
static{
init();
}
public static HttpClient getHttpClient(){
return new DefaultHttpClient(cm, params);
}
}
手机版新浪微博登录141008
package test;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CookieStore;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
* 新浪微博手机版登录-新版
*
* @author sikaijian
* */
public class SinaWeiboLogin {
public static void main(String[] args) {
// 登录入口
String url = "http://login.weibo.cn/login/?ns=1&revalid=2&backURL=http%3A%2F%2Fweibo.cn%2F&backTitle=%CE%A2%B2%A9&vt=";
// 登录提交base
String postUrl = "http://login.weibo.cn/login/";
// 准备cookie存储
CookieStore cookieStore = new BasicCookieStore();
HttpContext httpContext = new BasicHttpContext();
httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
DefaultHttpClient httpClient = new DefaultHttpClient();
try {
/**
* 第一步先访问登录页面
* 1. 获取cookie
* 2. 获取form中所有不按套路出牌的key， 比如 <intput type="password" name="password_8873"/> 8873是后台传过来的
* 3. 获取form中所有hidden的key和value
*/
String loginHtml = "f:/loginWeibo.html";
HttpGet get = new HttpGet(url);
get.setHeader("Host", "login.weibo.cn");
get
.setHeader("User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0");
HttpResponse resp = httpClient.execute(get, httpContext);
extractStreamToFile(resp.getEntity().getContent(), loginHtml, false);
get.releaseConnection();
Map<String, String> contextMap = parseContext(loginHtml);
postUrl += contextMap.get("action");
String passwordKey = contextMap.get("password");
String vk = contextMap.get("vk");
/**
* 第二步：
* form提交，需要所有字段（可用浏览器查看form的源码）， form中的 部分字段是固定值
*
* 注意！！！！ 请用自己的帐号密码
*/
HttpPost post = new HttpPost(postUrl);
post.setHeader("Host", "login.weibo.cn");
post
.setHeader("User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0");
post
.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
post.setHeader("Accept-Language",
"zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
post.setHeader("Accept-Encoding", "gzip, deflate");
post
.setHeader(
"Referer",
"http://login.weibo.cn/login/?ns=1&revalid=2&backURL=http%3A%2F%2Fweibo.cn%2F&backTitle=%CE%A2%B2%A9&vt=");
post.setHeader("Connection", "keep-alive");
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
// TODO please use your own account
nvps.add(new BasicNameValuePair("mobile", "ACCOUNT"));
// TODO please use your own password
nvps.add(new BasicNameValuePair(passwordKey, "PASSWORD"));
nvps.add(new BasicNameValuePair("remember", "on"));
nvps.add(new BasicNameValuePair("backURL", "http://weibo.cn/"));
nvps.add(new BasicNameValuePair("backTitle", "微博"));
nvps.add(new BasicNameValuePair("tryCount", ""));
nvps.add(new BasicNameValuePair("vk", vk));
nvps.add(new BasicNameValuePair("submit", "登录"));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(nvps);
entity.setContentType("application/x-www-form-urlencoded");
post.setEntity(entity);
HttpResponse resp2 = httpClient.execute(post, httpContext);
System.out.println(resp2.getStatusLine());
InputStream in = resp2.getEntity().getContent();
extractStreamToFile(in, "f:/weibo.html", true);
/**
* 第三步:
* 重定向访问到首页
*
* 所有的上下文都会在httpcontext中保存，这一步之后，只要使用该httpcontext就可以随心所欲的访问。
*/
Header[] headers = resp2.getHeaders("Location");
if(null!=headers && headers.length>0){
String redirectUrl = headers[0].getValue();
HttpGet rget = new HttpGet(redirectUrl);
HttpResponse resp3 = httpClient.execute(rget, httpContext);
System.out.println(resp3.getStatusLine());
extractStreamToFile(resp3.getEntity().getContent(), "f:/weibo2.html", false);
}
System.out.println("DONE");
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 解析下上下文字段
* @param fileName
* @return
*/
public static Map<String, String> parseContext(String fileName) {
Map<String, String> paramMap = new HashMap<String, String>();
Document doc;
try {
doc = Jsoup.parse(new File(fileName), "utf-8");
Elements elms = doc.getElementsByAttributeValue("method", "post");
Element elm = elms.get(0);
String action = elm.attr("action");
if(null==action || "".equals(action)){
throw new RuntimeException("action not found!");
}
paramMap.put("action", action);
elms = elm.getElementsByAttributeValue("type", "password");
String password = elms.get(0).attr("name");
if(null==password || "".equals(password)){
throw new RuntimeException("password key not found!");
}
paramMap.put("password", password);
elms = elm.getElementsByAttributeValue("name", "vk");
String vk = elms.get(0).attr("value");
if(null==vk || "".equals(vk)){
throw new RuntimeException("vk value not found!");
}
paramMap.put("vk", vk);
} catch (IOException e) {
e.printStackTrace();
}
return paramMap;
}
/**
* 流转文件
* @param in 输入流
* @param filePath 文件路径
* @param isGzip 是否是gzip流
*/
public static void extractStreamToFile(InputStream in, String filePath,
boolean isGzip) {
InputStreamReader reader;
try {
if (isGzip) {
GZIPInputStream gzin;
gzin = new GZIPInputStream(in);
reader = new InputStreamReader(gzin, "utf-8");
} else {
reader = new InputStreamReader(in, "utf-8");
}
PrintWriter out = new PrintWriter(new File(filePath));
char[] charBuff = new char[1024];
int charReaded = -1;
while ((charReaded = reader.read(charBuff)) != -1) {
out.write(charBuff, 0, charReaded);
}
out.flush();
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
问题记录
1. 挑机器的UnknowHostException
一般情况都是url地址不存在之类的，但是在做本文章中的sina登录，发现我的代码在在别人的机器上无法运行，而在自己的机器上毫无问题。这时候身边的人会说你代码不稳定之类的各种不太可能的假设。
其实原因很简单，检查下网络吧，域名解析有问题！