使用httpclient、htmlcleaner 、xpath 采集新浪微博3G站点数据

转载：http://blog.csdn.net/telnetor/article/details/8582045

0、背景

原来对新浪微博的采集主要靠对weibo.com这个入口进行，但是最近发现有人使用weibo.cn这个入口操作。应为weibo.cn对应的是微博的3g版本，基本没有广告，页面小，这样下载数据量会小很多，并且3g版本的采集比较简单。于是就有了这个3g版本的采集程序。

写出来，分享给大家，希望对有需要的朋友有所帮助。

使用到的类库：httpclient、htmlcleaner

httpclient负责处理http的get和post请求，下载页面；htmlcleaner负责将下载的页面转化为规范的xml，之后用xpath匹配所需内容。

1、基础的http请求类

 package cn.mingyuan.weibo.commons;

 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.impl.client.DefaultHttpClient;

 /**
  * http请求基类
  *
  * @author mingyuan
  *
  */
 public abstract class RequestCommons {
     protected HttpClient httpclient = null;

     public RequestCommons() {
         initHttpClient();
     }

     /**
      * 初始化httpclient
      */
     protected void initHttpClient() {
         httpclient = new DefaultHttpClient();
     }

     protected HttpClient getHttpClient() {
         return httpclient;
     }

     protected void addHeader(HttpRequestBase request, String key, String value) {
         request.addHeader(key, value);
     }

     protected void addCookie(HttpRequestBase request, String cookie) {
         addHeader(request, "Cookie", cookie);
     }

     protected void setCookie(HttpRequestBase request, String cookie) {
         request.setHeader("Cookie", cookie);
     }

     /**
      * 设置请求的header值
      *
      * @param request
      *            http的get或者post请求
      */
     protected void setHeader(HttpRequestBase request) {
         request.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
         request.setHeader("Accept-Language", "en-us,en;q=0.5");
         request.setHeader("Connection", "keep-alive");
         request.setHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:13.0) Gecko/20100101 Firefox/13.0.1");
     }
 }

2、weibo.cn登陆

 package cn.mingyuan.weibo.login;

 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;

 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.htmlcleaner.HtmlCleaner;
 import org.htmlcleaner.TagNode;
 import org.htmlcleaner.XPatherException;

 import cn.mingyuan.weibo.commons.RequestCommons;
 import cn.mingyuan.weibo.until.Constants;

 /**
  * 登陆 获取cookie
  *
  * @author mingyuan
  *
  */
 public class Login extends RequestCommons {
     /**
      * 获取登陆参数。主要有三个值：第一个是表单提交地址、第二个是密码输入框的名字、第三个是vk的值
      *
      * @return 返回登陆参数，string数组，里面的元素：第一个是表单提交地址、第二个是密码输入框的名字、第三个是vk的值
      */
     private String[] getLoginParameters() {
         HttpClient httpClient = getHttpClient();
         String location = "http://3g.sina.com.cn/prog/wapsite/sso/login.php?backURL=http%3A%2F%2Fweibo.cn%2F&backTitle=%D0%C2%C0%CB%CE%A2%B2%A9&vt=4&revalid=2&ns=1";
         HttpGet get = new HttpGet(location);
         setHeader(get);
         HttpResponse response;
         InputStream content;
         String retAction = null;
         String retPassword = null;
         String retVk = null;
         try {
             response = httpClient.execute(get);
             HttpEntity entity = response.getEntity();
             content = entity.getContent();

             // 提取登陆参数
             HtmlCleaner cleaner = new HtmlCleaner();
             TagNode tagNode = cleaner.clean(content, "utf-8");
             Object[] action = tagNode.evaluateXPath("//form/@action");

             if (action.length > 0) {
                 retAction = action[0].toString();
             }
             Object[] passwordKey = tagNode.evaluateXPath("//form//input[@type='password']/@name");

             if (passwordKey.length > 0) {
                 retPassword = passwordKey[0].toString();
             }
             Object[] vkKey = tagNode.evaluateXPath("//form//input[@name='vk']/@value");

             if (vkKey.length > 0) {
                 retVk = vkKey[0].toString();
             }
             EntityUtils.consume(entity);
         } catch (ClientProtocolException e) {
             System.out.println("获取登陆页面失败，location=" + location);
             e.printStackTrace();
         } catch (IOException e) {
             System.out.println("获取页面内容流失败");
             e.printStackTrace();
         } catch (XPatherException e) {
             System.out.println("解析登陆参数失败");
             e.printStackTrace();
         } finally {
             if (get != null) {
                 get.releaseConnection();
             }
         }

         System.out.println("请求页面：" + location);
         System.out.println("提交地址：" + retAction);
         System.out.println("密码输入框名称：" + retPassword);
         System.out.println("vk值：" + retVk);

         return new String[] { retAction, retPassword, retVk };
     }

     /**
      * 提交账号密码，开始登陆
      *
      * @param postAction
      *            登陆地址
      * @param userNameValue
      *            微博登陆账号
      * @param passwordValue
      *            微博登陆密码
      * @param passwordKey
      *            微博登陆框的name
      * @param vkValue
      *            vk的值
      * @return 返回取到的cookie与跳转地址，组合成一个String数组。第一个元素为cookie，第二个元素为跳转地址
      */
     private String[] submitPassword(String postAction, String userNameValue, String passwordValue, String passwordKey, String vkValue) {
         HttpClient httpclient = getHttpClient();
         String url = "http://3g.sina.com.cn/prog/wapsite/sso/" + postAction;
         System.out.println("开始提交账号密码：" + url);
         HttpPost post = new HttpPost(url);
         setHeader(post);
         List<NameValuePair> nvps = new ArrayList<NameValuePair>();
         nvps.add(new BasicNameValuePair("mobile", userNameValue));
         nvps.add(new BasicNameValuePair(passwordKey, passwordValue));
         nvps.add(new BasicNameValuePair("remember", "on"));
         nvps.add(new BasicNameValuePair("vk", vkValue));
         nvps.add(new BasicNameValuePair("backURL", "http://weibo.cn/"));
         nvps.add(new BasicNameValuePair("backTitle", "新浪微博"));
         nvps.add(new BasicNameValuePair("submit", "登录"));
         HttpResponse response;
         String cookie = null;
         String location = null;
         try {
             post.setEntity(new UrlEncodedFormEntity(nvps));
             response = httpclient.execute(post);
             HttpEntity entity2 = response.getEntity();
             Header[] setCookie = response.getHeaders("Set-Cookie");

             if (setCookie != null) {
                 cookie = setCookie[0].getValue();
                 System.out.println("获取到Cookie：" + cookie);
             }
             Header[] locations = response.getHeaders("Location");
             if (locations != null) {
                 location = locations[0].getValue();
                 System.out.println("获取到跳转链接：" + location);
             }
             EntityUtils.consume(entity2);
         } catch (UnsupportedEncodingException e1) {
             e1.printStackTrace();
         } catch (ClientProtocolException e) {
             e.printStackTrace();
         } catch (IOException e) {
             e.printStackTrace();
         } finally {
             post.releaseConnection();
         }

         return new String[] { cookie, location };
     }

     /**
      * 获取重定向页面内容
      *
      * @param redirectUrl
      *            获取重定向页面地址
      * @return 获取cookie和要跳转的地址
      */
     private String[] getRedirectPageInfo(String redirectUrl) {
         System.out.println("开始获取跳转链接页面");
         HttpGet get = new HttpGet(redirectUrl);
         setHeader(get);
         HttpResponse redirectResponse;
         String cookie = null;
         String clickHref = null;
         try {
             redirectResponse = httpclient.execute(get);
             Header[] headers = redirectResponse.getHeaders("Set-Cookie");
             if (headers != null) {
                 cookie = headers[0].getValue();
                 String[] splits = cookie.split(";");

                 for (String str : splits) {
                     if (str.startsWith("gsid_CTandWM")) {
                         cookie = str;
                         break;
                     }
                 }
             }
             HttpEntity entity = redirectResponse.getEntity();
             InputStream content = entity.getContent();
             HtmlCleaner cleaner = new HtmlCleaner();
             TagNode tagNode = cleaner.clean(content, "utf-8");
             Object[] clickHrefs = tagNode.evaluateXPath("//div/a/@href");

             if (clickHrefs != null) {
                 clickHref = clickHrefs[0].toString();
                 System.out.println("获取到跳转链接地址：" + clickHref);

             }
             EntityUtils.consume(entity);
         } catch (ClientProtocolException e) {
             e.printStackTrace();
         } catch (IOException e) {
             e.printStackTrace();
         } catch (XPatherException e) {
             e.printStackTrace();
         } finally {
             get.releaseConnection();
         }
         return new String[] { cookie, clickHref };
     }

     /**
      * 跳转
      *
      * @param cookie
      *            上次请求取到的cookie
      * @param redirectUrl
      *            跳转url
      * @return 返回跳转后取得的cookie
      */
     private String doRedirection(String cookie, String redirectUrl) {
         HttpGet get = new HttpGet(redirectUrl);
         setHeader(get);
         get.setHeader("Cookie", cookie);
         HttpResponse response;
         try {
             response = httpclient.execute(get);
             HttpEntity entity = response.getEntity();
             Header[] headers2 = response.getHeaders("Set-Cookie");
             if (headers2 != null) {
                 cookie = headers2[0].getValue();
                 System.out.println("跳转页面取回的cookie：" + cookie);
                 String[] splits = cookie.split(";");
                 for (String str : splits) {
                     if (str.startsWith("_WEIBO_UID")) {
                         cookie = str;
                         break;
                     }
                 }
             }
             EntityUtils.consume(entity);
         } catch (ClientProtocolException e) {
             e.printStackTrace();
         } catch (IOException e) {
             e.printStackTrace();
         }
         return cookie;
     }

     /**
      * 提交账号密码，登陆
      *
      * @param userNameValue
      *            微博账号
      * @param passwordValue
      *            微博密码
      * @return 返回cookie
      */
     public String doLogin(String userNameValue, String passwordValue) {
         // 获取登陆页面的参数
         String[] loginParameters = getLoginParameters();
         String postAction = loginParameters[0];
         String passwordKey = loginParameters[1];
         String vkValue = loginParameters[2];

         // 提交账号密码，获取重定向页面链接与cookie
         String[] cookieRedirectLocation = submitPassword(postAction, userNameValue, passwordValue, passwordKey, vkValue);
         String cookie = cookieRedirectLocation[0];
         String redirectUrl = cookieRedirectLocation[1];
         // 获取重定向页面内容
         String[] redirectInfo = getRedirectPageInfo(redirectUrl);
         cookie = redirectInfo[0];
         redirectUrl = redirectInfo[1];
         System.out.println("准备跳转");
         try {
             TimeUnit.SECONDS.sleep(3);
         } catch (InterruptedException e) {
             e.printStackTrace();
         }
         System.out.println("开始跳转");
         String cookieOfRedirect = doRedirection(cookie, redirectUrl);

         StringBuffer sb = new StringBuffer(cookie);
         sb.append(';').append(cookieOfRedirect);
         System.out.println("登陆成功，最终cookie为：" + sb.toString());
         return sb.toString();
     }

     /**
      * 使用配置的账号、密码登陆
      *
      * @return 返回登陆cookie
      */
     public String doLogin() {
         return this.doLogin(Constants.LOGIN_USERNAME, Constants.LOGIN_PASSWORD);
     }
 }

3、测试

步骤：先登录取cookie再拿cookie填到请求里面取页面内容

 package cn.mingyuan.weibo.test;

 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;

 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.util.EntityUtils;

 import cn.mingyuan.weibo.commons.RequestCommons;
 import cn.mingyuan.weibo.login.Login;
 /**
  * 测试，取页面内容
  * @author mingyuan
  *
  */
 public class WeiboTest extends RequestCommons {

     /**
      * 打印流
      *
      * @param in
      *            InputStream
      */
     private void printContent(InputStream in) {
         BufferedReader reader = null;
         try {
             reader = new BufferedReader(new InputStreamReader(in, "utf-8"));
             String line;
             while ((line = reader.readLine()) != null) {
                 System.out.println(line);
             }
         } catch (Exception e) {
             e.printStackTrace();
         } finally {
             // 这里不关闭流，留作他用
         }
     }

     /**
      * 测试，读取一个微博地址，打印出页面内容
      *
      * @param finalCookie
      *            cookie
      */
     private void test(String finalCookie) {
         HttpGet get = new HttpGet("http://weibo.cn/irlucene");
         setHeader(get);
         get.setHeader("Cookie", finalCookie.toString());
         HttpResponse response;
         try {
             response = httpclient.execute(get);
             HttpEntity entity = response.getEntity();
             printContent(entity.getContent());
             EntityUtils.consume(entity);
         } catch (ClientProtocolException e) {
             e.printStackTrace();
         } catch (IOException e) {
             e.printStackTrace();
         } finally {
             get.releaseConnection();
         }
     }

     public static void main(String[] args) {
         Login login = new Login();
         String userNameValue = "username";
         String passwordValue = "password";
         String cookie = login.doLogin(userNameValue, passwordValue);
         System.out.println("final Cookie=" + cookie);
         new WeiboTest().test(cookie);
     }

 }