爬虫伪装方法分类

原文：http://pisces-java.iteye.com/blog/1471651

最近因为业务需要，要将豆瓣上的图书以及影视信息抓取下来，网页抓取其实很简单，很快就完成，但是系统还没上线就发现了问题，豆瓣会根据请求分析客户的行为，判断是否是机器人，如果判断为机器人，将要求输入验证码，最终导致抓取不到数据。

要解决这个问题，主要就两个思路，一个是识别验证码，当要求输入验证码时，识别图片中的验证码；另外一方案是伪装成正常访问，绕过豆瓣的分析程序。前一种方案受限于验证码识别率太低，很难满足需求，最终只能考虑第二种方案了。

通过分析豆瓣的分析判断机器人的实际情况，发现豆瓣是根据ip以及cookie信息统计访问频率来确定是否为“机器人”，有以下几种实际情况，1.不带cookie信息访问，快速访问一段时间，ip会被禁掉；2.带cookie访问，快速访问一段时间，请求会被禁掉，这时候清掉cookie，可以恢复正常访问。基于这个测试结果，可以采用一个简单的方案：

       1.第一次请求没有cookie信息，但记录下返回的cookie.

       2.后面每个请求都带上这个cookie信息.

       3.如果请求被跳转到验证码页面，就不带cookie重试，并记录返回的cookie信息

       4.下一次请求带上新的cookie信息。

重复2到4步骤。

  代码：

       1.简单的cookie manager：

 import java.io.Serializable;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 /**
  * 简单的Cookie Manager，按照顶级域名管理Cookie信息
  * @author 惊羽
  *
  */
 class CookieManager implements Serializable{
     private static final long serialVersionUID = 292218695837624307L;
     private static CookieManager cookieManager = new CookieManager();
     private Map> cookies = new ConcurrentHashMap>();

     private CookieManager(){}
     /**
      * 根据域名获取对应的Cookie
      * @param domain
      * @return
      */
     public String getCookies(String domain){
         Map domainCookies = cookies.get(getTopLevelDomain(domain));
         if(domainCookies != null){
             StringBuilder sb = new StringBuilder();
             boolean isFirst = true;
             for(Map.Entry cookieEntry : domainCookies.entrySet()){
                 if(!isFirst){
                     sb.append("; ");
                 }else{
                     isFirst = false;
                 }
                 sb.append(cookieEntry.getKey())
                   .append("=")
                   .append(cookieEntry.getValue());
             }
             return sb.toString();
         }
         return "";
     }

     /**
      *
      * 设置Cookie值
      * @param domain
      * @param cookiesString
      */
     public void setCookies(String domain,String cookiesString){
         Map domainCookies = cookies.get(getTopLevelDomain(domain));
         if(domainCookies == null){
             domainCookies = new ConcurrentHashMap();
             cookies.put(getTopLevelDomain(domain), domainCookies);
         }
         String[] cookies = cookiesString.split("; ");
         for (String cookie : cookies) {
             if(cookie != null && !cookie.trim().isEmpty()
                     && cookie.indexOf("=") > 0){
                 int equalMarkIndex = cookie.indexOf("=");
                 String key = cookie.substring(0,equalMarkIndex);
                 String value = cookie.substring(equalMarkIndex+1);
                 domainCookies.put(key, value);
             }
         }
     }
     /**
      * 删除域名下所有的Cookie
      * @param domain
      */
     public void removeCookies(String domain){
         cookies.remove(getTopLevelDomain(domain));
     }

     /**
      * 获取CookieManager的实例
      * @return
      */
     public static CookieManager getInstance(){
         return cookieManager;
     }

     /**
      * 获取域名的顶级域名
      * @param domain
      * @return
      */
     public String getTopLevelDomain(String domain){
         if(domain == null){
             return null;
         }
         if(!domainToTopLevelDomainMap.containsKey(domain)){
             String[] splits = domain.split("\\.");
             domainToTopLevelDomainMap.put(domain, (splits[splits.length-2] + "." + splits[splits.length -1]));
         }
         return domainToTopLevelDomainMap.get(domain);
     }
     /**
      * 存储域名与其顶级域名之间映射关系，避免重复的计算顶级域名
      */
     private Map domainToTopLevelDomainMap = new ConcurrentHashMap();
 }

     2. 包装HttpURLConnection，实现请求失败自动清除cookie并重试的功能。

 public class HttpURLConnectionWrapper extends HttpURLConnection {

     HttpURLConnection httpURLConnection;
     //简单的CookieManager
     CookieManager cookieManager = CookieManager.getInstance();

     public HttpURLConnectionWrapper(URL u) throws IOException {
         super(u);
         httpURLConnection = (HttpURLConnection) u.openConnection();
         setFollowRedirects(false);
         fillRequestHeadField();
     }
     /**
      * 填充Request Header信息
      */
     private void fillRequestHeadField(){
         httpURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
         httpURLConnection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
         httpURLConnection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
         httpURLConnection.setRequestProperty("Accept-Encoding", "GB2312,utf-8;q=0.7,*;q=0.7");
         httpURLConnection.setRequestProperty("Referer", "http://movie.douban.com/");
         httpURLConnection.setRequestProperty("Cache-Control", "max-age=0");
         httpURLConnection.setRequestProperty("Cookie", cookieManager.getCookies(url.getHost()));

     }

     @Override
     public InputStream getInputStream() throws IOException {
         InputStream is = httpURLConnection.getInputStream();
         //取到输入流中后处理Cookie信息
         resolveCookies();
         int responseCode = getResponseCode();
         if(responseCode != 200 && responseCode != 404 ){
             //清除cookie并重新发请求
             CookieManager.getInstance().removeCookies(url.getHost());
             try{
                 httpURLConnection.disconnect();
                 is.close();
             }catch (Exception e) {
             }
             httpURLConnection = (HttpURLConnection) this.getURL().openConnection();
             setFollowRedirects(false);
             fillRequestHeadField();
             is = httpURLConnection.getInputStream();
         }
         return is;
     }
     private void resolveCookies(){
         List setCookies = getHeaderFields().get("Set-Cookie");
         if(setCookies != null && !setCookies.isEmpty()){
             for (String setCookie : setCookies) {
                 cookieManager.setCookies(this.url.getHost(), setCookie);
             }
         }
     }
 .........
 }

......................................................

补充

如果用chrome浏览器可以按F12进入控制台

选择Network 选择左边name就可以看到一些request的头信息了

把这些头信息在代码里加入即可