伪装豆瓣爬虫程序——如何解决豆瓣禁用爬虫程序？

最近因为业务需要，要将豆瓣上的图书以及影视信息抓取下来，网页抓取其实很简单，很快就完成，但是系统还没上线就发现了问题，豆瓣会根据请求分析客户的行为，判断是否是机器人，如果判断为机器人，将要求输入验证码，最终导致抓取不到数据。

要解决这个问题，主要就两个思路，一个是识别验证码，当要求输入验证码时，识别图片中的验证码；另外一方案是伪装成正常访问，绕过豆瓣的分析程序。前一种方案受限于验证码识别率太低，很难满足需求，最终只能考虑第二种方案了。

通过分析豆瓣的分析判断机器人的实际情况，发现豆瓣是根据ip以及cookie信息统计访问频率来确定是否为“机器人”，有以下几种实际情况，1.不带cookie信息访问，快速访问一段时间，ip会被禁掉；2.带cookie访问，快速访问一段时间，请求会被禁掉，这时候清掉cookie，可以恢复正常访问。基于这个测试结果，可以采用一个简单的方案：

       1.第一次请求没有cookie信息，但记录下返回的cookie.

       2.后面每个请求都带上这个cookie信息.

       3.如果请求被跳转到验证码页面，就不带cookie重试，并记录返回的cookie信息

       4.下一次请求带上新的cookie信息。

重复2到4步骤。

 

       代码：

       1.简单的cookie manager：

import java.io.Serializable;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
 * 简单的Cookie Manager，按照顶级域名管理Cookie信息
 * @author 惊羽
 *
 */
class CookieManager implements Serializable{
	private static final long serialVersionUID = 292218695837624307L;
	private static CookieManager cookieManager = new CookieManager();
	private Map> cookies = new ConcurrentHashMap>();
	
	private CookieManager(){}
	/**
	 * 根据域名获取对应的Cookie
	 * @param domain
	 * @return
	 */
	public String getCookies(String domain){
		Map domainCookies = cookies.get(getTopLevelDomain(domain));
		if(domainCookies != null){
			StringBuilder sb = new StringBuilder();
			boolean isFirst = true;
			for(Map.Entry cookieEntry : domainCookies.entrySet()){
				if(!isFirst){
					sb.append("; ");
				}else{
					isFirst = false;
				}
				sb.append(cookieEntry.getKey())
			      .append("=")
			      .append(cookieEntry.getValue());
			}
			return sb.toString();
		}
		return "";
	}
	
	/**
	 *
	 * 设置Cookie值
	 * @param domain
	 * @param cookiesString
	 */
	public void setCookies(String domain,String cookiesString){
		Map domainCookies = cookies.get(getTopLevelDomain(domain));
		if(domainCookies == null){
			domainCookies = new ConcurrentHashMap();
			cookies.put(getTopLevelDomain(domain), domainCookies);
		}
		String[] cookies = cookiesString.split("; ");
		for (String cookie : cookies) {
			if(cookie != null && !cookie.trim().isEmpty()
					&& cookie.indexOf("=") > 0){
				int equalMarkIndex = cookie.indexOf("=");
				String key = cookie.substring(0,equalMarkIndex);
				String value = cookie.substring(equalMarkIndex+1);
				domainCookies.put(key, value);
			}
		}
	}
	/**
	 * 删除域名下所有的Cookie
	 * @param domain
	 */
	public void removeCookies(String domain){
		cookies.remove(getTopLevelDomain(domain));
	}
	
	/**
	 * 获取CookieManager的实例
	 * @return
	 */
	public static CookieManager getInstance(){
		return cookieManager;
	}
	
	/**
	 * 获取域名的顶级域名
	 * @param domain
	 * @return
	 */
	public String getTopLevelDomain(String domain){
		if(domain == null){
			return null;
		}
		if(!domainToTopLevelDomainMap.containsKey(domain)){
			String[] splits = domain.split("\\.");
			domainToTopLevelDomainMap.put(domain, (splits[splits.length-2] + "." + splits[splits.length -1]));
		}
		return domainToTopLevelDomainMap.get(domain);
	}
	/**
	 * 存储域名与其顶级域名之间映射关系，避免重复的计算顶级域名
	 */
	private Map domainToTopLevelDomainMap = new ConcurrentHashMap();
}

 

       2. 包装HttpURLConnection，实现请求失败自动清除cookie并重试的功能。

 

public class HttpURLConnectionWrapper extends HttpURLConnection {
	
	HttpURLConnection httpURLConnection;
	//简单的CookieManager
	CookieManager cookieManager = CookieManager.getInstance();
	
	public HttpURLConnectionWrapper(URL u) throws IOException {
		super(u);
		httpURLConnection = (HttpURLConnection) u.openConnection();
		setFollowRedirects(false);
		fillRequestHeadField();
	}
	/**
	 * 填充Request Header信息
	 */
	private void fillRequestHeadField(){
		httpURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
		httpURLConnection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
		httpURLConnection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
		httpURLConnection.setRequestProperty("Accept-Encoding", "GB2312,utf-8;q=0.7,*;q=0.7");
		httpURLConnection.setRequestProperty("Referer", "http://movie.douban.com/");
		httpURLConnection.setRequestProperty("Cache-Control", "max-age=0");
		httpURLConnection.setRequestProperty("Cookie", cookieManager.getCookies(url.getHost()));
		
	}

	@Override
	public InputStream getInputStream() throws IOException {
		InputStream is = httpURLConnection.getInputStream();
		//取到输入流中后处理Cookie信息
		resolveCookies();
		int responseCode = getResponseCode();
		if(responseCode != 200 && responseCode != 404 ){
			//清除cookie并重新发请求
			CookieManager.getInstance().removeCookies(url.getHost());
			try{
				httpURLConnection.disconnect();
				is.close();
			}catch (Exception e) {
			}
			httpURLConnection = (HttpURLConnection) this.getURL().openConnection();
			setFollowRedirects(false);
			fillRequestHeadField();
			is = httpURLConnection.getInputStream();
		}
		return is;
	}
	private void resolveCookies(){
		List setCookies = getHeaderFields().get("Set-Cookie");
		if(setCookies != null && !setCookies.isEmpty()){
			for (String setCookie : setCookies) {
				cookieManager.setCookies(this.url.getHost(), setCookie);
			}	
		}
	}
.........
}