Java爬虫技术之绕过百度云防护抓取网站内容

大家好，我是Coody

最近做文章采集，碰到一个有经过百度云加速的网站，由于打开浏览器需要安全检查，所以针对相关机制做了一下研究，故此封装了一个HTTP工具。

本文已发布之开源中国，由于csdn用户量巨大且易于搜索引擎收录，故此分享出来希望对特定的友友有所帮助。

直接贴代码，copy下来可以直接使用

如图：


输入图片说明


首先需要一个Http工具类：HttpHandle


package org.coody.robot.util;


import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;




public class HttpHandle {



public final static String POST="POST";
public final static String GET="GET";
public final static String HEAD="HEAD";
public final static String PUT="PUT";
public final static String CONNECT="CONNECT";
public final static String OPTIONS="OPTIONS";
public final static String DELETE="DELETE";
public final static String PATCH="PATCH";
public final static String PROPFIND="PROPFIND";
public final static String PROPPATCH="PROPPATCH";
public final static String MKCOL="MKCOL";
public final static String COPY="COPY";
public final static String MOVE="MOVE";
public final static String LOCK="LOCK";
public final static String UNLOCK ="UNLOCK";
public final static String TRACE="TRACE";

public final static String HTTP_GENERAL="HTTP_GENERAL";

public final static String HTTP_JSON="HTTP_JSON";

public HttpConfig config=new HttpConfig();




public HttpConfig getConfig() {
return config;
}


public void setConfig(HttpConfig config) {
this.config = config;
}
public static class HttpConfig{

private boolean allowRedirects=true;

private String cookie="";

private String encode="UTF-8";

private int timeOut=15;

private String httpModule=HTTP_GENERAL;

private Map headerMap=new HashMap();


public void setEncode(String encode) {
this.encode = encode;
}




public void setTimeOut(int timeOut) {
this.timeOut = timeOut;
}




public void setCookie(String cookie) {
this.cookie = cookie;
}




public void setHeaderMap(Map headerMap) {
this.headerMap = headerMap;
}


//设置Header头部
public void setRequestProperty(String fieldName,String value){
headerMap.put(fieldName, value);
}
//是否开启Gzip
public void setGzip(boolean isGzip){
if(isGzip){
headerMap.put("Accept-Encoding", "gzip, deflate, sdch");
return;
}
headerMap.put("Accept-Encoding", "*");
}
//是否保持连接
public void setKeepAlive(boolean keepAlive){
if(keepAlive){
headerMap.put("Connection", "keep-alive");
return;
}
headerMap.put("Connection", "close");
}

//是否允许重定向
public void allowRedirects(boolean allowRedirects){
this.allowRedirects=allowRedirects;
}
}

private HttpURLConnection createConnectionGeneral(String url) {
try {
HttpURLConnection conn = (HttpURLConnection) new URL(url)
.openConnection();
conn.addRequestProperty("Referer", getDomain(url));
conn.addRequestProperty(
"Accept",
"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*");
conn.addRequestProperty("Content-type",
"application/x-www-form-urlencoded");
conn.addRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
return conn;
} catch (Exception e) {
return null;
}
}

private HttpURLConnection createConnectionJson(String url) {
try {
HttpURLConnection conn = (HttpURLConnection) new URL(url)
.openConnection();
conn.addRequestProperty("Referer", getDomain(url));
conn.addRequestProperty(
"Accept",
"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*");
conn.addRequestProperty("Content-type",
"application/x-www-form-urlencoded");
conn.addRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
return conn;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}








//获取默认来源地址
public static String getDomain(String urlStr){
try {
URI uri=new URI(urlStr);
String result=uri.getScheme()+"://"+uri.getHost();
if(uri.getPort()>0&&uri.getPort()!=80){
result+=("/"+uri.getPort());
}
if(!result.endsWith("/")){
result+="/";
}
return result;
} catch (Exception e) {
e.printStackTrace();
return null;
}

}
//合并Cookie
private static String mergeCookie(String oldCookie, String newCookie) {
if (newCookie == null) {
return oldCookie;
}
Map cookieMap = new HashMap();
String[] cookTmp = null;
String[] cookieTab = null;
StringBuilder valueTmp = new StringBuilder();
String[] cookies = { oldCookie, newCookie };
for (String currCookie : cookies) {
if (StringUtil.isNullOrEmpty(currCookie)) {
continue;
}
cookieTab = currCookie.split(";");
for (String cook : cookieTab) {
cookTmp = cook.split("=");
if (cookTmp.length < 2) {
continue;
}
valueTmp = new StringBuilder();
for (int i = 1; i < cookTmp.length; i++) {
valueTmp.append(cookTmp[i]);
if (i < cookTmp.length - 1) {
valueTmp.append("=");
}
}
if (StringUtil.findNull(cookTmp[0], valueTmp) > -1) {
continue;
}
cookieMap.put(cookTmp[0], valueTmp.toString());
}
}
valueTmp = new StringBuilder();
for (String key : cookieMap.keySet()) {
valueTmp.append(key).append("=").append(cookieMap.get(key));
valueTmp.append(";");
}
return valueTmp.toString();
}

private HttpURLConnection getConnection(String url) {
if(config.httpModule.equals(HTTP_GENERAL)){
return createConnectionGeneral(url);
}
if(config.httpModule.equals(HTTP_JSON)){
return createConnectionJson(url);
}
return null;
}

public HttpEntity Get(String url){
return Conn(url, GET, null);
}

public HttpEntity Post(String url,String data){
return Conn(url, POST, data);
}

public HttpEntity Conn(String url, String method,
String postData){
if(url.contains(" ")){
url=url.replace(" ", "%20");
}
HttpURLConnection conn = getConnection(url);
if (conn == null) {
return null;
}
if (!StringUtil.isNullOrEmpty(config.headerMap)) {
for (String key : config.headerMap.keySet()) {
conn.setRequestProperty(key, config.headerMap.get(key));
key = null;
}
}
if(!config.allowRedirects){
conn.setInstanceFollowRedirects(false);
}
if (!StringUtil.isNullOrEmpty(config.cookie)) {
conn.setRequestProperty("Cookie", config.cookie);
}
try {
conn.setRequestMethod(method);
if (method.equalsIgnoreCase(POST)||method.equalsIgnoreCase(PUT)) {
conn.setDoOutput(true);
byte [] postByte=postData.getBytes(config.encode);
conn.setRequestProperty("Content-Length", String.valueOf(postByte.length));
conn.getOutputStream().write(postByte);
conn.connect();
conn.getOutputStream().flush();
conn.getOutputStream().close();
}
} catch (Exception e) {
e.printStackTrace();
}
conn.setConnectTimeout(config.timeOut*1000);
InputStream ins = null;
HttpEntity hEntity = new HttpEntity();
String key = "";
StringBuilder cookie = new StringBuilder();
try {
Integer status=conn.getResponseCode();
if (status !=HttpURLConnection.HTTP_OK) {
ins=conn.getErrorStream();
}else{
ins=conn.getInputStream();
}
hEntity.setCode(conn.getResponseCode());
Map headMap=new HashMap();
for (int i = 1; (key = conn.getHeaderFieldKey(i)) != null; i++) {
headMap.put(key, conn.getHeaderField(key));
if (key.equalsIgnoreCase("set-cookie")) {
try {
cookie.append(conn.getHeaderField(i).replace("/", ""));
} catch (Exception e) {
}
}
}
config.cookie = mergeCookie(config.cookie, cookie.toString());
byte[] b = toByte(ins);
if((headMap.get("Content-Encoding")!=null && headMap.get("Content-Encoding").contains("gzip"))||(conn.getRequestProperty("Accept-Encoding")!=null&&conn.getRequestProperty("Accept-Encoding").contains("gzip"))){
b = GZIPUtils.uncompress(b);
}
hEntity.setEncode(config.encode);
hEntity.setBye(b);
hEntity.setCookie(config.cookie);
hEntity.setHeadMap(headMap);
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
ins.close();
} catch (Exception e2) {
}
}
return hEntity;
}


private byte[] toByte(InputStream ins) {
if(ins==null){
return null;
}
ByteArrayOutputStream swapStream = null;
try {
swapStream = new ByteArrayOutputStream();
byte[] buff = new byte[1024];
int rc = 0;
while ((rc = ins.read(buff, 0, 1024)) > 0) {
swapStream.write(buff, 0, rc);
}
return swapStream.toByteArray();
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
try {
swapStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}




}


其次需要一个Http响应对象类：HttpEntity


package org.coody.robot.util;


import java.util.HashMap;
import java.util.Map;


public class HttpEntity {



private String html;
private byte[] bye;
private String cookie;
private Integer code=-1;
private Map headMap;

public Map getHeadMap() {
return headMap;
}


public void setHeadMap(Map headMap) {
this.headMap = headMap;
}


private String encode="UTF-8";

public String getHtml() {
try {
if(html!=null){
return html;
}
if(bye==null){
return null;
}
String str= new String(bye, encode);
html=str;
return str;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

public String getHtml(boolean isGzip) {
try {
if(bye==null){
return null;
}
String str= new String(GZIPUtils.uncompress(bye), encode);
return str;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}


public String getEncode() {
return encode;
}


public void setEncode(String encode) {
this.encode = encode;
}


public void setHtml(String html) {
this.html = html;
}


public Integer getCode() {
return code;
}


public void setCode(Integer code) {
this.code = code;
}


public String getCookie() {
return cookie;
}


public void setCookie(String cookie) {
this.cookie = cookie;
}


public byte[] getBye() {
return bye;
}


public void setBye(byte[] bye) {
this.bye = bye;
}


public Map getCookieMap() {
if (cookie == null) {
return null;
}
Map cookieMap = new HashMap();
String[] cookies = cookie.split(";");
for (String cook : cookies) {
String[] tmps = cook.split("=");
if (tmps.length >= 2) {
String cookieValue = "";
for (int i = 1; i < tmps.length; i++) {
cookieValue += tmps[i];
if (i < tmps.length-1) {
cookieValue += "=";
}
}
cookieMap.put(tmps[0].trim(), cookieValue.trim());
}
}
return cookieMap;
}
}


某些网站是有Gzip压缩的，需要一个Gzip压缩类GzipUtils


package org.coody.robot.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;  
  
  
  
/** 
 *  
 * @author wenqi5 
 *  
 */  
public class GZIPUtils {  
  
    public static final String GZIP_ENCODE_UTF_8 = "UTF-8";  
  
  
    /** 
     * 字符串压缩为GZIP字节数组 
     *  
     * @param str 
     * @return 
     */  
    public static byte[] compress(String str) {  
        try {
return compress(str.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
return null;
}  
    }  
  
    /** 
     * 字符串压缩为GZIP字节数组 
     *  
     * @param str 
     * @param encoding 
     * @return 
     */  
    public static byte[] compress(byte[] data) {  
        if (data == null || data.length == 0) {  
            return null;  
        }  
        ByteArrayOutputStream out = new ByteArrayOutputStream();  
        GZIPOutputStream gzip = null;  
        try {  
            gzip = new GZIPOutputStream(out);  
            gzip.write(data);  
        } catch (IOException e) {  
        }finally {
        try {
        gzip.close();  
} catch (Exception e2) {
}
        try {
        out.close();
} catch (Exception e2) {
}
}
        return out.toByteArray();  
    }  
  
    /** 
     * GZIP解压�? 
     *  
     * @param bytes 
     * @return 
     */  
    public static byte[] uncompress(byte[] bytes) {  
        if (bytes == null || bytes.length == 0) {  
            return null;  
        }  
        ByteArrayOutputStream out = new ByteArrayOutputStream();  
        ByteArrayInputStream in = new ByteArrayInputStream(bytes);  
        try {  
            GZIPInputStream ungzip = new GZIPInputStream(in);  
            byte[] buffer = new byte[256];  
            int n;  
            while ((n = ungzip.read(buffer)) >= 0) {  
                out.write(buffer, 0, n);  
            }  
        } catch (IOException e) {  
        }  finally {
try {
in.close();
} catch (Exception e2) {
}
try {
out.close();
} catch (Exception e2) {
}
        }
  
        return out.toByteArray();  
    }  
}  
以上类均依赖一个StringUtil，笔者比较懒，也没有拆分出来


package org.coody.robot.util;


import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


//import oracle.sql.CLOB;


public class StringUtil {



public static Integer[] getIntegerParas(Object[] objs) {
if (isNullOrEmpty(objs)) {
return null;
}
Integer[] ints = new Integer[objs.length];
for (int i = 0; i < objs.length; i++) {
try {
ints[i] = Integer.valueOf(objs[i].toString());
} catch (Exception e) {
}
}
return ints;
}


/**
* 生成指定数目字符串按分隔符分�?
* 
* @param baseStr
* @param mosaicChr
* @param size
* @return
*/
public static String getByMosaicChr(String baseStr, String mosaicChr, Integer size) {
List list = new ArrayList();
for (int i = 0; i < size; i++) {
if (isNullOrEmpty(baseStr)) {
continue;
}
list.add(baseStr);
}
return collectionMosaic(list, mosaicChr);
}


/**
* 根据分割符将字符串分割成String数组
* 
* @param src
*            源字符串
* @param separator
*            分隔�??
* @return String数组
*/
public static String[] splitToStringArray(String src, String separator) {
Vector splitArrays = new Vector();
int i = 0;
int j = 0;
while (i <= src.length()) {
j = src.indexOf(separator, i);
if (j < 0) {
j = src.length();
}
splitArrays.addElement(src.substring(i, j));
i = j + 1;
}
int size = splitArrays.size();
String[] array = new String[size];
System.arraycopy(splitArrays.toArray(), 0, array, 0, size);
return array;
}


/**
* 根据分割符将字符串分割成Integer数组
* 
* @param src
*            源字符串
* @param separator
*            分隔�??
* @return Integer数组
*/
public static Integer[] splitToIntgArray(String src, String separator) {
String[] arr = splitToStringArray(src, separator);
Integer[] intArr = new Integer[arr.length];
for (int i = 0; i < arr.length; i++) {
intArr[i] = Integer.valueOf(arr[i]);
}
return intArr;
}


/**
* 根据分隔符将字符串分割成int数组
* 
* @param src
*            源字符串
* @param separator
*            分隔�??
* @return int数组
*/
public static int[] splitToIntArray(String src, String separator) {
String[] arr = splitToStringArray(src, separator);
int[] intArr = new int[arr.length];
for (int i = 0; i < arr.length; i++) {
intArr[i] = Integer.parseInt(arr[i]);
}
return intArr;
}


public static String getInPara(Integer size) {
return getByMosaicChr("?", ",", size);


}


public static String textCutCenter(String allTxt, String firstTxt, String lastTxt) {
try {
String tmp = "";
int n1 = allTxt.indexOf(firstTxt);
if (n1 == -1) {
return "";
}
tmp = allTxt.substring(n1 + firstTxt.length(), allTxt.length());
int n2 = tmp.indexOf(lastTxt);
if (n2 == -1) {
return "";
}
tmp = tmp.substring(0, n2);
return tmp;
} catch (Exception e) {
return "";
}
}


public static List textCutCenters(String allTxt, String firstTxt, String lastTxt) {
try {
List results = new ArrayList();
while(allTxt.contains(firstTxt)){
int n = allTxt.indexOf(firstTxt);
allTxt=allTxt.substring(n+firstTxt.length(), allTxt.length());
n=allTxt.indexOf(lastTxt);
if(n==-1){
return results;
}
String result=allTxt.substring(0, n);
results.add(result);
allTxt=allTxt.substring(n+firstTxt.length(), allTxt.length());
}
return results;
} catch (Exception e) {
return null;
}
}
public static String convertToUnicode(String source) {
String result = "";
char[] chrs = source.toCharArray();
for (int i = 0; i < chrs.length; i++) {
result += "&#" +Character.codePointAt(chrs, i);
}
return result;
}
public static Integer toInteger(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Integer.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}


public static String toString(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return String.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}


public static Double toDouble(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Double.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}


public static Float toFloat(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Float.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}


public static Long toLong(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Long.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}


public static Integer getRanDom(int start, int end) {
return (int) (Math.random() * (end - start + 1)) + start;
}


public static float getRanDom(Float start, Float end) {
String str = String.valueOf(start);
String[] tabs = str.split("\\.");
Integer startLength = 1;
if (tabs.length == 2) {
startLength = tabs[1].length();
}
str = String.valueOf(end);
tabs = str.split("\\.");
Integer endLength = 1;
if (tabs.length == 2) {
endLength = tabs[1].length();
}
if (endLength > startLength) {
startLength = endLength;
}
start = (float) (start * Math.pow(10, startLength));
end = (float) (end * Math.pow(10, startLength));
return (float) (getRanDom(start.intValue(), end.intValue()) / Math.pow(10, startLength));
}


public static String replaceBlank(String str) {
String dest = "";
if (str != null) {
Pattern p = Pattern.compile("\\s*|\t|\r|\n");
Matcher m = p.matcher(str);
dest = m.replaceAll("");
}
return dest;
}


public static Boolean isMatcher(String val, String matcher) {
Pattern p = Pattern.compile(matcher);
Matcher m = p.matcher(val);
return m.matches();
}


public static boolean isMobile(String mobile) {
if (isNullOrEmpty(mobile)) {
return false;
}
Pattern p = Pattern.compile("^((13[0-9])|(15[^4,\\D])|(17[^4,\\D])|(18[0,5-9]))\\d{8}$");
Matcher m = p.matcher(mobile);
return m.matches();
}


public static boolean isLegal(String str) {
if (isNullOrEmpty(str)) {
return false;
}
Pattern p = Pattern.compile("[A-Za-z0-9_]{3,16}");
Matcher m = p.matcher(str);
return m.matches();
}


public static boolean isEmail(String email) {
if (isNullOrEmpty(email)) {
return false;
}
Pattern p = Pattern.compile(
"^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$");
Matcher m = p.matcher(email);
return m.matches();
}


public static boolean isMd5(String md5) {
if (isNullOrEmpty(md5)) {
return false;
}
Pattern p = Pattern.compile("[A-Za-z0-9_]{16,40}");
Matcher m = p.matcher(md5);
return m.matches();
}




public static boolean isAllNull(Object... obj) {
if (obj == null || obj.length == 0) {
return true;
}
for (int i = 0; i < obj.length; i++) {
if (!isNullOrEmpty(obj[i])) {
return false;
}
}
return true;
}


public static boolean isAllNull(List