爬zoomeye遇到的坑和解决方案
坑一：
https的加密方式是TLSv1.2，这个导致很多以前的爬虫都不能用了，解决方法是用python 的ssl库代码如下：
参考的文章：http://www.cnblogs.com/vingi/articles/4131633.html
代码：
# -*- coding: utf-8 -*-
import httplib,ssl, urllib2, socket
class HTTPSConnectionV3(httplib.HTTPSConnection):
def __init__(self,*args,**kwargs):
httplib.HTTPSConnection.__init__(self,*args,**kwargs)
def connect(self):
sock= socket.create_connection((self.host,self.port),self.timeout)
if self._tunnel_host:
self.sock= sock
self._tunnel()
try:
self.sock= ssl.wrap_socket(sock,self.key_file,self.cert_file,ssl_version=ssl.PROTOCOL_TLSv1)
except ssl.SSLError,e:
print("TryingSSLv3.")
self.sock= ssl.wrap_socket(sock,self.key_file,self.cert_file,ssl_version=ssl.PROTOCOL_SSLv23)
class HTTPSHandlerV3(urllib2.HTTPSHandler):
def https_open(self,req):
return self.do_open(HTTPSConnectionV3,req)
#安装证书
urllib2.install_opener(urllib2.build_opener(HTTPSHandlerV3()))
if __name__== "__main__":
url="https://www.zoomeye.org/"
heads={"User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0"}
try:
r=urllib2.Request(url,headers=heads)
resual= urllib2.urlopen(r)
print resual.read()
except urllib2.URLError, e:
print e.read()
输出的是：
<script>var dc="";var t_d={hello:"world",t_c:function(x){if(x==="")return;if(x.slice(-1)===";"){x=x+" ";};if(x.slice(-2)!=="; "){x=x+"; ";};dc=dc+x;}};(function(a){eval(function(p,a,c,k,e,d){e=function(c){return(c<a?"":e(parseInt(c/a)))+((c=c%a)>35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)d[e(c)]=k[c]||e(c);k=[function(e){return d[e]}];e=function(){return'\\w+'};c=1;};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p;}('b d=[3,4,1,2,0];b o=[];b p=0;g(b i=0;i<d.f;i++){d[i]=a[d[i]]}d=d.j(\'\').l(\';\');g(b i=0;i<d.f;i++){h.m(d[i])}n("e.c=e.c.q(/[\\?|&]s-r/, \'\')",k);',29,29,'|||||||||||var|href||location|length|for|t_d||join|1500|split|t_c|setTimeout|||replace|challenge|captcha'.split('|'),0,{}));})(['14-Feb-16 04:46:09 GMT;Path=/;', 'XHhxKtGuWK1VUCL37XI%3D;Exp', 'ires=Sun, ', '__jsl_clearance=145542', '1569.853|0|kkQeowPJ']);document.cookie=dc;</script>
坑二：
虽然安装了正确的证书，但是还是不能拿到正常的网页，通过抓包，分析包，我发现必须带上正确的cookie才能请求到正常的数据
如何才能拿到正确的cookie呢？这里用到一个python的webkit：Ghost.py
参考的文章：
http://pyqt.sourceforge.net/Docs/PyQt4/qnetworkcookie.html#parseCookies
http://jeanphix.me/Ghost.py/
http://ghost-py.readthedocs.org/en/latest/
https://github.com/jeanphix/Ghost.py
代码：
import ghost
g = ghost.Ghost()
with g.start() as session:
head={"Referer": "https://www.zoomeye.org/"}
page, extra_resources = session.open("https://www.zoomeye.org/",method='get', headers=head,wait=True,encode_url=True, user_agent="Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0")
for element in session.cookies:
print element.toRawForm()
输出：
__jsluid=a94170e7f36908f508f4bdbfdcb69461; domain=www.zoomeye.org; path=/ __jsl_clearance=1455422745.375|0|ZTSyyuKTql2jEPtDBqdlp1xe1d8%3D; expires=Sun, 14-Feb-2016 05:05:45 GMT; domain=www.zoomeye.org; path=/
最后将两段代码整合一下就能得到正常的网页数据了
代码：
# -*- coding: utf-8 -*-
import httplib,ssl, urllib2, socket, ghost
from bs4 import BeautifulSoup
#https证书安装类
class HTTPSConnectionV3(httplib.HTTPSConnection):
def __init__(self,*args,**kwargs):
httplib.HTTPSConnection.__init__(self,*args,**kwargs)
def connect(self):
sock= socket.create_connection((self.host,self.port),self.timeout)
if self._tunnel_host:
self.sock= sock
self._tunnel()
try:
self.sock= ssl.wrap_socket(sock,self.key_file,self.cert_file,ssl_version=ssl.PROTOCOL_TLSv1) #选择证书类型
except ssl.SSLError,e:
print("TryingSSLv3.")
self.sock= ssl.wrap_socket(sock,self.key_file,self.cert_file,ssl_version=ssl.PROTOCOL_SSLv23)
class HTTPSHandlerV3(urllib2.HTTPSHandler):
def https_open(self,req):
return self.do_open(HTTPSConnectionV3,req)
#安装证书
urllib2.install_opener(urllib2.build_opener(HTTPSHandlerV3()))
#获取zoomeye的cookie
def getcookie():
g = ghost.Ghost()
with g.start() as session:
mycookielist=[]
head={"Referer": "https://www.zoomeye.org/"}
page, extra_resources = session.open("https://www.zoomeye.org/",method='get', headers=head,wait=True,encode_url=True, user_agent="Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0")
for element in session.cookies:
mycookielist.append(element.toRawForm().split(";"))
cookiestr= mycookielist[0][0]+";"+mycookielist[1][0]
return cookiestr
#发送http请求
def sendhttp():
url="https://www.zoomeye.org/"
heads={"User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0"}
#heads["Cookie"]="__jsluid=260219e8b0e3b3574a2e28d15e8249ff;__jsl_clearance=1455352838.397|0|uSSCfF4umEIhqMz8GpcSDVBL9MI%3D"
#heads["Cookie"]="__jsluid=9222a6c2cf192a864941182010a25d9b;__jsl_clearance=1455356240.603|0|50xi3gYoENuTPXaeWIhLW%2F00C2Y%3D;"
heads["Cookie"]=getcookie()
try:
r=urllib2.Request(url,headers=heads)
resual= urllib2.urlopen(r)
return resual.read()
except urllib2.URLError, e:
print e.read()
#主程序
if __name__== "__main__":
htmlstr =sendhttp()
print htmlstr