抓baidu数据,但跑不了多久就假死,无反应也不报错,初学python,搞了两礼拜没搞明白,望各位前辈指点下
#coding:utf-8
'''
百度排名查询,代理多线程版本
'''
import StringIO,pycurl,time,random,re,os,csv
from threading import Thread,Lock
from Queue import Queue
from bs4 import BeautifulSoup as bs
csvfile = open('serp_html.csv','wb') #存放关键词和搜索结果页源码的文件
bdjd_dict = {}
#bdjd_list = ["www.baidu.com","180.97.33.107","115.239.210.27","180.97.33.108","180.97.33.107","180.97.33.107","180.97.33.108","220.181.111.188","220.181.111.188","180.97.33.107","180.97.33.107","115.239.211.112","180.97.33.108","180.97.33.108","180.97.33.108","180.97.33.108","180.97.33.108","115.239.211.112","180.97.33.108","115.239.211.112","115.239.210.27","180.97.33.108","115.239.211.112","115.239.210.27","180.97.33.108","115.239.210.27","61.135.169.125","115.239.211.112","115.239.210.27","180.97.33.107","180.97.33.107","180.97.33.108","115.239.210.27","180.97.33.107","61.135.169.121","115.239.210.27","61.135.169.121","61.135.169.125","115.239.211.112","115.239.210.27","61.135.169.125","112.80.248.73","61.135.169.121","112.80.248.74","112.80.248.73","61.135.169.125","180.97.33.108","115.239.210.27","61.135.169.125","61.135.169.125","112.80.248.74","112.80.248.74","61.135.169.121","115.239.210.27","61.135.169.125","111.13.100.92","111.13.100.92","111.13.100.91","111.13.100.91","115.239.211.112","111.13.100.92","111.13.100.91","111.13.100.92","115.239.211.112","115.239.210.27","115.239.211.112","115.239.210.27","115.239.210.27","115.239.210.27","115.239.210.27"]
bdjd_list = ["www.baidu.com"]
#提取百度地域节点
def getBDJD(bdjd_str):
bdjd_list = bdjd_str.split(',')
bdjd = random.choice(bdjd_list)
return bdjd
def baidu_url(word): #百度搜索url
return 'http://www.baidu.com/s?wd=%s' % (bdjd,word)
daili_list = [] #存储代理ip
#读取代理文件,随机提取1个代理
def ip():
for x in open('hege_daili.txt'):
x = x.strip()
daili_list.append(x)
newip = random.choice(daili_list)
return newip
#如果代理不可用,则从代理文件中删除,此函数在baidu_cout中应用
def daili_delete(ip):
dailifile = open('daili_beifen.txt','w')
for line in open('hege_daili.txt'):
line = line.strip()
if ip not in line:
dailifile.write(line+"\n")
os.system("mv daili_beifen.txt hege_daili.txt")
def baidu_url(word): #百度搜索url
return 'http://www.baidu.com/s?wd=%s' % word
def getUA():
uaList = [
'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+1.1.4322;+TencentTraveler)',
'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+3.0.4506.2152;+.NET+CLR+3.5.30729)',
'Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.1+(KHTML,+like+Gecko)+Chrome/21.0.1180.89+Safari/537.1',
'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1)',
'Mozilla/5.0+(Windows+NT+6.1;+rv:11.0)+Gecko/20100101+Firefox/11.0',
'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+SV1)',
'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+GTB7.1;+.NET+CLR+2.0.50727)',
'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+KB974489)',
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36'
]
ua = random.choice(uaList)
return ua
def baidu_cont(url,headers,ip): #百度搜索结果页源码
while 1:
try:
c = pycurl.Curl()
c.setopt(pycurl.MAXREDIRS,5)
c.setopt(pycurl.REFERER, url)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.CONNECTTIMEOUT, 60)
c.setopt(pycurl.TIMEOUT,120)
c.setopt(pycurl.ENCODING,'gzip,deflate')
c.setopt(c.PROXY,ip)
c.fp = StringIO.StringIO()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.HTTPHEADER,headers)
c.setopt(c.WRITEFUNCTION, c.fp.write)
c.perform()
#code = c.getinfo(c.HTTP_CODE) 返回状态码
html = c.fp.getvalue()
if '="http://verify.baidu.com' in html:
time.sleep(1200)
print '重启'
continue
return html
except Exception, what:
information = '错误信息:%s' % what
return str(information)
continue
def search(req,line):
text = re.search(req,line)
if text:
data = text.group(1)
else:
data = 'no'
return data
url_list = []
for word in open('word'):
word = word.strip()
url_list.append(word)
class Fetcher:
def __init__(self,threads):
self.lock = Lock() #线程锁
self.q_req = Queue() #任务队列
self.q_ans = Queue() #完成队列
self.threads = threads
for i in range(threads):
t = Thread(target=self.threadget) #括号中的是每次线程要执行的任务
t.setDaemon(True) #设置子线程是否随主线程一起结束,必须在start()
#之前调用。默认为False
t.start() #启动线程
self.running = 0 #设置运行中的线程个数
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
self.q_req.join() #Queue等待队列为空后再执行其他操作
self.q_ans.join()
#返回还在运行线程的个数,为0时表示全部运行完毕
def taskleft(self):
return self.q_req.qsize()+self.q_ans.qsize()+self.running
def push(self,req):
self.q_req.put(req)
def pop(self):
return self.q_ans.get()
#线程执行的任务,根据req来区分
def threadget(self):
while True:
line = self.q_req.get()
word = line.strip()
'''
Lock.lock()操作,使用with可以不用显示调用acquire和release,
这里锁住线程,使得self.running加1表示运行中的线程加1,
如此做防止其他线程修改该值,造成混乱。
with下的语句结束后自动解锁。
'''
with self.lock:
self.running += 1
bdjd_str = ','.join(bdjd_list)
newip = ip()
bdjd = getBDJD(bdjd_str)
url = baidu_url(word)
headers = [
"Accept:*/*",
"Accept-Encoding:gzip, deflate, sdch",
"Accept-Language:zh-CN,zh;q=0.8,en;q=0.6",
"Connection:keep-alive",
#"Cookie:BIDUPSID=4DEE9B78AA3A97A51CFC916C43C30EC6; BAIDUID=380B58B009DF49761A4D9C30E19B34D0:FG=1; B64_BOT=1; PSTM=1432281826; BDRCVFR[G4oNZs7I7B3]=duGuzxHDPeTmy-lpA78QhPEUf; BDUSS=X5PYndScmJYc3lPSm1yQy15Vlg2YW45cWl3ZVFEalFjckFSNk16NHFIZTZkWVpWQVFBQUFBJCQAAAAAAAAAAAEAAAAJkstJv7TXvMTj1NnM-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALroXlW66F5Vc; SFSSID=ap4nu9shvsoc52f5nvnq6nev42; uc_login_unique=95684e36f6d0a03b984187546ec66f06; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a01823607022; BDRCVFR[ltbVPlNi2ac]=mk3SLVN4HKm; BD_HOME=1; BD_UPN=32; sug=3; sugstore=1; ORIGIN=0; bdime=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_645EC=5cdfPyG9PHeV%2BYlX03NQsmg4%2BZ0wlUr4XqxnORGloYCIlW8tFgZVbm9AliNr1w2Ls75g; BD_CK_SAM=1; BDSVRTM=106; H_PS_PSSID=13783_1454_13519_13075_12824_12867_14167_10562_12722_14155_14172_13202_14329_11951_13936_13741_14369_14182_8498_14195; WWW_ST=1432372813391",
"Host:www.baidu.com",
"RA-Sid:7739A016-20140918-030243-3adabf-48f828",
"RA-Ver:2.10.3",
"Referer:http://www.baidu.com/",
"X-Requested-With:XMLHttpRequest",
"User-Agent:%s" % getUA()
]
html = baidu_cont(url, headers, newip)
soup = bs(html)
b_tags = soup.find_all('div', {'class': 'result c-container '})
for line in b_tags:
newline = str(line)
number = search(r'id="(\d+)"',newline)
urldiv = search(r'(.*?)',newline) #获取源码中domain所在的
data = []
data.append(word)
data.append(newline)
writer = csv.writer(csvfile,dialect='excel')
writer.writerow(data)
if len(b_tags) == 0:
print html
else:
print '》》当前IP:%s,已抓取:%s,返回%s条结果' % (newip,word,len(b_tags))
#self.q_ans.put((req,ans)) # 将完成的任务压入完成队列,在主程序中返回
self.q_ans.put(word)
with self.lock:
self.running -= 1
self.q_req.task_done() # 在完成一项工作之后,Queue.task_done()
# 函数向任务已经完成的队列发送一个信号
time.sleep(0.1) # don't spam
if __name__ == "__main__":
#links = [ 'http://www.verycd.com/topics/%d/'%i for i in range(5420,5450) ]
f = Fetcher(threads=10) #设置线程数为10
for url in url_list:
f.push(url) #所有url推入下载队列
while f.taskleft(): #若还有未完成的的线程
f.pop() #从下载完成的队列中取出结果
'''
# 如果百度节点超时次数》10,则从百度节点列表中删除
# if '错误信息' in html:
# print html
# if 'Connection refused' in html:
# #判断访问超时的节点存入字典,若该节点已超过10次链接超时,则从节点列表中删除
# if bdjd_dict.has_key(bdjd):
# bdjd_dict[bdjd] += 1
# print '节点:%s,已%s次超时' % (bdjd,bdjd_dict[bdjd])
# if int(bdjd_dict[bdjd]) >= 10:
# bdjd_list.remove(bdjd)
# print "节点:%s 已删除" % bdjd
# else:
# bdjd_dict[bdjd] = 1
# continue
'''