python 百度贴吧爬虫（下载图片）
业余时用python写的百度贴吧爬虫程序，算是对学习python程序得一个练习。
本程序可以针对给定的贴吧链接，把帖子楼主的发言或者图片爬取出来，目前主要功能为下载所有楼主发的图片。爬取楼主发言的功能仅支持屏幕输出，没有保存到本地文件，有兴趣的朋友可以进行补充。仅供学习，转载请标明出处。
tieba_spider.py
#coding:utf-8
import urllib2,re,time,threading
import DownQueue
user_agent='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36' #模拟浏览器访问
url='http://tieba.baidu.com/p/3271638607?see_lz=1&pn=' #贴吧地址，只看楼主
header={'User-Agent' : user_agent}
g_worker=DownQueue.down() #下载器
class Tieba_Spider(threading.Thread):
def __init__(self,url,type):
threading.Thread.__init__(self)
self.url=url
self.type=type
self.num=0
def run(self):
self.start_spider()
def get_info(self):
try:
req=urllib2.Request(self.url,headers=header)
response=urllib2.urlopen(req)
htm=response.read().decode('gbk')
self.num=self.get_page_num(htm)
print 'It has %d page' % self.num
self.title=self.get_title(htm)
print 'It\'s title is %s'%self.title
except urllib2.URLError,e:
if hasattr(e,'code'):
print 'Error code :',e.code
if hasattr(e,'reason'):
print 'Reason :',e.reason
def start_spider(self):
global g_worker
self.get_info()
for i in range(1,self.num+1,1):
print 'start : ',i
try:
req=urllib2.Request(self.url+str(i),headers=header)
response=urllib2.urlopen(req)
htm=response.read().decode('gbk')
if self.type==0:
self.page_deal(htm)
elif self.type==1:
self.down_pic(htm)
except urllib2.URLError,e:
if hasattr(e,'code'):
print 'Error code :',e.code
if hasattr(e,'reason'):
print 'Reason :',e.reason
g_worker.set_flag(True)
def get_page_num(self,htm):
match=re.search(r'<span class="red">(\d*)</span>',htm)
if match:
return int(match.group(1))
else:
return 0
def get_title(self,htm):
match=re.search(r'class="core_title_txt(\s+)"(\s+)title="(.*?)"',htm)
if match:
return match.group(3)
else:
print 'no match title'
return ''
def page_deal(self,htm):
match=re.findall(r'id="post_content_(.*?)">(.*?)</div>',htm)
if match:
for it in match:
print it[1],'\n'
else:
print 'no deal'
def down_pic(self,htm):
global g_worker
match=re.findall(r'<img class="BDE_Image" pic_type=(.*?)src="(.*?)"',htm)
if match:
for it in match:
print 'picture url :',it[1],'\n'
g_worker.push(it[1])
else:
print 'no deal'
if __name__=='__main__':
spider=Tieba_Spider(url,1)#参数1为下载图片。默认为0，功能为抓取楼主的发言在屏幕上显示
spider.start()
g_worker.start()
DownQueue.py
#coding:utf-8
import threading,Queue,re,time
import urllib2
class down(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.queue=Queue.Queue(1000)
self.semaphore=threading.Semaphore(0)
self.flag=False #是否停止
def push(self,obj):
self.queue.put(obj)
self.semaphore.release()
def set_flag(self,f):
self.flag=f
def run(self):
while True:
if self.semaphore.acquire():
obj=self.queue.get()
data=urllib2.urlopen(obj).read()
pic=re.search(r'.*/(.*)',obj)
print 'dowing ',pic.group(1)
fd=open('./spider_pic/%s'%pic.group(1),'wb')
fd.write(data)
fd.close()
if self.queue.empty() and self.flag: #线程结束条件，队列为空并且退出标志为真
break
Tieba_Spider 类为爬虫类，负责爬出楼主发言中的图片链接，并将其推入down类的队列中。down类的工作为下载图片。两个类均继承自threading.Thread。仅供学习，转载请标明出处。