Python 旅游博客爬虫
爬的是柳絮同学的旅游博客，将所有文章
和文字内容爬取到一个文件夹里，以txt格式保存
其中有一行可能跟浏览器有关系 格式不太正确 下图这样
#coding:utf8
import urllib2
import re
import urlparse
import cookielib
from bs4 import BeautifulSoup
def download(url):
i_headers = {"User_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"}
if url is None:
return
request = urllib2.Request(url,i_headers)
response = urllib2.urlopen(url)
if response.getcode()!=200:
return
initial_html=response.read()
clean_html=''.join(re.split(r"<\!–\[if lte IE 6\]>|<\!\[endif\]–>", initial_html))
return clean_html
def pageselect(tag): #自定义的find函数
check_classes=['SG_pgon','SG_pgnext']
if(tag.has_attr('class')):
for class_ in tag.get('class'):
for check_class in check_classes:
if class_==check_class:
return False
return tag.name=='li'
def getpageurl(page_url,): #获取页面信息
soup = BeautifulSoup(open('output.html'), 'html.parser', from_encoding='utf-8')
lis=soup.find('ul',class_='SG_pages').find_all(pageselect)
for li in lis:
page_url.append(li.find('a')['href'])
def urlmanager(page_url,root_content,urls,titles):
getpageurl(page_url) #获取所有的页面url 其中root_url为第一页的内容
curr_content=root_content
count_=-1
len_page=len(page_url)
while True:
soup = BeautifulSoup(curr_content, 'html.parser', from_encoding='utf-8')
spans=soup.find_all('span',class_="atc_title")
for span in spans:
link= span.find('a')
urls.append(link['href'])
cleartitle=re.sub("[\?\*\/\<\>\\\]",'',link.get_text().replace(':','：')) #清除不能作为文件名的字符，并且将其中的冒号替换
titles.append(cleartitle)
count_=count_+1
if count_==len_page:
break
curr_content=download(page_url[count_]) #遍历所有页面
# return urls
def gettext(urls,texts,titles): #从所有url里面获取文章内容
count_=0
print len(urls)
for i in range(0,len(urls)):
soup=BeautifulSoup(download(urls[i]),'html.parser',from_encoding='utf-8')
textdiv=soup.find('div',id='sina_keyword_ad_area2')
texts.append(textdiv.get_text())
print "已经爬取第%s篇文章" % count_
filesrc=['travel\\',titles[i],'.txt']
fout=open(str(i).join(filesrc),"w+")
fout.write('%s'% texts[i])
fout.close()
if __name__ == '__main__':
url=[]
titles=[]
texts=[]
page_url=[]
root_url="http://blog.sina.com.cn/s/articlelist_1776757314_0_1.html"
root_content=download(root_url)
urlmanager(page_url,root_content,url,titles)
gettext(url,texts,titles)
结果(总计258篇）：
“