Python网络爬虫（二）：多线程爬取小说
这里我们爬取的小说是网站：笔趣阁，其中一本小说：一念永恒。
（一）准备阶段
1、网站URL：http://www.biqukan.com/1_1094/
2、浏览器：搜狗
3、我们打开网址，找到搜狗浏览器的审查元素（F12），可以看到class为”listmain”的div标签中的内容是我们想要的小说每个章节的链接地址。
4、我们单击其中一个章节的链接。我们发现，每个章节的正文内容放在class为”content”的div 标签中。
有了这些了解，接下来就可以编写代码。
（二）代码实现：
这里我们将整个代码工程写了四个文件，对应有四个类。
文件shenmu_main.py是程序运行的入口。
shenmu_main.py:
# -*- coding: utf-8 -*-
"""
主程序
"""
from xiaoshuo import shenmu_parse
from xiaoshuo import ProcessPoolDownload
from xiaoshuo import shenmu_downloader
class SpiderMain(object):
def __init__(self):
self.Htmldownloader= shenmu_downloader.HtmlDownLoader()
self.parse = shenmu_parse.HtmlParse()
self.PoolDown = ProcessPoolDownload.PoolDownLoader()
def TheStart(self,target_url):
#一念永恒小说目录地址
target_html=self.Htmldownloader.downloader(target_url)
new_urls=self.parse.parse(target_html)
self.PoolDown.oneprocess(new_urls)
#self.PoolDown.ThreadDown(new_urls)
if __name__ == "__main__":
target_url = 'http://www.biqukan.com/1_1094/'
Spider=SpiderMain()
Spider.TheStart(target_url)
shenmu_downloader.py实现了下载小说整个主页面，也就是包含小说各个章节链接的页面。
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 29 18:12:11 2017
@author: Administrator
"""
from urllib import request,error
class HtmlDownLoader(object):
def downloader(self,target_url,retry_count=3):
if target_url is None:
return None
try:
#User-Agent
head = {}
head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19'
target_req = request.Request(url = target_url, headers = head)
target_response = request.urlopen(target_req)
target_html = target_response.read().decode('gbk','ignore')
#print("整个页面"+target_html)
except error.URLError as e:
print("HtmlDownLoader download error:",e.reason)
target_html=None
if retry_count>0: #重试下载
if hasattr(e, 'code') and 500 <= e.code < 600:
return self.downloader(target_url,retry_count-1)
return target_html
shenmu_parse.py解析shenmu_downloader传递来的HTML。分析出各个章节的链接地址返回。
# -*- coding: utf-8 -*-
"""
用BeautifulSoup分析小说每个章节的链接
"""
from bs4 import BeautifulSoup
import queue
class HtmlParse(object):
def parse(self,target_html,html_encode="utf-8"):
if target_html is None:
return
#print(target_html)
#创建BeautifulSoup对象
listmain_soup = BeautifulSoup(target_html,'lxml')
#搜索文档树,找出div标签中class为listmain的所有子标签
chapters = listmain_soup.find_all('div',class_ = 'listmain')
#使用查询结果再创建一个BeautifulSoup对象,对其继续进行解析
download_soup = BeautifulSoup(str(chapters), 'lxml')
# print(download_soup)
new_urls=self._get_new_urls(download_soup)
return new_urls
def _get_new_urls(self,download_soup):
#print(download_soup)
#new_urls集合用于存储每章的链接
new_urls=queue.Queue()
#开始记录内容标志位,只要正文卷下面的链接,最新章节列表链接剔除
begin_flag = False
#遍历dl标签下所有子节点
for child in download_soup.dl.children:
#滤除回车
if child != '\n':
#找到《神墓》正文卷,使能标志位
if child.string == u"《一念永恒》正文卷":
begin_flag = True
#爬取链接并下载链接内容
if begin_flag == True and child.a != None:
download_url = "http://www.biqukan.com" + child.a.get('href')
new_urls.put(download_url)
return new_urls
ProcessPoolDownload.py实现了从各个章节链接下载小说正文，并写入txt文件中。下载方式有单进程和多线程。
# -*- coding: utf-8 -*-
"""
下载每个章节的内容
"""
from urllib import request
from multiprocessing import Pool
from multiprocessing import Lock
from threading import Thread #线程
from bs4 import BeautifulSoup
import sys
import threading
import datetime
import queue
class PoolDownLoader(object):
def __init__(self):
self.new_urls=queue.Queue()
self.lock=threading.Lock()
self.lock1=threading.Lock()
def WriteText(self,download_name,texts):
file = open('一念永恒1.txt', 'a', encoding='utf-8')
soup_text = BeautifulSoup(str(texts), 'lxml')
write_flag = True
file.write(download_name + '\n\n')
for each in soup_text.div.text.replace('\xa0',''):
if each == 'h':
write_flag = False
if write_flag == True and each != ' ':
file.write(each)
if write_flag == True and each == '\r':
file.write('\n')
file.write('\n\n')
#打印爬取进度
print("已下载:"+download_name + '\r')
def funcDown(self,download_url):
head = {}
head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19'
download_req = request.Request(url = download_url, headers = head)
download_response = request.urlopen(download_req)
download_html = download_response.read().decode('gbk','ignore')
soup_texts = BeautifulSoup(download_html, 'lxml')
h1=soup_texts.h1.string
texts = soup_texts.find_all(id = 'content', class_ = 'showtxt')
#print(h1)
self.lock.acquire()
self.WriteText(h1,texts)
self.lock.release()
def PoolDown(self,new_urls):
lock = Lock()
p=Pool(processes=4,initargs=(lock,))
for download_url in new_urls:
#print(download_url)
p.apply_async(self.funcDown,args=(download_url,))
p.close()
p.join()
print("小说下载完成，已写入文档！\n")
def ThreadDown(self,new_urls):
starttime=datetime.datetime.now()
self.new_urls=new_urls
threads=[]
while threads or self.new_urls:
for thread in threads:
if not thread.isAlive():
threads.remove(thread)
while len(threads)<8 and not self.new_urls.empty():
new_url=self.new_urls.get()
thread =threading.Thread(target=self.funcDown,args=(new_url,))
thread.setDaemon(True)
thread.start()
threads.append(thread)
#thread.join()
endtime=datetime.datetime.now()
print("下载小说共用了;",(endtime-starttime).seconds)
def oneprocess(self,new_urls):
starttime=datetime.datetime.now()
self.new_urls=new_urls
while not self.new_urls.empty():
new_url=new_urls.get()
self.funcDown(new_url)
endtime=datetime.datetime.now()
print("下载小说共用了：",(endtime-starttime).seconds)
（三）结果分析：
单进程结果： 可以看到单进程，是严格按照顺序下载的，下载时间较短。
多线程结果： 可以看到多线程，不是严格按照顺序下载的，下载时间比单进程要长，是因为在写入文档时加了线程锁，大型工程多线程肯定要比单进程要高效。
（四）技术讲解：
先占个坑，以后写。