爬虫多进程

# coding=utf-8
import requests
from lxml import etree
import threading
from multiprocessing import Process
from multiprocessing import JoinableQueue as Queue


class QiubaiSpider:
    def __init__(self):
        self.url_temp = "https://www.qiushibaike.com/8hr/page/{}/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"}
        self.url_queue = Queue()  # 保存url
        self.html_queue = Queue()  # 保存html字符串
        self.content_queue = Queue()  # 保存提取到的数据

    def get_url_list(self):
        for i in range(1,21):
            self.url_queue.put(self.url_temp.format(i))

    def parse_url(self):
        while True:
            url = self.url_queue.get()
            print(url)
            html_str = requests.get(url, headers=self.headers).content.decode()
            self.html_queue.put(html_str)
            self.url_queue.task_done()

    def get_content_list(self):
        while True:
            html_str = self.html_queue.get()
            html = etree.HTML(html_str)
            div_list = html.xpath("//div[@id='content-left']/div")
            content_list = []
            for div in div_list:
                content = {}
                content["content"] = div.xpath(".//div[@class='content']/span/text()")
                content_list.append(content)
            self.content_queue.put(content_list)
            self.html_queue.task_done()

    def save_content_list(self):
        while True:
            content_list = self.content_queue.get()
            pass
            self.content_queue.task_done()

    def run(self):
        process_list = []
        # 1. url_list
        t_url = Process(target=self.get_url_list)
        process_list.append(t_url)
        # 2. 遍历，发送请求
        for i in range(10):  # 创建5个子进程
            t_parse = Process(target=self.parse_url)
            process_list.append(t_parse)
        # 3. 提取数据
        t_content = Process(target=self.get_content_list)
        process_list.append(t_content)
        # 4. 保存
        t_save = Process(target=self.save_content_list)
        process_list.append(t_save)

        for t in process_list:
            t.daemon = True  # 把进线程设置为守护线程，主进程技术，子进程结束
            t.start()

        for q in [self.url_queue, self.html_queue, self.content_queue]:
            q.join()  # 让主进程阻塞

        print("主进程结束")


if __name__ == '__main__':
    qiubai = QiubaiSpider()
    qiubai.run()