python 多线程爬虫(京东iPhone)

# 多线程模块
import threading
# 队列模块
import queue
import requests
from lxml import etree
import time
import random
import json
import pymongo

# 采集线程数
concurrent = 3
# 解析线程数
conparse = 3


class Crawl(threading.Thread):
    """
    主要作用根据特定URL获取响应信息
    """

    def __init__(self, number, req_list, data_list):
        # 调用Thread父类方法
        super(Crawl, self).__init__()
        # 初始化子类属性
        self.number = number
        self.req_list = req_list
        self.data_list = data_list
        self.headers = {
            "authority": "search.jd.com",
            "method": "GET",
            "path": "/s_new.php?keyword=iphone&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&suggest=1.his.0.0",
            "scheme": "https",
            "cookie": "shshshfpa=817b0820-cebe-1aa5-5eaa-8f8d3b5945b3-1564123066; shshshfpb=nyq8Po%20NsVQIn1ih2zomJlw%3D%3D; TrackID=167xINX9YcKrGWU-JQ0EQWhwYstFi3gNOxFq6Em4_l6J6OECY5-pwyzHxRFr6TTZkLHI1m_3orstgEzPhWGk1pkbfG_ASOMSSscDY_oEz4XQ; pinId=DRR64H7p6D2CxuR9knABB7V9-x-f3wj7; qrsc=3; __jdu=502413746; areaId=2; PCSYCityID=CN_310000_310100_310112; xtest=3925.cf6b6759; ipLoc-djd=2-2825-51931-0; rkv=V0600; user-key=88fa50f5-ec38-48ff-9efe-beecaa5ffc96; cn=0; unpl=V2_ZzNtbRdfQEF8DRMDeR9ZDGIHFAhKUhcRd1tBVnNLXAcwB0FbclRCFX0UR1xnGlgUZwMZWEpcRxVFCEdkeBBVAWMDE1VGZxBFLV0CFSNGF1wjU00zQwBBQHcJFF0uSgwDYgcaDhFTQEJ2XBVQL0oMDDdRFAhyZ0AVRQhHZH8YXwBnARRYRWdzEkU4dlB8G1oEVwIiXHIVF0l1CkJRfxkRAWYAF11AUUYSRQl2Vw%3d%3d; __jdv=76161171|baidu-pinzhuan|t_288551095_baidupinzhuan|cpc|0f3d30c8dba7459bb52f2eb5eba8ac7d_0_d92e84df3748457d94e53b639c13f5b7|1569383391707; __jda=122270672.502413746.1564123065.1569380256.1569383392.14; __jdc=122270672; shshshfp=f5c7274df7eb1773b86a72faa494fed4; 3AB9D23F7A4B3C9B=SHZYIPK2KZYMKRXEKXFSLWSYQOVLA745EZ4NXQTNMFLQMVFRJKZT7VYDEIY6L2USE2KDEOHO2IPNACME4W7GJ2LKTM",
            "referer": "https://search.jd.com/Search?keyword=iphone&enc=utf-8&suggest=1.his.0.0&wq=&pvid=ec4030074500424391b371d06a8a62fd",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
        }

        # 线程启动的时候调用

    def run(self):
        print("启动采集线程{}号".format(self.number))
        # 如果请求队列不为空， 则无限循环， 从请求队列里面请求URL
        while self.req_list.qsize() > 0:
            # 从请求队列里提取URL
            url = self.req_list.get()
            print("{}号线程采集：{}".format(self.number, url))
            # 防止请求频率过快，随机设置阻塞时间
            time.sleep(random.randint(1, 3))
            # 发起http请求，获取响应内容， 追加到数据队列里，等待解析
            response = requests.get(url, headers=self.headers)
            if response.status_code == 200:
                # 向数据队列里面追加
                self.data_list.put(response.text)


class Parse(threading.Thread):
    """
    将响应信息进行数据解析
    """

    def __init__(self, number, data_list, req_thread, f):
        super(Parse, self).__init__()
        # 线程编号
        self.number = number
        # 数据队列
        self.data_list = data_list
        # 请求队列，为了判断采集线程存活状态
        self.req_list = req_thread
        # 获取文件对象
        self.f = f
        # 判断是否从数据队列里提取数据
        self.is_parse = True

    def run(self):
        print("启动{}号解析线程".format(self.number))
        # 无线循环
        while True:
            # 如何判断解析线程的结束条件
            for t in self.req_list:
                # 判断解析线程是否存活
                if t.is_alive():
                    # 如果线程存活，代表请求没有完成，跳出整个循环，从新开始while True
                    break
            else:  # 如果循环完毕，没有执行break语句，则进入else
                if self.data_list.qsize() == 0:
                    # 设置解析False
                    self.is_parse = False
            # 判断是否继续解析
            if self.is_parse:
                try:
                    # 从数据队列里面提取一个数据
                    data = self.data_list.get(timeout=3)
                except Exception as E:  # 超时以后进入异常
                    data = None
                # 如歌成功拿到数据，则条用解析方法
                if data is not None:
                    self.parse(data)
            else:
                # 结束while 无线循环
                break
        print("退出{}号解析线程".format(self.number))

    def parse(self, data):
        html = etree.HTML(data)
        # 获取多有li
        lis = html.xpath('//li[contains(@class,"gl-item")]')
        for li in lis:
            href = li.xpath('div/div[@class="p-img"]/a/@href')
            title = li.xpath('div/div[@class="p-img"]/a/@title')
            price = li.xpath('div/div[@class="p-price"]/strong/i/text()')
            name = li.xpath('div/div[@class="p-name p-name-type-2"]/a/em/text()')
            commit = li.xpath('div/div[@class="p-commit"]//text()')
            shop = li.xpath('div/div[@class="p-shop"]/span/a/text()')
            icons = li.xpath('div/div[@class="p-icons"]/i/text()')
            stock = li.xpath('div/div[@class="p-stock"]/text()')
            item = {
                "href": "https:" + href[0],
                "title": str(title[0]).replace("\r\n\t", ""),
                "price": " ".join(price),
                "name": str(" ".join(name)).replace("\r\n\t", "").strip(),
                "commit": str(" ".join(commit)).replace("\r\n\t", "").strip(),
                "shop": " ".join(shop),
                "icons": " ".join(icons),
                "stock": " ".join(stock)
            }
            self.f.write(json.dumps(item, ensure_ascii=False) + "\n")
            self.save_to_mongo(item)

    def save_to_mongo(self, result):
        MONGO_URL = "localhost"
        MONGO_DB = "JingDong"
        MONGO_COLLECTION = "multithreading_iphone"
        client = pymongo.MongoClient(MONGO_URL)
        db = client[MONGO_DB]
        try:
            db[MONGO_COLLECTION].insert_one(result)
        except Exception:
            print("存储到MongoDB失败")


def main():
    # 生成请求队列
    req_list = queue.Queue()
    # 生成数据队列，请求以后，响应内容放到数据队列里
    data_list = queue.Queue()
    # 创建文件对象
    f = open(r"G:\个人总结\csv\JD_Iphone.json", "w", encoding="utf-8")
    # 循环生成多个请求URL
    ms = time.time()
    for i in range(100):
        base_url = None
        if (i + 1) % 2 != 0:
            base_url = "https://search.jd.com/s_new.php?keyword=iphone&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&suggest=1.his.0.0&page={0}&s={1}&click=0".format(
                i + 1, i * 30 + 1)
        else:
            base_url = "https://search.jd.com/s_new.php?keyword=iphone&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&suggest=1.his.0.0&page={0}&s={1}&scrolling=y&log_id={2}".format(
                i + 1, i * 30 + 1, '%.5f' % (ms)
            )
        req_list.put(base_url)
    # 生成N个采集线程
    req_thread = []
    for i in range(concurrent):
        t = Crawl(i + 1, req_list, data_list)
        t.start()
        req_thread.append(t)
    # 生成N个解析线程
    parse_thread = []
    for i in range(conparse):
        t = Parse(i + 1, data_list, req_thread, f)
        t.start()
        parse_thread.append(t)
    for t in req_thread:
        t.join()
        for tt in parse_thread:
            tt.join()
    f.close()


if __name__ == "__main__":
    main()