python爬虫的多进程，多线程和协程1

1. 多线程版本

import ssl
import threading
import time
import urllib.request
from queue import Queue
from lxml import etree
 
 
class myThread(threading.Thread):
    def __init__(self,url,q):
        super(myThread, self).__init__()
        self.q = q
        self.headers = {
 
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
 
        }
        self.url = url
 
    def run(self):
        self.parse_html()
 
    def send_request(self, url):
        request = urllib.request.Request(url=url, headers=self.headers)
        #我这里需要加上ssl认证
        ssl._create_default_https_context = ssl._create_unverified_context
        response = urllib.request.urlopen(request)
 
        html_response = response.read().decode('utf-8')
 
        return html_response
 
    def parse_html(self):
        resp = self.send_request(self.url)
        html = etree.HTML(resp)
        div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
        for div in div_list:
            name = div.xpath(".//img/@alt")[0]
            src = div.xpath(".//img/@src2")[0]
            # print(src)
            # print(name)
            self.q.put(name + '\t' + src)
 
def main():
    q = Queue()
    base_url = 'http://sc.chinaz.com/tupian/renwutupian_'
 
    url_list = [base_url + str(num)+'.html' for num in range(2, 100)]
    print(url_list)
    Thread_list = []
    # 创建并启动线程
    for url in url_list:
        p = myThread(url, q)
        p.start()
        Thread_list.append(p)
  # 让主线程等待子线程执行完成
    for i in Thread_list:
        i.join()
    #打印队列里存入的信息
    while not q.empty():
        print(q.get())
 
 
if __name__ == "__main__":
    start = time.time()
    main()
    print('总共耗时：%s' % (time.time() - start))

2. 多进程版本

import multiprocessing
import ssl
import time
import urllib.request
from queue import Queue
 
from lxml import etree
 
 
class ZhanzProcess(multiprocessing.Process):
 
    def __init__(self, url, q):
        super(ZhanzProcess, self).__init__()
 
        self.q = q
        self.url = url
        self.headers = {
 
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
 
        }
 
    def run(self):
        self.parse_html()
 
    def send_request(self, url):
        request = urllib.request.Request(url=url, headers=self.headers)
        ssl._create_default_https_context = ssl._create_unverified_context
        response = urllib.request.urlopen(request)
        html_response = response.read().decode('utf-8')
 
        return html_response
 
    def parse_html(self):
        res = self.send_request(self.url)
        html = etree.HTML(res)
        div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
        for div in div_list:
            name = div.xpath(".//img/@alt")[0]
            src = div.xpath(".//img/@src2")[0]
            # print(name + '\t' + src)
 
            self.q.put(name + '\t' + src)
 
 
def main():
    q = Queue()
    base_url = 'http://sc.chinaz.com/tupian/renwutupian_'
 
    url_list = [base_url + str(num) + '.html' for num in range(2, 100)]
    print(url_list)
    process_list = []
    for url in url_list:
        # print(url)
        p = ZhanzProcess(url, q)
        p.start()
        process_list.append(p)
 
    for i in process_list:
        i.join()
 
    while not q.empty():
         print(q.get())
 
 
 
if __name__ == "__main__":
    start = time.time()
    main()
    print('总共耗时：%s' % (time.time() - start))

3. 协程版本

import ssl
import time
import urllib.request
from queue import Queue
 
import gevent as gevent
from lxml import etree
 
# 打猴子补丁
from gevent import monkey
 
monkey.patch_all()
 
 
class Zzspider(object):
 
    def __init__(self):
        self.q = Queue()
        self.headers = {
 
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
 
        }
 
    def run(self, url):
        self.parse_html(url)
 
    def send_request(self, url):
        request = urllib.request.Request(url=url, headers=self.headers)
        ssl._create_default_https_context = ssl._create_unverified_context
        response = urllib.request.urlopen(request)
        html_response = response.read().decode('utf-8')
 
        return html_response
 
    def parse_html(self, url):
        res = self.send_request(url)
        html = etree.HTML(res)
        div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
        for div in div_list:
            name = div.xpath(".//img/@alt")[0]
            src = div.xpath(".//img/@src2")[0]
 
            self.q.put(name + '\t' + src)
 
    def main(self):
        base_url = 'http://sc.chinaz.com/tupian/renwutupian_'
 
        url_list = [base_url + str(num) + '.html' for num in range(2, 100)]
        job_list = [gevent.spawn(self.run, url) for url in url_list]
        # 让线程等待所有任务完成，再继续执行。
        gevent.joinall(job_list)
        while not self.q.empty():
            print(self.q.get())
 
 
if __name__ == '__main__':
    start = time.time()
    zz = Zzspider()
    zz.main()
    print('总共耗时：%s' % (time.time() - start))

原文：https://blog.csdn.net/qq_32903613/article/details/87938409