python爬虫之用线程池改的Demo

  from concurrent.futures import ThreadPoolExecutor
  import requests
  from lxml.html import etree
  import threading
  #线程池的目的：创建一个线程池,里面有指定数量的线程,让线程执行任务
  def down_load_data(page):
  print(page)
  print('正在下载第' + str(page) + '页', threading.currentThread().name)
  full_url = 'http://blog.jobbole.com/all-posts/page/%s/' % str(page)
  req_header = {
      'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
  }
  response = requests.get(full_url, headers=req_header)
  if response.status_code == 200:
    # 将获取到的页面源码存到dataQueue队列中
      print('请求成功')
      return response.text,response.status_code
  def download_done(futures):
  print(futures.result())
  #可以在这里做数据的解析
  html = futures.result()[0]
  html_element = etree.HTML(html)
  articles = html_element.xpath('//div[@class="post floated-thumb"]')
  for article in articles:
      articleInfo = {}
      # 标题
      articleInfo['title'] = article.xpath('.//a[@class="archive-title"]/text()')[0]
      # 封面
      img_element = article.xpath('.//div[@class="post-thumb"]/a/img')
      if len(img_element) > 0:
          articleInfo['coverImage'] = img_element[0].xpath('./@src')[0]
      else:
          articleInfo['coverImage'] = '暂无图片'
      p_as = article.xpath('.//div[@class="post-meta"]/p[1]//a')
      if len(p_as) > 2:
          # tag类型
          articleInfo['tag'] = p_as[1].xpath('./text()')[0]
          # 评论量
          articleInfo['commentNum'] = p_as[2].xpath('./text()')[0]
      else:
          # tag类型
          articleInfo['tag'] = p_as[1].xpath('./text()')[0]
          # 评论量
          articleInfo['commentNum'] = '0'
      # 简介
      articleInfo['content'] = article.xpath('.//span[@class="excerpt"]/p/text()')[0]
      # 时间
      articleInfo['publishTime'] = ''.join(article.xpath('.//div[@class="post-meta"]/p[1]/text()')).replace('\n',
                                                                                                         '').replace(
          ' ', '').replace('\r', '').replace('·', '')
      print(articleInfo)
 if __name__ == '__main__':
  #创建线程池
  #max_workers:指点线程池中的线程数量
   pool = ThreadPoolExecutor(max_workers=10)
  for i in range(1, 201):
      #往线程池中添加任务
      handler = pool.submit(down_load_data,i)
      #设置回调方法
      handler.add_done_callback(download_done)
  #执行shutdown()内部实质是执行了join()方法
  pool.shutdown()