pycharm多线程爬取猫眼电影名字

pycharm多线程爬取猫眼电影名字

from multiprocessing.dummy import Pool as pl
from lxml import etree
import requests

headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0'}
pre_url = 'https://maoyan.com/board/4?offset='

def download(url):
    html = requests.get(url, headers=headers)
    return etree.HTML(html.text)

def spider(url):
    selector = download(url)
    dianying_list = selector.xpath('//*[@id="app"]/div/div/div[1]/dl/dd')
    for dianying in dianying_list:
        aparment = dianying.xpath("div/div/div[1]/p[1]/a/text()")
        print(aparment)

if __name__ == '__main__':
    pool = pl()
    maoyan_url = [pre_url+str(x*10) for x in range(0, 10)]
    pool.map(spider, maoyan_url)
    pool.close()
    pool.join()