Day02 Requests+正则表达式爬取猫眼电影

源代码如下：

useragents.py

import random

useragent = '''Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1
Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1
Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Maxthon2.0)
Mozilla/5.0(Macintosh;U;IntelMacOSX10_6_8;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50
Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50
Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;Trident/4.0)
Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Maxthon2.0)'''


def get_useragent():
    useragent_list = useragent.split('\n')
    length = len(useragent_list)
    return useragent_list[random.randint(0, length - 1)]


if __name__ == '__main__':
    print(get_useragent())

spider.py

import json
import requests
from requests.exceptions import RequestException
from useragent import get_useragent
import re
from multiprocessing import Pool


# 获取单个url的源文件
def get_one_page(url, headers):
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None


# 正则解析页面
def parse_one_page(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">(.*?)'
                         '.*?star.*?>(.*?)
.*?releasetime">(.*?)
'
                         '.*?integer">(.*?).*?fraction">(\d+).*?
', re.S)

    items = re.findall(pattern, html)  # items 是列表 ，元素是原组

    for item in items:
        yield {
            'index': item[0],
            'image_link': item[1],
            'title': item[2],
            'actor': item[3].strip()[3:],
            'time': item[4].strip()[5:],
            'score': item[5] + item[6]
        }


def write_to_file(content):
    with open('result.txt', 'a', encoding='UTF-8') as f:
        # json.dumps 是将字典转换成sr
        f.write(json.dumps(content, ensure_ascii=False) + '\n')  # 写入文件乱码：需要指定 open 和 json中的编码
        f.close()


def main(offset):
    url = "http://maoyan.com/board/4?offset=" + str(offset) # 添加offset 批量抓取页面
    headers = {"User-Agent": get_useragent()}  # 随机取得useragent

    html = get_one_page(url, headers)
    for item in parse_one_page(html):
        write_to_file(item)
        print(item)


if __name__ == '__main__':
    for i in range(10):
        main(i*10)

    # 使用线程池，但是有个问题，不是按照顺序抓取的
    # pool = Pool()
    # pool.map(main,[i*10 for i in range(10)])