利用Python爬虫抓取猫眼电影排行（BeautifulSoup方式初试手，欢迎改进）

from bs4 import BeautifulSoup
import requests
import json
import time
from requests.exceptions import RequestException

#获取响应内容
def get_html(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
        }

        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

#单个页面抓取
def parse_one_page(html):
    soup = BeautifulSoup(html, "lxml")
    items = soup.find_all(name='dd')
    for item in items:
        yield {
            'index': item.i.string,
            'image': item.find(class_='board-img')['data-src'],
            'title': item.find(class_='name').string,
            'actor': item.find(class_='star').string.strip()[3:],
            'time': item.find(class_='releasetime').string,
            'score': item.find(class_='score').find(class_='integer').string.strip() + item.find(class_='score').find(class_='fraction').string.strip()
        }


#写入文件
def write_to_file(content):
    with open('test.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'\n')

def main(offset):
    url = 'http://maoyan.com/board/4?offset='+str(offset)
    html = get_html(url)
    for item in parse_one_page(html):
        print(item)
        #写入文件
        #write_to_file(item)

if __name__ == '__main__':
    for i in range(10):
        main(offset=i*10)
        time.sleep(1)