数据通关分析demo1 # 爬取豆瓣电影的清单与得分

错误分析

下面展示源代码`。

// An highlighted block
# 简易demo，爬取豆瓣电影的清单与得分
import json
import re
import requests
from requests import RequestException

# 获取网页的内容，以文本形式返回
def get_page(url):
    headers = {
    'Cookie':'OCSSID=4df0bjva6j7ejussu8al3eqo03',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                 '(KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
    try:
        response = requests.get(url)#,headers=headers 
        print(response.status_code)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

# 对网页文本内容进行解析
def parse_page(html):
    pattern = re.compile('.*?', re.S)
    items = re.findall(pattern, html.decode('utf-8'))
    for item in items:#.encode('utf-8')
        yield{
            'title': item[0],
            'score': item[1],
            'image': item[2],}

# 爬虫主程序，同时print效果
def main():
    url = "https://movie.douban.com/cinema/nowplaying/beijing/"#https://movie.douban.com/cinema/nowplaying/beijing/
    html = get_page(url)
    print(html)
    for item in parse_page(html):
        print(item)

if __name__ == '__main__':
    main()

错误1：
修改如下

    items = re.findall(pattern, html.encode('utf-8').decode('utf-8'))

错误2：

此处可以看到print出来none ，

      print(response.status_code)

输出418
这是因为爬虫没有伪装成浏览器访问，所以要加上headers

    headers = {
    'Cookie':'OCSSID=4df0bjva6j7ejussu8al3eqo03',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                 '(KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
    try:
        response = requests.get(url,headers=headers )

此时解决错误