#抓取猫眼电影排行，以文件的形式保存结果
import json
import requests
from requests.exceptions import RequestException
import re
import time
#抓取第一页内容
def get_one_page(url):
try: #此处的cookies,headers,params需要根据自己浏览器登陆猫眼电影之后后台生成并经过相应网
#站处理，具体见之前的博客——爬取淘宝商品信息实例
cookies = {
'__mta': '208109101.1580868743910.1580869334708.1580869391871.6',
'uuid_n_v': 'v1',
'uuid': 'F1A445C047BC11EAB2946104D9850EAF183D937C41A04091ADBCEF33315FF34C',
'_csrf': '29dfa3ddd059c3a9db12143e29026730092990239ef75898af0b1751e3f8f4ff',
'_lx_utm': 'utm_source%3DBaidu%26utm_medium%3Dorganic',
'_lxsdk_cuid': '170131eb0cac8-06dda5778e29a9-b383f66-100200-170131eb0ca1',
'_lxsdk': 'F1A445C047BC11EAB2946104D9850EAF183D937C41A04091ADBCEF33315FF34C',
'mojo-uuid': 'b9015c74d0f312cee2a094515d087a8a',
'mojo-session-id': '{"id":"71271fe111c7821d0b4446a38b2b7dcc","time":1580868744683}',
'SL_GWPT_Show_Hide_tmp': '1',
'SL_wptGlobTipTmp': '1',
'lt': 'Yty7hi4TdDYZxEZhvpeYtaajBpoAAAAA-QkAAJKPH-4NCbTaSVLzCzf_TNVZrtzxSTBgGfPP4-CJmai8YCLwDdVe-7I0NZ-BGQdd1g',
'lt.sig': 'WXkP89KWgZ0hwZC1_IufMi11VrA',
'mojo-trace-id': '11',
'Hm_lvt_703e94591e87be68cc8da0da7cbd0be2': '1580868743,1580869392',
'Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2': '1580869392',
'_lxsdk_s': '170131eb0cc-ef-30a-66b%7C%7C12',
}
headers = {
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
'Sec-Fetch-User': '?1',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'cross-site',
'Sec-Fetch-Mode': 'navigate',
'Referer': 'https://passport.meituan.com/account/secondverify?response_code=ff1857e9b5eb422eb5da21966a3dd53d&request_code=da67f3c519af4f57a311dab06eb7cced',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
}
params = (
('offset', '0'),
)
response = requests.get(url, headers=headers, params=params, cookies=cookies)
response.encoding = response.apparent_encoding #设置编码方式与正文相同
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
#从网站控制台的Network中查看源码，不能从Element中查看，因为经过JS处理
#通过正则表达式解析页面
def parse_one_page(html):
pattern = re.compile('.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">(.*?).*?star">(.*?).*?releasetime">(.*?)'
+ '.*?integer">(.*?).*?fraction">(.*?).*?', re.S)
items = re.findall(pattern, html) #匹配HTML中的正则对象pattern
#数据按照类别整理
for item in items:
yield {
'index': item[0],
'image': item[1],
'title': item[2],
'actor': item[3].strip()[3:],#删除头尾空格换行符
'time': item[4].strip()[5:],
'score': item[5] + item[6]
}
#写入文件并存入指定目录
def write_to_file(content):
with open('D:/result.txt', 'a', encoding='utf-8') as f:
f.write(json.dumps(content, ensure_ascii=False) + '\n') #JSON库的dumps()方法实现字典的序列化，ensure_ascii=false保证中文输出
def main(offset):
url = 'http://maoyan.com/board/4?offset=' + str(offset) #拼接出所有页面的URL
html = get_one_page(url) #得到各个页面的HTML
for item in parse_one_page(html): #解析各个页面并将结果序列化输出并写入文件
print(item) #输出
write_to_file(item) #写入文件
#main函数调用这里实现URL的更改，得到所有页面的URL
if __name__ == '__main__':
for i in range(10):
main(offset=i * 10)
time.sleep(1)
