利用requests对猫眼电影排行前一百的电影信息进行爬取
利用requests对猫眼电影排行前一百的电影信息进行爬取
单线程获取猫眼电影排行榜前一百电影的信息。
实践环境：python3.6+pycharm2018
利用正则表达式解析目标链接：https://maoyan.com/board/4
代码:
import csv
import re
import requests
from requests import RequestException
def main():
start_url=“https://maoyan.com/board/4”
for i in range(0,100,10):
# 获取响应文本内容
html = get_one_page(url=start_url, offset=i) #获取链接参数，定义爬取起点
if html is None:#连接不存在 抛出异常
print(“链接：%s?offset=%s异常”.format(start_url,i))
continue
pass
for item in parse_one_page(html=html):
store_data(item)
download_thumb(item[“title”],item[“thumb”])#下载封面到字典
pass
pass
#内容获取和响应
def get_one_page(url,offset):
try:
headers = {
‘User-Agent’: ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36’}
response = requests.get(url=url, headers=headers,params={“offset”:offset})
if response.status_code==200:#响应成功
return response.text
pass
else:
return None
pass
pass
except RequestException as e:
return None
pass
pass
#页面解析
def parse_one_page(html):
#pattern = ‘
pattern = ‘
'(.?). ?releasetime">(.?). ?integer">(.?). ?fraction">(\d+).?
# re.S匹配多行
regex = re.compile(pattern, re.S)
items = regex.findall(html)
for item in items:#遍历 定义字典标签
yield {
‘index’: item[0],
‘thumb’: get_large_thumb(item[1]),
‘title’: item[2],
‘actors’: item[3].strip()[3:],
‘release_time’: get_release_time(item[4].strip()[5:]),
‘area’: get_release_area(item[4].strip()[5:]),
‘score’: item[5] + item[6]
}
# print(item)
pass
pass
#时间
def get_release_time(data):
pattern = ‘^(.?)((|$)’
regex = re.compile(pattern)
w = regex.search(data)
return w.group(1)
pass
#地区
def get_release_area(data):
pattern = '.((.*))’
regex = re.compile(pattern)
w = regex.search(data)
if w is None:
return’未知’
return w.group(1)
pass
pass
获取封面大图
def get_large_thumb(url):
pattern = ‘(.?)@.?’
regex = re.compile(pattern)
w = regex.search(url)
return w.group(1)
pass
下载封面图
def download_thumb(title,url):
try:
response = requests.get(url=url)
# 获取二进制数据
with open(‘thumb/’+title+’.jpg’, ‘wb’) as f:
f.write(response.content)#数据写入
f.close()#关闭保存
except RequestException as e:
print(e)
pass
#写入数据
def store_data(item):
with open(‘111.csv’,‘a’,newline=’’,encoding=‘utf-8’) as data_csv:
try:
csv_writer = csv.writer(data_csv)
csv_writer.writerow(
[item[‘index’], item[‘thumb’], item[‘title’], item[‘actors’], item[‘release_time’], item[‘area’],
item[‘score’]])
pass
except Exception as e:
print(e)
print(item)
pass
pass
pass
if name == ‘main’:
main()
print(“爬取完成!”)
pass
结果：
