Requests+BeautifulSoup+正则表达式爬取猫眼电影Top100（名称，演员，评分，封面，上映时间，简介）...

 

 1 # encoding:utf-8
 2 from requests.exceptions import RequestException
 3 import requests
 4 import re
 5 import json
 6 from multiprocessing import Pool
 7 
 8 def get_one_page(url):
 9     try:
10         response = requests.get(url)
11         if response.status_code == 200:
12             return response.text
13         return None
14     except RequestException:
15         return None
16 
17 def parse_one_page(html):
18     pattern = re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">'
19                          +'.*?>(.*?).*?star">(.*?)
.*?releasetime">(.*?)
'
20                           +'.*?integer">(.*?).*?fraction">(.*?).*?',re.S)
21     items = re.findall(pattern, html)
22     # print(items)
23     for item in items:
24         yield {
25             'index': item[0],
26             'image': item[1],
27             'title': item[2],
28             'actor': item[3].strip()[3:],
29             'time': item[4].strip()[5:],
30             'score': item[5]+item[6]
31         }
32 
33 def write_to_file(content):
34     with open('MaoyanTop100.txt', 'a', encoding='utf-8') as f:
35         f.write(json.dumps(content, ensure_ascii=False)+'\n')
36         f.close()
37 
38 def main(offset):
39     url = "http://maoyan.com/board/4?offset="+str(offset)
40     html = get_one_page(url)
41     # print(html)
42     # parse_one_page(html)
43     for item in parse_one_page(html):
44         print(item)
45         write_to_file(item)
46 
47 if __name__ == '__main__':
48     pool = Pool()
49     # for i in range(10):
50     #     main(i*10)
51     # 加快效率
52     pool.map(main, [i*10 for i in range(10)])

效果图：

更新（获取封面以及影片简介）：

 1 # encoding:utf-8
 2 from requests.exceptions import RequestException
 3 import requests
 4 import json
 5 import re
 6 from urllib import request
 7 from bs4 import BeautifulSoup
 8 
 9 def get_one_page(url):
10     try:
11         response = requests.get(url)
12         if response.status_code == 200:
13             return response.text
14         return None
15     except RequestException:
16         return None
17 
18 def parse_one_page(html):
19     pattern = re.compile('
.*?board-index.*?>(\d+).*?href="(.*?)".*?data-src="(.*?)".*?name">'
20                          +'.*?>(.*?).*?star">(.*?)
.*?releasetime">(.*?)
'
21                           +'.*?integer">(.*?).*?fraction">(.*?).*?',re.S)
22     items = re.findall(pattern, html)
23     # print(items)
24     for item in items:
25         yield {
26             'index': item[0],
27             'jump': item[1],
28             'image': item[2],
29             'title': item[3],
30             'actor': item[4].strip()[3:],
31             'time': item[5].strip()[5:],
32             'score': item[6]+item[7]
33         }
34 
35 def parse_summary_page(url):
36     # url = 'https://maoyan.com/films/1203'
37     head = {}
38     # 使用代理
39     head['User - Agent'] = 'User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2843.400'
40     req = request.Request(url, headers=head)
41     response = request.urlopen(req)
42     html = response.read()
43     # 创建request对象
44     soup = BeautifulSoup(html, 'lxml')
45     # 找出div中的内容
46     soup_text = soup.find('span', class_='dra')
47     # 输出其中的文本
48     # print(soup_text.text)
49     return soup_text
50 
51 def write_to_file(content):
52     with open('newMaoyanTop100.txt', 'a', encoding='utf-8') as f:
53         f.write(json.dumps(content, ensure_ascii=False)+'\n')
54         f.close()
55 
56 def main(offset):
57     url = "http://maoyan.com/board/4?offset="+str(offset*10)
58     html = get_one_page(url)
59 
60     for item in parse_one_page(html):
61         # print(item['number'])
62         # print(item['jump'])
63         jump_url = "https://maoyan.com"+str(item['jump']) 
64         item['summary'] = str(parse_summary_page(jump_url)).replace("","").replace("","")
65         print(item)
66         write_to_file(item)
67 
68     # 写txt
69     # for item in parse_one_page(html):
70     #     write_to_file(item['title'])
71 
72     # 爬取100张图片
73     # path = 'E:\\myCode\\py_test\\MaoyanTop100\\images\\'
74     # for item in parse_one_page(html):
75     #     urllib.request.urlretrieve(item['image'], '{}{}.jpg'.format(path, item['index']))
76 
77 if __name__ == '__main__':
78     for i in range(10):
79         main(i)

 

转载于:https://www.cnblogs.com/shixinzei/p/10264088.html