基础知识
1)实现分页爬虫
for i in range(0,250,25):
url= 'https://movie.douban.com/top250?start={}&filter='.format(i)
循环访问网址不同的start的值就是不同的访问页,其他的爬虫过程类似昨天的。
Python例子
一、在网上下载图片
◉ 代码
import requests
response = requests.get('http://g.hiphotos.baidu.com/image/pic/item/5366d0160924ab18014cefd83bfae6cd7a890b82.jpg')
# #获取Bytes类型的响应
data = response.content
with open('./python.png','wb') as f:
f.write(data)
◉ 结果截图
豆瓣TOP10爬虫
▶ 要求:
(1)获取:电影序号,电影名,电影介绍,电影星级,电影评价,电影描述,电影封面图地址
(2)输出所有的数据
(3)将所有的封面图片下载下来
▶ 代码:
#豆瓣TOP250爬虫
#导入
import requests
from lxml import etree
def spider_douban_top250():
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
for i in range(0,250,25):
url= 'https://movie.douban.com/top250?start={}&filter='.format(i)
#获取bytes类型的响应
data = requests.get(url,headers = headers).content
html = etree.HTML(data)
ol_list = html.xpath('//div[@id="content"]/div/div[1]/ol/li')
movies_info = []
for movie in ol_list:
#影片序号
serial_number = movie.xpath('./div/div[@class="pic"]/em/text()')
if len(serial_number) == 0:
serial_number = ''
else:
serial_number = serial_number[0]
# print(serial_number)
#电影的名字
movies_name = movie.xpath('div/div[@class="info"]/div[@class="hd"]/a/span[1]/text()')
if len(movies_name) == 0:
movies_name = ''
else:
movies_name = movies_name[0]
# print(movies_name)
#电影介绍 只能用于字典.strip() 用来去空格
movie_introduce = movie.xpath('div/div[@class="info"]/div[@class="bd"]/p/text()')
if len(movie_introduce) == 0:
movie_introduce = ''
else:
movie_introduce = movie_introduce[0].strip()
# print(movie_introduce)
#电影星级
star = movie.xpath('div/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')
if len(star) == 0:
star = ''
else:
star = star[0]
# print(star)
#电影评价
evalute = movie.xpath('div/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[4]/text()')
if len(evalute) == 0:
evalute = ""
else:
evalute = evalute[0]
evalute = evalute.replace('人评价', '')
# print(evalute)
#电影描述
describe = movie.xpath('div/div[@class="info"]/div[@class="bd"]/p[@class="quote"]/span[@class="inq"]/text()')
if len(describe) == 0:
describe = ''
else:
describe = describe[0]
# print(describe)
#电影封面地址
movie_img_url = movie.xpath('div/div[@class="pic"]/a/img/@src')
if len(movie_img_url) == 0:
movie_img_url = ''
else:
movie_img_url = movie_img_url[0]
# print(movie_img_url)
movies_info.append({
'serial_number':serial_number,
'movies_name':movies_name,
'movie_introduce':movie_introduce,
'star':star,
'evalute':evalute,
'describe':describe,
'movie_img_url':movie_img_url
})
# print("这是第{}个页面".format(i))
#遍历
for movie in movies_info:
print(movie)
#下载图片
for movie in movies_info:
url = movie['movie_img_url']
resp = requests.get(url)
if resp.status_code == 200:
img_name = 'hml00{}.jpg'.format(movie['serial_number'])
with open('./imgs/{}'.format(img_name),'wb') as f:
f.write(resp.content)
else:
print("图片下载失败!")
print("成功!")
spider_douban_top250()
▶ 数据截图
数据太多,这里就放部分数据。
▶ 项目截图