Python爬虫实战之爬取豆瓣详情以及影评
爬取豆瓣详情分为三步：
1.爬取豆瓣电影的所有标签，遍历标签，通过分析网址结构获得每一类标签下的电影url
2.通过url 爬取电影详情
3.导入数据库
爬虫代码如下：
from urllib import parse
import urllib
import random
from urllib.error import URLError
from urllib.request import ProxyHandler, build_opener
import json
from bs4 import BeautifulSoup
from urllib import request
import re
import pymysql
class Spider(object):
def get_ip(self):
fr=open('ip.txt','r')##代理ip文件
ips=fr.readlines()
new=[]
for line in ips:
temp=line.strip()
new.append(temp)
ip=random.choice(new)
proxy =ip
proxy_handler = ProxyHandler({
'http': 'http://' + proxy,
'https': 'https://' + proxy
})
opener = build_opener(proxy_handler)
self.get_info(opener)
def get_info(self,opener):
url='https://movie.douban.com/j/search_tags?type=movie&source='
try:
headers={'User-Agent':' Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'}
req=urllib.request.Request(url,headers=headers)
data=opener.open(req).read().decode('utf-8')
json_res=json.loads(data)
tags=json_res['tags']
i=0
for tag in tags:
print("正在爬取" + str(tag))
#tag={
# 'tag':tag,
# 'page_star':i
#}
start=0
step=1001 #控制长度
for i in range(start,21,step):
tag = urllib.request.quote(tag)
tag_url='https://movie.douban.com/j/search_subjects?type=movie&tag='+tag+'&sort=recommend&page_limit=20&page_start='+str(i)
req = request.Request(tag_url, headers=headers)
response=opener.open(tag_url).read().decode('utf-8')
response_dict=json.loads(response)
subjects=response_dict['subjects']
try:
for subject in subjects:
movie_id = subject['id']
movie_title = subject['title']
movie_rate=subject['rate']
movie_cover =subject['cover']
movie_url=subject['url']
print(movie_id)
print(movie_title)
print(movie_url)
html=request.urlopen(movie_url).read().decode('utf-8')
html = html[1:len(html) + 1]
bs = BeautifulSoup(html, 'lxml')
# print(bs.find(attrs={'id':'info'}).text)#电影信息
# print(bs.find('h2').text)
# print(bs.find_all('h2')[0].text)#简介列表
# print(bs.find(attrs={'class':'related-info'}).text.strip())
filmName = bs.h1.span.text
fileDIV = bs.find(attrs={'id': 'info'})
file1 = fileDIV.findAll(class_='p1')
spans = fileDIV.findAll('span')
span_atters = fileDIV.findAll(class_='attrs')
director_span = span_atters[0]
screenwriter_span = span_atters[1]
staring_span = span_atters[2]
director_str = ''
screenwriter_str = ''
staring_str = ''
director_a = director_span.findAll('a')
for director_a in director_a:
director_str = director_str + director_a.text + ';'
print(director_str) #导演
screenwriter_span = screenwriter_span.findAll('a')
for screenwriter_a in screenwriter_span:
screenwriter_str = screenwriter_str + screenwriter_a.text + ';'
print(screenwriter_str) #编剧
staring_span = staring_span.findAll('a')
for staring_a in staring_span:
staring_str = staring_str + staring_a.text + ';'
print(staring_str) #主演
file_type = ''
file_type_spans = bs.find_all('span', {'property': "v:genre"})
for file_type_span in file_type_spans:
file_type = file_type + file_type_span.text + ';'
print(file_type) #类型
pattern = re.compile("制片国家/地区:(.*)")
movie_country=pattern.findall(fileDIV.text)[0]
print(movie_country)
pattern2 = re.compile("语言:(.*)")
movie_language=pattern2.findall(fileDIV.text)[0]
print(movie_language)
movie_release_date = bs.find('span', {'property': 'v:initialReleaseDate'}).text
print(movie_release_date) #日期
movie_length = bs.find('span', {'property': "v:runtime"}).text
print(movie_length) #电影长度
pattern3 = re.compile("又名:(.*)")
movie_alias=pattern3.findall(fileDIV.text)[0]
print(movie_alias) #名字
pattern4 = re.compile("IMDb链接:(.*)")
imdb=pattern4.findall(fileDIV.text)[0]
print(imdb)
#brief=bs.find('span', {'property': 'v:summary'}).text.strip()
#print(brief.replace(' ', ''))
movies =(movie_id,movie_title,director_str,screenwriter_str,staring_str,file_type,movie_country,movie_language,movie_release_date,movie_length,movie_alias,imdb)
self.con_mysql(movies,movie_title,movie_id)
except URLError as e:
print(e.reason)
except URLError as e:
print(e.reason)
def con_mysql(self,movies,movie_title,movie_id):
try:
host = 'localhost'
user = 'root'
pwd = ''
database = 'douban'
db = pymysql.connect(host, user, pwd, database,charset="utf8")
cursor = db.cursor() # 获取一个邮标，增删减除
sql = "select * from movie3 where movie_id=%s" % movie_id
cursor.execute(sql)
one = cursor.fetchone() # 返回单个元组
db.commit()
if (one is not None):
print(str(movie_title)+"已存在该ID")
pass
else:
cursor.execute("insert ignore into movie3 values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % movies)
two=cursor.fetchone()
db.commit()
if (two is None):
print('插入失败')
else:
print(str(movie_title) + "插入成功")
except URLError as e:
print(e.reason)
spider=Spider()
spider.get_ip()
运行结果如下：
