import requests
import pymysql
import time
import re
import xlwt
from lxml import etree
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36',
'Cookie': 'gr_user_id = c6f58a39 - ea25 - 4f58 - b448 - 545070192c4e;59a81cc7d8c04307ba183d331c373ef6_gr_session_id = e8e4b66f - 440a - 4ae7 - a76a - fe2dd2b34a26;59a81cc7d8c04307ba183d331c373ef6_gr_last_sent_sid_with_cs1 = e8e4b66f - 440a - 4ae7 - a76a - fe2dd2b34a26;59a81cc7d8c04307ba183d331c373ef6_gr_last_sent_cs1 = N % 2FA;59a81cc7d8c04307ba183d331c373ef6_gr_session_id_e8e4b66f - 440a - 4ae7 - a76a - fe2dd2b34a26 = true;grwng_uid = 9ec14ad9 - 5ac0 - 4bb1 - 81c1 - bc60d2685710;abtest_ABTest4SearchDate = b;xzuuid = 79426b52;_uab_collina = 154660443606130958890473;TY_SESSION_ID = 907f32df - c060 - 49ca - b945 - 98215cc03475;rule_math = pvzq3r06hi'}
conn = pymysql.connect(host= 'localhost',user= 'root',passwd='momiao5201314',db='doubanmovie',port=3306,charset='utf8')
cursor = conn.cursor() #创建光标对象
'''
# 创建一个workbook设置编码
workbook = xlwt.Workbook(encoding = 'utf-8')
# 创建一个worksheet
worksheet = workbook.add_sheet('My Worksheet')
#定义表头
header = ['movie_name','director','actors,style','country','release_time','time','score']
for h in range(len(header)):
workbook.write(0,h,header[h])
'''
def get_movie_url(url):
html = requests.get(url,headers=headers)
selector = etree.HTML(html.text)
movie_urls = selector.xpath('//div[@class="hd"]/a/@href')
for movie_url in movie_urls:
#print(movie_url)
get_movie_info(movie_url)
def get_movie_info(url):
html = requests.get(url,headers=headers)
selector = etree.HTML(html.text)
try:
movie_name = selector.xpath('//*[@id="content"]/h1/span[1]/text()') #1电影名称
#print(movie_name)
director = selector.xpath('//*[@id="info"]/span[1]/span[2]/a/text()') #2导演
#print(director)
actors = selector.xpath('//*[@id="info"]/span[3]/span[2]')[0] #Xpath疑问?
actor = actors.xpath('string(.)') #3演员
#print(actor)
style = re.findall('(.*?)',html.text,re.S)[0] + re.findall('(.*?)',html.text,re.S)[1] #4类型
#print(style)
country = re.findall('制片国家/地区:(.*?)
',html.text,re.S) #5制片地区
#print(country)
release_time = re.findall('上映日期:.*?>(.*?)',html.text,re.S) #6上映时间
#print(release_time)
time = re.findall('片长:.*?>(.*?)',html.text,re.S) #7片长
#print(time)
score = selector.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()') #8评分
#print(score)
print(str(movie_name))
#sql = 'insert into doubanmovie(name,director,actor,style,country,release_time,time,score,) values("{}","{}","{}","{}","{}","{}","{}","{}")'.format(movie_name,director,actor,style,country,release_time,time,score) #多一个逗号
cursor.execute("insert into doubanmovie(name,director,actor,style,country,release_time,time,score) values(%s,%s,%s,%s,%s,%s,%s,%s)",(str(movie_name),str(director),str(actor),str(style),str(country),str(release_time),str(time),str(score)))
except IndexError:
pass
if __name__ == '__main__':
urls = ['https://movie.douban.com/top250?start={}&filter='.format(num)for num in range(0,250,25)]
for url in urls:
get_movie_url(url)
time.sleep(2)
conn.commit()
