爬虫程序-豆瓣电影

import requests
import re
from bs4 import BeautifulSoup   #从bs4这个库中导入BeautifulSoup
import lxml
import csv
movie_list = ['电影名']
m_eng = ['英文名或别名']
link_list = ['豆瓣链接']
info_list = ['详细信息']
director_list = ['导演']
star_list = ['主演']
score_list = ['评分']
country_list = ['国家']
year_list = ['年份']
type_list = ['类型']
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36','Connection': 'close'}

for i in range(10):

    link = "https://movie.douban.com/top250?start=" + str(i*25) +'&filter='
    r = requests.get(link, headers=headers,timeout = 30,verify = False)
    print(str(i+1),'页码响应码：', r.status_code)
    soup = BeautifulSoup(r.text,'html.parser')
    html = lxml.etree.HTML(r.text)
    link_all = html.xpath('//div[@class = "hd"]/a[1]')
    title_all = html.xpath('//div[@class = "hd"]/a/span[1]')
    etitle_all = html.xpath('//div[@class = "hd"]/a/span[2]')
    info_all = html.xpath('//div[@class = "bd"]/p[1]/text()')
    score_all = html.xpath('//div[@class = "star"]/span[2]/text()')
    r.close()
    #title_movie = title_all[0]
    #title_eng = title_all[1]
    #title_other = title_all[2]
    #movie_list.append(title_movie)
    #m_eng.append(title_eng)
    #m_other.append(title_other)
    #print(score_all)
    for index in range(len(link_all)):
        movie_list.append(title_all[index].text)
        m_eng.append(' '.join(etitle_all[index].text.split()))
        link_list.append(link_all[index].attrib['href'])
        score_list.append(score_all[index])
        info_list.append(' '.join(info_all[2*index].split()))
        info_str = ' '.join(info_all[2*index].split())
        info2 = re.match(r'(.*?)/(.*)/(.*)',info_all[2*index+1])
        print(info_all[2*index+1])
        print(info2)
        country_list.append(info2.group(2).strip())
        year_list.append(info2.group(1).strip())
        type_list.append(info2.group(3).strip())
        #print(country_list)
        #print(index)
        #print(info_str)
        direc = re.match(r'导演: (.*) 主',info_str)
        if direc == None :
            director_list.append('')
        else:
            #print(direc.group(1))
            director_list.append(direc.group(1))
        star = re.search(r'主演: (.*)', info_str)
        #print(star)
        if star == None :
            star_list.append('')
        else:
            star_list.append(star.group(1))
            #print(star.group(1))




print(movie_list)
print(m_eng)
print(link_list)
#print(info_list)
print(director_list)
print(star_list)
print(score_list)
print(country_list)
print(year_list)
print(type_list)
movie_info = []
with open('Top250 movie.csv','a+',encoding='UTF-8',newline='')as csvfile:
    w = csv.writer(csvfile,dialect='excel')
    for i, element in enumerate(score_list):
        movie_info = [movie_list[i],m_eng[i],country_list[i],year_list[i],type_list[i],director_list[i],star_list[i],score_list[i],link_list[i]]
        w.writerow(movie_info)



#print (movie_list)