python3爬虫：爬取电影天堂电影信息

python3爬虫：爬取电影天堂电影信息

#爬取电影天堂电影信息
#爬取电影天堂电影信息
#爬取电影天堂电影信息
from lxml import etree
import requests

BASE_DOMAIN = 'https://www.ygdy8.net'
HEADERS = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0'
}
# 获取电影详情页url
def get_detail_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode(encoding='gbk', errors='ignore')
    # print(text)
    html = etree.HTML(text)
    detail_urls = html.xpath("//table[@class='tbspan']//a/@href")
    # for detail_url in detail_urls:
    #     print(BASE_DOMAIN + detail_url)
    detail_urls = map(lambda url:BASE_DOMAIN+url,detail_urls)
    return detail_urls


#解析电影详情页url
def parse_detail_page(url):
    movie = {}
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode(encoding='gbk', errors='ignore')
    html = etree.HTML(text)
    title = html.xpath("//div[@class='title_all']//font[@color='#07519a']/text()")
    # print(title)
    movie['title'] = title
    # for x in title:
    #     print(etree.tostring(x,encoding='utf-8').decode("utf-8"))

    zoomE = html.xpath("//div[@id='Zoom']")[0]
    imgs = zoomE.xpath(".//img/@src")
    cover = imgs[0]
    #screenshot = imgs[1]
    movie['cover'] = cover
    #movie['screenshot'] = screenshot

    #定义筛选函数
    def parse_info(info,rule):
        return info.replace(rule,"").strip()

    infos = zoomE.xpath(".//text()")
    for index,info in enumerate(infos):
        if info.startswith("◎年　　代"):
            info = parse_info(info,"◎年　　代")
            movie['year'] = info
        elif info.startswith("◎产　　地"):
            info = parse_info(info,"◎年　　代")
            movie['country'] = info
        elif info.startswith("◎类　　别"):
            info = parse_info(info,"◎类　　别")
            movie['category'] = info
        elif info.startswith("◎豆瓣评分"):
            info = parse_info(info,"◎豆瓣评分")
            movie['douban_rating'] = info
        elif info.startswith("◎片　　长"):
            info = parse_info(info, "◎片　　长")
            movie['duration'] = info
        elif info.startswith("◎导　　演"):
            info = parse_info(info, "◎导　　演")
            movie['director'] = info
        elif info.startswith("◎主　　演"):
            info = parse_info(info, "◎主　　演")
            actors = [info]
            for x in range(index+1,len(infos)):
                actor = infos[x].strip()
                if actor.startswith("◎"):
                    break
                actors.append(actor)
            movie['actors'] = actors
        elif info.startswith("◎简　　介"):
            info = parse_info(info, "◎简　　介")
            for x in range(index + 1, len(infos)):
                profile = infos[x].strip()
                #if profile.startswith("【下载地址】"):
                    #break
                #print(profile)
                movie["profile"] = profile

    download_url = html.xpath("//td[@bgcolor='#fdfddf']/a/@href")
    movie['download_url'] = download_url
    return movie



#得到电影天堂最新电影前7页，可更改for循环改变页数
def spider():
    base_url = "https://www.ygdy8.net/html/gndy/dyzz/list_23_{}.html"
    movies = []
    for x in range(1,8):
        url = base_url.format(x)
        detail_urls = get_detail_urls(url)
        for detail_url in detail_urls:
            #print(detail_url)
            movie = parse_detail_page(detail_url)
            movies.append(movie)
            print(movie)
    #print(movies)
if __name__ == '__main__':
    spider()