python网络爬虫--BeautifulSoup提取猫眼TOP100电影

import requests
from bs4 import BeautifulSoup
import bs4
import re
import json

def getPage(url):
    try:
        headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
    }
        response = requests.get(url,headers = headers)
        if response.status_code == 200:
            response.encoding = "UTF-8"
            return response.text
    except requests.ConnectionError:
　　　　　return None


def parsePage(html):
    soup = BeautifulSoup(html,"html.parser")
    for child in soup.find("dl").children:
        if isinstance(child,bs4.element.Tag):
            index = child.find(class_ = re.compile("^board-index")) #提取电影排名
            title = child.find(class_ = "name") # 提取电影名称，此处用的时标签属性，还有其它的方法，比如说CSS。 child.select(".name")
            releasetime = child.find(class_ ="releasetime") # 提取上映时间
            actor = child.find(class_ = "star")  # 提取主演
            score = child.find(class_ = "score") # 提取评分
            totalScore = ""
            for sco in score:
                totalScore = totalScore + sco.string
            imageLink = child.find(class_ = "board-img") # 提取电影封面链接
            yield{
            "index":index.string,
            "imageLink":imageLink["data-src"],
            "title":title.string,
            "releasetime":releasetime.string.strip()[5:],
            "actor":actor.string.strip()[3:],
            "score":totalScore
            }
        
            


def writeInfomation(content):
    with open("Movie BeautifulSoup.text","a") as f:
        f.write(json.dumps(content,ensure_ascii = False)+"\n")
        


def main(offset):
    url = "https://maoyan.com/board/4?offset="+str(offset)
    html = getPage(url)
    for item in parsePage(html):
        writeInfomation(item)

if __name__ == "__main__":
    for i in range(10):
        main(offset = i*10)
    

 

转载于:https://www.cnblogs.com/sakura-d/p/10911509.html