豆瓣Top250数据爬取小项目

• 导入相关的包

from bs4 import BeautifulSoup
from urllib.request import urlopen
import urllib
import lxml
import re
import random

• 需要改掉文件路径

def get_html():
    base_url = "https://baike.baidu.com"
    his = ["/item/%E8%BD%A6%E8%BD%AE/1468488"]
    
    for i in range(20):
        url = base_url + his[-1]
        html = urlopen(url).read().decode('utf-8')
        soup = BeautifulSoup(html, features='lxml')
        print(soup.find('h1').get_text(),' url', his[-1])
        sub_urls = soup.find_all("a", {"target":"_blank", "href":re.compile("/item/(%.{2})+$")})
        
        if len(sub_urls) != 0:
            his.append(random.sample(sub_urls, 1)[0]['href'])
        else:
            his.pop()

• 主函数

if __name__ == "__main__":
    get_html()

• 附上结果图