爬取菜鸟教程

import requests  # 用于请求和获取响应数据等
from lxml import etree  # 解析爬取到的HTML页面
from fake_useragent import UserAgent  # 生成请求头
import os  # 文件/目录方法
from time import time

start_time = time()
ua = UserAgent()
headers = {
     
    'User-Agent': ua.random  # 生成随机请求头
}
url = 'https://www.runoob.com/'
res = requests.get(url=url, headers=headers)
res.encoding = res.apparent_encoding  # 自动解析网页编码
text = res.text
html = etree.HTML(text=text)
module_list = html.xpath('/html/body/div[4]/div/div[2]/div')  # 获取每个大模块

for module in module_list[10:]:
    module_title = str(module.xpath('./h2/text()')[0]).strip().replace(' ', '')
    module_title = module_title.replace('/', '_')  # 获取每个大模块的标题
    
    os.makedirs('runoob/' + module_title)  # 递归创建每个大模块的目录
    sub_module = module.xpath('./a/h4/text()')  # 获取子模块的标题
    sub_module_link = ['https:' + link for link in module.xpath("./a/@href")]  # 获取子模块的链接
    sub_module_info = zip(sub_module, sub_module_link)  # 将对应的子模块的标题及链接封装成元组
    
    for sub_module in sub_module_info:
        title = sub_module[0].replace('/', '_')  # 获取子模块的目录
        request = requests.get(url=sub_module[1], headers=headers)
        request.encoding = request.apparent_encoding
        html = etree.HTML(text=request.text)
        for a_label in html.xpath('//*[@id="leftcolumn"]/a'):
            a_label_title = a_label.xpath('./text()')[0].strip()
            a_label_href = 'https://www.runoob.com' + a_label.xpath('./@href')[0]
            with open('runoob/' + module_title + '/' + title + '.txt', encoding='utf-8', mode='a+') as file:
                file.write(a_label_title + '\t' + a_label_href + '\n')  # 以子模块为名，目录和链接为内容写进文件中
        print('{}: {} 爬取成功'.format(module_title, title))

print('爬取耗时：{}秒'.format(time() - start_time))

运行结果：