python爬虫之古诗词分类爬取加存储

分析网站

目标网站的url=“https://www.gushiwen.org/shiju/xiejing.aspx”

对网站的目录进行简单的分析，这个是各古诗的分类标题。

def get_div_text(html_text):
    soup=BeautifulSoup(html_text,'html.parser')
    divs=soup.find_all('div',{
     "class":"cont"}) 	#对网站的目录div标签抓取
    urls=[]
    for div in divs:
        div=div.find_all(href=re.compile('shiju'))
         #re.compile('shiju')对多字段进行shiju关键字获取
    for url in div:
        urls.append(durl+url['href'])	#爬虫专用进行网址的存储
    return urls

分析数据

	for url in urls:
        time.sleep(10) #进行延迟爬取
        url=requests.get(url).text
        
        soup=BeautifulSoup(url,'html.parser')
        #取古诗的内容
        divs=soup.find_all("div", {
     "class": "cont"})
        #取古诗的标题 strip()去除句子的样式
        gushibt=soup.find('div',{
     "class":"title"}).text.strip()
        
        gush=[]
        gush.append(gushibt)
        for gushi in divs:
            gushci=gushi.find_all('a')
            gushci1=gushci[0].string #进行古诗爬取
            gushci2=gushci[1].string#进行作者爬取
            gush.append(gushci1+'——————————'+gushci2)
        gush.pop()#删除最后一项
        

写入文件

for gush1 in gush:
            with open("gushi.txt", "a+", encoding='utf-8') as f:
            #进行简单的文件写入
                f.write(json.dumps(gush1, ensure_ascii=False) + "\n")

原代码

import time
import json
import requests
from bs4 import BeautifulSoup
import re
durl='https://www.gushiwen.org/'
#爬取各标题
def get_html_text():
    url="https://www.gushiwen.org/shiju/xiejing.aspx"
    return requests.get(url).text
    
#进行爬取各诗词的
def get_div_text(html_text):
    soup=BeautifulSoup(html_text,'html.parser')
    divs=soup.find_all('div',{
     "class":"cont"})
    urls=[]
    for div in divs:
        div=div.find_all(href=re.compile('shiju'))
    for url in div:
        urls.append(durl+url['href'])
    return urls
    
def save_text(urls):
    for url in urls:
        time.sleep(10)
        url=requests.get(url).text
        
        soup=BeautifulSoup(url,'html.parser')
        #取古诗的内容
        divs=soup.find_all("div", {
     "class": "cont"})
        #取古诗的标题 strip()去除句子的样式
        gushibt=soup.find('div',{
     "class":"title"}).text.strip()
        
        gush=[]
        gush.append(gushibt)
        for gushi in divs:
            gushci=gushi.find_all('a')
            gushci1=gushci[0].string
            gushci2=gushci[1].string
            gush.append(gushci1+'——————————'+gushci2)
        gush.pop(1)
        gush.pop()
        for gush1 in gush:
            with open("gushi.txt", "a+", encoding='utf-8') as f:
                f.write(json.dumps(gush1, ensure_ascii=False) + "\n")
        
        
    
    
html_text=get_html_text()

url=get_div_text(html_text)

save_text(url)

以上为源码，如有疑问欢迎私信我，我会第一时间改正进行优化此项目。

人生苦短 我用python