AttributeError: 'NoneType' object has no attribute 'text'
import requests
from bs4 import BeautifulSoup
import lxml
#对首页页面进行数据采集
url = "https://www.shicimingju.com/book/sanguoyanyi.html"
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
page_text = requests.get(url=url,headers=header).text
#在首页解析出章节的标题和详细页源码数据加载到该对象中
#1.实例BeautifulSoup对象,需要将页面数据加载到该对象
soup = BeautifulSoup(page_text,'lxml')
#解析章节标题和详细页url
li_list = soup.select('.book-mulu > ul >li')
fp = open('./sanguo.txt',"w",encoding="utf-8")
for li in li_list:
title = li.a.string
datail_url = "https://www.shicimingju.com/book/" + li.a['href']
#对详情也发请求
datail_text = requests.get(url=datail_url,headers=header).text
detail_soup = BeautifulSoup(datail_text,'lxml')
div_tag = detail_soup.find('div',class_="chapter_content")
content = div_tag.text
fp.write(title+':'+content+'\n')
print(title,"爬取成功!!")
import requests
from bs4 import BeautifulSoup
import lxml
#对首页页面进行数据采集
if __name__ == "__main__":
url = "https://www.shicimingju.com/book/sanguoyanyi.html"
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
page_text = requests.get(url=url,headers=header).content.decode("utf-8")
#在首页解析出章节的标题和详细页源码数据加载到该对象中
#1.实例BeautifulSoup对象,需要将页面数据加载到该对象
soup = BeautifulSoup(page_text,'lxml')
#解析章节标题和详细页url
li_list = soup.select('.book-mulu > ul >li')
fp = open('./sanguo.txt',"w",encoding="utf-8")
for li in li_list:
title = li.a.string
datail_url = "https://www.shicimingju.com" + li.a['href']
#对详情也发请求
datail_text = requests.get(url=datail_url,headers=header).content.decode("utf-8")
detail_soup = BeautifulSoup(datail_text,'lxml')
div_tag = detail_soup.find('div',class_="chapter_content")
content = div_tag.text
fp.write(title+':'+content+'\n')
print(title,"爬取成功!!")
1.该情况是因为我的user-agent访问过于频繁 ,需要更改其他的user-agent
2.中文乱码问题(requests请求后面)加上.content.decode("utf-8")