百度爬虫搜索接口1.0版:
经过测试,初始时拼接url,只需要加入keyword参数即可,为了便于之后如果有需要扩充的参数进行扩展,可以写在不同的函数里,进行功能解耦,通过params的方式进行参数的传递,及拼接url;
具体方法可以参照链接:
Python骚操作总结
通过headers等参数来请求得到响应,中间加上RequestException的异常处理,因为返回的是文本text数据,所以返回response.text。如果是动态加载的xhr数据,则返回的数据则用response.json;实现如下:
def get_page(url):
try:
headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
'accept-language': 'zh-CN,zh;q=0.9',
'cache-control': 'max-age=0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
}
response = requests.get(url=url,headers=headers)
# 更改编码方式,否则会出现乱码的情况
response.encoding = "utf-8"
print(response.status_code)
# print(response.text)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
首先我们要清楚我们要抓取的内容有url,title,以及abstract text文本的内容。
根据div栏中的id号,我们可以找到我们想要的数据,但是要注意其中的文本内容包括带图片的,以及纯文本,这两种内容的xpath不同。
import requests
import urllib.parse
from requests.exceptions import RequestException
from urllib.parse import urljoin
from lxml import etree
import re
import json
# 百度搜索接口
def format_url(url, params: dict=None) -> str:
query_str = urllib.parse.urlencode(params)
return f'{ url }?{ query_str }'
def get_url(keyword):
params = {
'wd': str(keyword)
}
url = "https://www.baidu.com/s"
url = format_url(url, params)
# print(url)
return url
def get_page(url):
try:
headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
'accept-language': 'zh-CN,zh;q=0.9',
'cache-control': 'max-age=0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
}
response = requests.get(url=url,headers=headers)
# 更改编码方式,否则会出现乱码的情况
response.encoding = "utf-8"
print(response.status_code)
# print(response.text)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
def parse_page(url,page):
for i in range(1,int(page)+1):
print("正在爬取第{}页....".format(i))
title = ""
sub_url = ""
abstract = ""
flag = 11
if i == 1:
flag = 10
html = get_page(url)
content = etree.HTML(html)
for j in range(1,flag):
data = {}
res_title = content.xpath('//*[@id="%d"]/h3/a' % ((i - 1) * 10 + j))
if res_title:
title = res_title[0].xpath('string(.)')
sub_url = content.xpath('//*[@id="%d"]/h3/a/@href' % ((i - 1) * 10 + j))
if sub_url:
sub_url = sub_url[0]
res_abstract = content.xpath('//*[@id="%d"]/div[@class="c-abstract"]'%((i-1)*10+j))
if res_abstract:
abstract = res_abstract[0].xpath('string(.)')
else:
res_abstract = content.xpath('//*[@id="%d"]/div/div[2]/div[@class="c-abstract"]'%((i-1)*10+j))
if res_abstract:
abstract = res_abstract[0].xpath('string(.)')
# res_abstract = content.xpath('//*[@id="%d"]/div/div[2]/p[1]'%((i-1)*10+j))
# if not abstract:
# abstract = content.xpath('//*[@id="%d"]/div/div[2]/p[1]'%((i-1)*10+j))[0].xpath('string(.)')
data['title'] = title
data['sub_url'] = sub_url
data['abstract'] = abstract
rel_url = content.xpath('//*[@id="page"]/a[{}]/@href'.format(flag))
if rel_url:
url = urljoin(url, rel_url[0])
else:
print("无更多页面!~")
return
yield data
def main():
keyword = input("输入关键字:")
page = input("输入查找页数:")
url = get_url(keyword)
results = parse_page(url,page)
# 写入文件
file = open("data.json", 'w+', encoding='utf-8')
for result in results:
print(result)
file.write(json.dumps(result, indent=2, ensure_ascii=False))
if __name__ == '__main__':
main()