python爬虫小案例

案例1：爬取搜狗指定词条对应的搜索结果（简易网页采集器）

import requests
url = "https://www.sogou.com/web"
#处理URL携带的参数：
kw =input("输入要搜索的关键字")
param={
    'query':kw
}
#对指定的url发起的请求的url是携带参数的，并在请求过程中处理了参数
response = requests.get(url=url,params=param)
page_text = response.text
fileName =kw+'.html'
with open(fileName,'w',encoding='utf-8') as fp:
    fp.write(page_text)
print(fileName+"保存成功！！！")

反爬：#UA伪装

#UA伪装
#User-Agent
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"
}

response = requests.get(url=url,params=param,headers=headers)

案例2：破解百度翻译

import requests
import json
#1.指定url
post_url ="https://fanyi.baidu.com/langdetect"
#2.UA伪装
headers ={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"
}
#3.post请求参数处理（同get请求一样）
data = {
    # 'from': 'en',
    # 'to': 'zh',
    'query': 'dog',

}
#4.请求发送
response = requests.post(url=post_url,data=data,headers=headers)
#5.获取响应数据 .json()方法返回的是obj（如果确认响应数据是json类型的，才可以用json（））
dic_obj = response.json()
print(dic_obj)
#进行持久化存储
fp =open('./dog.json','w',encoding='utf-8')
json.dump(dic_obj,fp=fp,ensure_ascii=False)
fp.close()
print("over")

案例3：爬取豆瓣电影分类排行榜 https://movie.douban.com/中的电影详情数据

#-*- coding = utf-8 -*-
#@Time : 2022/2/19 17:33
#@File : requests实战之豆瓣电影.py
#@software : PyCharm

import requests
import json

headers ={    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"
}
url = "https://movie.douban.com/j/chart/top_list"
param ={
'type': '24',
'interval_id': '100:90',
'action':'',
'start': '40',
'limit': '20'
}
response = requests.get(url=url,params=param,headers=headers)
list_data = response.json()
fp =open('./douban.json','w',encoding='utf-8')
json.dump(list_data,fp=fp,ensure_ascii=False)
fp.close()
print("over")

案例4：爬取肯德基餐厅查询 http://www.kfc.com.cn/kfccda/index.aspx中指定地点的餐厅数

#-*- coding = utf-8 -*-
#@Time : 2022/2/19 18:01
#@File : requests实战之肯德基.py
#@software : PyCharm
import requests
import json
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"
}
url = "http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword"
kw = input("请输入要查询的城市名称")
param ={
'cname':'',
'pid': '',
'keyword': kw,
'pageIndex': '1',
'pageSize': '10'
}
response = requests.get(url=url,params=param,headers=headers)
page_text = response.text
with open('./kfc.text','w',encoding='utf-8') as fp:
    fp.write(page_text)


print("over")

案例5：爬取国家药品监督管理总局中基于中华人民共和国化妆品生产许可证相关数据

http://scxk.nmpa.gov.cn:81/xk/

#-*- coding = utf-8 -*-
#@Time : 2022/2/19 19:08
#@File : requests实战之药监总局相关数据爬取.py
#@software : PyCharm

import requests
import json
#批量获取不同企业的id值
if __name__ =="__main__":

    id_list = []  # 存储企业的id
    all_data_list = []  # 存储所有企业详情
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"
    }
    url = "http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList"

    #参数的封装
    for page in range(1,6):
        page =str(page)
        data ={
        'on': 'true',
        'page': page,
        'pageSize': '15',
        'productName': '',
        'conditionType': '1',
        'applyname': '',
        'applysn': '',
        }

        json_ids = requests.post(url=url,headers=headers,data=data).json()
        for dic in json_ids['list']:
            id_list.append(dic['ID'])
    #详情页url的域名都是一样的，只有携带的参数不同
    #获取企业详情信息

    for id  in id_list:
        data_xq ={
        "id": id
        }
        url_xq = "http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById"
        json_xq = requests.post(url=url_xq,headers=headers,data=data_xq).json()
        print(json_xq)
        all_data_list.append(json_xq)
        # fp = open('.json', 'w', encoding='utf-8')
        # json.dump(json_xq, fp=fp, ensure_ascii=False)
        # fp.close()
        #持久化存储
    fp =open('./allData.json','w',encoding='utf-8')
    json.dump(all_data_list,fp =fp,ensure_ascii=False)
    print("ov")