Python 爬虫之 requests模块（ua伪装、代理、cookies、防盗链 Referer）、re模块、xpath模块、selenium

Python 爬虫

requests

下载

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple requests

发送 get 请求

案例：百度

import requests

url = "http://www.baidu.com"

# 发送 get 请求
response = requests.get(url)

# 设置字符集 (根据爬取网页 charset=utf-8)
response.encoding = 'utf8'

# 获取网页源代码
print(response.text)

发送 post 请求

案例：百度翻译

import requests

url = "https://fanyi.baidu.com/sug"

# 搜索词
data = {
    "kw":"love"
}
# 发送 get 请求
response = requests.post(url,data)

# 如果返回值是 json 数据，可直接调用 json() 方法
print(response.json())

UA 伪装

headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.52"
}

response = requests.get(url, headers=headers)

代理

proxy = {
    "http": "121.233.212.34:3432"
}

response = requests.get(url, proxies=proxy)

cookies 登录

案例：17k小说网

import requests

sessions = requests.session()

url = 'https://passport.17k.com/ck/user/login'
data = {
    "loginName": "15675521581",
    "password": "**"
}
res = sessions.post(url, data=data)
# print(res.cookies)

result = sessions.get(url = 'https://user.17k.com/ck/author/shelf?page=1&appKey=2406394919')
print(result.json())

防盗链 Referer

案例：梨视频

import requests

# 抓包的视频地址 https://video.pearvideo.com/mp4/short/20161103/1683211212907-10041181-hd.mp4
# 实际播放的地址 https://video.pearvideo.com/mp4/short/20161103/cont-1007270-10041181-hd.mp4
# 比较两个地址需要把抓包地址后面的时间戳替换为 cont-id
url = 'https://www.pearvideo.com/videoStatus.jsp?contId=1007270&mrd=0.2400885482825692'
refererUrl = "https://www.pearvideo.com/video_1007270"
contId = refererUrl.split('_')[1]

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
    "Referer": refererUrl
}

res = requests.get(url, headers=headers)
# 抓包返回的数据
dict = res.json()
print(dict)
# 抓包返回的时间戳
systemTime = dict['systemTime']
# 抓包返回的视频地址
srcUrl = dict['videoInfo']['videos']['srcUrl']
# 地址处理
newUrl = srcUrl.replace(systemTime, f"cont-{contId}")
print(newUrl)

# 下载视频到本地
with open('test.mp4', mode='wb') as f:
    f.write(requests.get(newUrl).content)

re 模块

findall

import re

result = re.findall(r"\d+","今天写了2行代码，赚了200元。")
print(result) # ['2', '200'] 返回列表

search

import re

result = re.search(r"\d+","今天写了2行代码，赚了200元。")
print(result) #  只匹配第一个值，然后返回
print(result.group()) # 2  从 match 获取匹配的值

finditer

import re

result = re.finditer(r"\d+","今天写了2行代码，赚了200元。")
print(result) #  把所有结果放在迭代器里

for item in result:
    # 2
    #  200
    print(item,item.group()) 

预加载

提前写好正则表达式

import re

obj = re.compile(r"\d+")

result = obj.findall("今天写了2行代码，赚了200元。")
print(result) # ['2', '200']

匹配换行 re.S

obj = re.compile(r"\d+",re.S)

在 html 中使用

import re

s = """

    
Baidu
    
weixin

"""
# 输出如下格式
# http://www.baidu.com Baidu
# http://www.weixin.com weixin

obj = re.compile(r'
(.*?)
')

result = obj.finditer(s)
for item in result:
    print(item.group(1),item.group(2))

原子组使用别名

obj = re.compile(r'
(?P