| '''
'''
'''
证书验证(大部分网站都是https)
'''
import requests
# # 如果是ssl请求,首先检查证书是否合法,不合法则报错,程序终端
# response = requests.get('https://www.xiaohuar.com')
# print(response.status_code)
# 改进1:去掉报错,但是会报警告
# import requests
# response = requests.get('https://www.xiaohuar.com', verify=False)
# # 不验证证书,报警告,返回200
# print(response.status_code)
# 改进2:去掉报错,并且去掉警报信息
# import requests
# import urllib3
# urllib3.disable_warnings() # 关闭警告
# response = requests.get('https://www.xiaohuar.com', verify=False)
# print(response.status_code)
# 改进3:加上证书
# 很多网站都是https,但是不用证书也可以访问,大多数情况都是可以携带也可以不携带证书
# 知乎\百度等都是可带可不带
# 有硬性要求的,则必须带，比如对于定向的用户,拿到证书后才有权限访问某个特定网站
# import requests
# import urllib3
# # urllib3.disable_warnings() # 关闭警告
# # 伪代码
# response = requests.get(
# 'https://www.xiaohuar.com',
# # verify=False,
# # /path/server.crt证书的存放目录， /path/key
# cert=('/path/server.crt', '/path/key'))
# print(response.status_code)
'''
超时设置
'''
# 超时设置
# 两种超时:float or tuple
# timeout=0.1 # 代表接收数据的超时时间
# # timeout=(0.1,0.2) # 0.1代表链接超时 0.2代表接收数据的超时时间
#
# import requests
# response = requests.get('https://www.baidu.com',
# timeout=0.0001
# print(response.elapsed)
# print(response.status_code)
'''
代理设置:先发送请求给代理,然后由代理帮忙发送(封ip是常见的事情)
'''
# import requests
# proxies={
# # 带用户名密码的代理,@符号前是用户名与密码
# 'http':'http://tank:123@localhost:9527',
# # 'http':'http://localhost:9527',
# # 'https':'https://localhost:9527',
# }
# response=requests.get('https://www.12306.cn',
# proxies=proxies)
#
# print(response.status_code)
#
'''
爬取西刺免费代理：
1.访问西刺免费代理页面
2.通过re模块解析并提取所有代理
3.通过ip测试网站对爬取的代理进行测试
4.若test_ip函数抛出异常代表代理作废，否则代理有效
5.利用有效的代理进行代理测试
 |
| 112.85.131.99 |
| 9999 |
| 江苏南通
|
| 高匿 |
| HTTPS |
|
|
|
|
| 6天 |
| 19-05-16 11:20 |
re:
(.*?).*?| (.*?) | '''
import requests
import re
import time
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}
def get_index(url):
time.sleep(1)
response = requests.get(url, headers=HEADERS)
return response
def parse_index(text):
ip_list = re.findall('.*?| (.*?) | .*?(.*?) | ', text, re.S) for ip_port in ip_list:
ip = ':'.join(ip_port)
yield ip
def test_ip(ip):
print('测试ip: %s' % ip)
try:
proxies = {
'https': ip
}
# ip测试网站
ip_url = 'https://www.ipip.net/'
# 使用有效与无效的代理对ip测试站点进行访问，若返回的结果为200则代表当前测试ip正常
response = requests.get(ip_url, headers=HEADERS, proxies=proxies, timeout=1)
if response.status_code == 200:
print(f'有用的ip：{ip}')
return ip
# 若ip代理无效则抛出异常
except Exception as e:
print(e)
# # 使用代理爬取nba
def spider_nba(good_ip):
url = 'https://china.nba.com/'
proxies = {
'https': good_ip
}
response = requests.get(url, headers=HEADERS, proxies=proxies)
print(response.status_code)
print(response.text)
if __name__ == '__main__':
base_url = 'https://www.xicidaili.com/nn/{}'
for line in range(1, 3677):
ip_url = base_url.format(line)
response = get_index(ip_url)
# 解析西刺代理获取每一个ip列表
ip_list = parse_index(response.text)
# 循环每一个ip
for ip in ip_list:
# print(ip)
# 对爬取下来的ip进行测试
good_ip = test_ip(ip)
if good_ip:
# 真是代理，开始测试
spider_nba(good_ip)
'''
认证设置
'''
import requests
# 通过访问github的api来测试
# url = 'https://api.github.com/user'
# HEADERS = {
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
# }
# 测试1，失败返回401
# response = requests.get(url, headers=HEADERS)
# print(response.status_code) # 401
# print(response.text)
'''
打印结果:
{
"message": "Requires authentication",
"documentation_url": "https://developer.github.com/v3/users/#get-the-authenticated-user"
}
'''
#
# # 测试2，通过requests.auth内的HTTPBasicAuth进行认证，认证成功返回用户信息
# from requests.auth import HTTPBasicAuth
# response = requests.get(url, headers=HEADERS, auth=HTTPBasicAuth('tankjam', 'kermit46709394'))
# print(response.text)
#
# 测试3，通过requests.get请求内的auth参数默认就是HTTPBasicAuth，认证成功返回用户信息
# response = requests.get(url, headers=HEADERS, auth=('tankjam', 'kermit46709394'))
# print(response.text)
'''
上传文件
'''
# import requests
# 上传文本文件
# files1 = {'file': open('user.txt', 'rb')}
# # files参数是POST请求固定参数
# response = requests.post('http://httpbin.org/post', files=files1)
# print(response.status_code) # 200
# print(response.text) # 200
# 上传图片文件
# files2 = {'jpg': open('一拳.jpg', 'rb')}
# response = requests.post('http://httpbin.org/post', files=files2)
# print(response.status_code) # 200
# print(response.text) # 200
#
# 上传视频文件
# files3 = {'movie': open('love_for_GD.mp4', 'rb')}
# response = requests.post('http://httpbin.org/post', files=files3)
# print(response.status_code) # 200
# print(response.text) # 200
| 
