1、urllib2.urlopen() 发送GET请求
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# 1、导入urllib2库
import urllib2
# 2、设置要访问的网络地址
response = urllib2.urlopen("http://www.baidu.com/")
# 3、read()方法就是读取文件里的全部内容，返回字符串
html = response.read()
print html
2、urllib2.Request() 设置请求头
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib2
# 1、请求头参数键值对
ua_headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
}
# 2、urllib2.Request() 方法构造一个请求对象
request = urllib2.Request(
"http://www.baidu.com/", # 固定参数，后面是一串【变参列表】
headers = ua_headers # 【变参列表】后的固定参数，必须写上参数名
)
# 3、向指定的url地址发送请求，并返回服务器响应的类文件对象
response = urllib2.urlopen(request)
# 4、read()方法就是读取文件里的全部内容，返回字符串
html = response.read()
# 5、返回 HTTP的响应码，成功返回200，4服务器页面出错，5服务器问题
print response.getcode()
# 6、返回 返回实际数据的实际URL，防止重定向问题
print response.geturl()
# 7、返回 服务器响应的HTTP报头
print response.info()
# 8、打印响应内容
print html
3、请求头 - UserAgent
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib2
import random
url = "http://www.baidu.com/"
# 1、可以是User-Agent列表，也可以是代理列表
ua_list = [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv2.0.1) Gecko/20100101 Firefox/4.0.1",
"Mozilla/5.0 (Windows NT 6.1; rv2.0.1) Gecko/20100101 Firefox/4.0.1",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"
]
# 2、在User-Agent列表里随机选择一个User-Agent
user_agent = random.choice(ua_list)
# 3、构造一个请求
request = urllib2.Request(url)
# 4、add_header()方法 添加/修改 一个HTTP报头
request.add_header("User-Agent", user_agent)
# 5、get_header() 获取一个已有的HTTP报头的值，注意只能是第一个字母大写，其他的必须小写
print request.get_header("User-agent")
4、发送 GET 请求
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib
import urllib2
url = "http://www.baidu.com/s"
headers = {"User-Agent" : "Mozilla 。。。。"}
keyword = raw_input("请输入需要查询的关键字： ")
wd = {"wd" : keyword}
# 通过urllib.urlencode() 参数是一个dict类型
wd = urllib.urlencode(wd)
# 拼接完整的url
fullurl = url + "?" + wd
# 构造请求对象
request = urllib2.Request(fullurl, headers = headers)
# 发送网络请求，并获取结果数据
response = urllib2.urlopen(request)
print response.read()
5、发送 POST 请求
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib
import urllib2
# 1、通过抓包的方式获取的url，并不是浏览器上显示的url
url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule&smartresult=ugc&sessionFrom=null"
# 2、完整的请求头headers字典
headers = {
"Accept" : "application/json, text/javascript, */*; q=0.01",
"X-Requested-With" : "XMLHttpRequest",
"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
"Content-Type" : "application/x-www-form-urlencoded; charset=UTF-8",
}
# 3、用户接口输入
key = raw_input("请输入需要翻译的文字:")
# 4、发送到web服务器的表单数据
formdata = {
"type" : "AUTO",
"i" : key,
"doctype" : "json",
"xmlVersion" : "1.8",
"keyfrom" : "fanyi.web",
"ue" : "UTF-8",
"action" : "FY_BY_CLICKBUTTON",
"typoResult" : "true"
}
# 5、经过urlencode转码
data = urllib.urlencode(formdata)
# 6、如果Request()方法里的data参数有值，那么这个请求就是POST
request = urllib2.Request(url, data = data, headers = headers)
# 7、发送网络请求
print urllib2.urlopen(request).read()
6、设置 cookie 请求头参数
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib2
url = "http://www.renren.com/410043129/profile"
# 1、请求头参数键值对
headers = {
"Host" : "www.renren.com",
"Connection" : "keep-alive",
#"Upgrade-Insecure-Requests" : "1",
"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Referer" : "http://www.renren.com/SysHome.do",
#"Accept-Encoding" : "gzip, deflate, sdch",
"Cookie" : "anonymid=ixrna3fysufnwv; _r01_=1; depovince=GW; jebe_key=f6fb270b-d06d-42e6-8b53-e67c3156aa7e%7Cc13c37f53bca9e1e7132d4b58ce00fa3%7C1484060607478%7C1%7C1484400895379; jebe_key=f6fb270b-d06d-42e6-8b53-e67c3156aa7e%7Cc13c37f53bca9e1e7132d4b58ce00fa3%7C1484060607478%7C1%7C1484400890914; JSESSIONID=abcX8s_OqSGsYeRg5vHMv; jebecookies=0c5f9b0d-03d8-4e6a-b7a9-3845d04a9870|||||; ick_login=8a429d6c-78b4-4e79-8fd5-33323cd9e2bc; _de=BF09EE3A28DED52E6B65F6A4705D973F1383380866D39FF5; p=0cedb18d0982741d12ffc9a0d93670e09; ap=327550029; first_login_flag=1; [email protected]; ln_hurl=http://hdn.xnimg.cn/photos/hdn521/20140529/1055/h_main_9A3Z_e0c300019f6a195a.jpg; t=56c0c522b5b068fdee708aeb1056ee819; societyguester=56c0c522b5b068fdee708aeb1056ee819; id=327550029; xnsid=5ea75bd6; loginfrom=syshome",
"Accept-Language" : "zh-CN,zh;q=0.8,en;q=0.6",
}
# 2、创建请求对象
request = urllib2.Request(url, headers = headers)
# 3、创建响应
response = urllib2.urlopen(request)
# 4、发送网络请求，获取响应数据
print response.read()
7、忽略https验证
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib2
import ssl
url = "https://www.12306.cn/mormhweb/"
# 1、忽略SSL安全认证
context = ssl._create_unverified_context()
# 2、请求头参数键值对
headers = {
"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
}
# 3、创建请求
request = urllib2.Request(url, headers=headers)
# 4、忽略SSL安全认证，添加到context参数里
response = urllib2.urlopen(request, context=context)
# 5、读取响应数据
print response.read()
8、urllib2.build_opener()
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib2
# 1、构建一个HTTPHandler处理器对象，支持处理HTTP的请求
# -1）增加参数"debuglevel=1"将会自动打开Debug log 模式，
# -2）程序在执行的时候会打印收发包的信息
# http_handler = urllib2.HTTPHandler()
http_handler = urllib2.HTTPHandler(debuglevel=1)
# 2、调用build_opener()方法构建一个自定义的opener对象
opener = urllib2.build_opener(http_handler)
# 3、构造请求参数
request = urllib2.Request("http://www.baidu.com/")
# 4、opner().open()发送网络请求
response = opener.open(request)
# 5、读取结果数据
print response.read()
