Python爬虫之使用代理爬取

#代理服务器需要购买，也有免费的https://www.kuaidaili.com/  http://www.xicidaili.com/

import urllib.request
url = 'http://www.baidu.com/s?ie=utf-8&wd=ip'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
}
request = urllib.request.Request(url=url,headers=headers)

'''

response = urllib.request.urlopen(request)
# print(response.read().decode('utf-8'))
with open('baidu.html','wb') as fp:
    fp.write(response.read())
'''
#配置代理
# 注意这里的http或者https必须与上面访问的url对应起来
handler = urllib.request.ProxyHandler({'http':'61.155.164.107:3128',}) #里面写代理服务器，是一个字典
#创建opener
opener = urllib.request.build_opener(handler)
response = opener.open(request)
with open('ip_daili_baidu.html','wb') as fp:
    fp.write(response.read())