python简单爬取安居客

爬取安居客信息

-- coding:utf-8 --

‘’’
让我们的爬虫程序，伪装成浏览器发送请求

1. 定义User-agent
2. 设置请求头
   ‘’’

#引入需要的模块
import urllib.request
import random

#定义的可用的User-agent
ua=[
‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36’,
‘User-Agent:Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Trident/5.0;’,
‘User-Agent:Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;Trident/4.0)’,
‘User-Agent:Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.0)’
]

#随机选择一个ua
user_agent=random.choice(ua)

#定义请求头
my_header = {
‘User-agent’:user_agent
}

#封装请求对象，并且设置请求头数据
url =“https://www.taobao.com”
request=urllib.request.Request(url,headers=my_header)

#发送请求并且获取数据
response=urllib.request.urlopen(request)

print(response.read())
with open(‘gf.txt’, ‘w’) as f:
f.write(response.read())