python 爬虫 伪装成 chrome

利用 urllib 发起的请求，UA 默认是 Python-urllib/3.5 而在 chrome 中访问 UA 则是 User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36，因为服务器根据 UA 来判断拒绝了 python 爬虫。

import urllib  
from urllib import request  
page = 1  
url = 'http://www.qiushibaike.com/hot/page/'+str(page)  
#url = 'http://www.baidu.com'  
def getHTML(url):  
    headers = {'User-Agent': 'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}  
    req = request.Request(url, headers=headers)  
    return request.urlopen(req)  
  
try:  
    response = getHTML(url)  
    print(response.read())  
except urllib.request.URLError as e:  
    if hasattr(e,'code'):  
        print(e.code())  
    if hasattr(e,'reason'):  
        print(e,reason())