python3代理爬虫

抓取索引页内容

这里写代码片

代理设置

这里写代码片

分析详情页内容

这里写代码片

将数据保存到数据库

完整代码

import  requests
from  urllib.parse import urlencode
from  pyquery import PyQuery  as pq
from setting import  *
import pymysql

base_url = 'http://weixin.sogou.com/weixin?'
keyword = 'nba排名'
proxy_pool_url = 'http://localhost:5000/get'

proxy=None
max_count=5

headers = {
    'Cookie' :'IPLOC=CN3205; SUV=00C12A5E7550DE2E5A362D333D8AA437; CXID=C44BCB28F7BD7663A5494737D6EA1A61; ad=HZllllllll2z2eU4lllllVI0EWwlllllnHn8fkllll9lllllVqxlw@@@@@@@@@@@; SUID=12EE52753665860A5A07E844000E7FF5; wuid=AAHlthigHQAAAAqLK0Z4/QUAGwY=; ABTEST=8|1517105588|v1; SNUID=F5D12491070264C4A3BD171908D3114D; weixinIndexVisited=1; sct=2; JSESSIONID=aaauC50avZYdCc7j0xCew; ppinf=5|1517107611|1518317211|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo2OiVEMSU4NHxjcnQ6MTA6MTUxNzEwNzYxMXxyZWZuaWNrOjY6JUQxJTg0fHVzZXJpZDo0NDpvOXQybHVNcWhZS250cll6Wi1SQW40c1JWSkw4QHdlaXhpbi5zb2h1LmNvbXw; pprdig=eq_cq2lEyvPuPu2S1-6fOrxEvdikZmFAzrsTvHVFNmG-cPN7GOvVhYsjF78ABcjjPbmIP_5PrnaJTfYQT4dvnYik2fvfbODVT67n7MonAutWspNOSv7snXR0ClDVAEb3JDBGV0xFVT0tsdHxi9MkvAAzo2LqidqovzvZnH11EAo; sgid=11-33218981-AVptOZu2Lgu0umQEuMTbkM4; ppmdig=1517107611000000f4537b3ad16dd1b8ec8ce8d2767eca92' ,
    'Host' : 'weixin.sogou.com',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/57.0'
}

def get_proxy():
    try:
        respone = requests.get(proxy_pool_url)
        if respone.status_code == 200:
            return  respone.text
        get_proxy()
    except ConnectionError:
        get_proxy()

def get_html(url,count=1):
    print('Crawling',url)
    print('Tring Count',count)
    global proxy
    if count >= max_count:
        print('Tried Too Many Counts')
        return None
    try:
        if proxy:
            proxies = {
                'http':'http://'+proxy
            }
            response = requests.get(url,allow_redirects=False , headers=headers,proxies=proxies)
        else:
            response = requests.get(url,allow_redirects=False , headers=headers)
        if response.status_code == 200:
            return  response.text
        if response.status_code == 302:
            print('302')
            proxy = get_proxy()
            if proxy:
                print('Using Proxy',proxy)
                return get_html(url)
            else:
                print('Get Proxy Failed')
                return get_html(url)
    except ConnectionError as  e:
        print('ConnectionError Occurred',e.args)
        proxy = get_proxy()
        count +=1
        return  get_html(url,count)
    except TimeoutError as e:
        print('TimeoutError Occurred', e.args)
        proxy = get_proxy()
        return get_html(url)
    except ConnectionRefusedError as e:
        print('ConnectionRefusedError Occurred', e.args)
        proxy = get_proxy()
        return get_html(url)

def get_index(keyword,page):
    data = {
        'query': keyword,
        'type' : 2,
        'page' : page,
        'ie'  : 'utf8'
    }
    queries = urlencode(data)
    url = base_url + queries
    html = get_html(url)
    return html

def parse_index(html):
    doc = pq(html)
    items = doc('#sogou_vr_11002601_title_0').items()
    for item in items:
        yield item.attr('href')

def get_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        else:
            return None
    except ConnectionError:
        return None

def parse_detail(html):
    doc = pq(html)
    title = doc('.rich_media_title').text()
    content = doc('.rich_media_content').text()
    date = doc('#post-date').text()
    nickname = doc('.profile_nickname').text()
    wechat = doc('#js_profile_qrcode .profile_inner .profile_meta .profile_meta_value').eq(0).text()
    return (title,content,date,nickname,wechat)

def main():
    #for page in range(1,2):
        html = get_index(keyword,1)
        if html:
            article_urls = parse_index(html)
            for article_url in article_urls:
                article_html = get_detail(article_url)
                if article_url:
                    article_data =  parse_detail(article_url)
                    conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, charset=charset)
                    cur = conn.cursor()
                    cur.execute("insert into nba_ranking (title, content, date, nickname, wechat) values (%s,%s,%s,%s,%s)",article_data)
                    conn.commit()
                    conn.close()

if __name__ == '__main__':
    main()