import requests
from urllib.parse import urlencode
from requests.exceptions import ConnectionError
from pyquery import PyQuery as pq
import pymongo
import sys
sys.setrecursionlimit(1000000)
client = pymongo.MongoClient('localhost')
db = client['weixin']
headers = {
'Cookie': 'SUV=0050272B0E1788145AD0D1D0B5C42214; dt_ssuid=3835736327; pex=C864C03270DED3DD8A06887A372DA219231FFAC25A9D64AE09E82AED12E416AC; ssuid=8355709570; SUID=1488170E3665860A5B933B700007EAF7; SMYUV=1540625126607399; CXID=0E31D6148DFE7E3823CFFE36FA7A5EDF; pgv_pvi=389063680; GOTO=; toutiao_city_news=%E5%8C%97%E4%BA%AC; _ga=GA1.2.1568155290.1580978361; wuid=AAG/ghcULgAAAAqLMXVRzQkAGwY=; IPLOC=CN4401; ABTEST=0|1602147107|v1; SNUID=3CF06076777DC65C89455BFC78EA55FC; weixinIndexVisited=1; JSESSIONID=aaakEGIn8Y8Nx7fLl-atx',
'Host': 'weixin.sogou.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
base_url = 'https://weixin.sogou.com/weixin?'
def get_index(keyword,page):
data = {
'query':keyword,
'type':2,
'page':page
}
queries = urlencode(data)
url = base_url + queries
html = get_html(url)
return html
keyword = '风景'
proxy_pool_url = 'http://127.0.0.1:5555/random'
proxy = None
max_count =5
def get_html(url,count=1):
print('Crawling',url)
print('Trying Count',count)
global proxy
if count >= max_count:
print('Tried Too Many Counts')
return None
try:
if proxy:
proxies = {
'http':'http://'+proxy
}
response = requests.get(url, allow_redirects=False, headers=headers,proxies=proxies)
else:
response = requests.get(url, allow_redirects=False, headers=headers)
if response.status_code == 200:
return response.text
if response.status_code == 302:
print('302')
proxy = get_proxy()
if proxy:
print('Using Proxy',proxy)
return get_html(url)
else:
print('Get Proxy Failed')
return None
except ConnectionError as e:
print('Error occurred',e.args)
proxy = get_proxy()
count +=1
return get_html(url,count)
def get_proxy():
try:
response = requests.get(proxy_pool_url)
if response.status_code == 200:
return response.text
return None
except ConnectionError:
return None
def parse_index(html):
doc = pq(html)
items = doc('.news-box .news-list li .txt-box h3 a').items()
for item in items:
yield item.attr('href')
def get_detail(url):
try:
response = requests.get(url)
if response.status_code == 200:
return response.text
return None
except ConnectionError:
return None
def parse_detail(html):
doc = pq(html)
title = doc('#img-content .rich_media_title').text()
nickname = doc('.rich_media_meta_list .rich_media_meta_nickname').text()
date = doc ('.rich_media_meta_list #post-date').text()
wechat = doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
content = doc('.rich_media_content').text()
return {
'title':title,
'nickname':nickname,
'date':date,
'wechat':wechat,
'content':content
}
def save_to_mongo(data):
if db['articles'].update({'title':data['title']},{'$set':data},True):
print('Save to Mongo',data['title'])
else:
print('Save to Mongo Failed',data['title'])
def main():
for page in range(1,101):
html = get_index(keyword,page)
if html:
article_urls = parse_index(html)
for article_url in article_urls:
article_html = get_detail(article_url)
if article_html:
article_data = parse_detail(article_html)
print(article_data)
if article_data:
save_to_mongo(article_data)
if __name__ == '__main__':
main()