求救爬虫的疑难杂症!!!requests.exceptions.MissingSchema: Invalid URL ‘xxx‘ No schema supplied.

#引入模块
import requests
from urllib.parse import urlencode
from requests.exceptions import ConnectionError
from pyquery import PyQuery as pq
import pymongo
import sys
sys.setrecursionlimit(1000000)

client = pymongo.MongoClient('localhost')
db = client['weixin']
#参数设置
headers = {
    'Cookie': 'SUV=0050272B0E1788145AD0D1D0B5C42214; dt_ssuid=3835736327; pex=C864C03270DED3DD8A06887A372DA219231FFAC25A9D64AE09E82AED12E416AC; ssuid=8355709570; SUID=1488170E3665860A5B933B700007EAF7; SMYUV=1540625126607399; CXID=0E31D6148DFE7E3823CFFE36FA7A5EDF; pgv_pvi=389063680; GOTO=; toutiao_city_news=%E5%8C%97%E4%BA%AC; _ga=GA1.2.1568155290.1580978361; wuid=AAG/ghcULgAAAAqLMXVRzQkAGwY=; IPLOC=CN4401; ABTEST=0|1602147107|v1; SNUID=3CF06076777DC65C89455BFC78EA55FC; weixinIndexVisited=1; JSESSIONID=aaakEGIn8Y8Nx7fLl-atx',
    'Host': 'weixin.sogou.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}

#构造URL,进行微信关键词搜索
base_url = 'https://weixin.sogou.com/weixin?'
def get_index(keyword,page):
    data = {
        'query':keyword,
        'type':2,
        'page':page
    }
    queries = urlencode(data)
    url = base_url + queries
    html = get_html(url)
    return html


keyword = '风景'
proxy_pool_url = 'http://127.0.0.1:5555/random'

proxy = None
max_count =5


#请求URL,得到索引页HTML
def get_html(url,count=1):
    print('Crawling',url)
    print('Trying Count',count)
    global proxy
    if count >= max_count:
        print('Tried Too Many Counts')
        return None
    try:
        if proxy:
            proxies = {
                'http':'http://'+proxy
            }
            response = requests.get(url, allow_redirects=False, headers=headers,proxies=proxies)
        else:
            response = requests.get(url, allow_redirects=False, headers=headers)
        if response.status_code == 200:
            return response.text
        if response.status_code == 302:
            #Need Proxy
            print('302')
            proxy = get_proxy()
            if proxy:
                print('Using Proxy',proxy)
                return get_html(url)
            else:
                print('Get Proxy Failed')
                return None

    except ConnectionError as e:
        print('Error occurred',e.args)
        proxy = get_proxy()
        count +=1
        return get_html(url,count)


def get_proxy():
    try:
        response = requests.get(proxy_pool_url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None

def parse_index(html):
    doc = pq(html)
    items = doc('.news-box .news-list li .txt-box h3 a').items()
    for item in items:
        yield item.attr('href')

def get_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None


def parse_detail(html):
    doc = pq(html)
    title = doc('#img-content .rich_media_title').text()
    nickname = doc('.rich_media_meta_list .rich_media_meta_nickname').text()
    date = doc ('.rich_media_meta_list #post-date').text()
    wechat = doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
    content = doc('.rich_media_content').text()
    return {
        'title':title,
        'nickname':nickname,
        'date':date,
        'wechat':wechat,
        'content':content
    }


def save_to_mongo(data):
    if db['articles'].update({'title':data['title']},{'$set':data},True):
        print('Save to Mongo',data['title'])
    else:
        print('Save to Mongo Failed',data['title'])



def main():
    for page in range(1,101):
        html = get_index(keyword,page)
        #print(html)
        if html:
            article_urls = parse_index(html)
            for article_url in article_urls:
                #print(article_url)
                article_html = get_detail(article_url)
                if article_html:
                    article_data = parse_detail(article_html)
                    print(article_data)
                    if article_data:
                        save_to_mongo(article_data)





if __name__ == '__main__':
    main()

求救爬虫的疑难杂症!!!requests.exceptions.MissingSchema: Invalid URL ‘xxx‘ No schema supplied._第1张图片

你可能感兴趣的:(求救爬虫的疑难杂症!!!requests.exceptions.MissingSchema: Invalid URL ‘xxx‘ No schema supplied.)