requests多进程爬取今日头条街拍--记录

spider.py

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import requests
import re
import json
import pymongo
import os
from requests.exceptions import RequestException
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from hashlib import md5
from multiprocessing import Pool
from config import *

#定义数据库
client = pymongo.MongoClient(IP, PORT)
db = client[DB]


def open_url(url,encode=None,imgurl=False):
    '''定义网址打开方法'''
    if encode == None:
        response = requests.get(url)
    else:
        response = requests.get(url + urlencode(encode))
    try:
        if response.status_code == 200 and imgurl == False:
            return response.text
        elif response.status_code == 200 and imgurl == True:
            save_images(response.content)
            print('%s 下载完成..' % url)
        return None
    except RequestException:
        print('请求失败..')
        return None

def get_page_index(offset,keyword):
    '''POST的请求参数'''
    data = {
            'offset': offset,
            'format': 'json',
            'keyword': keyword,
            'autoload': 'true',
            'count': '20',
            'cur_tab': '3',
            'from':'gallery',
        }
    return data

def parse_page_index(html):
    '''获取详情页的URL'''
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            if item.get('article_url').startswith('http://toutiao.com'):
                yield item.get('article_url')

def get_pic_url(url):
    '''提取出图片地址'''
    html = open_url(url)
    soup = BeautifulSoup(html,'lxml')
    title = soup.select('title')[0].get_text()
    images_pattern = re.compile('gallery: JSON.parse\("(.*?)"\),',re.S)
    result = re.search(images_pattern,html).group(1).replace('\\','')
    url_pattern = re.compile('"url_list":\[{"url":"(.*?)"},',re.S)
    url_result = re.findall(url_pattern,result)
    for imgurl in url_result:open_url(imgurl,imgurl=True)

    return {
        'title':title,
        'images':url_result,
        'url':url
    }

def save_to_mongo(result):
    '''存储到数据库'''
    if db[TABLE].insert(result):
        return True
    return False

def save_images(content):
    '''保存图片'''
    if os.path.exists(os.getcwd() + '\\images'):
        file_path = '{0}\\images\\{1}.{2}'.format(os.getcwd(),md5(content).hexdigest(),'jpg')
    else:
        os.mkdir(os.getcwd() + '\\images')
        file_path = '{0}\\images\\{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')

    if not os.path.exists(file_path):
        with open(file_path,'wb') as fp:
            fp.write(content)

def run(offset):
    '''整体运行'''
    jsondata = open_url('https://www.toutiao.com/search_content/?',get_page_index(offset,KEYWORD))
    for picurl in parse_page_index(jsondata):
        data = get_pic_url(picurl)
        save_to_mongo(data)


if __name__ == '__main__':
    groups = [x*20 for x in range(GROUP_START,GROUP_END + 1)]
    pool = Pool()
    pool.map(run,groups)
    pool.close()
    pool.join()

config.py

IP = 'localhost'
PORT = 27017
DB = 'photo'
TABLE = 'jp'

GROUP_START = 1
GROUP_END = 20
KEYWORD = '街拍'