爬虫实战：头条图集Ajax抓取

"""
filename: toutiaophoto/spider.py
python: 3.7.0
description: 使用requests爬取今日头条图集存入mongodb

"""

import requests
from requests.exceptions import RequestException
import json
import re
import random
from config import *
from pymongo import MongoClient
from hashlib import md5

client = MongoClient(MONGO_URL)

def get_index_page(url, params):
    """获取索引页"""
    try:
        response = requests.get(url, params=params)
        print("[+] " +response.request.url)
        if response.status_code == 200:
            return response.text
        return "[-] status code(" + response.status_code + "): " + url
    except RequestException as e:
        print("[-] 获取索引页错误(" + e + "): " + url)

def parse_index_page(data):
    """解析索引页"""
    data = json.loads(data)
    items = data['data']
    for item in items:
        if 'article_url' in item.keys():
            if "toutiao.com" in item['article_url']:
                yield item['article_url']

def get_details_page(url, headers):
    """获取详情页"""
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.content.decode('utf8')
        return "[-] status code(" + str(response.status_code) + "): " + url
    except RequestException as e:
        print("[-] 获取详情页错误(" + e + "): " + url)

def parse_details_page(content):
    """解析详情页"""

    # 获取标题
    if re.search("title: '(.*?)'", content) is None:
        return None
    title = re.search("title: '(.*?)'", content).group(1)

    # 获取图片url
    pattern = re.compile(r'uri\\":\\"(.*?)"')
    items = pattern.findall(content)
    image_urls = []
    for item in items:
        image_urls.append("http://p99.pstatp.com/" + item.replace(r'\\', "", -1)[:-1])

    # 获取时间
    if re.search("publish_time: '(.*?)',", content) is None:
        time = None
    else:
        time = re.search("publish_time: '(.*?)',", content).group(1)

    # 构造规则结果
    item = {
        'title': title,
        'time': time,
        'image_urls': image_urls
    }
    return item


def main(offset):
    start_url = "https://www.toutiao.com/search_content/?"
    params = {
        'offset': offset,
        'format': 'json',
        'keyword': KEYWORD,
        'autoload': 'true',
        'count': '20',
        'cur_tab': '3',
        'from': 'gallery',
    }

    ua = [
        # 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
        # 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
        # "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        # "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
        "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50",
        ]
    headers = {
        "user-agent": random.choice(ua)
    }

    data = get_index_page(start_url, params=params)

    for details_url in parse_index_page(data):
        content = get_details_page(details_url, headers)
        item = parse_details_page(content)
        if item is not None:
            print(item['title'])
            item['_id'] = "ObjectId('" + md5(item['title'].encode('utf8')).hexdigest() + "')"
            # 唯一写入
            if client[MONGO_DB][MONGO_TABLE].find_one({"_id": item['_id']}):
                continue
            # 写入mongodb
            client[MONGO_DB][MONGO_TABLE].insert(item, continue_on_error=True)



if __name__ == '__main__':

    for i in range(START_PAGE, END_PAGE):
        main(i*10)

image.png

注意：有反爬，需要更换user-agent和代理