今日头条街拍美图爬取

代码

'''
目标网站:https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D
目标:爬取图片
方法:Ajax 爬取
'''

import requests
from pyquery import PyQuery as pq
from urllib.parse import urlencode
import json
import time

# http://www.toutiao.com/search_content/?
base_url = 'https://www.toutiao.com/api/search/content/?'
headers = {
    'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
    'x-requested-with' : 'XMLHttpRequest'
}

'''
    功能:爬取页面
    入参:keyword
    出参:爬取的网页
'''
def get_page(keyword, page):
    parms = {
        'aid' : '24',
        'app_name' : 'web_search',
        'offset' : '0',
        'format' : 'json',
        'keyword' : keyword,
        'autoload' : 'true',
        'count' : str(20*page),
        'en_qc' : '1',
        'cur_tab' : '1',
        'from' : 'search_tab',
        'pd' : 'synthesis',
        'timestamp' : str(int(round(time.time() * 1000)))
    }
    url = base_url + urlencode(parms)
    try:
        response = requests.get(url, headers = headers)
        print(url)
        if response.status_code == 200:
            return response.json()
        else:
            print(response.status_code)
    except request.ConnectionError as e:
        print('Error', e.args)

def save_json(file, json_data):
    file = file + '.json'
    with open(file, 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_data, indent=2, ensure_ascii=False))

keyword = '街拍'
json_data = get_page(keyword, page=1)
save_json(keyword, json_data)

问题:爬取到 html,但 data 数据是空的?

你可能感兴趣的:(问题集锦)