- scrapy框架爬取今日头条数据，主要实现一下几个主要功能：
- 数据存储到mongodb数据库
- 图片下载
- 随机切换User-Agent
- 对接IP代理池
- 实现邮件发送
1.首先按F12打开开发者工具，如图：
- 由于今日头条的数据是js动态加载，我们需要找到加载数据的接口，进攻分析，数据接口链接如上图所示。
-
首先查看Headers信息：
headers
我们发现query string parameters 中有as和cp这两个参数，这是今日头条一种反扒措施，我们需要编写算法来生成这两个参数（网上找的）
-
我们再看response也就是返回的数据信息：
response
我们将数据拷贝到在线的json格式转换器中，方便我们对数据进行分析：
json数据 - 上图我们可以看到，返回的数据包含来新闻的全部信息，其中在最后还有一个max_behot_time参数，通过改变这个参数信息，我们可以实现新闻数据循环抓取
- spider代码如下：
# -*- coding: utf-8 -*-
import scrapy
import json
import time
import hashlib
import random
import requests
from datetime import datetime
from ..emailsend import EmailSend
from toutiao_two.items import ToutiaoTwoItem
class ToutiaoSpiderSpider(scrapy.Spider):
name = 'toutiao_spider'
allowed_domains = ['www.toutiao.com']
headers = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',
'Host': 'www.toutiao.com',
'Accept': 'application/json, text/plain, */*',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4,zh-TW;q=0.2,mt;q=0.2',
'Connection': 'keep-alive',
'X-Requested-With': 'XMLHttpRequest',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
}
cookies = {'tt_webid': '6722356446824613389'}
start_url = 'https://www.toutiao.com/api/pc/feed/?category=news_hot&utm_source=toutiao&widen=1&max_behot_time='
max_behot_time = '0'
D = {'hot_time': '0'}
def get_as_cp(self): # 该函数主要是为了获取as和cp参数，程序参考今日头条中的加密js文件：home_4abea46.js
zz = {}
now = round(time.time())
print(now) # 获取当前计算机时间
e = hex(int(now)).upper()[2:] # hex()转换一个整数对象为16进制的字符串表示
print('e:', e)
a = hashlib.md5() # hashlib.md5().hexdigest()创建hash对象并返回16进制结果
print('a:', a)
a.update(str(int(now)).encode('utf-8'))
i = a.hexdigest().upper()
print('i:', i)
if len(e) != 8:
zz = {'as': '479BB4B7254C150',
'cp': '7E0AC8874BB0985'}
return zz
n = i[:5]
a = i[-5:]
r = ''
s = ''
for i in range(5):
s = s + n[i] + e[i]
for j in range(5):
r = r + e[j + 3] + a[j]
zz = {
'as': 'A1' + s + e[-3:],
'cp': e[0:3] + r + 'E1'
}
print('zz:', zz)
return zz
def start_requests(self):
global start_time
start_time = datetime.now()
ascp = self.get_as_cp()
yield scrapy.FormRequest(url=self.start_url + self.max_behot_time + '&max_behot_time_tmp=' + self.max_behot_time + '&tadrequire=true&as=' + ascp[
'as'] + '&cp=' + ascp['cp'],
method='GET',
headers=self.headers,
cookies=self.cookies,
callback=self.parse,
)
def parse(self, response):
json_result = json.loads(response.text)
# if json_result is None:
# print(self.D['hot_time'], '=====')
# time.sleep(20)
# yield scrapy.FormRequest(
# url=self.start_url + self.D['hot_time'] + '&max_behot_time_tmp=' + self.D['hot_time'] + '&tadrequire=true&as=' +
# 'A115DD5DE72AC29' + '&cp=' + '5DD7FA9C02D90E1',
# method='GET',
# headers=self.headers,
# cookies=self.cookies,
# callback=self.parse,
# )
item = ToutiaoTwoItem()
infos = json_result['data']
for info in infos:
image_url_list = []
item['abstract'] = info['abstract'] if info.get('abstract') else ''
item['chinese_tag'] = info['chinese_tag'] if info.get('chinese_tag') else ''
item['title'] = info['title'] if info.get('title') else ''
item['source'] = info['source'] if info.get('source') else ''
image_urls = info['image_list'] if info.get('image_list') else ''
for image_url in image_urls:
url = 'https:' + image_url['url']
image_url_list.append(url)
item['image_url'] = image_url_list
yield item
time.sleep(random.randint(1, 4))
print(self.D['hot_time'])
if json_result.get('next'):
next = json_result['next']
if next.get('max_behot_time'):
max_behot_time = str(json_result['next']['max_behot_time'])
self.D.update({'hot_time': max_behot_time})
ascp = self.get_as_cp()
yield scrapy.FormRequest(
url=self.start_url + max_behot_time + '&max_behot_time_tmp=' + max_behot_time + '&tadrequire=true&as=' +
str(ascp['as']) + '&cp=' + str(ascp['cp']),
method='GET',
headers=self.headers,
cookies=self.cookies,
callback=self.parse,
)
def closed(self, reason):
# 爬虫关闭的时候，会调用这个方法
email = EmailSend()
# 爬虫耗时
use_time = datetime.now() - start_time
close_time = 'toutiao爬虫开始时间{};结束时间:{};爬虫耗时:{}'.format(start_time, datetime.now(), use_time)
content = '爬虫关闭原因:{}'.format(reason)
email.send_text_email('发送者邮箱@qq.com', '接受者邮箱@qq.com', close_time, content)
实现IP代理池对接，在middlewares.py中实现,增加如下代码，并在settings中配置：
# 对接IP代理池
class ProxyMiddleware():
def __init__(self, proxy_url):
self.logger = logging.getLogger(__name__)
self.proxy_url = proxy_url
def get_random_proxy(self):
try:
response = requests.get(self.proxy_url)
if response.status_code == 200:
proxy = response.text
return proxy
except requests.ConnectionError:
return False
def process_request(self, request, spider):
if request.meta.get('retry_times'):
proxy = self.get_random_proxy()
if proxy:
uri = 'https://{proxy}'.format(proxy=proxy)
self.logger.debug('使用代理 ' + proxy)
request.meta['proxy'] = uri
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
return cls(
proxy_url=settings.get('PROXY_URL')
)
- setteings
# 抛出可用IP地址
PROXY_URL = 'http://localhost:5555/random'
设置随机切换User-Agent，同样在middlewares.py中实现，并在settings配置：
# 随机切换User-Agent
class RandomUserAgent(object):
"""Randomly rotate user agents based on a list of predefined ones"""
def __init__(self, agents):
self.agents = agents
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings.getlist('USER_AGENTS'))
def process_request(self, request, spider):
#print "**************************" + random.choice(self.agents)
request.headers.setdefault('User-Agent', random.choice(self.agents))
- settings
USER_AGENTS = [
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
]
DOWNLOADER_MIDDLEWARES = {
'toutiao_two.middlewares.RandomUserAgent': 543,
'toutiao_two.middlewares.ProxyMiddleware': 550,
}
在pipipelines中实现图片下载
import copy
import pymongo
from pymongo import MongoClient
from scrapy import Request
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
# 下载图片
class ToutaioImagePipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None):
url = request.url
file_name = url.split('/')[-1] + '.jpg'
return file_name
def item_completed(self, results, item, info):
image_paths = [x['path'] for ok, x in results if ok]
if not image_paths:
raise DropItem('Image DownloadedFailed')
return item
def get_media_requests(self, item, info):
for image_url in item['image_url']:
yield Request(image_url)
- settings中配置图片存储路径：
IMAGES_STORE = './images'
- 同样在pipipelines中实现数据存储
# mongodb
class ToutiaoTwoMongoPipeline():
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls, crawler):
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DB')
)
def open_spider(self, spider):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
def close_spider(self, spider):
self.client.close()
def process_item(self, item, spider):
"""
问题：数据存入数据库之后，出现大量重复数据
解决思路：
在process_item中执行数据插入之前，先对变量进行复制copy，再用复制copy的变量进行操作，通过互斥确保变量不被修改。因此，修正这个问题，我们只需要调整优化下process_item()方法。
解决代码：process_item() - copy.deepcopy(item) ->导入copy包
"""
asynItem = copy.deepcopy(item)
infos = {'chinese_tag': asynItem['chinese_tag'], 'title': asynItem['title'], 'source': asynItem['source'],
'image_url': asynItem['image_url']}
self.db.toutiao.insert(infos)
return item
- settings中配置数据库链接信息：
ITEM_PIPELINES = {
'toutiao_two.pipelines.ToutiaoTwoMongoPipeline': 300,
'toutiao_two.pipelines.ToutaioImagePipeline': 300,
}
MONGO_URI = 'localhost'
MONGO_DB = 'scrapy_toutiao'
实现邮件发送，在settings同级目录下新增emailsend.py文件，写入如下内容：
#-*- coding: utf-8 -*-
'''
---------------
Description of this file
:author: Luopeng
:date created: 2019-12-04
:python version: 3.6
---------------
'''
import smtplib
from email.mime.text import MIMEText
import logging
class EmailSend(object):
def __init__(self):
self.logging = logging.getLogger('Waring')
self.email_host = 'smtp.qq.com'
self.email_port = '465'
self.email_pass = '*********' # 自己的授权码
def send_text_email(self, from_addr, to_addrs, subject, content):
self.logging.warning('send_text_email is willed 丢弃')
self.logging.error('send_text_email is None')
message_text = MIMEText(content, 'plain', 'utf8')
message_text['From'] = from_addr
message_text['To'] = to_addrs
message_text['Subject'] = subject
try:
# 在创建客户端对象的同时，连接到邮箱服务器。
client = smtplib.SMTP_SSL(host=self.email_host, port=self.email_port)
login_result = client.login(from_addr, self.email_pass)
if login_result and login_result[0] == 235:
print('登录成功')
client.sendmail(from_addr, to_addrs, message_text.as_string())
print('邮件发送成功')
else:
print('邮件发送异常：', login_result[0], login_result[1])
except Exception as e:
# print('连接邮箱服务器异常：',e)
self.logging.error('连接邮箱服务器异常：{}'.format(e))
def send_image_email(self):
pass
def send_word_email(self):
pass
def send_video_email(self):
pass
- 具体调用请参考spider里面的代码