scrapy 爬取当当网-图书排行榜-多条件爬取

自学爬虫框架scrapy，爬取当当网-图书排行榜练手

• 目标：爬取当当网-图书畅销榜中的图书数据，要求各种条件的数据都要有。

  
  
  dangdang.png
• spider

# -*- coding: utf-8 -*-
import scrapy
from dd_book.items import DdBookItem
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class DdbookSpider(scrapy.Spider):
    name = 'ddbook'
    start_urls = ['http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-1']

    def __init__(self):
        self.driver = webdriver.PhantomJS()
        self.item = DdBookItem()

    def parse(self, response):
        '''
        该方法用于从start_urls定义的初始url中获取需要爬取的几个不同条件的url
        条件分为
            近24小时，近七日，近30日
            http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-1
            http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-recent7-0-0-1-1
            http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-recent30-0-0-1-1
            今年1月，2月...本月
            往年：2015,2016,2017,2018
            条件不同，url不同，但是有规律
        每种条件的url不同
        '''
        self.driver.get(response.url)
        wait = WebDriverWait(self.driver,3)
        wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.bang_list_date')))
        p_lis = response.css('.bang_list_date p')
        self.item = DdBookItem()
        # 分析页面，获取不同的条件url
        for p in p_lis:
            a_lis = p.css('span.date_list a')
            for x in a_lis:
                url = x.css('a::attr(href)').extract_first()
                arr = url.split('/')
                res = (arr[len(arr) - 1]).split('-')
                self.item['cate'] = res[1]      # 获取分类信息，存储时方便区分
                self.item['category'] = arr[4]  # #获取分类信息，存储时方便区分
                yield scrapy.Request(url, meta={'item':self.item},callback=self.parse_condition_page)

    def parse_condition_page(self, response):
        '''
        该方法用于从一个特定的条件分类中执行翻页操作
        例如：条件为近30日下有25页数据
        '''
        self.driver.get(response.url)
        wait = WebDriverWait(self.driver,3)
        wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.bang_list_box .bang_list li')))
        maxpage = int(response.xpath('//li[@class="next"]/preceding-sibling::li[1]/a/text()').extract_first())
        for i in range(1,maxpage+1):
            url = response.url[0:-1] + str(i)
            yield scrapy.Request(url, meta={'item':response.meta['item']}, callback=self.parse_book_infos)

    def parse_book_infos(self, response):
        '''
        该方法用于进入到每本图书的图书详情页
        '''
        lis = response.css('.bang_list_box ul.bang_list li')
        for li in lis:
            full_url = li.css('div.pic a::attr(href)').extract_first()
            yield scrapy.Request(full_url, meta={'item':response.meta['item']}, callback=self.get_bookinfos)

    def get_bookinfos(self, response):
        '''
        该方法用于解析页面，获取需要的信息
        '''
        item = response.meta['item']
        item['cate'] = item['cate']
        item['category'] = item['category']
        item['bookname'] = response.css('.sale_box_left h1::attr(title)').extract_first()
        item['author'] = response.css('.messbox_info span#author a:nth-of-type(1)::text').extract_first()
        item['publisher'] = response.css('.messbox_info span[ddt-area="003"] a::text').extract_first()
        publishtime = response.css('.messbox_info span[ddt-area="003"]+span.t1::text').extract_first()
        item['publishtime'] = publishtime.strip()[5:] if publishtime != None else ''
        price = response.css('#dd-price::text').extract()
        item['price'] = price[1].strip()
        transfer = response.css('.messbox_info span#author a:nth-of-type(2)::text').extract_first()
        item['transfer'] = transfer if transfer != None else ''
        yield item

• settings

BOT_NAME = 'dd_book'

SPIDER_MODULES = ['dd_book.spiders']
NEWSPIDER_MODULE = 'dd_book.spiders'
AUTOTHROTTLE_START_DELAY = 3
AUTOTHROTTLE_ENABLED = True
MONGO_URI = 'localhost'
MONGO_DB = 'dangdangbook'

ITEM_PIPELINES = {
    'dd_book.pipelines.MongoPipeline' : 300
}
ROBOTSTXT_OBEY = False

• Pipelines

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo


class DdBookPipeline(object):
    def process_item(self, item, spider):
        return item

class MongoPipeline(object):
    def __init__(self,mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri = crawler.settings.get('MONGO_URI'),
            mongo_db = crawler.settings.get('MONGO_DB')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def process_item(self,item, spider):
        name = 'dangdang_book'
        self.db[name].insert(dict(item))
        return item

    def close_spider(self, spider):
        self.client.close()

• items

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
class DdBookItem(scrapy.Item):
    bookname = scrapy.Field()
    author = scrapy.Field()
    publisher = scrapy.Field()
    publishtime = scrapy.Field()
    rating_list = scrapy.Field()
    price = scrapy.Field()
    transfer = scrapy.Field()
    image = scrapy.Field()
    cate = scrapy.Field()
    category = scrapy.Field()

• mongo数据

  
  
  1.png

2.png

问题：爬取过程中没有报错，但是部分数据丢失，按照条件算起来应该有5000+的数据，但是最后统计只有1590条数据被抓取到

3.png

• 现在不知道原因到底是什么，比较懊恼，哪位看官可以帮忙解答一下？