初识Scrapy

一.基本命令

创建项目 scrapy startproject 项目名字
创建爬虫文件 scrapy genspider 文件名 域名
运行爬虫项目 scrapy crawl name属性
查看可运行的爬虫 scrapy list

二.小试牛刀（scrapy.Spider）

全书网

1. 创建项目

scrapy startproject qsw_spider

2.创建爬虫文件

cd  qsw_spider
scrapy genspider qsw_test www.quanshuwang.com

3.设置字段，，，，，items.py

spider02.png

import scrapy
class QswSpiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    author = scrapy.Field()
    context = scrapy.Field()
    link_url = scrapy.Field()

4.编写爬虫文件，，，qsw_test.py

spider01.png

# -*- coding: utf-8 -*-
import scrapy
from ..items import QswSpiderItem

class QswTestSpider(scrapy.Spider):
    name = 'qsw_test'
    # allowed_domains = ['www.quanshuwang.com']
    start_urls = ['http://www.quanshuwang.com/list/1_1.html']
    def parse(self, response):
        '''
        解析网页
        :param response:
        :return:
        '''
        link_url_list = response.xpath(r'//ul[@class="seeWell cf"]/li/a/@href').extract()
        title_list = response.xpath(r'//ul[@class="seeWell cf"]/li/span/a[@target="_blank"]/text()').extract()
        author_list = response.xpath(r'//ul[@class="seeWell cf"]/li/span/a[2]/text()').extract()
        context_list = response.xpath(r'//ul[@class="seeWell cf"]/li/span/em/text()').extract()

        for title,author,context,link_url in zip(title_list,author_list,context_list,link_url_list):
            items = QswSpiderItem()
            items['title']= title
            items['author']= author
            items['context']= context
            items['link_url']= link_url
            yield items

        next_url = response.xpath('//div[@class="pages"]/div[@class="pagelink"]/a[@class="next"]/@href').extract()
        if next_url:
            yield scrapy.Request(next_url[0], callback=self.parse)

5.设置管道，，，pipelines.py

spider03.png

class QswSpidertestPipeline(object):
    def __init__(self):
        self.fa = open('result.json','w+',encoding='utf-8')
    def process_item(self, item, spider):
        self.fa.write(str(item)+'\r\n')
        return item
    def close_spider(self,spider):
        self.fa.close()

6. 激活 管道，，，，settings.py

ITEM_PIPELINES = {
   'qsw_spider.pipelines.QswSpidertestPipeline': 300,
}

7. 运行爬虫

scrapy crawl  qsw_test

三.另一个爬虫（CrawlSpider）

,需要设置UA，打开settings.py

DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
   'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0',
}

1. 创建文件

   
   
   spider01.png

cd/d D:\Python_scrapy_test\spider_qsw\qsw_spider>
#创建CrawlSpider爬虫文件
scrapy genspider -t crawl js jianshu.com

2.设置需要的字段 items.py

spider04.png

class JianshuItem(scrapy.Item):
    title = scrapy.Field()
    url = scrapy.Field()
    author = scrapy.Field()
    pub_time = scrapy.Field()
    read_count = scrapy.Field()
    word_count = scrapy.Field()

3. 编写爬虫文件 js.py

   
   
   spider02.png

spider03.png

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class JsSpider(CrawlSpider):
    name = 'js'
    # allowed_domains = ['jianshu.com']
    start_urls = ['https://www.jianshu.com/']

    rules = (
        Rule(LinkExtractor(allow=r'https://www.jianshu.com/p/.*?'), callback='parse_item', follow=True),  #后面的是uid，匹配uid即可
    )

    def parse_item(self, response):
        item = {}
        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
        #item['name'] = response.xpath('//div[@id="name"]').get()
        #item['description'] = response.xpath('//div[@id="description"]').get()
        item['title'] = response.xpath(r'//h1[@class="_1RuRku"]/text()').extract_first()
        item['author'] = response.xpath(r'//span[@class="_22gUMi"]/text()').extract_first()
        item['pub_time'] = response.xpath(r'//time/text()').extract_first()
        item['url'] = response.url
        count = response.xpath(r'//div[@class="s-dsoj"]/span[2]/text()').extract_first()
        if '字数' in count:
            item['word_count'] = count.split(' ')[-1]
            item['read_count'] = response.xpath(r'//div[@class="s-dsoj"]/span[3]/text()').extract_first().split(' ')[-1]
        elif '阅读' in count:
            item['read_count'] = count.split(' ')[-1]
            item['word_count'] = response.xpath(r'//div[@class="s-dsoj"]/span[1]/text()').extract_first().split(' ')[-1]
        yield item

4. 设置管道，，，pipelines.py

   
   
   spider05.png

class JianShuPipeline(object):
    def __init__(self):
        self.f = open('result_jianshu.json','w+',encoding='utf-8')
    def process_item(self, item, spider):
        self.f.write(str(item)+'\r\n')
        return item
    def close_spider(self,spider):
        self.f.close()

5.激活管道 settings.py

spider06.png

ITEM_PIPELINES = {
   'qsw_spider.pipelines.JianShuPipeline': 301,
}

6. 运行爬虫

scrapy crawl  js

7. 效果图

   
   
   spider07.png