使用scrapy框架写的爬虫项目代码

源代码文件：

import scrapy
import json
from..items import Db250Item
class W666Spider(scrapy.Spider):
    name = 'w666'
    allowed_domains = ['movie.douban.com']
    start_urls = ['http://movie.douban.com/top250']
    page_num=0

    def parse(self, response):
        node_list=response.xpath('//div[@class="info"]')
        if node_list:
            for i  in node_list:
                movies_name=i.xpath('.//div/a/span/text()').get()
                director = i.xpath('./div/p/text()').get().strip()
                score = i .xpath('//span[@class="rating_num"]/text()').get()
                item=Db250Item()
                item["movies_name"]=movies_name
                item["director"]=director
                item["score"]=score
                yield item

            self.page_num +=1
            new_url='https://movie.douban.com/top250?start={}&filter='.format(self.page_num*25)

            yield scrapy.Request(new_url,callback=self.parse)
        else:
            return

 items.py:

import scrapy


class Db250Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    movies_name=scrapy.Field()
    director=scrapy.Field()
    score=scrapy.Field()

 管道pipelines.py:

​​​​​​

import json

class Db250Pipeline:
    def open_spider(self,spider):
        self.f = open('wxin.txt','w',encoding='utf-8')
    def process_item(self, item, spider):
        json_str=json.dumps(dict(item),ensure_ascii=False)+'\n'
        self.f.write(json_str)
        return item
    def close_spider(self):
        self.f.close()

 settings.py需要修改以及激活的内容:

ROBOTSTXT_OBEY =False #这个默认的是Ture，必须该成Flase,要不然就爬取不到数据

DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agent':' Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36'
}    #这个必须要去network里找到user-agent，然后添加进去，如果不加大概率是爬不到数据的

ITEM_PIPELINES = {
   'db250.pipelines.Db250Pipeline': 300,
}   #这个储存管道设定后必须激活的,才能进行持久化保存