关于scrapy爬取51job网以及智联招聘信息存储文件的设置

  通过这两个文件,,可以存储数据(但是注意在爬虫文件中也在写相应的代码

具体参考51job网和智联招聘两个文件)

1.先设置items文件

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class JobspiderItem(scrapy.Item):
    # define the fields for your item here like:
    job_name = scrapy.Field()
    fan_kui_lv = scrapy.Field()
    job_company_name = scrapy.Field()
    job_salary = scrapy.Field()
    job_place = scrapy.Field()
    job_type = scrapy.Field()
    job_time = scrapy.Field()

2.设置管道文件

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

#pipeline:俗称管道,用于接收爬虫返回的item数据

class JobspiderPipeline(object):
    def process_item(self, item, spider):
        return item


class TocsvPipeline(object):

    def process_item(self, item, spider):
        with open("job.csv", "a",encoding="gb18030") as f:
            job_name = item['job_name']
            fan_kui_lv = item['fan_kui_lv']
            job_company_name = item['job_company_name']
            job_salary = item['job_salary']
            job_place = item['job_place']
            job_type = item['job_type']
            job_time = item['job_time']

            job_info = [job_name, fan_kui_lv,
                        job_company_name,
                        job_salary, job_place,
                        job_type, job_time,'\n']
            f.write(",".join(job_info))
        #把item传递给下一个pipeline
        return item