scrapy框架学习-爬取腾讯社招信息-item字段和管道文件

item

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TenxunItem(scrapy.Item):
    # define the fields for your item here like:
    job_name = scrapy.Field()

    job_link = scrapy.Field()

    job_type = scrapy.Field()

    job_people_num = scrapy.Field()

    job_site = scrapy.Field()

    publish_time = scrapy.Field()

pipelines

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


import json


class TenxunPipeline(object):
    def __init__(self):
        self.f = open("tencent.json", "w")

    def process_item(self, item, spider):
        content = json.dumps(dict(item),
                             ensure_ascii=False) + ",\n"                                                             "\n"
        self.f.write(content)
        return item

    def close_spider(self, spider):
        self.f.close()