scrapy框架学习-爬取腾讯社招信息-tencent.py

功能：设置起始URL和爬取范围，设置要提取的数据路径，返回item或下一个URL地址

# -*- coding: utf-8 -*-
import scrapy
from Tenxun.items import TenxunItem


class TencentSpider(scrapy.Spider):
    name = 'tencent'
    allowed_domains = ['tencent.com']
    start_urls = ['http://hr.tencent.com/position.php?&start=0']

    def parse(self, response):

        nodelist = response.xpath("//tr[@class='even'] | //tr[@class='odd']")

        for node in nodelist:
            item = TenxunItem()

            item['job_name'] = node.xpath("./td[1]/a/text()").extract()[0]
            item['job_link'] = node.xpath("./td[1]/a/@href").extract()[0]
            if len(node.xpath("./td[2]/text()")):
                item['job_type'] = node.xpath("./td[2]/text()").extract()[0]
            else:
                item['job_type'] = "NULL"
            item['job_people_num'] = node.xpath("./td[3]/text()").extract()[0]
            item['job_site'] = node.xpath("./td[4]/text()").extract()[0]
            item['publish_time'] = node.xpath("./td[5]/text()").extract()[0]

            yield item

        if len(response.xpath("//a[@class='noactive' and @id='next']")) == 0:
            url = response.xpath("//a[@id='next']/@href").extract()[0]
            yield scrapy.Request("http://hr.tencent.com/" + url, callback=self.parse)