python crawlspider 例子

rules=(
Rule(LinkExtractor(allow=r'/web/site0/tab5240/info\d+.htm'),callback='parse_item'),
Rule(LinkExtractor(allow=r'/web/site0/tab5240/module14430/page\d+.htm'),follow=True),
)
1、##

-- coding: utf-8 --

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import re

class CfSpider(CrawlSpider):
name = 'cf'
allowed_domains = ['bxjg.circ.gov.cn']
start_urls = ['http://bxjg.circ.gov.cn/web/site0/tab5240/Default.htm']

rules = (
    Rule(LinkExtractor(allow=r'/web/site0/tab5240/info\d+\.htm'), callback='parse_item', ),
    Rule(LinkExtractor(allow=r'/web/site0/tab5240/module14430/page\d+\.htm'),follow=True, ),
)

def parse_item(self, response):
    item = {}
    item['title']=response.xpath('//title/text()').extract()
    item['publish_date'] = response.xpath("//meta[@name='docdate']/@content").extract()
    print(item)

2、settings.py

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'
DOWNLOAD_DELAY=1