Scrapy爬虫爬取天气数据存储为txt和json等多种格式

一、创建Scrrapy项目

scrapy startproject weather

  

   二、 创建爬虫文件

scrapy genspider wuhanSpider wuhan.tianqi.com

 

   三、SCrapy项目各个文件

   (1) items.py

import scrapy


class WeatherItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    cityDate = scrapy.Field()
    week = scrapy.Field()
    img = scrapy.Field()
    temperature = scrapy.Field()
    weather = scrapy.Field()
    wind = scrapy.Field()

  

 (2)wuhanSpider.py

# -*- coding: utf-8 -*-
import scrapy
from weather.items import WeatherItem

class WuhanspiderSpider(scrapy.Spider):
    name = "wuHanSpider"
    allowed_domains = ["tianqi.com"]
    citys = ['wuhan','shanghai']
    start_urls = []
    for city in citys:
        start_urls.append('http://' + city + '.tianqi.com/')

    def parse(self, response):
        subSelector = response.xpath('//div[@class="tqshow1"]')
        items = []
        for sub in subSelector:
            item = WeatherItem()
            cityDates = ''
            for cityDate in sub.xpath('./h3//text()').extract():
                cityDates += cityDate
            item['cityDate'] = cityDates
            item['week'] = sub.xpath('./p//text()').extract()[0]
            item['img'] = sub.xpath('./ul/li[1]/img/@src').extract()[0]
            temps = ''
            for temp in sub.xpath('./ul/li[2]//text()').extract():
                temps += temp
            item['temperature'] = temps
            item['weather'] = sub.xpath('./ul/li[3]//text()').extract()[0]
            item['wind'] = sub.xpath('./ul/li[4]//text()').extract()[0]
            items.append(item)
        return items

  (3)pipelines.py，处理spider的结果

import time
import os.path
import urllib2
#将获得的数据存储到txt文件
class WeatherPipeline(object):
    def process_item(self, item, spider):
		today = time.strftime('%Y%m%d', time.localtime())
		fileName = today + '.txt'
		with open(fileName,'a') as fp:
			fp.write(item['cityDate'].encode('utf8') + '\t')
			fp.write(item['week'].encode('utf8') + '\t')
			imgName = os.path.basename(item['img'])
			fp.write(imgName + '\t')
			if os.path.exists(imgName):
				pass
			else:
				with open(imgName, 'wb') as fp:
					response = urllib2.urlopen(item['img'])
					fp.write(response.read())	
			fp.write(item['temperature'].encode('utf8') + '\t')
			fp.write(item['weather'].encode('utf8') + '\t')
			fp.write(item['wind'].encode('utf8') + '\n\n')
			time.sleep(1)
		return item

import time
import json
import codecs
#将获得的数据存储到json文件
class WeatherPipeline(object):
    def process_item(self, item, spider):
		today = time.strftime('%Y%m%d', time.localtime())
		fileName = today + '.json'
		with codecs.open(fileName, 'a', encoding='utf8') as fp:
			line = json.dumps(dict(item), ensure_ascii=False) + '\n'
			fp.write(line)
		return item

import MySQLdb
import os.path
#将获得的数据存储到mysql数据库
class WeatherPipeline(object):
    def process_item(self, item, spider):
		cityDate = item['cityDate'].encode('utf8')
		week = item['week'].encode('utf8') 
		img = os.path.basename(item['img'])
		temperature = item['temperature'].encode('utf8')
		weather = item['weather'].encode('utf8')
		wind = item['wind'].encode('utf8')

		conn = MySQLdb.connect(
				host='localhost',
				port=3306,
				user='crawlUSER',
				passwd='crawl123',
				db='scrapyDB',
				charset = 'utf8')
		cur = conn.cursor()
		cur.execute("INSERT INTO weather(cityDate,week,img,temperature,weather,wind) values(%s,%s,%s,%s,%s,%s)", (cityDate,week,img,temperature,weather,wind))
		cur.close()
		conn.commit()
		conn.close()

		return item

  (4)settings.py，决定 由哪个文件来处理获取的数据

BOT_NAME = 'weather'

SPIDER_MODULES = ['weather.spiders']
NEWSPIDER_MODULE = 'weather.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'weather (+http://www.yourdomain.com)'

#### user add
ITEM_PIPELINES = {
'weather.pipelines.WeatherPipeline':1,
'weather.pipelines2json.WeatherPipeline':2,
'weather.pipelines2mysql.WeatherPipeline':3
}

  (5)爬取命令

scrapy crawl wuHanSpider

  (6)结果显示

  1.txt数据

 
   2.json数据

 

  3. 存储到mysql数据库