在爬虫中获取数据后的各种存储方法
一、文本存储
大体结构是这样的
with open('名字.text','w',encoding='utf-8') as f:
f.write()
f.close()
实例如下
class NovelSpider(scrapy.Spider):
name = 'novel'
allowed_domains = ['tieba.baidu.com']
start_urls = ['https://tieba.baidu.com/p/5815118868?pn=1']
f = open('tieba.txt','a', encoding='utf-8')
def parse(self, response):
# print(response.text)
# print("==========================================================")
# 找到指定的div的标签，该标签内贴吧内容和作者的集合体
# div_list = response.xpath('//div[@class="d_post_content j_d_post_content "]')
div_list = response.xpath('//div[@class="l_post l_post_bright j_l_post clearfix "]')
# print(div_list)
# print("===================================")
# 找到作者
for div in div_list:
# 获取含有louzhubiaoshi_wrap 类名的标签
# 改类名只有楼主才有
author = div.xpath('.//div[@class="louzhubiaoshi_wrap"]').extract()
print("----------------------------------")
print(author)
if len(author) !=0:
# 获取标签内全部文本的方式
# 1.获取最外层标签，遍历内部所有的子标签，获取标签文本
# 2，正则去掉所有标签 <.*?> re.comlile
# 3.text 获取标签的文本 //text（）获取标签及子标签的文本
# 4.使用xpath（string（.））这种方式来获取所有文本并且拼接
# content_list = div.xpath('.//div[@class="d_post_content j_d_post_content "]/text()').extract()
content_list = div.xpath('.//div[@class="d_post_content j_d_post_content "]').xpath('string(.)').extract()[0] + '\n'
# print(content_list)
# print("000000000000000000000000000000000000000000000000000")
# self.f.write(content)
remove = re.compile('\s')
douhao = re.compile(',')
content =[]
for string in content_list:
string = string.sub(remove,'',string)
string = re.sub(douhao,' ',string)
content += string + ','
# print(content)
self.f.write(content)
self.f.write('\n')
#如要将数据存储成json格式的话，具体实现如下
import codecs,os,json
class HongxiutianxiangPipeline(object):
print("========================================")
def __init__(self):
self.file = codecs.open(filename='book.json' ,mode='w+',encoding='utf-8')
self.file.write('"book_list":[')
def process_item(self, item, spider):
res = dict(item)
str = json.dumps(res,ensure_ascii=False)
self.file.write(str)
self.file.write(',\n')
return item
def close_spider(self ,spSEider):
self.file.seek(-1,os.SEEK_END)
self.file.truncate()
self.file.seek(-1,os.SEEK_END)
self.file.truncate()
self.file.write(']')
self.file.close()还有下面的主要是用于数据转码的，防止出现乱码，也是json格式、
# 用来打开指定文件，并且对文件进行转码，防止出现乱码问题
import codecs
import json
class XiaoshuoPipeline(object):
def __init__(self):
# w 写文件
# w+ 读写文件 r 读写文件
# 以上两个的区别，前者如果文件不存在则创建，后者速写文件，如果文件不存在，则抛出异常
self.file = codecs.open(filename='book.json',mode='w+',encoding='utf-8')
# self.file.write("list":"[]")
# 如果想要将数据写入本地或者使用数据库的时候，这个方法需要保留
def process_item(self, item, spider):
# print(item,'==================')
# 将item对象转化成字典对象
res = dict(item)
# print(res,'-------------------')
# 将字典对象转化成字符串， ascii编码是否可用
# 如果直接将字典形式的数据写入到文件当中，会发生错误，转化成字符串写入到文件当中
str = json.dumps(res,ensure_ascii=False)
# print(str,'1111111111111111111111111111111111')
# 将数据写入到文件当中
self.file.write(str)
self.file.write('\n')
def open_spider(self,spider):
print("爬虫开始了")二、sqlite3微型数据库存储
在爬虫中一般都是在pipeline里面设置的，然后再setting里面的ITEM_PIPELINES解注释，添加
ITEM_PIPELINES ={
'Job.pipelines.Jobpipeline':300,
}
项目名+要开启的管道名+在管道里新添加的类或者是说要用到的类，这里还有优先级，数字越小优先级越高
import sqlite3
class JobPipeline(object):
def process_item(self, item, spider):
return item
class JobDBPipeline(object):
def open_spider(self,spider):
self.connect = sqlite3.connect('JobDB')
self.cursor = self.connect.cursor()
self.cursor.execute(
'create table if not exists JobTable(job_name,job_company,job_place,job_time,min_salary,max_salary)')
self.connect.commit()
def process_item(self, item,spider):
self.cursor.execute('insert into JobTable(job_name,job_company,job_place,job_time,min_salary,max_salary) VALUES ("{}","{}","{}","{}","{}","{}")'.format(item['job_name'],item['job_company'],item['job_place'],item['job_time'],item['min_salary'],item['max_salary']))
self.connect.commit()
def close_spider(self,spider):
self.cursor.close()
self.connect.close()三、将数据存储到excel表格，以豆瓣电影存储到excel表格为例
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from fake_useragent import UserAgent
import xlwt,requests
from lxml import etree
class DBmovie(object):
def __init__(self):
self.base_url = 'https://movie.douban.com/top250'
self.headers = UserAgent()
self.workBooke = None
self.sheet = None
self.record = 1
def spider_manage(self):
self.excel_build()
self.get_url_code()
self.workBook.save('电影表.xls')
def excel_build(self):
self.workBook = xlwt.Workbook(encoding='utf-8')
self.sheet = self.workBook.add_sheet('电影排行榜')
self.sheet.write(0,0,'电影排名')
self.sheet.write(0,1,'电影名称')
self.sheet.write(0,2,'演员与导演')
self.sheet.write(0,3,'电影评分')
self.sheet.write(0,4,'电影影评')
# self.sheet.write(0,5,'电影内容')
def get_url_code(self, url =''):
headers = {
'User-Agent': self.headers.random
}
full_url = self.base_url +url
# print(full_url)
response = requests.get(full_url, headers=headers).text
# print(response)
code = etree.HTML(response)
print(code)
item_div = code.xpath('//div[@class="item"]')
# print(item_div)
for tag in item_div:
movie_name = tag.xpath('.//div[@class="hd"]/a/span/text()')
# print(movie_name)
name=''
for movie in movie_name:
name += movie
# print(name)
movie_rank = tag.xpath('div/em[@class=""]/text()')[0]
# print(movie_rank)
movie_author = tag.xpath('.//div[@class="bd"]/p/text()')[0]
movie_author = movie_author.strip('\n').replace(' ','')
# print(movie_author)
movie_grade = tag.xpath('.//span[@class="rating_num"]/text()')[0]
# print(movie_grade)
movie_comment = tag.xpath('.//div[@class="star"]/span[last()]/text()')[0]
movie_comment = movie_comment[0:-3]
# print(movie_comment)
# movie_content = tag.xpath('.//p[@class="quote"]/span/text()')[0]
# movie_content = movie_content.strip('\n').replace('。','')
# print(movie_content)
# print(movie_content)
self.sheet.write(self.record,0,movie_rank)
self.sheet.write(self.record,1,name)
self.sheet.write(self.record,2,movie_author)
self.sheet.write(self.record,3,movie_grade)
self.sheet.write(self.record,4,movie_comment)
# self.sheet.write(self.record,5,movie_content)
self.record += 1
self.get_next_page(code)
def get_next_page(self,code):
next_url = code.xpath('//span[@class="next"]/a/@href')
# print(next_url)
if len(next_url) == 0:
print("已经是最后一页了")
return
self.get_url_code(next_url[0])
movie = DBmovie()
movie.spider_manage()
四、mysql数据库存储
class DianyingtiantangPipeline(object):
def __init__(self):
self.connect = pymysql.connect(host='localhost',user='root',password='123456',db='Movie',port=3305)
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
self.cursor.execute('insert into movieTable (name ,href) VALUES ("{}","{}")'.format(item['name'],item['href']))
self.connect.commit()
return item
def close_spider(self ,spider):
self.cursor.close()
self.connect.close()
五、在爬虫终端命令形式存成表格形式
scrapy crawl 爬虫名 -o 爬虫名.csv
六、终端命令将数据的解码方式改成utf-8,防止中文乱码
scrapy crawl 爬虫名 -o 爬虫名.json -s FEED_EXPORT_ENCORDING=UTF-8
七、将数据中的图片存到本地文件夹
在setting里面添加
IMAGES_STORE = '***'（文件名、默认是在当前目录的父级，在这里你也可选择已存在文件夹的路径）
IMAGES_URLS_FIELD = '***' （一般填写图片的下载路径，在爬虫文件中要特别注意，要传的路径最好带【】就是将得到的数据转储到items里面的时候）
将文本文件存到本地（例如小说之类）
FILES_STORES = '****' (性质和存图片的一样)
FILES_URLS_FIELD = '***'(同上)