利用python爬取博客信息并保存在Excel中
只放代码
import re,json,requests
from lxml import etree
import xlwt
# 分页
# 找页码变化的规律
for i in range(1,6):
base_url = 'https://blog.csdn.net/qq_42374697/article/list/%s'%(i)
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36',
}
response = requests.get(base_url,headers=headers)
html = etree.HTML(response.text)
div_list = html.xpath('//div[@class="article-item-box csdn-tracking-statistics"]')
# print(dd_list)
info_list = []
for div in div_list:
type_ = div.xpath('./h4/a/span/text()')[0]
title = div.xpath('./h4/a/text()')[1].strip()
date = div.xpath('.//span[@class="date"]/text()')[0].strip()
read_num = div.xpath('.//span[@class="read-num"]/text()')[0]
item = {}
item['文章类型'] = type_
item['标题'] = title
item['日期'] = date
item['阅读量'] = read_num
info_list.append(item)
filename = 'C:/爬取数据.xls'
import xlwt
import os
import xlrd
from xlutils.copy import copy
class ExcelUtils(object):
#工具类的方法：不适用外部变量
#静态方法：直接可以用类名.方法名来调用
# @staticmethod
#类变量：
#实例变量
#类方法
@staticmethod
def write_to_excel(filename,sheetname,word_list):
'''
写入excel
:param filename: 文件名
:param sheetname: 表单名
:param word_list: [item,item,{}]
:return:
'''
try:
# 创建workbook
workbook = xlwt.Workbook(encoding='utf-8')
# 给工作表添加sheet表单
sheet = workbook.add_sheet(sheetname)
# 设置表头
head = []
for i in word_list[0].keys():
head.append(i)
# print(head)
# 将表头写入excel
for i in range(len(head)):
sheet.write(0, i, head[i])
# 写内容
i = 1
for item in word_list:
for j in range(len(head)):
sheet.write(i, j, item[head[j]])
i += 1
# 保存
workbook.save(filename)
print('写入excle成功！')
except Exception as e:
print(e)
print('写入失败！')
@staticmethod
def write_to_excel_append(filename,infos):
'''
追加excel的方法
:param filename: 文件名
:param infos: 【item，item】
:return:
'''
#打开excle文件
work_book = xlrd.open_workbook(filename)
#获取工作表中的所有sheet表单名称
sheets = work_book.sheet_names()
#获取第一个表单
work_sheet = work_book.sheet_by_name(sheets[0])
#获取已经写入的行数
old_rows = work_sheet.nrows
#获取表头的所有字段
keys = work_sheet.row_values(0)
print('===================',keys)
#将xlrd对象转化成xlwt，为了写入
new_work_book = copy(work_book)
#获取表单来添加数据
new_sheet = new_work_book.get_sheet(0)
i = old_rows
for item in infos:
for j in range(len(keys)):
new_sheet.write(i, j, item[keys[j]])
i += 1
new_work_book.save(filename)
print('追加成功！')
if os.path.exists(filename):
#如果文件存在就追加
ExcelUtils.write_to_excel_append(filename,info_list)
else:
#不存在就新建
ExcelUtils.write_to_excel(filename,'sheet',info_list)
多线程
from selenium import webdriver
from lxml import etree
import threading
import os
from queue import Queue
import xlwt
import xlrd
from xlutils.copy import copy
class ExcelUtils(object):
#工具类的方法：不适用外部变量
#静态方法：直接可以用类名.方法名来调用
# @staticmethod
#类变量：
#实例变量
#类方法
@staticmethod
def write_to_excel(filename,sheetname,word_list):
'''
写入excel
:param filename: 文件名
:param sheetname: 表单名
:param word_list: [item,item,{}]
:return:
'''
try:
# 创建workbook
workbook = xlwt.Workbook(encoding='utf-8')
# 给工作表添加sheet表单
sheet = workbook.add_sheet(sheetname)
# 设置表头
head = []
for i in word_list[0].keys():
head.append(i)
# print(head)
# 将表头写入excel
for i in range(len(head)):
sheet.write(0, i, head[i])
# 写内容
i = 1
for item in word_list:
for j in range(len(head)):
sheet.write(i, j, item[head[j]])
i += 1
# 保存
workbook.save(filename)
print('写入excle成功！')
except Exception as e:
print(e)
print('写入失败！')
@staticmethod
def write_to_excel_append(filename,infos):
'''
追加excel的方法
:param filename: 文件名
:param infos: 【item，item】
:return:
'''
#打开excle文件
work_book = xlrd.open_workbook(filename)
#获取工作表中的所有sheet表单名称
sheets = work_book.sheet_names()
#获取第一个表单
work_sheet = work_book.sheet_by_name(sheets[0])
#获取已经写入的行数
old_rows = work_sheet.nrows
#获取表头的所有字段
keys = work_sheet.row_values(0)
print('===================',keys)
#将xlrd对象转化成xlwt，为了写入
new_work_book = copy(work_book)
#获取表单来添加数据
new_sheet = new_work_book.get_sheet(0)
i = old_rows
for item in infos:
for j in range(len(keys)):
new_sheet.write(i, j, item[keys[j]])
i += 1
new_work_book.save(filename)
print('追加成功！')
class SPIDER(threading.Thread):
def __init__(self,url,queue_page,name,filename):
super().__init__() #调用父类的init方法
self.url=url
self.queue_page=queue_page
self.name=name
self.filename=filename
def run(self):
while True:
# 一定要先做跳出循环的条件准备
if self.queue_page.empty(): #如果队列中页码为空了，就跳出循环
break
#取页码
page = self.queue_page.get() #出队操作
#请求+解析
self.parse_page(page)
def parse_page(self,page):
driver = webdriver.PhantomJS() #创建一个驱动
driver.get(self.url.format(page))
html = etree.HTML(driver.page_source)
info_list=[]
ii_list = html.xpath('//div[@class="article-item-box csdn-tracking-statistics"]')
for ii in ii_list:
try:
##提取
type_ = ii.xpath('./h4/a/span/text()')[0]
title = ii.xpath('./h4/a/text()')[1].strip()
date = ii.xpath('.//span[@class="date"]/text()')[0].strip()
read_num = ii.xpath('.//span[@class="read-num"]/text()')[0]
#获取到的信息放入字典
item = {}
item['文章类型'] = type_
item['标题'] = title
item['日期'] = date
item['阅读量'] = read_num
info_list.append(item)
except Exception:
pass
# 保存
if os.path.exists(self.filename):
#如果文件存在就追加
ExcelUtils.write_to_excel_append(self.filename,info_list)
else:
#不存在就新建
ExcelUtils.write_to_excel(self.filename,'sheet1',info_list)
if __name__ == '__main__':
#基础url
base_url = 'https://blog.csdn.net/qq_42374697/article/list/{}'
#第一步：创建任务队列并初始化
queue_page = Queue()
# 找页码变化的规律
for i in range(1,3):
queue_page.put(i) #入队操作 就是 0,1,2,3,4，...，9 页码
name_list = ['a','b','c','d','e']
thread_list = []
for i in name_list:
#创建线程
#queue_page:将创建好的队列传进去
#传线程名称
t = SPIDER(base_url,queue_page,i,'C:/爬取数据.xls')
t.start()
thread_list.append(t)
for t in thread_list:
t.join()
