python爬虫
- 用requests请求列表页
- selenium请求详情页
- lxml解析网页
- 保存成csv
抓取范围：（按时间排序）前100页，每页16个，共计1600个众筹项目
具体看代码
'''
2020/3/11
京东众筹
https://z.jd.com/bigger/search.html
------------------------------
项目名称、支持者数量、关注数量、剩余天数、话题数、类别
------------------------------
'''
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import requests
import csv
import time
from itertools import count
from lxml import etree
import xlwt
import os
from urllib.parse import urlsplit
import sys
import datetime
import re
class LoopOver(Exception):
def __init__(self, *args, **kwargs):
pass
class Spider:
def __init__(self):
self.path = '.'
self.csvfilenamegbk = 'datas-gbk.csv'
self.csvfilename = 'datas.csv'
self.retry_time = 1
self.host = 'https://z.jd.com'
self.listurl = 'https://z.jd.com/bigger/search.html'
self.user_input_sales = 100
self.browser = webdriver.Chrome()
self.wait = WebDriverWait(self.browser, 20)
def run(self):
strat = time.time()
for text in self.get_list(self.listurl):
for page_url in self.parse_list(text):
item = self.parse_page(self.get_page(page_url))
self.save_data(item)
end = time.time()
self.runtime = end - strat
def get_list(self, url):
for index in range(1, 101):
data = {
'productEnd': '-28',
'sort': 'zxsx'
}
print('当前在第{}页'.format(index))
data['page'] = index
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36',
}
response = requests.post(url, data=data, headers=headers)
yield response.text
def parse_list(self, text):
html = etree.HTML(text)
urls = html.xpath('//ul[@class="infos clearfix"]/li/a/@href')
urls = [self.host + i for i in urls]
if len(urls) == 0:
self.save_html(text)
raise LoopOver
for url in urls:
yield url
def get_page(self, url):
self.browser.get(url)
self.wait.until(EC.presence_of_element_located(
(By.XPATH, '//*[@id="topicBtn"]/span')))
time.sleep(3)
return self.browser.page_source
def parse_page(self, text):
# 项目名称、支持者数量、关注数量、剩余天数、话题数、类别
html = etree.HTML(text)
title = html.xpath('//h1[@class="p-title"]/text()')[0].strip()
typee = html.xpath('//title/text()')[0].strip().split('-')[-2].replace('众筹', '')
try:
p_num = re.findall(
r"\d+\.?\d*", html.xpath('//p[@class="p-progress"]/span[@class="fr"]/text()')[0].strip())[0]
except IndexError:
p_num = '0'
like_num = re.findall(
r"\d+\.?\d*", html.xpath('//*[@id="focusCount"]/text()')[0].strip())[0]
try:
l_day = html.xpath(
'//*[@id="projectMessage"]/span[last()]/text()')[0].strip()
except IndexError:
l_day = 0
try:
talk_num = html.xpath('//*[@id="topicBtn"]/span/text()')[0].strip()
except IndexError:
talk_num = 0
return [title, typee, p_num, like_num, l_day, talk_num]
def save_data(self, item):
'''
保存文件
'''
print('>>>', item)
with open('{}/{}'.format(self.path, self.csvfilename), 'a', encoding='utf_8', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(item)
def save_html(self, text):
with open('test.html', 'w', encoding='utf-8') as f:
f.write(text)
@property
def time(self):
return '总共用时：{}秒'.format(self.runtime)
if __name__ == '__main__':
spider = Spider()
spider.run()
print(spider.time) # 运行总时间
