利用Python爬虫爬取京东（小规模）
一.开发环境
本文运行环境为Windows10+Python3.7
使用的第三方库有selenium（操作浏览器）+pymysql（数据库）+bs4（解析）+chrome+ChromeDriver
二.先决条件
利用京东的搜索结果，然后把结果保存起来
那么接下来就是找到京东搜索的相关url。因为可见即可爬。
base_url = "https://search.jd.com/Search?"
params = {
"keyword": "android",
"enc": "utf-8",
"qrst": 1,
"page": offset * 2 + 1,
"wq": "android"
}
browser.get(base_url + urlencode(params))
上面的一段代码并不完整，这个只是示例如何构建一个模拟浏览器搜索的操作
三.完整代码
所以到这里，所有的思路就是：
1）先找到要爬取的界面
2）分析界面dom，找到需要的信息
3）解析
4）保存到数据库
第一步，先创建数据库
def init_database():
create_table_sql = "CREATE TABLE IF NOT EXISTS jd_android (id INT NOT NULL, name VARCHAR(255) NOT NULL)"
try:
db = pymysql.connect(host="localhost", user="root", password="root", port=3306, db="spiders")
cursor = db.cursor()
cursor.execute(create_table_sql)
finally:
if db:
db.close()
第二步，开始模拟浏览器操作
def spider_by_chrome():
browser = webdriver.Chrome()
db = pymysql.connect(db="spiders", user="root", password="root", port=3306)
for index in range(10):
get_source_page(browser, index, db)
browser.close()
注意这里，数据库操作没有做相关处理（诸如单例），在小规模不会出问题，当数据量变大时可能会出现读写瓶颈。这里在未来会进行优化。
第三步，解析并保存到数据库
def get_source_page(browser, offset, db):
'''
:param browser: the browser we use to spider
:param offset: the page to search
:param db: database to store data
:return: the final html page
'''
params = {
"keyword": "android",
"enc": "utf-8",
"qrst": 1,
"page": offset * 2 + 1,
"wq": "android"
}
browser.get(base_url + urlencode(params))
# input = browser.find_element_by_id("key")
# input.send_keys("android")
# input.send_keys(Keys.ENTER)
wait = WebDriverWait(browser, 10)
wait.until(EC.presence_of_element_located((By.ID, "J_searchWrap")))
html = BeautifulSoup(browser.page_source, "lxml")
for item in html.find_all("li", class_="gl-item"):
book_name = item.select("em")[1]
print(book_name.text)
global book_id
cursor = db.cursor()
cursor.execute("INSERT INTO jd_android (id, name) values (%s, %s)", (book_id, book_name.text))
book_id = book_id + 1
# item_xml = BeautifulSoup(item, "lxml")
# print(item_xml.find_all("em"))
db.commit()#保存
同样的，数据库操作都十分简陋，有很大的优化空间。错误回滚等操作都没有处理。
综上，是基本的爬虫操作。
四.完整代码
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from urllib.parse import urlencode
import pymysql
from bs4 import BeautifulSoup
base_url = "https://search.jd.com/Search?"
book_id = 1
def get_source_page(browser, offset, db):
'''
:param browser: the browser we use to spider
:param offset: the page to search
:param db: database to store data
:return: the final html page
'''
params = {
"keyword": "android",
"enc": "utf-8",
"qrst": 1,
"page": offset * 2 + 1,
"wq": "android"
}
browser.get(base_url + urlencode(params))
# input = browser.find_element_by_id("key")
# input.send_keys("android")
# input.send_keys(Keys.ENTER)
wait = WebDriverWait(browser, 10)
wait.until(EC.presence_of_element_located((By.ID, "J_searchWrap")))
html = BeautifulSoup(browser.page_source, "lxml")
for item in html.find_all("li", class_="gl-item"):
book_name = item.select("em")[1]
print(book_name.text)
global book_id
cursor = db.cursor()
cursor.execute("INSERT INTO jd_android (id, name) values (%s, %s)", (book_id, book_name.text))
book_id = book_id + 1
# item_xml = BeautifulSoup(item, "lxml")
# print(item_xml.find_all("em"))
db.commit()
def spider_by_chrome():
browser = webdriver.Chrome()
db = pymysql.connect(db="spiders", user="root", password="root", port=3306)
for index in range(10):
get_source_page(browser, index, db)
browser.close()
def init_database():
create_table_sql = "CREATE TABLE IF NOT EXISTS jd_android (id INT NOT NULL, name VARCHAR(255) NOT NULL)"
try:
db = pymysql.connect(user="root", password="root", port=3306, db="spiders")
cursor = db.cursor()
cursor.execute(create_table_sql)
finally:
if db:
db.close()
init_database()
spider_by_chrome()