python selenium4 使用无界面浏览器 爬虫 并存储mysql数据库

浏览器驱动 

需要查看对应浏览器版本进行下载

selenium · PyPIhttps://pypi.org/project/selenium/浏览器设置中查看当前版本

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
# select 选择框需要引入 select 类
from selenium.webdriver.support.select import Select
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import time
import lxml
import pymysql
db = pymysql.connect(
        host='127.0.0.1', 
        user='', # 用户名
        password='', #用户密码
        database=''# 数据库名称
        )

# 设置无界面打开浏览器 后台运行   无浏览器模式  注释则是浏览器模式
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

s = Service("下载的浏览器驱动文件位置")
driver = webdriver.Chrome(service=s, options=chrome_options)
#  打开网站
driver.get("私聊获取示例链接")
# 姓
xing = '李'
# 名
name = '王'
# 输入姓氏内容 到姓氏文本框内
driver.find_element(By.ID,"ctl00_ContentPlaceHolder1_tbXing").send_keys(xing)
time.sleep(1)
# 输入名字内容 到对应文本框中
driver.find_element(By.ID,"ctl00_ContentPlaceHolder1_tbMing").send_keys(name)
time.sleep(1)
# 点击选择 性别
# driver.find_element(By.ID,'ctl00_ContentPlaceHolder1_rbFemale').click()
time.sleep(1)
# 下拉选择  日期类型
select_type = driver.find_element(By.ID,'ctl00_ContentPlaceHolder1_ddlType')
# 创建一个Select对象
select_type_object = Select(select_type)
# 根据 value 选中内容
select_type_object.select_by_value('3')
time.sleep(1)
# 根据可以看见的文本内容 选中
select_type_object.select_by_visible_text("农历")
time.sleep(1)
# 根据选项序号 选中
select_type_object.select_by_index(0)
time.sleep(1)

# 下拉选择  年份
select_year = driver.find_element(By.ID,"ctl00_ContentPlaceHolder1_ddlYear")
select_year_object = Select(select_year)
select_year_object.select_by_value("1999")
time.sleep(1)
# 下拉选择  月份
select_month = driver.find_element(By.ID,"ctl00_ContentPlaceHolder1_ddlMonth")
select_month_object = Select(select_month)
select_month_object.select_by_value("5")
time.sleep(1)
# 下拉选择  日期
select_day = driver.find_element(By.ID,"ctl00_ContentPlaceHolder1_ddlDay")
select_day_object = Select(select_day)
select_day_object.select_by_value("25")
time.sleep(1)


# 提交表单
driver.find_element(By.ID,'ctl00_ContentPlaceHolder1_btSubmit').click()
#
# driver.page_source 获取页面内容 配合 BeautifulSoup 执行爬虫 获取数据
# 解析器 lxml 容错能力强、执行速度快 需要安装c语言库
time.sleep(2)
soup = BeautifulSoup(driver.page_source,"lxml")
# 获取对应 模块内容
content = soup.select('#ctl00_ContentPlaceHolder1_plResult > table:nth-child(4)')
str = content[0]

# 使用cursor()方法获取操作游标
cursor = db.cursor()

# SQL 插入语句
sql = "INSERT INTO python(content,`name`) \
         VALUES ('%s','%s')" % \
      (str,xing+name)
try:
        # 执行sql语句
        cursor.execute(sql)
        # 提交到数据库执行
        db.commit()
except:
        # 如果发生错误则回滚
        db.rollback()

# 关闭数据库连接
db.close()
time.sleep(1)
# 返回上一页
driver.back()
time.sleep(1)
# driver.find_element(By.ID,"ctl00_ContentPlaceHolder1_tbMing").send_keys('')


# 停止执行 8秒
time.sleep(8)
# 退出
driver.quit()

mysql 表 创建命令

DROP TABLE IF EXISTS `python`;
CREATE TABLE `python`  (
  `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
  `content` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL,
  `name` varchar(55) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
  PRIMARY KEY (`id`) USING BTREE
) ENGINE = MyISAM AUTO_INCREMENT = 6 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

SET FOREIGN_KEY_CHECKS = 1;