Day03:Selenium,BeautifulSoup4

选择器
element: 查找一个
elements: 查找多个

by_id
by_class_name
by_name
by_link_text
by_partial_link_text
by_css_selector

 

Selenium剩余部分

1.元素交互操作:

• 点击、清除 click clear

• - ActionChains
  是一个动作链对象，需要把driver驱动传给它。
  动作链对象可以操作一系列设定好的动作行为。

  - iframe的切换
  driver.switch_to.frame('iframeResult')

  - 执行js代码
  execute_script()

• 元素交互操作

  from selenium import webdriver
  from selenium.webdriver import ActionChains
  from selenium.webdriver.common.keys import Keys
  import time
  
  driver=webdriver.Chrome(r'C:\Users\HP\Desktop\chromedriver.exe')
  
  try:
      driver.implicitly_wait(10)
      driver.get('https://www.jd.com/')
      time.sleep(5)
  
      input=driver.find_element_by_id('key')
      input.send_keys('围城')
  
      search=driver.find_element_by_class_name('button')
      search.click()
  
      time.sleep(3)
  
      input2=driver.find_element_by_id('key')
      input2.clear()
  
      time.sleep(1)
  
      input2.send_keys('墨菲定律')
      input2.send_keys(Keys.ENTER)
  
      time.sleep(10)
  
  finally:
      driver.close()

   

• ActionChains: 动作链

• from selenium import webdriver
  from selenium.webdriver import ActionChains
  from selenium.webdriver.common.keys import Keys
  import time
  
  driver=webdriver.Chrome(r'C:\Users\HP\Desktop\chromedriver.exe')
  
  try:
      driver.implicitly_wait(10)
      driver.get('https://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
      time.sleep(5)
  
      driver.switch_to.frame('iframeResult')
      time.sleep(1)
  
      source=driver.find_element_by_id('draggable')
  
      target=driver.find_element_by_id('droppable')
  
      print(source.size)
      print(source.tag_name)
      print(source.text)
      print(source.location)
  
      distance=target.location['x']-source.location['x']
  
      ActionChains(driver).click_and_hold(source).perform()
  
      s=0
      while s<distance:
          ActionChains(driver).move_by_offset(xoffset=2,yoffset=0).perform()
          s+=2
          time.sleep(0.1)
      ActionChains(driver).release().perform()
      time.sleep(10)
  
  finally:
      driver.close()

• 模拟浏览器的前进后退

• import time
  from selenium import webdriver
  
  browser = webdriver.Chrome()
  browser.get('https://www.baidu.com')
  
  browser.get('https://www.taobao.com')
  
  browser.get('http://www.sina.com.cn/')
  
  # 回退
  browser.back()
  
  time.sleep(5)
  
  # 前进
  browser.forward()
  
  time.sleep(3)
  
  browser.close()

   

• 爬取京东商品信息

• import time
  from selenium import webdriver
  from selenium.webdriver.common.keys import Keys
  
  def get_good(driver):
      num=1
      try:
          time.sleep(5)
          js_code='''
              window.scrollTo(0,5000)
          '''
          driver.execute_script(js_code)
          time.sleep(5)
          good_list=driver.find_elements_by_class_name('gl-item')
          for good in good_list:
              good_name=good.find_element_by_css_selector('.p-name em').text
              good_url=good.find_element_by_css_selector('.p-name a').get_attribute('href')
              good_price=good.find_element_by_class_name('p-price').text
              good_commit=good.find_element_by_class_name('p-commit').text
  
              good_content=f'''
              num:{num}
              商品名称：{good_name}
              商品链接：{good_url}
              商品价格：{good_price}
              商品评价：{good_commit}
              \n
              '''
              print(good_content)
              with open('jd.text','a',encoding='utf-8')as f:
                  f.write(good_content)
              num += 1
          print('商品信息写入成功！')
          next_tag=driver.find_elements_by_class_name('pn-next')
          next_tag.click()
          time.sleep(5)
          get_good(driver)
      finally:
          driver.close()
  
  if __name__ == '__main__':
      driver = webdriver.Chrome(r'C:\Users\HP\Desktop\chromedriver.exe')
      try:
          driver.implicitly_wait(10)
          driver.get('http://www.jd.com/')
  
          input_tag=driver.find_element_by_id('key')
          input_tag.send_keys('墨菲定律')
          input_tag.send_keys(Keys.ENTER)
          get_good(driver)
      finally:
          driver.close()

   

• 二 BeautifulSoup4 BS4

• 1.什么BeautifulSoup？
  bs4是一个解析库，可以通过某种(解析器)来帮我们提取想要的数据。

  2.为什么要使用bs4？
  因为它可以通过简洁的语法快速提取用户想要的数据内容。

  3.解析器的分类
  - lxml
  - html.parser

  4.安装与使用
  - 遍历文档树
  - 搜索文档树

• 补充知识点:

• 数据格式:

  json数据:
  {
  "name": "tank"
  }

  XML数据:
  tank

  HTML:
  

  生成器: yield 值（把值放进生成器中）
  def f():

  # return 1
  yield 1
  yield 2
  yield 3

  g = f()
  print(g)

  for line in g:
  print(line)

• bs4安装与使用

• ''''''
  '''
  安装解析器:
  pip3 install lxml
  
  安装解析库:
  pip3 install bs4
  '''
  html_doc = """
  
  
  class="sister">$37
  class="story" id="p">Once upon a time there were three little sisters; and their names were
  "http://example.com/elsie" class="sister" >Elsie,
  "http://example.com/lacie" class="sister" id="link2">Lacie and
  "http://example.com/tillie" class="sister" id="link3">Tillie;
  and they lived at the bottom of a well.
  
  class="story">...
  """
  from bs4 import BeautifulSoup
  
  # python自带的解析库
  # soup = BeautifulSoup(html_doc, 'html.parser')
  
  # 调用bs4得到一个soup对象
  soup = BeautifulSoup(html_doc, 'lxml')
  
  # bs4对象
  print(soup)
  
  # bs4类型
  print(type(soup))
  
  # 美化功能
  html = soup.prettify()
  print(html)

   

• bs4解析库之遍历文档树

• html_doc = """
  
  
  class="sister">$37
  class="story" id="p">Once upon a time there were three little sisters; and their names were
  "http://example.com/elsie" class="sister" >Elsie,
  "http://example.com/lacie" class="sister" id="link2">Lacie and
  "http://example.com/tillie" class="sister" id="link3">Tillie;
  and they lived at the bottom of a well.
  
  class="story">...
  """
  from bs4 import BeautifulSoup
  
  soup = BeautifulSoup(html_doc, 'lxml')
  # print(soup)
  # print(type(soup))
  # 遍历文档树
  # 1、直接使用  *****
  print(soup.html)
  print(type(soup.html))
  print(soup.a)
  print(soup.p)
  
  # 2、获取标签的名称
  print(soup.a.name)
  
  # 3、获取标签的属性   *****
  print(soup.a.attrs)  # 获取a标签中所有的属性
  print(soup.a.attrs['href'])
  
  # 4、获取标签的文本内容  *****
  print(soup.p.text)  # $37
  
  # 5、嵌套选择
  print(soup.html.body.p)
  
  # 6、子节点、子孙节点
  print(soup.p.children)  # 返回迭代器对象
  print(list(soup.p.children))  # [$37]
  
  # 7、父节点、祖先节点
  print(soup.b.parent)
  print(soup.b.parents)
  print(list(soup.b.parents))
  
  # 8、兄弟节点  （sibling: 兄弟姐妹）
  print(soup.a)
  # 获取下一个兄弟节点
  print(soup.a.next_sibling)
  
  # 获取下一个的所有兄弟节点,返回的是一个生成器
  print(soup.a.next_siblings)
  print(list(soup.a.next_siblings))
  
  # 获取上一个兄弟节点
  print(soup.a.previous_sibling)
  # 获取上一个的所有兄弟节点，返回的是一个生成器
  print(list(soup.a.previous_siblings))

   

• bs4之搜索文档树

• find: 找第一个
  find_all: 找所有
  
  标签查找与属性查找:
  name 属性匹配
  
      name 标签名
      attrs 属性查找匹配
      text 文本匹配
  
      标签:
          - 字符串过滤器   
              字符串全局匹配
  
          - 正则过滤器
              re模块匹配
  
          - 列表过滤器
              列表内的数据匹配
  
          - bool过滤器
              True匹配
  
          - 方法过滤器
              用于一些要的属性以及不需要的属性查找。
      属性:
          - class_
          - id

• html_doc = """
  $37
  Once upon a time there were three little sisters; and their names wereElsieLacie andTillieand they lived at the bottom of a well.
  ...
  
  """
  from bs4 import BeautifulSoup
  
  soup = BeautifulSoup(html_doc, 'lxml')
  
  # name 标签名
  # attrs 属性查找匹配
  # text 文本匹配
  # find与find_all搜索文档
  
  '''
  
  字符串过滤器
  '''
  p = soup.find(name='p')
  p_s = soup.find_all(name='p')
  
  print(p)
  print(p_s)
  
  # name + attrs
  p = soup.find(name='p', attrs={"id": "p"})
  print(p)
  
  # name + text
  tag = soup.find(name='title', text="The Dormouse's story")
  print(tag)
  
  # name + attrs + text
  tag = soup.find(name='a', attrs={"class": "sister"}, text="Elsie")
  print(tag)
  
  '''
  - 正则过滤器
  re模块匹配
  '''
  import re
  # name
  # 根据re模块匹配带有a的节点
  a = soup.find(name=re.compile('a'))
  print(a)
  
  a_s = soup.find_all(name=re.compile('a'))
  print(a_s)
  
  
  # attrs
  a = soup.find(attrs={"id": re.compile('link')})
  print(a)
  
  
  # - 列表过滤器
  # 列表内的数据匹配
  print(soup.find(name=['a', 'p', 'html', re.compile('a')]))
  print(soup.find_all(name=['a', 'p', 'html', re.compile('a')]))
  
  
  # - bool过滤器
  # True匹配
  print(soup.find(name=True, attrs={"id": True}))
  
  # - 方法过滤器
  # 用于一些要的属性以及不需要的属性查找。
  
  def have_id_not_class(tag):
      # print(tag.name)
      if tag.name == 'p' and tag.has_attr("id") and not tag.has_attr("class"):
          return tag
  
  # print(soup.find_all(name=函数对象))
  print(soup.find_all(name=have_id_not_class))
  
  
  # 补充知识点:
  # id
  a = soup.find(id='link2')
  print(a)
  
  # class
  p = soup.find(class_='sister')
  print(p)

   

 

转载于:https://www.cnblogs.com/zhoujie333/p/11128756.html