python爬虫xpath，BeautifulSoup和正则用法全

from lxml import etree
from bs4 import BeautifulSoup
import re

html = """

• 时间


• 地点


• 任务

这里是个小标题

1. 1


2. 2


3. 3

• 84


• 104


• 223

这里是H3的内容
百度一下

• test1


• test2

• 1


• 2


• 3


• 4


• 5


• 6


• 7


• 8


• 9


• 10

"""

def title():

#第一种，xpath提取
html_etree = etree.HTML(html)
# print(type(html_etree))  #
# result = etree.tostring(html_etree)  #如果标签不全，tostring()可以补全
# print(result.decode('utf-8'))  #tostring()后的数据类型是bytes，需要decode()转成str
title_xpath1 = html_etree.xpath('/html/head/title/text()')  #需要text()把文字解析出来
print('用xpath绝对路径方法提取title:', title_xpath1)  #xpath返回的是列表
title_xpath2 = html_etree.xpath('//head/title/text()')  #效果一样，/表示绝对路径，//表示相对路径
print('用xpath相对路径方法提取title:', title_xpath2)

#第二种，BeautifulSoup提取
soup = BeautifulSoup(html, 'lxml')
# print(soup)
# print(type(soup))  #
title_soup = soup.select('title')  #soup.select返回的也是列表，需要提取出来在用get_text()拿出文字
# css选择器，标签名不加修饰，类名前加点，id名前加#，可组合查找
# print(title_soup)
# print(type(title_soup))  #list
title_BeautifuleSoup = title_soup[0].get_text()
# title_BeautifuleSoup = soup.title.get_text()
print('用BeautifulSoup方法提取title:', title_BeautifuleSoup)

#第三种，正则表达式提取
re_pattern = re.compile(r'', re.S)  #(.*?)是需要匹配返回的字符串,re.S可换行匹配
# print(type(re_pattern))  #re.compile返回的是数据类型正则表达式：
title_re_compile = re.findall(re_pattern, html)
print('用正则表达式方法提取title:', title_re_compile)
#可以不使用re.compile
title_re = re.findall(r'', html)
print('用正则表达式跳过re.compile提取title:', title_re)

def price():

#第一种，xpath提取
html_etree = etree.HTML(html)
# price_xpath = html_etree.xpath('/html/body/div/@price')
# price_xpath = html_etree.xpath('/html/body/child::*/@price')  #child::*  选取当前节点所有子元素
# price_xpath = html_etree.xpath('/html/body/child::div/@price')  # child::div  子节点定位div标签
# price_xpath = html_etree.xpath('//@price')  #相对路径，且price属性只有一个
# price_xpath = html_etree.xpath("//div[@id='testid']/ancestor::div")  #ancestor::  提取所有父辈div元素
# price_xpath = html_etree.xpath("//div[@id='testid']/ancestor::div/@price")  #父辈定位div元素price属性
price_xpath = html_etree.xpath("//div[@id='testid']/ancestor-or-self::div/@price")  # 父辈及当前节点div元素
print('用xpath方法提取price:', price_xpath)

#第二种，BeautifulSoup提取
soup = BeautifulSoup(html, 'lxml')
price_BeautifulSoup = soup.div.attrs['price']
# price_BeautifulSoup = soup.find('div').attrs['price']
# price_BeautifulSoup = soup.select('div')[0].attrs['price']
print('用BeautifulSoup方法提取price:', price_BeautifulSoup)

#第三种，正则表达式提取
re_pattern = re.compile(r'
', re.S)
price_re = re.findall(re_pattern, html)
print('用正则表达式跳过re.compile提取price:', price_re)

提取第一个div下ul下li的文字

def ul_li():

# 第一种，xpath提取
html_etree = etree.HTML(html)
# ul_li = html_etree.xpath('//div/div[1]/ul/child::*/text()')  #child::节点子元素方法
# ul_li = html_etree.xpath('//div/div[1]/ul/li/text()')
# ul_li = html_etree.xpath("//div[@id='testid']/preceding::div/ul/li/text()")  #preceding::  当前节点标签之前的所有节点，可定点
ul_li = html_etree.xpath("//div[@id='testid']/preceding::li/text()")  #preceding::  可避免重复节点带来的麻烦
print('用xpath方法提取ul标签下的li的内容:', ul_li)

# 第二种，BeautifulSoup提取
soup = BeautifulSoup(html, 'lxml')
# 第一种BeautifulSoup方法
# ul_li = soup.select('ul')[0].select('li')
# ul_li = [i.get_text() for i in ul_li]
#另外一种BeautifulSoup方法
ul_li = soup.div.div.get_text()
ul_li = ul_li.strip()  #删除首尾空格
ul_li = ul_li.split('\n')  #按换行符分割字符串
print('用BeautifulSoup方法提取ul_li:', ul_li)

# 第三种，正则表达式提取
re_pattern = re.compile(r'
.*?
.*?
.*?
(.*?)
.*?
(.*?)
.*?
(.*?)
', re.S)
re_ul_li = re.findall(re_pattern, html)
print('用正则表达式跳过re.compile提取ul_li:', re_ul_li)

def first_id():

# 第一种，xpath提取
html_etree = etree.HTML(html)
first_id = html_etree.xpath('//div/div[2]/@id')
print('用xpath方法提取first_id的内容:', first_id)

# 第二种，BeautifulSoup提取
soup = BeautifulSoup(html, 'lxml')
first_id = soup.select('div')[2].attrs['id']
print('用BeautifulSoup方法提取first_id:', first_id)

# 第三种，正则表达式提取
re_comppile = re.compile(r"
", re.S)
first_id = re.findall(re_comppile, html)
print('用正则表达式跳过re.compile提取first_id:', first_id)

def h2():

# 第一种，xpath提取
html_etree = etree.HTML(html)
h2 = html_etree.xpath('//div/div[2]/h2/text()')
print('用xpath方法提取h2的内容:', h2)

# 第二种，BeautifulSoup提取
soup = BeautifulSoup(html, 'lxml')
# h2 = soup.select('h2')[0].get_text()
h2 = soup.div.h2.get_text()
print('用BeautifulSoup方法提取h2:', h2)

# 第三种，正则表达式提取
re_comppile = re.compile(r'
(.*?)
', re.S)
h2 = re.findall(re_comppile, html)
print('用正则表达式跳过re.compile提取h2:', h2)

def main():
title()
price()
ul_li()
first_id()
h2()

if name == 'main':
main()