【Python爬虫】信息组织与提取方法

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

三种信息标记形式的比较

 

 

 代码：

<person>
    <firstname>Tianfirstname>
    <lastname>Songlastname>
    <address>
        <streetAddr>中关村南大街5号streetAddr>
        <city>北京市city>
        <zipcode>100081zipcode>
    address>
    <prof>Computer Systemprof><prof>Securityprof>
person>

 

 代码：

{
    "firstName":"Tian",
    "lastName":"Song",
    "address":{
                "streetAddr":"中关村南大街5号",
                "city":"北京市",
                "zipcode":"100081"
               },
    "prof":["Computer System","Security"]
}

 

 代码：

firstName: Tian
lastName: Song
address: 
    streetAddr: 中关村南大街5号
    city: 北京市
    zipcode: 100081
prof:
-Computer System
-Security

 

 

 

信息提取的一般方法

 

 

 

 

 

 

 

 

 实践：

>>> import requests
>>> r=requests.get("https://www.baidu.com/")
>>> r.encoding='UTF-8'
>>> demo=r.text
>>> from bs4 import BeautifulSoup
>>> soup=BeautifulSoup(demo,"html.parser")
>>> for link in soup.find_all('a'):
    print(link.get('href'))

    
http://news.baidu.com
https://www.hao123.com
http://map.baidu.com
http://v.baidu.com
http://tieba.baidu.com
http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1
//www.baidu.com/more/
http://home.baidu.com
http://ir.baidu.com
http://www.baidu.com/duty/
http://jianyi.baidu.com/
>>> 

基于bs4库的HTML内容查找方法

 

 练习：

>>> soup.find_all('a')
[class="mnav" href="http://news.baidu.com" name="tj_trnews">新闻, class="mnav" href="https://www.hao123.com" name="tj_trhao123">hao123, class="mnav" href="http://map.baidu.com" name="tj_trmap">地图, class="mnav" href="http://v.baidu.com" name="tj_trvideo">视频, class="mnav" href="http://tieba.baidu.com" name="tj_trtieba">贴吧, class="lb" href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1" name="tj_login">登录, class="bri" href="//www.baidu.com/more/" name="tj_briicon" style="display: block;">更多产品, "http://home.baidu.com">关于百度, "http://ir.baidu.com">About Baidu, "http://www.baidu.com/duty/">使用百度前必读, class="cp-feedback" href="http://jianyi.baidu.com/">意见反馈]
>>> soup.find_all(['a','b'])
[class="mnav" href="http://news.baidu.com" name="tj_trnews">新闻, class="mnav" href="https://www.hao123.com" name="tj_trhao123">hao123, class="mnav" href="http://map.baidu.com" name="tj_trmap">地图, class="mnav" href="http://v.baidu.com" name="tj_trvideo">视频, class="mnav" href="http://tieba.baidu.com" name="tj_trtieba">贴吧, class="lb" href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1" name="tj_login">登录, class="bri" href="//www.baidu.com/more/" name="tj_briicon" style="display: block;">更多产品, "http://home.baidu.com">关于百度, "http://ir.baidu.com">About Baidu, "http://www.baidu.com/duty/">使用百度前必读, class="cp-feedback" href="http://jianyi.baidu.com/">意见反馈]
>>> for tag in soup.find_all(True):
    print(tag.name)

    
html
head
meta
meta
meta
link
title
body
div
div
div
div
div
div
img
form
input
input
input
input
input
input
span
input
span
input
div
a
a
a
a
a
noscript
a
script
a
div
div
p
a
a
p
a
a
img
>>> import re
>>> for tag in soup.find_all(re.compile('b')):
    print(tag.name)

    
body
>>> 

 

 练习：

>>> soup.find_all('a','nav')
[]
>>> soup.find_all('a','com')
[]
>>> r.text
'\r\n   
 
 
 
 
 
  
 
       
 
 
 
 新闻 hao123 地图 视频 贴吧   更多产品 
 
 
 
 
 
 关于百度 About Baidu 
 
©2017 Baidu 使用百度前必读  意见反馈 京ICP证030173号   
 
 
 
  \r\n'
>>> soup.find_all(id='wrapper')
[
"wrapper"> 
"head"> 
class="head_wrapper"> 
class="s_form"> 
class="s_form_wrapper"> 
"lg"> "129" hidefocus="true" src="//www.baidu.com/img/bd_logo1.png" width="270"/> 
 
"//www.baidu.com/s" class="fm" id="form" name="f"> "bdorz_come" type="hidden" value="1"/> "ie" type="hidden" value="utf-8"/> "f" type="hidden" value="8"/> "rsv_bp" type="hidden" value="1"/> "rsv_idx" type="hidden" value="1"/> "tn" type="hidden" value="baidu"/>class="bg s_ipt_wr">"off" autofocus="autofocus" class="s_ipt" id="kw" maxlength="255" name="wd" value=""/>class="bg s_btn_wr">"" class="bg s_btn" id="su" type="submit" value="百度一下"/> 
 
 
 
"u1"> class="mnav" href="http://news.baidu.com" name="tj_trnews">新闻 class="mnav" href="https://www.hao123.com" name="tj_trhao123">hao123 class="mnav" href="http://map.baidu.com" name="tj_trmap">地图 class="mnav" href="http://v.baidu.com" name="tj_trvideo">视频 class="mnav" href="http://tieba.baidu.com" name="tj_trtieba">贴吧   class="bri" href="//www.baidu.com/more/" name="tj_briicon" style="display: block;">更多产品 
 
 
 
"ftCon"> 
"ftConw"> 
"lh"> "http://home.baidu.com">关于百度 "http://ir.baidu.com">About Baidu 
 
"cp">©2017 Baidu "http://www.baidu.com/duty/">使用百度前必读  class="cp-feedback" href="http://jianyi.baidu.com/">意见反馈 京ICP证030173号  "//www.baidu.com/img/gs.gif"/> 
 
 
 
]
>>> soup.find_all('link')
["https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/bdorz/baidu.min.css" rel="stylesheet" type="text/css"/>]
>>> import re
>>> soup.find_all(id=re.compile('link'))
[]
>>> 

 

 练习：

>>> soup.find_all('a')
[class="mnav" href="http://news.baidu.com" name="tj_trnews">新闻, class="mnav" href="https://www.hao123.com" name="tj_trhao123">hao123, class="mnav" href="http://map.baidu.com" name="tj_trmap">地图, class="mnav" href="http://v.baidu.com" name="tj_trvideo">视频, class="mnav" href="http://tieba.baidu.com" name="tj_trtieba">贴吧, class="lb" href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1" name="tj_login">登录, class="bri" href="//www.baidu.com/more/" name="tj_briicon" style="display: block;">更多产品, "http://home.baidu.com">关于百度, "http://ir.baidu.com">About Baidu, "http://www.baidu.com/duty/">使用百度前必读, class="cp-feedback" href="http://jianyi.baidu.com/">意见反馈]
>>> soup.find_all('a',recursive=False)
[]
>>> 

 

 练习：

>>> soup


 "text/html;charset=utf-8" http-equiv="content-type"/>"IE=Edge" http-equiv="X-UA-Compatible"/>"always" name="referrer"/>"https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/bdorz/baidu.min.css" rel="stylesheet" type="text/css"/> "#0000cc"> 
"wrapper"> 
"head"> 
class="head_wrapper"> 
class="s_form"> 
class="s_form_wrapper"> 
"lg"> "129" hidefocus="true" src="//www.baidu.com/img/bd_logo1.png" width="270"/> 
 
"//www.baidu.com/s" class="fm" id="form" name="f"> "bdorz_come" type="hidden" value="1"/> "ie" type="hidden" value="utf-8"/> "f" type="hidden" value="8"/> "rsv_bp" type="hidden" value="1"/> "rsv_idx" type="hidden" value="1"/> "tn" type="hidden" value="baidu"/>class="bg s_ipt_wr">"off" autofocus="autofocus" class="s_ipt" id="kw" maxlength="255" name="wd" value=""/>class="bg s_btn_wr">"" class="bg s_btn" id="su" type="submit" value="百度一下"/> 
 
 
 
"u1"> class="mnav" href="http://news.baidu.com" name="tj_trnews">新闻 class="mnav" href="https://www.hao123.com" name="tj_trhao123">hao123 class="mnav" href="http://map.baidu.com" name="tj_trmap">地图 class="mnav" href="http://v.baidu.com" name="tj_trvideo">视频 class="mnav" href="http://tieba.baidu.com" name="tj_trtieba">贴吧   class="bri" href="//www.baidu.com/more/" name="tj_briicon" style="display: block;">更多产品 
 
 
 
"ftCon"> 
"ftConw"> 
"lh"> "http://home.baidu.com">关于百度 "http://ir.baidu.com">About Baidu 
 
"cp">©2017 Baidu "http://www.baidu.com/duty/">使用百度前必读  class="cp-feedback" href="http://jianyi.baidu.com/">意见反馈 京ICP证030173号  "//www.baidu.com/img/gs.gif"/> 
 
 
 
  

>>> soup.find_all(string="百度一下")
[]
>>> import re
>>> soup.find_all(string=re.compile("百度一下"))
['百度一下，你就知道']
>>>