BeautifulSoup爬取数据演示
BeautifulSoup爬取数据推荐操作简单，但是相比于xpath和正则匹配数据太慢，不建议使用
案例一： 双色球历史数据爬虫，爬取期数，开奖时间，红色球，蓝色球，一等奖，二等奖等信息
from urllib import request,parse
from urllib import error
import chardet
from lxml import etree
import csv
import re
from bs4 import BeautifulSoup
def shuangseqiu(url):
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
req = request.Request(url, headers=headers)
try:
response = request.urlopen(req)
resHtml = response.read()
resHtml = resHtml.decode("utf-8", 'ignore')
soup = BeautifulSoup(resHtml, 'lxml')
for i in soup.select('tr')[2:]:
# 期号的提取
qihao = i.select('td')[0].get_text()
print('------------------------------')
# 开奖日期的提取
date = i.select('td')[1].get_text()
# 总投注额（元）的提取
totalprice = i.select('td')[9].get_text()
# 一等奖（注数）的提取
OneNumber = i.select('td')[10].get_text()
# 一等奖（奖金（元））的提取
OnePrice = i.select('td')[11].get_text()
# 二等奖（注数）的提取
TwoNumber = i.select('td')[12].get_text()
# 二等奖（奖金（元））的提取
TwoPrice = i.select('td')[13].get_text()
# 奖池滚存（元）的提取
jackpot = i.select('td')[14].get_text()
# 红色球
red_qiu = i.select('td')[2].get_text() + i.select('td')[3].get_text() + i.select('td')[4].get_text() + i.select('td')[5].get_text() + i.select('td')[6].get_text() + i.select('td')[7].get_text()
# 蓝色球
blueqiu = i.select('td')[8].get_text()
except error.URLError as e:
print(e)
if __name__ == "__main__":
proxy = {"http": "118.31.220.3:8080"}
proxy_support = request.ProxyHandler(proxy)
opener = request.build_opener(proxy_support)
request.install_opener(opener)
url = "http://zst.aicai.com/ssq/openInfo/"
shuangseqiu(url)案例二： 爬取空姐网相册中照片
"""
需求：爬取空姐网相册中照片，保存在存储在images目录下，文件命名：uid + picid +'.jpg'
http://www.kongjie.com/home.php?mod=space&do=album&view=all&page=1
"""
from urllib import request,parse
from urllib import error
import chardet
import csv
import re,string
from bs4 import BeautifulSoup
def kongjie(url, beginPage, endPage):
"""
作用：负责处理url，分配每个url去发送请求
url：需要处理的第一个url
beginPage: 爬虫执行的起始页面
endPage: 爬虫执行的截止页面
"""
for page in range(beginPage, endPage):
pn = page
# 组合为完整的 url，并且pn值每次增加50
fullurl = url + "&page=" + str(pn)
headers = {"User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}
req = request.Request(fullurl, headers=headers)
response = request.urlopen(req)
html = response.read()
resHtml = html.decode("utf-8", 'ignore')
soup = BeautifulSoup(resHtml, 'lxml')
a = 0
for m in soup.select('div[class="c"]'):
url = m.a.get('href')
imgresponse = request.urlopen(url)
images = imgresponse.read()
resHtml = images.decode("utf-8", 'ignore')
soup = BeautifulSoup(resHtml, 'lxml')
pg = soup.find_all('label')
if pg !=[]:
for i in pg:
a = i.select('span')
if a !=[]:
for b in a:
ss = b.get('title')
num = re.findall("\d+", ss)[0]
for pn in range(0,int(num)):
kjurl = url+'&page='+ str(pn)
print(kjurl)
kjreq = request.Request(kjurl, headers=headers)
response = request.urlopen(kjreq)
html = response.read()
resHtml = html.decode("utf-8", 'ignore')
soup = BeautifulSoup(resHtml, 'lxml')
for i in soup.select('ul[class="ptw ml mlp cl"] li'):
url = i.a.get('href')
imgresponse = request.urlopen(url)
images = imgresponse.read()
resHtml = images.decode("utf-8", 'ignore')
soup = BeautifulSoup(resHtml, 'lxml')
for i in soup.select('div[id="photo_pic"] a'):
url = i.img.get('src')
print(url)
imageresponse = request.urlopen(url)
listimages = imageresponse.read()
a += 1
with open('./images/美女_%s.jpg' % (a), 'wb') as file:
file.write(listimages)
else:
for i in soup.select('ul[class="ptw ml mlp cl"] li'):
url = i.a.get('href')
imgresponse = request.urlopen(url)
images = imgresponse.read()
resHtml = images.decode("utf-8", 'ignore')
soup = BeautifulSoup(resHtml, 'lxml')
for i in soup.select('div[id="photo_pic"] a'):
url = i.img.get('src')
print(url)
imgresponse = request.urlopen(url)
images = imgresponse.read()
a += 1
with open('./images/空姐_%s.jpg' % (a), 'wb') as file:
file.write(images)
# 模拟 main 函数
if __name__ == "__main__":
proxy = {"http": "218.22.102.107:80"}
proxy_support = request.ProxyHandler(proxy)
opener = request.build_opener(proxy_support)
request.install_opener(opener)
# 输入起始页和终止页，str转成int类型
beginPage = int(input("请输入起始页："))
endPage = int(input("请输入终止页："))
url = "http://www.kongjie.com/home.php?mod=space&do=album&view=all"
# 组合后的url示例：http://tieba.baidu.com/f?kw=lol
url = parse.quote(url, safe=string.printable)
kongjie(url, beginPage, endPage)