BeautifulSoup编写脚本实现内网网页链接爬取
#!/usr/bin/python3 env
# -*- coding:utf-8 -*-
"""
auther:xiaohong.d
data:2020-04-30
description: this python3 script for groovy script use parameters to filter one module engine tag list
from now on. this script need python3 requests,json,BeautifulSoup only running on slave node
"""
import requests,json
from bs4 import BeautifulSoup
class RequestsParam:
"""参数组合"""
def __init__(self,url,prefix,module,suffix,timeout):
self.path = url+prefix+module+suffix
self.timeout = timeout
class Retrieve:
"""获取内容"""
def __init__(self,requestsparam):
self.req = requestsparam
def getPage(self):
r = requests.get(self.req.path,timeout=self.req.timeout)
return r.text
class ParsingHTML:
"""解析返回"""
def __init__(self,content):
self.content = content
def parseVersionLi(self):
soup = BeautifulSoup(self.content,'lxml')
newlist = [ i.string.replace("/", "") for i in soup.find_all("a")]
return newlist[1:]
if __name__ == '__main__':
try:
opts, args = getopt.getopt(sys.argv[1:], "ha:")
except Exception as ex:
print (ex)
sys.exit()
args = "shield"
for tmp1, tmp2 in opts:
if tmp1 == "-a":
args = tmp2
if tmp1 == "-h":
print ("this is a help message")
rp = RequestsParam('http://*****:9999/','box/',args+'/','ine/',0.300)
retrieve = Retrieve(rp)
res = retrieve.getPage()
ph = ParsingHTML(res)
li = ph.parseVersionLi()
for i in li:
print (i)