htmlunit 开源的java页面分析工具,读取页面后,使用htmlunit分析页面内容。模拟浏览器的运行,是一个没有界面的浏览器,运行迅速。
采用Rhinojs引擎。模拟js运行
引入依赖
4.0.0
com.open
HtmlUnitTest
0.0.1-SNAPSHOT
net.sourceforge.htmlunit
htmlunit
2.26
org.apache.httpcomponents
httpclient
4.5.2
import java.io.IOException;
import java.net.MalformedURLException;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class HtmlUnitTest {
public static void main(String[] args) {
WebClient webClient=new WebClient(); // 实例化Web客户端
try {
HtmlPage page=webClient.getPage("http://www.baidu.com"); // 解析获取页面
System.out.println("网页html:"+page.asXml());
System.out.println("==================================");
System.out.println("网页文本:"+page.asText());
} catch (FailingHttpStatusCodeException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
webClient.close(); // 关闭客户端,释放内存
}
}
}
htmlunit模拟浏览器
WebClient使用代理IP
setCssEnabled 取消支持提升解析速度
getHtmlElementById 根据id获取指定div元素
getElementsByTagName 根据标签名字获取a标签
getByXPath 根据dom获取元素,//任意路径下获取div 下 ul li的第二个 超链接的名称
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlListItem;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;
public class HtmlUnit {
public static void main(String[] args) {
WebClient webClient=new WebClient(BrowserVersion.FIREFOX_52,"1.61.148.187",25394); // 实例化Web客户端端
webClient.getOptions().setCssEnabled(false); // 取消css支持
webClient.getOptions().setJavaScriptEnabled(false); // 取消javascript支持
try {
HtmlPage page=webClient.getPage("http://www.baidu.com"); // 解析获取页面
Thread.sleep(10000); // 休息10秒 等待htmlunit执行Js
HtmlDivision div=page.getHtmlElementById("navMenu"); // 查找指定id的html dom元素
System.out.println(div.asXml());
System.out.println("==============");
HtmlForm form=page.getFormByName("myform"); // 获取搜索Form
HtmlTextInput textField=form.getInputByName("q"); // 获取查询文本框
HtmlSubmitInput button=form.getInputByName("submitButton"); // 获取提交按钮
textField.setValueAttribute("java"); // 文本框“填入”数据
HtmlPage resultPage=button.click(); // 模拟点击 获取查询结果页面
System.out.println(resultPage.asXml());
System.out.println("==============");
DomNodeList aList=page.getElementsByTagName("a"); // 根据tag名称查询所有tag
for(int i=0;i spanList=page.getByXPath("//div[@id='navMenu']/ul/li[2]/a/span");
System.out.println(spanList.get(0).asText());
} catch (FailingHttpStatusCodeException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
webClient.close(); // 关闭客户端,释放内存
}
}
}