HtmlUnit入门

HtmlUnit介绍

htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。是一个没有界面的浏览器，运行速度迅速。

入门案例

pom.xml加入依赖

    
        net.sourceforge.htmlunit
        htmlunit
        2.23
    

获取页面内容

        /**
         * 获取页面内容
         */
        public class HtmlUnitDemo03 {

            public static void main(String[] args) {
                //实例化web客户端
                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128);//使用代理IP
                try {
                    //解析获取页面
                    HtmlPage page = webClient.getPage("http://www.java1234.com");
                    HtmlElement div = page.getHtmlElementById("navMenu");
                    System.out.println(div.asXml());
                    System.out.println("---------------");
                    //获取a标签
                    DomNodeList dnList = page.getElementsByTagName("a");
                    for (int i = 0; i < dnList.getLength(); i++) {
                        DomElement element = dnList.get(i);
                        System.out.println(element.asXml());
                        System.out.println(element.asText());
                    }
                    System.out.println("------------------");
                    List itemList = (List) page.getByXPath("//div[@id='navMenu']/ul/li");
                    System.out.println(itemList.get(0).asXml());
                } catch (FailingHttpStatusCodeException e) {
                    e.printStackTrace();
                } catch (MalformedURLException e) {
                    e.printStackTrace();
                } catch (ElementNotFoundException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    //释放资源
                    webClient.close();
                }
            }
        }

取消css,js支持

        /**
         * 取消css,js支持
         */
        public class HtmlUnitDemo04 {

            public static void main(String[] args) {
                //实例化web客户端
                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128);//使用代理IP
                //取消css支持
                webClient.getOptions().setCssEnabled(false);
                //取消js支持
                webClient.getOptions().setJavaScriptEnabled(false);
                try {
                    //解析获取页面
                    HtmlPage page = webClient.getPage("http://www.java1234.com");
                    HtmlElement div = page.getHtmlElementById("navMenu");
                    System.out.println(div.asXml());
                    System.out.println("---------------");
                    //获取a标签
                    DomNodeList dnList = page.getElementsByTagName("a");
                    for (int i = 0; i < dnList.getLength(); i++) {
                        DomElement element = dnList.get(i);
                        System.out.println(element.asXml());
                    }
                    System.out.println("------------------");
                    List itemList = (List) page.getByXPath("//div[@id='navMenu']/ul/li");
                    System.out.println(itemList.get(0).asXml());
                } catch (FailingHttpStatusCodeException e) {
                    e.printStackTrace();
                } catch (MalformedURLException e) {
                    e.printStackTrace();
                } catch (ElementNotFoundException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    //释放资源
                    webClient.close();
                }
            }
        }

模拟表单搜索

        /**
         * 模拟表单搜索
         */
        public class HtmlUnitDemo05 {

            public static void main(String[] args) {
                //实例化web客户端
                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128);//使用代理IP
                try {
                    //解析获取页面
                    HtmlPage page = webClient.getPage("http://blog.java1234.com/index.html");
                    //获取搜索表单
                    HtmlForm form = page.getFormByName("myform");
                    //获取查询文本框
                    HtmlTextInput textInput = form.getInputByName("q");
                    //获取提交按钮
                    HtmlSubmitInput submitInput = form.getInputByName("submitButton");
                    //输入文本框
                    textInput.setValueAttribute("java");
                    //模拟提交(点击)
                    HtmlPage resultPage = submitInput.click();
                    System.out.println(resultPage.asXml());
                } catch (FailingHttpStatusCodeException e) {
                    e.printStackTrace();
                } catch (MalformedURLException e) {
                    e.printStackTrace();
                } catch (ElementNotFoundException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    //释放资源
                    webClient.close();
                }
            }
        }

获取ajax加载的内容

        /**
         * 获取ajax加载的内容
         */
        public class HtmlUnitDemo06 {

            public static void main(String[] args) {
                //实例化web客户端
                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45);//使用代理IP
                try {
                    //解析获取页面
                    HtmlPage page = webClient.getPage("https://pan.baidu.com/share/home?uk=305605848#category/type=0");
                    //等待htmlunit执行js
                    Thread.sleep(10000);
                    System.out.println("网页html:"+page.asXml());
                } catch (FailingHttpStatusCodeException e) {
                    e.printStackTrace();
                } catch (MalformedURLException e) {
                    e.printStackTrace();
                } catch (ElementNotFoundException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } finally {
                    //释放资源
                    webClient.close();
                }
            }
        }