htmlparser网页抓取

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

import org.apache.log4j.Logger;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.Tag;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.util.SimpleNodeIterator;

/**
* 分析www.cheshi.com首页新闻
* @author j.li
*/
public class HtmlParser {
    private static Logger logger;
    private Connection conn = null;
    private static final String SiteName = "";

    public void indexNewsContent(String sitepath) throws Exception {
        logger.info("分析网站【" + sitepath + "】首页的新闻列表，内容为【<div class=\"hotjd\"></div>】所有网页新闻地址的HTML内容。");
        Parser myParser = new Parser(sitepath);
        myParser.setEncoding("UTF-8");
        NodeList nodeList = myParser.extractAllNodesThatMatch(new NodeFilter() {
            public boolean accept(Node node) {
                return ((node instanceof Tag)
                        && !((Tag)node).isEndTag()
                        && ((Tag)node).getTagName().equals("DIV")
                        && ((Tag)node).getAttribute("class") != null
                        && ((Tag)node).getAttribute("class").equals("desc clearfix"));
            }
        });
        for(int i=0,len=nodeList.size();i<len;i++){
        Node node = nodeList.elementAt(i);
             logger.debug(node.toHtml());
             System.out.println(node.toHtml());
             System.out.println("------------------------------------------------------------------------------------------------------");
//             extractText(node.toHtml());
        }
      
    }
    
    public void extractText(String inputHtml) throws Exception {   
        Parser parser = Parser.createParser(inputHtml, "GBK");
        TagNameFilter filter = new TagNameFilter("a");
        NodeList nodeList = parser.extractAllNodesThatMatch(filter);
        NodeIterator it = nodeList.elements();
        getConnection();
        while (it.hasMoreNodes()) {
            LinkTag node = (LinkTag) it.nextNode();
            String href = node.getLink();
            String title = node.getLinkText();
            logger.info("分析首页新闻【"+title+"】，链接地址【"+href+"】");
            try {
                if(!newsExist(title)) {
                    insertDataBase(title, extractContent(href));
                } else {
                    logger.info("新闻【"+title+"】数据库中已经存在，忽略进入下一个新闻分析！");
                }
            } catch (SQLException e) {
                logger.error("插入数据库新闻记录异常！" + e.getMessage());
                e.printStackTrace();
            } catch (Exception e) {
                logger.error(e.getMessage());
                logger.info("分析新闻【"+title+"】，链接地址【"+href+"】失败，进入下一个新闻分析。");
                e.printStackTrace();
            }
        }
        closeConnection();
    }

    public String extractContent(String content) throws Exception {
        try {
            Parser myParser = new Parser(content);
            myParser.setEncoding("GBK");
            NodeList nodeList = myParser.extractAllNodesThatMatch(new NodeFilter() {
                public boolean accept(Node node) {
                    return ((node instanceof Tag)
                            && !((Tag)node).isEndTag()
                            && ((Tag)node).getTagName().equals("DIV")
                            && ((Tag)node).getAttribute("class") != null
                            && ((Tag)node).getAttribute("class").equals("cs_content"));
                }
            });
            int size = nodeList.size();
            Node node = nodeList.elementAt(size - 1);
            content = node.toHtml();
            logger.debug("==========extractContent==============");
            logger.debug(content);
        } catch (Exception pe) {
            logger.error("分析新闻页面出现异常！" + pe.getMessage() + "原因可能出现于新闻页面不存在<div class=\"cs_content\"></div>标记。");
            throw pe;
        }
        return removeTagA(content);
    }
    
    /**
     * 去除新闻中href包含cheshi.com的<a>标签
     * @param content 分析html内容
     * @return 分析处理后的html内容
     */
    public String removeTagA(String content) throws ParserException {
        Parser myParser = new Parser(content);
        myParser.setEncoding("GBK");
        NodeList nodeList = myParser.extractAllNodesThatMatch(new TagNameFilter("a"));
        SimpleNodeIterator it = nodeList.elements();
        while (it.hasMoreNodes()) {
            LinkTag node = (LinkTag)it.nextNode();
            logger.info("移除新闻内容中包含的文字、图片的链接【"+node.toHtml()+"】。");
            if(node.getLink().indexOf("cheshi.com") > -1)
                content = content.replace(node.toHtml(), node.getStringText());
        }
        logger.debug("==========removeTagA==============");
        logger.debug(content);
        return downloadImages(content, "D:\\autodata\\upload\\intersite", SiteName + "upload/intersite");
    }

    public String downloadImages(String content, String uploadImgPath, String localhost) throws ParserException {
        File f = new File(uploadImgPath);
        if(!f.exists()) {
            f.mkdirs();
        }
        Parser myParser = new Parser(content);
        myParser.setEncoding("GBK");
        NodeList nodeList = myParser.extractAllNodesThatMatch(new TagNameFilter("img"));
        SimpleNodeIterator it = nodeList.elements();
        while(it.hasMoreNodes()) {
            Tag tag = (Tag)it.nextNode();
            String src = tag.getAttribute("src");
            String filename = src.substring(src.lastIndexOf("/") + 1);
            InputStream is = null;
            FileOutputStream fos = null;
            try {
                URL url = new URL(src);
                is = url.openStream();
                int bytesRead = 0;
                byte[] buff = new byte[1024];
                fos = new FileOutputStream(uploadImgPath+"/"+filename);
                while((bytesRead = is.read(buff, 0, buff.length)) != -1){
                    fos.write(buff, 0, bytesRead);
                }
                content = content.replace(src, localhost + "/" + filename);
            } catch(FileNotFoundException notFoundException) {
                notFoundException.printStackTrace();
            } catch(IOException ioe) {
                ioe.printStackTrace();
            } finally {
                try {
                    if(fos != null) fos.close();
                    if(is != null) is.close();
                } catch(IOException ioe) {
                    ioe.printStackTrace();
                }
            }
        }
        logger.debug("=================downloadImages==================");
        logger.debug(content);
        return content;
    }
    
    public void getConnection() {
        try {
            Class.forName("com.microsoft.jdbc.sqlserver.SQLServerDriver");
            String strCon = "jdbc:microsoft:sqlserver://192.168.99.188:12580;databaseName=Project2009;SelectMethod=cursor";
            String strUserName = "sa";
            String strPWD = "qsyjcsxdl@@@web2009@@@";
            conn = DriverManager.getConnection(strCon, strUserName, strPWD);
        } catch (java.lang.ClassNotFoundException cnfe) {
            cnfe.printStackTrace();
        } catch (SQLException se) {
            se.printStackTrace();
        }
    }
    
    public void closeConnection() {
        try {
            if(conn!= null && !conn.isClosed()) conn.close();
        } catch (SQLException se) {
            se.printStackTrace();
        }
    }
    
    public void insertDataBase(String newsTitle, String newsContent) throws SQLException {
        PreparedStatement pstmt = null;
        try {
            pstmt = conn.prepareStatement("INSERT INTO FumNews(NewsTitle, NewsContext, NewsState) values(?, ?, ?)");
            pstmt.setString(1, newsTitle);
            pstmt.setString(2, newsContent);
            pstmt.setInt(3, 1);
            pstmt.executeUpdate();
        } catch(SQLException e) {
            throw e;
        } finally {
            try {
                if(pstmt != null) pstmt.close();
            } catch(SQLException e) {
                e.printStackTrace();
            }
        }
    }
    
    public boolean newsExist(String title) throws SQLException {
        PreparedStatement pstmt = null;
        try {
            pstmt = conn.prepareStatement("SELECT top 1 NewsId from FumNews where NewsTitle = ?");
            pstmt.setString(1, title);
            ResultSet rs = pstmt.executeQuery();
            return rs.next();
        } catch(SQLException e) {
            throw e;
        } finally {
            try {
                if(pstmt != null) pstmt.close();
            } catch(SQLException e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        HtmlParser html = new HtmlParser();
//      设置代理链接网络
//      System.getProperties().put("proxySet", "true");
//      System.getProperties().put("proxyHost", "192.168.99.100");
//      System.getProperties().put("proxyPort", "80");
//        URL url = html.getClass().getResource("log4j.properties");
//        PropertyConfigurator.configure("www.cheshi.com");
        logger = Logger.getLogger(HtmlParser.class);
        try {
            html.indexNewsContent("http://www.kaola.com/activity/detail/3245.html?navindex=1");
        } catch (Exception e) {
            e.printStackTrace();
            logger.error("分析网页遇到错误，原因："+e.getMessage());
        }
        logger.info("分析网页内容完成。");
    }
}