花瓣网图片采集

这段时间一直研究新闻采集，正好花瓣网刚刚兴起，突然萌发搞个程序下载花瓣网上图片的想法，程序很简单，希望大家指点，以及完善，共同提高。目前存在的问题是花瓣网采用分屏ajax处理，顾该采集只能采集第一页首页所包含的所有图片，正在研究当中，希望大家有好的想法或者处理方法，也多多指点和告知。

/**
 * 内容摘要：花瓣网采集类
 * 流程说明：
 * @author haozi
 * @return
 */
package test;

import org.htmlparser.Node;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;

import util.CrawNewsTools;

public class CrawPicForHuaBan {
    private static String baseUrl = "http://huaban.com";
    //从首页采集包含图片的URL列表
    public void crawImageList(String indexUrl, String encode) {
        // 采集源码
        String page = CrawNewsTools.getPage(indexUrl, encode);
        System.out.println("###采集到首页的内容:" + page);
        String templateNewsUrl = "/pins/(.*)";
        Node[] nodes = CrawNewsTools.getTagList(page, encode, LinkTag.class);
        for (int i = 0; i < nodes.length; i++) {
            String link = ((LinkTag) nodes[i]).getLink();
            boolean isRegex = CrawNewsTools.isRegex(link, templateNewsUrl);
            if (isRegex) {
                String completeLink = baseUrl + link;
                System.out.println("###符合规则的URL：" + completeLink);
                crawImage(completeLink, encode);
            }
        }

    }
    //采集图片
    public void crawImage(String url, String encode) {
        // 采集源码
        String page = CrawNewsTools.getPage(url, encode);
        String formatPage = CrawNewsTools.fromatBodyTag(page);
        String bodyPage = CrawNewsTools.getBody(formatPage);
        String body = CrawNewsTools.formatHtml(bodyPage);
        System.out.println("###格式后body源码:\r\n" + body);
        Node[] nodes = CrawNewsTools.getTagList(body, encode, ImageTag.class);
        for (int i = 0; i < nodes.length; i++) {
            String picUrl = ((ImageTag) nodes[i]).getImageURL();
            String patternUrl = "http://img.hb.aicdn.com/(.*)";
            boolean isRegex = CrawNewsTools.isRegex(picUrl, patternUrl);
            if (isRegex) {
                System.out.println("图片URL：" + picUrl);
            }
        }
    }

    public static void main(String[] args) {
        String url = "http://huaban.com/all/beauty/";
        String encode = "UTF-8";
        new CrawPicForHuaBan().crawImageList(url, encode);
    }

}


/**
 * 内容摘要：新闻采集工具类
 * 流程说明：
 * @author haozi
 * @return
 */
package util;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.tags.ScriptTag;
import org.htmlparser.tags.StyleTag;
import org.htmlparser.visitors.ObjectFindingVisitor;
import org.htmlparser.visitors.TextExtractingVisitor;

public class CrawNewsTools {

    // 使用HttpClient组件读取指定URL的页面HTML源码
    public static String getPage(String url, String encode) {
        String page = "";
        HttpClient httpClient = null;
        try {
            httpClient = new DefaultHttpClient();
            // 创建httpget
            HttpGet httpget = new HttpGet(url);
            System.out.println("请求URI路径:" + httpget.getURI());
            // 执行get请求
            HttpResponse response = httpClient.execute(httpget);
            // 获得响应实体
            HttpEntity httpEntity = response.getEntity();
            String charset = EntityUtils.getContentCharSet(httpEntity);
            System.out.println("###当前页面编码:" + charset);
            // 获取内容时，指定编码
            if (encode != null && !encode.trim().equals("")) {
                System.out.println("###采用指定编码:" + encode);
            } else if (charset != null) {
                System.out.println("###采用网页自身编码:" + charset);
                encode = charset;
            } else {
                System.out.println("###采用默认UTF-8编码:UTF-8");
                encode = "UTF-8";
            }
            page = EntityUtils.toString(httpEntity, encode);
            page = removeCssTag(page, encode);
            page = removeJsTag(page, encode);
            page = getBody(page);
            page = formatHtml(page);
        } catch (Exception ex) {
            ex.printStackTrace();
        } finally {
            httpClient.getConnectionManager().shutdown();
        }
        return page;
    }

    // 格式化body标签
    public static String fromatBodyTag(String htmlCode) {
        String result = htmlCode;
        if (htmlCode != null) {
            while (result.indexOf("<BODY") != -1) {
                result.replaceAll("<BODY", "<body");
            }
            while (result.indexOf("<Body") != -1) {
                resultresult = result.replaceAll("<Body", "<body");
            }
            while (result.indexOf("</BODY") != -1) {
                resultresult = result.replaceAll("</BODY", "</body");
            }
            while (result.indexOf("</Body") != -1) {
                resultresult = result.replaceAll("</Body", "</body");
            }
        }
        return result;
    }

    // 使用正则表达式提取body体内容
    public static String getBody(String htmlCode) {
        if (htmlCode == null) {
            return null;
        }
        Pattern pattern = Pattern.compile("<body(.*)>(.*)</body>",
                Pattern.MULTILINE | Pattern.DOTALL);
        Matcher matcher = pattern.matcher(htmlCode);
        if (matcher.find()) {
            return matcher.group();
        } else {
            return null;
        }
    }

    // 过滤css标签
    public static String removeCssTag(String htmlCode, String encode) {
        String htmlEndCode = htmlCode;
        try {
            Parser parser = Parser.createParser(htmlCode, encode);
            ObjectFindingVisitor visitor = new ObjectFindingVisitor(
                    StyleTag.class);
            parser.visitAllNodesWith(visitor);
            Node[] nodes = visitor.getTags();
            for (int i = 0; i < nodes.length; i++) {
                // System.out.println(nodes[i].toHtml());
                htmlEndCodehtmlEndCode = htmlEndCode.replace(nodes[i].toHtml(), "");
            }
            // System.out.println("###去除css标签后:" + htmlEndCode);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return htmlEndCode;
    }

    // 过滤js标签
    public static String removeJsTag(String htmlCode, String encode) {
        String htmlEndCode = htmlCode;
        try {
            Parser parser = Parser.createParser(htmlCode, encode);
            ObjectFindingVisitor visitor = new ObjectFindingVisitor(
                    ScriptTag.class);
            parser.visitAllNodesWith(visitor);
            Node[] nodes = visitor.getTags();
            for (int i = 0; i < nodes.length; i++) {
                // System.out.println(nodes[i].toHtml());
                htmlEndCodehtmlEndCode = htmlEndCode.replace(nodes[i].toHtml(), "");
            }
            // System.out.println("###去除js标签后:" + htmlEndCode);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return htmlEndCode;
    }

    // 格式化指定的HTML源码
    public static String formatHtml(String htmlcode) {
        String result = htmlcode;
        if (htmlcode != null && htmlcode.trim().length() > 0) {
            // 去除回车符
            while (result.indexOf("\r") != -1) {
                resultresult = result.replaceAll("\r", "");
            }
            // 去除换行符
            while (result.indexOf("\n") != -1) {
                resultresult = result.replaceAll("\n", "");
            }
            // 去除制表符
            while (result.indexOf("\t") != -1) {
                resultresult = result.replaceAll("\t", "");
            }
            // 去除多余空格
            while (result.indexOf("  ") != -1) {
                resultresult = result.replaceAll("  ", " ");
            }
            // 去除全角空格
            while (result.indexOf("　") != -1) {
                resultresult = result.replaceAll("　", "");
            }
            return result;
        } else {
            return null;
        }
    }

    // 使用HtmlParser组件去除内容中的HTML标签，得到纯文本内容
    public static String getText(String content, String encode) {
        String result = content;
        try {
            Parser parser = Parser.createParser(content, encode);
            // 创建TextExtractingVisitor对象
            TextExtractingVisitor visitor = new TextExtractingVisitor();
            // 去除网页中的所有标签,提出纯文本内容
            parser.visitAllNodesWith(visitor);
            result = visitor.getExtractedText();
            // System.out.println("###去除HTML标签:" + result);
        } catch (Exception ex) {
            ex.printStackTrace();
        }
        return result;
    }

    // 查询网页包含某种标签的集合数组
    @SuppressWarnings("unchecked")
    public static Node[] getTagList(String htmlCode, String encode, Class t) {
        Node[] nodes = null;
        try {
            Parser parser = Parser.createParser(htmlCode, encode);
            ObjectFindingVisitor visitor = new ObjectFindingVisitor(t);
            parser.visitAllNodesWith(visitor);
            nodes = visitor.getTags();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return nodes;
    }

    // 判断自定字符串是否符号某种正则规则
    public static boolean isRegex(String isRegexString, String regexString) {
        boolean regexStatus = Pattern.matches(regexString, isRegexString);
        return regexStatus;
    }
}

 

运行结果:

         图片上传有问题，大家可以运行一下查看结果。

         希望各位多多指点！