基于文本标签方差分析和链接分析的网页正文、正文图片地址抽取算法
public class CopyOfContentExtractor {
private static final int MIN_NODE_TEXT_LENGTH = 20; // //正文的最小长度
private static final int MIN_K = 30; // //有了链接分析，可以设置高一些，粗放一些
private static final double MAX_LINK_RATE = 0.5; // ///最小链接率
private double TEMP_MAX_LENGTH = 0;
private Node targetNode = null;
public String title = "";
public List<String> imgSrcList = new ArrayList<String>();
public String address;
/**
* 解析本地html文件
*
* @param address
*/
public void parseHTM(String location, String address) {
this.address = address;
try {
parse(new FileInputStream(new File(location)));
} catch (SAXException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 解析传入的url地址
*
* @param address
*/
public void parseURL(String address) {
this.address = address;
try {
URL url = new URL(address);
parse(url.openConnection().getInputStream());
} catch (SAXException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
private void parse(InputStream ips) throws SAXException, IOException {
DOMParser parser = new DOMParser();
parser.setProperty("http://cyberneko.org/html/properties/default-encoding", "utf-8");
parser.parse(new InputSource(ips));
Document doc = parser.getDocument();
Node body = doc.getElementsByTagName("body").item(0); // 抽取内容
if (body != null) {
NodeList list = body.getChildNodes();
for (int i = 0; i < list.getLength(); i++) {
Node node = list.item(i);
loop(node); // //////递归调用
}
// //////此时已对targetNode赋值
if (targetNode != null) {
linkNodeFilter(targetNode);
// //////进行链接分析继续对目标node的dom过滤
// refixTargetNodeByLinkAnalysis(this.targetNode);
// //////抽取目标node内的图片地址
imgExtractor(targetNode);
} else {
System.out.println("extractor is NULL!!! ");
}
}
}
public void loop(Node node) {
if (node.getNodeType() == Node.ELEMENT_NODE) {
Element elmt = (Element) node;
if (!elmt.getTagName().equals("STYLE") && !elmt.getTagName().equals("SCRIPT")) {
NodeList list = node.getChildNodes();
if (list.getLength() == 1) {
loop(list.item(0));
} else {
double[] lengthArr = new double[list.getLength()];
for (int i = 0; i < list.getLength(); i++) { /////计算每个子节点的文本长度
String text = textExtractor(list.item(i));
///System.out.println("text: " + text);
lengthArr[i] = text.length();
}
// ////////遍历每一个长度，判断走向
double sum = 0.0;
for (double d : lengthArr) {
sum += d;
}
if (sum > MIN_NODE_TEXT_LENGTH) {
double mean = sum / lengthArr.length;
double varianceSum = 0.0;
for (double d : lengthArr) {
varianceSum += (d - mean) * (d - mean);
}
double variance = varianceSum / lengthArr.length; // /////方差
double k = variance / sum;
////System.out.println("k:" + k);
if (k < MIN_K) {
////System.out.println(k + " " + textExtractor(node));
////抽取正文node
if (sum > TEMP_MAX_LENGTH) {
TEMP_MAX_LENGTH = sum;
targetNode = node;
}
} else {
////重新采用链接分析的办法筛选
for (int i = 0; i < list.getLength(); i++) { /////计算每个子节点的文本长度
lengthArr[i] = lengthArr[i] - linkAnalysis(list.item(i)); ///减去链接文本长度
}
int maxIndex = getMaxIndex(lengthArr);
loop(list.item(maxIndex));
}
}
}
}
}
}
// 抽取节点文本递归部分
public String textExtractor(Node root) {
// 若是文本节点的话，直接返回
if (root.getNodeType() == Node.TEXT_NODE) {
return root.getNodeValue().trim();
}
if (root.getNodeType() == Node.ELEMENT_NODE) {
Element elmt = (Element) root;
// 去除脚本
if (elmt.getTagName().equals("STYLE") || elmt.getTagName().equals("SCRIPT"))
return "";
NodeList children = elmt.getChildNodes();
StringBuilder text = new StringBuilder();
for (int i = 0; i < children.getLength(); i++) {
String innerText = textExtractor(children.item(i));
if (innerText.length() > 5) {
text.append(innerText + " ");
}
}
return text.toString();
}
// 对其它类型的节点，返回空值
return "";
}
// 抽取图片地址递归部分
private void imgExtractor(Node root) {
if (root.getNodeType() == Node.ELEMENT_NODE) {
Element elmt = (Element) root;
if (elmt.getTagName().equals("IMG") || elmt.getTagName().equals("img")) {
String src = elmt.getAttribute("real_src").isEmpty() ? elmt.getAttribute("src") : elmt.getAttribute("real_src");
if (!src.startsWith("http://")) { // //////如果图片地址是相对地址
String sub = address.substring(7);
int index = sub.indexOf("/");
src = "http://" + sub.substring(0, index) + src;
}
imgSrcList.add(src);
}
NodeList children = elmt.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
imgExtractor(children.item(i));
}
}
}
/**
* 计算节点链接文本长度，非破坏性
* @param root
* @return
*/
private double linkAnalysis(Node root) {
// 若是文本节点的话，直接返回
if (root.getNodeType() == Node.TEXT_NODE
&& (root.getParentNode().getNodeName().endsWith("a") || root.getParentNode().getNodeName().endsWith("A"))) {
return root.getNodeValue().trim().length();
}
if (root.getNodeType() == Node.ELEMENT_NODE) {
Element elmt = (Element) root;
// 去除脚本
if (elmt.getTagName().equals("STYLE") || elmt.getTagName().equals("SCRIPT"))
return 0.0;
NodeList children = elmt.getChildNodes();
double temp = 0.0;
for (int i = 0; i < children.getLength(); i++) {
double innerTemp = linkAnalysis(children.item(i));
temp += innerTemp;
}
return temp;
}
// 对其它类型的节点，返回空值
return 0.0;
}
private int getMaxIndex(double[] input) {
int index = 0;
double temp = 0.0;
for (int i = 0; i < input.length; i++) {
if (temp < input[i]) {
temp = input[i];
index = i;
}
}
return index;
}
/**
* 采用remove风格处理targetNode：链接分析
*/
private void linkNodeFilter(Node node) {
if (node != null) {
NodeList children = node.getChildNodes();
// //拼接网页正文
for (int i = 0; i < children.getLength(); i++) {
Node subNode = children.item(i);
// ///链接分析
String text = textExtractor(subNode);
double textLength = text.length();
if (textLength > 0) {
// System.out.println("Text:" + text);
// System.out.println(subNode.getNodeName());
double linkLen = linkAnalysis(subNode);
double linkRate = linkLen / textLength;
// System.out.println("linkLen: " + linkLen + "; textLen: "
// + textLength + "; linkRate: " + linkRate);
// System.out.println(100 * linkRate / (linkLen +
// textLength));
if (linkRate < MAX_LINK_RATE) {
linkNodeFilter(subNode);
} else {
node.removeChild(subNode);
}
}
}
}
}
/**
*
* @return
*/
private String getContent() {
if(targetNode != null){
return textExtractor(targetNode);
}else{
return "NULL";
}
}
// //////////
public static void main(String[] argv) throws IOException {
int count = 0;
for (File f : new File("D:/fxreader/data/2012-09-29-00-13").listFiles()) {
CopyOfContentExtractor extractor = new CopyOfContentExtractor();
extractor.parseHTM(f.getAbsolutePath(), Options.rules_startsWith);
String content = extractor.getContent();
if(content.length() > 200 ){
count++;
PrintWriter pw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(new File("D:/fxreader/content/2012-09-29-00-13/" + f.getName()+ "_" + content.length() + "_" + ".txt")), "utf-8"));
pw.append(content);
//FileUtils.writeStringToFile(new File("D:/fxreader/content/" + f.getName()+ "_" + content.length() + "_" + ".txt"), content, "utf-8");
pw.flush();
pw.close();
System.out.println("missing: " + f.getName());
}
System.out.println(count);
};
}
}
实验测试，错误率：42/500.其中，42篇中大多为原网页不含有“正文”的网页。