递归读取heritrix 爬下来的目录文件
递归读取heritrix 爬下来的目录文件
1. 在StringUtil中获取写一个根据路径递归的方法
public ArrayList<String> getAllPath(Stringpath){
File file = newFile(path);
File[] fileList = file.listFiles();
for(File f :fileList){
if(f.isDirectory()){
getAllPath(f.getAbsolutePath());
}else{
results.add(f.getAbsolutePath());
}
}
return results;
}
2. 测试：
@Test
public voidgetAllPathTest(){
String path = "F:\\lucene\\myheritrix\\jobs\\output\\mirror\\news.sohu.com";
ArrayList<String> lists = newStringUtil().getAllPath(path);
for(String s: lists){
System.out.println(s);
}
System.out.println(lists.size());
}
3. 整合起来，每次读取内容
public voidpaserSohuNews2(String inputPath,String outputPath){
File outputfile = newFile(outputPath);
try {
if(!outputfile.exists()){
outputfile.mkdirs();
}
ArrayList<String> pathLists = newStringUtil().getAllPath(inputPath);
for(Stringpath: pathLists){
if(!path.contains(".shtml")) continue;
String html = StringUtil.getContent(path);
//System.out.println(html);
String content = StringUtil.getContentUseParse2(html,"gbk", "h1","itemprop","headline");
content += ConstantString.WIN_NextLine+StringUtil.getContentUseParse2(html,"gbk", "div", "itemprop", "datePublished");
System.out.println(content);
content += ConstantString.WIN_NextLine+StringUtil.getContentUseParse2(html,"gbk", "div", "itemprop", "articleBody");
String name = StringUtil.getNameFromPath(path);
File outputFile = newFile(outputPath+"/"+name+ConstantString.postText);
if(!outputFile.exists()){
outputFile.createNewFile();
}
//System.out.println("outputpath:"+outputPath);
//System.out.println("outputFile:"+outputFile);
OutputStreamWriter ot = new OutputStreamWriter(newFileOutputStream(outputFile));
ot.write(content);
ot.close();
//System.out.println("---------------------------------------");
}