java文件处理测试
测试一下java处理文件
package name;
import df.util.Util;
import df.util.type.RandomUtil;
import df.util.type.StringUtil;
import df.util.type.SysLog;
import java.io.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* Created by andrew on 2015/7/12.
*/
public class NameApp implements Runnable {
private static final String TAG = Util.toTAG(NameApp.class);
Map<String, Integer> repeatNames = new HashMap<String, Integer>();
String ignoreChars = "";
@Override
public void run() {
File currentPath = new File("");
File dir = new File(currentPath.getAbsolutePath());
File parent = new File(dir.getParent());
if (null != parent
&& parent.isDirectory()) {
File res = new File(parent.getAbsolutePath() + "\\res");
if (!res.isDirectory()) {
res.mkdirs();
}
File ignoreFile = new File(parent.getAbsolutePath() + "\\ignore_chars.txt");
String ignoreEncodeType = "gbk";
if (!ignoreFile.isFile()) {
File ignoreCurrentPath = new File(ignoreFile.getAbsolutePath());
File ignoreParent = new File(ignoreCurrentPath.getParent());
ignoreParent.mkdirs();
String path = ignoreFile.getAbsolutePath();
ignoreFile = new File(path);
try {
ignoreFile.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
}
try {
InputStream ignoreIn = new BufferedInputStream(new FileInputStream(ignoreFile));
ignoreEncodeType = encodeType(ignoreIn);
byte[] ignoreBug = new byte[1024];
while (ignoreIn.read(ignoreBug) != -1) {
ignoreChars += new String(ignoreBug, ignoreEncodeType);
}
ignoreChars = ignoreChars.trim() + ' ';
SysLog.v(TAG, " ignoreChars=", ignoreChars);
} catch (IOException e) {
e.printStackTrace();
}
int saveFileIndex = 0;
while (true) {
File[] files = res.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
String name = pathname.getName();
if (name.toLowerCase().endsWith(".txt")) {
return true;
}
return false;
}
});
StringBuffer buf = new StringBuffer();
for (File f : files) {
InputStream reader = null;
OutputStream out = null;
OutputStream outRepeatName = null;
try {
File saveNameFile = null;
while (saveFileIndex < 99999) {
saveNameFile = new File(parent.getAbsolutePath() + "\\names_" + saveFileIndex + ".txt");
if (!saveNameFile.isFile()
|| saveNameFile.length() < 1024 * 1024 * 4) {
break;
}
saveFileIndex++;
}
out = new BufferedOutputStream(new FileOutputStream(saveNameFile, true));
outRepeatName = new BufferedOutputStream(new FileOutputStream(new File(parent.getAbsolutePath() + "\\repeat_names.txt"), true));
reader = new BufferedInputStream(new FileInputStream(f));
String charset = encodeType(reader);
if (StringUtil.empty(charset)) {
charset = "gbk";
}
byte[] line = new byte[1024 * 1024];
buf.setLength(0);
Arrays.fill(line, (byte) 0);
while (reader.read(line, 0, line.length) != -1) {
buf.append(new String(line, charset));
Arrays.fill(line, (byte) 0);
}
String txt = buf.toString().trim();
String repeatText = "";
String allNames = "";
String son = "";
String context = "";
String context2 = "";
while (son.length() < 2) {
int index = RandomUtil.toRandomInt(0, txt.length() - 2);
int index2 = RandomUtil.toRandomInt(0, txt.length() - 2);
char c1 = txt.charAt(index);
char c2 = txt.charAt(index2);
if (ignoreChars.contains("" + c1)
|| ignoreChars.contains("" + c2)) {
continue;
}
if (isLegal(c1) && isLegal(c2)) {
son += "" + c1 + c2;
son = son.trim();
if (son.length() == 2) {
final int gap = 3;
int front = index > gap ? index - gap : 0;
int behind = index + gap < txt.length() ? index + gap : txt.length();
context = txt.substring(front, behind);
context = context.replace("\r", "");
context = context.replace("\n", "");
context = context.replace(" ", "");
int front2 = index2 > gap ? index2 - gap : 0;
int behind2 = index2 + gap < txt.length() ? index2 + gap : txt.length();
context2 = txt.substring(front2, behind2);
context2 = context2.replace("\r", "");
context2 = context2.replace("\n", "");
context2 = context2.replace(" ", "");
String nameContext = son + " " + context + " " + context2;
allNames += nameContext + " " + f.getName() + "\r\n";
int repeatCount = 0;
if (repeatNames.containsKey(son)) {
repeatCount = repeatNames.get(son) + 1;
repeatText = nameContext + " " + repeatCount + " " + f.getName() + "\r\n";
}
repeatNames.put(son, repeatCount);
}
}
}
out.write(allNames.getBytes());
out.flush();
if (!StringUtil.empty(repeatText)) {
if (repeatNames.containsKey(son)) {
Integer val = repeatNames.get(son);
if (val > 5) {
outRepeatName.write(repeatText.getBytes());
outRepeatName.flush();
repeatNames.clear();
SysLog.v(TAG, repeatText, " , ", f.getName());
}
}
}
} catch (Exception e) {
SysLog.v(TAG, ", e=", e.getMessage());
e.printStackTrace();
} finally {
if (null != reader) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
try {
out.close();
} catch (Exception e) {
e.printStackTrace();
}
try {
outRepeatName.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
}
public static void main(String[] args) {
new Thread(new NameApp()).start();
}
public static String encodeType(InputStream reader) {
String type = "";
try {
reader.mark(0);
byte[] first3Bytes = new byte[3];
int read = reader.read(first3Bytes);
if (3 == read) {
//其中的 0xefbb、0xfffe、0xfeff、0x5c75这些都是这个文件的前面两个字节的16进制数
if (first3Bytes[0] == (byte) 0xFF && first3Bytes[1] == (byte) 0xFE) {
type = "UTF-16LE";
} else if (first3Bytes[0] == (byte) 0xFE && first3Bytes[1] == (byte) 0xFF) {
type = "UTF-16BE";
} else if (first3Bytes[0] == (byte) 0xEF && first3Bytes[1] == (byte) 0xBB && first3Bytes[2] == (byte) 0xBF) {
type = "UTF-8";
} else {
type = "gbk";
reader.reset();
}
} else {
type = "gbk";
reader.reset();
}
} catch (IOException e) {
e.printStackTrace();
}
return type;
}
public static boolean isLegal(char c) {
if ((c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9')
|| c == ' '
|| c == '。'
|| c == '，'
|| c == '"'
|| c == '“'
|| c == '”'
|| c == '、'
|| c == '?'
|| c == '？'
|| c == 12288) {
return false;
}
return true;
}
}