Java-实现文本数据去重

Java实现文本去重

直接看代码吧，具体写在注释里了

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

/**
 * Created by Jaylin on 2017/2/17.
 */
public class RepeatFilter {
    public static void main(String[] args) {
        File inFile = new File("C:\\Users\\Jaylin\\Desktop\\a.txt");
        if (!(inFile.exists() && inFile.isFile())) {
            System.out.println("---读取文件出错，请确认存在此文件---");
            return;
        }

        String fileName = inFile.getName();

        //输出文件名
        File outFile = new File(
                inFile.getParent()
                        + "\\result_"
                        + fileName
        );

        //用于去重的Set
        Set filter = new HashSet();

        try {
            //读取数据
            String content = FileUtils.readFileToString(inFile, "utf-8").trim();
            String[] lines = content.split("\n");
            int inLineNumber = lines.length;

            //通过Set处理重复数据
            for (String line : lines) {
                filter.add(line);
            }
            int outLineNumber = filter.size();

            //创建字符串生成器,节省内存开销
            StringBuilder sb = new StringBuilder();

            for (String line : filter) {
                sb.append(line + "\n");
            }

            //输出文件
            if (!(outFile.exists() && outFile.isFile())) {
                outFile.createNewFile();
            }

            FileUtils.writeStringToFile(
                    outFile,
                    sb.toString(),
                    "utf-8"
            );

            System.out.println("共处理了"
                    + inLineNumber
                    + "条数据"
                    + "\n去重"
                    + (inLineNumber - outLineNumber)
                    + "条"
            );

        } catch (IOException e) {
            e.printStackTrace();
        }

    }
}

用了 Apache Commons IO 这个包，简化了下文件的输入输出。