2019-05-31

Java去除文件中的重复行

public static void main(String[] args) throws IOException {

        LinkedHashMap map = new LinkedHashMap();

        ArrayList arrayList = new ArrayList<>(10000);

        List repeatList = new ArrayList(10000);

        String line="";

        File file =new File("D:/companyFile/TestToExcel/recursionzznissan.txt");

        File fileOutPut =new File("D:/companyFile/TestToExcel/recursionDistinct.txt");

        FileInputStream fis;InputStreamReader isr;BufferedReader br;

    try {

    fis = new FileInputStream(file);

    isr = new InputStreamReader(fis, "UTF-8");

    br = new BufferedReader(isr);

    while((line=br.readLine())!=null){

    String[] sArray = line.split(",");

                for (String s : sArray) {

                //set.add(s.substring(s.lastIndexOf("\\")+1, s.length()));

                //set.add(s.substring(0,s.lastIndexOf("\\")));

                map.put(s.toString(),"");

                //map.put(s.substring(s.lastIndexOf("\\")+1, s.length()),"");

                }

    }

                br.close();isr.close();fis.close();

                for(String key:map.keySet()){

                FileOutputStream fos = new FileOutputStream(fileOutPut,true);

                OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8");

                BufferedWriter bw = new BufferedWriter(osw);

                bw.write(key.toString());

                bw.newLine();

                bw.close();

                osw.close();

                fos.close();

    }

            }catch (Exception e){

    e.printStackTrace();

    }

 }