txt文件分割

1. background

   要在很大的数据量中查询其中某些flag的值是否正常，debug看肯定不行，我就把flag都输出到log中，然后下班回家。回到家中发现log有2G多，编辑器打不开，然后就想写个分割器。code 贴到下面

2. code

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.NoSuchElementException;
import java.util.Scanner;

public class FileSplit {
    public static String PATH = "D://FileSplit";
    public static String FILE_NAME = "E:\\xxx\\data\\logs\\LogWriterLog.txt";

    public static int MAX_LINE_NUM = 100000;
    public static int FILE_INDEX = 1;

    public static void main(String[] args) {
        FileInputStream fis = null;
        FileWriter fw = null;
        BufferedWriter bw = null;
        try {
            File file = new File(FILE_NAME);
            fis = new FileInputStream(file);
            Scanner sc = new Scanner(fis);
            String nextLine = sc.nextLine();
            String lineSeparator = System.getProperty("line.separator", "\r\n");  
            mkdirs(PATH);
            int i = 0;
            boolean needNewFile = true;
            while(nextLine != null){
                if(i < MAX_LINE_NUM) {
                    if(needNewFile) {
                        File temp = new File(PATH + File.separator + FILE_INDEX + ".txt");
                        fw = new FileWriter(temp);
                        bw = new BufferedWriter(fw);
                        needNewFile = false;
                    }
                    bw.append(nextLine);
                    bw.append(lineSeparator);
                    i++;
                } else {
                    bw.append(nextLine);
                    bw.append(lineSeparator);
                    bw.flush();
                    i = 0;
                    needNewFile = true;
                    FILE_INDEX ++;
                    if(bw != null) {
                        bw.close();
                    }
                    if(fw != null) {
                        fw.close();
                    }
                }
                try {
                    nextLine = sc.nextLine();
                } catch (NoSuchElementException e) {
                    System.exit(0);
                }
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
                try {
                    if(bw != null) {
                        bw.close();
                    }
                    if(fw != null) {
                        fw.close();
                    }
                    if(fis != null) {
                        fis.close();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
        }
    }

    private static void mkdirs(String path) {
        File file = new File(path);
        if(!file.exists()){
            file.mkdirs();
        }
    }
}

1. Note
   

代码很简单，但是要考虑内存问题，文件太大，如果都加载到内存很容易就OutOfMemory，所以我用了Scanner来扫描整个文件，这样只会扫描文件不会加载整个文件到内存，不用担心内存问题。