多线程实战运用【多线程解析文件获取信息，合并子线程输出文件】

package com.qtopay.common.localcache;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;

import com.alibaba.fastjson.JSONObject;

public class MultThread {

    public static void main(String[] args) {

        String[] files = new String[] { "D:\\20190909\\217log\\authentication-remote-digest.log.1", "D:\\20190909\\217log\\authentication-remote-digest.log.2",
                                        "D:\\20190909\\217log\\authentication-remote-digest.log.3", "D:\\20190909\\216log\\authentication-remote-digest.log.1",
                                        "D:\\20190909\\216log\\authentication-remote-digest.log.2", "D:\\20190909\\216log\\authentication-remote-digest.log.3", };
        List list = new ArrayList<>();
        StopWatch sw = new StopWatch();
        sw.start();
        for (int i = 0; i < files.length; i++) {
            File file = new File(files[i]);
            FileInputStream inputStream = null;
            InputStreamReader inputStreamReader = null;
            BufferedReader reader = null;
            try {
                inputStream = new FileInputStream(file);
                inputStreamReader = new InputStreamReader(inputStream);
                reader = new BufferedReader(inputStreamReader);
                String line = null;
                while ((line = reader.readLine()) != null) {
                    //CSV文件表格数据以逗号分隔表格数据
                    list.add(line);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                IOUtils.closeQuietly(inputStream);
                IOUtils.closeQuietly(reader);
                IOUtils.closeQuietly(inputStreamReader);
            }
        }
        sw.split();
        System.out.println("读取文件结束时间,split:" + sw.getSplitTime() + ", " + sw.toSplitString());

        int threadSize = list.size() / 10000;
        int mod = list.size() % 10000;
        if (mod != 0) {
            threadSize += 1;
        }

        ExecutorService threadPool = Executors.newFixedThreadPool(5);

        //主线程
        CountDownLatch countDown = new CountDownLatch(1);
        //子线程
        CountDownLatch await = new CountDownLatch(threadSize);
        try {

            for (int i = 0; i < threadSize; i++) {
                int st = i * 10000;
                int end = i == (threadSize - 1) ? (i * 10000) + mod : (i + 1) * 10000;
                ThreadM runnable = new ThreadM(countDown, await);
                runnable.setList(list.subList(st, end));
                runnable.setFileNname(i + ".txt");
                threadPool.execute(runnable);
            }
            threadPool.shutdown();

            //主线业务获取开始标记
            countDown.countDown();
            //等待子线业务完成
            await.await();

            sw.split();
            System.out.println("写文件结束时间,split:" + sw.getSplitTime() + ", " + sw.toSplitString());
            System.out.println("开始合并");

            //合并
            FileOutputStream fileOutputStream = new FileOutputStream("D:\\temp\\xlzx2019-08.txt");
            for (int i = 0; i < threadSize; i++) {
                FileInputStream newfileInputStream = new FileInputStream(new File("D:\\temp\\" + i + ".txt"));
                IOUtils.copy(newfileInputStream, fileOutputStream);
            }

            sw.split();
            System.out.println("合并文件结束时间,split:" + sw.getSplitTime() + ", " + sw.toSplitString());
        } catch (Exception e1) {
            e1.printStackTrace();
        }

    }
}

class TD implements Runnable {

    private List list;

    private final CountDownLatch countDown;
    private final CountDownLatch await;

    private String fileNname;

    public TD(CountDownLatch countDown, CountDownLatch await) {
        this.countDown = countDown;
        this.await = await;
    }

    public void setList(List list) {
        this.list = list;
    }

    public void setFileNname(String fileNname) {
        this.fileNname = fileNname;
    }

    @Override
    public void run() {
        try {

            //等待主线业务发起信号
            countDown.await();
            StopWatch sw = new StopWatch();
            sw.start();

            // 创建缓存区字符输出流，需要传入Write对象
            writeFile(list, fileNname);
            sw.split();
            System.out.println(Thread.currentThread().getName() + "子线程【" + fileNname + "】写文件结束时间,split:" + sw.getSplitTime() + ", " + sw.toSplitString());

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            //子线业务结束信号发出
            await.countDown();
        }
    }

    private void writeFile(List list, String fileNname) throws IOException {

        BufferedWriter out = new BufferedWriter(new FileWriter("D:\\temp\\" + fileNname));

        for (int i = 0; i < list.size(); i++) {
            JSONObject parseObject = null;
            //过滤格式异常数据
            try {
                parseObject = JSONObject.parseObject(list.get(i));
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println(list.get(i));
                continue;
            }

            //TODO    此处可添加过滤数据操作

            //输出信息
            String response = parseObject.getString("response");
            if (StringUtils.isNotBlank(response)) {

                out.write("输出处理后信息");
                out.newLine();
                out.flush();
            }
        }
        IOUtils.closeQuietly(out);

    }
}

核心的代码其实就3行       

ExecutorService threadPool = Executors.newFixedThreadPool(5);

//主线程
CountDownLatch countDown = new CountDownLatch(1);
//子线程
CountDownLatch await = new CountDownLatch(threadSize);

主要运用好CountDownLatch就可以实现对多线程处理逻辑控制。

有关多线程获取返回值的写法参考下一篇文章。