java多线程读取日志用于前端展示

最近公司有这样的业务场景, 我们有一个人工智能算法平台系统, 用户可以选择自己需要的模型, 利用我们的gpu资源来训练, 训练的过程中, 日志就是非常重要的一环了, 需要把TensorFlow或者kares训练的日志实时的展示到前端界面上, 用于让用户感知到训练过程.

大致的思路是前端跟java的服务器采用websocket通信, 后台检测到日志有变化的时候, 就主动把增量的日志文件推送到前端显示.


在java服务中,考虑需要对日志文件做轮询, 就想到的spring的ThreadPoolTaskScheduler, 创建一个定时任务的线程池, 配置为每5秒执行一次, 检测文件行数是否有变化, 如果文件行数有变化则返回增量数据, 并记录下目前已读取的行数.

service 代码如下:

@Component
public class TaskService {

private final ThreadPoolTaskScheduler taskScheduler;

private final SimpMessagingTemplate messagingTemplate;

private ScheduledFuture future;

private Map futureMap = new HashMap();
public TaskService(
    ThreadPoolTaskScheduler taskScheduler,
    SimpMessagingTemplate messagingTemplate) {
    this.taskScheduler = taskScheduler;
    this.messagingTemplate = messagingTemplate;
}


public void startReadLog(String jobPath){

    future = taskScheduler.schedule(new ReadLogThread(this, jobPath, messagingTemplate),new CronTrigger("*/5 * * * * ?"));

    futureMap.put(jobPath,future);
}


public void stopReadLog(String jobPath){
    ScheduledFuture temp = (ScheduledFuture) futureMap.get(jobPath);

    if(temp!=null){
        temp.cancel(true);
    }
}
}

thread 代码如下:

public class ReadLogThread implements Runnable {

private final TaskService taskService;

private final String jobPath;

private String charset;

private final SimpMessagingTemplate messagingTemplate;

private static final String PREAMBLE_STR = "\u001B[8mha:";
private static final String POSTAMBLE_STR = "\u001B[0m";

public ReadLogThread(TaskService taskService, String jobPath,
    SimpMessagingTemplate messagingTemplate) {
    this.taskService = taskService;
    this.jobPath = jobPath;
    this.messagingTemplate = messagingTemplate;
}


@Override
public void run() {
    try {
        List logs;
        if (TaskService.pageMap.containsKey(jobPath)) {
            long currentLine = (long) TaskService.pageMap.get(jobPath);

            logs = getLog(currentLine);

        } else {

            logs = getLog(0);

        }

        messagingTemplate.convertAndSend("/logs/" + jobPath, logs);

    } catch (IOException e) {
        e.printStackTrace();
    }

    System.out.println("DynamicTask.MyRunnable.run()，" + new Date());

}


    int lines = 0;
    long filePointer;
    final List lastLines = new ArrayList<>(128);
    final List bytes = new ArrayList<>();

    try (RandomAccessFile fileHandler = new RandomAccessFile(getLogFile(), "r")) {
        long fileLength = fileHandler.length() - 1;

        for (filePointer = fileLength; filePointer >maxLines;filePointer--) {
            fileHandler.seek(filePointer);
            byte readByte = fileHandler.readByte();

            if (readByte == 0x0A) {
                if (filePointer < fileLength) {
                    lines = lines + 1;
                    lastLines.add(convertBytesToString(bytes));
                    bytes.clear();
                }
            } else if (readByte != 0xD) {
                bytes.add(readByte);
            }
        }
        TaskService.pageMap.put(jobPath,fileLength);
    }

    if (lines != maxLines) {
        lastLines.add(convertBytesToString(bytes));
    }


    Collections.reverse(lastLines);

  

    return removeNotes(lastLines);
}

public File getLogFile() throws IOException {
    File rawF = new File(getRootDir(), "training.log");
    if (rawF.isFile()) {
        return rawF;
    }
    File gzF = new File(getRootDir(), "log.gz");
    if (gzF.isFile()) {
        return gzF;
    }
    //If both fail, return the standard, uncompressed log file
    return rawF;
}

public File getRootDir() throws IOException {
    return new File(getJobPath());
}

private String getJobPath() throws IOException {
    return new String(new sun.misc.BASE64Decoder().decodeBuffer(jobPath));
}

public static String humanReadableByteSize(long size) {
    String measure = "B";
    if (size < 1024) {
        return size + " " + measure;
    }
    Double number = new Double(size);
    if (number >= 1024) {
        number = number / 1024;
        measure = "KB";
        if (number >= 1024) {
            number = number / 1024;
            measure = "MB";
            if (number >= 1024) {
                number = number / 1024;
                measure = "GB";
            }
        }
    }
    DecimalFormat format = new DecimalFormat("#0.00");
    return format.format(number) + " " + measure;
}

private String convertBytesToString(List bytes) {
    Collections.reverse(bytes);
    Byte[] byteArray = bytes.toArray(new Byte[bytes.size()]);
    return new String(ArrayUtils.toPrimitive(byteArray), getCharset());
}


public final Charset getCharset() {
    if (charset == null) {
        return Charset.defaultCharset();
    }
    return Charset.forName(charset);
}

public static List removeNotes(Collection logLines) {
    List r = new ArrayList(logLines.size());
    for (String l : logLines) {
        r.add(removeNotes(l));
    }
    return r;
}

/**
 * Removes the embedded console notes in the given log line.
 *
 * @since 1.350
 */
public static String removeNotes(String line) {
    while (true) {
        int idx = line.indexOf(PREAMBLE_STR);
        if (idx < 0) {
            return line;
        }
        int e = line.indexOf(POSTAMBLE_STR, idx);
        if (e < 0) {
            return line;
        }
        line = line.substring(0, idx) + line.substring(e + POSTAMBLE_STR.length());
    }
}


}