单日志管理线程实现每秒百万级日志写入机制设计与实现

在任何系统中，日志都是必不可少的组成模块。但是在不同的应用场景中，对日志也提出了不同的要求。在大数据处理的应用场景中，日志的读写追求的是性能，比如文章《每秒百万级高效C++异步日志实践》介绍了一种基于C++的高性能的日志读写类，实现了每秒125万条长度为100字节的数据写入能发。

但是，其结构略微复杂，为了进一步简化，本文设计了一种更为简单的基于双缓冲的异步读写类，本项目也达到了百万级的日志写的性能，实际测试显示，可以达到110万条每秒长度为100字节的文件写。本项目保存在Gitee.com上，地址为：https://gitee.com/hwaust/HighPerformanceLogWriter

基本原理

1. 异步操作
   使用了一个独立的线程，进行相应的日志转换和写操作。
2. 基于数组缓冲
   由于数列的随机访问性能不如数组，所以使用数组进行缓冲。
3. 双缓冲
   使用了2个数组，一个数据作为主缓冲，接收所有的输入数据；另一个线程为副缓冲，当主缓冲满了以后，立即进行切换，从而保证业务正常运行。

源代码

/// 

/// 采用双缓冲和多线程机制的高性能日志处理类。
/// 
public class LogManager
{
    /// 

    /// 用于保存日志的文件。
    /// 
    public string LogPath { get; private set; }

    /// 

    /// 日志保存时间，默认为10毫秒。
    /// 
    public int SaveInterval { get; set; }

    /// 

    /// 缓冲大小，默认为100K个日志记录。
    /// 
    public int BufferSize { get; set; }

    // 用于控制线程启停
    bool isRunning = false;

    // 缓冲
    object[] buffer;

    // 待写入保存数据
    object[] toBeWritten;

    // 缓冲索引位置.
    int bufferIndex = 0;


    // 主线程
    Thread thread;

    /// 

    /// Construct a LogManager object.
    /// 
    public LogManager()
    {
        Initialize();
        File.Delete(LogPath);
    }

    /// 

    /// Construct a LogManager object.
    /// 
    /// The file path to store logs.
    /// Writing intervl.
    public LogManager(string logPath, int bufferSize = 200000, int saveInterval = 1)
    {
        Initialize(logPath, bufferSize, saveInterval);
    }

    private void Initialize(string logpath = "system.log", int bufferSize = 200000, int saveInterval = 1)
    {
        LogPath = logpath;
        BufferSize = bufferSize;
        SaveInterval = saveInterval;
        buffer = new object[BufferSize];
        bufferIndex = 0;
    }

    /// 

    /// 添加日志
    /// 
    /// 日志记录
    public void Add(object log)
    {
        lock (this)
        {
            if (bufferIndex >= buffer.Length)
            {
                SwichBuffer();
                Thread.Sleep(1);
            }

            buffer[bufferIndex++] = log;
        }
    }

    private void SwichBuffer()
    {
        while (isSaving)
            Thread.Sleep(0);

        toBeWritten = buffer;
        buffer = new object[BufferSize];
        bufferIndex = 0;
    }

    int count = 0;
    bool isSaving = false;

    /// 

    /// 对数据进行保存
    /// 
    public void Save()
    {
        // Obtain all logs and release lock
        if (toBeWritten == null || toBeWritten.Length == 0)
            return;

        // append new logs into log file.
        StringBuilder sb = new StringBuilder(BufferSize * 1024);
        isSaving = true;
        foreach (object log in toBeWritten)
        {
            if (log == null)
                break;
            count++;
            sb.AppendLine(log.ToString());
        }
        Console.WriteLine(count);
        toBeWritten = null;
        File.AppendAllText(LogPath, sb.ToString(), Encoding.UTF8);
        isSaving = false;
    }

    /// 

    /// main method to process logs.
    /// 
    private void Process()
    {
        isRunning = true;
        while (isRunning)
        {
            Save();
            Thread.Sleep(0);
        }
        SwichBuffer();
        Save();
        System.Console.WriteLine("Done. count = " + count);
    }

    /// 

    /// Starts thread of this log manager. 
    /// 
    public void Start()
    {
        if (thread != null)
            return;

        thread = new Thread(Process);
        thread.Start();
    }

    /// 

    /// Stops the thread of this log manager. 
    /// 
    public void Stop()
    {
        isRunning = false;
        Thread.Sleep(10);
    } 
}

Log类

    public class Log
    {
        /// 

        /// The occuring time.
        /// 
        public DateTime Time { get; set; }


        static Random random = new Random();
        static string[] types = { "LOG", "DBG", "SYS", "CRT", "ERR", "MSG" };
  
        public static Log Random()
        {
            return new Log(random.Next(10, 99), types[random.Next(types.Length)], "The content of the log.123456789012345678901");
        }

        /// 

        /// The type of the log.
        /// 
        public int LogType { get; set; }

        /// 

        /// Arguements of a log.
        /// 
        public string[] Arguments { get; set; }


        // Used to separate data fields of a log when serializing.
        char separator = (char)6;

        /// 

        /// Construct a Log object.
        /// 
        /// The type of log.
        /// Argument list.
        public Log(int logType, params object[] args)
        {
            Time = DateTime.Now;
            LogType = logType;
            Arguments = new string[args == null ? 0 : args.Length];
            for (int i = 0; i < Arguments.Length; i++)
                Arguments[i] = args[i] == null ? "" : args[i].ToString();
        }

        /// 

        /// Format: yyyyMMddHHmmss|LogType|Arguments
        /// 
        /// 
        public override string ToString()
        {
            StringBuilder sb = new StringBuilder();
            sb.Append((int)(Time - new DateTime(2019, 1, 1)).TotalSeconds);
            sb.Append(separator);
            sb.Append((int)LogType);
            sb.Append(separator);
            sb.Append(string.Join(separator.ToString(), Arguments));
            return sb.ToString();
        }

        /// 

        /// Parses a string into a Log object.
        /// 
        /// Input string.
        /// 
        public Log Parse(string str)
        {

            return null;
        }
    }

测试代码

        static void Main(string[] args)
        {
            DateTime t1 = DateTime.Now;
            Log[] logs = new Log[1024 * 1024];
            for (int i = 0; i < logs.Length; i++)
                logs[i] = Log.Random();
            DateTime t2 = DateTime.Now;
            Console.WriteLine("Logs.Length = " + logs.Length);
            Console.WriteLine("logs[0].Length = " + logs[0].ToString().Length);
            Console.WriteLine("logs[0].ToString() = " + logs[0].ToString());

            DateTime t3 = DateTime.Now;
            LogManager lm = new LogManager();
            lm.Start(); 
            for (int times = 0; times < 10; times++)
                for (int i = 0; i < logs.Length; i++)
                    lm.Add(logs[i]);

            lm.Stop();
            DateTime t4 = DateTime.Now; 
            Console.WriteLine("Init: " + (t2 - t1).TotalMilliseconds.ToString("0.00ms"));
            Console.WriteLine("Save: " + (t4 - t3).TotalMilliseconds.ToString("0.00ms")); 
        }

测试结果

测试使用了两台电脑：
机器2: 桌面电脑 i5 9600K(6T6C) 32GB DDR4 2166 512GB SSD
机器3: 桌面电脑 i3 9100(4T4C) 16 GB DDR4 2166 512GB SSD

测试结果如下表所示：

项目	机器2	机器3
1000万条写用时	9.394s	9.366s
1000万条写速度	106万条/秒	106.8万条/秒

Logs.Length = 1048576 x 10
logs[0].Length = 60
Init: 530.61ms
Save: 7714.36ms
130万条 60字节长度数据写入，文件大小 ：620MB

Logs.Length = 1048576 x 10
logs[0].Length = 100
Init: 527.56ms
Save: 9394.87ms
106万条 100字节长度数据写入，文件大小 ：1020MB

2019-7-29 11:54:33 测试结果
Log File: C:\Gitee\HighPerformanceLogWriter\HighPerformanceLog\bin\Debug\system.log
 # of Logs: 10485760
Content of logs[0]: 1810044487197CRTThe content of the log.1234567890123456789012345678901234567890123456789012345678
Length of logs[0]: 100
.........................................................................................................Done. count = 10485760
Init: 0.002s
Save: 9.366s
Speed: 1.119 M/s
Log Size: 2002.107 MB
请按任意键继续. . .
【i3 9100】测试结果
Init: 536.59ms
Save: 10782.14ms