java阿里云sls基于LoghubAppender自定义日志上传

1、背景：阿里sls日志提供快捷日志平台，平替elk公司使用这个日志服务，需要对接写入日志

目前日志集成有3种

        1）基于封装manager手动写日志手动send

              弊端：本地日志和阿里云日志共用日志代码很臃肿

        2）基于云服务器日志文件写入日志平台

                弊端：日志分割太碎，一次请求分割很多条日志

        3）代码LoghubAppender写入日志平台

                弊端：日志分割太碎，一次请求分割很多条日志

2、期望实现效果：日志按请求维度发送

3、实现思路：

      日志发送在LoghubAppender里面appendEvent(E eventObject)方法里面实现了日志内组装和发送，想法是拆分日志组装和发送，先组装不发送，在请求完成后再发送

 

3、实现方式：

自定义LoghubAppender

自定义log写入类:

 

完整appendEvent方法:

private void appendEvent(E eventObject) {
        if (eventObject instanceof LoggingEvent) {
            LoggingEvent event = (LoggingEvent) eventObject;
            List logItems = new ArrayList<>();
            LogItem item = new LogItem();
            logItems.add(item);
            item.SetTime((int) (event.getTimeStamp() / 1000L));
            if (this.formatter != null) {
                DateTime dateTime = new DateTime(event.getTimeStamp());
                item.PushBack("time", dateTime.toString(this.formatter));
            } else {
                Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
                item.PushBack("time", this.formatter1.format(instant));
            }

            item.PushBack("level", event.getLevel().toString());
            item.PushBack("thread", event.getThreadName());
            StackTraceElement[] caller = event.getCallerData();
            if (caller != null && caller.length > 0) {
                item.PushBack("location", caller[0].toString());
            }

            String message = event.getFormattedMessage();
            item.PushBack("message", message);
            IThrowableProxy iThrowableProxy = event.getThrowableProxy();
            if (iThrowableProxy != null) {
                String throwable = this.getExceptionInfo(iThrowableProxy);
                throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
                item.PushBack("throwable", throwable);
            }

            if (this.encoder != null) {
                item.PushBack("log", new String(this.encoder.encode(eventObject)));
            }

            Optional.ofNullable(this.mdcFields).ifPresent(f -> event.getMDCPropertyMap().entrySet().stream()
                    .filter(v -> Arrays.stream(f.split(",")).anyMatch(i -> i.equals(v.getKey())))
                    .forEach(map -> item.PushBack(map.getKey(), map.getValue())));
            AliyunLogManager aliyunLogManager = null;
            try {
                aliyunLogManager = SpringContext.getBean(AliyunLogManager.class);
            } catch (Exception e) {
                log.info("AliyunLogManager bean has not initialized");
            }

            if (aliyunLogManager != null && MDC.get(CommonConst.LOG_TRACE_ID) != null) {
                aliyunLogManager.writeLog("log", logItemToString(logItems));
            } else {
                try {
                    this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems
                            , new LoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems));
                } catch (Exception var9) {
                    this.addError("Failed to send log, project=");
                }
            }

        }
    }

 

阿里云日志工具类: 

主要思路：写日志时候放至threadlocal中

import com.aliyun.openservices.aliyun.log.producer.LogProducer;
import com.aliyun.openservices.aliyun.log.producer.Producer;
import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException;
import com.aliyun.openservices.log.common.LogItem;
import com.google.common.collect.Lists;
import com.ty.mid.common.base.thread.TyThreadPool;
import com.ty.mid.common.base.utils.LocalDateTimeUtil;
import com.ty.mid.common.log.config.AliYunAutoConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

/**
 * @author hll
 * @date 2022/9/20
 * 阿里云日志工具类
 */
@Component
@Slf4j
public class AliyunLogManager implements InitializingBean, DisposableBean {

    @Autowired
    private AliYunAutoConfig aliYunAutoConfig;

    private Producer producer;

    public static final ThreadLocal>> LOCAL_LOG = new ThreadLocal<>();
    public static final ThreadLocal>> LOCAL_TAG = new ThreadLocal<>();

    @Override
    public void afterPropertiesSet() {
        ProducerConfig producerConfig = new ProducerConfig();
        producerConfig.setBatchSizeThresholdInBytes(3145728);
        producerConfig.setBatchCountThreshold(40960);
        producerConfig.setIoThreadCount(this.aliYunAutoConfig.getThreadCount());
        producerConfig.setTotalSizeInBytes(this.aliYunAutoConfig.getTotalSize());
        this.producer = new LogProducer(producerConfig);
        this.producer.putProjectConfig(new ProjectConfig(this.aliYunAutoConfig.getProject(), this.aliYunAutoConfig.getEndpoint()
                , this.aliYunAutoConfig.getAccessKey(), this.aliYunAutoConfig.getAccessSecret()));
    }

    @Override
    public void destroy() throws Exception {
        if (this.producer != null) {
            try {
                this.producer.close();
            } catch (ProducerException var2) {
                log.error("Failed to close producer, e=", var2);
            }
        }

    }

    public void writeLog(String title, String value) {
        List> logList = LOCAL_LOG.get();
        if (logList == null) {
            logList = new ArrayList<>();
        }

        Map logMap = new HashMap<>(1);
        logMap.put(String.format("%s_%s", LocalDateTimeUtil.getSystemDateStr(), title), value);
        (logList).add(logMap);
        LOCAL_LOG.set(logList);
    }

    public void setTag(String tagName, String tagValue) {
        List> logList = LOCAL_TAG.get();
        if (logList == null) {
            logList = new ArrayList<>();
        }

        Map logMap = new HashMap<>(1);
        logMap.put(String.format("__tag__:__%s__", tagName), tagValue);
        (logList).add(logMap);
        LOCAL_TAG.set(logList);
    }

    public void send() {
        List> logList = LOCAL_LOG.get();
        List> tagList = LOCAL_TAG.get();
        //防止日志数量过大 阿里日志拒绝写入/读取
        List>> partition = Lists.partition(CollectionUtils.isEmpty(logList) ? new ArrayList<>() : logList, 500);
        for (List> maps : partition) {
            handleSend(maps, tagList);
        }
        LOCAL_LOG.remove();
        LOCAL_TAG.remove();
    }

    private void handleSend(List> logList, List> tagList) {
        if (CollectionUtils.isNotEmpty(logList)) {
            final List> logListCopy = new ArrayList<>();
            int i = 0;
            Map map;

            for (Map log : logList) {
                map = new HashMap<>(log.size());

                for (String key : log.keySet()) {
                    StringBuilder var10001 = (new StringBuilder()).append(key).append("_");
                    ++i;
                    if (i < 10) {
                        map.put(var10001.append("0").append(i).toString(), log.get(key));
                    } else {
                        map.put(var10001.append(i).toString(), log.get(key));
                    }

                }

                logListCopy.add(map);
            }

            //tag补充
            if (CollectionUtils.isNotEmpty(tagList)) {
                for (Map log : tagList) {
                    map = new HashMap<>(log.size());

                    for (String key : log.keySet()) {
                        StringBuilder var10001 = (new StringBuilder()).append(key).append("_");
                        ++i;
                        if (i < 10) {
                            map.put(var10001.append("0").append(i).toString(), log.get(key));
                        } else {
                            map.put(var10001.append(i).toString(), log.get(key));
                        }
                    }

                    logListCopy.add(map);
                }
            }


            ThreadPool.execute(() -> {
                try {
                    this.producer.send(this.aliYunAutoConfig.getProject(), this.aliYunAutoConfig.getLogStore()
                            , this.aliYunAutoConfig.getTopic(), this.aliYunAutoConfig.getSource(), this.generateLogItem(logListCopy), result -> {
                                if (!result.isSuccessful()) {
                                    log.error("send log failed");
                                }
                            });
                } catch (Exception var2) {
                    log.error(var2.getMessage(), var2);
                }
            });
        }
    }

    private LogItem generateLogItem(List> logList) {
        LogItem logItem = new LogItem();
        Iterator> var3 = logList.iterator();
        while (var3.hasNext()) {
            Map logMap = var3.next();
            for (Object o : logMap.entrySet()) {
                Map.Entry entry = (Map.Entry) o;
                logItem.PushBack(entry.getKey(), entry.getValue());
            }
        }

        return logItem;
    }
}

基于请求维度发送日志（logback日志为例）：

/**
 * @author hll
 * @date 2022/9/27
 * logback 阿里云日志对接处理
 */
@Component
@Order(-1)
public class LogbackRequestLoggingFilter extends AbstractRequestLoggingFilter {

    @Autowired
    private AliyunLogManager aliyunLogManager;

    @Override
    protected void beforeRequest(HttpServletRequest httpServletRequest, String s) {
        //http请求标记 便于直接按请求打印日志
        aliyunLogManager.setTag("traceId", MDC.get(CommonConst.LOG_TRACE_ID));
        aliyunLogManager.setTag("url", httpServletRequest.getRequestURI());
    }

    @Override
    protected void afterRequest(HttpServletRequest httpServletRequest, String s) {
        //set send
        aliyunLogManager.send();
    }
}

写日志：log.info即可 

最终效果：