Springboot整合Prometheus+Grafana自定义指标监控

加入依赖

  

        
            org.springframework.boot
            spring-boot-starter-actuator
        
        
            io.micrometer
            micrometer-registry-prometheus
        

配置文件，这里特别说明，要去掉db健康状态检测，感觉是版本不对，导致包依赖冲突，所以要去掉db,就正常，还有其它禁用的方式，如果不需要都可以 禁掉

management.health.mail.enabled
management.health.mongo.enabled
management.health.redis.enabled
management.health.solr.enabled
management.health.elasticsearch.enabled

# spring config
spring:
  application:
    name: dev-aiops
management:
  endpoints:
    web:
      exposure:
        include: "*"
  metrics:
    tags:
      dev-aiops: dev-aiops
  health:
    defaults:
      enabled: true
    db:
      enabled: false
    show-details: always

加载核心类，应用开启P90,P99统计

package com.sevenXnetworks.colossus.config;

import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.time.Duration;

/**
 * @author sky
 */
@Configuration
public class PrometheusInfo {

    // 注意，这个是注册的核心代码块

 /*   @Bean
    MeterRegistryCustomizer registry(@Value("${spring.application.name}") String applicationName) {
        return (registry) -> registry.config().commonTags("application", applicationName);
    }
    */

    @Bean
    MeterRegistryCustomizer metricsCommonTags(@Value("${spring.application.name}") String applicationName) {
        return registry -> {
            registry.config().meterFilter(
                    new MeterFilter() {
                        @Override
                        public DistributionStatisticConfig configure(@NotNull Meter.Id id, @NotNull DistributionStatisticConfig config) {
                            //匹配http开头并且是timer类型的监控指标
                            if (id.getType() == Meter.Type.TIMER & id.getName().matches("^(http){1}.*")) {
                                return DistributionStatisticConfig.builder()
                                        .percentilesHistogram(true)
                                        .percentiles(0.5, 0.90, 0.95, 0.99)
                                        .sla(Duration.ofMillis(50).toNanos(),
                                                Duration.ofMillis(100).toNanos(),
                                                Duration.ofMillis(200).toNanos(),
                                                Duration.ofSeconds(1).toNanos(),
                                                Duration.ofSeconds(5).toNanos())
                                        .minimumExpectedValue(Duration.ofMillis(1).toNanos())
                                        .maximumExpectedValue(Duration.ofSeconds(5).toNanos())
                                        .build()
                                        .merge(config);
                            } else {
                                return config;
                            }
                        }
                    }).commonTags("application", applicationName);
        };
    }

}

自定义的指标

package com.sevenXnetworks.colossus.config;

import com.sevenXnetworks.colossus.interceptor.UserInterceptor;
import com.sevenXnetworks.colossus.vo.SecurityUserVo;
import io.micrometer.core.instrument.Metrics;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import javax.servlet.http.HttpServletRequest;
import java.time.LocalDate;
import java.util.concurrent.TimeUnit;

/**
 * @author sky
 * date 2022-11-22 13:30
 */
@Component
@Aspect
public class PrometheusAPICounter {

    private final Log logger = LogFactory.getLog(this.getClass());

    @Autowired
    UserInterceptor userInterceptor;

    @Pointcut("execution(public * com.sevenXnetworks.colossus.controller.*.*(..))")
    public void pointCut() {

    }

    @Around("pointCut()")
    public Object metricsCollector(ProceedingJoinPoint joinPoint) throws Throwable {
        SecurityUserVo localUser = userInterceptor.getLocalUser();
        HttpServletRequest request = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest();
        String userName = localUser.getUserName() != null ? localUser.getUserName() : "no userId";//StringUtils.hasText( request.getParameter("userId"))?request.getParameter("userId"):"no userId";

        // 获取api url
        String api = request.getServletPath();
        // 获取请求方法
        String method = request.getMethod();
        long timeMillis = System.currentTimeMillis();
        LocalDate now = LocalDate.now();
        String[] tags = new String[8];
        tags[0]="api";
        tags[1] = api;
        tags[2]="method";
        tags[3]=method;
        tags[4]="date";
        tags[5]=now.toString();
        tags[6]="userName";
        tags[7]=userName;
        // 请求次数加1
        //自定义的指标名称：http_request_aiops_all，指标包含数据
        Metrics.counter("http_request_aiops_all",tags).increment();
        Object object;
        try {
            object = joinPoint.proceed();
        } catch (Exception e) {
            // 请求失败次数加1
            Metrics.counter("http_request_aiops_error",tags).increment();
            throw e;
        } finally {
            long f = System.currentTimeMillis();
            long l = f - timeMillis;
            //记录请求响应时间
            Metrics.timer("http_request_aiops_time", tags).record(l, TimeUnit.MILLISECONDS);
        }
        return object;
    }

}

 在grafana面板增加自己写的指标名，统计

每分钟请求总数
60*sum(rate(http_request_test_all_total{}[1m]))

统计自定义标签api=/test/user

60*sum(rate(http_request_test_all_total{api="/test/user"}[1m]))

请求平均耗时
(sum(http_request_test_time_seconds)/sum(http_request_test_all_total))

P90 P99指标统计
avg(http_request_test_time_seconds{job="prometheus-test" ,quantile =~ "0.9|0.99"}) by (job,quantile)

最大耗时
max(http_request_test_time_seconds_max{job="prometheus-test"})

如图所示