用ECharts绘制Prometheus图表，实现类似Grafana的自定义Dashboard

  大家一般都是用Grafana自定义Dashboard来监控Prometheus数据的，作者这次尝试用ECharts来绘制Prometheus数据图表，一方面可以减少依赖，另一方面可以将监控界面灵活的集成进应用系统。至于如何在被监测机器上安装NodeExporter以及如何部署Prometheus作者就不描述了，园子里有很多文章介绍。

一、数据查询及转换

  Prometheus提供了Http Api来执行promql查询，但需要将返回的数据格式转换为ECharts的格式，好在EChars的xAxis.type可以设置为’time’类型，与Prometheus返回的格式接近。作者写了个简单的服务来执行查询及转换数据，详见以下代码：

public class MetricService
{
    private static readonly HttpClient http = new HttpClient()
    {
        //请修改指向Prometheus地址
        BaseAddress = new Uri("http://10.211.55.2:9090/api/v1/"),
        Timeout = TimeSpan.FromSeconds(2)
    };

    public async Task<object> GetCpuUsages(string node, DateTime start, DateTime end)
    {
        var promql = $"100-irate(node_cpu{{instance='{node}:9100',mode='idle'}}[5m])*100";
        return await QueryRange(promql, start, end, 20, 2);
    }

    public async Task<object> GetMemUsages(string node, DateTime start, DateTime end)
    {
        var promql = $"(1-(node_memory_MemAvailable{{instance='{node}:9100'}}/(node_memory_MemTotal{{instance='{node}:9100'}})))*100";
        return await QueryRange(promql, start, end, 20, 2);
    }

    public async Task<object> GetNetTraffic(string node, DateTime start, DateTime end)
    {
        var downql = $"irate(node_network_receive_bytes{{instance='{node}:9100',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}}[5m])";
        var ls = await QueryRange(downql, start, end, 15/*4*/, 0);
        var upql = $"irate(node_network_transmit_bytes{{instance='{node}:9100',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}}[5m])";
        ls.Add(await QueryRange(upql, start, end, 15/*4*/, 0));
        return ls;
    }

    public async Task<object> GetDiskIO(string node, DateTime start, DateTime end)
    {
        var readql = $"irate(node_disk_bytes_read{{instance='{node}:9100'}}[1m])";
        var ls = await QueryRange(readql, start, end, 15/*10*/, 0);
        var writeql = $"irate(node_disk_bytes_written{{instance='{node}:9100'}}[1m])";
        ls.Add(await QueryRange(writeql, start, end, 15/*10*/, 0));
        return ls;
    }

    #region ====Parse PromQL====
    private static async Task<List<object>> QueryRange(string promql, DateTime start, DateTime end, int step, int round)
    {
        if (start >= end) throw new ArgumentOutOfRangeException();
        var ts1 = (int)(start.ToUniversalTime() - DateTime.UnixEpoch).TotalSeconds;
        var ts2 = (int)(end.ToUniversalTime() - DateTime.UnixEpoch).TotalSeconds;
        var res = await http.GetAsync($"query_range?query={promql}&start={ts1}&end={ts2}&step={step}s");
        var stream = await res.Content.ReadAsStreamAsync();
        using (var sr = new System.IO.StreamReader(stream))
        using (var jr = new JsonTextReader(sr))
        {
            return ParseToSeries(jr, round);
        }
    }

    private static List<object> ParseToSeries(JsonTextReader jr, int round)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "status")
            throw new Exception();
        var status = jr.ReadAsString();
        if (status != "success") throw new Exception();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "data")
            throw new Exception();

        if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "resultType")
            throw new Exception();
        var resultType = jr.ReadAsString();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "result")
            throw new Exception();

        return ReadResultArray(jr, round);
        //No need read others
    }

    private static List<object> ReadResultArray(JsonTextReader jr, int round)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartArray) throw new Exception();

        var ls = new List<object>();
        do
        {
            if (!jr.Read()) throw new Exception();
            if (jr.TokenType == JsonToken.EndArray) break;
            if (jr.TokenType != JsonToken.StartObject) throw new Exception();
            ls.Add(ReadResultItem(jr, round));
        } while (true);
        return ls;
    }

    private static List<double[]> ReadResultItem(JsonTextReader jr, int round)
    {
        //已读取StartObject标记
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "metric")
            throw new Exception();
        ReadMetric(jr);

        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "values")
            throw new Exception();
        var values = ReadValues(jr, round);
        if (!jr.Read() || jr.TokenType != JsonToken.EndObject) throw new Exception();
        return values;
    }

    private static void ReadMetric(JsonTextReader jr)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
        do
        {
            //PropertyName or EndObject
            if (!jr.Read()) throw new Exception();
            if (jr.TokenType == JsonToken.EndObject) return;
            //PropertyValue
            jr.Read();
        } while (true);
    }

    private static List<double[]> ReadValues(JsonTextReader jr, int round)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartArray) throw new Exception();

        var ls = new List<double[]>();
        do
        {
            if (!jr.Read()) throw new Exception();
            if (jr.TokenType == JsonToken.EndArray) break;
            if (jr.TokenType != JsonToken.StartArray) throw new Exception();
            var ts = jr.ReadAsDouble().Value * 1000; //PromQL时间*1000
            var value = Math.Round(double.Parse(jr.ReadAsString()), round, MidpointRounding.ToEven); //PromQL值为字符串
            ls.Add(new double[] { ts, value });
            if (!jr.Read() || jr.TokenType != JsonToken.EndArray) throw new Exception();
        } while (true);
        return ls;
    }
    #endregion

}

Tip: promql的写法可参考grafana网站相关Dashboard。

二、单指标Vue组件

  作者使用Vue-ECharts作为ECharts的包装，以CPU使用率Vue组件为例：

<v-chart theme="dark" autoresize :options="chartOptions" style="height:250px">
v-chart>

@Component
export default class CpuUsages extends Vue {
    /** 目标实例IP */
    @Prop({ type: String, default: '10.211.55.3' }) node
    /** 开始时间 */
    @Prop({ type: Date, default: () => { var now = new Date(); return new Date(now.getFullYear(), now.getMonth(), now.getDate()) } }) start
    /** 结束时间 */
    @Prop({ type: Date, default: () => { return new Date() } }) end

    chartOptions = {
        title: { text: 'Cpu Usages', x: 'center' },
        tooltip: { trigger: 'axis' },
        xAxis: { type: 'time' },
        yAxis: { min: 0, max: 100 },
        series: []
    }

    refresh() {
        sys.Services.MetricService.GetCpuUsages(this.node, this.start, this.end).then(res => {
                this.chartOptions.series.splice(0)
                for (var i = 0; i < res.length; ++i) {
                    var seria = { type: 'line', name: 'cpu' + i, data: res[i], showSymbol: false }
                    this.chartOptions.series.push(seria)
                }
            }).catch(err => {
                this.$message(err)
            })
    }

    mounted() {
        this.refresh()
    }
}

三、组合多个组件形成Dashboard

  根据需要可以灵活组合多个指标组件，形成相应的Dashboard界面（如下图所示）。

四、小结

  感谢Vue、ECharts、Vue-ECharts、Prometheus等项目，使得开发并集成监控Dashboard如此简单。另码文不易，码技术文更不易，所以请您多多推荐！