修改prometheus实现数据库存储报警规则和收集目标
从数据库获取报警规则和服务目标组
prometheus本身报警规则及服务发现策略基于文件配置很不方便，对于非K8S服务监控经常需要操作配置文件，不利于管理系统平台化建设。
实现思路：将相关配置信息存储在MySQL里，加入新的逻辑，实现保留文件加载配置的同时，加载MySQL中的信息，
动态生成static_config及alert_rule从而实现报警及监控目标的配置UI化.
MySQL配置
使用以下环境变量定义MySQL元信息
MYSQL_HOST #主机名/ip
MYSQL_PORT #端口
MYSQL_USER #用户名
MYSQL_PWD #密码
MYSQL_DB #数据库名
因为使用gorm实现，对于代码需要引入依赖
"github.com/jinzhu/gorm"
_ "github.com/go-sql-driver/mysql"
表结构定义
报警规则表结构定义
type AlertRule struct {
ID uint `gorm:"primary_key"`
CreatedAt time.Time
UpdatedAt time.Time
Group string `json:"group,omitempty"` // 存储报警规则的组
Alert string `json:"alert,omitempty"` // 存储报警规则的名称
Expr string `json:"expr,omitempty"` // 存储报警规则的表达式
For string `json:"for,omitempty"` // 存储报警规则的延迟时间
Labels JSON `sql:"type:json" json:"labels,omitempty"`// 存储报警规则的label
Annotations JSON `sql:"type:json" json:"annotations,omitempty"` // 存储报警规则的注释
}
func (self *AlertRule) TableName() string {
return "alert_rule"
}
type JSON []map[string]string
func (c JSON) Value() (driver.Value, error) {
b, err := json.Marshal(c)
return string(b), err
}
func (c *JSON) Scan(input interface{}) error {
return json.Unmarshal(input.([]byte), c)
}
代码逻辑
github.com/prometheus/prometheus/rules/manager.go LoadGroups方法 返回前加入以下代码
//出事连接
var rules []AlertRule
db_url:=fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",os.Getenv("MYSQL_USER"),os.Getenv("MYSQL_PWD"),os.Getenv("MYSQL_HOST"),os.Getenv("MYSQL_PORT"),os.Getenv("MYSQL_DB"))
db,err:=gorm.Open("mysql", fmt)
if err!=nil {
level.Error(m.logger).Log( "init db error: ",err)
}else {
level.Info(m.logger).Log("get db connection success")
}
//获取报警条目
err=db.Model(&AlertRule{}).Scan(&rules).Error
if err!=nil {
level.Error(m.logger).Log(err)
}else {
level.Info(m.logger).Log("get rules success")
}
defer func(db *gorm.DB) {
err:=db.Close()
if err!=nil {
level.Error(m.logger).Log("close db error: ",err)
}
}(db)
arules := make(map[string][]Rule)
for _,v:=range rules {
expr, err := promql.ParseExpr(v.Expr)
if err != nil {
return nil, []error{err}
}
var lbs=labels.Labels{labels.Label{
Name: "product",
Value: v.Group,
}}
for _,v:=range v.Labels {
lbs=append(lbs,labels.Label{
v["key"],v["value"],
})
}
var ans=labels.Labels{}
for _,v:=range v.Annotations {
ans=append(ans,labels.Label{
v["key"],v["value"],
})
}
arules[v.Group]= append(arules[v.Group], NewAlertingRule(v.Alert,expr,interval,lbs,ans,nil,shouldRestore,log.With(m.logger, "alert", v.Alert)))
}
level.Info(m.logger).Log(arules)
for k,v:=range arules {
rules:=make([]Rule,0,len(v))
rules=v[0:len(v)]
groups[groupKey(k,"rule.yaml")]=NewGroup(k,"rule.yaml",interval, rules,shouldRestore,m.opts)
}
job及targer代码
从数据库获取信息生成static config配置
结构定义
type AlertSdRule struct {
ID uint `gorm:"primary_key"`
CreatedAt time.Time `json:"create_at"`
UpdatedAt time.Time `json:"update_at"`
Target string `gorm:"type:varchar(200);not null" json:"target"`
Labels JSON `sql:"type:json" json:"type"`
GroupId int `gorm:"type:integer;not null" json:"group_id"`
Describe string `gorm:"type:varchar(200);not null" json:"describe"`
}
func (self *AlertSdRule) TableName() string {
return "alert_sd_rule"
}
type AlertSdGroup struct {
ID uint `gorm:"primary_key"`
CreatedAt time.Time `json:"create_at"`
UpdatedAt time.Time `json:"update_at"`
Name string `gorm:"type:varchar(200);not null" json:"name"`
MetricsPath string `gorm:"type:varchar(200);not null" json:"metrics_path"`
Labels JSON `sql:"type:json" json:"labels"`
Scheme string `gorm:"type:varchar(200);not null" json:"scheme"`
Interval int `gorm:"type:integer;not null" json:"interval"`
Timeout int `gorm:"type:integer;not null" json:"timeout"`
Describe string `gorm:"type:varchar(200);not null" json:"describe"`
AuthUser string `gorm:"type:varchar(200)" json:"auth_user"`
AuthPassword string `gorm:"type:varchar(200)" json:"auth_password"`
AuthToken string `gorm:"type:varchar(200)" json:"auth_token"`
}
func (self *AlertSdGroup) TableName() string {
return "alert_sd_group"
}
type JSON []map[string]string
func (c JSON) Value() (driver.Value, error) {
b, err := json.Marshal(c)
return string(b), err
}
func (c *JSON) Scan(input interface{}) error {
return json.Unmarshal(input.([]byte), c)
}
代码逻辑
main.go reloadConfig中
//初始化数据库连接
db_url:=fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",os.Getenv("MYSQL_USER"),os.Getenv("MYSQL_PWD"),os.Getenv("MYSQL_HOST"),os.Getenv("MYSQL_PORT"),os.Getenv("MYSQL_DB"))
conn, err := gorm.Open("mysql", db_url)
if err != nil {
panic(err)
}
var groups []AlertSdGroup
// 获取所有的JOB
err = conn.Model(&AlertSdGroup{}).Scan(&groups).Error
if err != nil {
panic(err)
}
for _, group := range groups {
var rules []AlertSdRule
// 获取指定job缩包含的targets
err = conn.Model(&AlertSdRule{}).Where("group_id = ?", group.ID).Scan(&rules).Error
if err != nil {
panic(err)
}
var targetgroups = []*targetgroup.Group{}
for _, rule := range rules {
var labels model.LabelSet = map[model.LabelName]model.LabelValue{}
for _, label := range group.Labels {
key := model.LabelName(label["key"])
value := model.LabelValue(label["value"])
labels[key] = value
}
for _, label := range rule.Labels {
key := model.LabelName(label["key"])
value := model.LabelValue(label["value"])
labels[key] = value
}
targetgroups = append(targetgroups, &targetgroup.Group{
Targets: []model.LabelSet{
model.LabelSet{
"__address__": model.LabelValue(rule.Target),
},
},
Labels: labels,
Source: fmt.Sprint(rule.ID),
})
}
//生成对应组的scrapeConfig
scrapeConfig := &config.ScrapeConfig{
JobName: group.Name,
Scheme: group.Scheme,
ScrapeInterval: model.Duration(time.Duration(group.Interval) * time.Second),
ScrapeTimeout: model.Duration(time.Duration(group.Timeout) * time.Second),
MetricsPath: group.MetricsPath,
HTTPClientConfig: config_util.HTTPClientConfig{
TLSConfig: config_util.TLSConfig{
InsecureSkipVerify: true,
},
},
ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
StaticConfigs: targetgroups,
},
}
//生成对应组的scrapeConfig的认证方式
if group.AuthUser != "" && group.AuthPassword != "" {
scrapeConfig.HTTPClientConfig.BasicAuth = &config_util.BasicAuth{
Username: group.AuthUser,
Password: config_util.Secret(group.AuthPassword),
}
} else if group.AuthToken != "" {
scrapeConfig.HTTPClientConfig.BearerToken = config_util.Secret(group.AuthToken)
}
conf.ScrapeConfigs = append(conf.ScrapeConfigs, scrapeConfig)
}