Prometheus Alertmanager Webhook 自定义告警

Build Webhook server

cat app.py

import json
from flask import Flask, request
from ciscosparkapi import CiscoSparkAPI
team_token=‘ZDQ0NTY2N2UtNTU2ZC00YzdjLWE0MjMtNTM3NzcyODFjNzU4OGEyNWEwODMtYmNm_PF84_1eb65fdf-9643-417f-9974-ad72cae0e10f’
api = CiscoSparkAPI(access_token=team_token)
team_room_id=‘xxx’

def send_message(text):
text=text
api.messages.create(team_room_id, markdown=text,mentionedPeople=‘Alex Ju’)

app = Flask(name)
@app.route(’/send’, methods=[‘POST’])
def send():
try:
data=json.loads(request.data)
alerts=data[‘alerts’]
for i in alerts:
status=str(i[‘status’])
if status==‘firing’:
emoji=‘’ #red apple
else:
emoji=‘’ #green apple
labels=i[‘labels’]
severity=str(labels[‘severity’])
alertname=str(labels[‘alertname’])
value=str(i[‘annotations’][‘value’])
labels.pop(‘severity’)
labels.pop(‘alertname’)
metric_labels=json.dumps(labels)
metric_labels=metric_labels.replace(’"’,’’)
metric_labels=metric_labels.replace(’: ‘,’=’)
message=emoji+’ ‘+severity.capitalize()+’’+’ - ‘+’’+alertname+’’+’ Metric Labels: ‘+metric_labels+’’+’ Value: ‘+value+’’
send_message(message)
except Exception as e:
print(e)
return ‘ok’

cat run.sh

#!/bin/bash
export FLASK_APP=app.py
flask run -h 0.0.0.0
chmod a+x run.sh

Build Docker image

cat Dockerfile

FROM python
RUN pip install flask
RUN pip install ciscosparkapi
COPY app.py /app.py
COPY run.sh /run.sh
RUN chmod +x /run.sh

EXPOSE 5000

ENTRYPOINT ["/run.sh"]

docker build -t alexju1988/prometheus-webhook:1.6 .
docker push alexju1988/prometheus-webhook:1.6 # please configure docker registry first.

Build K8S Deploy and Service

kubectl create -f prometheus-webhook.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: prometheus-webhook
name: prometheus-webhook
namespace: dev
spec:
podManagementPolicy: OrderedReady
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: prometheus-webhook
serviceName: “”
template:
metadata:
creationTimestamp: null
labels:
app: prometheus-webhook
spec:
containers:
- env:
name: prometheus-webhook
image: alexju1988/prometheus-webhook:1.6
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5000
name: webhook
protocol: TCP
resources:
limits:
cpu: “1”
memory: 1Gi
requests:
cpu: 50m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
updateStrategy:
rollingUpdate:
partition: 0
type: RollingUpdate

kubectl create -f prometheus-webhook-web.yaml

apiVersion: v1
kind: Service
metadata:
labels:
app: prometheus-webhook
name: prometheus-webhook-web
namespace: dev
spec:
externalTrafficPolicy: Cluster
ports:

• name: webhook-http
  nodePort: 31009
  port: 5000
  protocol: TCP
  targetPort: 5000
  selector:
  app: prometheus-webhook
  sessionAffinity: None
  type: NodePort

Configure Alertmanager

kubectl apply -f ps-prometheus-alertmanager-cm.yaml

apiVersion: v1
data:
alertmanager.yml: |
global:
resolve_timeout: 1m
smtp_smarthost: ‘cu-mail.i.jaspersystems.com:25’
smtp_from: ‘[email protected]’
smtp_auth_username: ‘[email protected]’
smtp_auth_password: ‘’
smtp_require_tls: false
route:
receiver: ‘webhook’
group_by: [‘alertname’]
group_wait: 1m
group_interval: 1m
repeat_interval: 1d

  routes:
  - match:
      severity: 'warning'
    receiver: 'webhook'

  - match:
      severity: 'critical'
    receiver: 'webhook'

receivers:

- name: 'ps-all'
  email_configs:
  - to: '[email protected], [email protected], [email protected], [email protected], [email protected], [email protected]'
    send_resolved: true

- name: 'webhook'
  webhook_configs:
  - send_resolved: true
    http_config: {}
    url: 'http://cn-k8s-master01:31009/send'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

kind: ConfigMap
metadata:
creationTimestamp: “2020-05-08T00:41:27Z”
labels:
app: prometheus
chart: prometheus-10.5.1
component: alertmanager
heritage: Tiller
release: ps
name: ps-prometheus-alertmanager
namespace: dev