1、部署相关文件已经打成tar包上传至云，链接: https://pan.baidu.com/s/1NflGeaYCWPi9243gM4gGdw
提取码：1234
使用tar包注意：
1、报警目录alert拷贝至data目录下
2、给node打标签确定下prometheus的node，标签nodetype=prometheus
[root@2dot241 sy] kubectl label node 2dot239 nodetype=prometheus
因想要让数据持久化存储，所以固定了prometheus所在的node 这个参数nodeSelector:所以要给相应的node上打个标签
[root@2dot239 ~] mkdir -p /data/prometheus/data && chmod 777 /data/prometheus/data
然后在对应的机器上创建data目录和修改权限---很重要，会报错
3、grafana模板 https://grafana.com/grafana/dashboards
我加载的几个模板
8919 --node
3146 --pod
创建namespaces名为monitor
[root@2dot241 sy]# cat namespaces.yml
apiVersion: v1
kind: Namespace
metadata:
name: monitor
labels:
name: monitor
[root@2dot241 sy]# kubectl apply -f namespaces.yml
创建rbac文件：为了能够让Prometheus能够访问收到认证保护的Kubernetes API，我们首先需要做的是，对Prometheus进行访问授权。在Kubernetes中主要使用基于角色的访问控制模型(Role-Based Access Control)，用于管理Kubernetes下资源访问权限。首先我们需要在Kubernetes下定义角色（ClusterRole），并且为该角色赋予响应的访问权限。同时创建Prometheus所使用的账号（ServiceAccount），最后则是将该账号与角色进行绑定（ClusterRoleBinding）
[root@2dot241 sy]# cat rbac.yml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups:
- extensions
resources:
- ingresses
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: monitor
[root@2dot241 sy]# kubectl apply -f rbac.yml
prometheus-deploy和prometheus-svc：创建prometheus服务、使用映射主机端口30003访问prometheus
首先创建configmap--就是对应prometheus的配置文件--其中主要的就是自动发现和监控规则
其中很多监控项我都注释了，报警太烦了、只留了报警就要处理（是不是活着），不要紧就不报警的东西
[root@2dot241 sy]# cat configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: monitor
data:
#被引用到/etc/prometheus/prometheus.yml
prometheus.yml: |
global:
#每15s采集一次数据和15s做一次告警检测
scrape_interval: 15s
evaluation_interval: 15s
#指定加载的告警规则文件
rule_files:
- /etc/prometheus/rules.yml
#将报警送至何地进行报警
alerting:
alertmanagers:
- static_configs:
- targets: ["192.168.2.241:9093"]
#指定prometheus要监控的目标
scrape_configs:
- job_name: 'k8s-node'
scrape_interval: 10s
static_configs:
- targets:
- '192.168.2.239:9100'
- '192.168.2.240:9100'
#自定义获取监控数据,每个 job_name 都是独立的
- job_name: 'tomcat-pods'
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_service_annotation_prometheus_io_jvm_scrape]
regex: true;true
action: keep
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_app_metrics_patn]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_service_annotation_prometheus_io_app_metrics_port]
action: replace
target_label: __address__
regex: (.+);(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_pod_host_ip]
action: replace
target_label: kubernetes_host_ip
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-nodes'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
- job_name: 'kubernetes-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
- job_name: 'kubernetes-services'
kubernetes_sd_configs:
- role: service
metrics_path: /probe
params:
module: [http_2xx]
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: kubernetes_name
- job_name: 'kubernetes-ingresses'
kubernetes_sd_configs:
- role: ingress
relabel_configs:
- source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
regex: (.+);(.+);(.+)
replacement: ${1}://${2}${3}
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_ingress_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_ingress_name]
target_label: kubernetes_name
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
- job_name: kube-state-metrics
static_configs:
- targets: ['kube-state-metrics.monitor.svc.cluster.local:8080']
# 监控规则文件,被引用到/etc/prometheus/rules.yml
rules.yml: |
groups:
- name: test-rule
rules:
############# Node监控 #############
- alert: k8s-node状态异常
expr: up{job="k8s-node"} != 1
for: 3m
labels:
team: k8s-node
annotations:
summary: "{{$labels.instance}}: Node节点状态异常"
description: "可能是重启了"
- alert: k8s-node节点CPU使用率
expr: (1 - avg(irate(node_cpu_seconds_total{job="k8s-node",mode="idle"}[1m])) by (instance)) * 100 > 95
for: 1m
labels:
team: k8s-node
annotations:
summary: "{{$labels.instance}}: Node节点CPU使用率超过95%"
description: "{{$labels.instance}}: Node节点当前CPU使用率为: {{ $value }}"
#- alert: k8s-node节点内存使用率
#expr: (sum(container_memory_working_set_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 95
#for: 1m
#labels:
#team: k8s-node
#annotations:
#description: "Node服务器[[ {{$labels.instance}} ]] 内存使用率超过95%"
#summary: "{{$labels.instance}} 当前内存使用率为: {{ $value }}"
############ Pod 监控 ############
#- alert: 监控k8s的pod状态异常
#expr: up{kubernetes_namespace="monitor"} != 1
#for: 3m
#labels:
#team: "kube-state-metrics"
#annotations:
#description: "{{$labels.kubernetes_namespace}} 内的 pod 状态有变动"
#summary: "此 Pod 用于获取 k8s 监控数据, 绑定在一个节点上"
#- alert: 应用的 pod 状态有变动
#expr: kube_pod_container_status_ready{namespace="test"} != 1
#for: 3m
#labels:
#status: "test 命名空间内的 pod {{$labels.pod}}有变动"
#annotations:
#description: " {{$labels.container}} 的 pod 状态有变动"
#summary: "可能是重启或者在升级版本,如果频繁重启,请跟踪排查问题"
#- alert: 以下应用的 pod 重启次数已经超过15,请查看原因
#expr: kube_pod_container_status_restarts_total{namespace="product"} > 15
#for: 3m
#labels:
#status: " pod {{$labels.pod}} 重启次数太多"
#annotations:
#description: "Deployment {{$labels.container}} 内的 pod 重启次数太多"
# summary: "重启次数太多,可能是因为 pod 内应用有问题"
########### Java 监控 ############
- alert: jvm线程数过高
expr: jvm_threads_current{job="tomcat-pods"}>2000
for: 1m
labels:
status: "空间内 jvm 的变动情况"
annotations:
description: "{{$labels.kubernetes_pod_name}}: Jvm线程数过高"
summary: '{{ $labels.kubernetes_pod_name }} : 当前你线程值为: {{ $value }}'
[root@2dot241 sy]# cat prometheus-deploy.yml
apiVersion: apps/v1beta2
kind: Deployment
metadata:
labels:
name: prometheus-deployment
name: prometheus
namespace: monitor
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
spec:
containers:
- image: prom/prometheus:latest
name: prometheus
imagePullPolicy: IfNotPresent
command:
- "/bin/prometheus"
args:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/home/prometheus"
- "--storage.tsdb.retention=168h"
- "--web.enable-lifecycle"
ports:
- containerPort: 9090
protocol: TCP
volumeMounts:
- mountPath: "/home/prometheus"
name: data
- mountPath: "/etc/prometheus"
name: config-volume
- mountPath: "/etc/localtime"
readOnly: false
name: localtime
resources:
requests:
cpu: 100m
memory: 2048Mi
limits:
cpu: 500m
memory: 3180Mi
serviceAccountName: prometheus
nodeSelector:
nodetype: prometheus
volumes:
- name: data
hostPath:
path: "/data/prometheus/data"
- name: config-volume
configMap:
name: prometheus-config
- name: localtime
hostPath:
path: "/etc/localtime"
type: File
[root@2dot241 sy] kubectl label node 2dot239 nodetype=prometheus
因想要让数据持久化存储，所以固定了prometheus所在的node 这个参数nodeSelector:所以要给相应的node上打个标签
[root@2dot239 ~] mkdir -p /data/prometheus/data && chmod 777 /data/prometheus/data
然后在对应的机器上创建data目录和修改权限
[root@2dot241 sy] cat prometheus-svc.yml
kind: Service
apiVersion: v1
metadata:
labels:
app: prometheus
name: prometheus
namespace: monitor
spec:
type: NodePort
ports:
- port: 9090
targetPort: 9090
nodePort: 30003
selector:
app: prometheus
[root@2dot241 sy]# kubectl apply -f configmap.yaml
[root@2dot241 sy]# kubectl apply -f prometheus-svc.yml
[root@2dot241 sy]# kubectl apply -f prometheus-deploy.yml
[root@2dot241 sy]# cat reload.sh
#!/bin/bash
#用于修改配置文件后让prometheus加载配置，如果在prometheus可视化窗口没看到修改的rule就多执行两下
kubectl apply -f configmap.yaml
sleep 30
curl -XPOST http://192.168.2.241:30003/-/reload
这个时候已经可以进入到prometheus窗口了接下来安装采集pod和service相关的插件
安装kube-state-metrics
git地址：https://github.com/kubernetes/kube-state-metrics/tree/v1.9.8 下面也写了很详细的安装步骤就是这句话（要部署此项目，您只需运行即可kubectl apply -f examples/standard创建 Kubernetes 服务和部署。（注意：如果你的kubernetes集群的版本不是1.8+，请调整一些资源的apiVersion，查看yaml文件了解更多信息）。）
#我选这个版本是因为超过这个版本的那些镜像写的都是google的还得找镜像地址。。。。我懒
#下载zip包--我用的1.9.7没啥区别，我更换测试了下
[root@2dot241 sy] unzip kube-state-metrics-1.9.7.zip
[root@2dot241 sy] cd kube-state-metrics-1.9.7/examples/standard/
#这个包里默认用的namespace是kube-system我们用的是monitor所以要修改下
[root@2dot241 standard] sed -i s/kube-system/monitor/g ./*
[root@2dot241 standard] kubectl apply -f .
没有pod数据的
这是我的grafana数据，是有很多pod数据的
剩下grafana和报警，我们使用docker安装
创建grafana数据目录
[root@2dot241 sy] mkdir -p /data/grafana/data && chmod -R 777 /data/grafana/
[root@2dot241 sy] cat start-grafana.sh
#!/bin/bash
docker stop `docker ps -a |awk '/grafana/{print $1}'`
docker rm `docker ps -a |awk '/grafana/{print $1}'`
docker run -d \
--name=grafana \
--restart=always \
-p 3000:3000 \
-m 4096m \
-v /data/grafana/data:/var/lib/grafana \
-v /data/grafana/log:/var/log/grafana \
grafana/grafana:8.0.6
[root@2dot241 sy] bash start-grafana.sh 创建完成后就可以访问宿主机对应的3000端口就能访问到grafana了
1、安装完之后，需要添加source，source直接点prometheus，链接就是[http://192.168.2.241:30003之前创建的prometheus界面]
2、添加模板dashboad（列出几个常用的）
点import导入，有俩种方式，直接填写官网模板号码比如下面的9276，或者导入json
https://grafana.com/dashboards/8919 node的cpu、内存等
https://grafana.com/dashboards/3146 pod 13105
https://grafana.com/dashboards/8588 deployment
安装alertmanager
我这只用了微信报警，想用钉钉和邮件的可以查看最下面的连接，他们配置的都比较多
创建alert数据目录
mkdir /data/alert/data
[root@2dot241 alert] cat alertmanager.yml
global:
resolve_timeout: 5m
templates:
- "/etc/alertmanager/template/*.tmpl" #报警模板文件位置`
route:
group_by: ['alertname']
group_wait: 10s
group_interval: 10s
repeat_interval: 6m
receiver: wechat
receivers:
- name: 'wechat'
wechat_configs:
- corp_id: 'wwd2849aba51a0c8220' #企业微信id 后台查看
to_party: '2' #微信部门ID
agent_id: '1000008' #企业微信应用程序id
api_secret: 'znhoVsyT8lafYe3ohELP01AzcV43ENkFd3to3aIYEn' #secret，后台应用程序中查看
send_resolved: true #报警解除后通知
message: '{{ template "wechat.default.message" . }}' #报警模板中字符串设置-要和报警模板对应（文本、html等类型）
#- name: 'default'
# email_configs:
# - to: ""
# send_resolved: true
# from: ""
# smarthost: "smtp.xxx.com:25"
# auth_username: ""
# auth_password: ""
# webhook_configs:
# - url: 'http://192.168.50.60:8060/dingtalk/ops_dingding/send'
# send_resolved: true
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname']
[root@2dot241 alert] mkdir template
[root@2dot241 template] cat wechat.tmpl 报警模板，在template目录中创建
{{ define "wechat.default.message" }}
{{- range .Alerts.Firing -}}
=====监控报警=====
告警状态：{{ .Status }}
告警级别：{{ .Labels.severity }}
告警类型：{{ .Labels.alertname }}
告警应用：{{ .Annotations.summary }}
告警主机：{{ .Labels.instance }}
告警详情：{{ .Annotations.description }}
告警时间：{{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}
=======end=======
{{ end }}
{{- range .Alerts.Resolved -}}
=====报警恢复=====
告警状态：{{ .Status }}
告警级别：{{ .Labels.severity }}
告警类型：{{ .Labels.alertname }}
告警应用：{{ .Annotations.summary }}
告警主机：{{ .Labels.instance }}
告警详情：{{ .Annotations.description }}
告警时间：{{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}
恢复时间: {{ .EndsAt.Local.Format "2006-01-02 15:04:05" }}
=======end=======
{{ end }}
{{- end }}
[root@2dot241 alert] cat start-alert.sh
#!/bin/bash
docker stop `docker ps -a |awk '/alertmanager/{print $1}'`
docker rm `docker ps -a |awk '/alertmanager/{print $1}'`
docker run -d \
--name alertmanager \
--restart=always \
-p 9093:9093 \
-v /etc/localtime:/etc/localtime:ro \
-v /data/alert/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
-v /data/alert/data:/alertmanager \
-v /data/alert/template:/etc/alertmanager/template \
prom/alertmanager:latest
[root@2dot241 alert]# ./start-alert.sh 执行脚本就可以了
添加数据源
写上prometheus的访问地址就行了
导入模板
选择数据来源
8919 模板
参考文档：
https://shenshengkun.github.io/posts/e2e612d1.html （多数镜像无法下载、报警需要修改报警信息）
https://blog.csdn.net/qq_31547771/article/details/119217023?spm=1001.2014.3001.5501 （无报警、监控信息、rule文件分离）
https://www.prometheus.wang/kubernetes/READMD.html （介绍详细，配置为阶段性，适用于学习了解）