Kubernetes1.27容器化部署Prometheus

GitHub链接

https://github.com/prometheus-operator/kube-prometheus

根据自己的k8s版本选择对应的版本

# git设置代理
git config --global http.proxy IP地址:端口

# 1.27对应的版本为release-0.13
git clone -b release-0.13 https://github.com/prometheus-operator/kube-prometheus.git

cd kube-prometheus/

修改镜像地址

cd manifests

# 可以通过如下命令来查看
ls | xargs -I {} grep -iH "image:" {}

# alertmanager-alertmanager.yaml
quay.io/prometheus/alertmanager:v0.26.0
swr.cn-north-4.myhuaweicloud.com/ctl456/alertmanager:v0.26.0

# blackboxExporter-deployment.yaml
quay.io/prometheus/blackbox-exporter:v0.24.0
swr.cn-north-4.myhuaweicloud.com/ctl456/blackbox-exporter:v0.24.0

jimmidyson/configmap-reload:v0.5.0
swr.cn-north-4.myhuaweicloud.com/ctl456/configmap-reload:v0.5.0

quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2

# grafana-deployment.yaml
grafana/grafana:9.5.3
swr.cn-north-4.myhuaweicloud.com/ctl456/grafana:9.5.3

# kubeStateMetrics-deployment.yaml
registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.9.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-state-metrics:v2.9.2

quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2

# nodeExporter-daemonset.yaml
quay.io/prometheus/node-exporter:v1.6.1
swr.cn-north-4.myhuaweicloud.com/ctl456/node-exporter:v1.6.1

quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2

# prometheusAdapter-deployment.yaml
registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.11.1
swr.cn-north-4.myhuaweicloud.com/ctl456/prometheus-adapter:v0.11.1

# prometheusOperator-deployment.yaml
quay.io/prometheus-operator/prometheus-operator:v0.67.1
swr.cn-north-4.myhuaweicloud.com/ctl456/prometheus-operator:v0.67.1

quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2

# prometheus-prometheus.yaml
quay.io/prometheus/prometheus:v2.46.0
swr.cn-north-4.myhuaweicloud.com/ctl456/prometheus:v2.46.0

部署命令

kubectl apply --server-side -f manifests/setup
kubectl wait \
	--for condition=Established \
	--all CustomResourceDefinition \
	--namespace=monitoring
kubectl apply -f manifests/

# 查看是否全部运行
kubectl get svc,pod -n monitoring

# 修改type为NodePort
kubectl edit svc grafana -n monitoring

# 删除规则
kubectl -n monitoring delete networkpolicy --all


# grafana默认账号密码
admin
admin

对Etcd集群进行监控（云原生监控）

Etcd的Metrics接口需要https协议访问

# 先获取证书文件目录
grep -E "key-file|cert-file" /etc/kubernetes/manifests/etcd.yaml

curl -s --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://IP地址:2379/metrics -k | tail -1

创建Etcd Service

vim etcd-svc.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app: etcd-prom
  name: etcd-prom
  namespace: kube-system
spec:
  ports:
  - name: https-metrics
    port: 2379
    protocol: TCP
    targetPort: 2379
  selector: # 添加标签选择器
    component: etcd # 容器etcd的标签
  type: ClusterIP
---
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
  labels:
    app: etcd-prom
    kubernetes.io/service-name: etcd-prom
  name: etcd-prom
  namespace: kube-system
addressType: IPv4
ports:
  - name: https-metrics
    port: 2379
    protocol: TCP
endpoints:
  - addresses:
    - "YOUR_ETCD_IP01" # Etcd集群的IP地址
    - "YOUR_ETCD_IP02"
    - "YOUR_ETCD_IP03"

kubectl apply -f etcd-svc.yaml
kubectl get svc -n kube-system etcd-prom

#通过curl测试访问
curl -s --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://CLUSTER-IP:2379/metrics -k | tail -1

创建Etcd证书的Secret

# 创建Etcd证书的Secret
kubectl create secret generic etcd-ssl --from-file=/etc/kubernetes/pki/etcd/ca.crt --from-file=/etc/kubernetes/pki/etcd/server.crt --from-file=/etc/kubernetes/pki/etcd/server.key -n monitoring

# 将证书挂载到容器里
kubectl edit prometheus k8s -n monitoring

# 检查是否挂载成功
kubectl exec -n monitoring prometheus-k8s-0 -c prometheus -- ls /etc/prometheus/secrets/etcd-ssl/

创建Etcd ServiceMonitor

vim servicemonitor.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd
  namespace: monitoring
  labels:
    app: etcd
spec:
  jobLabel: k8s-app
  endpoints:
    - interval: 30s
      port: https-metrics  # 这个port对应 Service.spec.ports.name
      scheme: https
      tlsConfig:
        caFile: /etc/prometheus/secrets/etcd-ssl/ca.crt #证书路径
        certFile: /etc/prometheus/secrets/etcd-ssl/server.crt
        keyFile: /etc/prometheus/secrets/etcd-ssl/server.key
        insecureSkipVerify: true  # 关闭证书校验
  selector:
    matchLabels:
      app: etcd-prom  # 跟Service的lables保持一致
  namespaceSelector:
    matchNames:
    - kube-system

kubectl create -f servicemonitor.yaml

grafana导入模板

成功截图

对MySQL进行监控（非云原生监控）

部署mysql

apiVersion: v1
kind: Service
metadata:
  name: wordpress-mysql
  labels:
    app: wordpress
spec:
  ports:
    - protocol: TCP
      port: 3306
      targetPort: 3306
  selector:
    app: wordpress
    tier: mysql
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: wordpress-mysql
  labels:
    app: wordpress
    tier: mysql
spec:
  selector:
    matchLabels:
      app: wordpress
      tier: mysql
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: wordpress
        tier: mysql
    spec:
      containers:
        - image: swr.cn-north-4.myhuaweicloud.com/ctl456/mysql:5.6
          name: mysql
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: changeme
          ports:
            - containerPort: 3306
              name: mysql
          volumeMounts:
            - name: tz-config
              mountPath: /etc/localtime
      volumes:
        - name: tz-config
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
            type: File

# 查看服务是否启动
kubectl get svc,pod

创建用户和权限

kubectl exec -it mysqlpod名称 -- bash

CREATE USER 'exporter'@'%' IDENTIFIED BY 'exporter';
GRANT PROCESS,REPLICATION CLIENT,SELECT ON *.* TO 'exporter'@'%';

配置Exporter

vim mysql-exporter.yaml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: mysql-exporter
  template:
    metadata:
      labels:
        k8s-app: mysql-exporter
    spec:
      containers:
      - name: mysql-exporter
        image: registry.cn-beijing.aliyuncs.com/dotbalo/mysqld-exporter 
        env:
         - name: DATA_SOURCE_NAME
           value: "exporter:exporter@(wordpress-mysql:3306)/" # 用户名:密码@(数据库地址:端口)
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9104
---
apiVersion: v1
kind: Service
metadata:
  name: mysql-exporter
  namespace: monitoring
  labels:
    k8s-app: mysql-exporter
spec:
  type: ClusterIP
  selector:
    k8s-app: mysql-exporter
  ports:
  - name: api
    port: 9104
    protocol: TCP

kubectl apply -f mysql-exporter.yaml

配置ServiceMonitor

vim mysql-sm.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: mysql-exporter
  namespace: monitoring
  labels:
    k8s-app: mysql-exporter
    namespace: monitoring
spec:
  jobLabel: k8s-app
  endpoints:
  - port: api
    interval: 30s
    scheme: http
  selector:
    matchLabels:
      k8s-app: mysql-exporter
  namespaceSelector:
    matchNames:
    - monitoring

grafana导入模板

6239

成功截图

静态配置

创建一个空文件

touch prometheus-additional.yaml

创建secret

kubectl create secret generic additional-config --from-file=prometheus-additional.yaml -n monitoring

编辑Prometheus的配置

kubectl edit prometheus -n monitoring

additionalScrapeConfigs:
    key: prometheus-additional.yaml
    name: additional-config
    optional: true

编辑prometheus-additional.yaml

vim prometheus-additional.yaml

- job_name: "blackbox"
  metrics_path: /probe
  params:
    module: [http_2xx]
  static_configs:
    - targets:
      - https://www.baidu.com
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      replacement: blackbox-exporter:19115

# 更新配置
kubectl create secret generic additional-config --from-file=prometheus-additional.yaml --dry-run=client -o yaml | kubectl replace -f - -n monitoring

grafana导入模板

13659

成功截图

监控Windows（外部）主机

下载Windows_exporter并安装

GitHub链接：https://github.com/prometheus-community/windows_exporter/releases

编辑prometheus-additional.yaml

vim prometheus-additional.yaml

- job_name: "WindowsServerMonitor"
  static_configs:
    - targets:
      - "192.168.1.5:9182" # windowsIP地址
      labels:
        server_type: "windows"
  relabel_configs:
    - source_labels: [__address__]
      target_label: instance

# 更新配置
kubectl create secret generic additional-config --from-file=prometheus-additional.yaml --dry-run=client -o yaml | kubectl replace -f - -n monitoring

grafana导入模板

12566

成功截图

Alertmanager告警

Alertmanager邮件通告

cd kube-prometheus/manifests/

vim alertmanager-secret.yaml

# global配置
smtp_from: "[email protected]" # 163邮箱地址
smtp_smarthost: "smtp.163.com:465" # 163邮箱SMTP服务器地址
smtp_hello: "163.com"
smtp_auth_username: "[email protected]" # 163邮箱地址
smtp_auth_password: "xxxxxx" # 授权码
smtp_require_tls: false

# receivers配置
"email_configs":
- to: "xxxxx" # 收件人邮箱地址
  send_resolved: true

# 使配置生效
kubectl replace -f alertmanager-secret.yaml