prometheus operator使用持久存储

kube-prometheus使用持久存储

prometheus operator部署参考：
https://blog.csdn.net/networken/article/details/85620793
持久存储配置参考：
https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/storage.md

prometheus默认使用empty-dir存储数据，pod迁移或重启监控数据将丢失，下面以nfs和local pv为例，持久化prometheus数据。

使用nfs存储

nfs server及动态提供存储参考：
https://blog.csdn.net/networken/article/details/86697018

下面是在部署kube-prometheus之前，修改官方manifests下的prometheus-prometheus.yaml内容，只在最后增加storage配置即可，注意指定storageClassName需要提前准备好。

# cat manifests/prometheus-prometheus.yaml
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  labels:
    prometheus: k8s
  name: k8s
  namespace: monitoring
spec:
  alerting:
    alertmanagers:
    - name: alertmanager-main
      namespace: monitoring
      port: web
  image: willdockerhub/prometheus:v2.15.2
  nodeSelector:
    kubernetes.io/os: linux
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  ruleSelector:
    matchLabels:
      prometheus: k8s
      role: alert-rules
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  version: v2.15.2
  storage:
    volumeClaimTemplate:
      spec:
        storageClassName: nfs-client
        resources:
          requests:
            storage: 40Gi

修改后按照官方部署方式部署prometheus即可，部署完成后可以看到创建的pv和pvc:

查看绑定的pv和pvc

[root@master01 ~]# kubectl -n monitoring get pvc
NAME                                 STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
prometheus-k8s-db-prometheus-k8s-0   Bound    pvc-a8b091d5-3929-4a29-b89f-fe47022aee55   40Gi       RWO            nfs-client     9h
prometheus-k8s-db-prometheus-k8s-1   Bound    pvc-1276a939-7d5d-43b9-ade9-ae0ef4108ecc   40Gi       RWO            nfs-client     9h

[root@master01 ~]# kubectl -n monitoring get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                           STORAGECLASS   REASON   AGE
pvc-1276a939-7d5d-43b9-ade9-ae0ef4108ecc   40Gi       RWO            Delete           Bound    monitoring/prometheus-k8s-db-prometheus-k8s-1   nfs-client              9h
pvc-a8b091d5-3929-4a29-b89f-fe47022aee55   40Gi       RWO            Delete           Bound    monitoring/prometheus-k8s-db-prometheus-k8s-0   nfs-client              9h
[root@master01 ~]#

删除pvc后保留pv需要更改pv回收策略为Retain：

kubectl patch pv <your-pv-name> -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'

使用local pv

创建storageclass yaml文件

cat > promethues-sc.yaml <kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
EOF

创建storage class

kubectl apply -f prometheus-sc.yaml

定义prometheus配置，同样修改 manifests/prometheus-prometheus.yaml文件，然后按照正常流程部署kube-prometheus即可。

# cat manifests/prometheus-prometheus.yaml
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  labels:
    prometheus: k8s
  name: k8s
  namespace: monitoring
spec:
  alerting:
    alertmanagers:
    - name: alertmanager-main
      namespace: monitoring
      port: web
  image: willdockerhub/prometheus:v2.15.2
  nodeSelector:
    kubernetes.io/os: linux
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  ruleSelector:
    matchLabels:
      prometheus: k8s
      role: alert-rules
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  version: v2.15.2
  storage:
    volumeClaimTemplate:
      spec:
        selector:
          matchLabels:
            app: prometheus
        storageClassName: local-storage
        resources:
          requests:
            storage: 20Gi
 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prom-local-pv-0
  labels:
    app: prometheus  
spec:
  capacity:
    storage: 20Gi
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local-storage
  local:
    path: /data/prometheus/data
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - node01                
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prom-local-pv-1
  labels:
    app: prometheus  
spec:
  capacity:
    storage: 20Gi
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local-storage
  local:
    path: /data/prometheus/data
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - node02

手动在pv指定的节点中创建挂载目录

mkdir -p /data/prometheus/data

部署完成后查看创建的pvc和pv

[root@master01 ~]# kubectl -n monitoring get pvc
NAME                                 STATUS   VOLUME            CAPACITY   ACCESS MODES   STORAGECLASS    AGE
prometheus-k8s-db-prometheus-k8s-0   Bound    prom-local-pv-1   20Gi       RWO            local-storage   42m
prometheus-k8s-db-prometheus-k8s-1   Bound    prom-local-pv-0   20Gi       RWO            local-storage   42m

[root@master01 ~]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS     CLAIM                                           STORAGECLASS    REASON   AGE
prom-local-pv-0                            20Gi       RWO            Retain           Bound      monitoring/prometheus-k8s-db-prometheus-k8s-1   local-storage            42m
prom-local-pv-1                            20Gi       RWO            Retain           Bound      monitoring/prometheus-k8s-db-prometheus-k8s-0   local-storage            42m