文章目录
-
- Kube-prometheus部署Prometheus
-
- 下载 kube-prometheus
- 修改类型为 NodePort
- kube-prometheus 集成 pushgateway
- 修改grafana配置
- Prometheus 持久化存储
- 配置Ingress
- 部署
- 监控k8s集群外节点
Kube-prometheus部署Prometheus
下载 kube-prometheus
wget https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/v0.10.0.tar.gz
cd /data/bigdata/kube-prometheus-0.10.0/manifests
grep -rn 'image: '
docker tag quay.io/prometheus/alertmanager:v0.23.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker tag quay.io/prometheus/blackbox-exporter:v0.19.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker tag quay.io/brancz/kube-rbac-proxy:v0.11.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker tag quay.io/prometheus/node-exporter:v1.3.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker tag quay.io/prometheus/prometheus:v2.32.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker tag quay.io/prometheus-operator/prometheus-config-reloader:v0.53.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker tag quay.io/prometheus-operator/prometheus-operator:v0.53.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker tag bitnami/kube-state-metrics:2.3.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker tag willdockerhub/prometheus-adapter:v0.9.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker tag grafana/grafana:8.3.3 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker tag prom/pushgateway:v1.4.3 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' alertmanager-alertmanager.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' blackboxExporter-deployment.yaml
sed -i 's/quay.io\/brancz/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' kubeStateMetrics-deployment.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' nodeExporter-daemonset.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' prometheus-prometheus.yaml
sed -i 's/quay.io\/prometheus-operator/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' prometheusOperator-deployment.yaml
修改类型为 NodePort
cd /data/bigdata/kube-prometheus-0.10.0/manifests
vim prometheus-service.yaml
spec:
type: NodePort
ports:
- name: web
port: 9090
targetPort: web
nodePort: 30090
- name: reloader-web
port: 8080
targetPort: reloader-web
vim alertmanager-service.yaml
spec:
type: NodePort
ports:
- name: web
port: 9093
targetPort: web
nodePort: 30093
- name: reloader-web
port: 8080
targetPort: reloader-web
vim grafana-service.yaml
spec:
type: NodePort
ports:
- name: http
port: 3000
targetPort: http
nodePort: 30300
kube-prometheus 集成 pushgateway
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: pushgateway
name: pushgateway
namespace: monitoring
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: pushgateway
template:
metadata:
labels:
app.kubernetes.io/name: pushgateway
spec:
nodeSelector:
kubernetes.io/os: linux
containers:
- image: bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
name: pushgateway
ports:
- containerPort: 9091
name: http
resources:
limits:
cpu: 200m
memory: 512Mi
requests:
cpu: 50m
memory: 128Mi
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/name: pushgateway
name: pushgateway
namespace: monitoring
spec:
type: NodePort
ports:
- name: http
port: 9091
targetPort: http
nodePort: 30391
selector:
app.kubernetes.io/name: pushgateway
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app.kubernetes.io/name: pushgateway
name: pushgateway
namespace: monitoring
spec:
endpoints:
- interval: 30s
path: /metrics
port: http
scheme: http
selector:
matchLabels:
app.kubernetes.io/name: pushgateway
修改grafana配置
vim grafana-config.yaml
grafana.ini: |
[security]
allow_embedding = true
[date_formats]
default_timezone = UTC
[auth.anonymous]
enabled = true
[database]
type = mysql
host = mysql-master.mysql:3306
name = grafana
user = grafana
password = YQZi5UUw825h
type: Opaque
Prometheus 持久化存储
vim sc.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: prometheus-local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Retain
vim pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-pv-1
spec:
storageClassName: prometheus-local-storage
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
local:
path: /data/zxl/prometheus1/
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- szzb-bg-dev-etl-4
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-pv-2
spec:
storageClassName: prometheus-local-storage
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
local:
path: /data/zxl/prometheus2/
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- szzb-bg-dev-etl-5
vim prometheus-prometheus.yaml
retention: 15d
storage:
volumeClaimTemplate:
spec:
storageClassName: prometheus-local-storage
resources:
requests:
storage: 10Gi
配置Ingress
vim ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: prometheus-ingress
namespace: monitoring
annotations:
kubernetes.io/ingress.class: "nginx"
prometheus.io/http_probe: "true"
spec:
rules:
- host: alert.k8s.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: alertmanager-main
port:
number: 9093
- host: bigdata-uat-k8s-grafana.ky-tech.com.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grafana
port:
number: 3000
tls:
- hosts:
- bigdata-uat-k8s-grafana.ky-tech.com.cn
secretName: grafana-secret
- host: bigdata-uat-k8s-prometheus.ky-tech.com.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: prometheus-k8s
port:
number: 9090
tls:
- hosts:
- bigdata-uat-k8s-prometheus.ky-tech.com.cn
secretName: prometheus-secret
kubectl create secret tls grafana-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring
kubectl create secret tls prometheus-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring
kubectl apply -rf .
部署
kubectl apply --server-side -f manifests/setup
kubectl wait \
--for condition=Established \
--all CustomResourceDefinition \
--namespace=monitoring
kubectl apply -f manifests/
kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup
监控k8s集群外节点
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
name: nginx
namespace: monitoring
spec:
endpoints:
- interval: 30s
port: metrcis
selector:
matchLabels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
namespaceSelector:
matchNames:
- monitoring
---
apiVersion: v1
kind: Service
metadata:
labels:
app: nginx-exporter
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
name: nginx
namespace: monitoring
spec:
type: ClusterIP
clusterIP: None
ports:
- name: metrcis
port: 9113
targetPort: 9113
protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
labels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
name: nginx
namespace: monitoring
subsets:
- addresses:
- ip: 192.168.26.11
- ip: 192.168.26.13
ports:
- name: metrcis
port: 9113
protocol: TCP