k8s进阶(三)

7. 基于nfs部署Redis服务实现持久化

7.1 构建Redis镜像

配置文件和启动脚本

root@k8s-ansible-client:~/yaml/20211016/dockerfile/redis# tree 
.
├── build-command.sh
├── Dockerfile
├── redis-4.0.14.tar.gz
├── redis.conf
└── run_redis.sh

root@k8s-ansible-client:~/yaml/20211016/dockerfile/redis# grep -v "^#" redis.conf |grep -v "^$"
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile ""
databases 16
always-show-logo yes
save 900 1
save 5 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error no
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /data/redis-data
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
requirepass 123456
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

root@k8s-ansible-client:~/yaml/20211016/dockerfile/redis# cat run_redis.sh 
#!/bin/bash

/usr/sbin/redis-server /usr/local/redis/redis.conf

tail -f  /etc/hosts

root@k8s-ansible-client:~/yaml/20211016/dockerfile/redis# cat build-command.sh 
#!/bin/bash
TAG=$1
docker build -t harbor.openscp.com/base/redis:${TAG} .
sleep 3
docker push  harbor.openscp.com/base/redis:${TAG}

root@k8s-ansible-client:~/yaml/20211016/dockerfile/redis# cat Dockerfile 
#Redis Image
FROM harbor.openscp.com/base/centos:centos7.9.2009


ADD redis-4.0.14.tar.gz /usr/local/src
RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
RUN ln -sv /usr/local/src/redis-4.0.14 /usr/local/redis && cd /usr/local/redis && make && cp src/redis-cli /usr/sbin/ && cp src/redis-server  /usr/sbin/ && mkdir -pv /data/redis-data 
ADD redis.conf /usr/local/redis/redis.conf 
ADD run_redis.sh /usr/local/redis/run_redis.sh

EXPOSE 6379

CMD ["/usr/local/redis/run_redis.sh"]

构建镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/redis# bash build-command.sh v1
...
Successfully tagged harbor.openscp.com/base/redis:v1
The push refers to repository [harbor.openscp.com/base/redis]
4b87fefa01f9: Pushed 
7cf99aab4830: Pushed 
498f7e20f28c: Pushed 
2853d7e8868a: Pushed 
4f43a69196ea: Pushed 
174f56854903: Mounted from base/tomcat-app1 
v1: digest: sha256:449ef47457b4161707f77f458e0e416be66f9f7fa7082d5afe71c982539d0936 size: 1581

创建一个pv

root@k8s-ansible-client:~/yaml/20211016/redis# cat redis-pv.yaml 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-datadir-pv-1
  namespace: pop
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/redis-datadir-1 
    server: 10.10.0.26

root@k8s-ansible-client:~/yaml/20211016/redis# kubectl apply -f redis-pv.yaml 
persistentvolume/redis-datadir-pv-1 created
root@k8s-ansible-client:~/yaml/20211016/redis# kubectl get pv -n pop
NAME                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                  STORAGECLASS   REASON   AGE
redis-datadir-pv-1   5Gi        RWO            Retain           Available                                                  39s

创建一个pvc

root@k8s-ansible-client:~/yaml/20211016/redis# cat redis-pvc.yaml 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: redis-datadir-pvc-1 
  namespace: pop
spec:
  volumeName: redis-datadir-pv-1 
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
root@k8s-ansible-client:~/yaml/20211016/redis# kubectl apply -f redis-pvc.yaml 
persistentvolumeclaim/redis-datadir-pvc-1 created                              11s
root@k8s-ansible-client:~/yaml/20211016/redis# kubectl get pvc -n pop
NAME                  STATUS   VOLUME               CAPACITY   ACCESS MODES   STORAGECLASS   AGE
redis-datadir-pvc-1   Bound    redis-datadir-pv-1   5Gi        RWO                           13s

7.2 部署Redis

Redis的Deployment文件

root@k8s-ansible-client:~/yaml/20211016/redis# cat redis.yaml 
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: devops-redis 
  name: deploy-devops-redis
  namespace: pop
spec:
  replicas: 1 
  selector:
    matchLabels:
      app: devops-redis
  template:
    metadata:
      labels:
        app: devops-redis
    spec:
      containers:
        - name: redis-container
          image: harbor.openscp.com/base/redis:v1
          imagePullPolicy: Always
          volumeMounts:
          - mountPath: "/data/redis-data/"
            name: redis-datadir
      volumes:
        - name: redis-datadir
          persistentVolumeClaim:
            claimName: redis-datadir-pvc-1 

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: devops-redis
  name: srv-devops-redis
  namespace: pop
spec:
  type: NodePort
  ports:
  - name: http
    port: 6379 
    targetPort: 6379
    nodePort: 36379 
  selector:
    app: devops-redis
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 10800

基于Deployment启动一个Redis pod容器

root@k8s-ansible-client:~/yaml/20211016/redis# kubectl apply -f redis.yaml 
deployment.apps/deploy-devops-redis created
service/srv-devops-redis created

root@k8s-ansible-client:~/yaml/20211016/redis# kubectl get pods,deploy -n pop
NAME                                      READY   STATUS    RESTARTS   AGE
pod/deploy-devops-redis-d9fd6594c-fvmmg   1/1     Running   0          44s

NAME                                  READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deploy-devops-redis   1/1     1            1           44s

验证

root@k8s-ansible-client:~/yaml/20211016/redis# kubectl exec -it pod/deploy-devops-redis-d9fd6594c-fvmmg bash -n pop
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@deploy-devops-redis-d9fd6594c-fvmmg /]# df -Th
Filesystem                           Type     Size  Used Avail Use% Mounted on
overlay                              overlay   20G  9.9G  8.8G  53% /
tmpfs                                tmpfs     64M     0   64M   0% /dev
tmpfs                                tmpfs    2.0G     0  2.0G   0% /sys/fs/cgroup
/dev/mapper/ubuntu--vg-ubuntu--lv    ext4      20G  9.9G  8.8G  53% /etc/hosts
10.10.0.26:/data/pop/redis-datadir-1 nfs4      20G   15G  4.2G  78% /data/redis-data
shm                                  tmpfs     64M     0   64M   0% /dev/shm
tmpfs                                tmpfs    3.2G   12K  3.2G   1% /run/secrets/kubernetes.io/serviceaccount
tmpfs                                tmpfs    2.0G     0  2.0G   0% /proc/acpi
tmpfs                                tmpfs    2.0G     0  2.0G   0% /proc/scsi
tmpfs                                tmpfs    2.0G     0  2.0G   0% /sys/firmware

8. 基于StatefulSet运行mysql一主多从

官网参考文档

8.1 StatefulSet介绍

有状态服务, 是一种给Pod提供唯一标志的控制器,它可以保证部署和扩展的顺序。

Pod一致性:包含次序(启动、停止次序)、网络一致性。此一致性与Pod相关,与被调度到哪个node节点无关。
稳定的次序:对于N个副本的StatefulSet,每个Pod都在[0,N)的范围内分配一个数字序号,且是唯一的。
稳定的网络:Pod的hostname模式为(statefulset名称)- (序号)。
稳定的存储:通过VolumeClaimTemplate为每个Pod创建一个PV。删除、减少副本,不会删除相关的卷

如果把之前无状态的服务比喻为牛、羊等牲畜,因为,这些到一定时候就可以”送出“。那么,有状态就比喻为:宠物,而宠物不像牲畜一样到达一定时候“送出”,人们往往会照顾宠物的一生。

8.2 StatefulSet实现原理

与 ReplicaSet 和 Deployment 资源一样,StatefulSet 也使用控制器的方式实现,它主要由 StatefulSetController、StatefulSetControl 和 StatefulPodControl 三个组件协作来完成 StatefulSet 的管理,StatefulSetController 会同时从 PodInformer 和 ReplicaSetInformer 中接受增删改事件并将事件推送到队列中。

控制器 StatefulSetController 会在 Run 方法中启动多个 Goroutine 协程,这些协程会从队列中获取待处理的 StatefulSet 资源进行同步,接下来我们会先介绍 Kubernetes 同步 StatefulSet 的过程。

8.3 StatefulSet组成部分

Headless Service:用来定义Pod网络标识( DNS domain)。

volumeClaimTemplates :存储卷申请模板,创建PVC,指定pvc名称大小,将自动创建pvc,且pvc必须由存储类供应。

StatefulSet :定义具体应用,名为Nginx,有三个Pod副本,并为每个Pod定义了一个域名部署statefulset。

为什么需要 headless service 无头服务?
在用Deployment时,每一个Pod名称是没有顺序的,是随机字符串,因此是Pod名称是无序的,但是在statefulset中要求必须是有序 ,每一个pod不能被随意取代,pod重建后pod名称还是一样的。而pod IP是变化的,所以是以Pod名称来识别。pod名称是pod唯一性的标识符,必须持久稳定有效。这时候要用到无头服务,它可以给每个Pod一个唯一的名称 。

为什么需要volumeClaimTemplate?
对于有状态的副本集都会用到持久存储,对于分布式系统来讲,它的最大特点是数据是不一样的,所以各个节点不能使用同一存储卷,每个节点有自已的专用存储,但是如果在Deployment中的Pod template里定义的存储卷,是所有副本集共用一个存储卷,数据是相同的,因为是基于模板来的 ,而statefulset中每个Pod都要自已的专有存储卷,所以statefulset的存储卷就不能再用Pod模板来创建了,于是statefulSet使用volumeClaimTemplate,称为卷申请模板,它会为每个Pod生成不同的pvc,并绑定pv,从而实现各pod有专用存储。这就是为什么要用volumeClaimTemplate的原因。

8.4 创建MYSQL一主多从环境

定义mysql的pv

root@k8s-ansible-client:~/yaml/20211016/mysql# cat mysql-pv.yaml 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-1
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/mysql-datadir-1 
    server: 10.10.0.26
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-2
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/mysql-datadir-2
    server: 10.10.0.26
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-3
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/mysql-datadir-3
    server: 10.10.0.26
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-4
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/mysql-datadir-4
    server: 10.10.0.26
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-5
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/mysql-datadir-5
    server: 10.10.0.26

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-6
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/pop/mysql-datadir-6
    server: 10.10.0.26


执行yaml文件

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl apply -f mysql-pv.yaml 
persistentvolume/mysql-datadir-1 created
persistentvolume/mysql-datadir-2 created
persistentvolume/mysql-datadir-3 created
persistentvolume/mysql-datadir-4 created
persistentvolume/mysql-datadir-5 created
persistentvolume/mysql-datadir-6 created

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl get pv -n pop
NAME                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                     STORAGECLASS   REASON   AGE
mysql-datadir-1      10Gi       RWO            Retain           Available                                                     9s
mysql-datadir-2      10Gi       RWO            Retain           Available                                                     9s
mysql-datadir-3      10Gi       RWO            Retain           Available                                                     9s
mysql-datadir-4      10Gi       RWO            Retain           Available                                                     9s
mysql-datadir-5      10Gi       RWO            Retain           Available                                                     9s
mysql-datadir-6      10Gi       RWO            Retain           Available                                                     9s

将MySQL配置定义到configMap

root@k8s-ansible-client:~/yaml/20211016/mysql# cat mysql-configmap.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql
  namespace: pop
  labels:
    app: mysql
data:
  master.cnf: |
    # Apply this config only on the master.
    [mysqld]
    log-bin
    log_bin_trust_function_creators=1
    lower_case_table_names=1
  slave.cnf: |
    # Apply this config only on slaves.
    [mysqld]
    super-read-only
    log_bin_trust_function_creators=1

执行yaml文件

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl apply -f mysql-configmap.yaml 
configmap/mysql created
root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl get configMap -n pop
NAME               DATA   AGE
kube-root-ca.crt   1      22h
mysql              2      9s

定义MySQL的Service

root@k8s-ansible-client:~/yaml/20211016/mysql# cat mysql-svc.yaml 
apiVersion: v1
kind: Service
metadata:
  namespace: pop
  name: mysql
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  clusterIP: None
  selector:
    app: mysql
---
apiVersion: v1
kind: Service
metadata:
  name: mysql-read
  namespace: pop
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  selector:
    app: mysql

执行yaml文件

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl apply -f mysql-svc.yaml 
service/mysql created
service/mysql-read created
root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl get svc -n pop
NAME               TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
mysql              ClusterIP   None                    3306/TCP         8s
mysql-read         ClusterIP   10.68.181.29            3306/TCP         8s
srv-devops-redis   NodePort    10.68.251.172           6379:30379/TCP   22h

定义MySQL的statefulset文件
镜像:
mysql: docker pull mysql:5.7.30
xtrabackup: docker pull anjia0532/google-samples.xtrabackup:1.0

root@k8s-ansible-client:~/yaml/20211016/mysql# cat mysql-statefulset.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
  namespace: pop
spec:
  selector:
    matchLabels:
      app: mysql
  serviceName: mysql
  replicas: 3
  template:
    metadata:
      labels:
        app: mysql
    spec:
      initContainers:
      - name: init-mysql
        image: harbor.openscp.com/base/mysql:5.7.30 
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Generate mysql server-id from pod ordinal index.
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          echo [mysqld] > /mnt/conf.d/server-id.cnf
          # Add an offset to avoid reserved server-id=0 value.
          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
          # Copy appropriate conf.d files from config-map to emptyDir.
          if [[ $ordinal -eq 0 ]]; then
            cp /mnt/config-map/master.cnf /mnt/conf.d/
          else
            cp /mnt/config-map/slave.cnf /mnt/conf.d/
          fi
        volumeMounts:
        - name: conf
          mountPath: /mnt/conf.d
        - name: config-map
          mountPath: /mnt/config-map
      - name: clone-mysql
        image: harbor.openscp.com/base/xtrabackup:1 
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Skip the clone if data already exists.
          [[ -d /var/lib/mysql/mysql ]] && exit 0
          # Skip the clone on master (ordinal index 0).
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          [[ $ordinal -eq 0 ]] && exit 0
          # Clone data from previous peer.
          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
          # Prepare the backup.
          xtrabackup --prepare --target-dir=/var/lib/mysql
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
      containers:
      - name: mysql
        image: harbor.openscp.com/base/mysql:5.7.30 
        env:
        - name: MYSQL_ALLOW_EMPTY_PASSWORD
          value: "1"
        ports:
        - name: mysql
          containerPort: 3306
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 500m
            memory: 1Gi
        livenessProbe:
          exec:
            command: ["mysqladmin", "ping"]
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
        readinessProbe:
          exec:
            # Check we can execute queries over TCP (skip-networking is off).
            command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
          initialDelaySeconds: 5
          periodSeconds: 2
          timeoutSeconds: 1
      - name: xtrabackup
        image: harbor.openscp.com/base/xtrabackup:1
        ports:
        - name: xtrabackup
          containerPort: 3307
        command:
        - bash
        - "-c"
        - |
          set -ex
          cd /var/lib/mysql
          # Determine binlog position of cloned data, if any.
          if [[ -f xtrabackup_slave_info ]]; then
            # XtraBackup already generated a partial "CHANGE MASTER TO" query
            # because we're cloning from an existing slave.
            mv xtrabackup_slave_info change_master_to.sql.in
            # Ignore xtrabackup_binlog_info in this case (it's useless).
            rm -f xtrabackup_binlog_info
          elif [[ -f xtrabackup_binlog_info ]]; then
            # We're cloning directly from master. Parse binlog position.
            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
            rm xtrabackup_binlog_info
            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
          fi
          # Check if we need to complete a clone by starting replication.
          if [[ -f change_master_to.sql.in ]]; then
            echo "Waiting for mysqld to be ready (accepting connections)"
            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
            echo "Initializing replication from clone position"
            # In case of container restart, attempt this at-most-once.
            mv change_master_to.sql.in change_master_to.sql.orig
            mysql -h 127.0.0.1 <

执行yaml文件

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl apply -f mysql-statefulset.yaml 
statefulset.apps/mysql created

可以通过运行以下命令查看启动进度

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl get pods -l app=mysql --watch -n pop
NAME      READY   STATUS     RESTARTS   AGE
mysql-0   2/2     Running    0          22s
mysql-1   0/2     Init:0/2   0          4s
mysql-1   0/2     Init:1/2   0          16s
mysql-1   0/2     Init:1/2   0          28s
mysql-1   0/2     PodInitializing   0          36s
mysql-1   1/2     Running           0          37s
mysql-1   2/2     Running           0          42s
mysql-2   0/2     Pending           0          0s
mysql-2   0/2     Pending           0          0s
mysql-2   0/2     Pending           0          2s
mysql-2   0/2     Init:0/2          0          2s
mysql-2   0/2     Init:1/2          0          17s
mysql-2   0/2     Init:1/2          0          30s
mysql-2   0/2     PodInitializing   0          37s
mysql-2   1/2     Error             0          38s
mysql-2   1/2     Running           1 (1s ago)   39s
mysql-2   2/2     Running           1 (6s ago)   44s

一段时间后,应该看到所有 3 个 Pod 进入 Running 状态:

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl get pods -l app=mysql -n pop
NAME      READY   STATUS    RESTARTS       AGE
mysql-0   2/2     Running   0              3m46s
mysql-1   2/2     Running   0              3m28s
mysql-2   2/2     Running   1 (2m8s ago)   2m46s

验证
在MySQL主(mysql-0)上新建一个数据库test

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl exec -it mysql-0 -n pop bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
root@mysql-0:/# mysql -u root -p
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 226
Server version: 5.7.30-log MySQL Community Server (GPL)

Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> CREATE DATABASE test;
Query OK, 1 row affected (0.01 sec)

mysql> show databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| test                   |
| xtrabackup_backupfiles |
+------------------------+
6 rows in set (0.01 sec)

mysql>

在从库(mysql-1,mysql-2)上查询数据库是否存在test

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl exec -it mysql-1 -n pop bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
root@mysql-1:/# mysql -u root -p
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 313
Server version: 5.7.30 MySQL Community Server (GPL)

Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| test                   |
| xtrabackup_backupfiles |
+------------------------+
6 rows in set (0.01 sec)

mysql> 

查看pv绑定情况

root@k8s-ansible-client:~/yaml/20211016/mysql# kubectl get pv -n pop
NAME                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                     STORAGECLASS   REASON   AGE
mysql-datadir-1      10Gi       RWO            Retain           Bound       pop/data-mysql-0                                  115m
mysql-datadir-2      10Gi       RWO            Retain           Bound       pop/data-mysql-1                                  115m
mysql-datadir-3      10Gi       RWO            Retain           Available                                                     115m
mysql-datadir-4      10Gi       RWO            Retain           Available                                                     115m
mysql-datadir-5      10Gi       RWO            Retain           Bound       pop/data-mysql-2                                  115m
mysql-datadir-6      10Gi       RWO            Retain           Available                                                     115m

nfs服务器上映射目录情况

root@k8s-ansible-client:/data/pop/mysql-datadir-1/mysql# ls
auto.cnf    client-cert.pem  ibdata1      ibtmp1              mysql-0-bin.000002  mysql-0-bin.index   public_key.pem   sys
ca-key.pem  client-key.pem   ib_logfile0  mysql               mysql-0-bin.000003  performance_schema  server-cert.pem  test
ca.pem      ib_buffer_pool   ib_logfile1  mysql-0-bin.000001  mysql-0-bin.000004  private_key.pem     server-key.pem   xtrabackup_backupfiles
root@k8s-ansible-client:/data/pop/mysql-datadir-1/mysql# pwd
/data/pop/mysql-datadir-1/mysql

9. 部署Jenkins

9.1 构建镜像

# dockerfile文件
root@k8s-ansible-client:~/yaml/20211016/dockerfile/jenkins# cat Dockerfile 
#Jenkins Version 2.190.1
FROM harbor.openscp.com/base/jdk-base:v8.212

ADD jenkins-2.190.1.war /apps/jenkins/
ADD run_jenkins.sh /usr/bin/


EXPOSE 8080 

CMD ["/usr/bin/run_jenkins.sh"]

# 启动脚本
root@k8s-ansible-client:~/yaml/20211016/dockerfile/jenkins# cat run_jenkins.sh 
#!/bin/bash
cd /apps/jenkins && java -server -Xms1024m -Xmx1024m -Xss512k -jar jenkins-2.190.1.war --webroot=/apps/jenkins/jenkins-data --httpPort=8080

# 构建镜像脚本
root@k8s-ansible-client:~/yaml/20211016/dockerfile/jenkins# cat build-command.sh 
#!/bin/bash
docker build -t  harbor.openscp.com/base/jenkins:v2.190.1 .
echo "镜像制作完成,即将上传至Harbor服务器"
sleep 1
docker push harbor.openscp.com/base/jenkins:v2.190.1
echo "镜像上传完成"

root@k8s-ansible-client:~/yaml/20211016/dockerfile/jenkins# ls
build-command.sh  Dockerfile  jenkins-2.190.1.war  run_jenkins.sh

制作镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/jenkins# bash build-command.sh 
Sending build context to Docker daemon  78.25MB
Step 1/5 : FROM harbor.openscp.com/base/jdk-base:v8.212
 ---> 5ff82b3545df
Step 2/5 : ADD jenkins-2.190.1.war /apps/jenkins/
 ---> 0ff146588622
Step 3/5 : ADD run_jenkins.sh /usr/bin/
 ---> f1845218b8e9
Step 4/5 : EXPOSE 8080
 ---> Running in b9385da9b115
Removing intermediate container b9385da9b115
 ---> 423d3faa4ee3
Step 5/5 : CMD ["/usr/bin/run_jenkins.sh"]
 ---> Running in 517b84d483af
Removing intermediate container 517b84d483af
 ---> ee7f6eb14d7a
Successfully built ee7f6eb14d7a
Successfully tagged harbor.openscp.com/base/jenkins:v2.190.1
镜像制作完成,即将上传至Harbor服务器
The push refers to repository [harbor.openscp.com/base/jenkins]
1435dd08975d: Pushed 
38926d9a4b7c: Pushed 
38dbe7a8225d: Mounted from base/tomcat-app1 
4cdbfe6aa3f6: Mounted from base/tomcat-app1 
3aec209f0edd: Mounted from base/tomcat-app1 
174f56854903: Mounted from base/redis 
v2.190.1: digest: sha256:a8c1486adf54c960f7a8bd1c33c86d164973115fbadd497f3f9ca151dd150db7 size: 1576
镜像上传完成

9.2 创建Jenkins基于nfs实现持久化存储

基于nfs创建pv

root@k8s-ansible-client:~/yaml/20211016/jenkins# cat jenkins-pv.yaml 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-datadir-pv
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 10.10.0.26
    path: /data/pop/jenkins-data 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-root-datadir-pv
  namespace: pop
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 10.10.0.26
    path: /data/pop/jenkins-root-data

root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl apply -f jenkins-pv.yaml 
persistentvolume/jenkins-datadir-pv created
persistentvolume/jenkins-root-datadir-pv created

root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl get pv -n pop
NAME                      CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                     STORAGECLASS   REASON   AGE
jenkins-datadir-pv        10Gi       RWO            Retain           Available                                                     11s
jenkins-root-datadir-pv   10Gi       RWO            Retain           Available                                                     11s

创建pvc

root@k8s-ansible-client:~/yaml/20211016/jenkins# cat jenkins-pvc.yaml 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-datadir-pvc
  namespace: pop
spec:
  volumeName: jenkins-datadir-pv
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-root-data-pvc
  namespace: pop
spec:
  volumeName: jenkins-root-datadir-pv 
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl apply -f jenkins-pvc.yaml 
persistentvolumeclaim/jenkins-datadir-pvc created
persistentvolumeclaim/jenkins-root-data-pvc created

root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl get pvc -n pop
NAME                    STATUS   VOLUME                    CAPACITY   ACCESS MODES   STORAGECLASS   AGE
jenkins-datadir-pvc     Bound    jenkins-datadir-pv        10Gi       RWO                           5s
jenkins-root-data-pvc   Bound    jenkins-root-datadir-pv   10Gi       RWO                           5s

创建Jenkins环境

root@k8s-ansible-client:~/yaml/20211016/jenkins# cat jenkins.yaml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: pop-jenkins
  name: pop-jenkins-deployment
  namespace: pop
spec:
  replicas: 1
  selector:
    matchLabels:
      app: pop-jenkins
  template:
    metadata:
      labels:
        app: pop-jenkins
    spec:
      containers:
      - name: pop-jenkins-container
        image: harbor.openscp.com/base/jenkins:v2.190.1
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http
        volumeMounts:
        - mountPath: "/apps/jenkins/jenkins-data/"
          name: jenkins-datadir-magedu
        - mountPath: "/root/.jenkins"
          name: jenkins-root-datadir
      volumes:
        - name: jenkins-datadir-magedu
          persistentVolumeClaim:
            claimName: jenkins-datadir-pvc
        - name: jenkins-root-datadir
          persistentVolumeClaim:
            claimName: jenkins-root-data-pvc

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: pop-jenkins
  name: pop-jenkins-service
  namespace: pop
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 31080
  selector:
    app: pop-jenkins

root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl apply -f jenkins.yaml 
deployment.apps/pop-jenkins-deployment created
service/pop-jenkins-service created


#查看pods、deployment信息
root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl get pods,deploy -n pop
NAME                                          READY   STATUS    RESTARTS      AGE
pod/deploy-devops-redis-d9fd6594c-fvmmg       1/1     Running   0             2d17h
pod/mysql-0                                   2/2     Running   0             41h
pod/mysql-1                                   2/2     Running   0             41h
pod/mysql-2                                   2/2     Running   1 (41h ago)   41h
pod/pop-jenkins-deployment-58d59b9bf5-llcqs   1/1     Running   0             27s

NAME                                     READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deploy-devops-redis      1/1     1            1           2d17h
deployment.apps/pop-jenkins-deployment   1/1     1            1           28s

# 查看svc信息
root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl get svc -n pop
NAME                  TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
pop-jenkins-service   NodePort    10.68.139.115           80:31080/TCP     37s

验证

image.png

获取Jenkins的admin密码

root@k8s-ansible-client:~/yaml/20211016/jenkins# kubectl exec -it pod/pop-jenkins-deployment-58d59b9bf5-llcqs -n pop bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@pop-jenkins-deployment-58d59b9bf5-llcqs /]# cat /root/.jenkins/secrets/initialAdminPassword
c174844ecff54595ba7ddea4e4400abf

查看nfs挂载硬盘信息

# pods里面挂载信息
[root@pop-jenkins-deployment-58d59b9bf5-llcqs /]# df -Th
Filesystem                             Type     Size  Used Avail Use% Mounted on
overlay                                overlay   20G  9.5G  9.2G  51% /
tmpfs                                  tmpfs     64M     0   64M   0% /dev
tmpfs                                  tmpfs    2.0G     0  2.0G   0% /sys/fs/cgroup
10.10.0.26:/data/pop/jenkins-root-data nfs4     295G  806M  279G   1% /root/.jenkins
/dev/mapper/ubuntu--vg-ubuntu--lv      ext4      20G  9.5G  9.2G  51% /etc/hosts
shm                                    tmpfs     64M     0   64M   0% /dev/shm
10.10.0.26:/data/pop/jenkins-data      nfs4     295G  806M  279G   1% /apps/jenkins/jenkins-data
tmpfs                                  tmpfs    3.2G   12K  3.2G   1% /run/secrets/kubernetes.io/serviceaccount
tmpfs                                  tmpfs    2.0G     0  2.0G   0% /proc/acpi
tmpfs                                  tmpfs    2.0G     0  2.0G   0% /proc/scsi
tmpfs                                  tmpfs    2.0G     0  2.0G   0% /sys/firmware

# nfs服务器上目录的信息
root@k8s-ansible-client:/data/pop/jenkins-data# ls
 bootstrap              dc-license.txt   help             jsbundles                      LogFileOutputStream.class       'MainDialog$1$1.class'   META-INF     WEB-INF
 ColorFormatter.class   executable       images          'LogFileOutputStream$1.class'  'Main$FileAndDescription.class'  'MainDialog$1.class'     robots.txt   winstone.jar
 css                    favicon.ico      JNLPMain.class  'LogFileOutputStream$2.class'   Main.class                       MainDialog.class        scripts
root@k8s-ansible-client:/data/pop/jenkins-data# cd ../jenkins-root-data/
root@k8s-ansible-client:/data/pop/jenkins-root-data# ls
config.xml                     jenkins.install.InstallUtil.installingPlugins  jobs              nodes       secret.key.not-so-secret  userContent
hudson.model.UpdateCenter.xml  jenkins.install.UpgradeWizard.state            logs              plugins     secrets                   users
identity.key.enc               jenkins.telemetry.Correlator.xml               nodeMonitors.xml  secret.key  updates
root@k8s-ansible-client:/data/pop/jenkins-root-data# 

10. k8s实战案例

10.1 web站点案例

实现nginx+php+wordpress+mysql环境

10.1.1 构建 php镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# ls
build-command.sh  Dockerfile  run_php.sh  www.conf
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# cat Dockerfile 
#PHP Base Image
FROM harbor.openscp.com/base/centos:centos7.9.2009

RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop &&  rm -rf /etc/localtime && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd  www -u 2020 && useradd nginx -u 2021
RUN yum install -y  https://mirrors.tuna.tsinghua.edu.cn/remi/enterprise/remi-release-7.rpm && yum install  php56-php-fpm php56-php-mysql -y 
ADD www.conf /opt/remi/php56/root/etc/php-fpm.d/www.conf
#RUN useradd nginx -u 2019
ADD run_php.sh /usr/local/bin/run_php.sh
EXPOSE 9000

CMD ["/usr/local/bin/run_php.sh"]

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# cat build-command.sh 
#!/bin/bash
TAG=$1
docker build -t harbor.openscp.com/base/wordpress-php-5.6:${TAG} .
echo "镜像制作完成,即将上传至Harbor服务器"
sleep 1
docker push harbor.openscp.com/base/wordpress-php-5.6:${TAG}
echo "镜像上传完成"

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# cat run_php.sh 
#!/bin/bash

/opt/remi/php56/root/usr/sbin/php-fpm
tail -f /etc/hosts

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# grep -v "^;" www.conf |grep -v "^$"
[www]
user = nginx
group = nginx
listen = 0.0.0.0:9000
pm = dynamic
pm.max_children = 50
pm.start_servers = 5
pm.min_spare_servers = 5
pm.max_spare_servers = 35
slowlog = /opt/remi/php56/root/var/log/php-fpm/www-slow.log
php_admin_value[error_log] = /opt/remi/php56/root/var/log/php-fpm/www-error.log
php_admin_flag[log_errors] = on
php_value[session.save_handler] = files
php_value[session.save_path]    = /opt/remi/php56/root/var/lib/php/session
php_value[soap.wsdl_cache_dir]  = /opt/remi/php56/root/var/lib/php/wsdlcache

执行脚本构建镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# chmod a+x *.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/php# bash build-command.sh v1
...
Successfully tagged harbor.openscp.com/base/wordpress-php-5.6:v1
镜像制作完成,即将上传至Harbor服务器
The push refers to repository [harbor.openscp.com/base/wordpress-php-5.6]
18125a50a52c: Pushed 
017c00d221f8: Pushed 
3bab9e7267e6: Pushed 
7fc5345cbe01: Mounted from base/nginx-web1 
174f56854903: Mounted from base/jenkins 
v1: digest: sha256:d5a6abed76905f428d164520a5c728e99625d88bc1dbc6c62d2bc6fe384a6714 size: 1369
镜像上传完成

10.1.2 构建NGINX镜像

先构建nginx的基础镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx-base# cat Dockerfile 
#Nginx Base Image
FROM harbor.openscp.com/base/centos:centos7.9.2009

RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop &&  rm -rf /etc/localtime && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd  www -u 2020 && useradd nginx -u 2021
ADD nginx-1.14.2.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.14.2 && ./configure --prefix=/apps/nginx  && make && make install && ln -sv  /apps/nginx/sbin/nginx /usr/sbin/nginx  &&rm -rf /usr/local/src/nginx-1.14.2.tar.gz 
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx-base# cat build-command.sh 
#!/bin/bash
docker build -t harbor.openscp.com/base/nginx-base-wordpress:v1.14.2  .
sleep 1
docker push  harbor.openscp.com/base/nginx-base-wordpress:v1.14.2

执行脚本构建nginx基础镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx-base# bash build-command.sh
...
Successfully built b73c4576e520
Successfully tagged harbor.openscp.com/base/nginx-base-wordpress:v1.14.2
The push refers to repository [harbor.openscp.com/base/nginx-base-wordpress]
154a43bb903c: Pushed 
3265817f225b: Mounted from base/nginx-web1 
7fc5345cbe01: Mounted from base/wordpress-php-5.6 
174f56854903: Mounted from base/wordpress-php-5.6 
v1.14.2: digest: sha256:52412ed50aff876003c4834f00a8d60d7624ab770444ff8f2cac6fd21712ced3 size: 1164

制作业务nginx镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# ls
build-command.sh  Dockerfile  index.html  nginx.conf  run_nginx.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# cat Dockerfile 
#FROM harbor.magedu.local/pub-images/nginx-base-wordpress:v1.14.2 
FROM harbor.openscp.com/base/nginx-base-wordpress:v1.14.2

ADD nginx.conf /apps/nginx/conf/nginx.conf
ADD run_nginx.sh /apps/nginx/sbin/run_nginx.sh
RUN mkdir -pv /home/nginx/wordpress
RUN chown nginx.nginx /home/nginx/wordpress/ -R

EXPOSE 80 443

CMD ["/apps/nginx/sbin/run_nginx.sh"] 
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# cat index.html 
nginx web1
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# cat nginx.conf 
user  nginx nginx;
worker_processes  auto;

events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    sendfile        on;

    keepalive_timeout  65;
    client_max_body_size 10M;
    client_body_buffer_size 16k;
    client_body_temp_path  /apps/nginx/tmp   1 2 2;
    gzip  on;


    server {
        listen       80;
        server_name  blogs.openscp.com;

        location / {
            root    /home/nginx/wordpress;
            index   index.php index.html index.htm;
        }

        location ~ \.php$ {
            root           /home/nginx/wordpress;
            fastcgi_pass   127.0.0.1:9000;
            fastcgi_index  index.php;
            fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
             include        fastcgi_params;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }

}
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# cat run_nginx.sh 
#!/bin/bash
/apps/nginx/sbin/nginx
tail -f /etc/hosts
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# cat build-command.sh 
#!/bin/bash
TAG=$1
docker build -t harbor.openscp.com/base/wordpress-nginx:${TAG} .
echo "镜像制作完成,即将上传至Harbor服务器"
sleep 1
docker push  harbor.openscp.com/base/wordpress-nginx:${TAG}
echo "镜像上传完成"

执行脚本构建业务Nginx镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# chmod a+x *.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/nginx# bash build-command.sh v1
...
Successfully tagged harbor.openscp.com/base/wordpress-nginx:v1
镜像制作完成,即将上传至Harbor服务器
The push refers to repository [harbor.openscp.com/base/wordpress-nginx]
2e4717b51034: Pushed 
e32627e01bb4: Pushed 
7a8e3034ed91: Pushed 
b79d4f07216e: Pushed 
25e48aff2729: Mounted from base/nginx-base-wordpress 
5b5981584815: Mounted from base/nginx-base-wordpress 
c6a91dc597a0: Mounted from base/nginx-base-wordpress 
174f56854903: Mounted from base/nginx-base-wordpress 
v1: digest: sha256:98d50b474682ef18369a6468a1e488d0baafa3324099fded4dc9ff754a84250d size: 1992
镜像上传完成

10.1.3 部署wordpress

yaml文件,基于nfs实现持久化存储

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/wordpress# cat wordpress.yaml 
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: wordpress-app
  name: wordpress-app-deployment
  namespace: pop
spec:
  replicas: 1
  selector:
    matchLabels:
      app: wordpress-app
  template:
    metadata:
      labels:
        app: wordpress-app
    spec:
      containers:
      - name: wordpress-app-nginx
        image: harbor.openscp.com/base/wordpress-nginx:v1
        imagePullPolicy: Always
        ports:
        - containerPort: 80
          protocol: TCP
          name: http
        - containerPort: 443
          protocol: TCP
          name: https
        volumeMounts:
        - name: wordpress
          mountPath: /home/nginx/wordpress
          readOnly: false

      - name: wordpress-app-php
        image: harbor.openscp.com/base/wordpress-php-5.6:v1
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 9000
          protocol: TCP
          name: http
        volumeMounts:
        - name: wordpress
          mountPath: /home/nginx/wordpress
          readOnly: false

      volumes:
      - name: wordpress
        nfs:
          server: 10.10.0.26
          path: /data/pop/wordpress 


---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: wordpress-app
  name: wordpress-app-spec
  namespace: pop
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30031
  - name: https
    port: 443
    protocol: TCP
    targetPort: 443
    nodePort: 30033
  selector:
    app: wordpress-app

基于上面MySQL主从数据库,创建数据库(wordpress)和帐号(wordpress 123456)

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/wordpress# kubectl exec -it mysql-0 -n pop bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
root@mysql-0:/# mysql -u root -p
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 91544
Server version: 5.7.30-log MySQL Community Server (GPL)

Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> create database wordpress;
Query OK, 1 row affected (0.01 sec)

mysql> 
mysql> grant all privileges on wordpress.* to wordpress@'%' identified by '123456';
Query OK, 0 rows affected, 1 warning (0.05 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.05 sec)

mysql>

执行yaml构建wordpress环境

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/wordpress# kubectl apply -f wordpress.yaml 
deployment.apps/wordpress-app-deployment created
service/wordpress-app-spec created

root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/wordpress# kubectl get pods,deploy -n pop
NAME                                            READY   STATUS    RESTARTS      AGE
pod/deploy-devops-redis-d9fd6594c-fvmmg         1/1     Running   0             2d18h
pod/mysql-0                                     2/2     Running   0             42h
pod/mysql-1                                     2/2     Running   0             42h
pod/mysql-2                                     2/2     Running   1 (42h ago)   42h
pod/wordpress-app-deployment-5f98868c5b-4775l   2/2     Running   0             4s

NAME                                       READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deploy-devops-redis        1/1     1            1           2d18h
deployment.apps/wordpress-app-deployment   1/1     1            1           4s
root@k8s-ansible-client:~/yaml/20211016/dockerfile/web/wordpress# kubectl get svc -n pop
NAME                 TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
mysql                ClusterIP   None                    3306/TCP                     44h
mysql-read           ClusterIP   10.68.181.29            3306/TCP                     44h
wordpress-app-spec   NodePort    10.68.208.19            80:30031/TCP,443:30033/TCP   115s

下载wordpress安装包https://cn.wordpress.org/download/

# 在nfs服务器上
root@k8s-ansible-client:/data/pop/wordpress# wget https://cn.wordpress.org/latest-zh_CN.tar.gz
# 解压安装包
root@k8s-ansible-client:/data/pop/wordpress# tar -zxvf latest-zh_CN.tar.gz
root@k8s-ansible-client:/data/pop/wordpress# ls
index.html  latest-zh_CN.tar.gz  wordpress
root@k8s-ansible-client:/data/pop/wordpress# rm -fr index.html latest-zh_CN.tar.gz 
root@k8s-ansible-client:/data/pop/wordpress# mv wordpress/* .
root@k8s-ansible-client:/data/pop/wordpress# ls
index.php    readme.html  wp-activate.php  wp-blog-header.php    wp-config-sample.php  wp-cron.php  wp-links-opml.php  wp-login.php  wp-settings.php  wp-trackback.php
license.txt  wordpress    wp-admin         wp-comments-post.php  wp-content            wp-includes  wp-load.php        wp-mail.php   wp-signup.php    xmlrpc.php
root@k8s-ansible-client:/data/pop/wordpress# rm -fr wordpress/

安装wordpress

image.png

注:如果权限有问题,将wordpress目录加上nginx权限

数据库名称和用户密码是上面创建的,连接地址:mysql-0.mysql.pop.svc.pop.local


image.png

image.png

登录并发表一篇文章


image.png

image.png

10.2 微服务案例

实现dubbo+zookeeper微服务环境

10.2.1 构建镜像

consumer

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# ls
build-command.sh  Dockerfile  dubbo-demo-consumer-2.1.5  dubbo-demo-consumer-2.1.5-assembly.tar.gz  run_java.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# cat Dockerfile 
#Dubbo consumer
FROM harbor.openscp.com/base/jdk-base:v8.212

RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop&&  rm -rf /etc/localtime && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd nginx -u 2001
RUN yum install file -y
RUN mkdir -p /apps/dubbo/consumer 
ADD dubbo-demo-consumer-2.1.5  /apps/dubbo/consumer
ADD run_java.sh /apps/dubbo/consumer/bin 
RUN chown nginx.nginx /apps -R
RUN chmod a+x /apps/dubbo/consumer/bin/*.sh

CMD ["/apps/dubbo/consumer/bin/run_java.sh"]

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# cat run_java.sh 
#!/bin/bash
su - nginx -c "/apps/dubbo/consumer/bin/start.sh"
tail -f /etc/hosts

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# cat build-command.sh 
#!/bin/bash
docker build -t harbor.openscp.com/base/dubbo-demo-consumer:v1  .
sleep 3
docker push harbor.openscp.com/base/dubbo-demo-consumer:v1

# 需要配置注册中心地址
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# grep -v "^#" dubbo-demo-consumer-2.1.5/conf/dubbo.properties
dubbo.container=log4j,spring
dubbo.application.name=demo-consumer
dubbo.application.owner=
dubbo.registry.address=zookeeper://zookeeper1.default.svc.pop.local:2181 | zookeeper://zookeeper2.default.svc.pop.local:2181 | zookeeper://zookeeper3.default.svc.pop.local:2181
dubbo.monitor.protocol=registry
dubbo.log4j.file=logs/dubbo-demo-consumer.log
dubbo.log4j.level=WARN

执行脚本构建镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# chmod a+x *.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/consumer# bash build-command.sh
...
Successfully tagged harbor.openscp.com/base/dubbo-demo-consumer:v1
The push refers to repository [harbor.openscp.com/base/dubbo-demo-consumer]
6138693121b5: Pushed 
4fae946992b0: Pushed 
1429d5765bc5: Pushed 
d75b5de4df30: Pushed 
c67ef8dd101d: Pushed 
a53b249d3f92: Pushed 
bc1760d32325: Pushed 
38dbe7a8225d: Mounted from base/jenkins 
4cdbfe6aa3f6: Mounted from base/jenkins 
3aec209f0edd: Mounted from base/jenkins 
174f56854903: Mounted from base/wordpress-nginx 
v1: digest: sha256:413825db8a03f0eed51fd3056041059bc35fd79bd18b4fd7285a9fedbfc3913f size: 2628

dubboadmin

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin# ls
build-command.sh  catalina.sh  Dockerfile  dubboadmin.war  dubboadmin.zip  logging.properties  run_tomcat.sh  server.xml
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin# cat Dockerfile 
#Dubbo dubboadmin
FROM harbor.openscp.com/base/tomcat-base:v8.5.43

RUN yum install -y vim unzip wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop&&  rm -rf /etc/localtime && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
ADD server.xml /apps/tomcat/conf/server.xml
ADD logging.properties /apps/tomcat/conf/logging.properties
ADD catalina.sh /apps/tomcat/bin/catalina.sh
ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh
ADD dubboadmin.war  /data/tomcat/webapps/dubboadmin.war
RUN cd /data/tomcat/webapps && unzip dubboadmin.war && rm -rf dubboadmin.war && chown -R nginx.nginx /data /apps

EXPOSE 8080 8443

CMD ["/apps/tomcat/bin/run_tomcat.sh"]

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin# cat build-command.sh 
#!/bin/bash
TAG=$1
docker build -t harbor.openscp.com/base/dubboadmin:${TAG}  .
sleep 3
docker push  harbor.openscp.com/base/dubboadmin:${TAG}

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin# cat run_tomcat.sh 
#!/bin/bash

su - nginx -c "/apps/tomcat/bin/catalina.sh start"
su - nginx -c "tail -f /etc/hosts"

# 需要配置注册中心地址
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin# cat dubboadmin/WEB-INF/dubbo.properties 
dubbo.registry.address=zookeeper://zookeeper1.default.svc.pop.local:2181 | zookeeper://zookeeper2.default.svc.pop.local:2181 | zookeeper://zookeeper3.default.svc.pop.local:2181
dubbo.admin.root.password=root
dubbo.admin.guest.password=guest

执行脚本构建镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin#  chmo a+x *.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/dubboadmin# bash build-command.sh v1
...
Successfully tagged harbor.openscp.com/base/dubboadmin:v1
The push refers to repository [harbor.openscp.com/base/dubboadmin]
98e0d611e025: Pushed 
6dc138ab6f03: Pushed 
82e5ee6a3c84: Pushed 
5ddfa7134fcc: Pushed 
44868cb8c702: Pushed 
aaefdcea4576: Pushed 
2ff59fc7c569: Pushed 
6dc737cf89d1: Pushed 
d5123d987925: Mounted from base/tomcat-app1 
afa3eb2a2173: Mounted from base/tomcat-app1 
7136febc3401: Mounted from base/tomcat-app1 
38dbe7a8225d: Mounted from base/dubbo-demo-consumer 
4cdbfe6aa3f6: Mounted from base/dubbo-demo-consumer 
3aec209f0edd: Mounted from base/dubbo-demo-consumer 
174f56854903: Mounted from base/dubbo-demo-consumer 
v1: digest: sha256:2f4e1148295e244c0188c2dd8048d673bbda3c0a11b421d1be25b42e81ca19b6 size: 3466

provider, 配置注册中心地址

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# ls
build-command.sh  Dockerfile  dubbo-demo-provider-2.1.5  dubbo-demo-provider-2.1.5-assembly.tar.gz  run_java.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# cat Dockerfile 
#Dubbo provider
FROM harbor.openscp.com/base/jdk-base:v8.212

RUN yum install -y vim wget tree file nc lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop&&  rm -rf /etc/localtime && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd nginx -u 2001
RUN mkdir -p /apps/dubbo/provider
ADD dubbo-demo-provider-2.1.5/  /apps/dubbo/provider
ADD run_java.sh /apps/dubbo/provider/bin 
RUN chown nginx.nginx /apps -R
RUN chmod a+x /apps/dubbo/provider/bin/*.sh

CMD ["/apps/dubbo/provider/bin/run_java.sh"]

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# cat build-command.sh 
#!/bin/bash
docker build -t harbor.openscp.com/base/dubbo-demo-provider:v1  .
sleep 3
docker push harbor.openscp.com/base/dubbo-demo-provider:v1

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# cat run_java.sh 
#!/bin/bash
su - nginx -c "/apps/dubbo/provider/bin/start.sh"
tail -f /etc/hosts

# 需要配置注册中心地址
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# grep -v "^#" dubbo-demo-provider-2.1.5/conf/dubbo.properties 
dubbo.container=log4j,spring
dubbo.application.name=demo-provider
dubbo.application.owner=
dubbo.registry.address=zookeeper://zookeeper1.default.svc.pop.local:2181 | zookeeper://zookeeper2.default.svc.pop.local:2181 | zookeeper://zookeeper3.default.svc.pop.local:2181
dubbo.monitor.protocol=registry
dubbo.protocol.name=dubbo
dubbo.protocol.port=20880
dubbo.log4j.file=logs/dubbo-demo-provider.log
dubbo.log4j.level=WARN

执行脚本构建镜像

root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# chmod a+x *.sh
root@k8s-ansible-client:~/yaml/20211016/dockerfile/dubbo/provider# bash build-command.sh
...
Successfully tagged harbor.openscp.com/base/dubbo-demo-provider:v1
The push refers to repository [harbor.openscp.com/base/dubbo-demo-provider]
c32486dc8751: Pushed 
719e351a9ee9: Pushed 
8f08fdf8a695: Pushed 
ee248e4a3744: Pushed 
fad539a3eb64: Pushed 
9ff991a489ff: Pushed 
38dbe7a8225d: Mounted from base/dubboadmin 
4cdbfe6aa3f6: Mounted from base/dubboadmin 
3aec209f0edd: Mounted from base/dubboadmin 
174f56854903: Mounted from base/dubboadmin 
v1: digest: sha256:5eeebdbbe537f385bbed92b7f24335af11c3f6d38285741390d2151c361010e8 size: 2416

10.2.2 部署微服务环境

yaml文件

root@k8s-ansible-client:~/yaml/20211016/dubbo# cat provider.yaml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: pop-provider
  name: pop-provider-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: pop-provider
  template:
    metadata:
      labels:
        app: pop-provider
    spec:
      containers:
      - name: pop-provider-container
        image: harbor.openscp.com/base/dubbo-demo-provider:v1
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 20880
          protocol: TCP
          name: http

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: pop-provider
  name: pop-provider-spec
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 30880
    #nodePort: 30001
  selector:
    app: pop-provider

root@k8s-ansible-client:~/yaml/20211016/dubbo# cat dubboadmin.yaml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: pop-dubboadmin
  name: pop-dubboadmin-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: pop-dubboadmin
  template:
    metadata:
      labels:
        app: pop-dubboadmin
    spec:
      containers:
      - name: pop-dubboadmin-container
        image: harbor.openscp.com/base/dubboadmin:v1
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: pop-dubboadmin
  name: pop-dubboadmin-service
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 30080
  selector:
    app: pop-dubboadmin

root@k8s-ansible-client:~/yaml/20211016/dubbo# cat consumer.yaml 
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: pop-consumer
  name: pop-consumer-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: pop-consumer
  template:
    metadata:
      labels:
        app: pop-consumer
    spec:
      containers:
      - name: pop-consumer-container
        image: harbor.openscp.com/base/dubbo-demo-consumer:v1
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 80
          protocol: TCP
          name: http

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: pop-consumer
  name: pop-consumer-server
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
    #nodePort: 30001
  selector:
    app: pop-consumer

启动provider(生产者)

root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl apply -f provider.yaml 
deployment.apps/pop-provider-deployment created
service/pop-provider-spec created

root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl get pods,deploy 
NAME                                           READY   STATUS             RESTARTS           AGE
pod/alpine-test                                1/1     Running            50 (53m ago)       28d
pod/kube100-site                               2/2     Running            0                  14d
pod/nginx                                      0/1     CrashLoopBackOff   1611 (3m21s ago)   4d
pod/nginx-test-001                             1/1     Running            26 (90m ago)       15d
pod/nginx-test1                                1/1     Running            50 (63m ago)       28d
pod/nginx-test2                                1/1     Running            50 (63m ago)       28d
pod/nginx-test3                                1/1     Running            50 (63m ago)       28d
pod/pop-provider-deployment-6dfd4d78db-gl4rt   1/1     Running            0                  20s
pod/zookeeper1-cdbb7fbc-5pgdg                  1/1     Running            1 (6d3h ago)       6d3h
pod/zookeeper2-f4944446d-2xnjd                 1/1     Running            0                  6d3h
pod/zookeeper3-589f6bc7-2mnz6                  1/1     Running            0                  6d3h

NAME                                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/pop-provider-deployment   1/1     1            1           20s
deployment.apps/zookeeper1                1/1     1            1           6d3h
deployment.apps/zookeeper2                1/1     1            1           6d3h
deployment.apps/zookeeper3                1/1     1            1           6d3h
root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl get svc
NAME                TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                                        AGE
kubernetes          ClusterIP   10.68.0.1              443/TCP                                        29d
pop-provider-spec   NodePort    10.68.4.34             80:31797/TCP                                   25s
zookeeper1          NodePort    10.68.42.189           2181:32181/TCP,2888:30923/TCP,3888:30168/TCP   6d3h
zookeeper2          NodePort    10.68.78.146           2181:32182/TCP,2888:31745/TCP,3888:30901/TCP   6d3h
zookeeper3          NodePort    10.68.199.44           2181:32183/TCP,2888:32488/TCP,3888:31621/TCP   6d3h

验证,使用window客户端zooinspector查看zookeeper里面的信息


image.png

image.png

可以发现信息已经注册到zookeeper里面

启动consumer(消费者)

root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl apply -f consumer.yaml 
deployment.apps/pop-consumer-deployment created
service/pop-consumer-server created

验证,查看provider(生产者)日志

root@k8s-ansible-client:~/yaml/20211010/03/dockerfile# kubectl exec -it pod/pop-provider-deployment-6dfd4d78db-gl4rt bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead
[root@pop-provider-deployment-6dfd4d78db-gl4rt logs]# tail -f *.log
...
==> stdout.log <==
    at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:137)
    at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:76)
    at org.jboss.netty.channel.Channels.write(Channels.java:632)
    at org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:70)
    at com.alibaba.dubbo.remoting.transport.netty.NettyHandler.writeRequested(NettyHandler.java:99)
    at org.jboss.netty.channel.Channels.write(Channels.java:611)
    at org.jboss.netty.channel.Channels.write(Channels.java:578)
    at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:251)
    at com.alibaba.dubbo.remoting.transport.netty.NettyChannel.send(NettyChannel.java:98)
    ... 6 more



[00:57:55] Hello world0, request from consumer: /172.20.213.27:35450
[00:57:57] Hello world1, request from consumer: /172.20.213.27:35450
[00:57:59] Hello world2, request from consumer: /172.20.213.27:35450
[00:58:01] Hello world3, request from consumer: /172.20.213.27:35450
[00:58:03] Hello world4, request from consumer: /172.20.213.27:35450
[00:58:05] Hello world5, request from consumer: /172.20.213.27:35450
[00:58:07] Hello world6, request from consumer: /172.20.213.27:35450

启动dubboadmin

root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl apply -f dubboadmin.yaml 
deployment.apps/pop-dubboadmin-deployment created
service/pop-dubboadmin-service created
root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl get pods -o wide
NAME                                        READY   STATUS             RESTARTS           AGE    IP              NODE             NOMINATED NODE   READINESS GATES
alpine-test                                 1/1     Running            50 (76m ago)       28d    172.20.108.65   192.168.20.236              
kube100-site                                2/2     Running            0                  14d    172.20.213.6    192.168.20.253              
nginx                                       0/1     CrashLoopBackOff   1617 (4m28s ago)   4d     172.20.213.20   192.168.20.253              
nginx-test-001                              1/1     Running            26 (113m ago)      15d    172.20.191.10   192.168.20.147              
nginx-test1                                 1/1     Running            50 (86m ago)       28d    172.20.191.2    192.168.20.147              
nginx-test2                                 1/1     Running            50 (86m ago)       28d    172.20.213.3    192.168.20.253              
nginx-test3                                 1/1     Running            50 (86m ago)       28d    172.20.191.3    192.168.20.147              
pop-consumer-deployment-54b54559d7-2dd2k    1/1     Running            0                  10m    172.20.213.27   192.168.20.253              
pop-dubboadmin-deployment-75f8d75df-drcz6   1/1     Running            0                  38s    172.20.108.91   192.168.20.236              
pop-provider-deployment-6dfd4d78db-gl4rt    1/1     Running            0                  23m    172.20.191.33   192.168.20.147              
zookeeper1-cdbb7fbc-5pgdg                   1/1     Running            1 (6d3h ago)       6d3h   172.20.191.27   192.168.20.147              
zookeeper2-f4944446d-2xnjd                  1/1     Running            0                  6d3h   172.20.108.81   192.168.20.236              
zookeeper3-589f6bc7-2mnz6                   1/1     Running            0                  6d3h   172.20.191.28   192.168.20.147              
root@k8s-ansible-client:~/yaml/20211016/dubbo# kubectl get svc
NAME                     TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                                        AGE
kubernetes               ClusterIP   10.68.0.1               443/TCP                                        29d
pop-consumer-server      NodePort    10.68.122.230           80:30926/TCP                                   12m
pop-dubboadmin-service   NodePort    10.68.60.83             80:30080/TCP                                   2m23s
pop-provider-spec        NodePort    10.68.4.34              80:31797/TCP                                   24m
zookeeper1               NodePort    10.68.42.189            2181:32181/TCP,2888:30923/TCP,3888:30168/TCP   6d3h
zookeeper2               NodePort    10.68.78.146            2181:32182/TCP,2888:31745/TCP,3888:30901/TCP   6d3h
zookeeper3               NodePort    10.68.199.44            2181:32183/TCP,2888:32488/TCP,3888:31621/TCP   6d3h

验证,登录帐号密码都是root


image.png

11. Ingress

11.1 什么是ingress

Ingress 公开了从集群外部到集群内服务的 HTTP 和 HTTPS 路由。 流量路由由 Ingress 资源上定义的规则控制。

简单的流程图,如下:


image.png

可以将 Ingress 配置为服务提供外部可访问的 URL、负载均衡流量、终止 SSL/TLS,以及提供基于名称的虚拟主机等能力。 Ingress 控制器 通常负责通过负载均衡器来实现 Ingress,尽管它也可以配置边缘路由器或其他前端来帮助处理流量。

Ingress 不会公开任意端口或协议。 将 HTTP 和 HTTPS 以外的服务公开到 Internet 时,通常使用 Service.Type=NodePort 或 Service.Type=LoadBalancer 类型的服务。

11.2 Ingress controller

ingress Controller 通过监听 Ingress这个api对象里的配置规则并转化成 Nginx 的配置(kubernetes声明式API和控制循环) , 然后对外部提供服务。

核心是一个deployment,实现方式有很多,比如nginx, Contour, Haproxy, trafik, Istio,需要编写的yaml有:Deployment, Service, ConfigMap, ServiceAccount(Auth),其中service的类型可以是NodePort或者LoadBalancer。

Ingress Controller是将Ingress这种变化生成一段Nginx的配置,然后将这个配置通过Kubernetes API写到Nginx的Pod中,然后reload.(注意:写入 nginx.conf 的不是service的地址,而是service backend 的 pod 的地址,避免在 service 在增加一层负载均衡转发)

image.png

从上图中可以很清晰的看到,实际上请求进来还是被负载均衡器拦截,比如 nginx,然后 Ingress Controller 通过跟 Ingress 交互得知某个域名对应哪个 service,再通过跟 kubernetes API 交互得知 service 地址等信息;综合以后生成配置文件实时写入负载均衡器,然后负载均衡器 reload 该规则便可实现服务发现,即动态映射

了解了以上内容以后,这也就很好的说明了我为什么喜欢把负载均衡器部署为 Daemon Set;因为无论如何请求首先是被负载均衡器拦截的,所以在每个 node 上都部署一下,同时 hostport 方式监听 80 端口;那么就解决了其他方式部署不确定 负载均衡器在哪的问题,同时访问每个 node 的 80 都能正确解析请求;如果前端再 放个 nginx 就又实现了一层负载均衡。

Ingress Controller 会根据你定义的 Ingress 对象,提供对应的代理能力。业界常用的各种反向代理项目,比如 Nginx、HAProxy、Envoy、Traefik 等,都已经为Kubernetes 专门维护了对应的 Ingress Controller。


image.png

11.3 ingress部署

ingress调度的是后端的service,而不是pod

ingress controller 是与k8s集群做交互的,感知etcd的变化

部署nginx-ingress-controller
https://kubernetes.github.io/ingress-nginx/deploy/
yaml地址

root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl apply -f aa.yaml 
namespace/ingress-nginx created
serviceaccount/ingress-nginx created
configmap/ingress-nginx-controller created
clusterrole.rbac.authorization.k8s.io/ingress-nginx created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created
role.rbac.authorization.k8s.io/ingress-nginx created
rolebinding.rbac.authorization.k8s.io/ingress-nginx created
service/ingress-nginx-controller-admission created
service/ingress-nginx-controller created
deployment.apps/ingress-nginx-controller created
ingressclass.networking.k8s.io/nginx created
validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created
serviceaccount/ingress-nginx-admission created
clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
role.rbac.authorization.k8s.io/ingress-nginx-admission created
rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
job.batch/ingress-nginx-admission-create created
job.batch/ingress-nginx-admission-patch created

root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl get pods,deploy -n ingress-nginx
NAME                                           READY   STATUS      RESTARTS   AGE
pod/ingress-nginx-admission-create--1-n7f7b    0/1     Completed   0          21s
pod/ingress-nginx-admission-patch--1-cd57p     0/1     Completed   1          21s
pod/ingress-nginx-controller-cfb66bc7b-bflgv   1/1     Running     0          21s

NAME                                       READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/ingress-nginx-controller   1/1     1            1           21s

11.3.1 基于域名的多虚拟主机

root@k8s-ansible-client:~/yaml/20211016/ingress# cat ingress_single-host.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-web
  namespace: pop
  annotations:
    kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
spec:
  rules: #路由规则
  - host: www.pop.com ##客户端访问的host域名
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: pop-tomcat-app1-service
            port:
               number: 80

root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl apply -f ingress_single-host.yaml 
ingress.networking.k8s.io/nginx-web created

root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl get ingress -n pop
NAME        CLASS    HOSTS         ADDRESS          PORTS   AGE
nginx-web      www.pop.com   192.168.20.253   80      2m44s

验证

image.png

11.3.2 基于域名的url转发

root@k8s-ansible-client:~/yaml/20211016/ingress# cat ingress-url.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-web
  namespace: pop
  annotations:
    kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
spec:
  rules:
  - host: www.pop.com
    http:
      paths:
      - path: /url1
        pathType: Prefix
        backend:
          service:
            name: pop-tomcat-app1-service
            port:
               number: 80
      - path: /url2
        pathType: Prefix
        backend:
          service:
            name: pop-tomcat-app2-service
            port:
               number: 80

root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl apply -f ingress-url.yaml 
ingress.networking.k8s.io/nginx-web created
root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl get ingress -n pop
NAME        CLASS    HOSTS         ADDRESS          PORTS   AGE
nginx-web      www.pop.com   192.168.20.253   80      9s

验证

image.png

image.png

11.3.3 基于多域名https

自签证书

# 域名 www.pop.com
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# openssl req -x509 -sha256 -newkey rsa:4096 -keyout ca.key -out ca.crt -days 3560 -nodes -subj '/CN=www.pop.com'
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# openssl req -new -newkey rsa:4096 -keyout server.key -out server.csr -nodes -subj '/CN=www.pop.com'
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# openssl x509 -req -sha256 -days 3650 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt
Signature ok
subject=CN = www.pop.com
Getting CA Private Key

# 域名 test.pop.net
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# openssl req -new -newkey rsa:4096 -keyout test.key -out test.csr -nodes -subj '/CN=test.pop.net'
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# openssl x509 -req -sha256 -days 3650 -in test.csr -CA ca.crt -CAkey ca.key -set_serial 01  -out test.crt
Signature ok
subject=CN = test.pop.net
Getting CA Private Key

证书上传至k8s

root@k8s-ansible-client:~/yaml/20211016/ingress/certs# kubectl  create secret generic www-tls-secret --from-file=tls.crt=server.crt --from-file=tls.key=server.key -n pop
secret/www-tls-secret created
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# kubectl  create secret generic test-tls-secret --from-file=tls.crt=test.crt --from-file=tls.key=test.key -n pop
secret/test-tls-secret created
root@k8s-ansible-client:~/yaml/20211016/ingress/certs# kubectl get secret -n pop
NAME                  TYPE                                  DATA   AGE
default-token-4nrw2   kubernetes.io/service-account-token   3      9d
test-tls-secret       Opaque                                2      11s
www-tls-secret        Opaque                                2      45s

root@k8s-ansible-client:~/yaml/20211016/ingress# cat ingress-https-multi.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-web
  namespace: pop
  annotations:
    kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
spec:
  tls:
  - hosts:
    - www.pop.com
    secretName: www-tls-secret
  - hosts:
    - test.pop.net
    secretName: test-tls-secret 
  rules:
  - host: www.pop.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: pop-tomcat-app1-service
            port:
               number: 80
  - host: test.pop.net
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: pop-tomcat-app2-service
            port:
               number: 80

root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl apply -f ingress-https-multi.yaml 
ingress.networking.k8s.io/nginx-web created
root@k8s-ansible-client:~/yaml/20211016/ingress# kubectl get ingress -n pop
NAME        CLASS    HOSTS                      ADDRESS          PORTS     AGE
nginx-web      www.pop.com,test.pop.net   192.168.20.253   80, 443   25s

验证

image.png

image.png

12. 基于HPA控制器实现控制pod副本数

12.1 HPA简介

HPA(Horizontal Pod Autoscaler)是kubernetes(以下简称k8s)的一种资源对象,能够根据某些指标对在statefulSet、replicaController、replicaSet等集合中的pod数量进行动态伸缩,使运行在上面的服务对指标的变化有一定的自适应能力。

HPA目前支持四种类型的指标,分别是Resource、Object、External、Pods。其中在稳定版本autoscaling/v1中只支持对CPU指标的动态伸缩,在测试版本autoscaling/v2beta2中支持memory和自定义指标的动态伸缩,并以annotation的方式工作在autoscaling/v1版本中。

12.2 HPA动态伸缩的原理及工作过程

HPA在k8s中也由一个controller控制,controller会间隔循环HPA,检查每个HPA中监控的指标是否触发伸缩条件,默认的间隔时间为15s。一旦触发伸缩条件,controller会向k8s发送请求,修改伸缩对象(statefulSet、replicaController、replicaSet)子对象scale中控制pod数量的字段。k8s响应请求,修改scale结构体,然后会刷新一次伸缩对象的pod数量。伸缩对象被修改后,自然会通过list/watch机制增加或减少pod数量,达到动态伸缩的目的。

kubectl autoscale 参数自动控制在k8s集群中运行的pod数量(水平自动伸缩),需要提前设置pod范围及触发条件。

k8s从1.1版本开始增加了名称为HPA(Horizontal Pod Autoscaler)的控制器,用于实现基于pod中资源
(CPU/Memory)利用率进行对pod的自动扩缩容功能的实现,早期的版本只能基于Heapster组件实现对CPU利用率
做为触发条件,但是在k8s 1.11版本开始使用Metrices Server完成数据采集(采集pod的CPU/内存),然后将采集到的数据通过
API(Aggregated API,汇总API),例如metrics.k8s.io、custom.metrics.k8s.io、external.metrics.k8s.io,然后
再把数据提供给HPA控制器进行查询,以实现基于某个资源利用率对pod进行扩缩容的目的。

控制管理器(kube-controller-manager)默认每隔15s(可以通过–horizontal-pod-autoscaler-sync-period修改)查询metrics的资源使用情况

kube-controller-manager支持以下三种metrics指标类型:
->预定义metrics(比如Pod的CPU)以利用率的方式计算
->自定义的Pod metrics,以原始值(raw value)的方式计算
->自定义的object metrics

支持两种metrics查询方式:
->Heapster
->自定义的REST API

支持多metrics,即支持多种数据采集方式,采集pod指标数据


image.png

运维通过master的API创建一个HPA控制器,由HPA控制器对deployment进行自动伸缩,通过调用kube-controller-manager对pod数量进行管理;
配置了HPA的deployment发现pod的资源利用率超过了之前所设定的值,通过调用kube-controller-manager服务创建pod,直到pod的资源利用率低于所设定的值;HPA会通过metrics API获取当前HPA所管理的deployment内部的POD的资源利用率

工作过程:
HPA通过master api(Metrices Server通过api server把采集到的数据记录到etcd中,HPA会通过master api获取到etcd中pod的数据)获取到deployment中pod的指标数据,再与HPA提前定义好的值做对比,如果资源利用率超出定义的值,则会通过kube-controller-manager新建pod,直到资源利用率低于hpa所定义的值;
kube-controller-manager默认每隔15s会查询metrics的资源使用情况

12.3 部署Metrics-server

root@k8s-ansible-client:~/yaml/20211016/hpa# cat metrics-server-v0.4.4.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    k8s-app: metrics-server
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
    rbac.authorization.k8s.io/aggregate-to-view: "true"
  name: system:aggregated-metrics-reader
rules:
- apiGroups:
  - metrics.k8s.io
  resources:
  - pods
  - nodes
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    k8s-app: metrics-server
  name: system:metrics-server
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - nodes
  - nodes/stats
  - namespaces
  - configmaps
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server-auth-reader
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server:system:auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    k8s-app: metrics-server
  name: system:metrics-server
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:metrics-server
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server
  namespace: kube-system
spec:
  ports:
  - name: https
    port: 443
    protocol: TCP
    targetPort: https
  selector:
    k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server
  namespace: kube-system
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  strategy:
    rollingUpdate:
      maxUnavailable: 0
  template:
    metadata:
      labels:
        k8s-app: metrics-server
    spec:
      containers:
      - args:
        - --cert-dir=/tmp
        - --secure-port=4443
        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
        - --kubelet-use-node-status-port
        image: harbor.openscp.com/base/metrics-server:v0.4.4
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /livez
            port: https
            scheme: HTTPS
          periodSeconds: 10
        name: metrics-server
        ports:
        - containerPort: 4443
          name: https
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /readyz
            port: https
            scheme: HTTPS
          periodSeconds: 10
        securityContext:
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 1000
        volumeMounts:
        - mountPath: /tmp
          name: tmp-dir
      nodeSelector:
        kubernetes.io/os: linux
      priorityClassName: system-cluster-critical
      serviceAccountName: metrics-server
      volumes:
      - emptyDir: {}
        name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
  labels:
    k8s-app: metrics-server
  name: v1beta1.metrics.k8s.io
spec:
  group: metrics.k8s.io
  groupPriorityMinimum: 100
  insecureSkipTLSVerify: true
  service:
    name: metrics-server
    namespace: kube-system
  version: v1beta1
  versionPriority: 100

执行yaml文件

root@k8s-ansible-client:~/yaml/20211016/hpa# kubectl apply -f metrics-server-v0.4.4.yaml 
serviceaccount/metrics-server created
clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
clusterrole.rbac.authorization.k8s.io/system:metrics-server created
rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
service/metrics-server created
deployment.apps/metrics-server created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created

root@k8s-ansible-client:~/yaml/20211016/hpa# kubectl top node
W1030 22:13:22.276731  170084 top_node.go:119] Using json format to get metrics. Next release will switch to protocol-buffers, switch early by passing --use-protocol-buffers flag
NAME             CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
192.168.20.147   229m         5%     1716Mi          53%       
192.168.20.189   137m         6%     1117Mi          86%       
192.168.20.201   251m         12%    1275Mi          99%       
192.168.20.236   243m         6%     1555Mi          48%       
192.168.20.249   309m         15%    1176Mi          91%       
192.168.20.253   275m         6%     1580Mi          48%

12.4 HPA自动伸缩pod数量

命令行模式

kubectl autoscale deployment/nginx-deploymen --min=2 --max=5 --cpu-percent=50 -n default
#自动扩容指定deployment的pod数量,当pod的cpu资源利用率达到50%,最大pod数量扩容到5个,最小2个pod数量

yaml文件
hpa的yml文件与服务的yml文件分开

root@k8s-ansible-client:~/yaml/20211016/hpa# cat hpa.yaml 
apiVersion: autoscaling/v1                #定义API版本
kind: HorizontalPodAutoscaler             #对象类型
metadata:                                 #定义对象元数据
  namespace: pop                        #hpa创建后隶属的namespace
  name: pop-tomcat-app1-podautoscaler   #hpa对象名称
  labels:                    
    app: pop-tomcat-app1   #自定义的label名称
    version: v2beta1         #自定义的api版本
spec:                        #定义对象具体信息
  scaleTargetRef:            #定义水平伸缩的目标对象,Deployment、ReplicationController/ReplicaSet     
    apiVersion: apps/v1      #目标对象的API版本
    #apiVersion: extensions/v1beta1 
    kind: Deployment         #目标对象类型为deployment
    name: pop-tomcat-app1-deployment   #目标deployment的具体名称
  minReplicas: 2                         #最小pod数
  maxReplicas: 4                         #最大pod数
  targetCPUUtilizationPercentage: 60     #CPU资源利用率

执行yaml, 可以看见默认启动一个pod,执行hpa之后,启动了两个pod,这里cpu利用率就不做测试

root@k8s-ansible-client:~/yaml/20211010/03/tomcat-app1# kubectl get pods,deploy -n pop
NAME                                              READY   STATUS    RESTARTS     AGE
pod/deploy-devops-redis-d9fd6594c-fvmmg           1/1     Running   0            9d
pod/mysql-0                                       2/2     Running   0            8d
pod/mysql-1                                       2/2     Running   0            8d
pod/mysql-2                                       2/2     Running   1 (8d ago)   8d
pod/pop-tomcat-app1-deployment-54bb9d8f8c-jblz6   1/1     Running   0            4h20m
pod/pop-tomcat-app2-deployment-5676bf7c9-2sc7g    1/1     Running   0            4h21m

NAME                                         READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deploy-devops-redis          1/1     1            1           9d
deployment.apps/pop-tomcat-app1-deployment   1/1     1            1           4h20m
deployment.apps/pop-tomcat-app2-deployment   1/1     1            1           4h21m

root@k8s-ansible-client:~/yaml/20211016/hpa# kubectl apply -f hpa.yaml 
horizontalpodautoscaler.autoscaling/pop-tomcat-app1-podautoscaler created

root@k8s-ansible-client:~/yaml/20211016/hpa# kubectl get pods,deploy -n pop
NAME                                              READY   STATUS    RESTARTS     AGE
pod/deploy-devops-redis-d9fd6594c-fvmmg           1/1     Running   0            9d
pod/mysql-0                                       2/2     Running   0            8d
pod/mysql-1                                       2/2     Running   0            8d
pod/mysql-2                                       2/2     Running   1 (8d ago)   8d
pod/pop-tomcat-app1-deployment-54bb9d8f8c-jblz6   1/1     Running   0            4h23m
pod/pop-tomcat-app1-deployment-54bb9d8f8c-psrr4   1/1     Running   0            7s
pod/pop-tomcat-app2-deployment-5676bf7c9-2sc7g    1/1     Running   0            4h24m

NAME                                         READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deploy-devops-redis          1/1     1            1           9d
deployment.apps/pop-tomcat-app1-deployment   2/2     2            2           4h23m
deployment.apps/pop-tomcat-app2-deployment   1/1     1            1           4h24m

你可能感兴趣的:(k8s进阶(三))