参考:
https://kubernetes.io/zh/docs/concepts/storage/storage-classes/
https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/
https://github.com/kubernetes-retired/external-storage/tree/master/nfs
https://www.jianshu.com/p/65ed4bdf0e89
https://www.cnblogs.com/panwenbin-logs/p/12196286.html
根据官网安装mysql,需要PersistentVolumeClaim,不然会报3 pod has unbound immediate PersistentVolumeClaims:
这个原因有很多,比如k8s gcr.io被墙导致ImagePullBackOff,或者节点内存不足导致caused "process_linux.go:101: executing setns process caused "exit status 1, 等。
如果k8s gcr.io被墙导致ImagePullBackOff,可以通过以下命令拉去镜像。
docker pull ist0ne/xtrabackup
docker tag ist0ne/xtrabackup:latest gcr.io/google-samples/xtrabackup:1.0
注意如果是云集群,那个每个节点都要执行以上命令拉去镜像。
这个是由nfs的权限设置导致的。具体解决办法见NFS安装步骤–主节点执行。
我的k8s有三个节点,一个master,两个node分别是以下三个。
192.168.0.11 k8s-master
192.168.0.22 k8s-node1
192.168.0.33 k8s-node2
yum -y install nfs-utils rpcbind
mkdir -p /home/nfs
vi /etc/exports
#加入,注意加入no_root_squash,这个是放开root权限,不加的话会报chown: changing ownership of '/var/lib/mysql/': Operation not permitted错误。
/home/nfs *(insecure,rw,async,no_root_squash)
#启动并设置开机启动
systemctl start rpcbind.service
systemctl status rpcbind.service
systemctl enable rpcbind.service
systemctl start nfs.service
systemctl enable nfs.service
systemctl status nfs.service
yum -y intall nfs-utils
showmount -e 192.1681.0.11
# 挂载至本地/mnt目录
mount -t nfs 192.1681.0.11:/home/nfs /mnt
df -h
umount /mnt
# rabc.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default #根据实际环境设定namespace,下面类同
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
# storage-class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: managed-nfs-storag
provisioner: nfs #这里的名称要和provisioner配置文件中的环境变量PROVISIONER_NAME保持一致
parameters:
type: gp2
reclaimPolicy: Retain
allowVolumeExpansion: true
mountOptions:
- debug
volumeBindingMode: Immediate
# nfs-provisioner.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
labels:
app: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default #与RBAC文件中的namespace保持一致
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: quay.io/external_storage/nfs-client-provisioner:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: nfs #provisioner名称,请确保该名称与 nfs-StorageClass.yaml文件中的provisioner名称保持一致
- name: NFS_SERVER
value: 192.168.0.11 #NFS Server IP地址
- name: NFS_PATH
value: /home/nfs #NFS挂载卷
volumes:
- name: nfs-client-root
nfs:
server: 192.168.0.11 #NFS Server IP地址
path: /home/nfs #NFS 挂载卷
# pvc-nfs.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
volumeMode: Filesystem
resources:
requests:
storage: 8Gi
storageClassName: managed-nfs-storag
# mysql-configmap.yaml
kind: ConfigMap
apiVersion: v1
metadata:
name: mysql
namespace: default
labels:
app: mysql
data:
master.cnf: |
# Apply this config only on the master.
[mysqld]
log-bin
slave.cnf: |
# Apply this config only on slaves.
[mysqld]
super-read-only
apiVersion: v1
kind: Service
metadata:
name: mysql
labels:
app: mysql
spec:
ports:
- name: mysql
port: 3306
clusterIP: None
selector:
app: mysql
---
apiVersion: v1
kind: Service
metadata:
name: mysql-read
labels:
app: mysql
spec:
ports:
- name: mysql
port: 3306
selector:
app: mysql
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: mysql
namespace: default
spec:
replicas: 3
selector:
matchLabels:
app: mysql
template:
metadata:
creationTimestamp: null
labels:
app: mysql
spec:
volumes:
- name: conf
emptyDir: {}
- name: config-map
configMap:
name: mysql
defaultMode: 420
initContainers:
- name: init-mysql
image: 'mysql:5.7'
command:
- bash
- '-c'
- |
set -ex
# Generate mysql server-id from pod ordinal index.
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
echo [mysqld] > /mnt/conf.d/server-id.cnf
# Add an offset to avoid reserved server-id=0 value.
echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
# Copy appropriate conf.d files from config-map to emptyDir.
if [[ $ordinal -eq 0 ]]; then
cp /mnt/config-map/master.cnf /mnt/conf.d/
else
cp /mnt/config-map/slave.cnf /mnt/conf.d/
fi
resources: {}
volumeMounts:
- name: conf
mountPath: /mnt/conf.d
- name: config-map
mountPath: /mnt/config-map
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
- name: clone-mysql
image: 'gcr.io/google-samples/xtrabackup:1.0'
command:
- bash
- '-c'
- >
set -ex
# Skip the clone if data already exists.
[[ -d /var/lib/mysql/mysql ]] && exit 0
# Skip the clone on master (ordinal index 0).
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
[[ $ordinal -eq 0 ]] && exit 0
# Clone data from previous peer.
ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C
/var/lib/mysql
# Prepare the backup.
xtrabackup --prepare --target-dir=/var/lib/mysql
resources: {}
volumeMounts:
- name: nfs-pvc
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
containers:
- name: mysql
image: 'mysql:5.7'
ports:
- name: mysql
containerPort: 3306
protocol: TCP
env:
- name: MYSQL_ALLOW_EMPTY_PASSWORD
value: '1'
resources:
requests:
cpu: 50m
memory: 50Mi
volumeMounts:
- name: nfs-pvc
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
livenessProbe:
exec:
command:
- mysqladmin
- ping
initialDelaySeconds: 30
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
readinessProbe:
exec:
command:
- mysql
- '-h'
- 127.0.0.1
- '-e'
- SELECT 1
initialDelaySeconds: 5
timeoutSeconds: 1
periodSeconds: 2
successThreshold: 1
failureThreshold: 3
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
- name: xtrabackup
image: 'gcr.io/google-samples/xtrabackup:1.0'
command:
- bash
- '-c'
- >
set -ex
cd /var/lib/mysql
# Determine binlog position of cloned data, if any.
if [[ -f xtrabackup_slave_info && "x$( change_master_to.sql.in
# Ignore xtrabackup_binlog_info in this case (it's useless).
rm -f xtrabackup_slave_info xtrabackup_binlog_info
elif [[ -f xtrabackup_binlog_info ]]; then
# We're cloning directly from master. Parse binlog position.
[[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
rm -f xtrabackup_binlog_info xtrabackup_slave_info
echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
fi
# Check if we need to complete a clone by starting replication.
if [[ -f change_master_to.sql.in ]]; then
echo "Waiting for mysqld to be ready (accepting connections)"
until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
echo "Initializing replication from clone position"
mysql -h 127.0.0.1 \
-e "$(