第10章 K8s进阶篇-高级调度计划任务，污点和容忍和Affinity
10.1 什么是Job？
job常用作初始化数据和基本的创建操作。
job创建成功后不会立即执行容器命令，只有suspend=true，才会执行。
10.2 Job使用入门
[root@k8s-master01 10st]# cat job.yaml
apiVersion: batch/v1
kind: Job
metadata:
labels:
job-name: echo
name: echo
namespace: default
spec:
#suspend: true # 1.21+
# ttlSecondsAfterFinished: 100
backoffLimit: 4
completions: 1
parallelism: 1
template:
spec:
containers:
- command:
- echo
- Hello,Job
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
name: echo
resources: {}
restartPolicy: Never
[root@k8s-master01 10st]# kubectl create -f job.yaml
job.yaml文件不可以修改后replace/apply,只能kubectl delete -f job,yaml删除后,修改后重新create。job一般是配合Helm进行部署。
[root@k8s-master01 10st]# cat job.yaml
apiVersion: batch/v1
kind: Job
metadata:
labels:
job-name: echo
name: echo
namespace: default
spec:
#suspend: true # 1.21+
# ttlSecondsAfterFinished: 100
backoffLimit: 4
completions: 5
parallelism: 3
template:
spec:
containers:
- command:
- echo
- Hello,Job
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
name: echo
resources: {}
restartPolicy: Never
[root@k8s-master01 10st]# 
10.3 更强大的计划任务CronJob
10.4 CronJob使用
[root@k8s-master01 10st]# cat cronjob.yaml
apiVersion: batch/v1beta1
kind: CronJob
metadata:
labels:
run: hello
name: hello
namespace: default
spec:
concurrencyPolicy: Allow
failedJobsHistoryLimit: 1
jobTemplate:
metadata:
spec:
template:
metadata:
labels:
run: hello
spec:
containers:
- args:
- /bin/sh
- -c
- date;echo Hello from the Kubernetes cluster
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: Always
name: hello
resources: {}
restartPolicy: OnFailure
securityContext: {}
schedule: '*/1 * * * *'
successfulJobsHistoryLimit: 3
suspend: false
您在 /var/spool/mail/root 中有新邮件
[root@k8s-master01 10st]# kubectl create -f cronjob.yaml
本案例中cronjob创建成功后会创建一个job,job会创建一个pod执行我们指令。
[root@k8s-master01 10st]# cat cronjob.yaml
apiVersion: batch/v1beta1
kind: CronJob
metadata:
labels:
run: hello
name: hello
namespace: default
spec:
concurrencyPolicy: Allow
failedJobsHistoryLimit: 1
jobTemplate:
metadata:
spec:
template:
metadata:
labels:
run: hello
spec:
containers:
- args:
- /bin/sh
- -c
- date;echo Hello from the Kubernetes cluster
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: Always
name: hello
resources: {}
restartPolicy: OnFailure
securityContext: {}
schedule: '*/1 * * * *'
successfulJobsHistoryLimit: 3
suspend: true
[root@k8s-master01 10st]# 10.5 InitContainer用途
不要以root身份运行业务容器，可以运行Init容器；
10.6 初始化容器和普通容器、PostStart
当pod里有多个Init容器时，只有第一个Init容器运行完成（以成功状态退出）后才会运行第二个，如果第一个Init运行失败，pod会一直运行第一个Init容器。比如pod里有4个Init容器（Init1-Init4），只有4个都执行成功，pod才会执行主程序的容器。
10.7 初始化容器配置解析
10.8 初始化容器使用示例
[root@k8s-master01 10st]# cat init.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: test-init
name: test-init
namespace: kube-public
spec:
replicas: 1
selector:
matchLabels:
app: test-init
template:
metadata:
labels:
app: test-init
spec:
volumes:
- name: data
emptyDir: {}
initContainers:
- command:
- sh
- -c
- touch /mnt/test-init.txt
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
name: init-touch
volumeMounts:
- name: data
mountPath: /mnt
containers:
- image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
# command: ["/bin/bash", "-ce", "tail -f /dev/null"]
#加上如下这句，不然容器启动后因为没有常驻进程导致退出，状态:CrashLoopBackOff
command: ["/bin/sh","-ce","sleep 3600"]
name: test-init
volumeMounts:
- name: data
mountPath: /mnt
[root@k8s-master01 10st]# kubectl create -f init.yaml 
#通过deploy扩容，观察先启动Init容器，后启动主进程
kubectl scale deploy test-init --replicas=6 -n kube-public 
测试多个Init容器
[root@k8s-master01 10st]# cat init.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: test-init
name: test-init
namespace: kube-public
spec:
replicas: 3
selector:
matchLabels:
app: test-init
template:
metadata:
labels:
app: test-init
spec:
volumes:
- name: data
emptyDir: {}
initContainers:
- command:
- sh
- -c
- touch /mnt/test-init.txt
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
name: init-touch
volumeMounts:
- name: data
mountPath: /mnt
- command:
- sh
- -c
- for i in `seq 1 10`;do echo $i;sleep 1;done
image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
name: echo
volumeMounts:
- name: data
mountPath: /mnt
containers:
- image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
imagePullPolicy: IfNotPresent
# command: ["/bin/bash", "-ce", "tail -f /dev/null"]
command: ["/bin/sh","-ce","sleep 3600"]
name: test-init
volumeMounts:
- name: data
mountPath: /mnt
[root@k8s-master01 10st]# kubectl create -f init.yaml
10.9 为什么要用临时容器？
临时容器:具有工具包+root权限，帮助排查pod里其他容器的问题，排查容器的容器。
10.10 使用临时容器在线debug
5节点都需要操作的步骤：master01-master03,node01-node02
vi /usr/lib/systemd/system/kube-proxy.service
--feature-gates=EphemeralContainers=true \
vi /etc/kubernetes/kubelet-conf.yml
featureGates:
EphemeralContainers: true
master01-master03需要操作的步骤
vi /usr/lib/systemd/system/kube-apiserver.service
--feature-gates=EphemeralContainers=true \
vi /usr/lib/systemd/system/kube-controller-manager.service
--feature-gates=EphemeralContainers=true \
vi /usr/lib/systemd/system/kube-scheduler.service
--feature-gates=EphemeralContainers=true \
5个节点操作
systemctl daemon-reload systemctl restart kube-apiserver kube-scheduler kube-controller-manager kubelet kube-proxy直接在你的pod上注入一个容器，并打开控制台；
进入临时容器方式1
kubectl debug metrics-server-595f65d8d5-zmc88 -ti --image=registry.cn-beijing.aliyuncs.com/dotbalo/debug-tools -n kube-system 
进入临时容器方式2
kubectl attach metrics-server-595f65d8d5-zmc88 -c debugger-b9s9j -i -t -n kube-system 进入临时容器方式3
kubectl get po metrics-server-595f65d8d5-zmc88 -n kube-system -oyaml
10.11 Taint和Toleration设计理念
Taint（污点--锁）作用在节点（node）上，Toleration（容忍--钥匙）是作用在pod上。
10.12 污点和容忍配置解析
如果希望pod部署到指定节点，需要通过nodeselector或者Affinity，因为容忍和污点没有强制依赖
遇到才会生效。含有容忍点的pod遇到了（被scheduler 调度）对应的污点能容忍它，遇不到就是遇不到。类比回家开门，有很多门有的上锁（污点node），有的没上锁（无污点的node）。碰到没上锁的门直接进去，有锁的才需要钥匙（容忍点）。如果要pod强制走带锁的门,可以使用nodeselector或者Affinity去限制pod调度。
10.13 污点和容忍配置示例
kubectl taint nodes k8s-node01 ssd=true:NoSchedule
驱除pod测试
kubectl taint nodes k8s-node01 ssd=true:NoExecute
kubectl label node k8s-node01 ssd=true
[root@k8s-master01 10st]# cat toleration.yaml
apiVersion: v1
kind: Pod
metadata:
name: nginx
labels:
env: test
spec:
containers:
- name: nginx
image: nginx
imagePullPolicy: IfNotPresent
nodeSelector:
ssd: "true"
tolerations:
- key: "ssd"
operator: "Exists"
[root@k8s-master01 10st]# kubectl create -f toleration.yaml 
注释掉容忍点测试
[root@k8s-master01 10st]# cat toleration.yaml
apiVersion: v1
kind: Pod
metadata:
name: nginx
labels:
env: test
spec:
containers:
- name: nginx
image: nginx
imagePullPolicy: IfNotPresent
nodeSelector:
ssd: "true"
#tolerations:
#- key: "ssd"
# operator: "Exists"
[root@k8s-master01 10st]# 
pending原因查看
10.14 内置污点
deployment无容忍配置，创建pod时k8s会自动给pod创建容忍的配置；
10.15 节点宕机秒级恢复应用
[root@k8s-master01 10st]# cat toleration_seconds.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: tolerations-second
name: tolerations-second
spec:
replicas: 1
selector:
matchLabels:
app: tolerations-second
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: tolerations-second
spec:
containers:
- image: registry.cn-beijing.aliyuncs.com/dotbalo/nginx
name: nginx
resources:
requests:
cpu: 10m
nodeSelector:
ssd: "true"
tolerations:
- key: ssd
operator: Equal
value: "true"
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 10
- effect: NoExecute
key: node.kubernetes.io/notready
operator: Exists
tolerationSeconds: 10
[root@k8s-master01 10st]# kubectl create -f toleration_seconds.yaml
此时将node01节点关机（vm控制台操作），40s后node01状态变为notready，观察pod是否漂移。
因为配置了nodeselector选项只选择node01，所以pod不会调度到其他节点。其他没有配置pod的状态上300s后才变为非running。
10.16 Taint命令入门
用的时候：key名,value和EFFECT一样，则判断为同一个taint;
配得时候，key名和effecf一样，则筛选为同一个taint修改；
查看某一个节点的污点
kubectl describe node k8s-node01 | grep Taints -A 10
完全匹配
kubectl taint node k8s-node01 ssd=true:NoExecute-污点已删除
污点重新添加上，继续测试
kubectl taint node k8s-node01 ssd=true:NoExecute
key名-
修改污点
kubectl taint node k8s-node01 ssd=fasle:NoExecute --overwrite 
查看帮助
kubectl taint node -h10.17 生产环境依旧存在的高可用率问题
10.18 Affinity分类
10.19 可用率保障-部署至不同宿主机
10.20 可用率保障-部署至不同机房或区
10.21 降低故障范围-不把鸡蛋放在同一个篮子里
10.22 节点亲和力配置详解
10.23 Pod亲和力和反亲和力配置详解
labselector是通过pod标签选择和哪个pod匹配；nodeslectors是通过节点（node）标签选择节点（node）。
10.24 实现同一个应用分布在不同的宿主机
[root@k8s-master01 10st]# cat pod-mulitinodes.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: must-be-diff-nodes
name: must-be-diff-nodes
namespace: kube-public
spec:
replicas: 3
selector:
matchLabels:
app: must-be-diff-nodes
#project: multi
template:
metadata:
labels:
app: must-be-diff-nodes
# project: multi
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- must-be-diff-nodes
topologyKey: kubernetes.io/hostname
containers:
- image: nginx
imagePullPolicy: IfNotPresent
name: must-be-diff-nodes
[root@k8s-master01 10st]# kubectl create -f pod-mulitinodes.yaml 
10.25 尽量将应用部署至高配置服务器
[root@k8s-master01 10st]# cat nodeAffinitySSD.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: prefer-ssd
name: prefer-ssd
namespace: kube-public
spec:
replicas: 1
selector:
matchLabels:
app: prefer-ssd
template:
metadata:
creationTimestamp: null
labels:
app: prefer-ssd
spec:
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- preference:
matchExpressions:
- key: ssd
operator: In
values:
- "true"
- key: gpu
operator: NotIn
values:
- "true"
weight: 100
- preference:
matchExpressions:
- key: type
operator: In
values:
- physical
weight: 10
containers:
- image: nginx
imagePullPolicy: IfNotPresent
name: prefer-ssd-1
[root@k8s-master01 10st]# kubectl create -f nodeAffinitySSD.yaml 
继续测试
kubectl label nodes k8s-node01 ssd-因为node02打了type=physical标签，所以被调度。
10.26 Topology的重要性
可以打区域标签，subnet网络区域标签；
打不同机房机柜标签，分到不同机房的不同机柜；逻辑上的划分，分为三个不同的域；
10.27 应用多地区多机房部署
实现同一应用多区域多机房部署
在一个域内，只能存在一个标签app=must-be-diff-zone的pod
[root@k8s-master01 10st]# cat must-be-diff-zone.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: must-be-diff-zone
name: must-be-diff-zone
namespace: kube-public
spec:
replicas: 3
selector:
matchLabels:
app: must-be-diff-zone
template:
metadata:
labels:
app: must-be-diff-zone
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- must-be-diff-zone
topologyKey: region
containers:
- image: nginx
imagePullPolicy: IfNotPresent
name: must-be-diff-zone
[root@k8s-master01 10st]# kubectl create -f must-be-diff-zone.yaml 
副本改为4继续测试
---------------教程来源：51cto 杜宽老师k8s课程的学习笔记 -------------