kubernetes使用jenkinsci/kubernetes-plugin完成持续构建与发布
github：https://github.com/jenkinsci/kubernetes-plugin
参考：https://blog.csdn.net/aixiaoyang168/article/details/79767649
一、部署jenkins master
首先从github的说明可以看到jenkins的配置文件在src/main/kubernetes路径里面
分别是jenkins.yml和service-account.yml
github是使用StatefulSet的方式部署jenkins，并且指定了ingress策略。这里我也使用StatefulSet的方式部署，由于本人使用的是kubernates v1.16.0，版本较新，所以需要修改github提供的配置文件才行
注：前提是已经做好storageClass动态申请存储卷，这里我是基于nfs做的动态存储卷
jenkins.yml
# jenkins
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: jenkins
labels:
name: jenkins
spec:
serviceName: jenkins
replicas: 1
selector:
matchLabels:
name: jenkins
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccount: jenkins
# serviceAccount: nfs-provisioner # 和创建的资源访问角色(nfs-provisioner)保持一致
containers:
- name: jenkins
# image: jenkins/jenkins:lts-alpine
image: jenkins/jenkins:lts-alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: jenkins-http
protocol: TCP
- containerPort: 50000
name: jenkins-agent
protocol: TCP
resources:
limits:
cpu: 2048m
memory: 2Gi
requests:
cpu: 512m
memory: 512Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
# value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:
- name: jenkins-home
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
#securityContext:
# runAsUser: 1000
# fsGroup: 1000
volumeClaimTemplates:
- metadata:
name: jenkins-home
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: nfs # 和定义的storageclass的名称保持一致
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
# type: LoadBalancer
selector:
name: jenkins
# ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)
#externalTrafficPolicy: Local
ports:
- name: jenkins-http
port: 8080
targetPort: 8080
protocol: TCP
- name: jenkins-agent
port: 50000
targetPort: 50000
protocol: TCP
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: jenkins
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
kubernetes.io/tls-acme: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
nginx.ingress.kubernetes.io/proxy-body-size: 50m
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
# For nginx-ingress controller < 0.9.0.beta-18
ingress.kubernetes.io/ssl-redirect: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
ingress.kubernetes.io/proxy-body-size: 50m
ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
rules:
- http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 8080 # 与service的端口对应
host: jenkins.example.com
tls:
- hosts:
- jenkins.example.com
secretName: tls-jenkins如果不想部署statefulset的jenkins，也可以修改一下yml，部署deployment的jenkins，如下
# jenkins
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
labels:
name: jenkins
spec:
replicas: 1
selector:
matchLabels:
name: jenkins
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccount: jenkins
# serviceAccount: nfs-provisioner # 和创建的资源访问角色(nfs-provisioner)保持一致
containers:
- name: jenkins
# image: jenkins/jenkins:lts-alpine
image: jenkins/jenkins:lts-alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: jenkins-http
protocol: TCP
- containerPort: 50000
name: jenkins-agent
protocol: TCP
resources:
limits:
cpu: 2048m
memory: 2Gi
requests:
cpu: 512m
memory: 512Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
# value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:
- name: jenkins-home
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
#securityContext:
# runAsUser: 1000
# fsGroup: 1000
volumes:
- name: jenkins-home
persistentVolumeClaim:
claimName: pvc-jenkins-home
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
# type: LoadBalancer
selector:
name: jenkins
# ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)
#externalTrafficPolicy: Local
ports:
- name: jenkins-http
port: 8080
targetPort: 8080
protocol: TCP
- name: jenkins-agent
port: 50000
targetPort: 50000
protocol: TCP
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: jenkins
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
kubernetes.io/tls-acme: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
nginx.ingress.kubernetes.io/proxy-body-size: 50m
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
# For nginx-ingress controller < 0.9.0.beta-18
ingress.kubernetes.io/ssl-redirect: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
ingress.kubernetes.io/proxy-body-size: 50m
ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
rules:
- http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 8080 # 与service的端口对应
host: jenkins.example.com
tls:
- hosts:
- jenkins.example.com
secretName: tls-jenkinsservice-account.yml
# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=|--group=]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: jenkins
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: jenkins
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
kubectl describe pods jenkins-0报错提示
pod has unbound immediate PersistentVolumeClaims查看pvc列表发现有个jenkins-home-jenkins-0的pvc，AGE竟然是45h，回想起来我之前在没部署好nfs-
[root@k8s-master ~/docker-about/image-and-yml/jenkins]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
jenkins-home-jenkins-0 Pending 45h
pvc-mysql-data Bound pv-mysql-data 20Gi RWO nfs 5d22h
pvc-redis-data Bound pv-redis-data 5Gi RWO nfs 4d6h
test-web-0 Bound pvc-d8cb8112-86f5-4b56-a830-04574412e165 1Gi RWO nfs 26h
test-web-1 Bound pvc-f2b7dbaf-6478-407d-9cd1-3d080efcfffa 1Gi RWO nfs 26h
注意： 初始化过程中，让输入 /var/jenkins_home/secret/initialAdminPassword 初始密码时，因为我们设置的 emptyDir: {}没有挂载到外部路径，可以进入到容器内部进行获取。
安装完插件，创建帐号后，就能看到界面
二、安装插件Kubernetes
管理员账户登录 Jenkins Master 页面，点击 “系统管理” —> “管理插件” —> “可选插件” —> “Kubernetes” 勾选安装即可。
`注：没看到有Kubernetes plugin插件，只有kubernetes，原来是插件名字变了`
点击安装，发现它一共安装两个插件kubernetes、kubernetes-client-api，由于我在线安装失败，所以到官网下载插件后，通过上传方式安装
下载的插件
通过上传方式安装，“系统管理” —> “管理插件” —> “可选插件” —> “高级”
安装后查看“已安装的插件”发现kubernetes plugin已经安装
三、Jenkins 配置 Kubernetes Plugin
安装完毕后，
点击 “系统管理” —> “系统设置” —> “新增一个云” —> 选择 “Kubernetes”，然后填写 Kubernetes 和 Jenkins 配置信息。
说明一下：
先查看自己的svc
kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
jenkins ClusterIP 10.1.106.26 8080/TCP,50000/TCP 4h22m
kubernetes ClusterIP 10.1.0.1 443/TCP 11d
kubernetes的svc名称是kubernetes，端口443
jenkins的svc名称是jenkins，端口8080
1、Name 处默认为 kubernetes，也可以修改为其他名称，如果这里修改了，下边在执行 Job 时指定 podTemplate() 参数 cloud 为其对应名称，否则会找不到，cloud 默认值取：kubernetes
2、Kubernetes URL 格式为
https://
https://
https://
我这里填写为https://kubernetes.default，命名空间就是default
3、Jenkins URL 格式跟上面类似
我填写为：https://jenkins.default:8080
配置完毕，可以点击 “测试连接” 按钮测试是否能够连接的到 Kubernetes，如果显示 Connection test successful 则表示连接成功，配置没有问题。
4、创建一个 Pipeline 并随便命名为 my-k8s-jenkins-pipeline，然后在 Pipeline 脚本处填写一个简单的测试脚本如下
//运行的pod的名字
def label = "mypod-${UUID.randomUUID().toString()}"
//使用pod模板运行
podTemplate(label: label, cloud: 'kubernetes') {
node(label) {
stage('Run shell') {
sh 'sleep 130s'
sh 'echo hello world.'
}
}
}注：使用默认的jenkins-jnlp-slave，尝试部署流水线，k8s会部署一个pod用于执行jenkins给的任务，执行完毕后，pod会自动销毁
四、使用自定义镜像执行流水线
如果想在jenkins的slave的pod运行时，做一系列工作，如使用maven对代码做构建工作。由于jenkins-jnlp-slave里面没有maven，可以用过dockerfile的方式把maven打进去镜像里面
1、创建自定义的jenkins-jnlp-slave镜像，并push到docker私库，我用的是docker register
# 指定基础镜像
FROM jenkins/jnlp-slave:latest
MAINTAINER my-jenkins
LABEL Description="This is a extend image base from jenkins/jnlp-slave which install maven in it."
# 切换到 root 账户进行操作
USER root
# 安装 maven-3.6.2
RUN wget http://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
tar -zxf apache-maven-3.6.3-bin.tar.gz && \
mv apache-maven-3.6.3 /usr/local && \
rm -f apache-maven-3.6.3-bin.tar.gz && \
ln -s /usr/local/apache-maven-3.6.3/bin/mvn /usr/bin/mvn && \
ln -s /usr/local/apache-maven-3.6.3 /usr/local/apache-maven
# 安装 node-v10
RUN wget https://nodejs.org/download/release/v10.17.0/node-v10.17.0-linux-x64.tar.gz && \
tar -xzf node-v10.17.0-linux-x64.tar.gz && \
mv node-v10.17.0-linux-x64 /usr/local && \
rm -r node-v10.17.0-linux-x64.tar.gz && \
ln -s /usr/local/node-v10.17.0-linux-x64/bin/node /usr/bin/node && \
ln -s /usr/local/node-v10.17.0-linux-x64/bin/npm /usr/bin/npm && \
ln -s /usr/local/node-v10.17.0-linux-x64 /usr/local/node-v10.17.0
USER jenkins
build镜像
docker build -t 192.168.200.207:5000/my-jenkins-jnlp-slave:latest -f ./my-dockerfile .创建一个 Pipeline 类型 Job 并命名为 my-k8s-jenkins-container-custom，然后在 Pipeline 脚本处填写一个简单的测试脚本如下：
def label = "mypod-${UUID.randomUUID().toString()}"
podTemplate(label: label, cloud: 'kubernetes',containers: [
containerTemplate(
name: 'jnlp',
image: '192.168.200.207:5000/my-jenkins-jnlp-slave:latest',
alwaysPullImage: false,
args: '${computer.jnlpmac} ${computer.name}'),
]) {
node(label) {
stage('stage1') {
stage('Show Maven version') {
sh 'mvn -version'
sh 'sleep 60s'
}
}
}
}
说明一下：这里 containerTemplate 的 name 属性必须叫 jnlp，Kubernetes 才能用自定义 images 指定的镜像替换默认的 jenkinsci/jnlp-slave 镜像。此外，args 参数传递两个 jenkins-slave 运行需要的参数。还有一点就是这里并不需要指定 container('jnlp'){...} 了，因为它被 Kubernetes 指定了要被执行的容器，所以直接执行 Stage 就可以了。
3、在jenkins的pipeline中，可以使用SSH Pipeline Steps插件做远程操作
jenkins-》系统管理-》插件管理-》可选插件，安装SSH Pipeline Steps
官网：https://jenkins.io/doc/pipeline/steps/ssh-steps/