五、容器化构建jenkins与slave的cicd流程

1、构建jenkins

jenkins是有状态的服务，我们需要建一个数据存储，这里我们试用nfs的动态存储StorageClass

准备nfs服务

每台机器都要安装nfs

yum install -y nfs-utils

这里我们将192.168.25.137作为nfs服务器
定义nfs服务路径与权限：

[root@node2 ~]# cat /etc/exports
/data/nfs_data 192.168.25.0/24(rw,no_root_squash)

定义一个storage

[root@master storage]# cat storageclass-nfs.yaml
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: fuseim.pri/ifs

部署授权

[root@master storage]# cat rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io

部署一个自动创建pv的服务

[root@master storage]# cat deployment-nfs.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: nfs-client-provisioner
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
    #  imagePullSecrets:
    #    - name: registry-pull-secret
      serviceAccount: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          #image: quay.io/external_storage/nfs-client-provisioner:latest
          image: lizhenliang/nfs-client-provisioner:v2.0.0
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              #这个值是定义storage里面的那个值
              value: fuseim.pri/ifs
            - name: NFS_SERVER
              value: 192.168.25.137
            - name: NFS_PATH
              value: /data/nfs_data
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.25.137
            path: /data/nfs_data

创建：

kubectl apply -f storageclass-nfs.yaml
kubectl apply -f rbac.yaml
kubectl apply -f deployment-nfs.yaml

查看创建好的storage:
[root@master storage]# kubectl get sc

NAME                  PROVISIONER      AGE
managed-nfs-storage   fuseim.pri/ifs   11h

nfs-client-provisioner 会以pod运行在k8s中，

[root@master storage]# kubectl get pod
NAME                                      READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-855887f688-hrdwj   1/1     Running   0          10h

部署jenkins

[root@master jenkins]# cat jenkins.yml
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: jenkins
  labels:
    name: jenkins
spec:
  serviceName: jenkins
  replicas: 1
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccountName: jenkins
      #imagePullSecrets:
      #  - name: registry-pull-secret
      containers:
        - name: jenkins
          image: jenkins/jenkins:lts
          imagePullPolicy: Always
          ports:
            - containerPort: 8080
            - containerPort: 50000
          resources:
            limits:
              cpu: 1
              memory: 1Gi
            requests:
              cpu: 0.5
              memory: 500Mi
          env:
            - name: LIMITS_MEMORY
              valueFrom:
                resourceFieldRef:
                  resource: limits.memory
                  divisor: 1Mi
            - name: JAVA_OPTS
              # value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
              value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
          volumeMounts:
            - name: data
              mountPath: /var/jenkins_home
          livenessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 200
            timeoutSeconds: 20
          readinessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 200
            timeoutSeconds: 20
      securityContext:
        fsGroup: 1000
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: jenkins-data

---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-data
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: managed-nfs-storage
  resources:
    requests:
      storage: 5Gi

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
spec:
  type: NodePort
  selector:
    name: jenkins
  ports:
    -
      name: http
      port: 80
      targetPort: 8080
      protocol: TCP
      nodePort: 30007
    -
      name: agent
      port: 50000
      protocol: TCP
      

创建jenkins；

kubectl apply -f jenkins.yml

2 配置jenkins

所有配置都跟在本地配置jenkins一样，只有一个地方有区别
添加Kubernetes配置：
在系统设置最下方添加一个云：

Kubernetes 地址：https://kubernetes.default
jenkins地址：http://jenkins.default
其他的配置参考上个章节，配置本地jenkins的cicd

3 构建jenkins salve

构建jenkins slave镜像
jenkins-slave文件参考：https://github.com/jenkinsci/docker-jnlp-slave
slave jar包地址：https://download.csdn.net/download/qq_25611295/10786753
构建jenkins salve所需文件：

[root@master jenkins_salve]# ls
apache-maven-3.3.9-bin.tar.gz  dockerfile-jenkins-slave  jenkins-slave  settings.xml  slave.jar

此处的setting.xml是maven的setting.xml文件，加入了国内源
jenkins-slave内容：

[root@master jenkins_salve]# cat jenkins-slave
#!/usr/bin/env sh

if [ $# -eq 1 ]; then

	# if `docker run` only has one arguments, we assume user is running alternate command like `bash` to inspect the image
	exec "$@"

else

	# if -tunnel is not provided try env vars
	case "$@" in
		*"-tunnel "*) ;;
		*)
		if [ ! -z "$JENKINS_TUNNEL" ]; then
			TUNNEL="-tunnel $JENKINS_TUNNEL"
		fi ;;
	esac

	# if -workDir is not provided try env vars
	if [ ! -z "$JENKINS_AGENT_WORKDIR" ]; then
		case "$@" in
			*"-workDir"*) echo "Warning: Work directory is defined twice in command-line arguments and the environment variable" ;;
			*)
			WORKDIR="-workDir $JENKINS_AGENT_WORKDIR" ;;
		esac
	fi

	if [ -n "$JENKINS_URL" ]; then
		URL="-url $JENKINS_URL"
	fi

	if [ -n "$JENKINS_NAME" ]; then
		JENKINS_AGENT_NAME="$JENKINS_NAME"
	fi

	if [ -z "$JNLP_PROTOCOL_OPTS" ]; then
		echo "Warning: JnlpProtocol3 is disabled by default, use JNLP_PROTOCOL_OPTS to alter the behavior"
		JNLP_PROTOCOL_OPTS="-Dorg.jenkinsci.remoting.engine.JnlpProtocol3.disabled=true"
	fi

	# If both required options are defined, do not pass the parameters
	OPT_JENKINS_SECRET=""
	if [ -n "$JENKINS_SECRET" ]; then
		case "$@" in
			*"${JENKINS_SECRET}"*) echo "Warning: SECRET is defined twice in command-line arguments and the environment variable" ;;
			*)
			OPT_JENKINS_SECRET="${JENKINS_SECRET}" ;;
		esac
	fi

	OPT_JENKINS_AGENT_NAME=""
	if [ -n "$JENKINS_AGENT_NAME" ]; then
		case "$@" in
			*"${JENKINS_AGENT_NAME}"*) echo "Warning: AGENT_NAME is defined twice in command-line arguments and the environment variable" ;;
			*)
			OPT_JENKINS_AGENT_NAME="${JENKINS_AGENT_NAME}" ;;
		esac
	fi

	#TODO: Handle the case when the command-line and Environment variable contain different values.
	#It is fine it blows up for now since it should lead to an error anyway.

	exec java $JAVA_OPTS $JNLP_PROTOCOL_OPTS -cp /usr/share/jenkins/slave.jar hudson.remoting.jnlp.Main -headless $TUNNEL $URL $WORKDIR $OPT_JENKINS_SECRET $OPT_JENKINS_AGENT_NAME "$@"
fi

构建的Dockerfile:

[root@master jenkins_salve]# cat dockerfile-jenkins-slave
FROM 192.168.25.135/library/java
MAINTAINER liaochao


ADD apache-maven-3.3.9-bin.tar.gz /tmp
RUN yum install curl git libtool-ltdl-devel -y && \
    yum clean all && \
    rm -rf /var/cache/yum/* && \
    mv /tmp/apache-maven-3.3.9 /usr/local/maven && \
    rm -rf /tmp/*.tar.gz && \
    mkdir -p /usr/share/jenkins && \
    mkdir -p /data/.maven_data
COPY slave.jar /usr/share/jenkins/slave.jar
COPY jenkins-slave /usr/bin/jenkins-slave
COPY settings.xml /usr/local/maven/conf/settings.xml
RUN chmod +x /usr/bin/jenkins-slave
ENV MAVEN_HOME=/usr/local/maven
ENV PATH=${PATH}:${MAVEN_HOME}/bin


ENTRYPOINT ["jenkins-slave"]

构建：

docker build -t 192.168.25.135/ops/jenkins_slave -f dockerfile-jenkins-slave .

推送到harbor:

docker login 192.168.25.135

账户密码用我们刚刚设置的liaochao Jlkj#123
推送：

docker push 192.168.25.135/ops/jenkins_slave

现在可以去harbo上检查镜像是否推送成功

4、编辑流水线文件jenkinsfile

这里我们将maven仓库数据存储挂载到nfs中，加快编译速度。
将kubectl 和对应的config文件也挂载到nfs中，这样可以让jenkins slave也能操作集群

这里我们还是将jenkinsfile放置在源码中，让jenkins去

[root@master deploy]# cat k8s_Jenkinsfile
podTemplate(label: 'jnlp-slave', cloud: 'kubernetes', containers: [
    containerTemplate(
        name: 'jnlp',
        image: '192.168.25.135/ops/jenkins_slave',
        alwaysPullImage: true
    ),
  ],
  volumes: [
    hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
    hostPathVolume(mountPath: '/usr/bin/docker', hostPath: '/usr/bin/docker'),
    nfsVolume(mountPath: '/data/.maven_data', readOnly: false, serverAddress: '192.168.25.137', serverPath: '/data/nfs_data/maven_data'),
    nfsVolume(mountPath: '/opt', readOnly:false, serverAddress: '192.168.25.137', serverPath: '/data/nfs_data/kubectl')
  ],
  imagePullSecrets: ['registry-pull-secret'],
)
{
node("jnlp-slave"){
  if ("${deploy_env}" == "rollback"){
     stage('start rollback'){
            sh '''
                  /opt/kubectl rollout undo deployment.apps/tomcat-java-demo -n blog --kubeconfig=/opt/config
               '''
           }
          }
    else{
      stage('Git Checkout'){
         checkout([$class: 'GitSCM', branches: [[name: '${Tag}']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '532a5d1f-f642-46db-9f61-f29fe4449b8f', url: '[email protected]:/home/git/solo.git']]])
      }
      stage('Unit Testing'){
      	echo "Unit Testing..."
      }
      stage('Maven Build'){
          sh "/usr/local/maven/bin/mvn clean package -Dmaven.test.skip=true"
      }
      stage('Build and Push Image'){
          sh '''
          docker login -u liaochao -p Jlkj#123 192.168.25.135
          docker build -t 192.168.25.135/project/tomcat-java-demo:${Tag} -f Dockerfile .
          docker push 192.168.25.135/project/tomcat-java-demo:${Tag}
          '''
      }
      stage('Deploy to K8S'){
          sh '''
           sed -i "/demo/{s/latest/${Tag}/}" deploy/deploy.yaml
        #  sed -i "s/environment/${Env}/" deploy/deploy.yaml
          '''
          kubernetesDeploy configs: 'deploy/deploy.yaml', kubeConfig: [path: ''], kubeconfigId: 'b89f5d1f-b381-4bd6-9e9e-9fe835d07447', secretName: '', ssh: [sshCredentialsId: '*', sshServer: ''], textCredentials: [certificateAuthorityData: '', clientCertificateData: '', clientKeyData: '', serverUrl: 'https://']
      }
      stage('Testing'){
          echo "Testing..."
      }
  }
 }
}

deploy.yaml不用变：

[root@master deploy]# cat deploy.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: tomcat-java-demo
  namespace: blog
spec:
  replicas: 1
  selector:
    matchLabels:
      project: www
      app: java-demo
  template:
    metadata:
      labels:
        project: www
        app: java-demo
    spec:
      imagePullSecrets:
      - name: registry-pull-secret
      containers:
      - name: tomcat
        image: 192.168.25.135/project/tomcat-java-demo:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          name: web
          protocol: TCP
        resources:
          requests:
            cpu: 0.5
            memory: 2Gi
          limits:
            cpu: 1
            memory: 2Gi
        livenessProbe:
          httpGet:
            path: /
            port: 8080
          initialDelaySeconds: 180
          timeoutSeconds: 30
        readinessProbe:
          httpGet:
            path: /
            port: 8080
          initialDelaySeconds: 200
          timeoutSeconds: 30

---
apiVersion: v1
kind: Service
metadata:
  name: tomcat-java-demo
  namespace: blog
spec:
  selector:
    project: www
    app: java-demo
  ports:
  - name: web
    port: 80
    targetPort: 8080
    nodePort: 30003
  type: NodePort

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  namespace: blog
  name: simple-nginx-example
  annotations:
    #配置重定向相关
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  rules:
  - host: www.liaochao.com
    http:
      paths:
      - path: /
        backend:
          #匹配需要代理的service名字
          serviceName: tomcat-java-demo
          #对应clusterip端口
          servicePort: 80

改完我们提交代码，编译发布：

git add .
git commit -m "v8.6"
git tag v8.6
 git push origin master
git push origin v8.6
 

回滚也是一样。
调用的kubectl undo语法