Jenkins

一. Jenkins

官网

1. 部署

# 解决报错: 无法验证 pkg.jenkins.io 的由 “/C=US/O=Let‘s Encrypt/CN=R3” 颁发的证书: 颁发的证书已经过期。 要以不安全的方式连接至 pkg.jenkins
yum install -y ca-certificates

wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo
rpm --import https://pkg.jenkins.io/redhat/jenkins.io.key

yum install fontconfig java-11-openjdk -y
yum install jenkins -y

• 或者这样

cat >>/etc/yum.repos.d/jenkins.repo<<EOF
[jenkins]
name=Jenkins-stable
baseurl=http://pkg.jenkins.io/redhat-stable
gpgcheck=1
EOF

rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key
yum install java-11-openjdk -y
yum install jenkins -y 
systemctl daemon-reload
systemctl enable jenkins;systemctl start jenkins

2. 插件

git
Chinese
pipeline
GitLab
Maven
Git Parameter				# git 参数化构建
Extended Choice Parameter	# 选项参数，可多选
Pipeline: Stage View		# 阶段视图
blue ocean					# 流水线视图
nodejs						# 前端打包npm

# workspace清理插件
ws-cleanup

# 连接k8s
Kubernetes CLI

# 角色控制
Role-based Authorization Strategy

# 钉钉消息
dingding

1). 离线安装插件

官网插件下载

阿里云插件下载

• 先下载插件到本地，然后如下，Advanced settings ==> Deploy Plugin ==> 上传

3. Manage and Assign Roles

• 权限管控

• 安装这个插件后, 需要在全局安全设置中设置, 才能打开, 旧版本可以不用设置

• • 系统管理==>全局安全设置

• 在授权策略中设置为: Role-based Strategy, 应用保存

• 之后返回系统管理界面, 可以看到Manage and Assign Roles

1). Manage Roles

• Global roles

• • 全局角色

• Item roles

• • 项目角色

• Node roles

• • 代理角色

• 需要先配置Manage Roles, 然后配置Assign Roles

2). 使用案例

• 创建zhangsan 和 lisi 两个用户

• 创建2个任务

• 设置 Manage Roles , 只需要给 Read 权限即可

• 点击 Assign Roles 配置

4. Kubernetes CLI

Kubernetes CLI | Jenkins

首先要在Jenkins上安装插件 Kubernetes CLI ,然后在Kubernetes上创建用户

[root@master-3 ~/images]# kubectl -n  create serviceaccount jenkins-robot

# 如果想给单独 Namespace 创建只需要创建 rolebinding， 如果是整个集群，就使用 clusterrolebinding
[root@master-3 ~/images]# kubectl -n  create clusterrolebinding jenkins-robot-binding --clusterrole=cluster-admin --serviceaccount=:jenkins-robot

# 获得 sa的 secret
[root@master-3 ~/images]# kubectl -n  get serviceaccount jenkins-robot -o go-template --template='{{range .secrets}}{{.name}}{{"\n"}}{{end}}'

[root@master-3 ~/images]# kubectl -n hhy get secrets `kubectl -n hhy  get sa jenkins-robot -o go-template --template='{{range .secrets}}{{.name}}{{"\n"}}{{end}}'` -o go-template --template '{{index .data "token"}}' | base64 -d

然后再 Jenkins 上创建 Secret text 类型的凭据，将上面 base64 -d 获得的信息 贴进去，就可以在 Jenkins 上使用 withKubeConfig() 连接集群

测试该角色是否有权限，随便找个名称空间下，看是否可以获得他们的信息

curl -k  -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkcyTjNrRFYwNGZmNl9Qc203N2YySUllNUg2T3lTSWg2cm9FUDdiUWdZbnMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJqZW5raW5zLXJvYm90LXRva2Vxxxxxxxxxxxxxx" https://192.99.99.99:6443/api/v1/namespaces/ingress-nginx/pods

Jenkins在集群中可以这样写

container('kubectl') {
    withKubeConfig([credentialsId: "Kubernetes Token 凭据 ID",serverUrl: "https://kubernetes.default.svc.cluster.local"]) {
        sh "kubectl get nodes"
    }
}

5. jenkins SSH拉取代码

# jenkin 上创建秘钥对
ssh-keygen

# 复制到 gitlab 上， https://gitlab.yee.net.cn/-/profile/keys
cat ~/.ssh/id_rsa.pub

# Jenkins 上创建 "SSH Username with private key" 类型的 credentials（凭据）

# 测试是否通
ssh -T [email protected]

安全方面提示：

You're using 'Known hosts file' strategy to verify ssh host keys, but your known_hosts file does not exist, please go to 'Manage Jenkins' -> 'Security' -> 'Git Host Key Verification Configuration' and configure host key verification.

• Manage Jenkins ==> Security ==> Git Host Key Verification Configuration

凭据创建

报错：

Couldn't find any revision to build. Verify the repository and branch configuration for this job

• 因为最新的 gitlab 的默认分支不再是master，而是main分支

• 可以在 Gitlab上修改默认分支为 master

pipeline {
    agent any
    environment {
        GIT_PATH = "[email protected]"
        PROJECT_NAME = "${JOB_NAME}"
        IMAGE_HOST_TEST = "harbor.yee.net.cn"
        MVN_CMD = "mvn clean -U install -Pprd -Dmaven.test.skip=true"
    }

    tools {
        maven "maven-3.3.9"
    }

    stages {
        stage('Init') {
            steps {
                script{
                    images = []
                   _imageTag = ""
                   _imageGroup = env.IMAGE_GROUP
                   _dockerFile = "dockerfile"
                   _deployServiceNames = []
                   _deployEnv = params.deploy_env //test
                   _isDev = false
                   _canDeploy = true
                   _serviceNames = []
                   if(params.service_types!=""){
                       for (type in params.service_types.tokenize(',')){
                            _serviceNames.add("${PROJECT_NAME}-${type}")
                       }
                   }else{
                       echo "[Error]没有选择服务"
                   }
                   
                   if(_imageGroup==null || _imageGroup==""){
                       _imageGroup = "${PROJECT_NAME.split('-')[0]}"
                   }
                   
                   _imageHost = "${IMAGE_HOST_TEST}"
                   echo "发布环境: ${_deployEnv}"
                   
                  if(params.branch_or_tag == "" || _serviceNames.size()==0){
                      _canDeploy = false
                  }
                }
            }
        }
        stage('Checkout') {
            steps {
                git branch: 'main', credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"
                

                # 这种方法也可以
                // // https://plugins.jenkins.io/git-parameter/
                // checkout([$class: 'GitSCM', 
                //     branches: [[name: "${branch_or_tag}"]], 
                //     extensions: [], 
                //     userRemoteConfigs: [[credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"]]
                // ])
            }
        }
    }
}

6. maven

maven3.39版本下载一

maven3.39版本下载二

# 插件目录
/var/lib/jenkins/plugin

7. 案例

使用 Username with password 类型的凭据

    environment {
        // HARBOR="harbor.devopsing.site"
        HARBOR_ACCESS_KEY = credentials('harbor-userpwd-pair')
            }
    .....

    docker login --username=${HARBOR_ACCESS_KEY_USR} --password=${HARBOR_ACCESS_KEY_PSW} ${HARBOR}

JAVA

pipeline {
    agent any
    environment {
        GIT_PATH = "[email protected]"
        HARBOR_PRJ = "yee"
        DOCKER_FILE = "dockerfile"        
        PROJECT_NAME = "${JOB_NAME}"
        IMAGE_HOST_TEST = "harbor.yee.cn"
        MVN_CMD = "mvn clean -U install -Pprd -Dmaven.test.skip=true"
    }

    tools {
        maven "maven-3.3.9"
    }

    stages {
        stage('Init') {
            steps {
                script{
                    images = []
                   _imageTag = ""
                   _imageGroup = env.IMAGE_GROUP
                   _deployServiceNames = []
                   _deployEnv = params.deploy_env //test
                   _isDev = false
                   _canDeploy = true
                   _serviceNames = []
                   if(params.service_types!=""){
                       for (type in params.service_types.tokenize(',')){
                            _serviceNames.add("${type}")
                       }
                   }else{
                       echo "[Error]没有选择服务"
                   }
                   
                   if(_imageGroup==null || _imageGroup==""){
                       _imageGroup = "${PROJECT_NAME.split('-')[0]}"
                   }
                   
                   _imageHost = "${IMAGE_HOST_TEST}"
                   echo "发布环境: ${_deployEnv}"
                   
                  if(params.branch_or_tag == "" || _serviceNames.size()==0){
                      _canDeploy = false
                  }
                }
            }
        }
        stage('Checkout') {
            steps {
                // git branch: 'main', credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"
                
                echo "git: ${GIT_PATH}"
                echo "branch: ${branch_or_tag}"
                
                https://plugins.jenkins.io/git-parameter/
                checkout([$class: 'GitSCM', 
                    branches: [[name: "${branch_or_tag}"]], 
                    extensions: [], 
                    userRemoteConfigs: [[credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"]]
                ])
            }
        }
        stage(' maven ') {
            when { expression { _canDeploy } }
            steps {
                script{
                    sh "${MVN_CMD}"
                }
            }
        }
        stage( ' deploy ' ) {
	       steps {
	           script{
	               def imageTag = sh(script:"date '+%Y%m%d%H%M%S'", returnStdout: true).replace("\n","")
                   for (serviceName in _serviceNames){
                        def workDic = "${WORKSPACE}/${PROJECT_NAME}-${serviceName}"
                        HARBOR_IMAGE_PUSH = "${IMAGE_HOST_TEST}/${HARBOR_PRJ}/${PROJECT_NAME}-${serviceName}:${imageTag}"
                        def CONFIG_FILE = "/data/k8s-deploy-scripts/test-${PROJECT_NAME}-${serviceName}.yaml"
                        
                        dir("${workDic}/target/"){
                            sh "docker build . -f ${JENKINS_HOME}/${DOCKER_FILE} -t ${HARBOR_IMAGE_PUSH}"
                            sh "docker push ${HARBOR_IMAGE_PUSH}"
                            sh "docker rmi ${HARBOR_IMAGE_PUSH}"                            
                        }
                        _deployServiceNames.add("${PROJECT_NAME}-${serviceName}");
                        images.add("${HARBOR_IMAGE_PUSH}")
                        
                        sh """
	                      ssh [email protected] 'yq -i ".spec.template.spec.containers[].image = \\"${HARBOR_IMAGE_PUSH}\\"" ${CONFIG_FILE}'
	                    """
	                    sh(script: "ssh [email protected] kubectl apply -f ${CONFIG_FILE}")
                   }
	           }
	       }
        }
        
        stage(' output ') {
            steps {
                script{
                    echo "服务名称：${_deployServiceNames}"
                    echo "镜像：${images}"
                }
            }
        }         
    }
}

前端项目

//前端项目打包构建；支持多环境
pipeline {
    agent any
    environment {
        GIT_PATH = "[email protected]"
        DOCKER_FILE = "dockerfile"
        HARBOR_PRJ = "yee"
        IMAGE_HOST_TEST = "harbor.yee.cn"
        PROJECT_NAME = "${JOB_NAME}"
    }

    stages {
        stage('Init') {
            steps {
                script{
                    dev_imgFile = ""
                    prd_imgFile = ""
                   _imageTagForTest = ""
                   _imageTag = ""
                   _imageGroup = env.IMAGE_GROUP
                   _envs = ["prod","test"]
                
                   _envTexts = [prod:'生产环境',test:'测试环境']
                   
                   //if(!params.build_prd){
                   //   _envs = ["test"]
                   //}
                   
                   if(_imageGroup==null || _imageGroup==""){
                       _imageGroup = "${PROJECT_NAME.split('-')[0]}"
                   }
                }
            }
        }
        stage('Checkout') {
            steps {
                echo "git: ${GIT_PATH}"
                echo "branche: ${branch_or_tag}"
                
                checkout([$class: 'GitSCM', 
                    branches: [[name: "${branch_or_tag}"]], 
                    extensions: [], 
                    userRemoteConfigs: [[credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"]]
                ])
                
                echo "code checkout completed"
            }
        }
        stage(' npm ') {
            when { expression {  params.branch_or_tag != "" } }
            steps {
                script{
                    def imageTag = sh(script:"date '+%Y%m%d%H%M%S'", returnStdout: true).replace("\n","")
                    def workDic = "${WORKSPACE}"
                    def imgName = "/${_imageGroup}/${PROJECT_NAME}"
                    nodejs('node14.17.3') {
                        sh "node -v"
                        sh "npm -v"
                        if(params.clean_cache){
                            sh "rm -rf package-lock.json"
                            sh "rm -rf node_modules/"
                        }
                        sh "npm install --unsafe-perm --legacy-peer-deps --registry https://registry.npmmirror.com"   
                        sh "npm run build:test"
                        dir("${workDic}/dist"){
                            HARBOR_IMAGE_PUSH = "${IMAGE_HOST_TEST}/${HARBOR_PRJ}/${PROJECT_NAME}:${imageTag}"
                            sh "docker build . -f ${JENKINS_HOME}/${DOCKER_FILE} -t ${HARBOR_IMAGE_PUSH}"
                            sh "docker push ${HARBOR_IMAGE_PUSH}"
                            sh "docker rmi ${HARBOR_IMAGE_PUSH}"                            
                        }
                    }

                }
            }
        }
        stage( ' deploy ' ) {
	       steps {
	           script{
	               def CONFIG_FILE = "/data/k8s-deploy-scripts/test-${PROJECT_NAME}.yaml"
	               
                   sh """
	                  ssh [email protected] 'yq -i ".spec.template.spec.containers[].image = \\"${HARBOR_IMAGE_PUSH}\\"" ${CONFIG_FILE}'
	               """
	               
	               sh(script: "ssh [email protected] kubectl apply -f ${CONFIG_FILE}")
	           }
	       }
        }
        
        stage(' output ') {
            steps {
                script{
                    echo "服务名称：${PROJECT_NAME}"
                    echo "镜像：${HARBOR_IMAGE_PUSH}"
                }
            }
        }
    }
}

8. 登录 Jenkin 用户

su -s /bin/bash jenkins

9 参数

清除旧的构建

参数使用，一次性多选服务

10. slave

# 类似这种报错， 需要勾选： Use WebSocket
java.io.IOException: http://192.168.99.185:8000/ provided port:50000 is not reachable

二. 容器部署

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: argocd
spec:
  replicas: 1
  selector:
    matchLabels:
      devops: jenkins
  template:
    metadata:
      labels:
        devops: jenkins
    spec:
      nodeName: k8s-208092-prd
      hostAliases:
       - ip: "xx.xx.xx.xx"
         hostnames:
         - "harbor.yee.net.cn"
      serviceAccount: jenkins
      initContainers:
      - name: fix-permissions
        image: busybox
        imagePullPolicy: IfNotPresent
        command:
        - /bin/sh
        - -c
        - |
          chown -R 1000:1000 /var/jenkins_home
        securityContext:
          privileged: true
        volumeMounts:
        - name: jenkinshome
          mountPath: /var/jenkins_home
      containers:
      - name: jenkins
        #image: jenkinsci/blueocean:1.25.5
        image: harbor.yee.com:8443/library/jenkins:1.25.5
        imagePullPolicy: IfNotPresent
        ports:
        - name: http
          containerPort: 8080
        - name: slavelistener
          containerPort: 50000
        volumeMounts:
        - name: jenkinshome
          mountPath: /var/jenkins_home
        - name: date-config
          mountPath: /etc/localtime
        - name: deploy-sh
          mountPath: /manifest 
        env:
        - name: JAVA_OPTS
          value: "-Xms4g -Xmx8g -Duser.timezone=Asia/Shanghai -Dhudson.model.DirectoryBrowserSupport.CSP="
      volumes:
      - name: jenkinshome
        persistentVolumeClaim:
          claimName: jenkins-pvc
      - name: date-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      - name: deploy-sh
        configMap:
          name: jenkins-deploy-sh

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins-vie
  namespace: argocd
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  - name: slavelistener
    port: 50000
    targetPort: 50000
  type: ClusterIP
  selector:
    devops: jenkins
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jenkins
  namespace: argocd
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  rules:
  - host: jenkins-vie.yee.com
    http:
      paths:
      - backend:
          service:
            name: jenkins-vie
            port:
              number: 8080
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - jenkins-ops.yee.com
    secretName: yee.com