先决条件
- 需要安装 kubectl 命令行工具
- 已有云Kubernetes，本文以阿里云的Kubernetes为例
制作container镜像
Dockerfile
FROM jenkins/jenkins:2.150.3
# set timezone for Java runtime arguments #TODO: FIXME security vulnerability
ENV JAVA_OPTS='-Duser.timezone=Asia/Shanghai -Dpermissive-script-security.enabled=no_security'
# set timezone for OS by root
USER root
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# Plugins
COPY plugins.txt /usr/share/jenkins/ref/plugins.txt
RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt
# Local Plugins
COPY hpi/* /usr/share/jenkins/ref/plugins/
# install Maven
USER root
RUN sed -i "s@http://deb.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list
RUN sed -i "s@http://security.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list
RUN apt-get update && apt-get install -y maven vim
RUN update-ca-certificates --fresh
# Add vault + consul-template descriped in https://ifritltd.com/2018/03/18/advanced-jenkins-setup-creating-jenkins-configuration-as-code-and-applying-changes-without-downtime-with-java-groovy-docker-vault-consul-template-and-jenkins-job/
RUN curl https://raw.githubusercontent.com/georgedriver/devops-tools/master/vault_1.0.3_linux_amd64.zip -o vault_1.0.3_linux_amd64.zip
RUN unzip vault_1.0.3_linux_amd64.zip -d /usr/local/bin/ && rm -fr vault_1.0.3_linux_amd64.zip
RUN curl https://raw.githubusercontent.com/georgedriver/devops-tools/master/consul-template?raw=true -o consul-template
RUN mv consul-template /usr/local/bin/ && rm -fr consul-template
RUN chmod 775 /usr/local/bin/consul-template
# Init scripts
COPY script/ /usr/share/jenkins/ref/init.groovy.d/
RUN chown jenkins:jenkins -R /usr/share/jenkins/ref/init.groovy.d/
USER jenkins
plugins.txt
ssh-slaves:1.29.4
mailer:1.23
email-ext:2.65
slack:2.23
htmlpublisher:1.18
greenballs:1.15
simple-theme-plugin:0.5.1
kubernetes:1.14.8
workflow-aggregator:2.6
git:3.9.3
blueocean:1.13.2
docker-build-publish:1.3.2
http_request:1.8.22
github:1.29.4
pipeline-githubnotify-step:1.0.4
sidebar-link:1.11.0
hashicorp-vault-plugin:2.2.0
role-strategy:2.10
audit-trail:2.4
basic-branch-build-strategies:1.1.1
permissive-script-security:0.3
sonar:2.8.1
jacoco:3.0.4
fireline:1.6.10
parameterized-trigger:2.35.2
checkstyle:4.0.0
warnings-ng:4.0.0
pipeline-utility-steps:2.3.0
github-oauth:0.32
datadog:0.7.1
编译image并上传
docker build -t georgesre/jenkins-master:latest .
docker push georgesre/jenkins-master:latest
创建Jenkins持久化磁盘
pv-test.yaml
---
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1
metadata:
name: alicloud-disk-ssd-shanghai-f
provisioner: alicloud/disk
reclaimPolicy: Retain
parameters:
type: cloud_ssd
regionid: cn-shanghai
zoneid: cn-shanghai-f
fstype: "ext4"
readonly: "false"
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: jenkins.pvc-disk
spec:
accessModes:
- ReadWriteOnce
storageClassName: alicloud-disk-ssd-shanghai-f
resources:
requests:
storage: 20Gi
创建这些PV和PVC
kubectl apply -f pv-test.yaml
创建Secret用来存储Jenkins master启动时的密码token等
Kubernetes可以使用自带的secret来存储一些敏感信息，创建Secret之前我们需要先把secret的值做一次base64 encode操作
echo -n 'dummy_token' | base64
然后用以下yaml内容创建k8s Secret资源
secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: jenkins.service-secrets
type: Opaque
data:
github_token: ZHVtbXlfdG9rZW4=
创建Jenkins master Deployment
有了以上的PVCs和secret后，现在我们已经可以创建出Jenkins master Deployment了。
deploy.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: jenkins
name: jenkins
spec:
replicas: 1
template:
metadata:
labels:
app: jenkins
spec:
containers:
- name: jenkins
image: georgesre/jenkins-master:latest
resources:
limits:
cpu: "2"
memory: 2Gi
requests:
cpu: "0.5"
memory: 500Mi
volumeMounts:
- mountPath: /var/jenkins_home
name: disk-pvc
env:
- name: SERVICE_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
name: jenkins.service-secrets
key: github_token
ports:
- containerPort: 8080
name: http
protocol: TCP
- containerPort: 50000
name: jnlp
protocol: TCP
volumes:
- name: disk-pvc
persistentVolumeClaim:
claimName: jenkins.pvc-disk
几点说明：
GITHUB_TOKEN会在Jenkins启动的时候运行groovy脚本配置
github server，也用来创建Github Credential会用于后面拉取私有代码，所以要给足GitHub token的权限。
通过负载均衡（Server Load Balancer）访问服务
当我们的Jenkins master启动完成之后，我们使用负载均衡类型的SVC来访问服务
svc.yaml
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
app: jenkins
name: jenkins
spec:
externalTrafficPolicy: Cluster
ports:
- name: http
port: 8080
protocol: TCP
targetPort: 8080
- name: jnlp
port: 50000
protocol: TCP
targetPort: 50000
selector:
app: jenkins
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer: {}
Sha-51664-Mbp:jenkins-master georgehe$ kubectl apply -f deploy.yaml -n George
deployment.extensions "jenkins" created
Sha-51664-Mbp:jenkins-master georgehe$ kubectl create -f svc.yaml -n George
service "jenkins" created
georgehe@Sha-51664-Mbp ~ kubectl get pod
NAME READY STATUS RESTARTS AGE
jenkins-65d595984d-pftq7 1/1 Running 0 4h
georgehe@Sha-51664-Mbp ~ kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
jenkins LoadBalancer 172.21.5.134 47.101.74.28 8080:31026/TCP,50000:31494/TCP 32m
从外部访问Jenkins
http://172.21.5.134:8080
完成
请确保访问8080和5000端口均有正确的输出
image.png
image.png
我们按照指示来完成初始化的配置即可。
Jenkins的全局配置
我们的全局配置都已经在script/init.groovy.override中完成，Jenkins每次启动都会加载这个groovy脚本。
// ==== Let's configure label of master
import jenkins.*
import hudson.model.Node.Mode
Jenkins jenkins = Jenkins.getInstance()
jenkins.setLabelString('do-not-use-master')
jenkins.setMode(Mode.EXCLUSIVE)
println 'Configured label of master.'
// ==== Let's remove all the init credential
import com.cloudbees.plugins.credentials.domains.Domain
def credentialsStore = jenkins.model.Jenkins.instance.getExtensionList('com.cloudbees.plugins.credentials.SystemCredentialsProvider')[0].getStore()
allCreds = credentialsStore.getCredentials(Domain.global())
allCreds.each{
if (it.id == "github_token_string_cred") {
credentialsStore.removeCredentials(Domain.global(), it)
}
if (it.id == "github_token_userpass_cred") {
credentialsStore.removeCredentials(Domain.global(), it)
}
if (it.id == "vault_token") {
credentialsStore.removeCredentials(Domain.global(), it)
}
if (it.id == "jenkins_config_as_code") {
credentialsStore.removeCredentials(Domain.global(), it)
}
}
// ==== Let's setup the very initial github-token for webhook =====
import jenkins.model.*
import com.cloudbees.plugins.credentials.*
import com.cloudbees.plugins.credentials.common.*
import com.cloudbees.plugins.credentials.domains.*
import com.cloudbees.plugins.credentials.impl.*
import com.cloudbees.jenkins.plugins.sshcredentials.impl.*
import org.jenkinsci.plugins.plaincredentials.*
import org.jenkinsci.plugins.plaincredentials.impl.*
import hudson.util.Secret
import hudson.plugins.sshslaves.*
import org.apache.commons.fileupload.*
import org.apache.commons.fileupload.disk.*
import java.nio.file.Files
import com.datapipe.jenkins.vault.credentials.VaultTokenCredential;
domain = Domain.global()
store = Jenkins.instance.getExtensionList('com.cloudbees.plugins.credentials.SystemCredentialsProvider')[0].getStore()
String fileContentsGithubToken = System.getenv('GITHUB_TOKEN') ?: 'DUMMY_GITHUB_TOKEN'
String fileContentsVaultToken = System.getenv('VAULT_TOKEN') ?: 'DUMMY_VAULT_TOKEN'
secretTextGithub = new StringCredentialsImpl( CredentialsScope.GLOBAL, "github_token_string_cred", "github_token_string_cred", Secret.fromString(fileContentsGithubToken))
secretUserPassGithub = new UsernamePasswordCredentialsImpl( CredentialsScope.GLOBAL, "github_token_userpass_cred", "github_token_userpass_cred", "georgedriver", fileContentsGithubToken)
secretTextVault = new VaultTokenCredential(CredentialsScope.GLOBAL, "vault_token", "vault_token", Secret.fromString(fileContentsVaultToken));
store.addCredentials(domain, secretTextGithub)
store.addCredentials(domain, secretUserPassGithub)
store.addCredentials(domain, secretTextVault)
println 'Configured Credentials: vault_token vault_token.'
// ==== Let's config github server
import com.cloudbees.plugins.credentials.CredentialsScope
import com.cloudbees.plugins.credentials.domains.Domain
import hudson.util.Secret
import jenkins.model.JenkinsLocationConfiguration
import org.jenkinsci.plugins.github.GitHubPlugin
import org.jenkinsci.plugins.github.config.GitHubPluginConfig
import org.jenkinsci.plugins.github.config.GitHubServerConfig
import org.jenkinsci.plugins.plaincredentials.impl.StringCredentialsImpl
// configure github plugin
GitHubPluginConfig pluginConfig = GitHubPlugin.configuration()
GitHubServerConfig serverConfig = new GitHubServerConfig('github_token_string_cred')
serverConfig.name = "My GitHub.com"
serverConfig.manageHooks = true
pluginConfig.setConfigs([serverConfig])
pluginConfig.save()
println 'Configured GitHub plugin.'
// ===== Let's configure Vault
// https://github.com/buildit/jenkins-startup-scripts
import com.datapipe.jenkins.vault.configuration.GlobalVaultConfiguration
import com.datapipe.jenkins.vault.configuration.VaultConfiguration
import jenkins.model.GlobalConfiguration
String vault_addr = System.getenv('VAULT_ADDR') ?: 'DUMMY_VAULT_ADDR'
GlobalVaultConfiguration globalConfig = GlobalConfiguration.all().get(GlobalVaultConfiguration.class)
globalConfig.setConfiguration(new VaultConfiguration(vault_addr, 'vault_token'))
globalConfig.save()
println 'Configured Vault plugin.'
// ===== Let's configure Datadog
import jenkins.model.*
import org.datadog.jenkins.plugins.datadog.DatadogBuildListener
String dd_api_key = System.getenv('DD_API_KEY') ?: 'DUMMY_DD_API_KEY'
String service_namespace = System.getenv('SERVICE_NAMESPACE') ?: 'DUMMY_SERVICE_NAMESPACE'
def j = Jenkins.getInstance()
def d = j.getDescriptor("org.datadog.jenkins.plugins.datadog.DatadogBuildListener")
d.setHostname('tooling-'+ service_namespace + '-jenkins')
d.setTagNode(true)
d.setApiKey(dd_api_key)
d.setBlacklist('job1,job2')
d.setGlobalJobTags('region=china\n(.*?)/(.*?)/.*, mission:$1, project:$2')
d.save()
println 'Configured datadog plugin.'
- 我们的Jenkins本身的label设置为
do-not-use-master，除非是必须使用Jenkins来完成某些特定任务，我们都不应该使用它来跑任务- init credential: 删除老的credentials，从环境变量中获取新值创建新的credentials，主要有github_token_string_cred，github_token_userpass_cred，vault_token，jenkins_config_as_code，如果没有相应的环境变量，那么会有dummy value来替代。
- 其他的配置请自行阅读
更多
- 为了能够让Jenkins master本身
agent { label "do-not-use-master" }能够build docker镜像（04 Jenkins Kubernetes插件动态创建slave agent），我们提交了PR-1到Jenkins master
diff --git a/Dockerfile b/Dockerfile
index 658f177..af17295 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,10 +3,24 @@ FROM jenkins/jenkins:2.150.3
# set timezone for Java runtime arguments #TODO: FIXME security vulnerability
ENV JAVA_OPTS='-Duser.timezone=Asia/Shanghai -Dpermissive-script-security.enabled=no_security'
+# docker daemonの動いているホストのGIDを指定する
+# docker run -v /var/run/docker.sock:/var/run/docker.sock で
+# ホストのdocker daemonを共有する前提
+ENV DOCKER_GROUP_GID 501
+
# set timezone for OS by root
USER root
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+# docker のバイナリをinstall
+RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-18.03.1-ce.tgz
+RUN tar -xvf docker-18.03.1-ce.tgz
+RUN mv docker/* /usr/bin/
+
+# jenkins userでもdockerが使えるようにする
+RUN groupadd -o -g ${DOCKER_GROUP_GID} docker
+RUN usermod -g docker jenkins
+
# Plugins
COPY plugins.txt /usr/share/jenkins/ref/plugins.txt
RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt
diff --git a/deploy.yaml b/deploy.yaml
index 70ab293..2244b72 100644
--- a/deploy.yaml
+++ b/deploy.yaml
@@ -35,6 +35,8 @@ spec:
secretKeyRef:
name: jenkins.service-secrets
key: github_token
+ - name: DOCKER_HOST
+ value: tcp://localhost:2375
ports:
- containerPort: 8080
name: http
@@ -42,7 +44,17 @@ spec:
- containerPort: 50000
name: jnlp
protocol: TCP
+ - name: dind
+ image: docker:dind
+ imagePullPolicy: Always
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: dind-storage
+ mountPath: /var/lib/docker
volumes:
- name: disk-pvc
persistentVolumeClaim:
claimName: jenkins.pvc-disk
+ - name: dind-storage
+ emptyDir: {}
利用dind作为sidecar，将来Jenkins master上所有关于docker的命令都会运行在这个dind sidecar中
问题
- No valid crumb was included in the request
请到http:///configureSecurity/下关闭CSRF配置
image.png
更多
- 回到Jenkins系列目录
云平台开发运维解决方案@george.sre
个人主页：https://geekgoogle.com
GitHub: https://github.com/george-sre
Mail: [email protected]
: georgesre -
欢迎交流～