Kubernetes+Gitlab+Jenkins构建镜像并创建Pod

1.重新构建 jnlp-slave

# ls
config  docker-18.06.1-ce.tgz  Dockerfile  kubectl

A.创建 config

# vim config
apiVersion: v1
clusters:
- cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
    server: https://192.168.100.180:8443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: admin
  name: kubernetes
current-context: kubernetes
kind: Config
preferences: {}
users:
- name: admin
  user:
    client-certificate: /etc/kubernetes/ssl/admin.pem
    client-key: /etc/kubernetes/ssl/admin-key.pem

B.创建 Dockerfile

# vim Dockerfile
FROM jenkins/jnlp-slave:3.23-1-alpine
MAINTAINER zhi 

USER root
ARG DOCKER_GID=994
ENV DOCKER_VERSION=18.06.1-ce
ENV SSL_DIR /etc/kubernetes/ssl

# 提取 docker 二进制文件 
COPY docker-${DOCKER_VERSION}.tgz /var/tmp/
RUN  tar --strip-components=1 -xvzf /var/tmp/docker-${DOCKER_VERSION}.tgz -C /usr/local/bin \
    && rm -rf /var/tmp/docker-${DOCKER_VERSION}.tgz \
    && chmod -R 775 /usr/local/bin/docker
    
# 安装 kubectl
COPY kubectl /usr/local/bin/

# 此处文件均为空文件将在运行时由 ConfigMap 挂载为 Volume 填充真实证书文件
COPY config ${SSL_DIR}/
RUN mkdir -p /root/.kube/ && mkdir -p ${SSL_DIR} \
    touch ${SSL_DIR}/ca.pem \
    touch ${SSL_DIR}/admin.pem \
    touch ${SSL_DIR}/admin-key.pem  
RUN export KUBE_CONFIG==${SSL_DIR}/config && kubectl config view 
RUN addgroup -g ${DOCKER_GID} docker && adduser jenkins docker 

# 暴露证书文件所在文件夹，在运行时由 ConfigMap 挂载为 Volume 填充真实证书文件
VOLUME ${SSL_DIR}

USER jenkins:${DOCKER_GID}

注: docker 和 kubectl 跟 kubernetes 集群的版本一致。同时说明一下，这里GitLab 、Jenkins都部署在kube-ops这个namespace下。而docker使用了docker in docker 挂载宿主机/var/run/docker.dock进行操作。

C.构建 jnlp-slave 并推送至私有镜像仓库

# docker build -t 192.168.100.100/jenkinsci/jnlp-slave:latest .
# docker images|grep jnlp-slave
192.168.100.100/jenkinsci/jnlp-slave         latest            d3b287300a83        6 days ago          377MB
# docker push 192.168.100.100/jenkinsci/jnlp-slave:latest

2.创建并查看 configmap

# kubectl create configmap kubectl-cert-cm --from-file=/etc/kubernetes/ssl -n kube-ops
# kubectl describe configmap kubectl-cert-cm -n kube-ops

3.部署 Jenkins

注: 由于无法解决配置了专有 secret 而 Jenkins 依然去找 default 的 secret 的问题，这里重新配置了 RBAC 以使用 default 的 secret 并绑定了 cluster-admin 角色。

A.创建 RBAC 

# vim jenkins-rbac.yaml
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: default
  namespace: kube-ops
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","get","update","list","watch","patch","delete"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","get","update","list","watch","patch","delete"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: default
  namespace: kube-ops
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: default
  namespace: kube-ops
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: default
  namespace: kube-ops
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: default
subjects:
- kind: ServiceAccount
  name: default
  namespace: kube-ops

B.创建PVC

# vim jenkins-pvc.yaml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-pv-claim
  namespace: kube-ops
  labels:
    app: jenkins
spec:
  storageClassName: kube-ops
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: "10Gi"

C.创建 deployment 和 service

# vim jenkins.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: jenkins
  namespace: kube-ops
spec:
  replicas: 1
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      #serviceAccountName: jenkins
      serviceAccountName: default
      securityContext:
        runAsUser: 1000
        fsGroup: 1000
      containers:
      - name: jenkins
        image: 192.168.100.100/jenkinsci/jenkins:lts
        resources:
          requests:
            memory: "1Gi"
            cpu: "1000m"
          limits:
            memory: "2Gi"
            cpu: "2000m"
        env:
        - name: JAVA_OPTS
          value: "-Duser.timezone=Asia/Shanghai"  
        ports:
        - name: web
          containerPort: 8080
        - name: agent
          containerPort: 50000
        volumeMounts:
        - mountPath: /var/jenkins_home
          name: jenkins-data
        volumeMounts:
        - mountPath: /var/run/docker.sock
          readOnly: false
          name: docker-sock
      volumes:
      - name: jenkins-data
        persistentVolumeClaim:
          claimName: jenkins-pv-claim
      - name: docker-sock
        hostPath: 
          path: /var/run/docker.sock
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: kube-ops
  labels:
    name: jenkins
spec:
  type: LoadBalancer
  ports:
    - name: web
      port: 8080
      targetPort: web
    - name: agent
      port: 50000
      targetPort: agent
  selector:
    name: jenkins

注:这里就不在赘述Jenkins的部署。也不再赘述gitlab的部署，需要更改的地方大同小异。

4.配置Jenkins

5.新建项目

# git clone http://192.168.100.185:32030/wangzhijian/grafana.git
正克隆到 'grafana'...
Username for 'http://192.168.100.185:32030': wangzhijian
Password for 'http://[email protected]:32030': 
remote: Counting objects: 37, done.
remote: Compressing objects: 100% (21/21), done.
remote: Total 37 (delta 7), reused 37 (delta 7)
Unpacking objects: 100% (37/37), done.
# ls grafana/
build  grafana.yaml
# ls grafana/build/
build.sh  Dockerfile

注:该项非必须，只是想显示一下文件存放目录。本来我想使用本地添加hosts，然后使用域名来git，但是不知道是因为添加了traefik代理的原因还是traefik的SSL没有完全配置好的原因，总之这里没有解决使用域名来git的问题。

使用IP进行git操作的格式:git clone http://nodeip:port/yourname/project.git

A.配置Dockerfile

# vim Dockerfile 
ARG GRAFANA_VERSION="latest"

FROM 192.168.100.100/grafana/grafana:${GRAFANA_VERSION}

USER grafana

ARG GF_INSTALL_PLUGINS=""

RUN if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then \
    OLDIFS=$IFS; \
        IFS=','; \
    for plugin in ${GF_INSTALL_PLUGINS}; do \
        IFS=$OLDIFS; \
        grafana-cli --pluginsDir "$GF_PATHS_PLUGINS" plugins install ${plugin}; \
    done; \

B.配置构建脚本

# vim build.sh 
#!/bin/bash

IMAGE="192.168.100.100/grafana/grafana-plugins"

# 编译镜像
docker build -f build/Dockerfile -t $IMAGE \
  --build-arg "GRAFANA_VERSION=latest" \
  --build-arg "GF_INSTALL_PLUGINS=grafana-worldmap-panel,grafana-clock-panel,grafana-piechart-panel,alexanderzobnin-zabbix-app,grafana-kubernetes-app,grafana-simple-json-datasource,michaeldmoore-annunciator-panel" .
  
# 登录Harbor镜像仓库
docker login -u admin -p Harbor12345 192.168.100.100

# 上传镜像 
docker push $IMAGE

# 清理镜像
docker rmi $IMAGE

C.创建yaml文件

# vim grafana.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: grafana
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      name: grafana
      labels:
        k8s-app: grafana
    spec:
      containers:
      - name: grafana
        image: 192.168.100.100/grafana/grafana-plugins:latest
        ports:
        - containerPort: 3000
          protocol: TCP
        env:
        - name: GF_SERVER_HTTP_PORT
          value: "3000"
        - name: GF_AUTH_BASIC_ENABLED
          value: "true"
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: "false"
        - name: GF_SECURITY_ADMIN_USER
          value: "admin"
        - name: GF_SECURITY_ADMIN_PASSWORD
          value: "wangzhijian"
        - name: GF_SERVER_ROOT_URL
          value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
          value: /
        - name: GF_SMTP_ENABLED
          value: "true"
        - name: GF_SMTP_HOST
          value: smtp.qq.com:465
        - name: GF_SMTP_USER
          value: [email protected]
        - name: GF_SMTP_PASSWORD
          value: ********
        - name: GF_SMTP_FROM_ADDRESS
          value: [email protected]
---
apiVersion: v1
kind: Service
metadata:
  labels:
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Grafana
  name: grafana
  namespace: kube-system
spec:
  type: LoadBalancer
  ports:
  - port: 80
    targetPort: 3000
  selector:
    k8s-app: grafana

6.使用Jenkins新建任务

7.构建输出

A.构建镜像并推送到私有镜像仓库

B.部署grafana

C.查看部署情况

# kubectl cluster-info|grep Grafana
Grafana is running at https://192.168.100.180:8443/api/v1/namespaces/kube-system/services/grafana/proxy
# kubectl -n kube-system get pod|grep grafana
grafana-576fd64977-2kj6c                1/1       Running   0          1d
# kubectl -n kube-system get service|grep grafana
grafana                   LoadBalancer   10.244.249.26         80:36902/TCP      1d

这里使用浏览器输入http://nodeip:port,使用变量配置的用户名(admin)和密码(wangzhijian)进行登录

构建镜像时添加的一些插件显示如下:

特别感谢:

为了将kubectl运行起来，试过来很多方法，也搜了很多资料，但是依然不得其所，感谢作者和他的这篇文章:

基于 IBM Cloud Private 的 DevOps 实践