【kubernetes】使用KubeSphere devops部署我的微服务系统
KubeSphere Devops
入门使用KubeSphere的Devops功能部署"我的微服务系统"
（内容学习于尚硅谷云原生课程）kubesphere devops官方文档：
https://v3-1.docs.kubesphere.io/zh/docs/devops-user-guide/how-to-use/create-a-pipeline-using-jenkinsfile/
代码准备
暂时部署这4个服务，auth服务、crm服务、gateway服务、前端ui服务
Dockerfile
前端kwsphere
# 基础镜像
FROM nginx
# author
MAINTAINER kw
# 挂载目录
VOLUME /home/kw-microservices/ui/kwsphere
# 创建目录
RUN mkdir -p /home/kw-microservices/ui/kwsphere
# 指定路径
WORKDIR /home/kw-microservices/ui/kwsphere
# 复制conf文件到路径
COPY ./nginx.conf /etc/nginx/nginx.conf
# 复制html文件到路径
COPY ./dist /home/kw-microservices/ui/kwsphere
nginx配置，内容来源于ruoyi
worker_processes 1;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
server {
listen 80;
server_name localhost;
location / {
root /home/kw-microservices/ui/kwsphere;
try_files $uri $uri/ /index.html;
index index.html index.htm;
}
location /prod-api/{
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://kw-gateway:8080/;
}
# 避免actuator暴露
if ($request_uri ~ "/actuator") {
return 403;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
后端
后端这个3个服务的dockefile的内容是一样的
# 基础镜像
FROM openjdk:8-jre
# author
MAINTAINER kw
# 挂载目录
VOLUME /home/kw-microservices
# 创建目录
RUN mkdir -p /home/kw-microservices
# 指定路径
WORKDIR /home/kw-microservices
# 复制jar文件到路径
COPY target/*.jar /home/kw-microservices/app.jar
# 启动网关服务
ENTRYPOINT ["java","-jar","app.jar"]
k8s deploy.yaml
前端kwsphere
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: kwsphere
name: kwsphere
namespace: my-server #一定要写名称空间
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app: kwsphere
strategy:
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
type: RollingUpdate
template:
metadata:
labels:
app: kwsphere
spec:
imagePullSecrets:
- name: aliyun-registry-secret #提前在项目下配置访问阿里云的账号密码
containers:
- image: $REGISTRY/$DOCKERHUB_NAMESPACE/kwsphere:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER
# readinessProbe:
# httpGet:
# path: /actuator/health
# port: 8080
# timeoutSeconds: 10
# failureThreshold: 30
# periodSeconds: 5
imagePullPolicy: Always
name: app
ports:
- containerPort: 80
protocol: TCP
resources:
limits:
cpu: 300m
memory: 600Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
labels:
app: kwsphere
name: kwsphere
namespace: my-server
spec:
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
nodePort: 31234
selector:
app: kwsphere
sessionAffinity: None
type: NodePort
后端kw-gateway
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: kw-gateway
name: kw-gateway
namespace: my-server #一定要写名称空间
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app: kw-gateway
strategy:
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
type: RollingUpdate
template:
metadata:
labels:
app: kw-gateway
spec:
imagePullSecrets:
- name: aliyun-registry-secret #提前在项目下配置访问阿里云的账号密码
containers:
- image: $REGISTRY/$DOCKERHUB_NAMESPACE/kw-gateway:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER
# readinessProbe:
# httpGet:
# path: /actuator/health
# port: 8080
# timeoutSeconds: 10
# failureThreshold: 30
# periodSeconds: 5
imagePullPolicy: Always
name: app
ports:
- containerPort: 8080
protocol: TCP
resources:
limits:
cpu: 300m
memory: 600Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
labels:
app: kw-gateway
name: kw-gateway
namespace: my-server
spec:
ports:
- name: http
port: 8080
protocol: TCP
targetPort: 8080
selector:
app: kw-gateway
sessionAffinity: None
type: ClusterIP
后端kw-auth-server
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: kw-auth-server
name: kw-auth-server
namespace: my-server #一定要写名称空间
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app: kw-auth-server
strategy:
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
type: RollingUpdate
template:
metadata:
labels:
app: kw-auth-server
spec:
imagePullSecrets:
- name: aliyun-registry-secret #提前在项目下配置访问阿里云的账号密码
containers:
- image: $REGISTRY/$DOCKERHUB_NAMESPACE/kw-auth-server:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER
# readinessProbe:
# httpGet:
# path: /actuator/health
# port: 8080
# timeoutSeconds: 10
# failureThreshold: 30
# periodSeconds: 5
imagePullPolicy: Always
name: app
ports:
- containerPort: 8080
protocol: TCP
resources:
limits:
cpu: 300m
memory: 600Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
labels:
app: kw-auth-server
name: kw-auth-server
namespace: my-server
spec:
ports:
- name: http
port: 8080
protocol: TCP
targetPort: 8080
selector:
app: kw-auth-server
sessionAffinity: None
type: ClusterIP
后端kw-crm-server
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: kw-crm-server
name: kw-crm-server
namespace: my-server #一定要写名称空间
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app: kw-crm-server
strategy:
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
type: RollingUpdate
template:
metadata:
labels:
app: kw-crm-server
spec:
imagePullSecrets:
- name: aliyun-registry-secret #提前在项目下配置访问阿里云的账号密码
containers:
- image: $REGISTRY/$DOCKERHUB_NAMESPACE/kw-crm-server:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER
# readinessProbe:
# httpGet:
# path: /actuator/health
# port: 8080
# timeoutSeconds: 10
# failureThreshold: 30
# periodSeconds: 5
imagePullPolicy: Always
name: app
ports:
- containerPort: 8080
protocol: TCP
resources:
limits:
cpu: 300m
memory: 600Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
labels:
app: kw-crm-server
name: kw-crm-server
namespace: my-server
spec:
ports:
- name: http
port: 8080
protocol: TCP
targetPort: 8080
selector:
app: kw-crm-server
sessionAffinity: None
type: ClusterIP
环境准备
- 阿里云镜像服务（电脑计算资源有限，暂时使用阿里云，后续搭建harbor）
- k8s集群
- KubeSphere DevOps
使用Docker部署服务测试
使用k8s部署，先用docker部署一次，测试镜像和服务访问是否有问题。
本次搭建的服务镜像Dockerfile文件配置，参考若依微服务系统代码。
创建DevOps工程
maven镜像地址配置
登录kubesphere管理员权限账号
在这里搜索devops
修改配置
增加maven国内镜像仓库地址配置
<mirrors>
<mirror>
<id>alimavenid>
<name>aliyun mavenname>
<url>http://maven.aliyun.com/nexus/content/groups/public/url>
<mirrorOf>centralmirrorOf>
mirror>
mirrors>
访问凭证配置
- gitee代码凭证
- 阿里云镜像私有仓库访问凭证
- k8s访问凭证
创建流水线
下图为最终的4个流水线；
先部署成功一个，后面的服务直接复制流水线，通过修改Jenkinsfile，即可快速部署服务；
初学参考官方提供的流水线模板
编辑流水线Jenkinsfile
SpringBoot服务
后端kw-gateway
pipeline {
agent {
node {
label 'maven'
}
}
stages {
stage('拉取代码') {
agent none
steps {
container('maven') {
git(url: 'https://gitee.com/kkmy/kw-microservices.git', credentialsId: 'kw-microservices-gitee', changelog: true, poll: false, branch: 'master')
sh 'ls -l'
}
}
}
stage('项目编译打包jar') {
agent none
steps {
container('maven') {
sh 'ls -l'
sh 'mvn clean package \'-Dmaven.test.skip=true\' -Pk8s'
}
}
}
stage('构建镜像') {
agent none
steps {
container('maven') {
sh '''ls kw-gateway/target -l
cat docker/micro-server/gateway/Dockerfile'''
sh 'docker build -f docker/micro-server/gateway/Dockerfile -t kw-gateway:latest ./kw-gateway/'
}
}
}
stage('推送镜像') {
agent none
steps {
container('maven') {
withCredentials([usernamePassword(credentialsId : 'aliyun-docker-registry' ,passwordVariable : 'DOCKER_ALIYUN_PWD' ,usernameVariable : 'DOCKER_ALIYUN_USER' ,)]) {
sh 'echo "$DOCKER_ALIYUN_PWD" | docker login $REGISTRY -u "$DOCKER_ALIYUN_USER" --password-stdin'
sh 'docker tag kw-gateway:latest $REGISTRY/$DOCKERHUB_NAMESPACE/kw-gateway:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
sh 'docker push $REGISTRY/$DOCKERHUB_NAMESPACE/kw-gateway:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
}
}
}
}
stage('部署dev环境') {
agent none
steps {
container ('maven') {
withCredentials([
kubeconfigFile(
credentialsId: env.KUBECONFIG_CREDENTIAL_ID,
variable: 'KUBECONFIG')
]) {
sh 'envsubst < deploy/gateway/deploy.yaml | kubectl apply -f -'
}
}
}
}
}
environment {
DOCKER_CREDENTIAL_ID = 'dockerhub-id'
GITHUB_CREDENTIAL_ID = 'github-id'
KUBECONFIG_CREDENTIAL_ID = 'kubeconfig'
REGISTRY = 'registry.cn-hangzhou.aliyuncs.com'
DOCKERHUB_NAMESPACE = 'kk_hub'
GITHUB_ACCOUNT = 'kubesphere'
APP_NAME = 'kw-gateway'
}
parameters {
string(name: 'TAG_NAME', defaultValue: '', description: '')
}
}
后端kw-crm-server
pipeline {
agent {
node {
label 'maven'
}
}
stages {
stage('拉取代码') {
agent none
steps {
container('maven') {
git(url: 'https://gitee.com/kkmy/kw-microservices.git', credentialsId: 'kw-microservices-gitee', changelog: true, poll: false, branch: 'master')
sh 'ls -l'
}
}
}
stage('项目编译打包jar') {
agent none
steps {
container('maven') {
sh 'ls -l'
sh 'mvn clean package \'-Dmaven.test.skip=true\' -Pk8s'
}
}
}
stage('构建镜像') {
agent none
steps {
container('maven') {
sh '''ls kw-modules/kw-crm-service/target -l
cat docker/micro-server/crm-server/Dockerfile'''
sh 'docker build -f docker/micro-server/crm-server/Dockerfile -t kw-crm-server:latest ./kw-modules/kw-crm-service/'
}
}
}
stage('推送镜像') {
agent none
steps {
container('maven') {
withCredentials([usernamePassword(credentialsId : 'aliyun-docker-registry' ,passwordVariable : 'DOCKER_ALIYUN_PWD' ,usernameVariable : 'DOCKER_ALIYUN_USER' ,)]) {
sh 'echo "$DOCKER_ALIYUN_PWD" | docker login $REGISTRY -u "$DOCKER_ALIYUN_USER" --password-stdin'
sh 'docker tag kw-crm-server:latest $REGISTRY/$DOCKERHUB_NAMESPACE/kw-crm-server:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
sh 'docker push $REGISTRY/$DOCKERHUB_NAMESPACE/kw-crm-server:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
}
}
}
}
stage('部署dev环境') {
agent none
steps {
container('maven') {
withCredentials([
kubeconfigFile(
credentialsId: env.KUBECONFIG_CREDENTIAL_ID,
variable: 'KUBECONFIG')
]) {
sh 'envsubst < deploy/crm-server/deploy.yaml | kubectl apply -f -'
}
}
}
}
}
environment {
DOCKER_CREDENTIAL_ID = 'dockerhub-id'
GITHUB_CREDENTIAL_ID = 'github-id'
KUBECONFIG_CREDENTIAL_ID = 'kubeconfig'
REGISTRY = 'registry.cn-hangzhou.aliyuncs.com'
DOCKERHUB_NAMESPACE = 'kk_hub'
GITHUB_ACCOUNT = 'kubesphere'
APP_NAME = 'kw-crm-server'
}
parameters {
string(name: 'TAG_NAME', defaultValue: '', description: '')
}
}
后端kw-auth-server
pipeline {
agent {
node {
label 'maven'
}
}
stages {
stage('拉取代码') {
agent none
steps {
container('maven') {
git(url: 'https://gitee.com/kkmy/kw-microservices.git', credentialsId: 'kw-microservices-gitee', changelog: true, poll: false, branch: 'master')
sh 'ls -l'
}
}
}
stage('项目编译打包jar') {
agent none
steps {
container('maven') {
sh 'ls -l'
sh 'mvn clean package \'-Dmaven.test.skip=true\' -Pk8s'
}
}
}
stage('构建镜像') {
agent none
steps {
container('maven') {
sh '''ls kw-auth/kw-auth-server/target -l
cat docker/micro-server/auth-server/Dockerfile'''
sh 'docker build -f docker/micro-server/auth-server/Dockerfile -t kw-auth-server:latest ./kw-auth/kw-auth-server/'
}
}
}
stage('推送镜像') {
agent none
steps {
container('maven') {
withCredentials([usernamePassword(credentialsId : 'aliyun-docker-registry' ,passwordVariable : 'DOCKER_ALIYUN_PWD' ,usernameVariable : 'DOCKER_ALIYUN_USER' ,)]) {
sh 'echo "$DOCKER_ALIYUN_PWD" | docker login $REGISTRY -u "$DOCKER_ALIYUN_USER" --password-stdin'
sh 'docker tag kw-auth-server:latest $REGISTRY/$DOCKERHUB_NAMESPACE/kw-auth-server:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
sh 'docker push $REGISTRY/$DOCKERHUB_NAMESPACE/kw-auth-server:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
}
}
}
}
stage('部署dev环境') {
agent none
steps {
container ('maven') {
withCredentials([
kubeconfigFile(
credentialsId: env.KUBECONFIG_CREDENTIAL_ID,
variable: 'KUBECONFIG')
]) {
sh 'envsubst < deploy/auth-server/deploy.yaml | kubectl apply -f -'
}
}
}
}
}
environment {
DOCKER_CREDENTIAL_ID = 'dockerhub-id'
GITHUB_CREDENTIAL_ID = 'github-id'
KUBECONFIG_CREDENTIAL_ID = 'kubeconfig'
REGISTRY = 'registry.cn-hangzhou.aliyuncs.com'
DOCKERHUB_NAMESPACE = 'kk_hub'
GITHUB_ACCOUNT = 'kubesphere'
APP_NAME = 'kw-auth-server'
}
parameters {
string(name: 'TAG_NAME', defaultValue: '', description: '')
}
}
Nodejs服务
前端kwsphere
pipeline {
agent {
node {
label 'nodejs'
}
}
stages {
stage('拉取代码') {
agent none
steps {
container('nodejs') {
git(url: 'https://gitee.com/kkmy/kw-microservices.git', credentialsId: 'kw-microservices-gitee', changelog: true, poll: false, branch: 'master')
sh 'ls kw-ui/kwsphere -l'
}
}
}
stage('项目编译dist') {
agent none
steps {
container('nodejs') {
sh 'npm config set user 0'
sh 'npm config set unsafe-perm true'
sh 'npm uninstall node-sass'
sh 'npm i [email protected] --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ --unsafe-perm'
sh 'npm install --prefix ./kw-ui/kwsphere --registry=https://registry.npm.taobao.org'
sh 'npm --prefix ./kw-ui/kwsphere run build:prod'
sh 'ls -l'
}
}
}
stage('构建镜像') {
agent none
steps {
container('nodejs') {
sh 'cat docker/micro-server/ui/Dockerfile'
sh 'docker build -f docker/micro-server/ui/Dockerfile -t kwsphere:latest ./kw-ui/kwsphere/'
}
}
}
stage('推送镜像') {
agent none
steps {
container('nodejs') {
withCredentials([usernamePassword(credentialsId : 'aliyun-docker-registry' ,passwordVariable : 'DOCKER_ALIYUN_PWD' ,usernameVariable : 'DOCKER_ALIYUN_USER' ,)]) {
sh 'echo "$DOCKER_ALIYUN_PWD" | docker login $REGISTRY -u "$DOCKER_ALIYUN_USER" --password-stdin'
sh 'docker tag kwsphere:latest $REGISTRY/$DOCKERHUB_NAMESPACE/kwsphere:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
sh 'docker push $REGISTRY/$DOCKERHUB_NAMESPACE/kwsphere:SNAPSHOT-$BRANCH_NAME-$BUILD_NUMBER'
}
}
}
}
stage('部署dev环境') {
agent none
steps {
container('nodejs') {
withCredentials([
kubeconfigFile(
credentialsId: env.KUBECONFIG_CREDENTIAL_ID,
variable: 'KUBECONFIG')
]) {
sh 'envsubst < deploy/kwsphere/deploy.yaml | kubectl apply -f -'
}
}
}
}
}
environment {
DOCKER_CREDENTIAL_ID = 'dockerhub-id'
GITHUB_CREDENTIAL_ID = 'github-id'
KUBECONFIG_CREDENTIAL_ID = 'kubeconfig'
REGISTRY = 'registry.cn-hangzhou.aliyuncs.com'
DOCKERHUB_NAMESPACE = 'kk_hub'
GITHUB_ACCOUNT = 'kubesphere'
APP_NAME = 'kwsphere'
}
parameters {
string(name: 'TAG_NAME', defaultValue: '', description: '')
}
}
问题记录
Kubernetes Deploy插件问题
尚硅谷教程中的k8s deploy插件运行没问题，但在目前搭建过程中不好了，会报错；
#没有找到
Starting Kubernetes deployment
Loading configuration: /home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere-ui/deploy/gateway/deploy.yml
ERROR: ERROR: java.lang.RuntimeException: io.kubernetes.client.openapi.ApiException: Bad Request
"message": "the export parameter, deprecated since v1.14, is no longer supported",
问了kubespere群中的大佬，说是该插件已废弃，官网有新示例。
就是不能再用这个插件了
改用shell命令运行
sh 'envsubst < deploy/kwsphere/deploy.yaml | kubectl apply -f -'
service指定nodeport报错
#k8s指定端口报错
The Service "kwsphere" is invalid: spec.ports[0].nodePort: Invalid value: 43113: provided port is not in the valid range. The range of valid ports is 30000-32767
前端服务build报错
#npm install报错
#解决，install之前，运行这两个命令
npm config set user 0
npm config set unsafe-perm true
Unable to save binary /home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/node-sass/vendor/linux-x64-64 : { Error: EACCES: permission denied, mkdir '/home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/node-sass/vendor'
at Object.mkdirSync (fs.js:757:3)
at sync (/home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/mkdirp/index.js:74:13)
at Function.sync (/home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/mkdirp/index.js:80:24)
at checkAndDownloadBinary (/home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/node-sass/scripts/install.js:114:11)
at Object.<anonymous> (/home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/node-sass/scripts/install.js:157:1)
at Module._compile (internal/modules/cjs/loader.js:778:30)
at Object.Module._extensions..js (internal/modules/cjs/loader.js:789:10)
at Module.load (internal/modules/cjs/loader.js:653:32)
at tryModuleLoad (internal/modules/cjs/loader.js:593:12)
at Function.Module._load (internal/modules/cjs/loader.js:585:3)
errno: -13,
syscall: 'mkdir',
code: 'EACCES',
path:
'/home/jenkins/agent/workspace/kw-devopsw2gh4/kubesphere/kw-ui/kwsphere/node_modules/node-sass/vendor' }
#又报错，解决：流水线 步骤中添加执行如下命令
#npm uninstall node-sass
#npm i [email protected] --sass_binary_site=https://npm.taobao.org/mirrors/node-sass/ --unsafe-perm
npm ERR! code ELIFECYCLE
npm ERR! errno 1
npm ERR! [email protected] postinstall: `node scripts/build.js`
npm ERR! Exit status 1
npm ERR!
npm ERR! Failed at the [email protected] postinstall script.
npm ERR! This is probably not a problem with npm. There is likely additional logging output above.
npm ERR! A complete log of this run can be found in:
npm ERR! /root/.npm/_logs/2023-08-27T08_57_22_270Z-debug.log
script returned exit code 1
代码多目录build node.js服务问题
我的前后端代码在一个工程中，导致在运行npm命令，找不到package.json文件
尝试在构建过程中，先执行cd命令，进入到前端代码的目录下，但是不好使，
最后找到npm命令如何指定package.json文件路径解决
#使用--prefix参数
npm install --prefix ./kw-ui/kwsphere --registry=https://registry.npm.taobao.org
npm --prefix ./kw-ui/kwsphere run build:prod
结语
以上仅仅是入门学习使用，devops流水线，还有很多地方可以简化配置