容器化微服务项目
熟悉Spring Cloud微服务项目
1639991817856.png
源代码编译构建
#安装jdk、maven
[root@prometheus simple-microservice-dev3]# yum install java-1.8.0-openjdk maven
#修改product、stock、order连接数据库地址
url: jdbc:mysql://192.168.153.27:3306/tb_order?characterEncoding=utf-8
url: jdbc:mysql://192.168.153.27:3306/tb_product?characterEncoding=utf-8
url: jdbc:mysql://192.168.153.27:3306/tb_stock?characterEncoding=utf-8
#编译构建
[root@prometheus simple-microservice-dev3]# mvn clean package -Dmaven.test.skip=true
构建项目镜像并推送到镜像仓库
登录harbor
[root@prometheus harbor]# docker login 192.168.153.20
eureka
[root@prometheus eureka-service]# docker build -t 192.168.153.20/ms/eureka:v1 .
[root@prometheus eureka-service]# docker push 192.168.153.20/ms/eureka:v1
K8s中部署Eureka集群
安装Ingress
[root@k8s-m1 k8s]# kubectl apply -f ingress-controller.yaml
[root@k8s-m1 k8s]# kubectl get pods -n ingress-nginx -o wide
NAME READY STATUS IP NODE
nginx-ingress-controller-5dc64b58f-stb5j 1/1 Running 192.168.153.25 k8s-m1
创建registry-pull-secret
[root@k8s-m1 k8s]# kubectl create secret docker-registry registry-pull-secret --docker-username=admin --docker-password=Harbor12345 --docker-server=192.168.153.20
secret/registry-pull-secret created
部署eureka集群
[root@k8s-m1 k8s]# kubectl apply -f eureka.yaml
[root@k8s-m1 k8s]# kubectl get pod,svc -n ms
NAME READY STATUS RESTARTS AGE
pod/eureka-0 1/1 Running 0 6m55s
pod/eureka-1 1/1 Running 0 2m43s
pod/eureka-2 1/1 Running 0 95s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/eureka ClusterIP None 8888/TCP 6m55s
---------------------------------------------------------------------------------
hosts:
192.168.153.27 eureka.ctnrs.com
#访问：
http://eureka.ctnrs.com/
K8s中部署MySQL
部署mysql
#拉起MySQL镜像(:5.7 表示5.7版本)
docker pull mysql:5.7
#运行MySQL容器
docker run -d -p 3306:3306 --privileged=true -v /docker/mysql/conf/my.cnf:/etc/my.cnf -v /docker/mysql/data:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=123456 --name mysql mysql:5.7 --character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci
#参数说明:
run run 是运行一个容器
-d 表示后台运行
-p 表示容器内部端口和服务器端口映射关联
--privileged=true 设值MySQL 的root用户权限, 否则外部不能使用root用户登陆
-v /docker/mysql/conf/my.cnf:/etc/my.cnf 将服务器中的my.cnf配置映射到docker中的/docker/mysql/conf/my.cnf配置
-v /docker/mysql/data:/var/lib/mysql 同上,映射数据库的数据目录, 避免以后docker删除重新运行MySQL容器时数据丢失
-e MYSQL_ROOT_PASSWORD=123456 设置MySQL数据库root用户的密码
--name mysql 设值容器名称为mysql
mysql:5.7 表示从docker镜像mysql:5.7中启动一个容器
--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci 设值数据库默认编码
#赋予远程登录权限
[root@xdclass ~]# docker exec -it mysql bash
root@ce7e026432b3:/# mysql -u root -p
mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> FLUSH PRIVILEGES;
#导入数据
order.sql stock.sql product.sql
在K8s中部署微服务
product
#构建镜像
[root@prometheus product-service-biz]# docker build -t 192.168.153.20/ms/product:v1 .
#推送镜像
[root@prometheus product-service-biz]# docker push 192.168.153.20/ms/product:v1
#k8s集群中发布
[root@k8s-m1 k8s]# kubectl apply -f product.yaml
order
#构建镜像
[root@prometheus order-service-biz]# docker build -t 192.168.153.20/ms/order:v1 .
#推送镜像
[root@prometheus order-service-biz]# docker push 192.168.153.20/ms/order:v1
#k8s集群中发布
[root@k8s-m1 k8s]# kubectl apply -f order.yaml
stock
#构建镜像
[root@prometheus stock-service-biz]# docker build -t 192.168.153.20/ms/stock:v1 .
#推送镜像
[root@prometheus stock-service-biz]# docker push 192.168.153.20/ms/stock:v1
#k8s集群中发布
[root@k8s-m1 k8s]# kubectl apply -f stock.yaml
gateway
#构建镜像
[root@prometheus gateway-service]# docker build -t 192.168.153.20/ms/gateway:v1 .
#推送镜像
[root@prometheus gateway-service]# docker push 192.168.153.20/ms/gateway:v1
#k8s集群中发布
[root@k8s-m1 k8s]# kubectl apply -f gateway.yaml
#hosts
192.168.153.27 gateway.ctnrs.com
portal
#构建镜像
[root@prometheus portal-service]# docker build -t 192.168.153.20/ms/portal:v1 .
#推送镜像
[root@prometheus portal-service]# docker push 192.168.153.20/ms/portal:v1
#k8s集群中发布
portal.ctnrs.com
查看相关服务
[root@k8s-m1 k8s]# kubectl get pod,svc,ing -n ms
NAME READY STATUS RESTARTS AGE
pod/eureka-0 1/1 Running 2 86m
pod/eureka-1 1/1 Running 2 84m
pod/eureka-2 1/1 Running 1 83m
pod/gateway-6c7b6f7c85-g9srj 1/1 Running 1 70m
pod/order-65b848c67c-r7stp 1/1 Running 0 6m58s
pod/portal-78ccc5768c-wvt5f 1/1 Running 1 70m
pod/product-59c88fbf7f-snrkf 1/1 Running 0 7m4s
pod/stock-c9b89d8b-p4wvd 1/1 Running 0 6m51s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/eureka ClusterIP None 8888/TCP 86m
service/gateway ClusterIP 10.0.0.101 9999/TCP 70m
service/portal ClusterIP 10.0.0.44 8080/TCP 70m
NAME CLASS HOSTS ADDRESS PORTS AGE
ingress.networking.k8s.io/eureka eureka.ctnrs.com 80 86m
ingress.networking.k8s.io/gateway gateway.ctnrs.com 80 70m
ingress.networking.k8s.io/portal portal.ctnrs.com 80 70m
1640014289523.png
1640058334935.png
1640058349907.png
http://gateway.ctnrs.com/product/queryAllProduct?page=1&limit=10
{"status":200,"msg":"success","result":[{"id":1,"productName":"测试商品1","price":99.99,"stock":99},{"id":2,"productName":"美女","price":999.0,"stock":87},{"id":3,"productName":"Q币","price":100.0,"stock":77},{"id":4,"productName":"貂皮大衣很厚很厚的那种","price":9999.0,"stock":66}]}
http://gateway.ctnrs.com/order/queryAllOrder
{"status":200,"msg":"success","result":[{"id":1,"orderNumber":"0j889r86wo0tng9x","orderProductName":"美女","orderPrice":999.0,"count":1,"buyDate":"2021-12-21T03:40:32.000+0000"}]}
Skywalking
介绍
• 多种监控手段。可以通过语言探针和 service mesh 获得监控是数据。
• 多个语言自动探针。包括 Java，.NET Core 和 Node.JS。
• 轻量高效。无需大数据平台，和大量的服务器资源。
• 模块化。UI、存储、集群管理都有多种机制可选。
• 支持告警。
• 优秀的可视化解决方案
架构
1640089757346.png
部署
部署ES数据库
docker run --name elasticsearch -p 9200:9200 -e "discovery.type=single-node" -d elasticsearch:7.7.0
部署Skywalking OAP
[root@k8s-m1 ~]# yum install java-11-openjdk –y
[root@k8s-m1 ~]# tar zxvf apache-skywalking-apm-es7-8.3.0.tar.gz
[root@k8s-m1 ~]# cd apache-skywalking-apm-bin-es7/
[root@k8s-m1 ~]# vi config/application.yml
storage:
selector: ${SW_STORAGE:elasticsearch7} #这里使用elasticsearch7
...
elasticsearch7:
nameSpace: ${SW_NAMESPACE:""}
clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:192.168.0.10:9200} # 指定ES地址
#启动OAP和UI：
[root@k8s-m1 bin]# ./startup.sh
SkyWalking OAP started successfully!
SkyWalking Web Application started successfully!
#访问UI：
http://192.168.153.25:8080
#collector.backend_service为Skywalking服务器，11800端口复制收集数据
[root@k8s-m1 agent]# ss -antp|grep 11800
LISTEN 0 128 :::11800 :::* users:(("java",pid=59156,fd=269))
1640092815351.png
Dockerfile
#启动Java程序以探针方式集成Agent(以eureka为例)，每个服务都要加，重新构建：
java -jar -javaagent:/skywalking/skywalking-agent.jar=agent.service_name=ms-eureka,agent.instance_name=$(echo $HOSTNAME | awk -F- '{print $1"-"$NF}'),
collector.backend_service=192.168.153.25:11800 -Deureka.instance.hostname=${MY_POD_NAME}.eureka.ms /eureka-service.jar
构建发布
#启动mysql服务
docker start mysql
#启动es
docker start elasticsearch
#启动Skywalking OAP和UI：
[root@k8s-m1 bin]# ./startup.sh
SkyWalking OAP started successfully!
SkyWalking Web Application started successfully!
#访问UI：
http://192.168.153.25:8080
#collector.backend_service为Skywalking服务器，11800端口复制收集数据
[root@k8s-m1 agent]# ss -antp|grep 11800
LISTEN 0 128 :::11800 :::* users:(("java",pid=59156,fd=269))
#修改Dockerfile重新构建推送
docker build -t 192.168.153.20/ms/eureka:v2 .
docker push 192.168.153.20/ms/eureka:v2
......
#k8s发布过程中出现oomkill问题
解决：将limit设置的更大
[root@k8s-m1 k8s]# kubectl get pod,svc,ing -n ms
NAME READY STATUS RESTARTS AGE
pod/eureka-0 1/1 Running 0 62m
pod/eureka-1 1/1 Running 0 61m
pod/eureka-2 1/1 Running 0 60m
pod/gateway-77776889-r29dt 1/1 Running 0 34m
pod/order-846f7c95b9-dpqh8 1/1 Running 0 30m
pod/portal-66cf475fc4-9ww57 1/1 Running 1 49m
pod/product-554d7d554c-6g87b 1/1 Running 0 30m
pod/stock-546b455df8-nblxn 1/1 Running 0 30m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/eureka ClusterIP None 8888/TCP 62m
service/gateway ClusterIP 10.0.0.173 9999/TCP 34m
service/portal ClusterIP 10.0.0.94 8080/TCP 49m
NAME CLASS HOSTS ADDRESS PORTS AGE
ingress.networking.k8s.io/eureka eureka.ctnrs.com 80 62m
ingress.networking.k8s.io/gateway gateway.ctnrs.com 80 34m
ingress.networking.k8s.io/portal portal.ctnrs.com 80 49m
效果展示
1640175090524.png
1640176967598.png
生产环境踩坑经验分享
限制了容器资源，还经常被杀死？
在JAVA1.9版本之前，是不能自动发现docker设置的内存限制，随着应用负载起伏就会造成内存使用过大，超过limits限制，从而触发K8s杀掉该容器。
解决办法：
• 手动指定JVM堆内存大小
CMD java -jar $JAVA_OPTS /gateway-service.jar
env:
- name: JAVA_OPTS
value: "-Xmx1g"
resources:
requests:
cpu: 0.5
memory: 256Mi
limits:
cpu: 1
memory: 1Gi
滚动更新期间造成流量丢失
滚动更新触发，Pod在删除过程中，有些节点kube-proxy还没来得及同步iptables规则，从而部分流量请求到Terminating的Pod上，导致请求出错。
解决办法：配置preStop回调，在容器终止前优雅暂停5秒，给kube-proxy多预留一点时间
lifecycle:
preStop:
exec:
command:
- sh
- -c
- "sleep 5"
还可以做一些回调处理，curl......
滚动更新之健康检查重要性
滚动更新是默认发布策略，当配置健康检查时，滚动更新会根据Probe状态来决定是否继续更新以及是否允许接入流量，这样在整个滚动更新过程中可保证始终会有可用的Pod存在，达到平滑升级
readinessProbe:
tcpSocket:
port: 9999
initialDelaySeconds: 60
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 9999
initialDelaySeconds: 60
periodSeconds: 10