k8s运行java项目，使用harbor作为私有镜像仓库
目录
- 一、环境说明
- 二、环境检查
- 三、获取基础服务镜像、构建项目镜像
- 四、使用k8s集群部署java项目
- 五、提供外部访问
- 1、使用service
- 2、使用ingress提供外部访问
- 六、验证
一、环境说明
操作系统：centos7
kubernetes：16.0
docker：18.06
| 主机名 | IP地址 | 类型 |
|---|---|---|
| [k8s-master | 192.168.1.191 | masters |
| [k8s-node01 | 192.168.1.192 | nodes |
| [harbor | 192.168.1.193 | harbor |
二、环境检查
[root@k8s-master ~]# cat /etc/docker/daemon.json
{
"registry-mirrors": ["https://eyg9yi6d.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"insecure-registries":[ "192.168.1.193" ],
"storage-driver": "overlay2"
}
[root@k8s-node01 ~]# cat /etc/docker/daemon.json
{
"registry-mirrors": ["https://eyg9yi6d.mirror.aliyuncs.com"],
"insecure-registries":[ "192.168.1.193" ],
"exec-opts": ["native.cgroupdriver=systemd"]
}
三、获取基础服务镜像、构建项目镜像
## 下载基础镜像
[root@harbor ~]# docker pull tomcat
[root@harbor ~]# docker images | grep tomcat
tomcat latest 4e7840b49fad 2 weeks ago 529MB
# 打标签，推送基础镜像到镜像仓库
[root@harbor ~]# docker tag tomcat 192.168.1.193/tomcat/tomcat:v1
#登录到仓库地址
[root@harbor ~]# docker login --username=admin --password=123456 192.168.1.193
WARNING! Using --password via the CLI is insecure. Use --password-stdin.
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@harbor ~]# docker push 192.168.1.193/tomcat/tomcat:v1
## 编译项目
[root@harbor java-demo]# pwd
/data/java/java-demo
[root@harbor java-demo]# ll
总用量 40
drwxr-xr-x. 2 root root 4096 5月 21 2019 db
-rw-r--r--. 1 root root 128 3月 12 11:01 Dockerfile
drwxr-xr-x. 2 root root 4096 11月 30 11:58 k8s-yaml
-rw-r--r--. 1 root root 11357 5月 21 2019 LICENSE
-rw-r--r--. 1 root root 1930 5月 21 2019 pom.xml
-rw-r--r--. 1 root root 88 5月 21 2019 README.md
drwxr-xr-x. 3 root root 4096 5月 27 2019 src
drwxr-xr-x. 7 root root 4096 3月 12 18:14 target
[root@harbor java-demo]# mvn clean package -D maven.test.skip=true
## 构建项目镜像
[root@harbor java-demo]# cat Dockerfile
FROM 192.168.1.193/tomcat/tomcat:v1
RUN rm -rf /usr/local/tomcat/webapps/*
COPY target/*.war /usr/local/tomcat/webapps/ROOT.war
[root@harbor java-demo]# docker build -t java-demo .
[root@harbor java-demo]# docker tag java-demo:latest 192.168.1.193/java/java-demo:v1
[root@harbor java-demo]# docker push 192.168.1.193/java/java-demo:v1
四、使用k8s集群部署java项目
[root@k8s-master ~]# kubectl create namespace java-dev
namespace/java-dev created
[root@k8s-master ~]# kubectl create secret docker-registry harbor-secret --docker-username=admin --docker-password=123456 --docker-server=192.168.1.193 -n java-dev -o yaml --dry-run > /data/java/harbor-secret.yaml
## 修改/data/java/java-demo.yaml文件，添加imagePullSecrets字段
[root@k8s-master ~]# cat /data/java/java-demo.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: java-web
name: java-web
namespace: java-dev
spec:
replicas: 1
selector:
matchLabels:
app: java-web
template:
metadata:
labels:
app: java-web
spec:
imagePullSecrets:
- name: "harbor-secret"
containers:
- image: 192.168.1.193/java/java-demo:v1
name: java-demo
resources: {}
[root@k8s-master ~]# kubectl apply -f /data/java/harbor-secret.yaml
secret/harbor-secret created
[root@k8s-master ~]# kubectl apply -f /data/java/java-demo.yaml
deployment.apps/java-web created
## 检查是否创建成功
[root@k8s-master ~]# kubectl get pod -n java-dev
NAME READY STATUS RESTARTS AGE
java-web-7f67c7fd65-qg57r 1/1 Running 0 50s
[root@k8s-master ~]# kubectl get secret -n java-dev
NAME TYPE DATA AGE
default-token-sjl5f kubernetes.io/service-account-token 3 9m2s
harbor-secret kubernetes.io/dockerconfigjson 1 64s
五、提供外部访问
1、使用service
## 创建service
[root@k8s-master ~]# kubectl expose deployment java-web --port=80 --target-port=8080 --type=NodePort -o yaml --dry-run > /data/java/java-service.yaml
[root@k8s-master ~]# cat /data/java/java-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: java-web
name: java-web
namespace: java-dev
spec:
ports:
- port: 80
protocol: TCP
targetPort: 8080
nodePort: 30018 #自定义nodeport的端口
selector:
app: java-web
type: NodePort
[root@k8s-master ~]# kubectl apply -f /data/java/java-service.yaml
[root@k8s-master ~]# kubectl get pod,svc -n java-dev
NAME READY STATUS RESTARTS AGE
pod/java-web-7f67c7fd65-qg57r 1/1 Running 0 23m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/java-web NodePort 10.1.219.227 <none> 80:30018/TCP 8m40s
## 集群内部访问
[root@k8s-master ~]# curl 10.1.219.227:30018
<!DOCTYPE html>
<html>
<head lang="en">
…………
</body>
</html>
## 集群外部访问：
[root@k8s-master ~]# curl 192.168.1.192:30018
<!DOCTYPE html>
<html>
<head lang="en">
…………
</body>
</html>
2、使用ingress提供外部访问
#继续上面的service操作，访问方式为service+ingress
[root@k8s-master ~]# cd /data/java/
[root@k8s-master java]# wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/mandatory.yaml
## 修改镜像地址为阿里云，容器的网络为hostNetwork=true模式，删掉 replicas: 这一行，apps/v1的kind改为：DaemonSet
[root@k8s-master java]# cat mandatory.yaml
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "-"
# Here: "-"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
hostNetwork: true
serviceAccountName: nginx-ingress-serviceaccount
nodeSelector:
kubernetes.io/os: linux
containers:
- name: nginx-ingress-controller
image: registry.aliyuncs.com/google_containers/nginx-ingress-controller:0.30.0
args:
- /nginx-ingress-controller
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
allowPrivilegeEscalation: true
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 101
runAsUser: 101
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
protocol: TCP
- name: https
containerPort: 443
protocol: TCP
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
---
apiVersion: v1
kind: LimitRange
metadata:
name: ingress-nginx
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
limits:
- min:
memory: 90Mi
cpu: 100m
type: Container
[root@k8s-master ~]# kubectl apply -f mandatory.yaml
[root@k8s-master java]# cat java-service1.yaml
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
app: java-web
name: java-web
namespace: java-dev
spec:
ports:
- port: 80
protocol: TCP
targetPort: 8080
nodePort: 30018
selector:
app: java-web
type: NodePort
六、验证
在主机添加hosts文件，路径：C:\Windows\System32\drivers\etc\host
192.168.1.192 java.example.com
