kubernetes部署-ingress(十一)
部署ingress
获取ingress的yaml文件
https://github.com/kubernetes/ingress-nginx/tree/nginx-0.20.0/deploy
也可以wget直接下载下来
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml
mandatory.yaml 需要修改images的镜像地址下面的我已经都改好了可以直接用
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: default-http-backend
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
namespace: ingress-nginx
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
spec:
terminationGracePeriodSeconds: 60
containers:
- name: default-http-backend
# Any image is permissible as long as:
# 1. It serves a 404 page at /
# 2. It serves 200 on a /healthz endpoint
image: googlecontainer/defaultbackend-amd64:1.5
livenessProbe:
httpGet:
path: /healthz
port: 8080
scheme: HTTP
initialDelaySeconds: 30
timeoutSeconds: 5
ports:
- containerPort: 8080
resources:
limits:
cpu: 10m
memory: 20Mi
requests:
cpu: 10m
memory: 20Mi
---
apiVersion: v1
kind: Service
metadata:
name: default-http-backend
namespace: ingress-nginx
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
spec:
ports:
- port: 80
targetPort: 8080
selector:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "-"
# Here: "-"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
serviceAccountName: nginx-ingress-serviceaccount
containers:
- name: nginx-ingress-controller
image: siriuszg/nginx-ingress-controller:0.20.0
args:
- /nginx-ingress-controller
- --default-backend-service=$(POD_NAMESPACE)/default-http-backend
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
---
查看pod是否已经起来
kubectl get pod -n ingress-nginx
NAME READY STATUS RESTARTS AGE
default-http-backend-cbd6d5994-hzj9l 1/1 Running 0 15h
这里其实是有问题的从yaml里可以看出会有两个pod
gat deploy查看一下
kubectl get deploy -n ingress-nginx
NAME READY UP-TO-DATE AVAILABLE AGE
default-http-backend 1/1 1 1 15h
nginx-ingress-controller 0/2 0 0 15h
nginx-ingress-controller这个并没有正确的交给pod来运行
通过以上流程图可以看出deploy把任务交给rs，rs在分别启动pod现在都看不到pod问题应该在rs上
get rs
kubectl get rs -n ingress-nginx
NAME DESIRED CURRENT READY AGE
default-http-backend-cbd6d5994 1 1 1 15h
nginx-ingress-controller-57c8dd86d4 2 0 0 15h
状态不正确
通过describe查看一下详情
kubectl describe rs/nginx-ingress-controller-57c8dd86d4 -n ingress-nginx
Warning FailedCreate 49m replicaset-controller Error creating: pods "nginx-ingress-controller-57c8dd86d4-97gl6" is forbidden: Secur
ityContext.RunAsUser is forbidden
通过排查判断是apiserver的参数问题
检查apiserver配置文件： 去除：–enable-admission-plugins中的SecurityContextDeny部分，分别重启apiserver
get一下pod看起来了没
kubectl get pod -n ingress-nginx -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS
GATESdefault-http-backend-cbd6d5994-9bkh8 1/1 Running 0 99s 172.17.82.9 10.167.130.210 <none> <none>
nginx-ingress-controller-57c8dd86d4-p8dz9 1/1 Running 0 97s 172.17.71.8 10.167.130.206 <none> <none>
通过curl来测试是否正常
测试默认返回404页面的主机http-backend
curl http://172.17.82.9
default backend - 404
测试直接访问nginx代理看是否会转发到404
curl http://172.17.71.8
default backend - 404
更改部署方式
此刻问题来了通过yaml创建的deploy以及server来看好像并没有把nginx端口映射到宿主机上,那么我访问宿主机ip就不会有任何返回,这里可以通过hostport+DaemonSet来解决这个问题
修改yaml文件
1.修改nginx 部署方式为DaemonSet
2.注释replicas: 1
3.增加 hostNetwork: true 在spec: 段内增加
4.增加hostPort 在Ports段内增加
kubectl delete -f ingress.yaml #删除重新创建
kubectl apply -f ingress.yaml
附上修改后的yaml文件内容
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: default-http-backend
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
namespace: ingress-nginx
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
spec:
terminationGracePeriodSeconds: 60
containers:
- name: default-http-backend
# Any image is permissible as long as:
# 1. It serves a 404 page at /
# 2. It serves 200 on a /healthz endpoint
image: googlecontainer/defaultbackend-amd64:1.5
livenessProbe:
httpGet:
path: /healthz
port: 8080
scheme: HTTP
initialDelaySeconds: 30
timeoutSeconds: 5
ports:
- containerPort: 8080
resources:
limits:
cpu: 10m
memory: 20Mi
requests:
cpu: 10m
memory: 20Mi
---
apiVersion: v1
kind: Service
metadata:
name: default-http-backend
namespace: ingress-nginx
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
spec:
ports:
- port: 80
targetPort: 8080
selector:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "-"
# Here: "-"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
# replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
serviceAccountName: nginx-ingress-serviceaccount
hostNetwork: true
containers:
- name: nginx-ingress-controller
image: siriuszg/nginx-ingress-controller:0.20.0
args:
- /nginx-ingress-controller
- --default-backend-service=$(POD_NAMESPACE)/default-http-backend
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
hostPort: 80
- name: https
containerPort: 443
hostPort: 443
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
---
ingress部署参考文档
kubernetes ingress nginx
创建一个tomcat并用ingress7层代理转发
创建tomcat服务以及deploy
cat tomcat-ingress.yaml
apiVersion: v1
kind: Service
metadata:
name: tomcat
namespace: default
spec:
type: ClusterIP
selector:
app: tomcat
release: canary
ports:
- name: http
port: 8080
targetPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tomcat-deploy
spec:
replicas: 1
selector:
matchLabels:
app: tomcat
release: canary
template:
metadata:
labels:
app: tomcat
release: canary
spec:
containers:
- name: tomcat
image: tomcat:7-alpine
ports:
- name: httpd
containerPort: 8080
查看一下状态
kubectl get pod | grep tomcat
tomcat-deploy-64b488b68-wk45q 1/1 Running 0 29m
kubectl get svc | grep tomcat
tomcat ClusterIP 10.0.0.183 <none> 8080/TCP 29m
创建ingress绑定
cat ingress-tomcat.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-tomcat
namespace: default
annotations:
kubernets.io/ingress.class: "nginx"
spec:
rules:
- host: www.aa.com #用来解析的域名地址
http:
paths:
- path:
backend:
serviceName: tomcat #集群服务的名字
servicePort: 8080 #集群服务开放的端口
访问测试
curl -H "host:www.aa.com" http://10.167.130.206:80 #IP地址为运行ingress-nginx-controller的主机地址,因为只有运行了这个容器才会监听宿主的80端口。
<!DOCTYPE html>
<html lang="en">
<head>
<title>Apache Tomcat/7.0.91</title>
可用命令查看ingress列表
kubectl get ingress
NAME HOSTS ADDRESS PORTS AGE
ingress-tomcat www.aa.com 80 34m
kubectl describe ingress ingress-tomcat
Name: ingress-tomcat
Namespace: default
Address:
Default backend: default-http-backend:80 (<none>)
Rules:
Host Path Backends
---- ---- --------
www.aa.com
tomcat:8080 (<none>)
用ingress来负载代理tcp,udp请求**
前面已经展示了7层代理,那么4层代理比如mysql对3306端口代理怎么办呢？
ingress也可以做4层代理配置
具体配置如下
创建我们用来测试使用的mysql实例两个
cat mysql.yaml
apiVersion: v1
kind: Service
metadata:
name: mysql
namespace: default
spec:
type: ClusterIP
selector:
app: mysql
release: canary
ports:
- name: mysql
port: 3306
targetPort: 3306
---
apiVersion: apps/v1
kind: DaemonSet #每个node都运行一个pod，我就两个node正好用来测试负载效果
metadata:
name: mysql-daemonset
spec:
# replicas: 1
selector:
matchLabels:
app: mysql
release: canary
template:
metadata:
labels:
app: mysql
release: canary
spec:
containers:
- name: mysql
image: mysql
env:
- name: MYSQL_ROOT_PASSWORD #mysql镜像必须的变量,不写这个变量mysql跑不起来
value: "mysql"
ports:
- name: mysql
containerPort: 3306
kubectl apply -f mysql.yaml #部署mysql pod
kubectl get pod
mysql-daemonset-2xdr7 1/1 Running 0 63m
mysql-daemonset-stvhf 1/1 Running 0 63m
新建一个configmap文件tcp,udp是需要写这个文件来发布服务的
cat configmap.yaml
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
data:
3306: "default/mysql:3306" #我们的mysql是在默认命名空间里创建的这个自行查看更改
kubectl apply -f configmap.yaml
[root@node1 ~]# netstat -nlp | grep 3306 #去node节点查看3306端口是否已经监听起来
tcp 0 0 0.0.0.0:3306 0.0.0.0:* LISTEN 25785/nginx: master
tcp6 0 0 :::3306
注:添加或增减configmap直接在这个configmap文件中新增或去除即可,用kubectl apply重新应用,尽量不要直接kubectl delete -f configmap.yaml 因为这样会把整个tcp-services都删掉,删掉后node节点检测不到数据就不会对规则更新,这个和7层代理不太一样,7层可以一个服务创建一个name,4层在创建ingress服务时候就指定tcp-services和udp-services两个文件了,定义位置可以看ingress.yaml的281-282行,还有一种编辑方法
kubectl edit configmap/tcp-services -n ingress-nginx
# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
data:
"3306": default/mysql:3306 #在下面增加一行就可以,不需要的可以注释或者删除
kind: ConfigMap
metadata:
另外有一点,因为创建完ingress时候Node节点就是监听80和443的,在配置这个mysql时排错过程中发现,3306端口并不会默认监听,只有ingress可以正常连接到mysql集群时,node才会去监听3306端口,有错误可以按照这个思路排错,配置过程中也遇到很多问题,排错思路,容器>容器ip>集群ip>ingress。
断开连接几次试试，应该是轮训算法，分别在两个pod的数据库里写了a和b用来测试负载效果
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| a |
| information_schema |
| mysql |
| performance_schema |
| sys |
+--------------------+
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| b |
| information_schema |
| mysql |
| performance_schema |
| sys |
+--------------------+