kubernetes之Ingress controller

前言：

  traefik

  Traefik是一个用Golang开发的轻量级的Http反向代理和负载均衡器。由于可以自动配置和刷新backend节点，目前可以被绝大部分容器平台支持，例如Kubernetes，Swarm，Rancher等。由于traefik会实时与Kubernetes API交互,所以对于Service的节点变化，traefik的反应会更加迅速。总体来说traefik可以在Kubernetes中完美的运行.

  Nginx-Ingress-Controller

  Nginx-Ingress-Controller对于绝大多数刚刚接触k8s的人来说都比较熟悉，一个对外暴露service的7层反向代理。目前最新代号0.9.0-beta.15，可见目前nginx-ingress-control仍然处于beta版本。不过接触过的人还是明白nginx-ingress-control强大的Annotate配置，可以为service提供丰富的个性化配置，这点对于traefik来说是目前还无法打到的地步。

部署：

  要使用 traefik，我们同样需要部署 traefik 的 Pod，由于我们演示的集群中只有 master 节点有外网网卡，所以我们这里只有 master 这一个边缘节点，我们将 traefik 部署到该节点上即可。首先，为安全起见我们这里使用 RBAC 安全认证方式：(rbac.yaml)：

vim traefik-rbac.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: traefik-ingress-controller
  namespace: kube-ops
---  
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik-ingress-controller
rules:
  - apiGroups:
      - ""
    resources:
      - services
      - endpoints
      - secrets
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik-ingress-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
  name: traefik-ingress-controller
  namespace: kube-ops

  kubectl apply -f traefik-rbac.yaml  

[root@kubemaster traefik]# kubectl get ClusterRole -n kube-ops|grep traefik
traefik-ingress-controller                                             11m
[root@kubemaster traefik]# kubectl get ClusterRoleBinding -n kube-ops|grep traefik
traefik-ingress-controller                             2m36s
[root@kubemaster traefik]# kubectl get sa -n kube-ops
NAME                         SECRETS   AGE
default                      1         44h
prometheus                   1         14h
traefik-ingress-controller   1         11m
[root@kubemaster traefik]# 
可以查看到SA、ClusterRole和ClusterRoleBinding资源

  vim traefik-deployment.yaml

---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: traefik-ingress-controller
  namespace: kube-ops
  labels:
    k8s-app: traefik-ingress-lb
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: traefik-ingress-lb
  template:
    metadata:
      labels:
        k8s-app: traefik-ingress-lb
        name: traefik-ingress-lb
    spec:
      serviceAccountName: traefik-ingress-controller
      terminationGracePeriodSeconds: 60
      containers:
      - image: traefik
        name: traefik-ingress-lb
        ports:
        - name: http
          containerPort: 80
          hostPort: 80
        - name: admin
          containerPort: 8080
        args:
        - --api
        - --kubernetes
        - --logLevel=INFO
---
kind: Service
apiVersion: v1
metadata:
  name: traefik-ingress-service
  namespace: kube-ops
spec:
  selector:
    k8s-app: traefik-ingress-lb
  ports:
    - protocol: TCP
      port: 80
      name: web
    - protocol: TCP
      port: 8080
      name: admin
  type: NodePort

此处在containerPort里面的字段hostPort指定了，此容器的端口直接映射到宿主机的80端口，在创建Ingress资源之前，我们先需要创建一个演示的web应用
  
我开始部署一个测试的app应用，vim traefik-backend-app.yaml 部署了一个deployment和service,然后测试访问.这里我们部署的应用只能通过ClusterIP访问，而且ClusterIP只能是K8S集群内部才能访问的。如果需要从宿主机的外部访问到这个app应用，就需要把Service修改成NodePort的类型。加入有上百个应用在一个宿主机上面运行，那么修改成NodePort的类型的Service，一个宿主机的Iptables防火墙需要增加上百条策略，而且每一个宿主机都需要这样操作，势必会带来管理上的不便。这也就是为什么会产生Ingress资源的原因。客户访问k8s集群里面的web应用的流程应该是首先访问到公司的外部SLB设备（可以是硬件的负载均衡器比如F5等,也可以是软件比如LVS等。然后在从外部的LB设备到k8s集群的Ingress Controller。Ingress Controller就是k8s集群的访问入口，相当于nginx服务器一样。Ingress Controller既可以支持https协议，也可以通过虚拟主机或者URL映射的方式调用后端的upstream服务器。后端的upstream服务器就是真正运行的Pod.所以k8s集群只需要将Ingress Controller映射出去即可；

[root@kubemaster traefik]# kubectl get pods -n kube-ops 
NAME                                          READY   STATUS    RESTARTS   AGE
myapp-deploy-6b56d98b6b-65jc9                 1/1     Running   0          7m30s
myapp-deploy-6b56d98b6b-r92p8                 1/1     Running   0          7m30s
myapp-deploy-6b56d98b6b-rrb5b                 1/1     Running   0          7m30s
node-exporter-788bd                           1/1     Running   1          43h
node-exporter-7vfs7                           1/1     Running   1          43h
node-exporter-xkj2b                           1/1     Running   1          43h
prometheus-848d44c7bc-zwlb8                   1/1     Running   0          15h
redis-58c6c94968-qcq6p                        2/2     Running   2          44h
traefik-ingress-controller-86d4b5fcbf-6pfm5   1/1     Running   0          25m
traefik-ingress-controller-86d4b5fcbf-bs69c   1/1     Running   0          25m
[root@kubemaster traefik]# kubectl get svc -n kube-ops 
NAME                      TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                       AGE
myapp                     ClusterIP   10.98.239.156            80/TCP                        8m47s
prometheus                NodePort    10.109.108.37            9090:31312/TCP                44h
redis                     ClusterIP   10.100.225.179           6379/TCP,9121/TCP             44h
traefik-ingress-service   NodePort    10.111.9.88              80:30582/TCP,8080:30048/TCP   25m

[root@kubemaster traefik]# curl 10.98.239.156
Hello MyApp | Version: v2 | Pod Name
---
apiVersion: v1
kind: Service
metadata:
  name: myapp
  namespace: kube-ops 
spec:
  selector:
    app: myapp
    release: canary
  ports:
  - name: http 
    targetPort: 80
    port: 80
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-deploy 
  namespace: kube-ops
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      release: canary
  template:
    metadata:
      labels:
        app: myapp
        release: canary
    spec:
      containers:
      - name: myapp
        image: ikubernetes/myapp:v2
        ports：
        - name: http

  现在我们开始创建一个Ingress对象资源,vim traefik-ingress.yaml 

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-app
  namespace: kube-ops
  annotations:
    kubernetes.io/ingress.class: traefik
spec:
  rules:
  - host: myapp.maimaiti.cn
    http:
      paths:
      - backend:
          serviceName: myapp
          servicePort: 80
kubectl apply -f traefik-ingress.yaml
[root@kubemaster traefik]# kubectl get ingress -n kube-ops 
NAME          HOSTS               ADDRESS   PORTS   AGE
ingress-app   myapp.maimaiti.cn             80      8s

  现在我们开始在自己的电脑的hosts文件上面增加A记录，域名对应的IP地址就是运行traefik-ingress-controller的k8s node机器。由于我这边有两个node节点都运行了traefik-ingress-controller，所以绑定了连个地址  

10.83.32.146 myapp.maimaiti.cn
10.83.32.138 myapp.maimaiti.cn

  浏览器页面访问http://myapp.maimaiti.cn，输出的结果是
  Hello MyApp | Version: v2 | Pod Name


  我们除了通过Ingress Controller访问k8s集群的应用的Pod之外，traefik Ingress还有一个管理界面可以访问，现在我们再创建一个deployment,用于部署tomcat应用，然后也通过traefik Ingress Controller来提供流量访问入口  

apiVersion: v1
kind: Service
metadata:
  name: tomcat
  namespace: kube-ops 
spec:
  selector:
    app: tomcat
    release: canary
  ports:
  - name: http 
    targetPort: 8080
    port: 8080
  - name: ajp
    targetPort: 8009
    port: 8009
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deploy 
  namespace: kube-ops
spec:
  replicas: 3
  selector:
    matchLabels:
      app: tomcat
      release: canary
  template:
    metadata:
      labels:
        app: tomcat
        release: canary
    spec:
      containers:
      - name: tomcat
        image: tomcat:8.5.32-jre8-alpine
        ports:
        - name: http 
          containerPort: 8080
        - name: ajp
          containerPort: 8009
kubectl apply -f traefik-backend-tomcat.yaml

  然后开始重新修改一下Ingress资源的配置，将tomcat应用对应一个域名tomcat.maimaiti.cn来访问  

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-app
  namespace: kube-ops
  annotations:
    kubernetes.io/ingress.class: traefik
spec:
  rules:
  - host: myapp.maimaiti.cn
    http:
      paths:
      - backend:
          serviceName: myapp
          servicePort: 80
  - host: tomcat.maimaiti.cn
    http:
      paths:
      - backend:
          serviceName: tomcat
          servicePort: 8080
kubectl apply -f treafik-ingress.yaml

  现在我们开始在自己的电脑的hosts文件上面增加A记录，域名对应的IP地址就是运行traefik-ingress-controller的k8s node机器。由于我这边有两个node节点都运行了traefik-ingress-controller，所以绑定了连个地址  

10.83.32.146 myapp.maimaiti.cn tomcat.maimaiti.cn
10.83.32.138 myapp.maimaiti.cn tomcat.maimaiti.cn

2. traefik Ingress Controll https认证配置
  2.1. 配置traefik Ingress Controller的配置文件toml:
  vim traefik.toml  

defaultEntryPoints = ["http","https"]
[entryPoints]
  [entryPoints.http]
  address = ":80"
  [entryPoints.https]
  address = ":443"
    [entryPoints.https.tls]
      [[entryPoints.https.tls.certificates]]
      CertFile = "/ssl/tls.crt"
      KeyFile = "/ssl/tls.key"
[metrics]
  [metrics.prometheus]
    entryPoint = "traefik"
    buckets = [0.1, 0.3, 1.2, 5.0]
kubectl create configmap traefik-conf --from-file=traefik.toml -n kube-ops

[root@kubemaster traefik]# kubectl describe cm -n kube-ops traefik-conf 
Name:         traefik-conf
Namespace:    kube-ops
Labels:       
Annotations:  

Data
====
traefik.toml:
----
defaultEntryPoints = ["http","https"]
[entryPoints]
  [entryPoints.http]
  address = ":80"
  [entryPoints.https]
  address = ":443"
    [entryPoints.https.tls]
      [[entryPoints.https.tls.certificates]]
      CertFile = "/ssl/tls.crt"
      KeyFile = "/ssl/tls.key"
[metrics]
  [metrics.prometheus]
    entryPoint = "traefik"
    buckets = [0.1, 0.3, 1.2, 5.0]

Events:  
[root@kubemaster traefik]#

  配置文件主要包含了https接口访问的证书位置和prometheus的监控配置，接下来创建自签名证书

openssl req -newkey rsa:2048 -nodes -keyout tls.key -x509 -days 365 -out tls.crt
Generating a 2048 bit RSA private key
...........+++
................................................................+++
writing new private key to 'tls.key'
-----
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:GD
Locality Name (eg, city) [Default City]:SZ
Organization Name (eg, company) [Default Company Ltd]:MMT
Organizational Unit Name (eg, section) []:IT
Common Name (eg, your name or your server's hostname) []:gaoyang
Email Address []:[email protected]
[root@kubemaster traefik]# ll
total 32
-rw-r--r-- 1 root root 1367 Mar  7 14:55 tls.crt
-rw-r--r-- 1 root root 1708 Mar  7 14:55 tls.key
-rw-r--r-- 1 root root  601 Mar  7 10:55 traefik-backend-app.yaml
-rw-r--r-- 1 root root  718 Mar  7 13:44 traefik-backend-tomcat.yaml
-rw-r--r-- 1 root root 1028 Mar  7 11:02 traefik-deployment.yaml
-rw-r--r-- 1 root root  418 Mar  7 14:07 traefik-ingress.yaml
-rw-r--r-- 1 root root  800 Mar  7 10:28 traefik-rbac.yaml
-rw-r--r-- 1 root root  364 Mar  7 14:50 traefik.toml
#创建所需要的证书文件和Pod里面调用的secret资源
kubectl create secret generic traefik-cert --from-file=tls.crt --from-file=tls.key -n kube-ops

  接下来需要修改traefik Ingress Controll的deployment的配置，增加上读取configmap和secret的参数，并暴露443端口提供https的访问

---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: traefik-ingress-controller
  namespace: kube-ops
  labels:
    k8s-app: traefik-ingress-lb
spec:
  replicas: 2
  selector:
    matchLabels:
      k8s-app: traefik-ingress-lb
  template:
    metadata:
      labels:
        k8s-app: traefik-ingress-lb
        name: traefik-ingress-lb
    spec:
      serviceAccountName: traefik-ingress-controller
      terminationGracePeriodSeconds: 60
      volumes:
      - name: ssl
        secret:
          secretName: traefik-cert
      - name: config
        configMap:
          name: traefik-conf
      containers:
      - image: traefik
        name: traefik-ingress-lb
        volumeMounts:
        - name: "ssl"
          mountPath: "/ssl"
        - name: "config"
          mountPath: "/config"

        ports:
        - name: http
          containerPort: 80
          hostPort: 80
        - name: https
          containerPort: 443
          hostPort: 443
        - name: admin
          containerPort: 8080
        args:
        - --configfile=/config/traefik.toml
        - --api
        - --kubernetes
        - --logLevel=INFO
---
kind: Service
apiVersion: v1
metadata:
  name: traefik-ingress-service
  namespace: kube-ops
spec:
  selector:
    k8s-app: traefik-ingress-lb
  ports:
    - protocol: TCP
      port: 80
      name: web
    - protocol: TCP
      port: 8080
      name: admin
  type: NodePort
# 注意此处重新修改了deployment文件，增加了secret和configmap的挂载，增加了启动读取配置文件的参数

  接下来需要修改Ingress资源的配置，增加上https访问

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-app
  namespace: kube-ops
  annotations:
    kubernetes.io/ingress.class: traefik
spec:
  tls:
  - hosts:
    - myapp.maimaiti.cn
    secretName: traefik-cert
  rules:
  - host: myapp.maimaiti.cn
    http:
      paths:
      - backend:
          serviceName: myapp
          servicePort: 80
  - host: tomcat.maimaiti.cn
    http:
      paths:
      - backend:
          serviceName: tomcat
          servicePort: 8080
kubectl apply -f traefik-ingress.yaml

  现在就可以用https访问tomcat和app



推荐关注我的个人微信公众号 “云时代IT运维”,周期性更新最新的应用运维类技术文档。关注虚拟化和容器技术、CI/CD、自动化运维等最新前沿运维技术和趋势；