K8S 部署 traefik 2.2.1 ingress controller

准备traefik K8S 部署相关的yaml 文件

# 有*号替换的问题请查看我自己的github
https://github.com/qist/k8s/tree/master/k8s-yaml/traefik2
# 官网规则地址：
# https://docs.traefik.io/v2.2/routing/providers/kubernetes-ingress/
# https://docs.traefik.io/v2.2/routing/providers/kubernetes-crd/
# 创建新命名空间 ingress-system
vim 0traefik-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: ingress-system
---
# 创建rbac 及Definitions 
vim traefik-rbac.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ingre***outes.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: Ingre***oute
    plural: ingre***outes
    singular: ingre***oute
  scope: Namespaced

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: middlewares.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: Middleware
    plural: middlewares
    singular: middleware
  scope: Namespaced

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ingre***outetcps.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: Ingre***outeTCP
    plural: ingre***outetcps
    singular: ingre***outetcp
  scope: Namespaced

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ingre***outeudps.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: Ingre***outeUDP
    plural: ingre***outeudps
    singular: ingre***outeudp
  scope: Namespaced

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: tlsoptions.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: TLSOption
    plural: tlsoptions
    singular: tlsoption
  scope: Namespaced

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: tlsstores.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: TLSStore
    plural: tlsstores
    singular: tlsstore
  scope: Namespaced

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: traefikservices.traefik.containo.us

spec:
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: TraefikService
    plural: traefikservices
    singular: traefikservice
  scope: Namespaced
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik

rules:
  - apiGroups:
      - ""
    resources:
      - services
      - endpoints
      - secrets
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
    resources:
      - ingresses/status
    verbs:
      - update
  - apiGroups:
      - traefik.containo.us
    resources:
      - middlewares
      - ingre***outes
      - traefikservices
      - ingre***outetcps
      - ingre***outeudps
      - tlsoptions
      - tlsstores
    verbs:
      - get
      - list
      - watch

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik

roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: traefik
subjects:
  - kind: ServiceAccount
    name: traefik
    namespace: ingress-system
# 创建traefik daemonset yaml 私有环境daemonset 方式部署
vim traefik-daemonset-https.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: traefik
  namespace: ingress-system
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
  namespace: ingress-system
  name: traefik
  labels:
    k8s-app: traefik
spec:
  selector:
    matchLabels:
      k8s-app: traefik
  template:
    metadata:
      labels:
        k8s-app: traefik
      annotations:
        prometheus.io/port: "8082"
        prometheus.io/scrape: 'true'
    spec:
      serviceAccountName: traefik
      terminationGracePeriodSeconds: 60
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
        - name: traefik
          image: traefik:v2.2.1
          args:
            - --api.insecure
            - --api.dashboard
            - --log
            - --log.level=INFO
            - --accesslog
            - --accessLog.fields.headers.defaultMode=redact
            - --entrypoints.web.Address=:80
            - --entrypoints.websecure.Address=:443
            - --providers.kubernetescrd
            - --metrics.prometheus
            - --metrics.prometheus.entrypoint=metrics
            - --metrics.prometheus.addEntryPointsLabels=true
            - --entryPoints.metrics.address=:8082
            - --serverstransport.insecureskipverify=true
            - --providers.kubernetesingress.disablepasshostheaders=true
          ports:
            - name: web
              containerPort: 80
              hostPort: 80
            - name: websecure
              containerPort: 443
              hostPort: 443
            - name: admin
              containerPort: 8080
              hostPort: 8080
            - name: http-metrics
              containerPort: 8082
              hostPort: 8082
          securityContext:
            capabilities:
              drop:
              - ALL
              add:
              - NET_BIND_SERVICE
      #nodeSelector:
        #ingress: "yes"
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/ingress
        operator: Equal
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1

---
apiVersion: v1
kind: Service
metadata:
  labels:
    k8s-app: traefik  
  name: traefik
  namespace: ingress-system
spec:
  clusterIP: None
  type: ClusterIP
  ports:
    - protocol: TCP
      name: web
      port: 80
    - protocol: TCP
      name: admin
      port: 8080
    - protocol: TCP
      name: websecure
      port: 443
    - protocol: TCP
      name: http-metrics
      port: 8082
  selector:
    k8s-app: traefik
# 创建 traefik dashboard  ingress
vim traefik-dashboard.yaml
---
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: traefik-dashboard
  namespace: ingress-system
spec:
  entryPoints:
    - web
  routes:
  - match: Host(`traefik.tycng.com`)
    kind: Rule
    services:
    - name: api@internal
      kind: TraefikService

部署traefik ingress

kubectl apply -f .
root@Qist:/mnt/g/work/k8s/k8s-yaml/traefik2# kubectl apply -f .
namespace/ingress-system created
customresourcedefinition.apiextensions.k8s.io/ingre***outes.traefik.containo.us created
customresourcedefinition.apiextensions.k8s.io/middlewares.traefik.containo.us created
customresourcedefinition.apiextensions.k8s.io/ingre***outetcps.traefik.containo.us created
customresourcedefinition.apiextensions.k8s.io/ingre***outeudps.traefik.containo.us created
customresourcedefinition.apiextensions.k8s.io/tlsoptions.traefik.containo.us created
customresourcedefinition.apiextensions.k8s.io/tlsstores.traefik.containo.us created
customresourcedefinition.apiextensions.k8s.io/traefikservices.traefik.containo.us created
clusterrole.rbac.authorization.k8s.io/traefik created
clusterrolebinding.rbac.authorization.k8s.io/traefik created
ingre***oute.traefik.containo.us/traefik-dashboard created
serviceaccount/traefik created
daemonset.apps/traefik created
service/traefik created
# 如果报错请多试一次
unable to recognize "traefik-dashboard.yaml": no matches for kind "Ingre***oute" in version "traefik.containo.us/v1alpha1"
# 再次执行kubectl apply -f . 就好了

验证traefik ingress 部署是否正常

# traefik 有自己的dashboard 端口是8080 任意节点访问
http://192.168.2.175:8080/dashboard/#/
# dns 解析traefik-dashboard traefik.tycng.com # 域名改成自己的

# 域名访问结果

创建应用对外提供访问

# 记得域名修改成自己的，然后dns 做好解析
# 创建测项目
#  部署一个应用
kubectl create deployment myip --image=cloudnativelabs/whats-my-ip 
# 暴露端口
kubectl expose deployment myip --port=8080 --target-port=8080
# 兼容 K8S 
cat << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: myip
    namespace: default
spec:
  ingressClassName: traefik
  rules:
  - host: prometheus.tycng.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          serviceName: myip 
          servicePort: 8080
EOF

# 能够正常提供访问
# kubernetes-crd 创建对我服务网 并测试ipv4 ipv6
# 创建应用IPV4 service
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  labels:
    app: myip
  name: myip-ipv4
  namespace: default
spec:
  ipFamily: IPv4
  ports:
  - port: 8080
    protocol: TCP
    targetPort: 8080
  selector:
    app: myip
  sessionAffinity: None
  type: ClusterIP
EOF
# 创建应用IPV6 service
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  labels:
    app: myip
  name: myip-ipv6
  namespace: default
spec:
  ipFamily: IPv6
  ports:
  - port: 8080
    protocol: TCP
    targetPort: 8080
  selector:
    app: myip
  sessionAffinity: None
  type: ClusterIP
EOF
# 创建应用IPV4 Ingress
cat << EOF | kubectl apply -f -
---
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: myip-ipv4
  namespace: default
spec:
  entryPoints:
    - web
  routes:
  - match: Host(\`ipv4.tycng.com\`)
    kind: Rule
    services:
    - kind: Service
      name: myip-ipv4
      port: 8080
      sticky:
        cookie:
          httpOnly: true
          name: cookie
          secure: true
          sameSite: none
      passHostHeader: true
      responseForwarding:
        flushInterval: 100ms
EOF
# 创建应用IPV6Ingress
cat << EOF | kubectl apply -f -
---
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: myip-ipv6
  namespace: default
spec:
  entryPoints:
    - web
  routes:
  - match: Host(\`ipv6.tycng.com\`)
    kind: Rule
    services:
    - kind: Service
      name: myip-ipv6
      port: 8080
      sticky:
        cookie:
          httpOnly: true
          name: cookie
          secure: true
          sameSite: none
      passHostHeader: true
      responseForwarding:
        flushInterval: 100ms
EOF
# 分别访问 ipv4 ipv6 域名

#查看日志
# http://prometheus.tycng.com/
192.168.0.151 - - [09/May/2020:02:17:55 +0000] "GET / HTTP/1.1" 200 48 "-" "REDACTED" 5 "myip-default-prometheus-tycng-com@kubernetes" "http://10.81.251.198:8080" 1ms
192.168.0.151 - - [09/May/2020:02:17:55 +0000] "GET /favicon.ico HTTP/1.1" 200 48 "REDACTED" "REDACTED" 6 "myip-default-prometheus-tycng-com@kubernetes" "http://10.81.251.198:8080" 1ms
192.168.0.151 - - [09/May/2020:02:17:55 +0000] "GET / HTTP/1.1" 200 48 "-" "REDACTED" 7 "myip-default-prometheus-tycng-com@kubernetes" "http://10.81.251.198:8080" 1ms
192.168.0.151 - - [09/May/2020:02:17:55 +0000] "GET /favicon.ico HTTP/1.1" 200 48 "REDACTED" "REDACTED" 8 "myip-default-prometheus-tycng-com@kubernetes" "http://10.81.251.198:8080" 1ms
## http://ipv4.tycng.com/
192.168.0.151 - - [09/May/2020:02:17:38 +0000] "GET / HTTP/1.1" 200 48 "-" "REDACTED" 7 "default-myip-ipv4-e434de741cf720e0f177@kubernetescrd" "http://10.81.251.198:8080" 3ms
192.168.0.151 - - [09/May/2020:02:17:38 +0000] "GET /favicon.ico HTTP/1.1" 200 48 "REDACTED" "REDACTED" 8 "default-myip-ipv4-e434de741cf720e0f177@kubernetescrd" "http://10.81.251.198:8080" 2ms
192.168.0.151 - - [09/May/2020:02:17:40 +0000] "GET / HTTP/1.1" 200 48 "-" "REDACTED" 9 "default-myip-ipv4-e434de741cf720e0f177@kubernetescrd" "http://10.81.251.198:8080" 2ms
192.168.0.151 - - [09/May/2020:02:17:40 +0000] "GET /favicon.ico HTTP/1.1" 200 48 "REDACTED" "REDACTED" 10 "default-myip-ipv4-e434de741cf720e0f177@kubernetescrd" "http://10.81.251.198:8080" 3ms
# http://ipv6.tycng.com/
fc00:bd4:efa8:1002:2c5b:6d16:5d76:db04 - - [09/May/2020:02:20:14 +0000] "GET / HTTP/1.1" 200 48 "-" "REDACTED" 11 "default-myip-ipv6-93107a0b989d93ecac85@kubernetescrd" "http://fd00::1:fbc6:8080" 7ms
fc00:bd4:efa8:1002:2c5b:6d16:5d76:db04 - - [09/May/2020:02:20:14 +0000] "GET /favicon.ico HTTP/1.1" 200 48 "REDACTED" "REDACTED" 12 "default-myip-ipv6-93107a0b989d93ecac85@kubernetescrd" "http://fd00::1:fbc6:8080" 3ms

traefik HTTPS 配置

vim traefik-secret.yaml
---
apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: tls-cert
  name: tls-cert
  namespace: ingress-system
type: Opaque
data:
  tls.crt:  # 域名的证书对应nginx 配置证书  base64 加密   tls.crt 名字不可以修改
  tls.key:  # 域名的私钥对应 nginx 配置私钥 base64 加密   tls.key名字不可以修改 

# 提交 Secret
 kubectl apply -f tls-cert
# 下面是http 强制https 配置写法
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: jaeger
  namespace: ingress-system
spec:
  entryPoints:
    - websecure
  routes:
  - match: Host(`trae.tycng.com`)
    kind: Rule
    priority: 12
    services:
    - name: jaeger-query
      port: 80
      weight: 1
     # 开启粘性会话
      sticky:
        cookie:
          httpOnly: true
          name: cookie
          secure: true
          sameSite: none
      passHostHeader: true
      responseForwarding:
        flushInterval: 100ms
  tls:
    options: 
      name: default
      namespace: ingress-system
    secretName: tls-cert
---
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: jaegerhttp
  namespace: ingress-system
spec:
  entryPoints:
    - web
  routes:
  - match: Host(`trae.tycng.com`)
    kind: Rule
    priority: 12
    services:
    - name: jaeger-query
      port: 80
      weight: 1
      # 开启粘性会话
      sticky:
        cookie:
          httpOnly: true
          name: cookie
          secure: true
          sameSite: none
      passHostHeader: true
      responseForwarding:
        flushInterval: 100ms
    middlewares:
      - name: redirect

---
apiVersion: traefik.containo.us/v1alpha1
kind: Middleware
metadata:
  name: redirect
  namespace: ingress-system
spec:
  redirectScheme:
    scheme: https
---
apiVersion: traefik.containo.us/v1alpha1
kind: TLSOption
metadata:
  name: default
  namespace: ingress-system

spec:
  minVersion: VersionTLS12

---
apiVersion: traefik.containo.us/v1alpha1
kind: TLSOption
metadata:
  name: mintls13
  namespace: ingress-system

spec:
  minVersion: VersionTLS13
# websockets 转发写法
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: rancherhttp
  namespace: cattle-system
spec:
  entryPoints:
    - web
  routes:
  - match: Host(`rke.tycng.com`)
    kind: Rule
    priority: 12
    services:
    - name: rancher
      port: 80
    middlewares:
      - name: redirect
---
apiVersion: traefik.containo.us/v1alpha1
kind: Ingre***oute
metadata:
  name: rancher
  namespace: cattle-system
spec:
  entryPoints:   
    - websecure
  routes:
    - match: Host(`rke.tycng.com`)
      middlewares:
        - name: x-forwarded-proto-allow
          namespace: cattle-system
      kind: Rule
      services:
        - name: rancher
          port: 80
  tls:
    secretName: tls-rancher-ingress    
---    
apiVersion: traefik.containo.us/v1alpha1
kind: Middleware
metadata:
  name: redirect
  namespace: cattle-system
spec:
  redirectScheme:
    scheme: https    
---    
apiVersion: traefik.containo.us/v1alpha1
kind: TLSOption
metadata:
  name: default
  namespace: cattle-system

spec:
  minVersion: VersionTLS12  
--- 
apiVersion: traefik.containo.us/v1alpha1
kind: Middleware
metadata:
  name: x-forwarded-proto-allow
  namespace: cattle-system
spec:
  headers:
    customRequestHeaders:
      X-Forwarded-Proto: https
---
apiVersion: traefik.containo.us/v1alpha1
kind: TLSOption
metadata:
  name: mintls13
  namespace: cattle-system

spec:
  minVersion: VersionTLS13