k8s内部署rabbitmq高可用集群

一、创建NFS动态卷持久化存储

1、创建NFS服务端

[root@work03 ~]# yum install nfs-utils rpcbind -y
[root@work03 ~]# systemctl start nfs
[root@work03 ~]# systemctl start rpcbind
[root@work03 ~]# systemctl enable nfs
[root@work03 ~]# systemctl enable rpcbind
[root@work03 ~]# mkdir -p /data/mongodb/
[root@work03 ~]# vim /etc/exports
/data/mongodb/    *(rw,sync,no_root_squash,no_all_squash)
[root@work03 ~]# systemctl restart rpcbind
[root@work03 ~]# systemctl restart nfs
[root@work03 ~]# showmount -e localhost
Export list for localhost:
/data/nfs *

2、创建动态卷提供者

（1）创建service-rbac

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: kube-system
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f  service-rbac.yaml

（2）创建nfs-provisioner-deploy

vim nfs-provisioner-deploy.yaml

kind: Deployment
apiVersion: apps/v1
metadata:
  name: rabbitmq-nfs
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: rabbitmq-nfs
  replicas: 3
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: rabbitmq-nfs
    spec:
      serviceAccount: nfs-client-provisioner
      imagePullSecrets:
      - name: regcred
      containers:
        - name: rabbitmq-nfs
          image: 192.168.0.107:80/heosun/nfs-client-provisioner:v1.0
          volumeMounts:
            - name: rabbitmq-nfs-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: asd
            - name: NFS_SERVER
              value: 192.168.0.108
            - name: NFS_PATH
              value: /data/rabbitmq
      volumes:
        - name: rabbitmq-nfs-root
          nfs:
            server: 192.168.0.108
            path: /data/rabbitmq

（4）创建storageclass

vim storageclass.yaml

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: rabbitmq-nfs
provisioner: asd

kubectl apply -f storageclass.yaml

二、部署rabbitmq

一、下载插件

mkdir plugins && cd plugins

wget https://github.com/rabbitmq/rabbitmq-autocluster/releases/download/0.10.0/autocluster-0.10.0.ez

wget https://github.com/rabbitmq/rabbitmq-autocluster/releases/download/0.10.0/rabbitmq_aws-0.10.0.ez

二、创建Dockerfile

cd ../

vim Dockerfile

FROM rabbitmq:3.6.8
MAINTAINER Idiom
ENV RABBITMQ_USE_LONGNAME=true \
    AUTOCLUSTER_LOG_LEVEL=debug \
    AUTOCLUSTER_CLEANUP=true \
    CLEANUP_INTERVAL=60 \
    CLEANUP_WARN_ONLY=false \
    AUTOCLUSTER_TYPE=k8s \
    LANG=en_US.UTF-8
ADD plugins/*.ez /usr/lib/rabbitmq/lib/rabbitmq_server-3.6.8/plugins/
RUN rabbitmq-plugins enable --offline autocluster
RUN rabbitmq-plugins enable --offline rabbitmq_management

docker build -t 172.16.0.14:5000/pointsmart/rabbitmq3.7:v4 .

docker push 172.16.0.14:5000/pointsmart/rabbitmq3.7:v4 （没有私有harbor可以省略此步骤）

三、生成cookie文件

vim erlang-cookie.sh

#! /bin/bash
echo $(openssl rand -base64 32) > erlang.cookie
kubectl create secret generic erlang.cookie --from-file=erlang.cookie

sh erlang-cookie.sh

四、配置rbac

vim rabbitmq-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: rabbitmq
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: endpoint-reader
rules:
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: endpoint-reader
subjects:
- kind: ServiceAccount
  name: rabbitmq
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: endpoint-reader

五、编写service

vim serveice-rabbitmq.yaml

kind: Service
apiVersion: v1
metadata:
  name: rabbitmq-service
spec:
  type: NodePort
  ports:
    - name: mangement
      protocol: TCP
      port: 15672
      nodePort: 32001
    - name: smp 
      protocol: TCP
      port: 5672
      nodePort: 32002
  selector:
    app: rabbitmq
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq
  labels:
    app: rabbitmq
spec:
  clusterIP: None
  ports:
  - port: 5672
    name: amqp
  selector:
    app: rabbitmq

六、编写StatefulSet

vim StatefulSet.yaml

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: rabbitmq
spec:
  serviceName: rabbitmq
  replicas: 3
  template:
    metadata:
      labels:
        app: rabbitmq
    spec:
      serviceAccountName: rabbitmq
      imagePullSecrets: 
        - name: regsecret
      containers:
      - name: rabbitmq
        image: 172.16.0.14:5000/pointsmart/rabbitmq3.7:v4 
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            memory: "256Mi"
            cpu: "150m"
          limits:
            memory: "512Mi"
            cpu: "250m"
        volumeMounts:
          - name: rabbitmq-data
            mountPath: /var/lib/rabbitmq/mnesia
        ports:
        - containerPort: 5672
          name: amqp
        env:
          - name: RABBITMQ_DEFAULT_USER
            value: comma 
          - name: RABBITMQ_DEFAULT_PASS
            value: commaai2017
          - name: RABBITMQ_ERLANG_COOKIE
            valueFrom:
              secretKeyRef:
                name: erlang.cookie
                key: erlang.cookie
          - name: MY_POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: K8S_SERVICE_NAME
            value: "rabbitmq"
          - name: RABBITMQ_USE_LONGNAME
            value: "true"
          - name: RABBITMQ_NODENAME
            value: "rabbit@$(MY_POD_NAME).$(K8S_SERVICE_NAME)"
          - name: RABBITMQ_NODE_TYPE
            value: disc
          - name: AUTOCLUSTER_TYPE
            value: "k8s"
          - name: AUTOCLUSTER_DELAY
            value: "10"
          - name: AUTOCLUSTER_CLEANUP
            value: "true"
          - name: CLEANUP_WARN_ONLY
            value: "false"
          - name: K8S_ADDRESS_TYPE
            value: "hostname"
          - name: K8S_HOSTNAME_SUFFIX
            value: ".$(K8S_SERVICE_NAME)"
  volumeClaimTemplates:
  - metadata:
      name: rabbitmq-data
      annotations:
        volume.beta.kubernetes.io/storage-class: rabbitmq-nfs
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1Gi

kubectl create -f storageclass.yaml
kubectl create -f nfs-provisioner-deploy.yaml
kubectl create -f rabbitmq-statefulset.yaml
kubectl create -f serveice-rabbitmq.yaml
kubectl create -f rabbitmq-rbac.yaml

七、进入主容器创建用户

kubectl exec -it rabbitmq-0 -- /bin/bash #进入容器
rabbitmqctl stop_app #停止rabbitmq
rabbitmqctl reset #设置rabbitmq可更改
rabbitmqctl start_app #开启rabbitmq
rabbitmqctl delete_user guest #删除guest用户
rabbitmqctl add_user 用户名 密码 #创建用户
rabbitmqctl set_user_tags 用户名 administrator #增加权限

八、登录方式
node节点:32001