k8s使用StatefulSet部署MongoDB集群

1.1 前提准备

• 制作MongoDB 镜像，可从 Docker Hub 官网 下载镜像，目的是解决提示权限mongo-sidecar提示权限错误问题（官方镜像基础上添加了密码认证）

        生成 keyfile ，编写Dockerfile，构建镜像

# 生成 keyfile
[root@registry mongodb]# openssl rand -base64 741 > mongodb-keyfile
# 查看文件信息
[root@registry mongodb]# ls
Dockerfile  mongodb-keyfile
# 创建Dockerfile
[root@registry mongodb]# cat Dockerfile 
FROM mongo:3.6.4
ADD mongodb-keyfile /data/config/mongodb-keyfile
RUN chown mongodb:mongodb /data/config/mongodb-keyfile && chmod 600 /data/config/mongodb-keyfile

# 构建镜像
[root@registry mongodb]# docker build -f Dockerfile -t jinyuyun.top/mongo:3.6.4 .

构建镜像

• 外部ceph集群搭建好，(使用cephfs-csi 做数据持久卷，作为 MongoDB用来存放数据)

1.2 编写 yaml 文件

编写 mongodb-statefulset.yaml 文件，创建ServiceAccount，RBAC，StatefulSet以及提供了一个对外暴露的 NodePort 类型的 Service，用于外部访问。

# cat mongodb-statefulset.yaml
---
apiVersion: v1
kind: Namespace
metadata:
  name: mongodb
  labels:
    name: mongo
---
apiVersion: v1
kind: ServiceAccount      #集群访问apiserver的凭证
metadata:
  name: mongo
  namespace: mongodb
 
---
# rbac配置
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: mongo-default-view
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: view
subjects:
  - kind: ServiceAccount
    name: mongo
    namespace: mongodb


---
#mongo部署service
apiVersion: v1
kind: Service
metadata:
  name: mongo
  namespace: mongodb
  labels:
    name: mongo
spec:
  ports:
  - port: 27017
    targetPort: 27017
  clusterIP: None
  selector:
    role: mongo
---
#mongo暴露外部端口用于外部访问
apiVersion: v1
kind: Service
metadata: 
  name: mongo-service
  namespace: mongodb
  labels: 
    name: mongo
spec: 
  ports: 
  - name: mongo
    port: 27017
    nodePort: 27017
  selector: 
    role: mongo
  type: NodePort
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mongo
  namespace: mongodb
spec:
  selector: 
    matchLabels: 
      role: mongo
      environment: prod
  serviceName: "mongo"
  replicas: 2
  template:
    metadata:
      labels:
        role: mongo
        environment: prod
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccountName: mongo
      containers:
      - name: mongo
        image: jinyuyun.top/mongo:3.6.4
        imagePullPolicy: IfNotPresent
        resources:
          limits:            # 限定资源
            cpu: 500m
            memory: 500Mi
          requests:
            cpu: 100m
            memory: 50Mi
        env:
          - name: MONGO_INITDB_ROOT_USERNAME
            value: root
          - name: MONGO_INITDB_ROOT_PASSWORD
            value: 123
        args: 此处需要将command改为args ，否则 MONGO_INITDB_ROOT_USERNAME，MONGO_INITDB_ROOT_PASSWORD会被覆盖不能生效
        - mongod
        - "--replSet"
        - rs0
        - "--bind_ip"
        - 0.0.0.0
        - --clusterAuthMode
        - keyFile
        - --keyFile
        - /data/config/mongodb-keyfile
  #      - "--smallfiles"
  #      - "--noprealloc"
        ports:
          - containerPort: 27017
        volumeMounts:
          - name: mongo-data
            mountPath: /data/db
      - name: mongo-sidecar
        image: jinyuyun.top/mongo-k8s-sidecar
        imagePullPolicy: IfNotPresent
        resources:
          limits:            # 限定资源
            cpu: 500m
            memory: 500Mi
          requests:
            cpu: 100m
            memory: 50Mi
        env:
          - name: KUBE_NAMESPACE
            value: mongodb
          - name: MONGODB_USERNAME
            value: root
          - name: MONGODB_PASSWORD
            value: 123
          - name: MONGO_SIDECAR_POD_LABELS
            value: "role=mongo,environment=prod"
          - name: MONGODB_DATABASE
            value: admin
            
  volumeClaimTemplates:
    - metadata:
        name: mongo-data
      spec:
        accessModes: [ "ReadWriteMany" ]
        storageClassName: "jyy-cephfs-sc"
        resources:
          requests:
            storage: 3Gi

1.3 部署mongodb-statefullset.yaml文件

[root@master mongodb]# kubectl apply -f mongodb-statefulset.yaml 
namespace/mongodb created
serviceaccount/mongo created
clusterrolebinding.rbac.authorization.k8s.io/mongo-default-view created
persistentvolumeclaim/mongodb-pvc created
service/mongo created
service/mongo-service created
statefulset.apps/mongo created

1.4 查看部署的mongo集群

[root@master mongodb]# kubectl get all -o wide
NAME                               TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                              AGE   SELECTOR
service/csi-metrics-cephfsplugin   ClusterIP   10.103.172.38           8080/TCP                             35d   app=csi-cephfsplugin
service/kubernetes                 ClusterIP   10.96.0.1               443/TCP                              58d   
service/nfs-provisioner            ClusterIP   10.110.5.164            2049/TCP,20048/TCP,111/TCP,111/UDP   58d   app=nfs-provisioner
[root@master mongodb]# kubectl get all -n mongodb
NAME          READY   STATUS    RESTARTS   AGE
pod/mongo-0   2/2     Running   0          55m
pod/mongo-1   2/2     Running   0          54m

NAME                    TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)           AGE
service/mongo           ClusterIP   None                     27017/TCP         55m
service/mongo-service   NodePort    10.102.239.141           27017:27017/TCP   55m

NAME                     READY   AGE
statefulset.apps/mongo   2/2     55m

1.4 进入容器查看集群状态

# 进入mongo-0 pod, 使用密码登录
[root@master mongodb]# kubectl exec -it mongo-0 -n mongodb -- mongo
# 进入db:admin
rs0:PRIMARY> use admin;
# 认证
rs0:PRIMARY> db.auth("root","123");
# 查看数据库
rs0:PRIMARY> show dbs;
admin   0.000GB
config  0.000GB
local   0.000GB

# 查看集群状态
rs0:PRIMARY> rs.status()


# 可知集群状态信息
mongo-0    PRIMARY
mongo-1    SECONDARY

 1.5 高可用主从读写测试

# 主节点存储数据
rs0:PRIMARY> db.test.insert({"name":"zhangshan"})
WriteResult({ "nInserted" : 1 })
rs0:PRIMARY> show dbs
admin   0.000GB
config  0.000GB
local   0.000GB
test    0.000GB
rs0:PRIMARY> exit

# 从节点查看数据库数据
rs0:SECONDARY> use admin
switched to db admin
rs0:SECONDARY> db.auth("root","root123");
1
rs0:SECONDARY> rs.slaveOk()
rs0:SECONDARY> show dbs
admin   0.000GB
config  0.000GB
local   0.000GB
test    0.000GB
rs0:SECONDARY> use test
switched to db test
rs0:SECONDARY> db.test.find().pretty()
{ "_id" : ObjectId("618a307a30ed71e51682d041"), "name" : "zhangshan" }
rs0:SECONDARY> exit

 1.5 使用访问MongoDB

mongo cluster访问默认连接为：

mongodb://mongo1,mongo2,mongo3:27017/dbname_?

在kubernetes中最常用的FQDN连接服务的连接为：

#appName.$HeadlessServiceName.$Namespace.svc.cluster.local

因为我们采用statefulset部署的pod，所以命名均有规则，所以实际上如果连接到副本的mongodb cluster，上面的默认连接该为（默认为namespace之外）：

mongodb://mongo-0.mongo.mongodb.svc.cluster.local:27017,mongo-1.mongo.mongodb.svc.cluster.local:27017/?replicaSet=rs0

mongodb集群部署完成。

1.6 参考 

 k8s 搭建mongodb多副本集群 | 易学教程

https://segmentfault.com/a/1190000017321906

Kubernetes部署高可用MongoDB集群 - EvenChan - 博客园

https://github.com/cvallance/mongo-k8s-sidecar