CentOS7部署Ceph nautilus集群 - 傻瓜式教程

1 环境介绍与配置

1.1 ceph介绍

#ceph架构
#ceph支持的三种接口：
1 Object：有原生的API，而且也兼容Swift和S3的API。
2 Block：支持精简配置、快照、克隆。
3 File：Posix接口，支持快照。

#ceph的三种类型的优缺点：

1.2 环境介绍

[root@ceph131 ~]# cat /etc/redhat-release
CentOS Linux release 7.8.2003 (Core)

#ceph 
nautilus 14.2.10-0.el7

#网络设计
172.16.1.0/24 #Management Network 可不用设
172.16.2.0/24 #Public Network
172.16.3.0/24 #Cluster Network

#每台ceph节点下除系统盘外，挂1个32G硬盘
ceph131 eth0:172.16.1.131 eth1:172.16.2.131 eth2:172.16.3.131 1u2g
ceph132 eth0:172.16.1.132 eth1:172.16.2.132 eth2:172.16.3.132 1u2g
ceph133 eth0:172.16.1.133 eth1:172.16.2.133 eth2:172.16.3.133 1u2g

1.3 基础环境准备

1.3.1 关闭selinux、防火墙

#关闭防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
#关闭SElinux
sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config
sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config
grep --color=auto '^SELINUX' /etc/selinux/config
setenforce 0
reboot

1.3.2 设置主机名，每台设置

hostnamectl set-hostname ceph135
su -

1.3.3 设置网卡对应网卡IP（自行更改网卡名进行替换IP）

#vim /etc/sysconfig/network-scripts/ifcfg-eth0

NetName=eth0
rm -f /etc/sysconfig/network-scripts/ifcfg-$NetName
nmcli con add con-name $NetName ifname $NetName autoconnect yes type ethernet \
ip4 172.16.1.131/24 ipv4.dns "114.114.114.114" ipv4.gateway "172.16.1.254"
#设置完成后执行下reload网络
systemctl network restart

#（可选）如要指定默认路由，只要在对应的网卡配置加上以下配置，如：
#vim /etc/sysconfig/network-scripts/ifcfg-eth0

IPV4_ROUTE_METRIC=0

1.3.4 在hosts里添加对应的ceph节点信息

#vim /etc/hosts

#[ceph14]
172.16.2.131 ceph131
172.16.2.132 ceph132
172.16.2.133 ceph133

1.3.5 添加Ceph nautilus版本源

#系统yum源更换成阿里源，并更新yum文件缓存

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo 
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache

#添加nautilus源

cat << EOM > /etc/yum.repos.d/ceph.repo
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://download.ceph.com/rpm-nautilus/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
EOM

#更新应用版本

yum update -y

1.3.6 时间同步

#个人比较喜欢用以下方式来同步时间

ntpdate ntp3.aliyun.com 

echo "*/3 * * * * ntpdate ntp3.aliyun.com  &> /dev/null" > /tmp/crontab

crontab /tmp/crontab

##3# 1.3.7（可选）安装基础软件

yum install net-tools wget vim bash-completion lrzsz unzip zip -y

2 ceph安装与配置

2.1 ceph-deploy工具部署

#如果是在15版本，支持使用cephadm工具部署，ceph-deploy在14版本前都支持
#安装ceph-deploy，这里选用ceph131做为管理节点，因此ceph-deploy部署在这台上

yum install ceph-deploy -y

#这里安装的是版本是 ceph-deploy-2.0.1-0.noarch

#Ceph-deploy必须作为具有无密码sudo特权的用户登录到Ceph节点，因为它需要在不提示密码的情况下安装软件和配置文件
#在每个Ceph节点上创建ceph-deploy的用户，密码为 ceph.123

useradd -d /home/cephdeploy -m cephdeploy
passwd cephdeploy
usermod -G root cephdeploy

#对于添加到每个Ceph节点的用户，确保该用户具有sudo特权。

echo "cephdeploy ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephdeploy
chmod 0440 /etc/sudoers.d/cephdeploy

#登陆cephdeploy启用没有密码的ssh，在ceph131执行即可，三次回车！

[cephdeploy@ceph131 ~]$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:CsvXYKm8mRzasMFwgWVLx5LvvfnPrRc5S1wSb6kPytM root@ceph131
The key's randomart image is:
+---[RSA 2048]----+
|  +o.            |
| =oo.       .    |
|. oo         o . |
|   ..  .    . =  |
|. ....+ S  . *   |
| + o.=.+    O    |
|  + * oo.. + *   |
|   B *o  .+.E .  |
|  o *  ...++.    |
+----[SHA256]-----+

#将密钥复制到每个Ceph节点

ssh-copy-id cephdeploy@ceph131
ssh-copy-id cephdeploy@ceph132
ssh-copy-id cephdeploy@ceph133

#验证，不需要输入密码即为成功

[cephdeploy@ceph131 ~]$ ssh 'cephdeploy@ceph133'
Last failed login: Thu Jul  2 11:18:06 +08 2020 from 172.16.1.131 on ssh:notty
There were 2 failed login attempts since the last successful login.
Last login: Thu Jul  2 10:32:51 2020 from 172.16.1.131
[cephdeploy@ceph133 ~]$ exit
logout
Connection to ceph133 closed.

2.2 创建并配置ceph集群

2.2.1 创建ceph配置目录和集群

#创建集群目录，用于维护ceph-deploy为集群生成的配置文件和密钥。

su cephdeploy
mkdir ~/cephcluster && cd ~/cephcluster
[cephdeploy@ceph131 cephcluster]$ pwd
/home/cephdeploy/cephcluster

#注意点
#1 不要使用sudo调用ceph-deploy，如果您是以另一个用户登录，也不要以root身份运行它，因为它不会发出远程主机上所需的sudo命令。
#2 ceph-deploy会将文件输出到当前目录。如果在执行ceph-deploy时，一定要确保您在这个目录中。

#附：节点清理
#如果在任何时候你遇到麻烦，你想重新开始，执行以下清除Ceph包，并擦除其所有数据和配置:

ceph-deploy purge ceph131 ceph132 ceph133
ceph-deploy purgedata ceph131 ceph132 ceph133
ceph-deploy forgetkeys
rm ceph.*

#在cephcluster目录下执行创建集群

[cephdeploy@ceph131 cephcluster]$ ceph-deploy new ceph131 ceph132 ceph133
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephdeploy/.cephdeploy.conf
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /bin/ceph-deploy new ceph131 ceph132 ceph133
[ceph_deploy.cli][INFO  ] ceph-deploy options:
[ceph_deploy.cli][INFO  ]  username                      : None
[ceph_deploy.cli][INFO  ]  func                          : <function new at 0x7f9f54674e60>
[ceph_deploy.cli][INFO  ]  verbose                       : False
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
[ceph_deploy.cli][INFO  ]  quiet                         : False
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f9f541fa9e0>
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
[ceph_deploy.cli][INFO  ]  ssh_copykey                   : True
[ceph_deploy.cli][INFO  ]  mon                           : ['ceph131', 'ceph132', 'ceph133']
[ceph_deploy.cli][INFO  ]  public_network                : None
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
[ceph_deploy.cli][INFO  ]  cluster_network               : None
[ceph_deploy.cli][INFO  ]  default_release               : False
[ceph_deploy.cli][INFO  ]  fsid                          : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO  ] making sure passwordless SSH succeeds
[ceph131][DEBUG ] connection detected need for sudo
[ceph131][DEBUG ] connected to host: ceph131
[ceph131][DEBUG ] detect platform information from remote host
[ceph131][DEBUG ] detect machine type
[ceph131][DEBUG ] find the location of an executable
[ceph131][INFO  ] Running command: sudo /usr/sbin/ip link show
[ceph131][INFO  ] Running command: sudo /usr/sbin/ip addr show
[ceph131][DEBUG ] IP addresses found: [u'172.16.1.131', u'172.16.2.131', u'172.16.3.131']
[ceph_deploy.new][DEBUG ] Resolving host ceph131
[ceph_deploy.new][DEBUG ] Monitor ceph131 at 172.16.1.131
[ceph_deploy.new][INFO  ] making sure passwordless SSH succeeds
[ceph132][DEBUG ] connected to host: ceph131
[ceph132][INFO  ] Running command: ssh -CT -o BatchMode=yes ceph132
[ceph132][DEBUG ] connection detected need for sudo
[ceph132][DEBUG ] connected to host: ceph132
[ceph132][DEBUG ] detect platform information from remote host
[ceph132][DEBUG ] detect machine type
[ceph132][DEBUG ] find the location of an executable
[ceph132][INFO  ] Running command: sudo /usr/sbin/ip link show
[ceph132][INFO  ] Running command: sudo /usr/sbin/ip addr show
[ceph132][DEBUG ] IP addresses found: [u'172.16.2.132', u'172.16.1.132', u'172.16.3.132']
[ceph_deploy.new][DEBUG ] Resolving host ceph132
[ceph_deploy.new][DEBUG ] Monitor ceph132 at 172.16.1.132
[ceph_deploy.new][INFO  ] making sure passwordless SSH succeeds
[ceph133][DEBUG ] connected to host: ceph131
[ceph133][INFO  ] Running command: ssh -CT -o BatchMode=yes ceph133
[ceph133][DEBUG ] connection detected need for sudo
[ceph133][DEBUG ] connected to host: ceph133
[ceph133][DEBUG ] detect platform information from remote host
[ceph133][DEBUG ] detect machine type
[ceph133][DEBUG ] find the location of an executable
[ceph133][INFO  ] Running command: sudo /usr/sbin/ip link show
[ceph133][INFO  ] Running command: sudo /usr/sbin/ip addr show
[ceph133][DEBUG ] IP addresses found: [u'172.16.2.133', u'172.16.1.133', u'172.16.3.133']
[ceph_deploy.new][DEBUG ] Resolving host ceph133
[ceph_deploy.new][DEBUG ] Monitor ceph133 at 172.16.1.133
[ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph131', 'ceph132', 'ceph133']
[ceph_deploy.new][DEBUG ] Monitor addrs are ['172.16.1.131', '172.16.1.132', '172.16.1.133']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...

2.2.2 ceph配置文件修改

#创建公共访问的网络
#vim /root/cephcluster/ceph.conf

[global]
fsid = 76235629-6feb-4f0c-a106-4be33d485535
mon_initial_members = ceph131, ceph132, ceph133
mon_host = 172.16.1.131,172.16.1.132,172.16.1.133
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public_network = 172.16.1.0/24
cluster_network = 172.16.2.0/24
#设置副本数
osd_pool_default_size = 3
#设置最小副本数
osd_pool_default_min_size = 2
#设置时钟偏移0.5s
mon_clock_drift_allowed = .50

2.2.3 各节点安装基础包

#指定源安装ceph安装包，一定要在目录里执行
#输出无报错即可

ceph-deploy install --repo-url https://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-nautilus/el7/ --gpg-url https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc ceph131 ceph132 ceph133

2.2.4 部署初始mon并生成密钥:

#初始化mon，执行过程无报错即可

ceph-deploy mon create-initial

#这时候目录底下应该会多以下的key

[cephdeploy@ceph131 cephcluster]$ ll
total 220
-rw------- 1 cephdeploy cephdeploy    113 Jul  2 14:16 ceph.bootstrap-mds.keyring
-rw------- 1 cephdeploy cephdeploy    113 Jul  2 14:16 ceph.bootstrap-mgr.keyring
-rw------- 1 cephdeploy cephdeploy    113 Jul  2 14:16 ceph.bootstrap-osd.keyring
-rw------- 1 cephdeploy cephdeploy    113 Jul  2 14:16 ceph.bootstrap-rgw.keyring
-rw------- 1 cephdeploy cephdeploy    151 Jul  2 14:16 ceph.client.admin.keyring
-rw-rw-r-- 1 cephdeploy cephdeploy    453 Jul  2 14:05 ceph.conf
-rw-rw-r-- 1 cephdeploy cephdeploy 177385 Jul  2 14:16 ceph-deploy-ceph.log
-rw------- 1 cephdeploy cephdeploy     73 Jul  2 14:03 ceph.mon.keyring

#使用Ceph -deploy将配置文件和管理密钥复制到您的管理节点和Ceph节点

[cephdeploy@ceph131 cephcluster]$ ceph-deploy admin ceph131 ceph132 ceph133
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephdeploy/.cephdeploy.conf
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /bin/ceph-deploy admin ceph131 ceph132 ceph133
[ceph_deploy.cli][INFO  ] ceph-deploy options:
[ceph_deploy.cli][INFO  ]  username                      : None
[ceph_deploy.cli][INFO  ]  verbose                       : False
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
[ceph_deploy.cli][INFO  ]  quiet                         : False
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7ff46c014638>
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
[ceph_deploy.cli][INFO  ]  client                        : ['ceph131', 'ceph132', 'ceph133']
[ceph_deploy.cli][INFO  ]  func                          : <function admin at 0x7ff46c8b52a8>
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
[ceph_deploy.cli][INFO  ]  default_release               : False
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph131
[ceph131][DEBUG ] connection detected need for sudo
[ceph131][DEBUG ] connected to host: ceph131
[ceph131][DEBUG ] detect platform information from remote host
[ceph131][DEBUG ] detect machine type
[ceph131][DEBUG ] write cluster configuration to /etc/ceph/{
     cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph132
[ceph132][DEBUG ] connection detected need for sudo
[ceph132][DEBUG ] connected to host: ceph132
[ceph132][DEBUG ] detect platform information from remote host
[ceph132][DEBUG ] detect machine type
[ceph132][DEBUG ] write cluster configuration to /etc/ceph/{
     cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph133
[ceph133][DEBUG ] connection detected need for sudo
[ceph133][DEBUG ] connected to host: ceph133
[ceph133][DEBUG ] detect platform information from remote host
[ceph133][DEBUG ] detect machine type
[ceph133][DEBUG ] write cluster configuration to /etc/ceph/{
     cluster}.conf

2.2.5 部署mgr服务

#创建mgr，执行过程无报错即可

ceph-deploy mgr create ceph131 ceph132 ceph133

2.2.6 添加OSD

#创建osd，执行过程无报错即可

ceph-deploy osd create --data /dev/sdb ceph131
ceph-deploy osd create --data /dev/sdb ceph132
ceph-deploy osd create --data /dev/sdb ceph133

2.2.7 验证集群状态

[root@ceph132 ~]# ceph osd tree
ID CLASS WEIGHT  TYPE NAME        STATUS REWEIGHT PRI-AFF
-1       0.09357 root default
-3       0.03119     host ceph131
 0   hdd 0.03119         osd.0        up  1.00000 1.00000
-5       0.03119     host ceph132
 1   hdd 0.03119         osd.1        up  1.00000 1.00000
-7       0.03119     host ceph133
 2   hdd 0.03119         osd.2        up  1.00000 1.00000

3 扩展集群的服务 - 请根据需求开启

#所有的ceph-deploy操作都必须是使用cephdeploy用户并在配置文件目录底下执行！！！

[cephdeploy@ceph131 cephcluster]$ pwd
/home/cephdeploy/cephcluster

3.1 添加元数据服务器（mds）

ceph-deploy mds create ceph132 

3.2 添加监控端（mon）

ceph-deploy mon add ceph132 ceph133

#一旦您添加了新的Ceph监视器，Ceph将开始同步监视器并形成一个quorum。您可以通过执行以下操作来检查仲裁状态:

ceph quorum_status --format json-pretty

3.3 增加守护进程（mgr）

#Ceph Manager守护进程以活动/备用模式操作。部署其他管理器守护进程可以确保，如果一个守护进程或主机失败，另一个守护进程可以接管，而不会中断服务。

ceph-deploy mgr create ceph132 ceph133

#验证

ceph -s

3.4 增加对象存储网关（rgw）

ceph-deploy rgw create ceph131

#默认情况下，RGW实例将监听端口7480。这可以通过在运行RGW的节点上编辑ceph.conf来改变，如下所示:

[client]
rgw frontends = civetweb port=80

#修改完端口需要重启服务

systemctl restart ceph-radosgw.service

4 存储部署

4.1 启用CephFS

4.1.1 请确认至少有一台节点启用了mds服务，pg数是有算法的，可以使用官网计算器去计算！

PG数量的预估 集群中单个池的PG数计算公式如下：PG 总数 = (OSD 数 * 100) / 最大副本数 / 池数 (结果必须舍入到最接近2的N次幂的值)
#在ceph集群，其中一台执行命令即可，这里用ceph132

[root@ceph132 ~]# ceph osd pool create cephfs_data 16
pool 'cephfs_data' created
[root@ceph132 ~]# ceph osd pool create cephfs_metadata 16
pool 'cephfs_metadata' created
[root@ceph132 ~]# ceph fs new cephfs_storage cephfs_metadata cephfs_data
new fs with metadata pool 2 and data pool 1
[root@ceph132 ~]# ceph df
RAW STORAGE:
    CLASS     SIZE       AVAIL      USED       RAW USED     %RAW USED
    hdd       96 GiB     93 GiB     11 MiB      3.0 GiB          3.14
    TOTAL     96 GiB     93 GiB     11 MiB      3.0 GiB          3.14

POOLS:
    POOL                ID     STORED      OBJECTS     USED        %USED     MAX AVAIL
    cephfs_data          1         0 B           0         0 B         0        29 GiB
    cephfs_metadata      2     2.2 KiB          22     1.5 MiB         0        29 GiB

4.1.2 挂载cephfs

#在客户端创建密码文件

[root@ceph133 ~]# cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
	key = AQC5e/1eXm9WExAAmlD9aZoc2dZO6jbU8UXSqg==
	caps mds = "allow *"
	caps mgr = "allow *"
	caps mon = "allow *"
	caps osd = "allow *"
[root@ceph133 ~]# vim admin.secret
[root@ceph133 ~]# ll
total 8
-rw-r--r--  1 root root   41 Jul  2 15:17 admin.secret

#挂载cephfs文件夹，并验证

[root@ceph133 ~]# mkdir /mnt/cephfs_storage
[root@ceph133 ~]# mount -t ceph 172.16.1.133:6789:/ /mnt/cephfs_storage -o name=admin,secretfile=admin.secret
[root@ceph133 ~]# df -Th
Filesystem          Type      Size  Used Avail Use% Mounted on
devtmpfs            devtmpfs  983M     0  983M   0% /dev
tmpfs               tmpfs     995M     0  995M   0% /dev/shm
tmpfs               tmpfs     995M  8.6M  987M   1% /run
tmpfs               tmpfs     995M     0  995M   0% /sys/fs/cgroup
/dev/sda1           xfs        20G  2.2G   18G  11% /
tmpfs               tmpfs     199M     0  199M   0% /run/user/0
tmpfs               tmpfs     995M   52K  995M   1% /var/lib/ceph/osd/ceph-2
172.16.1.133:6789:/ ceph       30G     0   30G   0% /mnt/cephfs_storage

4.2 启用块存储

4.2.1 在ceph集群，其中一台执行命令即可，这里用ceph133

#初始化rbd池
PG数量的预估 集群中单个池的PG数计算公式如下：PG 总数 = (OSD 数 * 100) / 最大副本数 / 池数 (结果必须舍入到最接近2的N次幂的值)

[root@ceph132 ~]# ceph osd pool create rbd_storage 16 16 replicated
pool 'rbd_storage' created

#创建一个块设备

[root@ceph132 ~]# rbd create --size 1024 rbd_image -p rbd_storage
[root@ceph132 ~]# rbd ls rbd_storage
rbd_image

#删除命令

rbd rm rbd_storage/rbd_image

#附rbd池的相关操作命令

4.2.2 挂载rbd块设备

#将块设备映射到系统内核

[root@ceph133 ~]# rbd map rbd_storage/rbd_image
/dev/rbd0
[root@ceph133 ~]# lsblk
NAME                                                                                                  MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
rbd0                                                                                                  252:0    0   1G  0 disk
sdb                                                                                                     8:16   0  32G  0 disk
└─ceph--376ebd83--adf0--4ee1--b7e4--b4b79bae048e-osd--block--4b0444fa--535e--40c7--b55a--167ab21dbf9b 253:0    0  32G  0 lvm
sr0                                                                                                    11:0    1   4M  0 rom
sda                                                                                                     8:0    0  20G  0 disk
└─sda1                                                                                                  8:1    0  20G  0 part /

#格式化rbd设备

[root@ceph133 ~]# mkfs.ext4 -m0 /dev/rbd/rbd_storage/rbd_image
mke2fs 1.42.9 (28-Dec-2013)
Discarding device blocks: done
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=16 blocks, Stripe width=16 blocks
65536 inodes, 262144 blocks
0 blocks (0.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=268435456
8 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
	32768, 98304, 163840, 229376

Allocating group tables: done
Writing inode tables: done
Creating journal (8192 blocks): done
Writing superblocks and filesystem accounting information: done

#挂载rbd设备

[root@ceph133 ~]# mkdir /mnt/rbd_storage
[root@ceph133 ~]# mount /dev/rbd/rbd_storage/rbd_image /mnt/rbd_storage
[root@ceph133 ~]# df -Th
Filesystem          Type      Size  Used Avail Use% Mounted on
devtmpfs            devtmpfs  983M     0  983M   0% /dev
tmpfs               tmpfs     995M     0  995M   0% /dev/shm
tmpfs               tmpfs     995M  8.6M  987M   1% /run
tmpfs               tmpfs     995M     0  995M   0% /sys/fs/cgroup
/dev/sda1           xfs        20G  2.2G   18G  11% /
tmpfs               tmpfs     199M     0  199M   0% /run/user/0
tmpfs               tmpfs     995M   52K  995M   1% /var/lib/ceph/osd/ceph-2
172.16.1.133:6789:/ ceph       30G     0   30G   0% /mnt/cephfs_storage
/dev/rbd0           ext4      976M  2.6M  958M   1% /mnt/rbd_storage

#取消内核挂载

rbd unmap /dev/rbd0

4.3 启用RGW对象存储

4.3.1 请确认至少有一台节点启用了rgw服务

#启用rgw见3.4
#使用浏览器，查看 http://172.16.1.131:7480/，说明已经启用成功

5 启用ceph dashboard

5.1 启用dashboard

#Ceph仪表板是一个内置的基于web的Ceph管理和监视应用程序，用于管理集群。
#节点默认没有安装mgr-dashboard程序，因此要先安装

yum install ceph-mgr-dashboard -y

#任一节点启用dashboard，没有提示

[root@ceph132 ~]# ceph mgr module enable dashboard

#配置登陆认证

[root@ceph132 ~]# ceph dashboard create-self-signed-cert
Self-signed certificate created

#配置登陆账户

[root@ceph132 ~]# ceph dashboard ac-user-create admin admin.123 administrator
{
     "username": "admin", "lastUpdate": 1593679011, "name": null, "roles": ["administrator"], "password": "$2b$12$kQYtMXun1jKdTDKwjfuNj.WYyJcr3vSHLTMWfXIi.wrKkFRCmmC1.", "email": null}

#测试登陆，浏览器查看：https://172.16.1.131:8443/ 用户名：admin 密码：admin.123

[root@ceph132 ~]# ceph mgr services
{
     
    "dashboard": "https://ceph131:8443/"
}

5.2 Dashboard中启用RGW

#创建rgw的用户

[root@ceph131 ~]# radosgw-admin user create --uid=rgwadmin --display-name=rgwadmin --system
{
     
    "user_id": "rgwadmin",
    "display_name": "rgwadmin",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
     
            "user": "rgwadmin",
            "access_key": "VHK8BZYA3F5BDWMJFUER",
            "secret_key": "WVBdt66ZbmGe0l6Wu3LoPKM1GlzF6V35JCpNKPJw"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "system": "true",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
     
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
     
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

#设置凭证，取创建用户生成的key

[root@ceph131 ~]# ceph dashboard set-rgw-api-access-key VHK8BZYA3F5BDWMJFUER
Option RGW_API_ACCESS_KEY updated
[root@ceph131 ~]# ceph dashboard set-rgw-api-secret-key WVBdt66ZbmGe0l6Wu3LoPKM1GlzF6V35JCpNKPJw
Option RGW_API_SECRET_KEY updated

#禁用SSL

[root@ceph131 ~]# ceph dashboard set-rgw-api-ssl-verify False
Option RGW_API_SSL_VERIFY updated

#启用rgw的dashboard

[root@ceph131 ~]# ceph dashboard set-rgw-api-host 172.16.1.131
Option RGW_API_HOST updated
[root@ceph131 ~]# ceph dashboard set-rgw-api-port 7480
Option RGW_API_PORT updated
[root@ceph131 ~]# ceph dashboard set-rgw-api-scheme http
Option RGW_API_SCHEME updated
[root@ceph131 ~]# ceph dashboard set-rgw-api-admin-resource admin
Option RGW_API_ADMIN_RESOURCE updated
[root@ceph131 ~]# ceph dashboard set-rgw-api-user-id rgwadmin
Option RGW_API_USER_ID updated
[root@ceph131 ~]# systemctl restart ceph-radosgw.target

X.部署过程遇到的问题

eg1.[root@ceph131 cephcluster]# ceph-deploy mon create-initial
[ceph131][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 5
[ceph_deploy.mon][WARNIN] waiting 5 seconds before retrying
[ceph131][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 4
[ceph_deploy.mon][WARNIN] waiting 10 seconds before retrying
[ceph131][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 3
[ceph_deploy.mon][WARNIN] waiting 10 seconds before retrying
[ceph131][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 2
[ceph_deploy.mon][WARNIN] waiting 15 seconds before retrying
[ceph131][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 1
[ceph_deploy.mon][WARNIN] waiting 20 seconds before retrying
[ceph_deploy.mon][ERROR ] Some monitors have still not reached quorum:
[ceph_deploy.mon][ERROR ] ceph131
解决方案：
1 查看ceph.com的配置，public_network 或者mon_host 是否配置正确,
2 查看hosts配置是不是有增加了ipv6的解析，直接去掉即可再重新部署

eg2.[ceph_deploy][ERROR ] IOError: [Errno 13] Permission denied: '/root/cephcluster/ceph-deploy-ceph.log'
解决方案：
[root@ceph131 cephcluster]# echo "cephdeploy ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephdeploy
cephdeploy ALL = (root) NOPASSWD:ALL
[root@ceph131 cephcluster]# chmod 0440 /etc/sudoers.d/cephdeploy
[root@ceph131 cephcluster]# cat /etc/sudoers.d/cephdeploy
cephdeploy ALL = (root) NOPASSWD:ALL
[root@ceph131 cephcluster]# usermod -G root cephdeploy

eg3.[root@ceph131 ~]# ceph -s
  cluster:
    id:     76235629-6feb-4f0c-a106-4be33d485535
    health: HEALTH_WARN
            clock skew detected on mon.ceph133
原因是：时钟偏移问题
解决方案：
1 在deploy节点的部署目录底下修改ceph.conf文件
[cephdeploy@ceph131 cephcluster]$ pwd
/home/cephdeploy/cephcluster
2 编辑ceph.conf，并添加以下字段保存退出
#设置时钟偏移
mon clock drift allowed = 2
mon clock drift warn backoff = 30
3 重新下发配置
[cephdeploy@ceph131 cephcluster]$ ceph-deploy --overwrite-conf config push ceph{
     131..133}
4 至对应节点重启mon
[root@ceph133 ~]# systemctl restart [email protected]

eg4.[root@ceph132 ~]# ceph mgr module enable dashboard
Error ENOENT: all mgr daemons do not support module 'dashboard', pass --force to force enablement
原因是：节点没有安装ceph-mgr-dashboard，在mgr的节点上安装。
yum install ceph-mgr-dashboard

eg5.[root@ceph133 ~]# ceph -s
  cluster:
    id:     76235629-6feb-4f0c-a106-4be33d485535
    health: HEALTH_WARN
            application not enabled on 1 pool(s)
[root@ceph133 ~]# ceph health detail
HEALTH_WARN application not enabled on 1 pool(s)
POOL_APP_NOT_ENABLED application not enabled on 1 pool(s)
    application not enabled on pool 'rbd_storage'
    use 'ceph osd pool application enable  ', where <app-name> is 'cephfs', 'rbd', 'rgw', or freeform for custom applications.
解决方案：
ceph osd pool application enable rbd_storage rdb