Ceph实战入门系列（一）——三节点Ceph集群的安装与部署

配置环境

环境说明

• 在Win7下安装vmware软件，用CentOS-7.2创建一台centos7.2-mini虚拟机，安装完后把内存改为512MB就足够了，如果是用来实践的话。
• 基于上面的虚拟机链接克隆3台虚拟机

修改hostname

# 分别在3个host上执行
[root@localhost ~]# hostnamectl set-hostname ceph6
[root@localhost ~]# hostnamectl set-hostname ceph7
[root@localhost ~]# hostnamectl set-hostname ceph8

• 1
• 2
• 3
• 4

配置解析

# 全体都有,并测试网络是否连通
[root@ceph8 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.103.139 ceph6
192.168.103.140 ceph7
192.168.103.138 ceph8

• 1
• 2
• 3
• 4
• 5
• 6
• 7

配置防火墙启用端口

# 全体都有
[root@ceph6 ~]# firewall-cmd --zone=public --add-port=6789/tcp --permanent 
success
[root@ceph6 ~]# firewall-cmd --zone=public --add-port=6800-7100/tcp --permanent 
success
[root@ceph6 ~]# firewall-cmd --reload
success
[root@ceph6 ~]# firewall-cmd --zone=public --list-all 
public (default, active)
  interfaces: eno16777736
  sources: 
  services: dhcpv6-client ssh
  ports: 6789/tcp 6800-7100/tcp
  masquerade: no
  forward-ports: 
  icmp-blocks: 
  rich rules: 

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18

禁用SELINUX

# 全体都有
[root@ceph6 ~]# setenforce 0
[root@ceph6 ~]# vim /etc/selinux/config 
[root@ceph6 ~]# cat /etc/selinux/config | grep disabled
#     disabled - No SELinux policy is loaded.
SELINUX=disabled

• 1
• 2
• 3
• 4
• 5
• 6

配置网络

# 全体都有；公司内部开发网需要单独设置
[root@ceph6 ~]# cat /etc/yum.conf 
[main]
cachedir=/var/cache/yum/$basearch/$releasever
keepcache=0
debuglevel=2
logfile=/var/log/yum.log
exactarch=1
obsoletes=1
gpgcheck=1
plugins=1
installonly_limit=5
bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum
distroverpkg=centos-release
Proxy=http://dev-proxy.oa.com:8080/

[root@ceph6 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.14.87.100 dev-proxy.oa.com

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21

设置全局代理

# 全体都有；公司网络限制，需要设置全局代理，需要重启主机（poweroff命令，再开机）
http_proxy=dev-proxy.oa.com:8080
https_proxy=dev-proxy.oa.com:8080
export http_proxy https_proxy

• 1
• 2
• 3
• 4

安装配置ntp服务

# 全体都有， 10.14.0.131是公司的ntp server
[root@ceph6 ~]# cat /etc/ntp.conf | grep server | grep -v '#'
server 10.14.0.131

• 1
• 2
• 3
• 4

wget下载rpm；代理

# wget代理下载rpm，发现不能install，不知道原因
[root@ceph6 ~]# wget http://dev-proxy.oa.com:8080/ http://download.ceph.com/rpm-giant/el7/noarch/ceph-release-1-0.el7.noarch.rpm -e http_proxy=dev-proxy.oa.com:8080/

• 1
• 2

Ceph安装与部署

安装部署工具

[root@ceph6 ~]# yum install -y ceph-deploy

• 1
• 2

初始化monitor

# 在ceph6上操作
[root@ceph6 ~]# mkdir /etc/ceph
[root@ceph6 ~]# cd /etc/ceph/
[root@ceph6 ceph]# ceph-deploy new ceph6
[root@ceph6 ceph]# ls
ceph.conf  ceph.log  ceph.mon.keyring

• 1
• 2
• 3
• 4
• 5
• 6

免密码登录

# 在ceph6上操作
[root@ceph6 ~]# ssh-keygen 
[root@ceph6 ~]# ssh-copy-id root@ceph7
[root@ceph6 ~]# ssh-copy-id root@ceph8

• 1
• 2
• 3
• 4

远程安装部署

# 网络问题;通过第一步的设置全局代理并未解决
[root@ceph6 ~]# ceph-deploy install ceph6 ceph7 ceph8
# 省略
[ceph6][DEBUG ] 
[ceph6][DEBUG ] Complete!
[ceph6][DEBUG ] Configure Yum priorities to include obsoletes
[ceph6][WARNIN] check_obsoletes has been enabled for Yum priorities plugin
[ceph6][INFO  ] Running command: rpm --import https://download.ceph.com/keys/release.asc
[ceph6][WARNIN] curl: (7) Failed to connect to 2607:f298:6050:51f3:f816:3eff:fe71:9135: Network is unreachable
[ceph6][WARNIN] error: https://download.ceph.com/keys/release.asc: import read failed(2).
[ceph6][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy][ERROR ] RuntimeError: Failed to execute command: rpm --import https://download.ceph.com/keys/release.asc

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12

独立安装部署

[root@ceph7 ~]# ceph-deploy install ceph7
[root@ceph8 ~]# ceph-deploy install ceph8

• 1
• 2

搭建集群

创建OSD

# 列出ceph6上所有的可用磁盘
[root@ceph6 ceph]# ceph-deploy disk list ceph6
# disk zap 子命令删除现有分区表和磁盘内容
[root@ceph6 ceph]# ceph-deploy disk zap ceph6:sdb ceph6:sdc ceph6:sdd
[root@ceph6 ceph]# ceph-deploy disk zap ceph7:sdb ceph7:sdc ceph7:sdd
[root@ceph6 ceph]# ceph-deploy disk zap ceph8:sdb ceph8:sdc ceph8:sdd
# osd create 子命令首先准备磁盘（用xfs文件系统格式化磁盘），然后激活磁盘的第一、第二个分区，作为数据分区和日志分区
[root@ceph6 ceph]# ceph-deploy osd create ceph6:sdb ceph6:sdc ceph6:sdd
[root@ceph6 ceph]# ceph-deploy osd create ceph7:sdb ceph7:sdc ceph7:sdd
[root@ceph6 ceph]# ceph-deploy osd create ceph8:sdb ceph8:sdc ceph8:sdd
# 检查ceph的状态
[root@ceph6 ceph]# ceph -s
    cluster 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
     health HEALTH_WARN
            too few PGs per OSD (21 < min 30)
     monmap e1: 1 mons at {ceph6=192.168.103.139:6789/0}
            election epoch 3, quorum 0 ceph6
     osdmap e43: 9 osds: 9 up, 9 in
            flags sortbitwise
      pgmap v95: 64 pgs, 1 pools, 0 bytes data, 0 objects
            307 MB used, 134 GB / 134 GB avail
                  64 active+clean
# 解决HEALTH_WARN
[root@ceph6 ceph]# ceph osd pool set rbd pg_num 128
[root@ceph6 ceph]# ceph osd pool set rbd pg_num 128
[root@ceph6 ceph]# ceph -s
    cluster 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
     health HEALTH_OK
     monmap e1: 1 mons at {ceph6=192.168.103.139:6789/0}
            election epoch 3, quorum 0 ceph6
     osdmap e48: 9 osds: 9 up, 9 in
            flags sortbitwise
      pgmap v130: 128 pgs, 1 pools, 0 bytes data, 0 objects
            309 MB used, 134 GB / 134 GB avail
                 128 active+clean

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36

部署奇数个数的monitor

# 将公共网络地址添加到配置文件
[root@ceph6 ceph]# cat /etc/ceph/ceph.conf 
[global]
fsid = 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
mon_initial_members = ceph6,ceph7,ceph8
mon_host = 192.168.103.139,192.168.103.140,192.168.103.138
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public network = 192.168.103.0/24

# 使用ceph-deploy命令在ceph7和ceph8上创建一个monitor
[root@ceph6 ceph]# ceph-deploy mon create ceph7 --overwrite-conf
[root@ceph6 ceph]# ceph-deploy --overwrite-conf mon create ceph8

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14

实践使用ceph集群

检查Ceph的状态

[root@ceph6 ceph]# ceph -s
    cluster 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
     health HEALTH_OK
     monmap e3: 3 mons at {ceph6=192.168.103.139:6789/0,ceph7=192.168.103.140:6789/0,ceph8=192.168.103.138:6789/0}
            election epoch 8, quorum 0,1,2 ceph8,ceph6,ceph7
     osdmap e48: 9 osds: 9 up, 9 in
            flags sortbitwise
      pgmap v130: 128 pgs, 1 pools, 0 bytes data, 0 objects
            309 MB used, 134 GB / 134 GB avail
                 128 active+clean
[root@ceph6 ceph]# ceph status
    cluster 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
     health HEALTH_OK
     monmap e3: 3 mons at {ceph6=192.168.103.139:6789/0,ceph7=192.168.103.140:6789/0,ceph8=192.168.103.138:6789/0}
            election epoch 8, quorum 0,1,2 ceph8,ceph6,ceph7
     osdmap e48: 9 osds: 9 up, 9 in
            flags sortbitwise
      pgmap v130: 128 pgs, 1 pools, 0 bytes data, 0 objects
            309 MB used, 134 GB / 134 GB avail
                 128 active+clean

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21

观察集群健康状态

[root@ceph6 ceph]# ceph -w
    cluster 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
     health HEALTH_OK
     monmap e3: 3 mons at {ceph6=192.168.103.139:6789/0,ceph7=192.168.103.140:6789/0,ceph8=192.168.103.138:6789/0}
            election epoch 8, quorum 0,1,2 ceph8,ceph6,ceph7
     osdmap e48: 9 osds: 9 up, 9 in
            flags sortbitwise
      pgmap v130: 128 pgs, 1 pools, 0 bytes data, 0 objects
            309 MB used, 134 GB / 134 GB avail
                 128 active+clean

2016-11-09 10:01:39.675477 mon.0 [INF] from='client.4174 :/0' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.bootstrap-mds", "caps": ["mon", "allow profile bootstrap-mds"]}]: dispatch

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13

检查Ceph monitor仲裁状态

[root@ceph6 ceph]# ceph quorum_status --format json-pretty

{
    "election_epoch": 8,
    "quorum": [
        0,
        1,
        2
    ],
    "quorum_names": [
        "ceph8",
        "ceph6",
        "ceph7"
    ],
    "quorum_leader_name": "ceph8",
    "monmap": {
        "epoch": 3,
        "fsid": "8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5",
        "modified": "2016-11-09 10:01:31.732730",
        "created": "2016-11-08 22:36:48.791105",
        "mons": [
            {
                "rank": 0,
                "name": "ceph8",
                "addr": "192.168.103.138:6789\/0"
            },
            {
                "rank": 1,
                "name": "ceph6",
                "addr": "192.168.103.139:6789\/0"
            },
            {
                "rank": 2,
                "name": "ceph7",
                "addr": "192.168.103.140:6789\/0"
            }
        ]
    }
}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40

导出Ceph monitor信息

[root@ceph6 ceph]# ceph mon dump
dumped monmap epoch 3
epoch 3
fsid 8ea4fa79-3b6b-4de3-8cfb-a0922d6827c5
last_changed 2016-11-09 10:01:31.732730
created 2016-11-08 22:36:48.791105
0: 192.168.103.138:6789/0 mon.ceph8
1: 192.168.103.139:6789/0 mon.ceph6
2: 192.168.103.140:6789/0 mon.ceph7

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10

检查集群使用状态

[root@ceph6 ceph]# ceph df
GLOBAL:
    SIZE     AVAIL     RAW USED     %RAW USED 
    134G      134G         309M          0.22 
POOLS:
    NAME     ID     USED     %USED     MAX AVAIL     OBJECTS 
    rbd      0         0         0        45942M           0 

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8

检查Ceph monitor、OSD和PG（配置组）状态

[root@ceph6 ceph]# ceph mon stat
e3: 3 mons at {ceph6=192.168.103.139:6789/0,ceph7=192.168.103.140:6789/0,ceph8=192.168.103.138:6789/0}, election epoch 8, quorum 0,1,2 ceph8,ceph6,ceph7
[root@ceph6 ceph]# ceph osd stat
     osdmap e48: 9 osds: 9 up, 9 in
            flags sortbitwise
[root@ceph6 ceph]# ceph pg stat
v130: 128 pgs: 128 active+clean; 0 bytes data, 309 MB used, 134 GB / 134 GB avail

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8

列表PG

[root@ceph6 ceph]# ceph pg dump

• 1

列表Ceph存储池

[root@ceph6 ceph]# ceph osd lspools
0 rbd,

• 1
• 2
• 3

检查OSD的CRUSH map

[root@ceph6 ceph]# ceph osd tree
ID WEIGHT  TYPE NAME      UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 0.13129 root default                                     
-2 0.04376     host ceph6                                   
 0 0.01459         osd.0       up  1.00000          1.00000 
 1 0.01459         osd.1       up  1.00000          1.00000 
 2 0.01459         osd.2       up  1.00000          1.00000 
-3 0.04376     host ceph7                                   
 3 0.01459         osd.3       up  1.00000          1.00000 
 4 0.01459         osd.4       up  1.00000          1.00000 
 5 0.01459         osd.5       up  1.00000          1.00000 
-4 0.04376     host ceph8                                   
 6 0.01459         osd.6       up  1.00000          1.00000 
 7 0.01459         osd.7       up  1.00000          1.00000 
 8 0.01459         osd.8       up  1.00000          1.00000 

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16

列表集群的认证密钥

[root@ceph6 ceph]# ceph auth list