通过ceph-ansible安装ceph

作者：吴业亮

博客：wuyeliang.blog.csdn.net

一、通用配置
1、修改主机名

hostnamectl set-hostname  node

2、关闭selinux和firewalld

setenforce 0
sed -i  "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
systemctl disable firewalld.service
systemctl stop firewalld.service

3、配置源

yum -y install epel-release centos-release-ceph-nautilus centos-release-openstack-stein

4、配置hosts文件

cat <<"EOF">/etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.80.90 node1
192.168.80.91 node2
192.168.80.92 node3
192.168.80.93 node4
192.168.80.94 node5
192.168.80.95 node6
EOF

二、配置ceph-absible节点

1、安装ceph-ansible

yum -y install ceph-ansible

2、配置变量
使用阿里源

# cat  /usr/share/ceph-ansible/group_vars/all.yml | grep -v ^# 
ceph_origin: repository
ceph_repository: community
ceph_mirror: http://mirrors.aliyun.com/ceph
ceph_stable_key: http://mirrors.aliyun.com/ceph/keys/release.asc
ceph_stable_release: nautilus
ceph_stable_repo: "{{ ceph_mirror }}/rpm-{{ ceph_stable_release }}"
public_network: 192.168.80.0/24
cluster_network: "{{ public_network }}"
monitor_interface: eth0

使用默认源

# cat  /usr/share/ceph-ansible/group_vars/all.yml | grep -v ^# 
ceph_origin: repository
ceph_repository: community
ceph_repository_type: cdn
ceph_stable_release: nautilus
fetch_directory: ~/ceph-ansible-keys
monitor_interface: eth0
public_network: 192.168.80.0/24
cluster_network: "{{ public_network }}"

配置osd

# cat /usr/share/ceph-ansible/group_vars/osds.yml
devices:
  - /dev/vdb
  - /dev/vdc
  - /dev/vdd
  - /dev/vde
  - /dev/vdf
  - /dev/vdg
  - /dev/vdh

配置主机

# cat  /etc/ansible/hosts | grep -v ^# | grep -v ^$
[all:vars]
ansible_connection=ssh
ansible_ssh_pass='Changeme_123'
dashboard_admin_password='Changeme_123' 
grafana_admin_password='Changeme_123'
[mons]
node1
node2
node3
[mgrs]
node1
node2
node3
[osds]
node1
node2
node3
node4
node5
node6
[grafana_server_group_name] 
node1
[grafana-server]
node1
[rgws]
node1
node2
node3
[mdss]
node1
node2
node3

# cd /usr/share/ceph-ansible
# cp -p site.yml.sample site.yml

注释site.yml一些组件

- hosts:
  - mons
  #- agents
  - osds
  #- mdss
  #- rgws
  #- nfss
  #- restapis
  #- rbdmirrors
  #- clients
  - mgrs
  #- iscsigws
  #- iscsi-gws # for backward compatibility only!

新建/usr/share/ceph-ansible/group_vars/rgws.yml，内容如下

radosgw_interface: eth0

开始安装

# cd /usr/share/ceph-ansible
# ansible-playbook site.yml

三、服务应用

1、安装完成登录web

# ceph mgr services
{
    "dashboard": "http://192.168.80.91:8443/",
    "prometheus": "http://node2:9283/"
}

登录web

http://192.168.80.91:8443/
admin/Changeme_123

2、使用cephfs
1）、查看状态

# ceph fs ls
name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]

# ceph mds stat
cephfs:1 {0=node2=up:active} 2 up:standby

# ceph fs status cephfs
cephfs - 0 clients
======
+------+--------+-------+---------------+-------+-------+
| Rank | State  |  MDS  |    Activity   |  dns  |  inos |
+------+--------+-------+---------------+-------+-------+
|  0   | active | node2 | Reqs:    0 /s |   10  |   13  |
+------+--------+-------+---------------+-------+-------+
+-----------------+----------+-------+-------+
|       Pool      |   type   |  used | avail |
+-----------------+----------+-------+-------+
| cephfs_metadata | metadata | 1536k | 2645G |
|   cephfs_data   |   data   |    0  | 2645G |
+-----------------+----------+-------+-------+
+-------------+
| Standby MDS |
+-------------+
|    node3    |
|    node1    |
+-------------+
MDS version: ceph version 14.2.8 (2d095e947a02261ce61424021bb43bd3022d35cb) nautilus (stable)

2）、cephfs客户端挂载

# yum -y install ceph-fuse

#获取admin key

# ssh node1 "ceph-authtool -p /etc/ceph/ceph.client.admin.keyring" > admin.key
root@node1''s password:
# chmod 600 admin.key

# mount -t ceph node1:6789:/ /mnt -o name=admin,secretfile=admin.key

[root@node2 ~]# df -h
Filesystem               Size  Used Avail Use% Mounted on
/dev/mapper/centos-root   72G  1.9G   70G   3% /
devtmpfs                 3.9G     0  3.9G   0% /dev
tmpfs                    3.9G     0  3.9G   0% /dev/shm
tmpfs                    3.9G  8.9M  3.9G   1% /run
tmpfs                    3.9G     0  3.9G   0% /sys/fs/cgroup
/dev/vda1               1014M  145M  870M  15% /boot
tmpfs                    783M     0  783M   0% /run/user/0
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-5
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-11
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-17
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-23
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-32
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-38
tmpfs                    3.9G   52K  3.9G   1% /var/lib/ceph/osd/ceph-41
overlay                   72G  1.9G   70G   3% /var/lib/docker/overlay2/c04b21ae413ef6251b1e74b29e0d9498603a70e11f95ab03799a47425e4bd98e/merged
shm                       64M     0   64M   0% /var/lib/docker/containers/4401499e9737dc4304d267f3c88726123e5c59e5afe4a63319bf7fe3f8459d39/shm
192.168.80.90:6789:/      2.6T     0  2.6T   0% /mnt

3、rgw使用

1）、创建用户wuyeliang

# radosgw-admin user create --uid=wuyeliang --display-name="wuyeliang" [email protected]{
    "user_id": "wuyeliang",
    "display_name": "wuyeliang",
    "email": "[email protected]",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "wuyeliang",
            "access_key": "X97S5ASHJG0N7168HA38",
            "secret_key": "39rrlxMyNUqKCQNgpV5bTvIyDhNA4iIm7phmDHAG"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

2）、查看用户

# radosgw-admin user list
[
    "ceph-dashboard",
    "wuyeliang"
]

[root@node2 ~]# radosgw-admin user info --uid=wuyeliang
{
    "user_id": "wuyeliang",
    "display_name": "wuyeliang",
    "email": "[email protected]",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "wuyeliang",
            "access_key": "X97S5ASHJG0N7168HA38",
            "secret_key": "39rrlxMyNUqKCQNgpV5bTvIyDhNA4iIm7phmDHAG"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

3）、通过s3接口去使用rgw

# yum -y install python-boto

新建s3_test.py，内容如下

import sys
import boto
import boto.s3.connection

# user's access-key and secret-key you added on [2] section
ACCESS_KEY = 'X97S5ASHJG0N7168HA38'
SECRET_KEY = '39rrlxMyNUqKCQNgpV5bTvIyDhNA4iIm7phmDHAG'

# Object Gateway's hostname and listeing port
HOST = 'node1'
PORT = 8080

conn = boto.connect_s3(
    aws_access_key_id = ACCESS_KEY,
    aws_secret_access_key = SECRET_KEY,
    port = PORT,
    host = HOST,
    is_secure = False,
    calling_format = boto.s3.connection.OrdinaryCallingFormat(),
)

# create [wyl-new-bucket] bucket
bucket = conn.create_bucket('wyl-new-bucket')

# list own bucket list
for bucket in conn.get_all_buckets():
    print "{name}\t{created}".format(
        name = bucket.name,
        created = bucket.creation_date,
    )

# python s3_test.py
wyl-new-bucket 2020-04-06T05:37:13.653Z