ceph下创建rbd
[root@ceph01 ~]# ceph -s
cluster:
id: e6c031c4-5482-4029-ab47-ef4193bd0bbd
health: HEALTH_OK
services:
mon: 1 daemons, quorum ceph01
mgr: ceph01(active)
osd: 6 osds: 6 up, 6 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 bytes
usage: 6338 MB used, 54495 MB / 60833 MB avail
pgs:
[root@ceph01 ~]# ceph osd pool create sas 256
pool 'sas' created
[root@ceph01 ~]# rbd create testrbd --size 10240 --pool sas
[root@ceph01 ~]# rbd map testrbd -p sas
rbd: sysfs write failed
RBD image feature set mismatch. Try disabling features unsupported by the kernel with "rbd feature disable".
In some cases useful info is found in syslog - try "dmesg | tail".
rbd: map failed: (6) No such device or address
[root@ceph01 ~]# rbd info sas/testrbd
rbd image 'testrbd':
size 10240 MB in 2560 objects
order 22 (4096 kB objects)
block_name_prefix: rbd_data.1025643c9869
format: 2
features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
flags:
create_timestamp: Tue Jan 2 10:25:47 2018
[root@ceph01 ~]# rbd feature disable sas/testrbd fast-diff
[root@ceph01 ~]# rbd feature disable sas/testrbd deep-flatten
[root@ceph01 ~]# rbd feature disable sas/testrbd object-map
[root@ceph01 ~]# rbd feature disable sas/testrbd exclusive-lock
[root@ceph01 ~]# rbd info sas/testrbd
rbd image 'testrbd':
size 10240 MB in 2560 objects
order 22 (4096 kB objects)
block_name_prefix: rbd_data.1025643c9869
format: 2
features: layering
flags:
create_timestamp: Tue Jan 2 10:25:47 2018
[root@ceph01 ~]# rbd map testrbd -p sas
/dev/rbd0
设置crushmap
[root@ceph01 ~]#
[root@ceph01 ~]# yum install scsi-target-utils -y
Loaded plugins: fastestmirror, priorities
Loading mirror speeds from cached hostfile
* base: mirrors.aliyun.com
* epel: mirrors.ustc.edu.cn
* extras: mirrors.cn99.com
* updates: mirrors.aliyun.com
8 packages excluded due to repository priority protections
Resolving Dependencies
--> Running transaction check
---> Package scsi-target-utils.x86_64 0:1.0.55-4.el7 will be installed
--> Processing Dependency: librdmacm for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: librdmacm.so.1(RDMACM_1.0)(64bit) for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: lsof for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: perl(Config::General) for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: sg3_utils for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: librdmacm.so.1()(64bit) for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Running transaction check
---> Package librdmacm.x86_64 0:13-7.el7 will be installed
---> Package lsof.x86_64 0:4.87-4.el7 will be installed
---> Package perl-Config-General.noarch 0:2.61-1.el7 will be installed
---> Package sg3_utils.x86_64 0:1.37-12.el7 will be installed
--> Processing Dependency: sg3_utils-libs = 1.37-12.el7 for package: sg3_utils-1.37-12.el7.x86_64
--> Processing Dependency: libsgutils2.so.2()(64bit) for package: sg3_utils-1.37-12.el7.x86_64
--> Running transaction check
---> Package sg3_utils-libs.x86_64 0:1.37-12.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
============================================================================================================
Package Arch Version Repository Size
============================================================================================================
Installing:
scsi-target-utils x86_64 1.0.55-4.el7 epel 209 k
Installing for dependencies:
librdmacm x86_64 13-7.el7 base 59 k
lsof x86_64 4.87-4.el7 base 331 k
perl-Config-General noarch 2.61-1.el7 epel 75 k
sg3_utils x86_64 1.37-12.el7 base 644 k
sg3_utils-libs x86_64 1.37-12.el7 base 64 k
Transaction Summary
============================================================================================================
Install 1 Package (+5 Dependent packages)
Total download size: 1.3 M
Installed size: 3.5 M
Downloading packages:
(1/6): librdmacm-13-7.el7.x86_64.rpm | 59 kB 00:00:00
(2/6): lsof-4.87-4.el7.x86_64.rpm | 331 kB 00:00:00
(3/6): perl-Config-General-2.61-1.el7.noarch.rpm | 75 kB 00:00:00
(4/6): sg3_utils-libs-1.37-12.el7.x86_64.rpm | 64 kB 00:00:00
(5/6): scsi-target-utils-1.0.55-4.el7.x86_64.rpm | 209 kB 00:00:00
(6/6): sg3_utils-1.37-12.el7.x86_64.rpm | 644 kB 00:00:00
------------------------------------------------------------------------------------------------------------
Total 1.1 MB/s | 1.3 MB 00:00:01
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : sg3_utils-libs-1.37-12.el7.x86_64 1/6
Installing : sg3_utils-1.37-12.el7.x86_64 2/6
Installing : perl-Config-General-2.61-1.el7.noarch 3/6
Installing : lsof-4.87-4.el7.x86_64 4/6
Installing : librdmacm-13-7.el7.x86_64 5/6
Installing : scsi-target-utils-1.0.55-4.el7.x86_64 6/6
Verifying : librdmacm-13-7.el7.x86_64 1/6
Verifying : sg3_utils-1.37-12.el7.x86_64 2/6
Verifying : lsof-4.87-4.el7.x86_64 3/6
Verifying : perl-Config-General-2.61-1.el7.noarch 4/6
Verifying : scsi-target-utils-1.0.55-4.el7.x86_64 5/6
Verifying : sg3_utils-libs-1.37-12.el7.x86_64 6/6
Installed:
scsi-target-utils.x86_64 0:1.0.55-4.el7
Dependency Installed:
librdmacm.x86_64 0:13-7.el7 lsof.x86_64 0:4.87-4.el7
perl-Config-General.noarch 0:2.61-1.el7 sg3_utils.x86_64 0:1.37-12.el7
sg3_utils-libs.x86_64 0:1.37-12.el7
Complete!
[root@ceph01 ~]# service tgtd start
Redirecting to /bin/systemctl start tgtd.service
[root@ceph01 ~]# systemctl status tgtd
?.tgtd.service - tgtd iSCSI target daemon
Loaded: loaded (/usr/lib/systemd/system/tgtd.service; disabled; vendor preset: disabled)
Active: active (running) since Tue 2018-01-02 10:32:58 CST; 5s ago
Process: 20995 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v ready (code=exited, status=0/SUCCESS)
Process: 20989 ExecStartPost=/usr/sbin/tgt-admin -e -c $TGTD_CONFIG (code=exited, status=0/SUCCESS)
Process: 20987 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v offline (code=exited, status=0/SUCCESS)
Process: 20986 ExecStartPost=/bin/sleep 5 (code=exited, status=0/SUCCESS)
Main PID: 20985 (tgtd)
CGroup: /system.slice/tgtd.service
?..20985 /usr/sbin/tgtd -f
Jan 02 10:32:53 ceph01 systemd[1]: Starting tgtd iSCSI target daemon...
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: iser_ib_init(3436) Failed to initialize RDMA; load kernel modules?
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: work_timer_start(146) use timer_fd based scheduler
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: bs_init_signalfd(267) could not open backing-store module ...store
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: bs_init(386) use signalfd notification
Jan 02 10:32:58 ceph01 systemd[1]: Started tgtd iSCSI target daemon.
Hint: Some lines were ellipsized, use -l to show in full.
[root@ceph01 ~]# ps aux | grep tgt
root 20985 0.0 0.0 42276 1572 ? Ss 10:32 0:00 /usr/sbin/tgtd -f
root 21000 0.0 0.0 112660 976 pts/1 R+ 10:33 0:00 grep --color=auto tgt
[root@ceph01 ~]# chkconfig tgtd on
Note: Forwarding request to 'systemctl enable tgtd.service'.
Created symlink from /etc/systemd/system/multi-user.target.wants/tgtd.service to /usr/lib/systemd/system/tgtd.service.
[root@ceph01 ~]# vi /etc/tgt/targets.conf
[root@ceph01 ~]# vi /etc/tgt/targets.conf
backing-store /dev/rbd0
initiator-address 172.16.0.100
[root@ceph01 ~]# systemctl restart tgtd
[root@ceph01 ~]# systemctl status tgtd
?.tgtd.service - tgtd iSCSI target daemon
Loaded: loaded (/usr/lib/systemd/system/tgtd.service; enabled; vendor preset: disabled)
Active: active (running) since Tue 2018-01-02 10:36:17 CST; 8s ago
Process: 21033 ExecStop=/usr/sbin/tgtadm --op delete --mode system (code=exited, status=0/SUCCESS)
Process: 21030 ExecStop=/usr/sbin/tgt-admin --update ALL -c /dev/null (code=exited, status=0/SUCCESS)
Process: 21028 ExecStop=/usr/sbin/tgtadm --op update --mode sys --name State -v offline (code=exited, status=0/SUCCESS)
Process: 21048 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v ready (code=exited, status=0/SUCCESS)
Process: 21040 ExecStartPost=/usr/sbin/tgt-admin -e -c $TGTD_CONFIG (code=exited, status=0/SUCCESS)
Process: 21038 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v offline (code=exited, status=0/SUCCESS)
Process: 21037 ExecStartPost=/bin/sleep 5 (code=exited, status=0/SUCCESS)
Main PID: 21036 (tgtd)
CGroup: /system.slice/tgtd.service
?..21036 /usr/sbin/tgtd -f
Jan 02 10:36:12 ceph01 systemd[1]: Starting tgtd iSCSI target daemon...
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: iser_ib_init(3436) Failed to initialize RDMA; load kernel modules?
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: work_timer_start(146) use timer_fd based scheduler
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: bs_init_signalfd(267) could not open backing-store module ...store
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: bs_init(386) use signalfd notification
Jan 02 10:36:17 ceph01 systemd[1]: Started tgtd iSCSI target daemon.
Hint: Some lines were ellipsized, use -l to show in full.
[root@ceph01 ~]# tgt-admin --show
Target 1: iqn.2017-08.com.ceph-dev.test:target1
System information:
Driver: iscsi
State: ready
I_T nexus information:
LUN information:
LUN: 0
Type: controller
SCSI ID: IET 00010000
SCSI SN: beaf10
Size: 0 MB, Block size: 1
Online: Yes
Removable media: No
Prevent removal: No
Readonly: No
SWP: No
Thin-provisioning: No
Backing store type: null
Backing store path: None
Backing store flags:
Account information:
ACL information:
172.16.0.100
iscsiadm -m discovery -t sendtargets -p 172.16.1.40
172.16.1.40:3260,1 iqn.2017-08.com.ceph-dev.test:target1
ceph osd pool application enable testpool rbd
cluster:
id: e6c031c4-5482-4029-ab47-ef4193bd0bbd
health: HEALTH_OK
services:
mon: 1 daemons, quorum ceph01
mgr: ceph01(active)
osd: 6 osds: 6 up, 6 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 bytes
usage: 6338 MB used, 54495 MB / 60833 MB avail
pgs:
[root@ceph01 ~]# ceph osd pool create sas 256
pool 'sas' created
[root@ceph01 ~]# rbd create testrbd --size 10240 --pool sas
[root@ceph01 ~]# rbd map testrbd -p sas
rbd: sysfs write failed
RBD image feature set mismatch. Try disabling features unsupported by the kernel with "rbd feature disable".
In some cases useful info is found in syslog - try "dmesg | tail".
rbd: map failed: (6) No such device or address
[root@ceph01 ~]# rbd info sas/testrbd
rbd image 'testrbd':
size 10240 MB in 2560 objects
order 22 (4096 kB objects)
block_name_prefix: rbd_data.1025643c9869
format: 2
features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
flags:
create_timestamp: Tue Jan 2 10:25:47 2018
[root@ceph01 ~]# rbd feature disable sas/testrbd fast-diff
[root@ceph01 ~]# rbd feature disable sas/testrbd deep-flatten
[root@ceph01 ~]# rbd feature disable sas/testrbd object-map
[root@ceph01 ~]# rbd feature disable sas/testrbd exclusive-lock
[root@ceph01 ~]# rbd info sas/testrbd
rbd image 'testrbd':
size 10240 MB in 2560 objects
order 22 (4096 kB objects)
block_name_prefix: rbd_data.1025643c9869
format: 2
features: layering
flags:
create_timestamp: Tue Jan 2 10:25:47 2018
[root@ceph01 ~]# rbd map testrbd -p sas
/dev/rbd0
设置crushmap
[root@ceph01 ~]#
[root@ceph01 ~]# yum install scsi-target-utils -y
Loaded plugins: fastestmirror, priorities
Loading mirror speeds from cached hostfile
* base: mirrors.aliyun.com
* epel: mirrors.ustc.edu.cn
* extras: mirrors.cn99.com
* updates: mirrors.aliyun.com
8 packages excluded due to repository priority protections
Resolving Dependencies
--> Running transaction check
---> Package scsi-target-utils.x86_64 0:1.0.55-4.el7 will be installed
--> Processing Dependency: librdmacm for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: librdmacm.so.1(RDMACM_1.0)(64bit) for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: lsof for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: perl(Config::General) for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: sg3_utils for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Processing Dependency: librdmacm.so.1()(64bit) for package: scsi-target-utils-1.0.55-4.el7.x86_64
--> Running transaction check
---> Package librdmacm.x86_64 0:13-7.el7 will be installed
---> Package lsof.x86_64 0:4.87-4.el7 will be installed
---> Package perl-Config-General.noarch 0:2.61-1.el7 will be installed
---> Package sg3_utils.x86_64 0:1.37-12.el7 will be installed
--> Processing Dependency: sg3_utils-libs = 1.37-12.el7 for package: sg3_utils-1.37-12.el7.x86_64
--> Processing Dependency: libsgutils2.so.2()(64bit) for package: sg3_utils-1.37-12.el7.x86_64
--> Running transaction check
---> Package sg3_utils-libs.x86_64 0:1.37-12.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
============================================================================================================
Package Arch Version Repository Size
============================================================================================================
Installing:
scsi-target-utils x86_64 1.0.55-4.el7 epel 209 k
Installing for dependencies:
librdmacm x86_64 13-7.el7 base 59 k
lsof x86_64 4.87-4.el7 base 331 k
perl-Config-General noarch 2.61-1.el7 epel 75 k
sg3_utils x86_64 1.37-12.el7 base 644 k
sg3_utils-libs x86_64 1.37-12.el7 base 64 k
Transaction Summary
============================================================================================================
Install 1 Package (+5 Dependent packages)
Total download size: 1.3 M
Installed size: 3.5 M
Downloading packages:
(1/6): librdmacm-13-7.el7.x86_64.rpm | 59 kB 00:00:00
(2/6): lsof-4.87-4.el7.x86_64.rpm | 331 kB 00:00:00
(3/6): perl-Config-General-2.61-1.el7.noarch.rpm | 75 kB 00:00:00
(4/6): sg3_utils-libs-1.37-12.el7.x86_64.rpm | 64 kB 00:00:00
(5/6): scsi-target-utils-1.0.55-4.el7.x86_64.rpm | 209 kB 00:00:00
(6/6): sg3_utils-1.37-12.el7.x86_64.rpm | 644 kB 00:00:00
------------------------------------------------------------------------------------------------------------
Total 1.1 MB/s | 1.3 MB 00:00:01
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : sg3_utils-libs-1.37-12.el7.x86_64 1/6
Installing : sg3_utils-1.37-12.el7.x86_64 2/6
Installing : perl-Config-General-2.61-1.el7.noarch 3/6
Installing : lsof-4.87-4.el7.x86_64 4/6
Installing : librdmacm-13-7.el7.x86_64 5/6
Installing : scsi-target-utils-1.0.55-4.el7.x86_64 6/6
Verifying : librdmacm-13-7.el7.x86_64 1/6
Verifying : sg3_utils-1.37-12.el7.x86_64 2/6
Verifying : lsof-4.87-4.el7.x86_64 3/6
Verifying : perl-Config-General-2.61-1.el7.noarch 4/6
Verifying : scsi-target-utils-1.0.55-4.el7.x86_64 5/6
Verifying : sg3_utils-libs-1.37-12.el7.x86_64 6/6
Installed:
scsi-target-utils.x86_64 0:1.0.55-4.el7
Dependency Installed:
librdmacm.x86_64 0:13-7.el7 lsof.x86_64 0:4.87-4.el7
perl-Config-General.noarch 0:2.61-1.el7 sg3_utils.x86_64 0:1.37-12.el7
sg3_utils-libs.x86_64 0:1.37-12.el7
Complete!
[root@ceph01 ~]# service tgtd start
Redirecting to /bin/systemctl start tgtd.service
[root@ceph01 ~]# systemctl status tgtd
?.tgtd.service - tgtd iSCSI target daemon
Loaded: loaded (/usr/lib/systemd/system/tgtd.service; disabled; vendor preset: disabled)
Active: active (running) since Tue 2018-01-02 10:32:58 CST; 5s ago
Process: 20995 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v ready (code=exited, status=0/SUCCESS)
Process: 20989 ExecStartPost=/usr/sbin/tgt-admin -e -c $TGTD_CONFIG (code=exited, status=0/SUCCESS)
Process: 20987 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v offline (code=exited, status=0/SUCCESS)
Process: 20986 ExecStartPost=/bin/sleep 5 (code=exited, status=0/SUCCESS)
Main PID: 20985 (tgtd)
CGroup: /system.slice/tgtd.service
?..20985 /usr/sbin/tgtd -f
Jan 02 10:32:53 ceph01 systemd[1]: Starting tgtd iSCSI target daemon...
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: iser_ib_init(3436) Failed to initialize RDMA; load kernel modules?
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: work_timer_start(146) use timer_fd based scheduler
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: bs_init_signalfd(267) could not open backing-store module ...store
Jan 02 10:32:53 ceph01 tgtd[20985]: tgtd: bs_init(386) use signalfd notification
Jan 02 10:32:58 ceph01 systemd[1]: Started tgtd iSCSI target daemon.
Hint: Some lines were ellipsized, use -l to show in full.
[root@ceph01 ~]# ps aux | grep tgt
root 20985 0.0 0.0 42276 1572 ? Ss 10:32 0:00 /usr/sbin/tgtd -f
root 21000 0.0 0.0 112660 976 pts/1 R+ 10:33 0:00 grep --color=auto tgt
[root@ceph01 ~]# chkconfig tgtd on
Note: Forwarding request to 'systemctl enable tgtd.service'.
Created symlink from /etc/systemd/system/multi-user.target.wants/tgtd.service to /usr/lib/systemd/system/tgtd.service.
[root@ceph01 ~]# vi /etc/tgt/targets.conf
[root@ceph01 ~]# vi /etc/tgt/targets.conf
backing-store /dev/rbd0
initiator-address 172.16.0.100
[root@ceph01 ~]# systemctl restart tgtd
[root@ceph01 ~]# systemctl status tgtd
?.tgtd.service - tgtd iSCSI target daemon
Loaded: loaded (/usr/lib/systemd/system/tgtd.service; enabled; vendor preset: disabled)
Active: active (running) since Tue 2018-01-02 10:36:17 CST; 8s ago
Process: 21033 ExecStop=/usr/sbin/tgtadm --op delete --mode system (code=exited, status=0/SUCCESS)
Process: 21030 ExecStop=/usr/sbin/tgt-admin --update ALL -c /dev/null (code=exited, status=0/SUCCESS)
Process: 21028 ExecStop=/usr/sbin/tgtadm --op update --mode sys --name State -v offline (code=exited, status=0/SUCCESS)
Process: 21048 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v ready (code=exited, status=0/SUCCESS)
Process: 21040 ExecStartPost=/usr/sbin/tgt-admin -e -c $TGTD_CONFIG (code=exited, status=0/SUCCESS)
Process: 21038 ExecStartPost=/usr/sbin/tgtadm --op update --mode sys --name State -v offline (code=exited, status=0/SUCCESS)
Process: 21037 ExecStartPost=/bin/sleep 5 (code=exited, status=0/SUCCESS)
Main PID: 21036 (tgtd)
CGroup: /system.slice/tgtd.service
?..21036 /usr/sbin/tgtd -f
Jan 02 10:36:12 ceph01 systemd[1]: Starting tgtd iSCSI target daemon...
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: iser_ib_init(3436) Failed to initialize RDMA; load kernel modules?
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: work_timer_start(146) use timer_fd based scheduler
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: bs_init_signalfd(267) could not open backing-store module ...store
Jan 02 10:36:12 ceph01 tgtd[21036]: tgtd: bs_init(386) use signalfd notification
Jan 02 10:36:17 ceph01 systemd[1]: Started tgtd iSCSI target daemon.
Hint: Some lines were ellipsized, use -l to show in full.
[root@ceph01 ~]# tgt-admin --show
Target 1: iqn.2017-08.com.ceph-dev.test:target1
System information:
Driver: iscsi
State: ready
I_T nexus information:
LUN information:
LUN: 0
Type: controller
SCSI ID: IET 00010000
SCSI SN: beaf10
Size: 0 MB, Block size: 1
Online: Yes
Removable media: No
Prevent removal: No
Readonly: No
SWP: No
Thin-provisioning: No
Backing store type: null
Backing store path: None
Backing store flags:
Account information:
ACL information:
172.16.0.100
iscsiadm -m discovery -t sendtargets -p 172.16.1.40
172.16.1.40:3260,1 iqn.2017-08.com.ceph-dev.test:target1
ceph osd pool application enable testpool rbd