基于iSCSI的IPSAN存储系统

第一步：服务端  target




1、配置ip地址，关闭iptables 和selinux


[root@localhost Desktop]# ifconfig eth0 10.0.0.1 netmask 255.0.0.0


[root@localhost Desktop]# service iptables stop
i
[root@localhost Desktop]# setenforce 0


2、软件安装




[root@localhost Desktop]# cd /etc/yum.repos.d/
[root@localhost yum.repos.d]# ll
total 16
-rw-r--r--. 1 root root 1926 Nov 27  2013 CentOS-Base.repo
-rw-r--r--. 1 root root  638 Nov 27  2013 CentOS-Debuginfo.repo
-rw-r--r--. 1 root root  630 Nov 27  2013 CentOS-Media.repo
-rw-r--r--. 1 root root 3664 Nov 27  2013 CentOS-Vault.repo
[root@localhost yum.repos.d]# rm -rf CentOS-Base.repo 
[root@localhost yum.repos.d]# rm -rf CentOS-Debuginfo.repo 
[root@localhost yum.repos.d]# rm -rf CentOS-Vault.repo 
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# ll
total 4
-rw-r--r--. 1 root root 630 Nov 27  2013 CentOS-Media.repo
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# vim CentOS-Media.repo 
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# tail -6 CentOS-Media.repo 
[c6-media]
name=CentOS-$releasever - Media
baseurl=file:///mnt
gpgcheck=0
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# mount /dev/sr0 /mnt
mount: block device /dev/sr0 is write-protected, mounting read-only
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# yum clean all
Loaded plugins: fastestmirror, refresh-packagekit, security
Cleaning repos: c6-media
Cleaning up Everything
[root@localhost yum.repos.d]# 


安装软件scsi-target-utils


[root@localhost ~]# 
[root@localhost ~]# yum -y install scsi-target-utils
[root@localhost ~]# rpm -qa |grep scsi
scsi-target-utils-1.0.24-10.el6.x86_64
[root@localhost ~]# 




3、添加硬盘，配置本地存储系统提供远程访问


[root@localhost ~]# 
[root@localhost ~]# fdisk /dev/sdb    //注意只分区，而不格式化。格式化留给使用的客户端自己去格式化


[root@localhost ~]# vim /etc/tgt/targets.conf 
[root@localhost ~]# tail -5 /etc/tgt/targets.conf 
                  //表示IQN号,可以随便取,但是要遵循这个格式（这个格式必须这样子，网址倒着写）
      vendor_id sohu                                 //说明id号，无所谓的东西
      backing-store /dev/sdb                         //提供ISCSI服务的硬盘分区.
      initiator-address 10.0.0.2                     //指定允许访问的客户端ip（表示允许这两个IP的地址访问SCSI磁盘.）

[root@localhost ~]# 
[root@localhost ~]#


5、强scsi服务器端启动并设置开机自动启动


root@localhost ~]# 
[root@localhost ~]# service tgtd start
Starting SCSI target daemon:                               [  OK  ]
[root@localhost ~]# 
[root@localhost ~]# service tgtd status
tgtd (pid 13150 13148) is running...
[root@localhost ~]# 
[root@localhost ~]# 
[root@localhost ~]# chkconfig tgtd on               //设置开机自动启动
[root@localhost ~]# 
[root@localhost ~]# netstat -antup |grep 3260       //tgtd的端口号为3260          
tcp        0      0 0.0.0.0:3260                0.0.0.0:*                   LISTEN      13148/tgtd          
tcp        0s      0 :::3260                     :::*                        LISTEN      13148/tgtd          
[root@localhost ~]# 






第二步：配置客户端  initaitor


1、配置ip，iptables selinux、


[root@localhost Desktop]# ifconfig eth0 10.0.0.2 netmask 255.0.0.0
[root@localhost Desktop]# ip a
[root@localhost Desktop]# service iptables stop
[root@localhost Desktop]# setenforce 0


2、安装客户端软件


[root@localhost Desktop]# rpm -ivh /media/CentOS_6.5_Final/Packages/iscsi-initiator-utils-6.2.0.873-10.el6.x86_64.rpm 
warning: /media/CentOS_6.5_Final/Packages/iscsi-initiator-utils-6.2.0.873-10.el6.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY
Preparing...                ########################################### [100%]
   1:iscsi-initiator-utils  ########################################### [100%]
[root@localhost Desktop]# 
[root@localhost Desktop]# rpm -qa |grep iscsi
iscsi-initiator-utils-6.2.0.873-10.el6.x86_64
[root@localhost Desktop]# 


3、使用iscsi发现目标


注意：各种启动没成功，因为需要先发现再启


[root@localhost ~]# service iscsid restart
Stopping iscsid: 
[root@localhost ~]#                                        [FAILED]
[root@localhost ~]# service iscsid start                                  //各种启动没成功，因为需要先发现再启动
[root@localhost ~]# service iscsid status           
iscsid is stopped
[root@localhost ~]# 




iscsiadm -m discovery -t sendtargets -p 10.0.0.1:3260


iscsiadm --mode discoverydb --type sendtargets --portal 10.0.0.1 –discover


[root@localhost ~]# 
[root@localhost ~]# iscsiadm -m discovery -t sendtargets -p 10.0.0.1:3260   //发现目标。过程自动启动iscsid服务
Starting iscsid:                                           [  OK  ]
10.0.0.1:3260,1 iqn.2015-11.com.sohu:san1
[root@localhost ~]#  
[root@localhost ~]# service iscsid status       //服务处于启动状态。在发现目标是自动启动的。
iscsid (pid  2778) is running...
[root@localhost ~]# 
[root@localhost ~]# chkconfig iscsid on         //设置开启自动启动
[root@localhost ~]# service iscsi start
[root@localhost ~]# chkconfig iscsi on          //设置开启自动启动


注意：只有iscsi和iscsid都启动了，才能看到，服务器共享出来的硬盘 sdb


[root@localhost ~]# ll /dev/sd*
brw-rw----. 1 root disk 8,  0 Nov 26 00:19 /dev/sda
brw-rw----. 1 root disk 8,  1 Nov 26 00:19 /dev/sda1
brw-rw----. 1 root disk 8,  2 Nov 26 00:19 /dev/sda2
brw-rw----. 1 root disk 8,  3 Nov 26 00:19 /dev/sda3
brw-rw----. 1 root disk 8, 16 Nov 26 01:36 /dev/sdb
brw-rw----. 1 root disk 8, 17 Nov 26 01:36 /dev/sdb1
[root@localhost ~]# 


4、格式化挂载使用


[root@localhost ~]# 
[root@localhost ~]# mkdir /wt
[root@localhost ~]# mount /dev/sdb /wt
mount: you must specify the filesystem type
[root@localhost ~]# 
[root@localhost ~]# mkfs.ext4 /dev/sdb
mke2fs 1.41.12 (17-May-2010)
/dev/sdb is entire device, not just one partition!
Proceed anyway? (y,n) y
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
65536 inodes, 262144 blocks
13107 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=268435456
8 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks: 
32768, 98304, 163840, 229376


Writing inode tables: done                            
Creating journal (8192 blocks): done
Writing superblocks and filesystem accounting information: done


This filesystem will be automatically checked every 34 mounts or
180 days, whichever comes first.  Use tune2fs -c or -i to override.
[root@localhost ~]# 
[root@localhost ~]# mount /dev/sdb /wt
[root@localhost ~]# ll /wt/
total 16
drwx------. 2 root root 16384 Nov 26 02:23 lost+found
[root@localhost ~]# 
[root@localhost ~]# cp -r /etc/passwd /wt
[root@localhost ~]# ll /wt
total 20
drwx------. 2 root root 16384 Nov 26 02:23 lost+found
-rw-r--r--. 1 root root  1423 Nov 26 02:24 passwd
[root@localhost]# 


查看uuid号用于开机自动挂载磁盘。因为sdb会由变动（比如，插入u就默认sdb），而uuid一直是不变的。


[root@localhost ~]# 
[root@localhost ~]# blkid /dev/sdb
/dev/sdb: UUID="c2a254fa-6ab5-4c85-a756-c8bf8c7242f4" TYPE="ext4" 
[root@localhost ~]# 


[root@localhost ~]# vim /etc/fstab                       //设置开机自动挂载
[root@localhost ~]# tail -1 !$
UUID="c2a254fa-6ab5-4c85-a756-c8bf8c7242f4"    /wt  ext4  _netdev       0 0 
[root@localhost ~]# 
 


注意，重启的时候要登出，不然，重启动系统的话会报错,导致很长时间起不来电脑




[root@localhost ~]# 
[root@localhost ~]# iscsiadm --mode node --targetname iqn.2015-11.com.sohu:san1 --portal 10.0.0.1:3260 --logout
Logging out of session [sid: 2, target: iqn.2015-11.com.sohu:san1, portal: 10.0.0.1,3260]
Logout of [sid: 2, target: iqn.2015-11.com.sohu:san1, portal: 10.0.0.1,3260] successful.
[root@localhost ~]# 
[root@localhost ~]# 


！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！
如下为登入命令。可以不用


[root@localhost ~]# iscsiadm --mode node --targetname iqn.2015-11.com.sohu:san1 --portal 10.0.0.1:3260 --login
[root@localhost ~]#


！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！


重启后，看是否能正常启动和挂载，并开启服务


[root@localhost ~]# reboot 
  
再看挂载ok，启动正常。数据依然在


[root@localhost Desktop]# tree /usr/bin/which
/usr/bin/which [error opening dir]


0 directories, 0 files
[root@localhost Desktop]# tree /var/lib/iscsi
/var/lib/iscsi
├── ifaces
├── isns
├── nodes
│?? └── iqn.2015-11.com.sohu:san1
│??     └── 10.0.0.1,3260,1
│??         └── default
├── send_targets
│?? └── 10.0.0.1,3260
│??     ├── iqn.2015-11.com.sohu:san1,10.0.0.1,3260,1,default -> /var/lib/iscsi/nodes/iqn.2015-11.com.sohu:san1/10.0.0.1,3260,1
│??     └── st_config
├── slp
└── static


10 directories, 2 files
[root@localhost Desktop]# 
[root@localhost Desktop]# 


##################################################################################################


卸载，挂载存储设备


1、卸载


如果，卸载不成功的话，将所有都卸了再挂载后（umount -a /mount -a）。用下面命令登出




[root@localhost Desktop]# 
[root@localhost Desktop]# iscsiadm -m node -T iqn.2015-11.com.sohu:san1 -u
Logging out of session [sid: 2, target: iqn.2015-11.com.sohu:san1, portal: 10.0.0.1,3260]
Logout of [sid: 2, target: iqn.2015-11.com.sohu:san1, portal: 10.0.0.1,3260] successful.
[root@localhost Desktop]# 


/dev/sdc         1032088   34056    945604   4% /wt
[root@localhost Desktop]# 
[root@localhost Desktop]# iscsiadm -m node -T iqn.2015-11.com.sohu:san1 -l
Logging out of session [sid: 2, target: iqn.2015-11.com.sohu:san1, portal: 10.0.0.1,3260]
Logout of [sid: 2, target: iqn.2015-11.com.sohu:san1, portal: 10.0.0.1,3260] successful.
[root@localhost Desktop]# 




总结：


卸载
卸载的时候，要先登出，否则下次启动的时候起不来，或启动慢。
登出：iscsiadm -m node -T iqn.2015-11.com.sohu:san1 -u   或  iscsiadm --mode node --targetname iqn.2015-11.com.sohu:san1 --portal 10.0.0.1:3260 --logout
这时候查看就没有挂载也没有共享的磁盘


挂载
方法一：重启即可自动挂载
方法二：先发现，再登入，最后挂载即可
发现：iscsiadm  -m discovery -t  sendtargets -p 10.0.0.1:3260
登入：iscsiadm -m node -T iqn.2008-09.com.sohu:wt -l
卸载重新挂载：umount -a    mount -a






详细如下：
root@localhost wt]# 
[root@localhost wt]# iscsiadm  -m discovery -t  sendtargets -p 10.0.0.1:3260
10.0.0.1:3260,1 iqn.2008-09.com.sohu:wt
[root@localhost wt]# 
[root@localhost wt]# df
Filesystem     1K-blocks    Used Available Use% Mounted on
/dev/sda2        8985528 2526196   6002884  30% /
tmpfs             502204     224    501980   1% /dev/shm
/dev/sdb         1032088   34056    945604   4% /wt
[root@localhost wt]# 
[root@localhost wt]# ll /wt
ls: reading directory /wt: Input/output error
total 0
[root@localhost wt]# 
[root@localhost wt]# mount -a
mount: special device UUID="106f7acc-ce28-4cf7-a5d7-1b18f4e8f6ed" does not exist
[root@localhost wt]# 
[root@localhost wt]# umount -a
umount: /wt: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /var/run/vmblock-fuse: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /dev/shm: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
[root@localhost wt]# mount -a
mount: special device UUID="106f7acc-ce28-4cf7-a5d7-1b18f4e8f6ed" does not exist
[root@localhost wt]# df
Filesystem     1K-blocks    Used Available Use% Mounted on
/dev/sda2        8985528 2526188   6002892  30% /
tmpfs             502204     224    501980   1% /dev/shm
/dev/sdb         1032088   34056    945604   4% /wt
/dev/sda1         297485   34642    247483  13% /boot
[root@localhost wt]# ll /wt
ls: reading directory /wt: Input/output error
total 0
[root@localhost wt]# iscsiadm -m node -T iqn.2008-09.com.sohu:wt -l
Logging in to [iface: default, target: iqn.2008-09.com.sohu:wt, portal: 10.0.0.1,3260] (multiple)
Login to [iface: default, target: iqn.2008-09.com.sohu:wt, portal: 10.0.0.1,3260] successful.
[root@localhost wt]# 
[root@localhost wt]# ll /wt
ls: reading directory /wt: Input/output error
total 0
[root@localhost wt]# mount -a
[root@localhost wt]# ll 
ls: reading directory .: Input/output error
total 0
[root@localhost wt]# cd
[root@localhost ~]# umount -a
umount: /var/run/vmblock-fuse: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /dev/shm: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
[root@localhost ~]# 
[root@localhost ~]# mount -a
[root@localhost ~]# 
[root@localhost ~]# ll
total 100
-rw-------. 1 root root  3322 Nov  4 08:55 anaconda-ks.cfg
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Desktop
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Documents
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Downloads
-rw-r--r--. 1 root root 41364 Nov  4 08:55 install.log
-rw-r--r--. 1 root root 15643 Nov  4 08:55 install.log.syslog
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Music
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Pictures
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Public
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Templates
drwxr-xr-x. 2 root root  4096 Nov  4 09:01 Videos
[root@localhost ~]# ll /wt
total 20
drwx------. 2 root root 16384 Dec  6 06:01 lost+found
-rw-r--r--. 1 root root  1429 Dec  6 06:01 passwd
[root@localhost ~]# 
[root@localhost ~]# 




#########################################################################################


target 删除了磁盘sdb，共享新盘sdc，客户端需要重新发现


1、服务端，target


删除原有磁盘，再添加新盘sdc


首先，客户端闲载
[root@localhost iscsi]# 
[root@localhost iscsi]# iscsiadm --mode node --targetname iqn.2015-11.com.sohu:san1 --portal 10.0.0.1:3260 --logout
或
[root@localhost Desktop]# iscsiadm -m node -T iqn.2015-11.com.sohu:san1 -u     
   
target端，添加硬盘sdc，并配置tgta服务


[root@localhost ~]# vim /etc/tgt/targets.conf 
[root@localhost ~]# tail -6 !$
tail -6 /etc/tgt/targets.conf
#

      vendor_id sohu
      backing-store /dev/sdc
     initiator-address 10.0.0.2

[root@localhost ~]# 






客户端 initiator（发起者）




[root@localhost iscsi]# 
[root@localhost iscsi]# service iscsi stop
Stopping iscsi:                                            [  OK  ]
[root@localhost iscsi]# 
[root@localhost iscsi]# 
[root@localhost iscsi]# ll
total 24
drwxr-xr-x. 2 root root 4096 Nov 23  2013 ifaces
drwxr-xr-x. 2 root root 4096 Nov 23  2013 isns
drwxr-xr-x. 3 root root 4096 Nov 26 01:20 nodes
drwxr-xr-x. 3 root root 4096 Nov 26 01:20 send_targets
drwxr-xr-x. 2 root root 4096 Nov 23  2013 slp
drwxr-xr-x. 2 root root 4096 Nov 23  2013 static
[root@localhost iscsi]# rm -rf /var/lib/iscsi/*
[root@localhost iscsi]# ll
total 0
[root@localhost iscsi]# 
[root@localhost iscsi]# service iscsi start
Starting iscsi:                                            [  OK  ]
[root@localhost iscsi]# 
[root@localhost ~]# 
[root@localhost ~]# iscsiadm -m discovery -t sendtargets -p 10.0.0.1:3260
10.0.0.1:3260,1 iqn.2015-11.com.sohu:san1
[root@localhost ~]# ll /dev/sd*
brw-rw----. 1 root disk 8,  0 Nov 26 02:56 /dev/sda
brw-rw----. 1 root disk 8,  1 Nov 26 02:56 /dev/sda1
brw-rw----. 1 root disk 8,  2 Nov 26 02:56 /dev/sda2
brw-rw----. 1 root disk 8,  3 Nov 26 02:56 /dev/sda3
brw-rw----. 1 root disk 8, 32 Nov 26 05:21 /dev/sdc
[root@localhost ~]# 
[root@localhost ~]# 
[root@localhost ~]# mkfs.ext4 /dev/sdc
[root@localhost ~]# umount -a
[root@localhost ~]# mount /dev/sdc /wt
[root@localhost ~]# ll /wt/
total 16
drwx------. 2 root root 16384 Nov 26 05:23 lost+found
[root@localhost ~]# 




#########################################################################################################3


在target服务端，再添加几个存储客户端，并实现同步数据。




1、在target服务端


[root@localhost ~]# vim /etc/tgt/targets.conf
[root@localhost ~]# tail -6 /etc/tgt/targets.conf 

      vendor_id sohu
      backing-store /dev/sdc
     initiator-address 10.0.0.2
     initiator-address 10.0.0.3

[root@localhost ~]# 
[root@localhost ~]# 
[root@localhost ~]# /etc/init.d/tgtd restart
Stopping SCSI target daemon: initiators still connected    [FAILED]
Starting SCSI target daemon:                               [  OK  ]
[root@localhost ~]#  


或


[root@localhost ~]# service tgtd restart
Stopping SCSI target daemon:                               [  OK  ]
Starting SCSI target daemon:                               [  OK  ]
[root@localhost ~]# 
[root@localhost ~]# 


2、再开2台客户端。


10.0.0.3
10.0.0.4


[root@localhost Desktop]# 
[root@localhost Desktop]# rpm -vih /media/CentOS_6.5_Final/Packages/iscsi-initiator-utils-6.2.0.873-10.el6.x86_64.rpm 
warning: /media/CentOS_6.5_Final/Packages/iscsi-initiator-utils-6.2.0.873-10.el6.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY
Preparing...                ########################################### [100%]
   1:iscsi-initiator-utils  ########################################### [100%]
[root@localhost Desktop]# 
[root@localhost Desktop]# mkdir /wt
[root@localhost Desktop]# 
[root@localhost Desktop]# iscsiadm -m discovery -t sendtargets -p 10.0.0.1:3260 
Starting iscsid:                                           [  OK  ]
iscsiadm: No portals found
[root@localhost Desktop]# 




root@localhost ~]# mount /dev/sdb /wt
[root@localhost ~]# ll /wt/
total 36
drwxr-xr-x. 2 root root  4096 Nov 26 09:52 22
drwxr-xr-x. 2 root root  4096 Nov 26 09:52 2222
drwxr-xr-x. 2 root root  4096 Nov 26 09:45 3
drwxr-xr-x. 2 root root  4096 Nov 26 09:49 33
drwxr-xr-x. 2 root root  4096 Nov 26 09:59 44444
-rw-r--r--. 1 root root     0 Nov 26 09:42 a
drwx------. 2 root root 16384 Nov 26 09:41 lost+found
[root@localhost ~]# 
[root@localhost ~]# 
[root@localhost ~]# 




总结，在10.0.0.2 10.0.0.3 10.0.0.4 上面分别挂载创建文件，如上，10.0.0.2创建22.2222.10.0.0.3创建3.33，依次来区分。然后，分别在各个系统上卸载后重新挂载，发现数据同步了。