配置环境
NFS： 192.168.1.121 CentOS6.7
Node1:192.168.1.131 CentOS7.2
Node2:192.168.1.132 CentOS7.2
Node3:192.168.1.133 CentOS7.2
配置前准备
[root@node1 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.131 node1
192.168.1.132 node2
[root@node1 ~]# ssh-keygen -t rsa -P ''
[root@node1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub node1
[root@node1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub node2
[root@node1 ~]# yum -y install ansible
[root@node1 ~]# vim /etc/ansible/hosts
[ha]
192.168.1.131
192.168.1.132
03 corosync集群配置精讲
Node1、Node2和node3
# yum -y install corosync pacemaker
[root@node1 ~]# cd /etc/corosync/
[root@node1 corosync]# cp corosync.conf.example corosync.conf
[root@node1 corosync]# vim corosync.conf
修改
crypto_cipher: none
crypto_hash: none
为
crypto_cipher: aes128
crypto_hash: sha1
setauth: on
修改
bindnetaddr: 192.168.1.0
为
bindnetaddr: 192.168.1.0 #设定本地网络，本示例不需要修改
在logging程序段之前添加
nodelist {
node {
ring0_addr: 192.168.1.131
nodeid: 1
}
node {
ring0_addr: 192.168.1.132
nodeid: 2
}
node {
ring0_addr: 192.168.1.133
nodeid: 3
}
}
corosync.conf文件
[root@node1 corosync]# grep -v "^[[:space:]]*#" corosync.conf
totem {
version: 2
crypto_cipher: aes128
crypto_hash: sha1
setauth: on
interface {
ringnumber: 0
bindnetaddr: 192.168.1.0
mcastaddr: 239.255.1.1
mcastport: 5405
ttl: 1
}
}
nodelist {
node {
ring0_addr: 192.168.1.131
nodeid: 1
}
node {
ring0_addr: 192.168.1.132
nodeid: 2
}
node {
ring0_addr: 192.168.1.133
nodeid: 3
}
}
logging {
fileline: off
to_stderr: no
to_logfile: yes
logfile: /var/log/cluster/corosync.log
to_syslog: no
debug: off
timestamp: on
logger_subsys {
subsys: QUORUM
debug: off
}
}
quorum {
provider: corosync_votequorum
}
生成认证文件
[root@node1 corosync]# corosync-keygen
认证文件
[root@node1 corosync]# ll
total 20
-r-------- 1 root root 128 Sep 22 13:46 authkey
[root@node1 corosync]# scp -p authkey corosync.conf node2:/etc/corosync/
[root@node1 corosync]# scp -p authkey corosync.conf node3:/etc/corosync/
[root@node1 corosync]# systemctl start corosync.service
[root@node2 corosync]# systemctl start corosync.service
[root@node3 ~]# systemctl start corosync.service
查看日志
[root@node2 corosync]# tail -f /var/log/cluster/corosync.log
Sep 22 16:23:01 [36631] node2 corosync notice [TOTEM ] A new membership (192.168.1.132:4) was formed. Members joined: 2
Sep 22 16:23:01 [36631] node2 corosync notice [QUORUM] Members[1]: 2
Sep 22 16:23:01 [36631] node2 corosync notice [MAIN ] Completed service synchronization, ready to provide service.
Sep 22 16:23:01 [36631] node2 corosync notice [TOTEM ] A new membership (192.168.1.131:8) was formed. Members joined: 1
Sep 22 16:23:01 [36631] node2 corosync notice [QUORUM] This node is within the primary component and will provide service.
Sep 22 16:23:01 [36631] node2 corosync notice [QUORUM] Members[2]: 1 2
Sep 22 16:23:01 [36631] node2 corosync notice [MAIN ] Completed service synchronization, ready to provide service.
Sep 22 16:23:09 [36631] node2 corosync notice [TOTEM ] A new membership (192.168.1.131:12) was formed. Members joined: 3
Sep 22 16:23:09 [36631] node2 corosync notice [QUORUM] Members[3]: 1 2 3
Sep 22 16:23:09 [36631] node2 corosync notice [MAIN ] Completed service synchronization, ready to provide service.
[root@node1 corosync]# vim /etc/sysconfig/pacemaker
去掉
PCMK_logfile=/var/log/pacemaker.log
前面的#号
[root@node1 corosync]# systemctl start pacemaker.service
[root@node2 corosync]# systemctl start pacemaker.service
[root@node3 corosync]# systemctl start pacemaker.service
[root@node2 corosync]# crm_mon
Last updated: Thu Sep 22 16:47:06 2016 Last change: Thu Sep 22 16:43:05 2016 by hacluster via crmd on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 0 resources configured
Online: [ node1 node2 node3 ]
至此集群配置完成，下面开始配置资源
[root@node1 ~]# ls *rpm
crmsh-2.1.4-1.1.x86_64.rpm pssh-2.3.1-4.2.x86_64.rpm python-pssh-2.3.1-4.2.x86_64.rpm
[root@node1 ~]# yum -y install *rpm
[root@node1 ~]# crm
crm(live)#
HA Web Service:
vip: 172.16.1.80, ocf:heartbeat:IPaddr
httpd: systemd
nfs shared storage: ocf:heartbeat:Filesystem
[root@NFS ~]# mkdir /www/htdocs -pv
[root@NFS ~]# echo "Test Page on NFS Server
" > /www/htdocs/index.html
[root@NFS ~]# vim /etc/exports
/www/htdocs 192.168.1.0/24(rw)
[root@NFS ~]# service nfsd start
Node1、Node2和node3
# yum -y install httpd
# mount -t nfs 192.168.1.121:/www/htdocs /var/www/html/
# systemctl start httpd.service
# systemctl stop httpd.service
# systemctl enable httpd.service
# umount /var/www/html/
错误：
[root@node1 ~]# crm_verify -L -V
error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
Errors found during check: config not valid
解决方法：
[root@node1 ~]# crm
crm(live)# configure property stonith-enabled=false
crm(live)# configure commit
INFO: apparently there is nothing to commit
INFO: try changing something first
crm(live)# configure
crm(live)configure# show
node 1: node1
node 2: node2
node 3: node3
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.13-10.el7_2.4-44eb2dd \
cluster-infrastructure=corosync \
stonith-watchdog-timeout=false \
stonith-enabled=false
[root@node1 ~]# crm configure
crm(live)configure# primitive webip ocf:heartbeat:IPaddr2 params ip="192.168.1.80" op monitor interval=30s timeout=20s
crm(live)configure# verify
crm(live)configure# show
node 1: node1
node 2: node2
node 3: node3
primitive webip IPaddr2 \
params ip=192.168.1.80 \
op monitor interval=30s timeout=20s
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.13-10.el7_2.4-44eb2dd \
cluster-infrastructure=corosync \
stonith-watchdog-timeout=false \
stonith-enabled=false
crm(live)configure# delete webip
crm(live)configure# show
node 1: node1
node 2: node2
node 3: node3
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.13-10.el7_2.4-44eb2dd \
cluster-infrastructure=corosync \
stonith-watchdog-timeout=false \
stonith-enabled=false
crm(live)configure# primitive webip ocf:heartbeat:IPaddr2 params ip="192.168.1.80" op monitor interval=20scrm(live)configure# primitive webip ocf:heartbeat:IPaddr2 params ip="192.168.1.80" op monitor interval=20s timeout=10s
crm(live)configure# verify
WARNING: webip: specified timeout 10s for monitor is smaller than the advised 20s timeout=10s
crm(live)configure# edit
node 1: node1
node 2: node2
node 3: node3
primitive webip IPaddr2 \
params ip=192.168.1.80 \
op monitor interval=20s timeout=20s
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.13-10.el7_2.4-44eb2dd \
cluster-infrastructure=corosync \
stonith-watchdog-timeout=false \
stonith-enabled=false
# vim: set filetype=pcmk:
crm(live)configure# verify
crm(live)configure# primitive webserver systemd:httpd op monitor interval=20s timeout=20scrm(live)configure# verify
crm(live)configure# primitive webstore ocf:heartbeat:Filesystem params device="192.168.1.121:/www/htdocs" directory="/var/www/html" fstype="nfs" op start timeout=60s op stop timeout=60s op monitor interval=20s timeout=40s
crm(live)configure# verify
crm(live)configure# colocation web_server_with_webstore_and_webip inf: webserver ( webip webstore )
crm(live)configure# verify
crm(live)configure# show
node 1: node1
node 2: node2
node 3: node3
primitive webip IPaddr2 \
params ip=192.168.1.80 \
op monitor interval=20s timeout=20s
primitive webserver systemd:httpd \
op monitor interval=20s timeout=20s
primitive webstore Filesystem \
params device="192.168.1.121:/www/htdocs" directory="/var/www/html" fstype=nfs \
op start timeout=60s interval=0 \
op stop timeout=60s interval=0 \
op monitor interval=20s timeout=40s
colocation web_server_with_webstore_and_webip inf: webserver ( webip webstore )
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.13-10.el7_2.4-44eb2dd \
cluster-infrastructure=corosync \
stonith-watchdog-timeout=false \
stonith-enabled=false
crm(live)configure# order webstore_after_webip Mandatory: webip webstore
crm(live)configure# order webserver_after_webstore Mandatory: webstore webserver
crm(live)configure# verify
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure# cd
crm(live)# status
Last updated: Fri Sep 23 09:35:56 2016 Last change: Fri Sep 23 09:35:34 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 node2 node3 ]
[root@node1 ~]# crm node standby
[root@node1 ~]# crm status
Last updated: Fri Sep 23 09:43:31 2016 Last change: Fri Sep 23 09:42:45 2016 by root via crm_attribute on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Node node1: standby
Online: [ node2 node3 ]
[root@node1 ~]# crm node online
crm(live)configure# location webservice_pref_node1 webip 100: node1
crm(live)configure# verify
crm(live)configure# commit
crm(live)# status
Last updated: Fri Sep 23 09:52:47 2016 Last change: Fri Sep 23 09:51:33 2016 by root via crm_attribute on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 node2 node3 ]
crm(live)configure# property default-resource-stickiness=50
crm(live)configure# commit
crm(live)configure# cd
crm(live)# node standby
crm(live)# status
Last updated: Fri Sep 23 09:56:48 2016 Last change: Fri Sep 23 09:56:41 2016 by root via crm_attribute on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Node node1: standby
Online: [ node2 node3 ]
crm(live)# node online
crm(live)# status
Last updated: Fri Sep 23 09:57:04 2016 Last change: Fri Sep 23 09:57:02 2016 by root via crm_attribute on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 node2 node3 ]
04 corosync集群配置-配置示例
清空集群配置
crm(live)configure# edit
删除后的资料如下所示：
node 1: node1 \
attributes standby=off
node 2: node2
node 3: node3
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.13-10.el7_2.4-44eb2dd \
cluster-infrastructure=corosync \
stonith-watchdog-timeout=false \
stonith-enabled=false \
default-resource-stickiness=50
# vim: set filetype=pcmk:
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure# cd
crm(live)# status
Last updated: Fri Sep 23 11:07:47 2016 Last change: Fri Sep 23 11:07:04 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 0 resources configured
Online: [ node1 node2 node3 ]
Node1、Node2和node3
# yum -y install pcs
# systemctl start pcsd.service
[root@node1 ~]# vim /etc/ansible/hosts
[ha]
192.168.1.131
192.168.1.132
192.168.1.133
[root@node1 ~]# ansible ha -m shell -a 'echo mageedu | passwd --stdin hacluster'
[root@node1 ~]# pcs cluster auth node1 node2 node3
Username: hacluster
Password:
node1: Authorized
node3: Authorized
node2: Authorized
[root@node1 ~]# pcs status
Cluster name:
WARNING: corosync and pacemaker node names do not match (IPs used in setup?)
Last updated: Fri Sep 23 12:17:48 2016 Last change: Fri Sep 23 11:07:04 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 0 resources configured
Online: [ node1 node2 node3 ]
Full list of resources:
PCSD Status:
node1 (192.168.1.131): Unable to authenticate
node2 (192.168.1.132): Unable to authenticate
node3 (192.168.1.133): Unable to authenticate
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
# vim /etc/corosync/corosync.conf
修改
setauth: on
为
setauth: off
[root@node1 ~]# ansible ha -m service -a 'name=corosync state=restarted'192.168.1.131 | SUCCESS => {
"changed": true,
"name": "corosync",
"state": "started"
}
192.168.1.132 | SUCCESS => {
"changed": true,
"name": "corosync",
"state": "started"
}
192.168.1.133 | SUCCESS => {
"changed": true,
"name": "corosync",
"state": "started"
}
[root@node1 ~]# ansible ha -m service -a 'name=pacemaker state=stopped'
[root@node1 ~]# ansible ha -m service -a 'name=corosync state=stoppedd'
[root@node1 ~]# pcs cluster setup --name=mycluster node1 node2 node3 --force
[root@node1 ~]# pcs cluster start --all
[root@node1 ~]# corosync-cfgtool -s
Printing ring status.
Local node ID 1
RING ID 0
id = 192.168.1.131
status = ring 0 active with no faults
[root@node1 ~]# pcs resource create webip ocf:heartbeat:IPaddr ip="192.168.1.80" op monitor interval=20s timeout=10s
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 12:40:55 2016 Last change: Fri Sep 23 12:40:29 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 1 resource configured
Online: [ node1 ]
OFFLINE: [ node2 node3 ]
Full list of resources:
webip (ocf::heartbeat:IPaddr): Started node1
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
[root@node1 ~]# pcs resource delete webip
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 12:44:06 2016 Last change: Fri Sep 23 12:43:20 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 0 resources configured
Online: [ node1 ]
OFFLINE: [ node2 node3 ]
Full list of resources:
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
[root@node1 ~]# pcs resource create webip ocf:heartbeat:IPaddr ip="192.168.1.80" op monitor interval=20s timeout=10s
[root@node1 ~]# pcs resource create webstore ocf:heartbeat:Filesystem device="192.168.1.121:/www/htdocs" directory="/var/www/html" fstype="nfs" op start timeout=60s op stop timeout=60s op monitor interval=20s timeout=40s
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 12:54:11 2016 Last change: Fri Sep 23 12:51:33 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 2 resources configured
Online: [ node1 ]
OFFLINE: [ node2 node3 ]
Full list of resources:
webip (ocf::heartbeat:IPaddr): Started node1
webstore (ocf::heartbeat:Filesystem): Started node1
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
[root@node1 ~]# pcs resource create webserver systemd:httpd op monitor interval=30s timeout=20s
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 12:55:56 2016 Last change: Fri Sep 23 12:55:36 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 ]
OFFLINE: [ node2 node3 ]
Full list of resources:
webip (ocf::heartbeat:IPaddr): Started node1
webstore (ocf::heartbeat:Filesystem): Started node1
webserver (systemd:httpd): Started node1
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
[root@node1 ~]# pcs resource group add webservice webip webstore webserver
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 12:59:38 2016 Last change: Fri Sep 23 12:59:15 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 ]
OFFLINE: [ node2 node3 ]
Full list of resources:
Resource Group: webservice
webip (ocf::heartbeat:IPaddr): Started node1
webstore (ocf::heartbeat:Filesystem): Started node1
webserver (systemd:httpd): Started node1
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
让node1、node2结点下线
[root@node1 ~]# pcs cluster standby node1
[root@node1 ~]# pcs cluster standby node2
让node1、node2结点上线
[root@node1 ~]# pcs cluster unstandby node1
[root@node1 ~]# pcs cluster unstandby node2
添加约束
[root@node1 ~]# pcs constraint location add webservice_pref_node1 webservice node1 100
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 13:22:02 2016 Last change: Fri Sep 23 13:21:24 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 node2 ]
OFFLINE: [ node3 ]
Full list of resources:
Resource Group: webservice
webip (ocf::heartbeat:IPaddr): Started node1
webstore (ocf::heartbeat:Filesystem): Started node1
webserver (systemd:httpd): Started node1
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
[root@node1 ~]# pcs constraint location add webservice_pref_node1 webip node1 INFINITY
[root@node1 ~]# pcs status
Cluster name: mycluster
Last updated: Fri Sep 23 13:24:57 2016 Last change: Fri Sep 23 13:24:21 2016 by root via cibadmin on node1
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7_2.4-44eb2dd) - partition with quorum
3 nodes and 3 resources configured
Online: [ node1 node2 ]
OFFLINE: [ node3 ]
Full list of resources:
Resource Group: webservice
webip (ocf::heartbeat:IPaddr): Started node1
webstore (ocf::heartbeat:Filesystem): Started node1
webserver (systemd:httpd): Started node1
PCSD Status:
node1: Online
node2: Online
node3: Online
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/disabled
查看全局属性
[root@node1 ~]# pcs property list --all