ansible管理机：192.168.8.35 名称：kick
tomcat主机: 192.168.8.234，192.168.8.235
VIP : 192.168.8.100
系统版本： Centos7.2
##### 实验软件下载地址
配置文件： http://pan.baidu.com/s/1eSou1Bs
NFS: http://pan.baidu.com/s/1kVieBgN
DRBD: http://pan.baidu.com/s/1nvkIxtR
######### 下载drbd 软件及模块
wget http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm
wget http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm
######### 将下载软件复制到各节点中去，也可以直接下载
[root@kick ~]# ansible store -m copy -a "src=drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm dest=/tmp/"
[root@kick ~]# ansible store -m copy -a "src=kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm dest=/tmp/"
######### 三步可以合一起直接yum -y install http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm
[root@kick ~]# ansible store -m shell -a "yum -y install /tmp/drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm"
[root@kick ~]# ansible store -m shell -a "yum -y reinstall /tmp/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm"
######### 加载drbd模块
[root@kick ~]# ansible store -m shell -a "modprobe drbd"
######### 查看drbd模块是否已加载上
[root@kick ~]# ansible store -m shell -a "lsmod | grep drbd"
192.168.8.235 | SUCCESS | rc=0 >>
drbd 392583 0
libcrc32c 12644 2 xfs,drbd
192.168.8.234 | SUCCESS | rc=0 >>
drbd 392583 0
libcrc32c 12644 2 xfs,drbd
# 新加一个磁盘分区不需要格式化
######### 配置drbd全局配置及资源
[root@kick ~]# ansible store -m copy -a "src=global_common.conf dest=/etc/drbd.d/"
[root@kick ~]# ansible store -m copy -a "src=mydrbd.res dest=/etc/drbd.d/"
[root@kick ~]# vim global_common.conf
global {
usage-count no;
}
common {
handlers {
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
}
startup {
}
options {
}
disk {
on-io-error detach;
}
net {
cram-hmac-alg "sha1";
shared-secret "xiong123";
}
syncer {
rate 1024M;
}
}
################# 资源叫mydrbd,主机名称一定要在本地的/etc/hosts中增加主机名称解析否则会报错
[root@kick ~]# vim mydrbd.res
resource mydrbd{
device /dev/drbd0;
disk /dev/sdb1;
meta-disk internal;
on store1 {
address 192.168.8.234:7789;
}
on store2 {
address 192.168.8.235:7789;
}
}
# 更改hosts主机名称
[root@kick ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.8.234 store1
192.168.8.235 store2
[root@kick ~]# ansible store -m copy -a "src=/etc/hosts dest=/etc/hosts"
######### 配置mydrbd资源并启动服务,查看资源信息两个没有主的时候都是seconary
[root@kick ~]# ansible store -m shell -a "drbdadm create-md mydrbd"
[root@kick ~]# ansible store -m shell -a "systemctl start drbd"
[root@kick ~]# ansible store -m shell -a "drbd-overview"
# 随便登陆一台设置成主节点
[root@store1 ~]# drbdadm primary --force mydrbd
# 测试查看
######### 安装nfs软件，也可以直接安装nfs-utils-1.3.0-0.33.el7.x86_64.rpm 这个版本，最后有错误记录
[root@kick ~]# ansible store -m yum -a "name=nfs-utils state=installed"
######### 配置keepalived，直接使用ansible安装即可
################# 主库 ##################################
! Configuration File for keepalived
global_defs {
notification_email {
xiong@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script check_run {
script "/etc/keepalived/notify_check.sh"
interval 10
weight 10
}
vrrp_instance nfs {
state MASTER
interface eno16780032
virtual_router_id 70
priority 100
advert_int 1
nopreempt # 不让它强制抢占资源
authentication {
auth_type PASS
auth_pass PPVk7CXXZ5U
}
track_script {
check_run weight 20 # 检查节点信息状态，关闭keepalived,之后它会调取notify_stop
}
notify_stop /etc/keepalived/notify_stop.sh # 当节点stop后停止所有服务
notify_master /etc/keepalived/notify_master.sh # 当节点成为master时开启所有服务
virtual_ipaddress {
192.168.8.100
}
}
################# 备库 ##################################
! Configuration File for keepalived
global_defs {
notification_email {
xiong@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script check_run {
script "/etc/keepalived/notify_check.sh"
interval 10
weight 10
}
vrrp_instance nfs {
state BACKUP
interface eno16780032
virtual_router_id 70
priority 95
advert_int 1
nopreempt # 不让它强制抢占资源
authentication {
auth_type PASS
auth_pass PPVk7CXXZ5U
}
track_script {
check_run weight 20
}
notify_stop /etc/keepalived/notify_stop.sh
notify_master /etc/keepalived/notify_master.sh
virtual_ipaddress {
192.168.8.100
}
}
#############################################################
######### 配置NFS
[root@kick ~]# cat /etc/exports
/data 192.168.8.0/24(rw,async,all_squash,anonuid=2021,anongid=2020)
######### 将配置文件复制到store两个drbd服务器中
[root@kick data]# ansible store -m copy -a "src=exports dest=/etc/"
######### 创建mysql用户mysql组 后续使用
[root@kick /]# ansible store -m shell -a "groupadd -g 2020 mysql"
[root@kick /]# ansible store -m shell -a "useradd -g 2020 -u 2021 mysql -s /sbin/nologin"
[root@kick /]# ansible store -m shell -a "chown mysql.mysql /data/ -R"
######### 复制脚本文件到各节点中
[root@kick ~]# ansible store -m copy -a "src=notify_master.sh dest=/etc/keepalived/"
[root@kick ~]# ansible store -m copy -a "src=notify_stop.sh dest=/etc/keepalived/"
[root@kick ~]# ansible store -m copy -a "src=notify_check.sh dest=/etc/keepalived/"
[root@kick ~]# ansible store -m copy -a "src=keepalived_store_drbd.conf dest=/etc/keepalived/keepalived.conf"
######################### 脚本内容 #####################################################
[root@kick ~]# cat notify_master.sh # 当keepalived为主节点时重启所有服务并设置drbd为主
#!/bin/bash
#
systemctl start rpcbind &>/dev/null
systemctl start nfs &>/dev/null
systemctl start drbd &>/dev/null
/usr/sbin/drbdadm primary mydrbd &>/tmp/node_change.txt
mount /dev/drbd0 /data &>/tmp/node_change.txt
[root@kick ~]# cat notify_stop.sh # 当检测节点为back时直接停停止各项服务
#!/bin/bash
#
umount /data
systemctl stop rpcbind &>/dev/null
systemctl stop nfs &>/dev/null
drbdadm secondary mydrbd &>/tmp/node_change.txt
echo -e "\n" &> /tmp/node_change.txt
systemctl stop keepalived &>/dev/null
[root@kick ~]# cat notify_check.sh # 检查服务的正常性 当NFS服务不正常时直接停止keepalived然后再调取notify_stop脚本
#!/bin/sh
###检查nfs可用性：进程和是否能够挂载 # 备用脚本###### check使用下面那个
systemctl status nfs &>/dev/null
if [ $? -ne 0 ];then
###如果服务状态不正常，先尝试重启服务
systemctl restart restart
systemctl status nfs &>/dev/null
if [ $? -ne 0 ];then
###若重启nfs服务后，仍不正常
###卸载drbd设备
umount /dev/drbd0
###将drbd主降级为备
drbdadm secondary r0
#关闭keepalived
systemctl stop keepalived
fi
fi
################################ 检查三项服务正常，当其中的任何一个不正常都直接干掉 ##################
#/bin/bash
#
systemctl status rpcbind &>/dev/null
VAR1=$?
systemctl status nfs &>/dev/null
VAR2=$?
systemctl status drbd &>/dev/null
VAR3=$?
if [[ $VAR1 -ne 0 || $VAR2 -ne 0 || $VAR3 -ne 0 ]];then
systemctl stop rpcbind &>/dev/null
systemctl stop nfs &>/dev/null
umount /data
drbdadm secondary mydrbd &>/tmp/node_change.txt
echo -e "\n" >> /tmp/node_change.txt
systemctl stop keepalived
systemctl restart drbd
fi
##########################################################################################
############## 启动服务 ###########
[root@kick data]# ansible store -m shell -a "systemctl start keepalived"
[root@kick data]# ansible store -m shell -a "systemctl start rpcbind"
[root@kick data]# ansible store -m shell -a "systemctl start nfs"
############### 配合日志查看 ############
[root@kick data]# ansible store -m shell -a "systemctl status keepalived"
[root@kick data]# ansible store -m shell -a "systemctl status nfs"
[root@kick data]# ansible store -m shell -a "cat /var/log/message"
##########################################################################################
# 检查VIP地址以及NFS是否挂载
[root@kick ~]# showmount -e 192.168.8.100
Export list for 192.168.8.100:
/data 192.168.8.0/24
################################## NFS 出现的一些错误 #################################
[root@kick ~]# mount -t nfs 192.168.8.234:/data /data
mount.nfs: access denied by server while mounting 192.168.8.234:/data
#在/etc/exports中增加insecure，http://blog.chinaunix.net/uid-20554957-id-3444786.html
/data 192.168.8.0(rw,insecure,no_root_squash,async)
[root@store1 data]# systemctl restart nfs
###### 一直报这个错 ###### 排查确认是 centos7 关于nfs 1.3.0-0.21.el7_2 版本的bug
Jun 27 09:29:52 store1 nfsdcltrack[12174]: sqlite_insert_client: insert statement prepare failed: table clients has 2 columns but 3 values were supplied
# 将版本直接升级至nfs-utils-1.3.0-0.33.el7.x86_64
# 需要的软件包 libtirpc需要大于0.2.4-0.7，libtirpc-devel，nfs-utils-1.3.0-0.33.el7
[root@kick ~]# ansible store -m copy -a "src=libtirpc-0.2.4-0.8.el7.x86_64.rpm dest=/tmp"
[root@kick ~]# ansible store -m copy -a "src=libtirpc-devel-0.2.4-0.8.el7.x86_64.rpm dest=/tmp"
[root@kick ~]# ansible store -m copy -a "src=nfs-utils-1.3.0-0.33.el7.x86_64.rpm dest=/tmp"
[root@kick ~]# ansible store -m shell -a "rpm -Uvh /tmp/libtirpc-0.2.4-0.8.el7.x86_64.rpm"
[root@kick ~]# ansible store -m shell -a "rpm -Uvh /tmp/libtirpc-devel-0.2.4-0.8.el7.x86_64.rpm"
[root@kick ~]# ansible store -m shell -a "rpm -Uvh /tmp/nfs-utils-1.3.0-0.33.el7.x86_64.rpm"