马哥N49第十六周作业

全程和就业

1、编写脚本，支持让用户自主选择，使用mysqldump还是xtraback全量备份。

1.1 编写脚本

#!/bin/bash

backup_dir="/data"
user="root"
password=""
host='127.0.0.1'

xtrabackup_all (){
    xtrabackup --user=${user} --password=${password} --host=${host}  --backup --target-dir=${backup_dir}/base &> /data/base.log
    [ $? -eq 0 ] && echo "xtrabackup done!" || echo "xtrabackup error, detail in /data/base.log"
}

mysqldump_all (){
    backup_time=`date +"%F-%H"`
    mysqldump --user=${user} --password=${password} -A -F -E -R --triggers \
    --single-transaction \
    --master-data=1 \
    --flush-privileges \
    --default-character-set=utf8 \
    --hex-blob >${backup_dir}/fullbak_${backup_time}.sql
    [ $? -eq 0 ] && echo "mysqldump done!" || echo "mysqldump error, detail in /var/log/mariadb/mariadb.log"
}

while true;do
    echo -en "\E[$[RANDOM%7+31];1m"
    cat <

1.2 测试脚本

[root@web1 ~]# bash backup_mysql.sh 
    请选择：
    1）xtrabackup全量备份数据库
    2）mysqldump全量备份数据库
    3) 退出

请输入数字选择备份方法：1
xtrabackup done!
[root@web1 ~]# bash backup_mysql.sh 
    请选择：
    1）xtrabackup全量备份数据库
    2）mysqldump全量备份数据库
    3) 退出

请输入数字选择备份方法：2
mysqldump: Got error: 1045: "Access denied for user 'root'@'localhost' (using password: YES)" when trying to connect
mysqldump error, detail in /var/log/mariadb/mariadb.log 
[root@web1 ~]# bash backup_mysql.sh 
    请选择：
    1）xtrabackup全量备份数据库
    2）mysqldump全量备份数据库
    3) 退出

请输入数字选择备份方法：2
mysqldump done!
[root@web1 ~]# 
[root@web1 ~]# bash backup_mysql.sh 
    请选择：
    1）xtrabackup全量备份数据库
    2）mysqldump全量备份数据库
    3) 退出

请输入数字选择备份方法：5
数字错误，请重新选择
    请选择：
    1）xtrabackup全量备份数据库
    2）mysqldump全量备份数据库
    3) 退出

请输入数字选择备份方法：3
[root@web1 ~]# 

2、配置Mysql主从同步

2.1 mysql-master

• 二进制安装mariadb-10.2.7

[root@mysql1 ~]# useradd -r -s /sbin/nologin mysql
[root@mysql1 ~]# tar xvf mariadb-10.2.7-linux-x86_64.tar.gz -C /usr/local/
[root@mysql1 ~]# cd /usr/local/
[root@mysql1 ~]# ln -sv mariadb-10.2.7-linux-x86_64 mysql
[root@mysql1 ~]# cd mysql/
[root@mysql1 ~]# chown -R root.root ./*
[root@mysql1 ~]# mkdir /data/mysql -p
[root@mysql1 ~]# chown -R mysql.mysql /data/mysql
[root@mysql1 ~]# mkdir /etc/mysql
[root@mysql1 ~]# cp support-files/my-huge.cnf  /etc/mysql/my.cnf

• 设置环境变量

[root@mysql1 ~]# vim /etc/profile.d/mysql.sh
PATH=/usr/local/mysql/bin/:$PATH
[root@mysql1 ~]# . /etc/profile.d/mysql.sh

• 运行初始化脚本，初始化数据库

[root@mysql1 ~]# yum -y install libaio -y
[root@mysql1 ~]# cd /usr/local/mysql/
[root@mysql1 ~]# scripts/mysql_install_db --user=mysql --datadir=/data/mysql

• 复制启动脚本，设置开机自启动

[root@mysql1 ~]# cp support-files/mysql.server /etc/rc.d/init.d/mysqld
[root@mysql1 ~]# chkconfig --add mysqld
[root@mysql1 ~]# service mysqld start

• 开启二进制日志，并设置全局唯一的server_id

[root@mysql1 ~]# yum -y install ncurses-compat-libs
[root@mysql1 ~]# mysql -uroot
[root@mysql1 ~]# vim /etc/mysql/my.cnf 
[mysqld]
server-id = 48
log_bin
log-basename = master
datadir = /data/mysql
skip_name_resolve = ON
port        = 3306
socket      = /tmp/mysql.sock

• 查看二进制日志位置，并创建主从复制账号

[root@mysql1 ~]# mysql
MariaDB [(none)]> show master logs;
MariaDB [(none)]> grant replication slave on *.* to repluser@'10.0.0.%' identified by 'magedu';
MariaDB [(none)]> create database wordpress;
MariaDB [(none)]> grant all on wordpress.* to wpuser@'10.0.0.%' identified by "wppass";
[root@mysql1 ~]# service mysqld restart

2.2 mysql-slave

• 二进制包安装mariadb-10.2.7

[root@mysql2 ~]# useradd -r -s /sbin/nologin mysql
[root@mysql2 ~]# tar xvf mariadb-10.2.7-linux-x86_64.tar.gz -C /usr/local/
[root@mysql2 ~]# cd /usr/local/
[root@mysql2 ~]# ln -sv mariadb-10.2.7-linux-x86_64 mysql
[root@mysql2 ~]# cd mysql/
[root@mysql2 ~]# chown -R root.root ./*
[root@mysql2 ~]# mkdir /data/mysql
[root@mysql2 ~]# chown -R mysql.mysql /data/mysql
[root@mysql2 ~]# mkdir /etc/mysql
[root@mysql2 ~]# cp support-files/my-huge.cnf /etc/mysql/my.cnf

• 启用二进制日志和设置server_id，server_id必须唯一

[root@mysql2 ~]# vim /etc/mysql/my.cnf 
         [mysqld]
         server_id = 58
         datadir = /data/mysql
         skip_name_resolve  = ON
         port            = 3306
         socket          = /tmp/mysql.sock

• 设置环境变量

[root@mysql2 ~]# vim /etc/profile.d/mysql.sh
         PATH=/usr/local/mysql/bin/:$PATH
[root@mysql2 ~]# . /etc/profile.d/mysql.sh

• 运行初始化脚本，初始化数据库

[root@mysql2 ~]# yum -y install libaio ncurses-compat-libs
[root@mysql2 ~]# cd /usr/local/mysql/
[root@mysql2 ~]# scripts/mysql_install_db --user=mysql --datadir=/data/mysql

• 复制启动脚本，并添加开机自启动

[root@mysql2 ~]# cp support-files/mysql.server /etc/rc.d/init.d/mysqld
[root@mysql2 ~]# chkconfig --add mysqld
[root@mysql2 ~]# service mysqld start

• 启动MySQL的slave进程

MariaDB [(none)]>CHANGE MASTER TO \
         MASTER_HOST='10.0.0.48',\
         MASTER_USER='repluser',\
         MASTER_PASSWORD='magedu',\
         MASTER_PORT=3306,\
         MASTER_LOG_FILE='mysql-bin.000002',\
         MASTER_LOG_POS=807,\
         MASTER_CONNECT_RETRY=10;\
MariaDB [(none)]>start slave;

2.3 测试主从复制

• 查看主从复制信息

MariaDB [(none)]> show slave status\G
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 10.0.0.48
                  Master_User: repluser
                  Master_Port: 3306
                Connect_Retry: 10
              Master_Log_File: mysql-bin.000002
          Read_Master_Log_Pos: 461407
               Relay_Log_File: mysql2-relay-bin.000005
                Relay_Log_Pos: 9321
        Relay_Master_Log_File: mysql-bin.000003
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
              Replicate_Do_DB: 
          Replicate_Ignore_DB: 
           Replicate_Do_Table: 
       Replicate_Ignore_Table: 
      Replicate_Wild_Do_Table: 
  Replicate_Wild_Ignore_Table: 
                   Last_Errno: 0
                   Last_Error: 
                 Skip_Counter: 0
          Exec_Master_Log_Pos: 461407
              Relay_Log_Space: 246663
              Until_Condition: None
               Until_Log_File: 
                Until_Log_Pos: 0
           Master_SSL_Allowed: No
           Master_SSL_CA_File: 
           Master_SSL_CA_Path: 
              Master_SSL_Cert: 
            Master_SSL_Cipher: 
               Master_SSL_Key: 
        Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
                Last_IO_Errno: 0
                Last_IO_Error: 
               Last_SQL_Errno: 0
               Last_SQL_Error: 
  Replicate_Ignore_Server_Ids: 
             Master_Server_Id: 24
               Master_SSL_Crl: 
           Master_SSL_Crlpath: 
                   Using_Gtid: No
                  Gtid_IO_Pos: 
      Replicate_Do_Domain_Ids: 
  Replicate_Ignore_Domain_Ids: 
                Parallel_Mode: conservative
                    SQL_Delay: 0
          SQL_Remaining_Delay: NULL
      Slave_SQL_Running_State: Slave has read all relay log; waiting for the slave I/O thread to update it
1 row in set (0.00 sec)

MariaDB [(none)]>

• 在master新建一个testdb的数据库

MariaDB [(none)]> create database testdb;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> 
MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testdb             |
+--------------------+
5 rows in set (0.00 sec)

MariaDB [(none)]> 

• 在slave查询testdb是否被复制过来

MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testdb             |
+--------------------+
5 rows in set (0.00 sec)

MariaDB [(none)]> 

3、使用MHA实现Mysql高可用。

3.1 环境准备

manager：10.0.0.20 安装mha4mysql-manager、mha4mysql-node
master：10.0.0.21 安装mha4mysql-node
slave1： 10.0.0.26 安装mha4mysql-node
slave2： 10.0.0.27 安装mha4mysql-node

• 下载mha4mysql-manager和mha4mysql-node软件包，由于年代久远，只有rhel6的安装包，在centos7还能安装，8不支持。
  https://github.com/yoshinorim/mha4mysql-manager/wiki/Downloads

3.2 mha的配置

• 在manager节点安装manager和node包

yum -y install mha4mysql-*

• 在master和所有slave安装node包

yum -y install mha4mysql-node-0.56-0.el6.noarch.rpm

• 所有节点实现ssh key验证登录

ssh-keygen
ssh-copy-id 10.0.0.20
rsync -av .ssh 10.0.0.21:/root/
rsync -av .ssh 10.0.0.26:/root/
rsync -av .ssh 10.0.0.27:/root/

• 在manager建立配置文件

[root@manager ~]# mkdir /etc/mastermha/
[root@manager ~]# vim /etc/mastermha/app1.cnf
[root@manager ~]# cat /etc/mastermha/app1.cnf
[server default]
user=mhauser
password=magedu
manager_workdir=/data/mastermha/app1/
manager_log=/data/mastermha/app1/manager.log
remote_workdir=/data/mastermha/app1/
ssh_user=root
repl_user=repluser
repl_password=magedu
ping_interval=1

[server1]
hostname=10.0.0.21
candidate_master=1

[server2]
hostname=10.0.0.26
candidate_master=1

[server3]
hostname=10.0.0.27

• 配置master

[root@master ~]# yum -y install mariadb-server
[root@master ~]# vim /etc/my.cnf
[mysqld]
log-bin
server_id=21
skip_name_resolve=1
[root@master ~]# mysql
MariaDB> show master logs;
+--------------------+-----------+
| Log_name           | File_size |
+--------------------+-----------+
| mariadb-bin.000001 |       245 |
+--------------------+-----------+

MariaDB> grant replication slave on *.* to repluser@'10.0.0.%' identified by 'magedu';
MariaDB> grant all on *.* to mhauser@'10.0.0.%' identified by 'magedu';

• slave1的配置

[root@slave1 ~]# yum -y install mariadb-server
[root@slave1 ~]# vim /etc/my.cnf
[mysqld]
server_id=26
log-bin
read-only
relay_log_purge=0
skip_name_resolve=1

[root@slave1 ~]# mysql
MariaDB> CHANGE MASTER TO MASTER_HOST='10.0.0.21', MASTER_USER='repluser',\
MASTER_PASSWORD='magedu', MASTER_LOG_FILE='mariadb-bin.000001',\
MASTER_LOG_POS=245;

MariaDB> start slave;
MariaDB> show slave status\G

• slave2的配置

[root@slave2 ~]# yum -y install mariadb-server
[root@slave2 ~]# vim /etc/my.cnf
[mysqld]
server_id=27
log-bin
read-only
relay_log_purge=0
skip_name_resolve=1

[root@slave2 ~]# mysql
MariaDB> CHANGE MASTER TO MASTER_HOST='10.0.0.21', MASTER_USER='repluser',\
MASTER_PASSWORD='magedu', MASTER_LOG_FILE='mariadb-bin.000001',\
MASTER_LOG_POS=245;

MariaDB> start slave;
MariaDB> show slave status\G

• 在manager检查mha ssh连接

[root@manager ~]# masterha_check_ssh --conf=/etc/mastermha/app1.cnf 
Mon Jan 25 19:11:42 2021 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Mon Jan 25 19:11:42 2021 - [info] Reading application default configuration from /etc/mastermha/app1.cnf..
Mon Jan 25 19:11:42 2021 - [info] Reading server configuration from /etc/mastermha/app1.cnf..
Mon Jan 25 19:11:42 2021 - [info] Starting SSH connection tests..
Mon Jan 25 19:11:44 2021 - [debug] 
Mon Jan 25 19:11:43 2021 - [debug]  Connecting via SSH from [email protected](10.0.0.27:22) to [email protected](10.0.0.21:22)..
Mon Jan 25 19:11:43 2021 - [debug]   ok.
Mon Jan 25 19:11:43 2021 - [debug]  Connecting via SSH from [email protected](10.0.0.27:22) to [email protected](10.0.0.26:22)..
Mon Jan 25 19:11:44 2021 - [debug]   ok.
Mon Jan 25 19:11:53 2021 - [debug] 
Mon Jan 25 19:11:42 2021 - [debug]  Connecting via SSH from [email protected](10.0.0.26:22) to [email protected](10.0.0.21:22)..
Mon Jan 25 19:11:53 2021 - [debug]   ok.
Mon Jan 25 19:11:53 2021 - [debug]  Connecting via SSH from [email protected](10.0.0.26:22) to [email protected](10.0.0.27:22)..
Mon Jan 25 19:11:53 2021 - [debug]   ok.
Mon Jan 25 19:11:58 2021 - [debug] 
Mon Jan 25 19:11:42 2021 - [debug]  Connecting via SSH from [email protected](10.0.0.21:22) to [email protected](10.0.0.26:22)..
Mon Jan 25 19:11:48 2021 - [debug]   ok.
Mon Jan 25 19:11:48 2021 - [debug]  Connecting via SSH from [email protected](10.0.0.21:22) to [email protected](10.0.0.27:22)..
Mon Jan 25 19:11:58 2021 - [debug]   ok.
Mon Jan 25 19:11:58 2021 - [info] All SSH connection tests passed successfully.

• 在manager检查集群复制状态

[root@keepalived1 ~]# masterha_check_repl --conf=/etc/mastermha/app1.cnf
Mon Jan 25 19:13:53 2021 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Mon Jan 25 19:13:53 2021 - [info] Reading application default configuration from /etc/mastermha/app1.cnf..
Mon Jan 25 19:13:53 2021 - [info] Reading server configuration from /etc/mastermha/app1.cnf..
Mon Jan 25 19:13:53 2021 - [info] MHA::MasterMonitor version 0.56.
Creating directory /data/mastermha/app1/.. done.
Mon Jan 25 19:13:54 2021 - [info] GTID failover mode = 0
Mon Jan 25 19:13:54 2021 - [info] Dead Servers:
Mon Jan 25 19:13:54 2021 - [info] Alive Servers:
Mon Jan 25 19:13:54 2021 - [info]   10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:13:54 2021 - [info]   10.0.0.26(10.0.0.26:3306)
Mon Jan 25 19:13:54 2021 - [info]   10.0.0.27(10.0.0.27:3306)
Mon Jan 25 19:13:54 2021 - [info] Alive Slaves:
Mon Jan 25 19:13:54 2021 - [info]   10.0.0.26(10.0.0.26:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:13:54 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:13:54 2021 - [info]     Primary candidate for the new Master (candidate_master is set)
Mon Jan 25 19:13:54 2021 - [info]   10.0.0.27(10.0.0.27:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:13:54 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:13:54 2021 - [info] Current Alive Master: 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:13:54 2021 - [info] Checking slave configurations..
Mon Jan 25 19:13:54 2021 - [warning]  relay_log_purge=0 is not set on slave 10.0.0.26(10.0.0.26:3306).
Mon Jan 25 19:13:54 2021 - [warning]  relay_log_purge=0 is not set on slave 10.0.0.27(10.0.0.27:3306).
Mon Jan 25 19:13:54 2021 - [info] Checking replication filtering settings..
Mon Jan 25 19:13:54 2021 - [info]  binlog_do_db= , binlog_ignore_db= 
Mon Jan 25 19:13:54 2021 - [info]  Replication filtering check ok.
Mon Jan 25 19:13:54 2021 - [info] GTID (with auto-pos) is not supported
Mon Jan 25 19:13:54 2021 - [info] Starting SSH connection tests..
Mon Jan 25 19:14:17 2021 - [info] All SSH connection tests passed successfully.
Mon Jan 25 19:14:17 2021 - [info] Checking MHA Node version..
Mon Jan 25 19:14:27 2021 - [info]  Version check ok.
Mon Jan 25 19:14:27 2021 - [info] Checking SSH publickey authentication settings on the current master..
Mon Jan 25 19:14:27 2021 - [info] HealthCheck: SSH to 10.0.0.21 is reachable.
Mon Jan 25 19:14:38 2021 - [info] Master MHA Node version is 0.56.
Mon Jan 25 19:14:38 2021 - [info] Checking recovery script configurations on 10.0.0.21(10.0.0.21:3306)..
Mon Jan 25 19:14:38 2021 - [info]   Executing command: save_binary_logs --command=test --start_pos=4 --binlog_dir=/var/lib/mysql,/var/log/mysql --output_file=/data/mastermha/app1//save_binary_logs_test --manager_version=0.56 --start_file=mariadb-bin.000001 
Mon Jan 25 19:14:38 2021 - [info]   Connecting to [email protected](10.0.0.21:22).. 
  Creating /data/mastermha/app1 if not exists.. Creating directory /data/mastermha/app1.. done.
   ok.
  Checking output directory is accessible or not..
   ok.
  Binlog found at /var/lib/mysql, up to mariadb-bin.000001
Mon Jan 25 19:14:48 2021 - [info] Binlog setting check done.
Mon Jan 25 19:14:48 2021 - [info] Checking SSH publickey authentication and checking recovery script configurations on all alive slave servers..
Mon Jan 25 19:14:48 2021 - [info]   Executing command : apply_diff_relay_logs --command=test --slave_user='mhauser' --slave_host=10.0.0.26 --slave_ip=10.0.0.26 --slave_port=3306 --workdir=/data/mastermha/app1/ --target_version=5.5.68-MariaDB --manager_version=0.56 --relay_log_info=/var/lib/mysql/relay-log.info  --relay_dir=/var/lib/mysql/  --slave_pass=xxx
Mon Jan 25 19:14:48 2021 - [info]   Connecting to [email protected](10.0.0.26:22).. 
Creating directory /data/mastermha/app1/.. done.
  Checking slave recovery environment settings..
    Opening /var/lib/mysql/relay-log.info ... ok.
    Relay log found at /var/lib/mysql, up to mariadb-relay-bin.000002
    Temporary relay log file is /var/lib/mysql/mariadb-relay-bin.000002
    Testing mysql connection and privileges.. done.
    Testing mysqlbinlog output.. done.
    Cleaning up test file(s).. done.
Mon Jan 25 19:14:58 2021 - [info]   Executing command : apply_diff_relay_logs --command=test --slave_user='mhauser' --slave_host=10.0.0.27 --slave_ip=10.0.0.27 --slave_port=3306 --workdir=/data/mastermha/app1/ --target_version=5.5.68-MariaDB --manager_version=0.56 --relay_log_info=/var/lib/mysql/relay-log.info  --relay_dir=/var/lib/mysql/  --slave_pass=xxx
Mon Jan 25 19:14:58 2021 - [info]   Connecting to [email protected](10.0.0.27:22).. 
Creating directory /data/mastermha/app1/.. done.
  Checking slave recovery environment settings..
    Opening /var/lib/mysql/relay-log.info ... ok.
    Relay log found at /var/lib/mysql, up to mariadb-relay-bin.000002
    Temporary relay log file is /var/lib/mysql/mariadb-relay-bin.000002
    Testing mysql connection and privileges.. done.
    Testing mysqlbinlog output.. done.
    Cleaning up test file(s).. done.
Mon Jan 25 19:14:58 2021 - [info] Slaves settings check done.
Mon Jan 25 19:14:58 2021 - [info] 
10.0.0.21(10.0.0.21:3306) (current master)
 +--10.0.0.26(10.0.0.26:3306)
 +--10.0.0.27(10.0.0.27:3306)

Mon Jan 25 19:14:58 2021 - [info] Checking replication health on 10.0.0.26..
Mon Jan 25 19:14:58 2021 - [info]  ok.
Mon Jan 25 19:14:58 2021 - [info] Checking replication health on 10.0.0.27..
Mon Jan 25 19:14:58 2021 - [info]  ok.
Mon Jan 25 19:14:58 2021 - [warning] master_ip_failover_script is not defined.
Mon Jan 25 19:14:58 2021 - [warning] shutdown_script is not defined.
Mon Jan 25 19:14:58 2021 - [info] Got exit code 0 (Not master dead).

MySQL Replication Health is OK.
[root@keepalived1 ~]#

• 启动mha

[root@manager ~]# nohup masterha_manager --conf=/etc/mastermha/app1.cnf &> /dev/null
[root@manager ~]# masterha_check_status --conf=/etc/mastermha/app1.cnf 
app1 (pid:1885) is running(0:PING_OK), master:10.0.0.21

• 模拟master宕机

systemctl stop mariadb

• 查看日志，最后可以看到master切换到10.0.0.26

[root@keepalived1 ~]# tail -f /data/mastermha/app1/manager.log 
10.0.0.21(10.0.0.21:3306) (current master)
 +--10.0.0.26(10.0.0.26:3306)
 +--10.0.0.27(10.0.0.27:3306)

Mon Jan 25 19:45:28 2021 - [warning] master_ip_failover_script is not defined.
Mon Jan 25 19:45:28 2021 - [warning] shutdown_script is not defined.
Mon Jan 25 19:45:28 2021 - [info] Set master ping interval 1 seconds.
Mon Jan 25 19:45:28 2021 - [warning] secondary_check_script is not defined. It is highly recommended setting it to check master reachability from two or more routes.
Mon Jan 25 19:45:28 2021 - [info] Starting ping health check on 10.0.0.21(10.0.0.21:3306)..
Mon Jan 25 19:45:28 2021 - [info] Ping(SELECT) succeeded, waiting until MySQL doesn't respond..









Mon Jan 25 19:45:51 2021 - [warning] Got error on MySQL select ping: 2006 (MySQL server has gone away)
Mon Jan 25 19:45:51 2021 - [info] Executing SSH check script: save_binary_logs --command=test --start_pos=4 --binlog_dir=/var/lib/mysql,/var/log/mysql --output_file=/data/mastermha/app1//save_binary_logs_test --manager_version=0.56 --binlog_prefix=mariadb-bin
Mon Jan 25 19:45:51 2021 - [info] HealthCheck: SSH to 10.0.0.21 is reachable.
Mon Jan 25 19:45:52 2021 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '10.0.0.21' (111))
Mon Jan 25 19:45:52 2021 - [warning] Connection failed 2 time(s)..
Mon Jan 25 19:45:53 2021 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '10.0.0.21' (111))
Mon Jan 25 19:45:53 2021 - [warning] Connection failed 3 time(s)..
Mon Jan 25 19:45:54 2021 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '10.0.0.21' (111))
Mon Jan 25 19:45:54 2021 - [warning] Connection failed 4 time(s)..
Mon Jan 25 19:45:54 2021 - [warning] Master is not reachable from health checker!
Mon Jan 25 19:45:54 2021 - [warning] Master 10.0.0.21(10.0.0.21:3306) is not reachable!
Mon Jan 25 19:45:54 2021 - [warning] SSH is reachable.
Mon Jan 25 19:45:54 2021 - [info] Connecting to a master server failed. Reading configuration file /etc/masterha_default.cnf and /etc/mastermha/app1.cnf again, and trying to connect to all servers to check server status..
Mon Jan 25 19:45:54 2021 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Mon Jan 25 19:45:54 2021 - [info] Reading application default configuration from /etc/mastermha/app1.cnf..
Mon Jan 25 19:45:54 2021 - [info] Reading server configuration from /etc/mastermha/app1.cnf..
Mon Jan 25 19:45:55 2021 - [info] GTID failover mode = 0
Mon Jan 25 19:45:55 2021 - [info] Dead Servers:
Mon Jan 25 19:45:55 2021 - [info]   10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:55 2021 - [info] Alive Servers:
Mon Jan 25 19:45:55 2021 - [info]   10.0.0.26(10.0.0.26:3306)
Mon Jan 25 19:45:55 2021 - [info]   10.0.0.27(10.0.0.27:3306)
Mon Jan 25 19:45:55 2021 - [info] Alive Slaves:
Mon Jan 25 19:45:55 2021 - [info]   10.0.0.26(10.0.0.26:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:55 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:55 2021 - [info]     Primary candidate for the new Master (candidate_master is set)
Mon Jan 25 19:45:55 2021 - [info]   10.0.0.27(10.0.0.27:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:55 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:55 2021 - [info] Checking slave configurations..
Mon Jan 25 19:45:55 2021 - [warning]  relay_log_purge=0 is not set on slave 10.0.0.26(10.0.0.26:3306).
Mon Jan 25 19:45:55 2021 - [warning]  relay_log_purge=0 is not set on slave 10.0.0.27(10.0.0.27:3306).
Mon Jan 25 19:45:55 2021 - [info] Checking replication filtering settings..
Mon Jan 25 19:45:55 2021 - [info]  Replication filtering check ok.
Mon Jan 25 19:45:55 2021 - [info] Master is down!
Mon Jan 25 19:45:55 2021 - [info] Terminating monitoring script.
Mon Jan 25 19:45:55 2021 - [info] Got exit code 20 (Master dead).
Mon Jan 25 19:45:55 2021 - [info] MHA::MasterFailover version 0.56.
Mon Jan 25 19:45:55 2021 - [info] Starting master failover.
Mon Jan 25 19:45:55 2021 - [info] 
Mon Jan 25 19:45:55 2021 - [info] * Phase 1: Configuration Check Phase..
Mon Jan 25 19:45:55 2021 - [info] 
Mon Jan 25 19:45:56 2021 - [info] GTID failover mode = 0
Mon Jan 25 19:45:56 2021 - [info] Dead Servers:
Mon Jan 25 19:45:56 2021 - [info]   10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:56 2021 - [info] Checking master reachability via MySQL(double check)...
Mon Jan 25 19:45:56 2021 - [info]  ok.
Mon Jan 25 19:45:56 2021 - [info] Alive Servers:
Mon Jan 25 19:45:56 2021 - [info]   10.0.0.26(10.0.0.26:3306)
Mon Jan 25 19:45:56 2021 - [info]   10.0.0.27(10.0.0.27:3306)
Mon Jan 25 19:45:56 2021 - [info] Alive Slaves:
Mon Jan 25 19:45:56 2021 - [info]   10.0.0.26(10.0.0.26:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:56 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:56 2021 - [info]     Primary candidate for the new Master (candidate_master is set)
Mon Jan 25 19:45:56 2021 - [info]   10.0.0.27(10.0.0.27:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:56 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:56 2021 - [info] Starting Non-GTID based failover.
Mon Jan 25 19:45:56 2021 - [info] 
Mon Jan 25 19:45:56 2021 - [info] ** Phase 1: Configuration Check Phase completed.
Mon Jan 25 19:45:56 2021 - [info] 
Mon Jan 25 19:45:56 2021 - [info] * Phase 2: Dead Master Shutdown Phase..
Mon Jan 25 19:45:56 2021 - [info] 
Mon Jan 25 19:45:56 2021 - [info] Forcing shutdown so that applications never connect to the current master..
Mon Jan 25 19:45:56 2021 - [warning] master_ip_failover_script is not set. Skipping invalidating dead master IP address.
Mon Jan 25 19:45:56 2021 - [warning] shutdown_script is not set. Skipping explicit shutting down of the dead master.
Mon Jan 25 19:45:57 2021 - [info] * Phase 2: Dead Master Shutdown Phase completed.
Mon Jan 25 19:45:57 2021 - [info] 
Mon Jan 25 19:45:57 2021 - [info] * Phase 3: Master Recovery Phase..
Mon Jan 25 19:45:57 2021 - [info] 
Mon Jan 25 19:45:57 2021 - [info] * Phase 3.1: Getting Latest Slaves Phase..
Mon Jan 25 19:45:57 2021 - [info] 
Mon Jan 25 19:45:57 2021 - [info] The latest binary log file/position on all slaves is mariadb-bin.000003:245
Mon Jan 25 19:45:57 2021 - [info] Latest slaves (Slaves that received relay log files to the latest):
Mon Jan 25 19:45:57 2021 - [info]   10.0.0.26(10.0.0.26:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:57 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:57 2021 - [info]     Primary candidate for the new Master (candidate_master is set)
Mon Jan 25 19:45:57 2021 - [info]   10.0.0.27(10.0.0.27:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:57 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:57 2021 - [info] The oldest binary log file/position on all slaves is mariadb-bin.000003:245
Mon Jan 25 19:45:57 2021 - [info] Oldest slaves:
Mon Jan 25 19:45:57 2021 - [info]   10.0.0.26(10.0.0.26:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:57 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:57 2021 - [info]     Primary candidate for the new Master (candidate_master is set)
Mon Jan 25 19:45:57 2021 - [info]   10.0.0.27(10.0.0.27:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:57 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:57 2021 - [info] 
Mon Jan 25 19:45:57 2021 - [info] * Phase 3.2: Saving Dead Master's Binlog Phase..
Mon Jan 25 19:45:57 2021 - [info] 
Mon Jan 25 19:45:57 2021 - [info] Fetching dead master's binary logs..
Mon Jan 25 19:45:57 2021 - [info] Executing command on the dead master 10.0.0.21(10.0.0.21:3306): save_binary_logs --command=save --start_file=mariadb-bin.000003  --start_pos=245 --binlog_dir=/var/lib/mysql,/var/log/mysql --output_file=/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog --handle_raw_binlog=1 --disable_log_bin=0 --manager_version=0.56
  Creating /data/mastermha/app1 if not exists..    ok.
 Concat binary/relay logs from mariadb-bin.000003 pos 245 to mariadb-bin.000003 EOF into /data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog ..
  Dumping binlog format description event, from position 0 to 245.. ok.
  Dumping effective binlog data from /var/lib/mysql/mariadb-bin.000003 position 245 to tail(264).. ok.
 Concat succeeded.
Mon Jan 25 19:45:58 2021 - [info] scp from [email protected]:/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog to local:/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog succeeded.
Mon Jan 25 19:45:58 2021 - [info] HealthCheck: SSH to 10.0.0.26 is reachable.
Mon Jan 25 19:45:58 2021 - [info] HealthCheck: SSH to 10.0.0.27 is reachable.
Mon Jan 25 19:45:58 2021 - [info] 
Mon Jan 25 19:45:58 2021 - [info] * Phase 3.3: Determining New Master Phase..
Mon Jan 25 19:45:58 2021 - [info] 
Mon Jan 25 19:45:58 2021 - [info] Finding the latest slave that has all relay logs for recovering other slaves..
Mon Jan 25 19:45:58 2021 - [info] All slaves received relay logs to the same position. No need to resync each other.
Mon Jan 25 19:45:58 2021 - [info] Searching new master from slaves..
Mon Jan 25 19:45:58 2021 - [info]  Candidate masters from the configuration file:
Mon Jan 25 19:45:58 2021 - [info]   10.0.0.26(10.0.0.26:3306)  Version=5.5.68-MariaDB (oldest major version between slaves) log-bin:enabled
Mon Jan 25 19:45:58 2021 - [info]     Replicating from 10.0.0.21(10.0.0.21:3306)
Mon Jan 25 19:45:58 2021 - [info]     Primary candidate for the new Master (candidate_master is set)
Mon Jan 25 19:45:58 2021 - [info]  Non-candidate masters:
Mon Jan 25 19:45:58 2021 - [info]  Searching from candidate_master slaves which have received the latest relay log events..
Mon Jan 25 19:45:58 2021 - [info] New master is 10.0.0.26(10.0.0.26:3306)
Mon Jan 25 19:45:58 2021 - [info] Starting master failover..
Mon Jan 25 19:45:58 2021 - [info] 
From:
10.0.0.21(10.0.0.21:3306) (current master)
 +--10.0.0.26(10.0.0.26:3306)
 +--10.0.0.27(10.0.0.27:3306)

To:
10.0.0.26(10.0.0.26:3306) (new master)
 +--10.0.0.27(10.0.0.27:3306)
Mon Jan 25 19:45:58 2021 - [info] 
Mon Jan 25 19:45:58 2021 - [info] * Phase 3.3: New Master Diff Log Generation Phase..
Mon Jan 25 19:45:58 2021 - [info] 
Mon Jan 25 19:45:58 2021 - [info]  This server has all relay logs. No need to generate diff files from the latest slave.
Mon Jan 25 19:45:58 2021 - [info] Sending binlog..
Mon Jan 25 19:45:59 2021 - [info] scp from local:/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog to [email protected]:/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog succeeded.
Mon Jan 25 19:45:59 2021 - [info] 
Mon Jan 25 19:45:59 2021 - [info] * Phase 3.4: Master Log Apply Phase..
Mon Jan 25 19:45:59 2021 - [info] 
Mon Jan 25 19:45:59 2021 - [info] *NOTICE: If any error happens from this phase, manual recovery is needed.
Mon Jan 25 19:45:59 2021 - [info] Starting recovery on 10.0.0.26(10.0.0.26:3306)..
Mon Jan 25 19:45:59 2021 - [info]  Generating diffs succeeded.
Mon Jan 25 19:45:59 2021 - [info] Waiting until all relay logs are applied.
Mon Jan 25 19:45:59 2021 - [info]  done.
Mon Jan 25 19:45:59 2021 - [info] Getting slave status..
Mon Jan 25 19:45:59 2021 - [info] This slave(10.0.0.26)'s Exec_Master_Log_Pos equals to Read_Master_Log_Pos(mariadb-bin.000003:245). No need to recover from Exec_Master_Log_Pos.
Mon Jan 25 19:45:59 2021 - [info] Connecting to the target slave host 10.0.0.26, running recover script..
Mon Jan 25 19:45:59 2021 - [info] Executing command: apply_diff_relay_logs --command=apply --slave_user='mhauser' --slave_host=10.0.0.26 --slave_ip=10.0.0.26  --slave_port=3306 --apply_files=/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog --workdir=/data/mastermha/app1/ --target_version=5.5.68-MariaDB --timestamp=20210125194555 --handle_raw_binlog=1 --disable_log_bin=0 --manager_version=0.56 --slave_pass=xxx
Mon Jan 25 19:45:59 2021 - [info] 
Applying differential binary/relay log files /data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog on 10.0.0.26:3306. This may take long time...
Applying log files succeeded.
Mon Jan 25 19:45:59 2021 - [info]  All relay logs were successfully applied.
Mon Jan 25 19:45:59 2021 - [info] Getting new master's binlog name and position..
Mon Jan 25 19:45:59 2021 - [info]  mariadb-bin.000002:245
Mon Jan 25 19:45:59 2021 - [info]  All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='10.0.0.26', MASTER_PORT=3306, MASTER_LOG_FILE='mariadb-bin.000002', MASTER_LOG_POS=245, MASTER_USER='repluser', MASTER_PASSWORD='xxx';
Mon Jan 25 19:45:59 2021 - [warning] master_ip_failover_script is not set. Skipping taking over new master IP address.
Mon Jan 25 19:45:59 2021 - [info] Setting read_only=0 on 10.0.0.26(10.0.0.26:3306)..
Mon Jan 25 19:45:59 2021 - [info]  ok.
Mon Jan 25 19:45:59 2021 - [info] ** Finished master recovery successfully.
Mon Jan 25 19:45:59 2021 - [info] * Phase 3: Master Recovery Phase completed.
Mon Jan 25 19:45:59 2021 - [info] 
Mon Jan 25 19:45:59 2021 - [info] * Phase 4: Slaves Recovery Phase..
Mon Jan 25 19:45:59 2021 - [info] 
Mon Jan 25 19:45:59 2021 - [info] * Phase 4.1: Starting Parallel Slave Diff Log Generation Phase..
Mon Jan 25 19:45:59 2021 - [info] 
Mon Jan 25 19:45:59 2021 - [info] -- Slave diff file generation on host 10.0.0.27(10.0.0.27:3306) started, pid: 3475. Check tmp log /data/mastermha/app1//10.0.0.27_3306_20210125194555.log if it takes time..
Mon Jan 25 19:46:00 2021 - [info] 
Mon Jan 25 19:46:00 2021 - [info] Log messages from 10.0.0.27 ...
Mon Jan 25 19:46:00 2021 - [info] 
Mon Jan 25 19:45:59 2021 - [info]  This server has all relay logs. No need to generate diff files from the latest slave.
Mon Jan 25 19:46:00 2021 - [info] End of log messages from 10.0.0.27.
Mon Jan 25 19:46:00 2021 - [info] -- 10.0.0.27(10.0.0.27:3306) has the latest relay log events.
Mon Jan 25 19:46:00 2021 - [info] Generating relay diff files from the latest slave succeeded.
Mon Jan 25 19:46:00 2021 - [info] 
Mon Jan 25 19:46:00 2021 - [info] * Phase 4.2: Starting Parallel Slave Log Apply Phase..
Mon Jan 25 19:46:00 2021 - [info] 
Mon Jan 25 19:46:00 2021 - [info] -- Slave recovery on host 10.0.0.27(10.0.0.27:3306) started, pid: 3477. Check tmp log /data/mastermha/app1//10.0.0.27_3306_20210125194555.log if it takes time..
Mon Jan 25 19:46:01 2021 - [info] 
Mon Jan 25 19:46:01 2021 - [info] Log messages from 10.0.0.27 ...
Mon Jan 25 19:46:01 2021 - [info] 
Mon Jan 25 19:46:00 2021 - [info] Sending binlog..
Mon Jan 25 19:46:00 2021 - [info] scp from local:/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog to [email protected]:/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog succeeded.
Mon Jan 25 19:46:00 2021 - [info] Starting recovery on 10.0.0.27(10.0.0.27:3306)..
Mon Jan 25 19:46:00 2021 - [info]  Generating diffs succeeded.
Mon Jan 25 19:46:00 2021 - [info] Waiting until all relay logs are applied.
Mon Jan 25 19:46:00 2021 - [info]  done.
Mon Jan 25 19:46:00 2021 - [info] Getting slave status..
Mon Jan 25 19:46:00 2021 - [info] This slave(10.0.0.27)'s Exec_Master_Log_Pos equals to Read_Master_Log_Pos(mariadb-bin.000003:245). No need to recover from Exec_Master_Log_Pos.
Mon Jan 25 19:46:00 2021 - [info] Connecting to the target slave host 10.0.0.27, running recover script..
Mon Jan 25 19:46:00 2021 - [info] Executing command: apply_diff_relay_logs --command=apply --slave_user='mhauser' --slave_host=10.0.0.27 --slave_ip=10.0.0.27  --slave_port=3306 --apply_files=/data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog --workdir=/data/mastermha/app1/ --target_version=5.5.68-MariaDB --timestamp=20210125194555 --handle_raw_binlog=1 --disable_log_bin=0 --manager_version=0.56 --slave_pass=xxx
Mon Jan 25 19:46:00 2021 - [info] 
Applying differential binary/relay log files /data/mastermha/app1//saved_master_binlog_from_10.0.0.21_3306_20210125194555.binlog on 10.0.0.27:3306. This may take long time...
Applying log files succeeded.
Mon Jan 25 19:46:00 2021 - [info]  All relay logs were successfully applied.
Mon Jan 25 19:46:00 2021 - [info]  Resetting slave 10.0.0.27(10.0.0.27:3306) and starting replication from the new master 10.0.0.26(10.0.0.26:3306)..
Mon Jan 25 19:46:00 2021 - [info]  Executed CHANGE MASTER.
Mon Jan 25 19:46:00 2021 - [info]  Slave started.
Mon Jan 25 19:46:01 2021 - [info] End of log messages from 10.0.0.27.
Mon Jan 25 19:46:01 2021 - [info] -- Slave recovery on host 10.0.0.27(10.0.0.27:3306) succeeded.
Mon Jan 25 19:46:01 2021 - [info] All new slave servers recovered successfully.
Mon Jan 25 19:46:01 2021 - [info] 
Mon Jan 25 19:46:01 2021 - [info] * Phase 5: New master cleanup phase..
Mon Jan 25 19:46:01 2021 - [info] 
Mon Jan 25 19:46:01 2021 - [info] Resetting slave info on the new master..
Mon Jan 25 19:46:01 2021 - [info]  10.0.0.26: Resetting slave info succeeded.
Mon Jan 25 19:46:01 2021 - [info] Master failover to 10.0.0.26(10.0.0.26:3306) completed successfully.
Mon Jan 25 19:46:01 2021 - [info] 

----- Failover Report -----

app1: MySQL Master failover 10.0.0.21(10.0.0.21:3306) to 10.0.0.26(10.0.0.26:3306) succeeded

Master 10.0.0.21(10.0.0.21:3306) is down!

Check MHA Manager logs at keepalived1.magedu.net:/data/mastermha/app1/manager.log for details.

Started automated(non-interactive) failover.
The latest slave 10.0.0.26(10.0.0.26:3306) has all relay logs for recovery.
Selected 10.0.0.26(10.0.0.26:3306) as a new master.
10.0.0.26(10.0.0.26:3306): OK: Applying all logs succeeded.
10.0.0.27(10.0.0.27:3306): This host has the latest relay log events.
Generating relay diff files from the latest slave succeeded.
10.0.0.27(10.0.0.27:3306): OK: Applying all logs succeeded. Slave started, replicating from 10.0.0.26(10.0.0.26:3306)
10.0.0.26(10.0.0.26:3306): Resetting slave info succeeded.
Master failover to 10.0.0.26(10.0.0.26:3306) completed successfully.

架构班
1、使用filebeta收集nginx访问日志，并通过geoip展示ip归属地
2、配置nginx虚拟主机，实现域名访问kibana。