Linux HA-OpenStack架构搭建详解

环境准备

• 密码统一六个零

主机名	IP
controller1	10.0.0.10
controller2	10.0.0.11
compute1	10.0.0.12
compute2	10.0.0.13
data1	10.0.0.14
data2	10.0.0.15
haproxy1	10.0.0.16
haproxy2	10.0.0.17

客户端系统	虚拟化工具	操作系统
Windows11	VMware15.5pro	Centos7.9

安装基本工具

• 所有机器

sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config; systemctl disable firewalld

yum install vim iotop bc gcc gcc-c++ glibc glibc-devel pcre pcre-devel openssl  openssl-devel zip unzip zlib-devel  net-tools lrzsz tree ntpdate telnet lsof tcpdump wget libevent libevent-devel bc  systemd-devel bash-completion traceroute bridge-utils  -y

cat >> /etc/hosts << EOF
10.0.0.10 controller1
10.0.0.11 controller2
10.0.0.12 compute1
10.0.0.13 compute2
10.0.0.14 data1
10.0.0.15 data2
10.0.0.16 haproxy1
10.0.0.17 haproxy2

10.0.0.100 openstack.vip.cn
EOF

Haproxy编译部署

• haproxy1与haproxy2机器

• 解决lua环境

  • 官网下载：http://www.lua.org/ftp/lua-5.4.4.tar.gz

# 安装基础命令及编译依赖环境
yum install -y gcc readline-devel

mkdir /apps

tar xvf lua-5.4.4.tar.gz -C /apps/

cd /apps/lua-5.4.4/

make linux test

# 查看编译安装的版本
src/lua -v

• haproxy1与haproxy2机器

  • 官网下载：http://www.haproxy.org/download/2.5/src/haproxy-2.5.7.tar.gz

yum install -y gcc openssl-devel pcre-devel systemd-devel

tar xvf haproxy-2.5.7.tar.gz

cd haproxy-2.5.7/

# 查看安装方法
ll Makefile

cat README

cat INSTALL

# 编译
make -j 4 TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 USE_PCRE=1 USE_SYSTEMD=1 LUA_INC=/apps/lua-5.4.4/src LUA_LIB=/apps/lua-5.4.4/src

# 安装
make install PREFIX=/apps/haproxy

ln -s /apps/haproxy/sbin/haproxy /usr/sbin/

haproxy -v

• haproxy启动文件(haproxy1与haproxy2机器)

vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID

[Install]
WantedBy=multi-user.target

mkdir /var/lib/haproxy/

mkdir /etc/haproxy/

# 配置man日志
vim /etc/man_db.conf
MANDATORY_MANPATH     /apps/haproxy/share/man/

# 更新man数据库
mandb

# 配置文件
vim /etc/haproxy/haproxy.cfg
global
   maxconn 100000
   chroot /apps/haproxy
   stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
   #uid 99
   #gid 99
   user haproxy
   group haproxy
   daemon
   # nbproc 4
   # cpu-map 1 0
   # cpu-map 2 1
   # cpu-map 3 2
   # cpu-map 4 3
   pidfile /var/lib/haproxy/haproxy.pid
   log 127.0.0.1 local2 info

defaults
   option http-keep-alive
   option forwardfor
   maxconn 100000
   mode http
   timeout connect 300000ms
   timeout client 300000ms
   timeout server 300000ms

listen stats
   mode http
   bind 0.0.0.0:9999
   stats enable
   log global
   stats uri    /haproxy-status
   stats auth   admin:123456

useradd -r -s /sbin/nologin -d /var/lib/haproxy haproxy


# haproxy需要vip地址才能启动，这里不检测vip也能启动
vim /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1

sysctl -p


systemctl daemon-reload

systemctl enable --now haproxy

Haproxy+Keepalived

安装服务

• haproxy1+haproxy2机器

yum install -y keepalived

配置haproxy高可用

• haproxy1与haproxy2机器

• 安装检测工具

yum install  -y psmisc

• 使用非抢占式

  • 效果是当主VIP宕机时VIP飘移过后，重启主VIP也不会将VIP夺回来

• haproxy1机器

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
   notification_email {
       [email protected]
       [email protected]
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id haprxy1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_haproxy {
   script "killall -0 haproxy || systemctl restart haproxy"
   interval 2
}

vrrp_instance HA_openstack {
   state BACKUP
   interface eth0
   virtual_router_id 66
   priority 100
   advert_int 2     # 调用脚本两次之间的间隔，默认为1秒
   nopreempt

   track_script {
      check_haproxy
   }

   virtual_ipaddress {
      10.0.0.100/24 dev eth0 label eth0:1       
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}
EOF



vim /etc/keepalived/notify.sh
#!/bin/bash
contact='[email protected]'
notify() {
   
  mailsubject="$(hostname) 切换到 $1, vip 地址发生漂移"    # 发送标题
  mailbody="$(date +'%F %T'): vip发生漂移, $(hostname) 切换到 $1"   # 发送内容
  echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
  notify master
  ;;
backup)
  notify backup
  ;;
fault)
  notify fault
  ;;
*)
  echo "Usage: $(basename $0) {master|backup|fault}"
  exit 1
  ;;
esac

chmod +x /etc/keepalived/notify.sh

yum install -y mailx

vim /etc/mail.rc
''''
set from=360120854@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=360120854@qq.com
set smtp-auth-password=ljroytmuhlkjbgje

systemctl restart keepalived

• haproxy2机器

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
   notification_email {
       [email protected]
       [email protected]
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id haprxy1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_haproxy {
   script "killall -0 haproxy || systemctl restart haproxy"
   interval 2
}

vrrp_instance HA_openstack {
   state BACKUP
   interface eth0
   virtual_router_id 66
   priority 80
   advert_int 2
   nopreempt

   track_script {
      check_haproxy
   }

   virtual_ipaddress {
      10.0.0.100/24 dev eth0 label eth0:1       
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}
EOF



vim /etc/keepalived/notify.sh
#!/bin/bash
contact='[email protected]'
notify() {
   
  mailsubject="$(hostname) 切换到 $1, vip 地址发生漂移"    # 发送标题
  mailbody="$(date +'%F %T'): vip发生漂移, $(hostname) 切换到 $1"   # 发送内容
  echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
  notify master
  ;;
backup)
  notify backup
  ;;
fault)
  notify fault
  ;;
*)
  echo "Usage: $(basename $0) {master|backup|fault}"
  exit 1
  ;;
esac

chmod +x /etc/keepalived/notify.sh

yum install -y mailx


vim /etc/mail.rc
''''
set from=360120854@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=360120854@qq.com
set smtp-auth-password=ljroytmuhlkjbgje

systemctl restart keepalived

OpenStack-data

mysql主主架构

• 采用二进制源码包部署，下载地址：https://downloads.mysql.com/archives/get/p/23/file/mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz
• 上传至data1机器，部署mysql

yum install -y libaio-devel

yum remove -y mariadb*

tar xf mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz 

useradd -s /sbin/nologin mysql

mkdir /application/mysql -pv

mkdir /data/mysql/data -pv

mkdir /data/mysql/binlog -pv

echo  "PATH=/application/mysql/bin:$PATH" > /etc/profile.d/mysql.sh

source  /etc/profile.d/mysql.sh

mv mysql-5.7.26-linux-glibc2.12-x86_64/* /application/mysql/

mysqld --initialize-insecure  --user=mysql --basedir=/application/mysql --datadir=/data/mysql/data 

vim  /etc/my.cnf 
[mysqld]
user=mysql
basedir=/application/mysql
datadir=/data/mysql/data
socket=/tmp/mysql.sock
server_id=6
port=3306
log_bin=/data/mysql/binlog/mysql-bin
character_set_server=utf8
[mysql]
socket=/tmp/mysql.sock

chown -R mysql.mysql /data/

cp /application/mysql/support-files/mysql.server /etc/init.d/mysqld

service mysqld start

mysql

# 为集群架构准备用户
grant replication slave on *.* to repl@'%' identified by '123';

• 上传至data2机器，部署mysql

yum install -y libaio-devel

yum remove -y mariadb*

tar xf mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz 

useradd -s /sbin/nologin mysql

mkdir /application/mysql -pv

mkdir /data/mysql/data -pv

mkdir /data/mysql/binlog -pv

echo  "PATH=/application/mysql/bin:$PATH" > /etc/profile.d/mysql.sh

source  /etc/profile.d/mysql.sh

mv mysql-5.7.26-linux-glibc2.12-x86_64/* /application/mysql/

mysqld --initialize-insecure  --user=mysql --basedir=/application/mysql --datadir=/data/mysql/data 

vim  /etc/my.cnf 
[mysqld]
user=mysql
basedir=/application/mysql
datadir=/data/mysql/data
socket=/tmp/mysql.sock
server_id=12
port=3306
log_bin=/data/mysql/binlog/mysql-bin
character_set_server=utf8
[mysql]
socket=/tmp/mysql.sock

chown -R mysql.mysql /data/

cp /application/mysql/support-files/mysql.server /etc/init.d/mysqld

service mysqld start

mysql

# 为集群架构准备用户
grant replication slave on *.* to repl@'%' identified by '123';

• 主主架构，data2机器执行

mysql

# data1节点查看二进制标识符与位置
show master status;

CHANGE MASTER TO
  MASTER_HOST='data1',
  MASTER_USER='repl',
  MASTER_PASSWORD='123',
  MASTER_PORT=3306,
  MASTER_LOG_FILE='mysql-bin.000001',
  MASTER_LOG_POS=437;
  
# 启动集群
start slave;

# 查看集群状态
show slave status\G

• 主主架构，data1机器执行

mysql

# data2节点查看二进制标识符与位置
show master status;

CHANGE MASTER TO
  MASTER_HOST='data2',
  MASTER_USER='repl',
  MASTER_PASSWORD='123',
  MASTER_PORT=3306,
  MASTER_LOG_FILE='mysql-bin.000001',
  MASTER_LOG_POS=437;
  
# 启动集群
start slave;

# 查看集群状态
show slave status\G




===========================================
如果出现失败或错误，那么执行如下清理

stop slave;

reset slave;

RabbitMQ集群

• data1与data2机器

yum install -y centos-release-openstack-train

yum install -y rabbitmq-server

systemctl enable --now rabbitmq-server

rabbitmqctl add_user openstack 000000

rabbitmqctl set_permissions openstack ".*" ".*" ".*"

rabbitmq-plugins enable rabbitmq_management

• data2机器

scp data1:/var/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/

systemctl restart rabbitmq-server.service

# 1.停止服务
rabbitmqctl stop_app
# 2.重置状态
rabbitmqctl reset
# 3.节点加入
rabbitmqctl join_cluster rabbit@data1
# 4.启动服务
rabbitmqctl start_app

memcached

• data1与data2机器

yum install memcached -y

sed -i "s/127.0.0.1/0.0.0.0/g" /etc/sysconfig/memcached

systemctl enable --now memcached

haproxy配置data机器高可用

• data1机器配置如下
  • data2暂时不配置，先将data1配置完成复制即可
• haproxy代理检测配置详细如下

check #对指定real进行健康状态检查，如果不加此设置，默认不开启检查,check后面没有其它配置也可以启用检查功能
	  #默认对相应的后端服务器IP和端口,利用TCP连接进行周期性健康性检查,注意必须指定端口才能实现健康性检查
 	addr <IP>    #可指定的健康状态监测IP，可以是专门的数据网段，减少业务网络的流量
 	port <num>   #指定的健康状态监测端口
 	inter <num>  #健康状态检查间隔时间，默认2000 ms，单位是毫秒 =2s
 	fall <num>   #后端服务器从线上转为线下的检查的连续失效次数，默认为3
 	rise <num>   #后端服务器从下线恢复上线的检查的连续有效次数，默认为2

• haproxy1机器

vim /etc/haproxy/haproxy.cfg
global
   maxconn 100000
   chroot /apps/haproxy
   stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
   #uid 99
   #gid 99
   user haproxy
   group haproxy
   daemon
   # nbproc 4
   # cpu-map 1 0
   # cpu-map 2 1
   # cpu-map 3 2
   # cpu-map 4 3
   pidfile /var/lib/haproxy/haproxy.pid
   log 127.0.0.1 local2 info

defaults
   option http-keep-alive
   option forwardfor
   maxconn 100000
   mode http
   timeout connect 300000ms
   timeout client 300000ms
   timeout server 300000ms

listen stats
   mode http
   bind 0.0.0.0:9999
   stats enable
   log global
   stats uri    /haproxy-status
   stats auth   admin:123456

listen mysql
   bind 10.0.0.100:3306
   mode tcp
   log global
   balance leastconn
   server data1 10.0.0.14:3306 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:3306 check inter 3000 fall 2 rise 5

listen rabbitmq
   bind 10.0.0.100:5672
   mode tcp
   log global
   balance leastconn
   server data1 10.0.0.14:5672 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:5672 check inter 3000 fall 2 rise 5

listen rabbitmq_web
   bind 10.0.0.100:15672
   mode http
   log global
   balance source
   server data1 10.0.0.14:15672 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:15672 check inter 3000 fall 2 rise 5

listen memcached
   bind 10.0.0.100:11211
   mode tcp
   log global
   balance source
   server data1 10.0.0.14:11211 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:11211 check inter 3000 fall 2 rise 5
   



# 平滑重启haproxy
systemctl reload haproxy.service

chrony部署

• 所有机器

# 安装时间服务
yum install -y chrony

• data1机器

vim /etc/chrony.conf 
server ntp6.aliyun.com iburst
allow all
local stratum 10

systemctl restart chronyd
clock -w

• 除data1机器所有机器

vim /etc/chrony.conf
server data1 iburst

systemctl restart chronyd
clock -w

keystone部署

• data1机器

mysql

CREATE DATABASE keystone;

GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%' IDENTIFIED BY 'keystone123';

• controller1机器

# 安装服务
yum install -y centos-release-openstack-train

yum install -y crudini python-openstackclient openstack-selinux

yum install -y openstack-keystone httpd mod_wsgi python2-PyMySQL python-memcached

# centos8或者其它环境安装区别
python3-PyMySQL
python3-mod_wsgi
#########################


# 备份过滤提前文件
cp /etc/keystone/keystone.conf{
   ,.bak}

grep -Ev "^$|#" /etc/keystone/keystone.conf.bak > /etc/keystone/keystone.conf



# 使用工具配置keystone文件
crudini --set /etc/keystone/keystone.conf database connection mysql+pymysql://keystone:[email protected]/keystone

crudini --set /etc/keystone/keystone.conf token provider fernet



# 同步数据库
su -s /bin/sh -c "keystone-manage db_sync" keystone



# 初始化 Fernet 密钥存储库(提供这些是为了允许在另一个操作系统用户/组下运行 keystone)
keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
keystone-manage credential_setup --keystone-user keystone --keystone-group keystone




# 引导身份
keystone-manage bootstrap --bootstrap-password 000000 \
  --bootstrap-admin-url http://openstack.vip.cn:5000/v3/ \
  --bootstrap-internal-url http://openstack.vip.cn:5000/v3/ \
  --bootstrap-public-url http://openstack.vip.cn:5000/v3/ \
  --bootstrap-region-id RegionOne


# 配置 ServerName选项以引用控制器节点
echo "ServerName controller1" >> /etc/httpd/conf/httpd.conf



# 创建/usr/share/keystone/wsgi-keystone.conf文件的链接
ln -s /usr/share/keystone/wsgi-keystone.conf /etc/httpd/conf.d/


# 开机自启启动服务
systemctl enable --now httpd.service

• haproxy1机器配置keystone代理

vim /etc/haproxy/haproxy.cfg
''''''
listen keystone
   bind 10.0.0.100:5000
   mode tcp
   log global
   balance random
   server controller1 10.0.0.10:5000 check inter 3000 fall 2 rise 5
   # controller2高可用没有做，先不要生效，配置完成再取消注释即可
   #server controller2 10.0.0.11:5000 check inter 3000 fall 2 rise 5
   
systemctl reload haproxy.service 

• controller1机器

cat > /etc/keystone/admin-openrc.sh << EOF
#!/bin/bash
export OS_PROJECT_DOMAIN_NAME=Default
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_NAME=admin
export OS_USERNAME=admin
export OS_PASSWORD=000000
export OS_AUTH_URL=http://openstack.vip.cn:5000/v3
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
EOF



# 创建service 项目
openstack project create --domain default --description "Service Project" service

# 验证
openstack token issue

glance部署

• data1机器

mysql

CREATE DATABASE glance;

GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'%' IDENTIFIED BY 'glance123';

• controller1机器

# 创建glance用户
openstack user create --domain default --password glance glance