部署Centos7下Haproxy实现Exchange反向代理负载并通过Keepalived主备负载

mini介质安装Centos7

1.Centos环境准备

1.1 启用root用户ssh登录

vi /etc/ssh/sshd_config

   PermitRootLogin yes

   

systemctl restart sshd.service

1.2环境准备及安装

yum -y update

yum install wget ftp ntp* mlocate openssl openssl-devel openssl-perl.x86_64 net-tools gcc automake autoconf libtool make -y

关闭SELINUX

vi /etc/selinux/config

SELINUX=enforcing改成SELINUX=disabled

getenforce

创建系统账号

useradd -s /sbin/nologin -M haproxy

id haproxy

配置NTP服务

vi /etc/ntp.config

添加如下内容

fudge 127.127.1.0 stratum 12

server ntp.api.bz iburst minpoll 6 maxpoll 7

server 0.cn.pool.ntp.org iburst minpoll 6 maxpoll 7

server 1.cn.pool.ntp.org iburst minpoll 6 maxpoll 7

server 2.cn.pool.ntp.org iburst minpoll 6 maxpoll 7

# service ntpd start

# systemctl enable ntpd.service

检查服务状态

# netstat -ano |grep :123

# ntpq -p

1.2.1 Cert证书准备

1.2.2 根证书

1.2.2.1检查根证书是否包含在主机内：

curl https://mail.alan.corp/owa

1.2.2.2 第三方根证书导入主机

root.cer（根证书） intermediate.cer 中间证书机构

Der格式证书转Base64格式

openssl x509 -in root.cer -inform der -outform pem -out root.pem

openssl x509 -in intermediate.cer -inform der -outform pem -out intermediate.pem 

将颁发证书机构导入本机证书

c_rehash .

cat 4b37341f.0 >> /etc/pki/tls/certs/ca-bundle.crt

1.2.2.3 将Exchange主机私有证书导入本机

mail.pfx（Exchange主机证书带私有证书，导出保存Base64格式）

openssl pkcs12 -in mail.pfx -nocerts -out exchange_private_key_passwordprotected.pem

输入pfx文件密码，输入Pem文件密码（4位以上）

openssl rsa -in exchange_private_key_passwordprotected.pem -out exchange_private_key_nopassword.pem

输入Pem密码

openssl pkcs12 -in mail.pfx -clcerts -nokeys -out exchange_certificate.pem

输入pfx密码

cat exchange_certificate.pem exchange_private_key_nopassword.pem > exchange_certificate_and_key_nopassword.pem

mv exchange_certificate_and_key_nopassword.pem /etc/ssl/certs/

1.3 安装haproxy

1.3.1软件下载编译及安装

cd /tmp

下载并解压缩

下载方法01：wget http://www.haproxy.org/download/1.9/src/haproxy-1.9.6.tar.gz

            tar -zxvf haproxy-1.9.6.tar.gz

下载方法02：curl --progress http://www.haproxy.org/download/1.9/src/haproxy-1.9.6.tar.gz | tar xz

cd haproxy-1.9.6

#安装haproxy

Hadir=/data/haproxy #安装目录

mkdir -p $Hadir

tar -axf haproxy-* && cd ./haproxy-*

make TARGET=linux310 ARCH=x86_64 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 PREFIX=$Hadir

make install PREFIX=$Hadir

$Hadir/sbin/haproxy -v

$Hadir/sbin/haproxy -vv

#内核优化

#NAT转发

sed -i '[email protected]_forward = [email protected]_forward = 1@g' /etc/sysctl.conf

grep ip_forward /etc/sysctl.conf

echo "net.ipv4.ip_nonlocal_bind = 1" >>/etc/sysctl.conf #允许没监听IP时启动

sysctl -p

1.3.2启动脚本配置

cp ./examples/haproxy.init $Hadir/haproxy

chmod 755 $Hadir/haproxy

sed -i '/^BIN=/cBIN='$Hadir'/sbin/$BASENAME' $Hadir/haproxy

sed -i '/^CFG=/cCFG='$Hadir'/$BASENAME.cfg' $Hadir/haproxy

1.3.3日志配置

sed -i 's/^#$ModLoad imudp/$ModLoad imudp/g' /etc/rsyslog.conf

sed -i 's/^#$UDPServerRun 514/$UDPServerRun 514/g' /etc/rsyslog.conf

echo 'local0.* /var/log/haproxy.log'>>/etc/rsyslog.conf #添加haproxy日志路径

systemctl restart rsyslog

echo "">$Hadir/haproxy.cfg

1.3.4 其他及防火墙配置

mkdir -p /var/lib/haproxy

#防火墙配置

firewall-cmd --permanent --add-port=443/tcp

firewall-cmd --permanent --add-port=80/tcp

firewall-cmd --permanent --add-port=25/tcp

firewall-cmd --permanent --add-port=110/tcp

firewall-cmd --permanent --add-port=143/tcp

firewall-cmd --permanent --add-port=465/tcp

firewall-cmd --permanent --add-port=587/tcp

firewall-cmd --permanent --add-port=993/tcp

firewall-cmd --permanent --add-port=995/tcp

firewall-cmd --permanent --add-port=9000/tcp

systemctl restart firewalld

1.3.5 创建配置文件

echo "

###########全局配置#########

    global

    log 127.0.0.1 local0

    log 127.0.0.1 local1 notice

    daemon

    #nbproc 1     #进程数量 

    maxconn 4096  #最大连接数 

    user haproxy  #运行用户  

    group haproxy #运行组 

    chroot /var/lib/haproxy

    pidfile /var/run/haproxy.pid

########默认配置############

    defaults

    log global

    mode http             #默认模式{ tcp|http|health }

    option httplog       #日志类别,采用httplog

    option dontlognull   #不记录健康检查日志信息  

    retries 2            #2次连接失败不可用

    option forwardfor    #后端服务获得真实ip

    option httpclose     #请求完毕后主动关闭http通道

    option abortonclose  #服务器负载很高，自动结束比较久的链接  

    maxconn 4096         #最大连接数  

    timeout connect 5m   #连接超时  

    timeout client 1m    #客户端超时  

    timeout server 31m   #服务器超时  

    timeout check 10s    #心跳检测超时  

    balance roundrobin   #负载均衡方式，轮询

#状态页面控制

listen stats

    bind *:9000 #伪装的端口号

    mode http #工作模式

balance #负载模式

    stats enable #显示状态页面

    stats hide-version #隐藏haproxy的版本号

    stats realm HAProxy\ Stats #提示信息

    stats auth admin:P@44w0rd #登录状态页面的帐号和密码

#   stats admin if TRUE #状态页面出现管理功能

    stats uri /haproxy?stats #访问入口

#转发配置

# Http 80 负载

frontend ft_exchange_HTTP

bind *:80 name web

maxconn 10000

default_backend bk_exchange_HTTP

backend bk_exchange_HTTP

server Node01 10.101.0.150:80 maxconn 10000 check

server Node02 10.101.0.151:80 maxconn 10000 check backup

# Https 443 负载

frontend ft_exchange_SSL

bind *:443 name ssl

maxconn 10000 #alctl: connection max (depends on capacity)

default_backend bk_exchange_SSL #alctl: default farm to use

backend bk_exchange_SSL

server Node01 10.101.0.150:443 maxconn 10000 check

server Node02 10.101.0.151:443 maxconn 10000 check backup

">$Hadir/haproxy.cfg

------------------------------------------------------------------------

# SMTP 25 负载

frontend ft_exchange_SMTP

bind *:25 name smtp

maxconn 10000

default_backend bk_exchange_SMTP

backend bk_exchange_SMTP

server Node01 10.101.0.150:25 maxconn 10000 check

server Node02 10.101.0.151:25 maxconn 10000 check backup

# SMTPS 465 负载

frontend ft_exchange_SMTP_Secure465

bind *:465 name smtpssl465

maxconn 10000

default_backend bk_exchange_SMTP_Secure465

backend bk_exchange_SMTP_Secure465

server Node01 10.101.0.150:465 maxconn 10000 check

server Node02 10.101.0.151:465 maxconn 10000 check backup

# SMTPS 587 负载

frontend ft_exchange_SMTP_Secure587

bind *:587 name smtpssl587

maxconn 10000

default_backend bk_exchange_SMTP_Secure587

backend bk_exchange_SMTP_Secure587

server Node01 10.101.0.150:587 maxconn 10000 check

server Node02 10.101.0.151:587 maxconn 10000 check backup

# IMTP 143 负载

frontend ft_exchange_IMAP

bind *:143 name imap

maxconn 10000

default_backend bk_exchange_IMAP

backend bk_exchange_IMAP

server Node01 10.101.0.150:143 maxconn 10000 check

server Node02 10.101.0.151:143 maxconn 10000 check backup

# IMTPS 993 负载

frontend ft_exchange_IMAP_Secure

bind *:993 name imapssl

maxconn 10000

default_backend bk_exchange_IMAP_Secure

backend bk_exchange_IMAP_Secure

server Node01 10.101.0.150:993 maxconn 10000 check

server Node02 10.101.0.151:993 maxconn 10000 check backup

# POP3 110 负载

frontend ft_exchange_POP3

bind *:110 name pop3

maxconn 10000

default_backend bk_exchange_POP3

backend bk_exchange_POP3

server Node01 10.101.0.150:110 maxconn 10000 check

server Node02 10.101.0.151:110 maxconn 10000 check backup

# POP3S 995 负载

frontend ft_exchange_POP3_Secure

bind *:995 name pop3ssl

maxconn 10000

default_backend bk_exchange_POP3_Secure

backend bk_exchange_POP3_Secure

server Node01 10.101.0.150:995 maxconn 10000 check

server Node02 10.101.0.151:995 maxconn 10000 check backup

----------------------------------------------------------------------------

1.4 #启动

/data/haproxy/haproxy start

netstat -antp|grep haproxy

ps -ef|grep haproxy

1.5 #添加自启动

ln -sf /data/haproxy/haproxy /etc/init.d/haproxy

chkconfig --add haproxy

chkconfig haproxy on

chkconfig --list haproxy

service haproxy restart

1.6 重启检查服务状态：

systemctl status haproxy

ps -A |grep haproxy

firewall-cmd --query-port 443/tcp

firewall-cmd --list-services            # 查看开放的服务

firewall-cmd --add-port=3306/tcp        # 开放通过tcp访问3306

firewall-cmd --remove-port=80tcp        # 阻止通过tcp访问3306

firewall-cmd --add-port=233/udp         # 开放通过udp访问233

firewall-cmd --list-ports               # 查看开放的端口

1.7 keepalived配置

安装前环境准备

yum -y install psmisc libnfnetlink-devel curl gcc openssl-devel libnl3-devel net-snmp-devel

1.7.1 下载与安装

软件目录规划

软件安装目录：/data/keepalived

日志文件单独存放在/var/log/keepalived/keepalived.log下

#配置主机名

hostnamectl set-hostname corp-haproxy-01

vi /etc/hosts

# 增加主机地址

172.16.0.222    corp-haproxy-01.localdomain

防火墙放行vrrp组播

firewall-cmd --direct --permanent --add-rule ipv4 filter INPUT 0 --in-interface ens160 --destination 224.0.0.18 --protocol vrrp -j ACCEPT

firewall-cmd --reload

1.7.3开始编译

1.7.3.1下载源码包

 下载站点：

 1、http://www.keepalived.org/download.html

 2、http://keepalived.org/software

 cd /tmp

 curl --progress http://keepalived.org/software/keepalived-2.0.13.tar.gz | tar xz

 

 cd /tmp

wget http://www.keepalived.org/software/keepalived-2.0.15.tar.gz

1.7.3.2 编译

kldir=/data/keepalived #安装目录

mkdir -p $kldir

tar -axf keepalived-* && cd ./keepalived-*

./configure  --prefix=$kldir

make && make install

1.7.3.3自启动脚本

检查脚本信息是否正确

# cat /usr/lib/systemd/system/keepalived.service 

[Unit]

Description=LVS and VRRP High Availability Monitor

After= network-online.target syslog.target

Wants=network-online.target

[Service]

Type=forking

PIDFile=/var/run/keepalived.pid

KillMode=process

EnvironmentFile=-/data/keepalived/etc/sysconfig/keepalived

ExecStart=/data/keepalived/sbin/keepalived $KEEPALIVED_OPTIONS

ExecReload=/bin/kill -HUP $MAINPID

[Install]

WantedBy=multi-user.target

!!!!默认的日志存放位置在/var/log/messages中。

echo 'local3.* /var/log/keepalived/keepalived.log' >>/etc/rsyslog.conf                             

然后需要修改keepalived.conf

创建默认启动文件

mkdir -p /etc/keepalived

cp /data/keepalived/etc/keepalived/keepalived.conf  /etc/keepalived/

cp /tmp/keepalived-2.0.15/keepalived/etc/init.d/keepalived  /etc/rc.d/init.d/

cp /data/keepalived/etc/sysconfig/keepalived  /etc/sysconfig/

 

# vi /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {

   notification_email {                    #指定keepalived在发生事情的时候，发送邮件告知，可以有多个地址，每行一个.

     [email protected]

   }

   notification_email_from [email protected]   #指定发件人

   smtp_server 192.168.200.1     #发送email的smtp地址

   smtp_connect_timeout 30       #超时时间

   router_id Haproxy_MASTER      #运行keepalived的机器的一个标识,多个节点标识可以相同，也可以不同

}

vrrp_script check_haproxy {        #killall (安装 yum install psmisc -y)

   script "killall -0 haproxy"

   interval 2

   weighit 2                        #权值脚本成功时（0）等于priority+weghit #否则为priority

   }

vrrp_instance  Haproxy_01 {

    state MASTER                    #指定当前节点为主节点 备用节点上设置为BACKUP即可

    interface ens160                #绑定虚拟IP的网络接口

    mcast_src_ip 172.16.0.222       #本机IP地址 

virtual_router_id 51            #VRRP组名，两个节点的设置必须一样，以指明各个节点属于同一VRRP组

    priority 100                    #主节点的优先级（1-254之间），备用节点必须比主节点优先级低

    advert_int 1                    #设置主备之间的检查时间，单位为s

    authentication {                #设置验证信息，两个节点必须一致

        auth_type PASS

        auth_pass 1111

    }

    virtual_ipaddress {                      #指定虚拟IP, 两个节点设置必须一样

        172.16.0.220/24 brd 172.16.0.255 dev ens160 label ens160:vip

    }

    track_script {

    check_haproxy

    }

    smtp_alert            #状态切换，使用邮件通知

}

 

重启服务即可。

1.7.3.4 设置开机启动

systemctl enable keepalived.service 

第二台主机修改:

1.主机名：

hostnamectl set-hostname SD-haproxy02

vi /etc/hosts

修改为第二台主机地址

10.101.0.154    SD-haproxy02.localdomain

2.修改IP

vi  /etc/sysconfig/network-scripts/ifcfg-ens160

修改为第二台主机地址

IPADDR=10.101.0.154

service network restart

3.修改keepalived配置

vi /etc/keepalived/keepalived.conf

修改如下行

   smtp_server 10.101.0.151 #发送email的smtp地址

   router_id  Haproxy_BACKUP #运行keepalived的机器的一个标识,多个节点标识可以相同，也可以不同

vrrp_instance Haproxy_BACKUP {

    state BACKUP #指定当前节点为主节点 备用节点上设置为BACKUP即可

    priority 99 #主节点的优先级（1-254之间），备用节点必须比主节点优先级低