EMQ集群搭建实现高可用和负载均衡(百万级设备连接)
一．EMQ集群搭建实现高可用和负载均衡
架构服务器规划
二．架构图
三．EMQ集群搭建
192.168.81.13 , 192.168.81.22 , 192.168.81.23 三台服务器作为emq集成服务器，三台都部署emqttd服务
3.1 环境安装
https://www.erlang-solutions.com/resources/download.html;https://github.com/rabbitmq/erlang-rpm/releases （erlang下载）
yum -y remove erlang
rpm -qa | grep erlang | xargs -I {} rpm -e {}
rpm -ivh erlang-21.3.8.1-1.el7.x86_64.rpm
socat安装
yum -y install socat
nginx安装
yum -y install nginx
3.2 SSL证书准备
mkdir –p /etc/nginx/ssl
mkdir –p /etc/nginx/ssl/ca
mkdir –p /etc/nginx/ssl/server
mkdir –p /etc/nginx/ssl/client
mkdir –p /etc/nginx/ssl/certs
mkdir –p /etc/nginx/ssl/crl
touch /etc/nginx/ssl/index.txt
cat
0
EOF
cat
01
EOF
mkdir -p /etc/nginx/ssl/newcerts/server
生成ca证书:
openssl genrsa -out /etc/nginx/ssl/ca/ca.key 1024
openssl req -out /etc/nginx/ssl/ca/ca.req -key /etc/nginx/ssl/ca/ca.key -new -subj “/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Xxxxxx/CN=xxxxxx/[email protected]”
openssl x509 -req -in /etc/nginx/ssl/ca/ca.req -out /etc/nginx/ssl/ca/ca.crt -sha1 -days 5000 -signkey /etc/nginx/ssl/ca/ca.key
rm -f /etc/nginx/ssl/ca/ca.req
生成server服务端证书:
openssl genrsa -out /etc/nginx/ssl/server/dev.xxxxxx.com.key 1024
openssl req -out /etc/nginx/ssl/server/dev.xxxxxx.com.req -key /etc/nginx/ssl/server/dev.xxxxxx.com.key -new -subj “/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/CN=dev.xxxxxx.com/[email protected]”
openssl x509 -req -in /etc/nginx/ssl/server/dev.xxxxxx.com.req -out /etc/nginx/ssl/server/dev.xxxxxx.com.crt -sha1 -CAcreateserial -days 5000 -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key
rm -f /etc/nginx/ssl/server/dev.xxxxxx.com.req
生成client客户端证书（若有先吊销然后再生成 ）:
openssl genrsa -out /etc/nginx/ssl/client/client.key 1024
openssl req -out /etc/nginx/ssl/client/client.req -key /etc/nginx/ssl/client/client.key -new -subj “/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/[email protected]”
openssl x509 -req -in /etc/nginx/ssl/client/client.req -out /etc/nginx/ssl/client/client.crt -sha1 -CAcreateserial -days 5000 -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key
rm -f /etc/nginx/ssl/client/client.req
3.3 修改openssl配置文件(centos7)
sed -i “s//etc/pki/CA//etc/nginx/ssl/g” /etc/pki/tls/openssl.cnf
sed -i “s/cacert.pem/ca/ca.crt/g” /etc/pki/tls/openssl.cnf
sed -i “s/private/cakey.pem/ca/ca.key/g” /etc/pki/tls/openssl.cnf
sed -i “s/private/.rand/ca/.rand/g” /etc/pki/tls/openssl.cnf
3.4 准备nginx的配置文件
配置nginx
cat
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '\$remote_addr - \$remote_user [\$time_local] "\$request" '
'\$status \$body_bytes_sent "\$http_referer" '
'"\$http_user_agent" "\$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
include /etc/nginx/conf.d/http/*.conf;
}
stream {
include /etc/nginx/conf.d/tcp/*.conf;
}
EOF
新建相关文件夹
mkdir -p /etc/nginx/conf.d/{http,tcp}
cat
upstream xxxxxx{
ip_hash;
server ${gateway_ip_address}:8800; #网关微服务地址和端口号
}
server {
error_log /var/log/nginx/80_error.log;
listen 80;
server_name xxxxxx.com;
location / {
index index.html index.htm;
root /usr/share/nginx/html/dist;
try_files \$uri \$uri/ /index.html;
}
location /api/ {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Credentials' 'true';
add_header 'Access-Control-Allow-Methods' '*';
add_header 'Access-Control-Max-Age' '18000L';
add_header 'Access-Control-Allow-Headers' '*';
client_max_body_size 1000m;
proxy_http_version 1.1;
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_pass http://xxxxxx/;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
EOF
cat
server {
error_log /var/log/nginx/1882_error.log;
listen 1882;
proxy_connect_timeout 600s;
proxy_pass xxxxxx;
}
EOF
cat
upstream xxxxxx{
server ${emq_ip_address}:1885; #这个配置无论写哪个tcp的配置文件里都可以，emq的端口必须写无鉴权的
}
server {
error_log /var/log/nginx/1883_error.log;
listen 1883 ssl;
proxy_connect_timeout 600s;
#proxy_timeout 3s;
proxy_pass xxxxxx;
ssl_certificate /etc/nginx/ssl/server/dev.xxxxxx.com.crt;
ssl_certificate_key /etc/nginx/ssl/server/dev.xxxxxx.com.key;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 4h;
ssl_handshake_timeout 30s;
}
EOF
cat
server {
error_log /var/log/nginx/1884_error.log;
listen 1884 ssl;
proxy_connect_timeout 600s;
#proxy_timeout 3s;
proxy_pass xxxxxx;
ssl_verify_client on;
ssl_client_certificate /etc/nginx/ssl/ca/ca.crt;
ssl_certificate /etc/nginx/ssl/server/dev.xxxxxx.com.crt;
ssl_certificate_key /etc/nginx/ssl/server/dev.xxxxxx.com.key;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 4h;
ssl_handshake_timeout 30s;
}
EOF
3.5 修改emq的配置文件
修改插件配置文件，更改域名地址
sed -i “s/test.xxxxxx.com/dev.xxxxxx.com/g” /usr/local/src/emqx-rel/deps/emqx_plugin_template/src/emqx_plugin_template.erl
cd /usr/local/src/emqx-rel/
重新编译，此操作会重新生成配置文件
make -C /usr/local/src/emqx-rel
修改emq配置文件，开启双向认证
sed -i “/^node.name = emqx@/cnode.name = [email protected]” /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf
配置端口号和证书路径
sed -i “/^listener.tcp.external =/clistener.tcp.external = 0.0.0.0:1885” /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf
sed -i “/^{deny, all, subscribe, /c%%{deny, all, subscribe, [”$SYS/#", {eq, “#”}]}." /usr/local/src/emqx-rel/_rel/emqx/etc/acl.conf
sed -i"/^cluster.discovery=/ccluster.discovery=static"/usr/local/src/emqx-rel/emqx/etc/emqx.conf
sed-i"/^##cluster.static.seeds=/[email protected],[email protected],[email protected]" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf
启动emq
/usr/local/src/emqx-rel/_rel/emqx/bin/emqx start
启动nginx
/usr/sbin/nginx -c /etc/nginx/nginx.conf
查看集群状态：
[root@dev-13 bin]# ./emqx_ctl status
Node ‘[email protected]’ is started
emqx a2a8e428 is running
[root@dev-13 bin]# ./emqx_ctl cluster status
Cluster status: [{running_nodes,[‘[email protected]’,‘[email protected]’,
'[email protected]']}]
3.6 Liunx和Erlang虚拟机调优
Linux 系统参数优化
修改系统所有进程可打开的文件数量:
vim /etc/sysctl.conf
fs.file-max = 2097152
fs.nr_open = 2097152
backlog - Socket 监听队列长度:
net.core.somaxconn=32768
net.ipv4.tcp_max_syn_backlog=16384
net.core.netdev_max_backlog=16384
可用知名端口范围:
net.ipv4.ip_local_port_range=‘1000 65535’
TCP Socket 读写 Buffer 设置:
net.core.rmem_default=262144
net.core.wmem_default=262144
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.core.optmem_max=16777216
#sysctl -w net.ipv4.tcp_mem=‘16777216 16777216 16777216’
net.ipv4.tcp_rmem=‘1024 4096 16777216’
net.ipv4.tcp_wmem=‘1024 4096 16777216’
TCP 连接追踪设置:
net.nf_conntrack_max=1000000
net.netfilter.nf_conntrack_max=1000000
net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
FIN-WAIT-2 Socket 超时设置:
net.ipv4.tcp_fin_timeout = 15
TIME-WAIT Socket 最大数量、回收与重用设置:
net.ipv4.tcp_max_tw_buckets=1048576
注意: 不建议开启該设置，NAT模式下可能引起连接RST
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
设置服务最大文件句柄数:
vim /etc/systemd/system.conf
DefaultLimitNOFILE=1048576
持久化设置允许用户/进程打开文件句柄数:
vim /etc/security/limits.conf
-
soft nofile 1048576
-
hard nofile 1048576
Erlang 虚拟机参数:
vim /usr/local/emqttd/etc/emq.conf
Erlang Process Limit
node.process_limit = 2097152
Sets the maximum number of simultaneously existing ports for this system
node.max_ports = 1048576
EMQ 最大允许连接数
Size of acceptor pool
listener.tcp.external.acceptors = 64
Maximum number of concurrent clients(以1G内存比5W进行配置)
listener.tcp.external.max_clients = 1000000
四．haproxy部署
192.168.81.12和192.168.81.21服务器部署和配置一样
4.1 haproxy安装
yum install -y pcre-devel bzip2-devel gcc gcc-c++ make
tar –zxvf haproxy-1.8.26.tar.gz
cd haproxy-1.8.26
make TARGET=linux2628 PREFIX=/usr/local/haproxy
make install PREFIX=/usr/local/haproxy
/usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 1.8.26 2020/08/03
Copyright 2000-2020 Willy Tarreau [email protected]
mkdir /etc/haproxy
groupadd haproxy
useradd -s /sbin/nologin -M -g haproxy haproxy //添加haproxy运行haproxy账号并设置及属主与属组
cp examples/haproxy.init /etc/init.d/haproxy
chmod 755 /etc/init.d/haproxy
chkconfig --add haproxy
cp /usr/local/haproxy/sbin/haproxy /usr/sbin/
4.2 增加配置文件
cat <>/etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
common defaults that all the ‘listen’ and ‘backend’ sections will
use if not designated in their block
#---------------------------------------------------------------------
defaults
log global
option dontlognull
option http-server-close
# option forwardfor
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 60s
timeout client 2m
timeout server 2m
timeout http-keep-alive 10s
timeout check 10s
frontend emqtt-front
bind *:1885
maxconn 1000000
mode tcp
default_backend emqtt-backend
backend emqtt-backend
balance roundrobin
# balance source
server emq1 192.168.81.13:1885 check inter 100000 fall 2 rise 5 weight 1
server emq2 192.168.81.22:1885 check inter 100000 fall 2 rise 5 weight 1
server emq3 192.168.81.23:1885 check inter 100000 fall 2 rise 5 weight 1
# source 0.0.0.0 usesrc clientip
frontend emqtt-admin-front
bind *:18083
mode http
default_backend emqtt-admin-backend
backend emqtt-admin-backend
mode http
balance roundrobin
server emq1 192.168.81.13:18083 check
server emq2 192.168.81.22:18083 check
server emq3 192.168.81.23:18083 check
listen admin_stats
stats enable
bind *:8080
mode http
option httplog
log global
maxconn 10
stats refresh 30s
stats uri /admin
stats realm haproxy
stats auth admin:admin
stats hide-version
stats admin if TRUE
4.3 启动haproxy
systemctl start haproxy
chkconfig haproxy on
五．keepalived部署
192.168.81.12和192.168.81.21服务器部署和配置略有差别
5.1 keepalived安装
yum –y Install keepalived
5.2 增加配置文件
! Configuration File for keepalived
global_defs {
router_id mqtt81
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP
interface ens192
virtual_router_id 81
nopreempt
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
mcast_src_ip 192.168.81.12
virtual_ipaddress {
192.168.81.101/24
}
}
5.3 启动keepalived并设置开机自启动
systemctl start keepalived
systemctl enable keepalived
六．架构测试
6.1 keepalived架构测试
关闭192.168.81.12或者192.168.81.21其中一台，查看服务是否还可以正常访问
6.2 haproxy负载均衡
客户端工具发起pub，看是否按照轮询方式发送到后端3台EMQ服务上面。
七．结果展示
7.1 emq集群展示
7.2 客户端发起6个连接测试
7.3 haproxy统计显示
