openresty利用haproxy2.0实现负载均衡

安装openresty

编译安装

yum install pcre-devel openssl-devel gcc curl wget 
wget https://openresty.org/download/openresty-1.15.8.3.tar.gz
tar -xzvf openresty-1.15.8.3.tar.gz
cd openresty-1.15.8.3
./configure
gmake
gmake install

yum 安装

yum install -y wget
wget -O /etc/yum.repos.d/openresty.repo https://openresty.org/package/centos/openresty.repo
yum install -y openresty openresty-resty

#默认安装编译参数检查
[root@localhost openresty-1.15.8.3]# openresty -V
nginx version: openresty/1.15.8.3
built by gcc 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC) 
built with OpenSSL 1.1.0l  10 Sep 2019
TLS SNI support enabled
configure arguments: --prefix=/usr/local/openresty/nginx --with-cc-opt='-O2 -DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/zlib/include -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl/include' --add-module=../ngx_devel_kit-0.3.1rc1 --add-module=../echo-nginx-module-0.61 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.31 --add-module=../ngx_lua-0.10.15 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.7 --add-module=../ngx_stream_lua-0.0.7 --with-ld-opt='-Wl,-rpath,/usr/local/openresty/luajit/lib -L/usr/local/openresty/zlib/lib -L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl/lib -Wl,-rpath,/usr/local/openresty/zlib/lib:/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl/lib' --with-cc='ccache gcc -fdiagnostics-color=always' --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-compat --with-stream --with-stream_ssl_preread_module --with-http_ssl_module


#安装php
yum install php-fpm php-mysql php-xml php -y
systemctl enable php-fpm.service --now

#虚拟站点配置文件
[root@localhost conf]# cat /usr/local/openresty/nginx/test/index.php 
echo hellowld
yayayayayale

[root@localhost conf]# cat /usr/local/openresty/nginx/conf.d/php.conf 
server {    
    
  #监听端口    
  listen 801;    
    
  #网站根目录    
  root /usr/local/openresty/nginx/test;    
    
  #虚拟主机名称    
  server_name 192.168.168.21;    
    
  #网站主页排序    
  index index.php index.html index.htm default.php default.htm default.html;    
    
  #网站访问、错误日志    
  access_log /usr/local/openresty/nginx/test/test.access.log;    
  error_log /usr/local/openresty/nginx/logs/test/test.error.log;    
    
  #流量限制（网站最大并发数500|单IP访问最大并发数50|每个请求流量上限1024KB）    
  #limit_conn perserver 500;    
  #limit_conn perip 50;    
  #limit_rate 1024k;    
      
  #配置错误页面    
  #error_page 404 /404.html;    
  #error_page 500 502 503 504 /50x.html;    
    
  #禁止访问文件和目录    
  location ~ ^/(\.user.ini|\.htaccess|\.git|\.svn|\.project|LICENSE|README.md) {    
    return 404;    
  }    
    
  #配置资源防盗链    
  location ~ .*\.(jpg|jpeg|gif|png|js|css)$ {    
    expires 30d;    
    access_log /dev/null;    
    valid_referers none blocked 192.168.168.21;    
    if ($invalid_referer) {    
      return 404;    
    }    
  }    
    
  #配置图片资源缓存时间    
  location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$ {    
    expires 30d;    
    error_log off;    
    access_log /dev/null;    
  }    
      
  #设置样式资源缓存时间    
  location ~ .*\.(js|css)?$ {    
    expires 12h;    
    error_log off;    
    access_log /dev/null;    
  }    
    
  #解析PHP    
  location ~* \.php$ {    
    fastcgi_index index.php;    
    fastcgi_pass 127.0.0.1:9000;    
    include fastcgi_params;    
    fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;    
    fastcgi_param SCRIPT_NAME $fastcgi_script_name;    
  }
}
#主配置文件
[root@localhost conf]# cat /usr/local/openresty/nginx/conf/nginx.conf

#user  nobody;
worker_processes  1;

error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    #加载虚拟主机配置文件    
    include /usr/local/openresty/nginx/conf.d/*.conf; 
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                 '$status $body_bytes_sent "$http_referer" '
    #                '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;
    log_format  json  '{"@timestamp":"$time_iso8601",'
                      '"@version":"1",'
                      '"client":"$remote_addr",'
                      '"url":"$uri",'
                      '"status":"$status",'
                      '"domain":"$host",'
                      '"host":"$server_addr",'
                      '"size":"$body_bytes_sent",'
                      '"responsentime":"$request_time",'
                      '"referer":"$http_referer",'
                      '"useragent":"$http_user_agent",'
                      '"upstreampstatus":"$upstream_status",'
                      '"upstreamaddr":"$upstream_addr",'
                      '"upstreamresponsetime":"$upstream_response_time"'
                       '}';

    access_log  logs/access_json.log  json; 
    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index1.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

安装haproxy

1.1需要先安装lua5.3版本，否则后面后报错

curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz 
yum install readline-devel gcc -y
tar zxvf lua-5.3.5.tar.gz
cd lua-5.3.5
make linux
make INSTALL_TOP=/usr/local/lua install
yum install systemd-devel wget openssl openssl-devel -y

1.2部署haproxy。

cdwget https://www.haproxy.org/download/2.0/src/haproxy-2.0.1.tar.gztar zxvf haproxy-2.0.1.tar.gz 
cd haproxy-2.0.1
make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 LUA_LIB=/usr/local/lua/lib/ LUA_INC=/usr/local/lua/include/ USE_PCRE=1 USE_SYSTEMD=1 
make install PREFIX=/usr/local/haproxycd

1.3创建运行用户（应该也可忽略，只是使用该用户运行，加强安全而已。我一般是直接使用root用户运行）

useradd haproxy -s /sbin/nologin

1.4创建配置文件haproxy.cfg。在安装目录/usr/local/hadproxy下新建haproxy.cfg

global　　#全局设置
　 daemon　　#以后台进程运行
　 maxconn 256　　#每个进程的最大连接数
　 nbproc 1　　#进程数，该值可以设置小于或等于cpu核心数
　 balance roundrobin #默认的负载均衡的方式,轮询方式 
  #balance source #默认的负载均衡的方式,类似nginx的ip_hash 
  #balance leastconn #默认的负载均衡的方式,最小连接 
defaults　　#默认设置
　　mode http　　#设置http(七层模式)，也可设置为tcp(四层模式)，另外还有一个Health健康监测模式。对mysql进行负载均衡的话，这里记得修改为tcp
　　timeout connect 5000ms
　　timeout client 50000ms
　　timeout server 50000ms
　　

listen admin_stats #配置haproxy管理页面
　　bind *:9999　　#访问端口为9999
　　mode http
　　option httplog
　　stats refresh 30s　　#自动刷新时间
　　stats uri /stats　　#项目名为status，ip+端口+项目名即可访问
　　stats auth admin:admin　　#配置管理用户账号密码
　　stats admin if TRUE
　　stats hide-version

frontend http-in　　#配置前端访问端口
　　bind *:1080　　#通过该端口进行负载均衡
　　default_backend servers　　#指定后端服务器

backend servers
　　server server1 172.16.1.230:80 check inter 2000 rise 3 fall 3 weight 1 maxconn 32　　

　　#建议加上check，否则后台服务器A宕机了，负载均衡还会把请求发送到该宕机服务器上，inter 2000指check检测时间为2000毫秒，rise 3检测3次均正常则表示后天服务器正常，fall 3检测3次失败，则会把该后天服务器标志宕机，不再玩该后台服务器发送请求，weight 1指权重，取消weight改为backup，则是所有后台服务器宕机后才会启用该backup后台服务器
　　server server2 172.16.1.227:80 check inter 2000 rise 3 fall 3 weight 1 maxconn 32



#参考yum 安装的配置文件
[root@node2 haproxy]# cat /etc/haproxy/haproxy.cfg |grep -v "^#"

global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend  main *:5000
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             app

backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

backend app
    balance     roundrobin
    server  app1 127.0.0.1:5001 check
    server  app2 127.0.0.1:5002 check
    server  app3 127.0.0.1:5003 check
    server  app4 127.0.0.1:5004 check
    
    

1.5使用指定配置文件运行haproxy。建议都haproxy和hadproxy.cfg都使用绝对路径

#/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg

1.6验证安装的haproxy版本

[root@localhost ~]# /usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 2.0.1 2019/06/26 - https://haproxy.org/

1.7配置开机自启

#修改haproxy cfg
[root@localhost haproxy]# cat /usr/local/haproxy/haproxy.cfg 
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /usr/local/haproxy
    pidfile     /usr/local/haproxy/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /usr/local/haproxy/socket

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend  main 
    bind 0.0.0.0:8000
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             app

backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

backend app
    balance     roundrobin
    server  app1 127.0.0.1:80 check
    server  app2 127.0.0.1:801 check
    #server  app3 127.0.0.1:5003 check
    #server  app4 127.0.0.1:5004 check
[root@localhost haproxy]# 

[root@localhost haproxy]# cat /usr/lib/systemd/system/haproxy.service 
[Unit]
Description=haporxy load bulancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg -c -q
ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /usr/local/haproxy/haproxy.cfg -p /usr/local/haproxy/haproxy.pid
ExecReload=/bin/kill -USER2 $MAINPID

[Install]
WantedBy=multi-user.target

systemctl enable haporxy --now

1.8测试效果

其中228的80端口上我启用了httpd，页面内容为this is 228.另外一台230上的80端口也是httpd，页面为默认内容。使用谷歌浏览器访问172.16.1.227，每次刷新的内容都和上一次不一样

curl 127.0.0.1:8000

[root@localhost haproxy]# curl 192.168.1.119:8000







Welcome to OpenResty!
If you see this page, the OpenResty web platform is successfully installed and
working. Further configuration is required.

For online documentation and support please refer to
openresty.org.

Commercial support is available at
openresty.com.

Thank you for flying OpenResty.



[root@localhost haproxy]# curl 192.168.1.119:8000
echo hellowld
yayayayayale