ab 灰度上线方案

后台服务AB部署

标签（空格分隔）： 运维

---

清单

├── apache-maven-3.5.2-bin.tar.gz
├── cronolog-1.6.2.tar.gz
├── gb-tomcat-cluster
│   ├── bin
│   ├── conf
│   ├── mode.last
│   ├── tomcat-customer-a
│   ├── tomcat-customer-b
│   ├── tomcat-seller-a
│   ├── tomcat-seller-b
│   ├── tomcat-statics-a
│   ├── tomcat-statics-b
│   ├── tomcat-wechat-a
│   └── tomcat-wechat-b
├── jdk-8u144-linux-x64.tar.gz
├── nginx-conf
│   ├── gb-conf
│   └── ngxFlowControl
└── nginx-install.tar.gz

---

服务器配置

• 统一JDK版本
• 统一maven版本

 # 安装JDK
    tar -zxvf jdk-8u144-linux-x64.tar.gz
    mv jdk1.8.0_144 /usr/local/jdk1.8.0_144
 # 安装maven
    tar -zxvf apache-maven-3.5.2-bin.tar.gz
    mv apache-maven-3.5.2 /usr/local/apache-maven-3.5.2
 # 安装cronolog
    tar -zxvf cronolog-1.6.2.tar.gz
    ./configure
    make && make install
 # 配置PATH
    vim /etc/profile
    PATH=$PATH:/usr/loacl/jad:/usr/loacl/jdk1.8.0_144/bin:/usr/local/apache-maven-3.5.2/bin
 # 复制maven本地仓库（加快构建速度，第一次从中央仓库、私服抓取比较慢）
 scp -r -P39876 root@172.16.100.63:/root/.m2/repository ./

Tomcat 相关

清单

├── bin
│   ├── gb-build
│   └── gb-server
├── code
│   └── greatbear-framework
├── conf
│   ├── customer
│   ├── seller
│   ├── statics
│   └── wechat
├── mode.last
├── tomcat-customer-a
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-customer-b
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-seller-a
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-seller-b
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-statics-a
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-statics-b
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-wechat-a
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── tomcat-wechat-b
│   ├── bin
│   ├── conf
│   ├── lib
│   ├── LICENSE
│   ├── logs
│   ├── NOTICE
│   ├── RELEASE-NOTES
│   ├── RUNNING.txt
│   ├── temp
│   ├── webapps
│   └── work
├── work-a
└── work-b

server.xml 修改

<Server port="-1" shutdown="SHUTDOWN">
  <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
  <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
  <Listener className="org.apache.catalina.core.JasperListener" />
  <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
  <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />

  <GlobalNamingResources>
    <Resource name="UserDatabase" auth="Container"
              type="org.apache.catalina.UserDatabase"
              description="User database that can be updated and saved"
              factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
              pathname="conf/tomcat-users.xml" />
  GlobalNamingResources>

  <Service name="Catalina">

<Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
            maxThreads="1500" minSpareThreads="30" maxIdleTime="60000"/>
    <Connector  executor="tomcatThreadPool"
                port="8070" protocol="org.apache.coyote.http11.Http11NioProtocol"
                maxHttpHeaderSize="8192"
                acceptCount="800"
                compression="off"
                compressionMinSize="2048"
                enableLookups="false"
                disableUploadTimeout="false"
                connectionTimeout="20000"
                redirectPort="8443"
                URIEncoding="UTF-8"/>

    <Engine name="Catalina" defaultHost="localhost">

      <Realm className="org.apache.catalina.realm.LockOutRealm">
        <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
               resourceName="UserDatabase"/>
      Realm>

      <Host name="localhost"  appBase="webapps"
            unpackWARs="true" autoDeploy="true">

        <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="localhost_access_log." suffix=".txt"
               pattern="%h %l %u %t "%r" %s %b" />
        <Valve className="org.apache.catalina.valves.RemoteIpValve"  
               remoteIpHeader="X-Forwarded-For"  
               protocolHeader="X-Forwarded-Proto"  
               protocolHeaderHttpsValue="https"/>
        <Context path="/greatbear-customer" reloadable="true" docBase="/data/customer/work/greatbear-customer">Context>
      Host>
    Engine>
  Service>
Server>

• 关闭 SHUTDOWN 端口（8005）（避免端口冲突）
• 关闭AJP端口（apache 使用的协议，我们用不到）（避免端口冲突）
• 统一Tomcat线程数，队列长度，请求长度限制
• 可灵活配置的JVM 内存配置
• 灵活配置 tomcat 数量
• 统一多tomcat日志到同一个文件，方便查找，方便管理。
• 共享一个catlina.sh，方便配置修改：比如日志切割等

脚本情况

• 构建脚本
  
  • 可选择tag（默认最新的5个tag）
  • 可选择构建环境（dev|test|prod）
  • 可选择部署方案（a|b）

#! /bin/sh
# chkconfig: 2345 96 14
GB_CLUSTER_PATH=/data/gb-tomcat-cluster
GIT_CODE_PATH=$GB_CLUSTER_PATH/code
PROJECT_NAME=greatbear-framework
GIT_REPOSITORY=ssh://git@172.16.117.224:39876/GreatBear/greatbear-framework.git

if [ ! -e "$GIT_CODE_PATH/$PROJECT_NAME/.git" ]
then
    cd $GIT_CODE_PATH
    git clone $GIT_REPOSITORY
fi

cd $GIT_CODE_PATH/$PROJECT_NAME

GIT_TAG_LIST=$(git describe --tags `git rev-list --tags --max-count=5`)
echo "" 
echo "请选择要发布的Tag,输入数字" 
select tag in ${GIT_TAG_LIST[@]}
do
    git fetch --all
    git checkout $tag
    git clean -df
    break
done

ENV_LIST=(
    "prod"
    "test"
    "dev"
)

echo "请选择构建环境"

select env in ${ENV_LIST[@]}
do
    if [ -n "$env" ]; then
            echo 'building $env ...'
        mvn clean package -Penv-$env -Dmaven.test.skip=true
        break
    fi
    echo "请选择正确的环境"
done


echo "构建成功"
echo "请选择要发布的模式, 上一次发布的模式为 : " `cat $GB_CLUSTER_PATH/mode.last`

MODE_LIST=(
    "a"
    "b"
)
select mode in ${MODE_LIST[@]}
do
    if [ -n "$mode" ]; then 
        read -r -p "请确定发布的模式: $mode? [y/n] " input
        if [ $input = "y" ]; then
            rm -fr $GB_CLUSTER_PATH/work-$mode/*
            cp -fr $GIT_CODE_PATH/$PROJECT_NAME/*/target/greatbear*  $GB_CLUSTER_PATH/work-$mode/
            echo "$mode" > $GB_CLUSTER_PATH/mode.last
        fi
            echo '发布成功，请重启服务 参见 service gb-server [start|stop|restart] '
            exit 1
        break
    fi
    echo "请输入数字"
done

• 服务管理脚本
  
  • 统一管理所有服务
  • 可选择运行的服务(seller|customer|statics|wechat)
  • 可选择（start|stop|restart）
  • 可选择运行模式（a|b）

#! /bin/sh
#Tomcat init script for linux
#
# chkconfig: 2345 96 14
# description: tomcat server init script
# Source Function Library

CRONO_LOG_PATH=`which cronolog`

if [ !"$CRONO_LOG_PATH" = "/usr/local/sbin/cronolog" ]; then
    echo "请安装日志分割工具 cronolog"
fi

MODE_LIST=(
        "a" 
        "b" 
)

GB_SERVICE=(
    "seller"
    "customer"
    "statics"
    "wechat"
)

JAVA_OPTS='-server -Xms2G -Xmx2G -Xmn512m -Xss256k'
DEPLOY_SERVICE=''
DEPLOY_MODE=''

echo '请选择要发布的项目'
select service in ${GB_SERVICE[@]}
do
    case $service in 
        seller | customer )
            JAVA_OPTS='-server -Xms12G -Xmx12G -Xmn3G -Xss256k'
            DEPLOY_SERVICE=$service
        ;;
        *)
            JAVA_OPTS='-server -Xms2G -Xmx2G -Xmn512m -Xss256k'
            DEPLOY_SERVICE=$service

    esac
    break
done

if [ -z "$DEPLOY_SERVICE" ];then
echo '无效的项目，退出'
exit 1
fi

echo "use JVM config : $JAVA_OPTS"
echo "请选择要使用的模式"
select mode in ${MODE_LIST[@]}
do
    if [ -n "$mode" ];then
        DEPLOY_MODE=$mode
        break
    fi  
    echo '请选择 a | b'
done

JAVA_HOME=/usr/bin/java
CATALINA_HOME=/data/gb-tomcat-cluster/tomcat-$DEPLOY_SERVICE-$DEPLOY_MODE
GB_HOME=/data/gb-tomcat-cluster
export JAVA_HOME CATALINA_HOME JAVA_OPTS

echo "use JVM config : $JAVA_OPTS"
read -r -p "请确认项目 $DEPLOY_SERVICE : 模式 : $DEPLOY_MODE ? [y/n] " input
if [ $input = "y" ]; then
    case $1 in
        start)
                exec find $GB_HOME/conf/$DEPLOY_SERVICE/$mode*.xml -exec $CATALINA_HOME/bin/catalina.sh start -config {} \;
        ;;
        stop)
                exec ps -ef |grep "$GB_HOME/conf/$DEPLOY_SERVICE/$mode.*xml start" |grep -v grep | grep -v python | awk '{print $2}' |xargs kill -9
        ;;
        restart)
                ps -ef |grep "$GB_HOME/conf/$DEPLOY_SERVICE/$mode.*xml start" |grep -v grep | grep -v python | awk '{print $2}' |xargs kill -9
                exec find $GB_HOME/conf/$DEPLOY_SERVICE/$mode*.xml -exec $CATALINA_HOME/bin/catalina.sh start -config {} \;
        ;;
        *)
                echo "USAGE:`basename $0` {start|stop|restart}"
                    exit 1
            ;;
    esac
fi  

echo 'exit'
exit 1

端口分布情况

├── customer
│   ├── a-8070.xml
│   ├── a-8071.xml
│   ├── a-8072.xml
│   ├── a-8073.xml
│   ├── a-8074.xml
│   ├── a-8075.xml
│   ├── a-8076.xml
│   ├── a-8077.xml
│   ├── a-8078.xml
│   ├── a-8079.xml
│   ├── b-8170.xml
│   ├── b-8171.xml
│   ├── b-8172.xml
│   ├── b-8173.xml
│   ├── b-8174.xml
│   ├── b-8175.xml
│   ├── b-8176.xml
│   ├── b-8177.xml
│   ├── b-8178.xml
│   └── b-8179.xml
├── seller
│   ├── a-8080.xml
│   ├── a-8081.xml
│   ├── a-8082.xml
│   ├── a-8083.xml
│   ├── a-8084.xml
│   ├── b-8180.xml
│   ├── b-8181.xml
│   ├── b-8182.xml
│   ├── b-8183.xml
│   └── b-8184.xml
├── statics
│   ├── a-8050.xml
│   └── b-8150.xml
└── wechat
    ├── a-8030.xml
    ├── a-8031.xml
    ├── b-8130.xml
    └── b-8131.xml

---

nginx 相关

安装步骤

#复制安装包到/usr/local/src/
    cp nginx-install.tar.gz /usr/local/src/
#解压
    tar -zxvf nginx-install.tar.gz
#执行安装
    sh nginx_https_lua_install.sh
#复制配置文件、lua脚本

配置文件及lua脚本清单

├── gb-conf
│   ├── greatbear-customer-a.conf
│   ├── greatbear-customer-b.conf
│   ├── greatbear-seller-a.conf
│   ├── greatbear-seller-b.conf
│   ├── nginx-customer.conf
│   └── nginx-seller.conf
└── ngxFlowControl
    └── ngxlua

ab 配置

• nginx-customer.conf

user  root;
worker_processes  32;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    use epoll;
    worker_connections  10240;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"'
                      'upstream_response_time $upstream_response_time request_time $request_time ';

    lua_package_path '/usr/local/nginx-customer-ab/ngxFlowControl/?.lua;;';
    lua_package_cpath '/usr/local/nginx-customer-ab/ngxFlowControl/ngxlua/lib/?.so;;';

    # access_log  logs/access.log  main;
    # error_log  logs/error.log;

    sendfile        on;
    #tcp_nopush     on;
    #keepalive_timeout  0;

    proxy_ignore_client_abort on;

    #gzip  on;
    #压缩配置#
    gzip  on;           #打开gzip压缩功能
    gzip_min_length 1k; #压缩阈值
    gzip_buffers 4 16k; #buffer 不用修改
    gzip_comp_level 8;  #压缩级别:1-10，数字越大压缩的越好，时间也越长
    gzip_types text/plain application/javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;  #        压缩文件类型
    gzip_vary off;      #跟Squid等缓存服务有关，on的话会在Header里增加 "Vary: Accept-Encoding"
    gzip_disable "MSIE [1-6]\.";  #IE1-6版本不支持gzip压缩



    include /usr/local/nginx-customer-ab/conf/vhost/greatbear-seller-a.conf;
}

• nginx-seller.conf

user  root;
worker_processes  32;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    use epoll;
    worker_connections  10240;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"'
                      'upstream_addr $upstream_addr '
                      'upstream_response_time $upstream_response_time request_time $request_time ';

    lua_package_path '/usr/local/nginx-seller-ab/ngxFlowControl-ab/?.lua;;';
    lua_package_cpath '/usr/local/nginx-https-ab/ngxFlowControl-ab/ngxlua/lib/?.so;;';


    #access_log  logs/access.log  main;
    #error_log  logs/error.log;

    sendfile        on;
    #tcp_nopush     on;
    #keepalive_timeout  0;
    keepalive_timeout  65;

    proxy_ignore_client_abort on;    

    #gzip  on;
    #压缩配置#
    gzip  on;           #打开gzip压缩功能
    gzip_min_length 1k; #压缩阈值
    gzip_buffers 4 16k; #buffer 不用修改
    gzip_comp_level 8;  #压缩级别:1-10，数字越大压缩的越好，时间也越长
    gzip_types text/plain application/javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;  #        压缩文件类型
    gzip_vary off;      #跟Squid等缓存服务有关，on的话会在Header里增加 "Vary: Accept-Encoding"
    gzip_disable "MSIE [1-6]\.";  #IE1-6版本不支持gzip压缩



    include /usr/local/nginx-seller-ab/conf/vhost/greatbear-seller-a.conf;
}

• greatbear-customer-a.conf

    upstream greatbear-server-http{
        server 172.16.100.51:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.52:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.62:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.21:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8010 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8020 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8030 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8040 max_fails=5 fail_timeout=10s;
    }

    upstream greatbear-customer-http{
        server 172.16.100.63:8070 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8071 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8072 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8073 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8074 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8075 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8076 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8077 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8078 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8079 max_fails=5 fail_timeout=10s;
    }

    upstream greatbear-customer-1 {
        server 127.0.0.1:8070 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8071 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-2 {
        server 127.0.0.1:8071 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8072 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-3 {
        server 127.0.0.1:8072 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8073 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-4 {
        server 127.0.0.1:8073 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8074 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-5 {
        server 127.0.0.1:8074 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8075 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-6 {
        server 127.0.0.1:8075 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8076 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-7 {
        server 127.0.0.1:8076 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8077 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-8 {
        server 127.0.0.1:8077 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8078 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-9 {
        server 127.0.0.1:8078 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8079 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-10 {
        server 127.0.0.1:8079 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8070 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }


    upstream greatbear-wx-kefu-1 {
        server 172.16.100.31:8030 max_fails=10 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-wx-kefu-2 {
        server 172.16.100.31:8031 max_fails=10 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-wx-kefu-default {
        server 172.16.100.31:8030 max_fails=10 fail_timeout=10s;
        keepalive 1000;
    }

    server {
        listen       80;
        listen       443 ssl;
        server_name opp.skynet.shangdejigou.cn greatbearopp.sunlands.com;
        include /usr/local/nginx-customer-ab/conf/vhost/deny.ip;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme; # ssl
        #charset koi8-r;
        large_client_header_buffers 4 16k;
        client_max_body_size 300m;
        client_body_buffer_size 128k;
        proxy_buffer_size 64k;
        proxy_buffers   4 32k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 64k;

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        location  / {
             proxy_pass   http://greatbear-server-http/;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             #proxy_request_buffering off;
             #proxy_buffering off;

        }


        location  /greatbear-customer {
             proxy_pass   http://greatbear-customer-http;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             #proxy_request_buffering off;
             #proxy_buffering off;

        }


        location ~ /greatbear-customer/customer/pu[ls][lh] {
                set $customer_back_num '10';
                set $cb_pname 'uid';
                set $cb_strategy 'balance_hash';
                set_by_lua_file $back_idx /usr/local/nginx-customer-ab/ngxFlowControl/ngxlua/server/customer_balance.lua;
                proxy_pass http://greatbear-customer-$back_idx;
                proxy_http_version 1.1;
                proxy_connect_timeout 600s;
                proxy_read_timeout 86400s;
                proxy_send_timeout 600s;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                keepalive_timeout  300s 300s;
        }
        location  /greatbear-wechat/customer/message {
                set $wx_back_num '2';
                set $back_idx '';
                rewrite_by_lua_file /usr/local/nginx-customer-ab/ngxFlowControl/ngxlua/server/wx_balance.lua;
                proxy_pass http://greatbear-wx-kefu-$back_idx;
                proxy_http_version 1.1;
                proxy_connect_timeout 600s;
                proxy_read_timeout 3600s;
                proxy_send_timeout 600s;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                keepalive_timeout  300s 300s;
                keepalive_requests 10000;

        }

        location /visit {
            root /data/greatbear-static;
            index index-*.html;
        }


        location ~* \.(mp3)$ {
            root /data/work/greatbear-web/assets;
            rewrite /greatbear-war/assets/(.*\.mp3) /$1;
        }

        location /nginx-status
        {
                stub_status on;
                access_log off;
                allow 172.16.102.131;
                allow 172.16.102.230;
                allow 127.0.0.1;
                deny all;
        }

        location /robots.txt {
            root /data;
        }
        location /report.html {
            root /usr/share/nginx/html/site;
            access_log off;
            allow 172.16.102.131;
            allow 172.16.202.10;
            deny all;
        }

        # ssl
        # ssl on;
        ssl_certificate star_sunlands_com.crt;
        ssl_certificate_key star_sunlands_com.key;
        ssl_session_timeout  5m;
        ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
        ssl_ciphers AES128-SHA:AES256-SHA:RC4-SHA:DES-CBC3-SHA:RC4-MD5;

        access_log /usr/local/nginx-customer/logs/greatbear.customer.access.log;
        error_log  /usr/local/nginx-customer/logs/greatbear.customer.error.log;
     }

• greatbear-customer-b.conf

    upstream greatbear-server-http{
        server 172.16.100.51:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.52:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.62:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.21:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8110 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8120 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8130 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8140 max_fails=5 fail_timeout=10s;
    }

    upstream greatbear-customer-http{
        server 172.16.100.63:8170 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8171 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8172 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8173 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8174 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8175 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8176 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8177 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8178 max_fails=5 fail_timeout=10s;
        server 172.16.100.63:8179 max_fails=5 fail_timeout=10s;
    }

    upstream greatbear-customer-1 {
        server 127.0.0.1:8170 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8171 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-2 {
        server 127.0.0.1:8171 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8172 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-3 {
        server 127.0.0.1:8172 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8173 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-4 {
        server 127.0.0.1:8173 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8174 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-5 {
        server 127.0.0.1:8174 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8175 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-6 {
        server 127.0.0.1:8175 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8176 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-7 {
        server 127.0.0.1:8176 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8177 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-8 {
        server 127.0.0.1:8177 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8178 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-9 {
        server 127.0.0.1:8178 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8179 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-customer-10 {
        server 127.0.0.1:8179 max_fails=10 fail_timeout=10s;
        server 127.0.0.1:8170 backup max_fails=5 fail_timeout=10s;
        keepalive 1000;
    }


    upstream greatbear-wx-kefu-1 {
        server 172.16.100.31:8130 max_fails=10 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-wx-kefu-2 {
        server 172.16.100.31:8131 max_fails=10 fail_timeout=10s;
        keepalive 1000;
    }

    upstream greatbear-wx-kefu-default {
        server 172.16.100.31:8130 max_fails=10 fail_timeout=10s;
        keepalive 1000;
    }

    server {
        listen       80;
        listen       443 ssl;
        server_name opp.skynet.shangdejigou.cn greatbearopp.sunlands.com;
        include /usr/local/nginx-customer-ab/conf/vhost/deny.ip;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme; # ssl
        #charset koi8-r;
        large_client_header_buffers 4 16k;
        client_max_body_size 300m;
        client_body_buffer_size 128k;
        proxy_buffer_size 64k;
        proxy_buffers   4 32k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 64k;

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        location  / {
             proxy_pass   http://greatbear-server-http/;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             #proxy_request_buffering off;
             #proxy_buffering off;

        }


        location  /greatbear-customer {
             proxy_pass   http://greatbear-customer-http;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             #proxy_request_buffering off;
             #proxy_buffering off;

        }


        location ~ /greatbear-customer/customer/pu[ls][lh] {
                set $customer_back_num '10';
                set $cb_pname 'uid';
                set $cb_strategy 'balance_hash';
                set_by_lua_file $back_idx /usr/local/nginx-customer-ab/ngxFlowControl/ngxlua/server/customer_balance.lua;
                proxy_pass http://greatbear-customer-$back_idx;
                proxy_http_version 1.1;
                proxy_connect_timeout 600s;
                proxy_read_timeout 86400s;
                proxy_send_timeout 600s;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                keepalive_timeout  300s 300s;
        }
        location  /greatbear-wechat/customer/message {
                set $wx_back_num '2';
                set $back_idx '';
                rewrite_by_lua_file /usr/local/nginx-customer-ab/ngxFlowControl/ngxlua/server/wx_balance.lua;
                proxy_pass http://greatbear-wx-kefu-$back_idx;
                proxy_http_version 1.1;
                proxy_connect_timeout 600s;
                proxy_read_timeout 3600s;
                proxy_send_timeout 600s;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                keepalive_timeout  300s 300s;
                keepalive_requests 10000;

        }

        location /visit {
            root /data/greatbear-static;
            index index-*.html;
        }


        location ~* \.(mp3)$ {
            root /data/work/greatbear-web/assets;
            rewrite /greatbear-war/assets/(.*\.mp3) /$1;
        }

        location /nginx-status
        {
                stub_status on;
                access_log off;
                allow 172.16.102.131;
                allow 172.16.102.230;
                allow 127.0.0.1;
                deny all;
        }

        location /robots.txt {
            root /data;
        }
        location /report.html {
            root /usr/share/nginx/html/site;
            access_log off;
            allow 172.16.102.131;
            allow 172.16.202.10;
            deny all;
        }

        # ssl
        # ssl on;
        ssl_certificate star_sunlands_com.crt;
        ssl_certificate_key star_sunlands_com.key;
        ssl_session_timeout  5m;
        ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
        ssl_ciphers AES128-SHA:AES256-SHA:RC4-SHA:DES-CBC3-SHA:RC4-MD5;

        access_log /usr/local/nginx-customer/logs/greatbear.customer.access.log;
        error_log  /usr/local/nginx-customer/logs/greatbear.customer.error.log;
     }

• greatbear-seller-a.conf

    upstream greatbear-server-iphash{

        server 172.16.100.62:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.21:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.51:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.52:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8084 max_fails=5 fail_timeout=10s;

    }

     upstream greatbear-server-{

        server 172.16.100.62:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.21:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.51:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8084 max_fails=5 fail_timeout=10s;

        server 172.16.100.52:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8084 max_fails=5 fail_timeout=10s;

    }


    upstream greatbear-server-1{
        server 172.16.100.62:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8081 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-2{
        server 172.16.100.62:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8082 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-3{
        server 172.16.100.62:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8083 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-4{
        server 172.16.100.62:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8084 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-5{
        server 172.16.100.62:8084 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8080 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-6{
        server 172.16.100.21:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8081 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-7{
        server 172.16.100.21:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8082 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-8{
        server 172.16.100.21:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8083 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-9{
        server 172.16.100.21:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8084 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-10{
        server 172.16.100.21:8084 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8080 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-11{
        server 172.16.100.51:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8081 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-12{
        server 172.16.100.51:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8082 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-13{
        server 172.16.100.51:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8083 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-14{
        server 172.16.100.51:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8084 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-15{
        server 172.16.100.51:8084 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8080 max_fails=5 fail_timeout=10s backup;
    }
    upstream greatbear-server-16{
        server 172.16.100.52:8080 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8081 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-17{
        server 172.16.100.52:8081 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8082 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-18{
        server 172.16.100.52:8082 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8083 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-19{
        server 172.16.100.52:8083 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8084 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-20{
        server 172.16.100.52:8084 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8080 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-blacklist {
        server 172.16.100.31:18050 max_fails=5 fail_timeout=10s;
    }

    upstream greatbear-statistics {
        server 172.16.100.31:8050 max_fails=5 fail_timeout=10s;
    }

    server {
        listen       80;
        listen       443 ssl;
        server_name  customer.skynet.shangdejigou.cn greatbearcst.sunlands.com stres.skynet.shangdejigou.cn;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        #charset koi8-r;
        large_client_header_buffers 4 16k;
        client_max_body_size 600m;
        client_body_buffer_size 128k;
        proxy_buffer_size 64k;
        proxy_buffers   4 32k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 64k;

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        location  / {
             proxy_pass   http://greatbear-server-iphash/;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             #proxy_request_buffering off;
             #proxy_buffering off;

        }

        location = /greatbear-war/ws/im/seller {
             set $back_num '20';
             set $uid '';
             set $param_name 'seller_id';
             set_by_lua_file $uid /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/counselor_balance.lua;
             proxy_pass http://greatbear-server-$uid;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location = /greatbear-war/erpCard/addERPCard {
             set $back_num '20';
             set $uid '';
             set $param_name 'seller_id';
             set_by_lua_file $uid /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/counselor_balance.lua;
             proxy_pass http://greatbear-server-$uid;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location = /greatbear-war/dataMonitor/findTodayData {
             set $back_num '20';
             set $uid '';
             set $param_name 'seller_id';
             set_by_lua_file $uid /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/counselor_balance.lua;
             proxy_pass http://greatbear-server-$uid;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location /greatbear-blacklist {

             proxy_pass   http://greatbear-blacklist;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location /greatbear-statistics {

             proxy_pass   http://greatbear-statistics;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }


        location /visit {
            proxy_http_version 1.1;
            proxy_set_header Host $proxy_host;
            proxy_read_timeout 3600s;
            proxy_send_timeout 600s;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

            set $ab_pname 'siteId';
            set_by_lua_file $ab_tag /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/ab_static.lua;

            #if ( $ab_tag = 'req_cdn' ){
            #     proxy_pass http://stres.skynet.shangdejigou.cn;
            #}

            if ( $ab_tag = 'req_b' ){
                rewrite ^/visit/(.*)$ /visitb/$1 last;
            }

            root /data/greatbear-static;
            index index-*.html;
        }

        location /visitb {
            root /data/greatbear-static-b;
            index index-*.html;
        }

        location ~* \.(mp3)$ {
            root /data/work/greatbear-web/assets;
            rewrite /greatbear-war/assets/(.*\.mp3) /$1;
        }

        location /nginx-status
        {
                stub_status on;
                access_log off;
                allow 172.16.102.131;
                allow 172.16.102.230;
                allow 127.0.0.1;
                deny all;
        }

        location /static {
            root /data/greatbear-static;
            index greatbear.html;
        }

        # 小程序验证
        location /8MvKoFJito.txt {
            root /data/greatbear-static/checkfile/;
        }

        # ssl
        # ssl on;
        ssl_certificate star_sunlands_com.crt;
        ssl_certificate_key star_sunlands_com.key;
        ssl_session_timeout  5m;
        ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
        ssl_ciphers AES128-SHA:AES256-SHA:RC4-SHA:DES-CBC3-SHA:RC4-MD5;

        access_log /usr/local/nginx-seller-ab/logs/greatbear.ssl.access.log main;
        error_log  /usr/local/nginx-seller-ab/logs/greatbear.ssl.error.log;
     }

• greatbear-seller-b.conf

    upstream greatbear-server-iphash{

        server 172.16.100.62:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.21:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.51:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.52:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8184 max_fails=5 fail_timeout=10s;

    }

     upstream greatbear-server-{

        server 172.16.100.62:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.21:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.51:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8184 max_fails=5 fail_timeout=10s;

        server 172.16.100.52:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8184 max_fails=5 fail_timeout=10s;

    }


    upstream greatbear-server-1{
        server 172.16.100.62:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8181 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-2{
        server 172.16.100.62:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8182 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-3{
        server 172.16.100.62:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8183 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-4{
        server 172.16.100.62:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8184 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-5{
        server 172.16.100.62:8184 max_fails=5 fail_timeout=10s;
        server 172.16.100.62:8180 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-6{
        server 172.16.100.21:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8181 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-7{
        server 172.16.100.21:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8182 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-8{
        server 172.16.100.21:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8183 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-9{
        server 172.16.100.21:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8184 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-10{
        server 172.16.100.21:8184 max_fails=5 fail_timeout=10s;
        server 172.16.100.21:8180 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-11{
        server 172.16.100.51:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8181 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-12{
        server 172.16.100.51:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8182 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-13{
        server 172.16.100.51:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8183 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-14{
        server 172.16.100.51:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8184 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-15{
        server 172.16.100.51:8184 max_fails=5 fail_timeout=10s;
        server 172.16.100.51:8180 max_fails=5 fail_timeout=10s backup;
    }
    upstream greatbear-server-16{
        server 172.16.100.52:8180 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8181 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-17{
        server 172.16.100.52:8181 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8182 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-18{
        server 172.16.100.52:8182 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8183 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-19{
        server 172.16.100.52:8183 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8184 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-server-20{
        server 172.16.100.52:8184 max_fails=5 fail_timeout=10s;
        server 172.16.100.52:8180 max_fails=5 fail_timeout=10s backup;
    }

    upstream greatbear-blacklist {
        server 172.16.100.31:18050 max_fails=5 fail_timeout=10s;
    }

    upstream greatbear-statistics {
        server 172.16.100.31:8150 max_fails=5 fail_timeout=10s;
    }

    server {
        listen       80;
        listen       443 ssl;
        server_name  customer.skynet.shangdejigou.cn greatbearcst.sunlands.com stres.skynet.shangdejigou.cn;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        #charset koi8-r;
        large_client_header_buffers 4 16k;
        client_max_body_size 600m;
        client_body_buffer_size 128k;
        proxy_buffer_size 64k;
        proxy_buffers   4 32k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 64k;

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        location  / {
             proxy_pass   http://greatbear-server-iphash/;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             #proxy_request_buffering off;
             #proxy_buffering off;

        }

        location = /greatbear-war/ws/im/seller {
             set $back_num '20';
             set $uid '';
             set $param_name 'seller_id';
             set_by_lua_file $uid /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/counselor_balance.lua;
             proxy_pass http://greatbear-server-$uid;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location = /greatbear-war/erpCard/addERPCard {
             set $back_num '20';
             set $uid '';
             set $param_name 'seller_id';
             set_by_lua_file $uid /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/counselor_balance.lua;
             proxy_pass http://greatbear-server-$uid;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location = /greatbear-war/dataMonitor/findTodayData {
             set $back_num '20';
             set $uid '';
             set $param_name 'seller_id';
             set_by_lua_file $uid /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/counselor_balance.lua;
             proxy_pass http://greatbear-server-$uid;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location /greatbear-blacklist {

             proxy_pass   http://greatbear-blacklist;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        location /greatbear-statistics {

             proxy_pass   http://greatbear-statistics;
             proxy_http_version 1.1;
             proxy_connect_timeout 600s;
             proxy_read_timeout 3600s;
             proxy_send_timeout 600s;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }


        location /visit {
            proxy_http_version 1.1;
            proxy_set_header Host $proxy_host;
            proxy_read_timeout 3600s;
            proxy_send_timeout 600s;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

            set $ab_pname 'siteId';
            set_by_lua_file $ab_tag /usr/local/nginx-seller-ab/ngxFlowControl/ngxlua/server/ab_static.lua;

            #if ( $ab_tag = 'req_cdn' ){
            #     proxy_pass http://stres.skynet.shangdejigou.cn;
            #}

            if ( $ab_tag = 'req_b' ){
                rewrite ^/visit/(.*)$ /visitb/$1 last;
            }

            root /data/greatbear-static;
            index index-*.html;
        }

        location /visitb {
            root /data/greatbear-static-b;
            index index-*.html;
        }

        location ~* \.(mp3)$ {
            root /data/work/greatbear-web/assets;
            rewrite /greatbear-war/assets/(.*\.mp3) /$1;
        }

        location /nginx-status
        {
                stub_status on;
                access_log off;
                allow 172.16.102.131;
                allow 172.16.102.230;
                allow 127.0.0.1;
                deny all;
        }

        location /static {
            root /data/greatbear-static;
            index greatbear.html;
        }

        # 小程序验证
        location /8MvKoFJito.txt {
            root /data/greatbear-static/checkfile/;
        }

        # ssl
        # ssl on;
        ssl_certificate star_sunlands_com.crt;
        ssl_certificate_key star_sunlands_com.key;
        ssl_session_timeout  5m;
        ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
        ssl_ciphers AES128-SHA:AES256-SHA:RC4-SHA:DES-CBC3-SHA:RC4-MD5;

        access_log /usr/local/nginx-seller-ab/logs/greatbear.ssl.access.log main;
        error_log  /usr/local/nginx-seller-ab/logs/greatbear.ssl.error.log;
     }

---

扩展

keepalived 高可用方案

https://blog.csdn.net/tianjun2012/article/details/53437792