生产环境下Tengine keepalived tomcat 安装和配置

一，tengine 安装和配置

1.基础库安装：pcre，openssl，geoip，luajit

#mkdir -p /var/log/nginx
#mkdir -p /var/nginx_temp/{nginx_client,nginx_proxy,nginx_fastcgi,nginx_temp}

#cd /data/soft
#wget 
#tar zxvf pcre-8.36.tar.gz
#cd pcre-8.36
#./configure --prefix=/usr/local
#make && make install


#cd /data/soft
#wget  
#cd openssl-1.0.1m
./config shared --prefix=/usr/local
#make && make install

#cd /data/soft
#wget 

#tar zxvf GeoIP-1.6.5.tar.gz
#cd GeoIP-1.6.5
./configure --prefix=/usr/local
#make && make install

#cd /data/soft
#wget 
#tar zxvf LuaJIT-2.0.4.tar.gz
#cd LuaJIT-2.0.4
#make PREFIX=/usr/local
make install  

#cd /data/soft
#tar xvzf tengine-1.5.2.tar.gz
#cd tengine-1.5.2
#./configure --prefix=/opt/nginx \
--lock-path=/var/lock/nginx.lock \
--pid-path=/var/run/nginx.pid \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--user=nobody \
--group=nobody \
--with-pcre=../pcre-8.36 \
--with-pcre-opt=-fPIC \
--with-openssl=../openssl-1.0.1m \
--with-openssl-opt=-fPIC \
--with-backtrace_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-http_realip_module \
--with-http_concat_module=shared \
--with-http_sysguard_module=shared \
--with-http_limit_conn_module=shared \
--with-http_limit_req_module=shared \
--with-http_split_clients_module=shared \
--with-http_footer_filter_module=shared \
--with-http_geoip_module=shared \
--with-http_sub_module=shared \
--with-http_access_module=shared \
--with-http_upstream_ip_hash_module=shared \
--with-http_upstream_least_conn_module=shared \
--with-http_referer_module=shared \
--with-http_rewrite_module=shared \
--with-http_memcached_module=shared \
--with-http_upstream_session_sticky_module=shared \
--with-http_addition_module=shared \
--with-http_xslt_module=shared \
--with-http_image_filter_module=shared \
--with-http_user_agent_module=shared \
--with-http_empty_gif_module=shared \
--with-http_browser_module=shared \
--with-http_map_module=shared \
--with-http_userid_filter_module=shared \
--with-http_charset_filter_module=shared \
--with-http_trim_filter_module=shared \
--with-http_lua_module=shared \
--without-http_fastcgi_module \
--without-http_uwsgi_module \
--without-http_scgi_module \
--without-select_module \
--without-poll_module \
-with-http_fastcgi_module=shared \
--with-http_uwsgi_module=shared  \
--http-client-body-temp-path=/var/nginx_temp/nginx_client \
--http-proxy-temp-path=/var/nginx_temp/nginx_proxy \
--http-fastcgi-temp-path=/var/nginx_temp/nginx_fastcgi \
 --with-cc-opt='-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro’
 
# make && make install

#mkdir -p /opt/nginx/conf/{upstream,vhost.d}
#cd /opt
#mv nginx tengine-1.5.2 && ln -s tengine-1.5.2 nginx
#rm -f /opt/nginx/conf/*.default





2.tengine 主配置
#vi /opt/nginx/conf/nginx.conf
user nobody nobody;
  
worker_processes auto;
worker_cpu_affinity auto;
  
error_log /var/log/nginx/error.log crit;
pid /var/run/nginx.pid;
  
worker_rlimit_nofile 65535;
  
dso {
    load ngx_http_rewrite_module.so;
    load ngx_http_access_module.so;
    load ngx_http_limit_conn_module.so;
    load ngx_http_limit_req_module.so;
    load ngx_http_sysguard_module.so;
    load ngx_http_lua_module.so;
}
  
events {
    use epoll;
    worker_connections 10240;
}
  
http {
    server_tokens off;
    server_tag off;
    autoindex off;
    access_log off;
    include mime.types;
    default_type application/octet-stream;
  
    server_names_hash_bucket_size 128;
    client_header_buffer_size 32k;
    large_client_header_buffers 4 32k;
    client_max_body_size 30m;
    client_body_buffer_size 256k;
  
    sendfile on;
    tcp_nopush on;
    keepalive_timeout 60;
    tcp_nodelay on;
  
    gzip on;
    gzip_min_length 1k;
    gzip_buffers 4 16k;
    gzip_http_version 1.1;
    gzip_comp_level 6;
    gzip_types text/plain application/x-javascript text/css application/xml application/javascript text/javascript image/jpeg image/gif image/png;
    gzip_vary on;
  
    proxy_connect_timeout 120;
    proxy_read_timeout 120;
    proxy_send_timeout 120;
    proxy_buffer_size 128k;
    proxy_buffers 4 128k;
    proxy_busy_buffers_size 256k;
    proxy_temp_file_write_size 256k;
    proxy_headers_hash_max_size 1024;
    proxy_headers_hash_bucket_size 128;
  
    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  
    proxy_temp_path /var/nginx_temp/nginx_temp;
  
  
  
  
  
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for" '
                    '$request_time';

log_format ha '$remote_addr\t-\t$remote_user\t[$time_local]\t"$request"\t'
                  '$status $body_bytes_sent\t"$http_referer"\t'
                  '"$http_user_agent"\t$request_time\t'
                  '$cookie_user_id__baidu_com\t$cookie_place_id__baidu_com\t'
                  '$cookie_elatitude\t$cookie_elongitude\t'
                  '$cookie_fixed_cid\t$cookie_temp_cid\t$http_x_requested_with\t$cookie_track_id\t$sent_http_content_type\t'
                  '$http_x_baidu_requestid\t$host'; 
  
    
    server {
        sysguard on;
        sysguard_load load=50 action=/loadlimit;
        sysguard_mem swapratio=24% action=/swaplimit;
        sysguard_mem free=100M action=/freelimit;
  
        location /loadlimit {
            return 503;
        }
  
        location /swaplimit {
            return 503;
        }
  
        location /freelimit {
            return 503;
        }
    }
  
    include upstream/*.conf;
    include vhost.d/*.conf;
}



3. 负载均衡池配置
#vim /opt/nginx/conf/upstream/baidi.com
upstream b_baidi_SERVER {
    consistent_hash $request_uri;
    server 100.10.111.210:8080 id=101 weight=8;
    server 100.10.111.180:8080 id=102 weight=10;
    server 100.10.210.139:8080 id=103 weight=10;
    server 100.10.112.163:8080 id=104 weight=5;
     
    check interval=3000 rise=2 fall=3 timeout=1000 type=http port=8080;
    check_http_send "OPTIONS http://baidi.com/api/check;
    check_http_expect_alive http_2xx http_3xx;
}

4vhost 设置
#vi /opt/nginx/conf/vhost.d/baidi.com.conf

server {
	listen 80;
	listen 443 ssl;
	server_name baidi.com;

	ssl_certificate /opt/nginx/conf/ssl/baidi.crt;
	ssl_certificate_key /opt/nginx/conf/ssl/baidi.key;
	ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

	set $rewrite_status 0;

	if ($https_status = off) {
		set $rewrite_status "${rewrite_status}1";
	}

	if ($scheme = http) {
		set $rewrite_status "${rewrite_status}2";
	}

	if ($https = on) {
		set $https_status $https;
	}

	if ($rewrite_status = 012) {
		rewrite / https://$http_host$request_uri permanent;
		break;
	}

	access_log /var/log/nginx/baidi.com.access.log ha;
	error_log  /var/log/nginx/baidi.com.error.log;

	add_header Access-Control-Allow-Origin *;
	add_header Access-Control-Allow-Credentials true;
	add_header Access-Control-Allow-Headers Content-Type;
	add_header Access-Control-Allow-Methods POST;

	location / {
	        proxy_next_upstream http_502 http_504 error timeout invalid_header;
	        proxy_redirect off;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header REMOTE-HOST $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-For $remote_addr;
		proxy_pass http://b_baidi_SERVER;
		proxy_set_header Host $host;
	}
	
	location ~ health_status {
        check_status;
        allow 127.0.0.1;
        allow 10.0.0.0/8;
        allow 172.0.0.0/8;
        allow 192.168.0.0/16;
        deny all;
    }
}




4 启动脚本
#vim /etc/init.d/nginx

#!/bin/sh
#
# nginx - this script start and stop the nginx daemon
#
# chkconfig: 2345
# description: Startup script for nginx
# processname: nginx
# config: /opt/nginx/conf/nginx.conf
# pidfile: /var/run/nginx.pid
#
# code by rocketzhang
#
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
  
DAEMON="/opt/nginx/sbin/nginx"
CNFFILE="/opt/nginx/conf/nginx.conf"
PIDFILE="/var/run/nginx.pid"
LOCKFILE="/var/lock/nginx.lock"
  
RETVAL=0
  
set -e
[[ -x "$DAEMON" ]] || exit 0
  
start() {
    [[ -x $DAEMON ]] || exit 5
    [[ -f $CNFFILE ]] || exit 6
  
    echo -n "Startting Nginx......"
    $DAEMON -c $CNFFILE || echo -n "Nginx already running!"
    RETVAL=$?
    echo
    [[ $RETVAL -eq 0 ]] && touch $LOCKFILE
}
  
stop() {
    echo -n "Stopping Nginx......"
  
    if [[ -n `ps aux | grep nginx | awk '/master/{print $2}'` ]]; then
        kill -QUIT `ps aux | grep nginx | awk '/master/{print $2}'`
        RETVAL=$?
        echo
        [[ $RETVAL -eq 0 ]] && rm -f $LOCKFILE $PIDFILE
    fi
}
  
reload() {
    echo -n "Reloading Nginx......"
  
    if [[ -n `ps aux | grep nginx | awk '/master/{print $2}'` ]]; then
        kill -HUP `ps aux | grep nginx | awk '/master/{print $2}'`
        RETVAL=$?
        echo
    fi
}
  
case "$1" in
start)
    start
    ;;
  
stop)
    stop
    ;;
  
reload)
    reload
    ;;
  
restart)
    stop
    sleep 1
    start
    ;;
  
*)
    echo "Usage: service nginx {start|stop|reload|restart}"
    RETVAL=1
    ;;
esac
  
exit $RETVAL


5.# 服务启动 
#chmod +x /etc/init.d/nginx
#chkconfig nginx on
#service nginx start





主节点配置
二，keepaived 安装和配置
1.基础安装
#yum -y install keepalived


#vim /etc/keepalived/keepalived.conf

global_defs {
    notification_email {
        [email protected]
    }
  
    notification_email_from [email protected]
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id NGX_DEVEL1
}
  
vrrp_script check_nginx_health {
    script "/opt/scripts/check_nginx_for_keepalived.sh"
    interval 1
    weight -2
}
  
vrrp_instance VI_1 {
    state MASTER
    interface bond0
    virtual_router_id 51
    priority 100
    advert_int 1
    smtp_alert
  
    authentication {
        auth_type PASS
        auth_pass 123456
    }
  
    virtual_ipaddress {
        10.0.20.200
    }
  
    track_script {
        check_nginx_health
    }
  
    notify_master "/opt/scripts/notify_for_keepalived.sh master"
    notify_bakcup "/opt/scripts/notify_for_keepalived.sh backup"
    notify_fault "/opt/scripts/notify_for_keepalived.sh fault"
}


备节点配置
#vim /etc/keepalived/keepalived.conf
global_defs {
    notification_email {
        [email protected]
    }
  
    notification_email_from [email protected]
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id NGX_DEVEL2
}
  
vrrp_script check_nginx_health {
    script "/opt/scripts/check_nginx_for_keepalived.sh"
    interval 1
    weight -2
}
  
vrrp_instance VI_1 {
    state BACKUP
    interface bond0
    virtual_router_id 51
    priority 90
    advert_int 1
    smtp_alert
  
    authentication {
        auth_type PASS
        auth_pass 123456
    }
  
    virtual_ipaddress {
        10.0.20.200
    }
  
    track_script {
        check_nginx_health
    }
  
    notify_master "/opt/scripts/notify_for_keepalived.sh master"
    notify_bakcup "/opt/scripts/notify_for_keepalived.sh backup"
    notify_fault "/opt/scripts/notify_for_keepalived.sh fault"
}


#脚本 
#vim /opt/scripts/check_nginx_for_keepalived.sh

#!/bin/sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
  
if [[ -z `ps aux | grep 'nginx:' | grep -v grep` ]]; then
    /etc/init.d/nginx start && sleep 5
  
    if [[ -z `ps aux | grep 'nginx:' | grep -v grep` ]]; then
        /sbin/service keepalived stop
    fi
fi

VIP 漂移短信通知脚本
# vim /opt/scripts/notify_for_keepalived.sh
#!/bin/sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
  
KEEPALIVE_CONF="/etc/keepalived/keepalived.conf"
VIP=`grep -A 1 virtual_ipaddress ${KEEPALIVE_CONF} | tail -1 | sed 's/\t//g; s/ //g'`
  
## 短信发送
API_KEY="xxxxxxx"
URL="<a href="http://baidi.com/api/send/" "="" style="color: rgb(59, 115, 175); text-decoration: none; border-radius: 0px !important; border: 0px !important; bottom: auto !important; float: none !important; height: auto !important; left: auto !important; margin: 0px !important; outline: 0px !important; overflow: visible !important; padding: 0px !important; position: static !important; right: auto !important; top: auto !important; vertical-align: baseline !important; width: auto !important; box-sizing: content-box !important; min-height: auto !important; background: none !important;">http://baidi.com/api/send/"
PHONE_NUM_FILE="/opt/scripts/phone_num_list.txt"
  
function SEND_SMS_INFO() {
    MESSAGE="$1"
  
    [[ ! -e $PHONE_NUM_FILE || -z `sed '/^$/d' $PHONE_NUM_FILE` ]] && exit 1
    [[ -n "$MESSAGE" ]] || exit 1
  
    while read PHONE_INFO
    do
        PHONE_NUM=`echo $PHONE_INFO | awk -F '|' '{print $1}'`
        curl -H "X-Authentication-Token:${TOKEN}" -d "sender_key=${API_KEY}&receiver=${PHONE_NUM}&message=${MESSAGE}" "${URL}"
    done < $PHONE_NUM_FILE
}
  
case "$1" in
master|backup|fault)
    SEND_SMS_INFO "(`hostname`) The search proxy server's vip has changed to master $1"
    exit 0
    ;;
  
*)
    echo "Usage: `basename $0` {master|backup|fault}"
    exit 1
esac

# service keepalived start

注意：
最终的实现要求是，VIP地址必须随着nginx服务的正常启动而漂移，否则就没有意义。
比如：主节点的nginx服务挂了，但没有被正常拉起来，而keepalived服务是正常的，
此时，由于主节点keepalived的优先级高，VIP地址就会漂移到主节点，就会导致无法正常
提供服务，所以必须考虑这种特殊情况。