LVS+keeplived+nginx+tomcat高可用、高性能jsp集群

#!/bin/bash
# BY kerryhu
# MAIL:[email protected]
# BLOG:http://kerry.blog.51cto.com
# Please manual operation yum of before Operation.....

#yum -y install gcc gcc-c++ autoconf libjpeg libjpeg-devel libpng libpng-devel \
#freetype freetype-devel libxml2 libxml2-devel zlib zlib-devel glibc glibc-devel \
#glib2 glib2-devel bzip2 bzip2-devel ncurses ncurses-devel curl curl-devel e2fsprogs \
#e2fsprogs-devel krb5 krb5-devel libidn libidn-devel openssl openssl-devel openldap \
#openldap-devel nss_ldap openldap-clients openldap-servers

一、网络拓扑:

lvs-master: 192.168.9.201
LVS-backup: 192.168.9.202
LVS-VIP: 192.168.9.200

web_1: 192.168.9.203
web_2: 192.168.9.204

目的：搭建高可用、高性能的jsp集群

二、初始化系统
#init system
系统环境: CentOS 5.5（定制安装）
组件：
Base
Development Libraries
Development Tools
Editors
Text-based Internet

./init_system.sh #此脚本参见http://kerry.blog.51cto.com/172631/555535

三、LVS+keeplived

#关于LVS+keeplived的配置请参考我的另一篇博文《CentOS5.5环境下布署LVS+keepalived 》http://kerry.blog.51cto.com/172631/401253

四、nginx、tomcat整合
#config web_1、web_2
#download software
cd /opt
wget http://labs.renren.com/apache-mirror/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz
wget http://download.oracle.com/otn-pub/java/jdk/6u25-b06/jdk-6u25-linux-i586.bin
wget http://nginx.org/download/nginx-1.0.0.tar.gz
wget http://ftp.exim.llorien.org/pcre/pcre-8.12.tar.gz
wget http://apache.etoak.com//tomcat/tomcat-connectors/native/1.1.20/source/tomcat-native-1.1.20-src.tar.gz
wget http://archive.apache.org/dist/apr/apr-1.4.2.tar.gz
wget http://archive.apache.org/dist/apr/apr-util-1.3.9.tar.gz

#================================ 1、安装tomcat、JDK =================================
mv apache-tomcat-6.0.32.tar.gz jdk-6u25-linux-i586.bin /usr/local/
tar -zxvf apache-tomcat-6.0.32.tar.gz
mv apache-tomcat-6.0.32 tomcat
chmod +x jdk-6u25-linux-i586.bin
./jdk-6u25-linux-i586.bin

#apr 与 tomcat-native 提供更好的伸缩性、性能和集成到本地服务器技术,如果没有apr技术，启动tomcat 时出现如下提示
#INFO: The APR based Apache Tomcat Native library which allows optimal performance in production environments was not found on the java.library.path: /usr/local/jdk1.6.0_25/jre/lib/i386/client:/usr/local/jdk1.6.0_25/jre/lib/i386:/usr/local/jdk1.6.0_25/jre/../lib/i386:/usr/java/packages/lib/i386:/lib:/usr/lib

rpm -e --nodeps apr-1.2.7-11.el5_3.1
rpm -e --nodeps apr-1.2.7-11.el5_3.1
tar -zxvf apr-1.4.2.tar.gz
cd apr-1.4.2
./configure
make;make install
cd ../
tar -zxvf apr-util-1.3.9.tar.gz
cd apr-util-1.3.9
./configure --with-apr=/usr/local/apr
make;make install
cd ../
cp tomcat-native-1.1.20-src.tar.gz /usr/local/tomcat/bin/
cd /usr/local/tomcat/bin
tar -zxvf tomcat-native.tar.gz
cd ./tomcat-native-1.1.12-src/jni/native/
./configure --with-apr=/usr/local/apr --with-java-home=/usr/local/jdk1.6.0_25
make;make install

#设置环境变量
cat >>/etc/profile<<EOF
export JAVA_HOME=/usr/local/jdk1.6.0_25
export CLASS_PATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export PATH=.:$PATH:$JAVA_HOME/bin
export CATALINA_HOME=/usr/local/tomcat
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/apr/lib
EOF

source /etc/profile

#查看java版本
java -version

#启动tomcat
/usr/local/tomcat/bin/startup.sh
echo "/usr/local/tomcat/bin/startup.sh" >> /etc/rc.local

#将tomcat添加到系统服务
cd /opt
wget http://forja.cenatic.es/plugins/scmsvn/viewcvs.php/*checkout*/servers/tomcat/bin/jsvc.tar.gz?root=norma&rev=82&pathrev=82
mv jsvc.tar.gz /usr/local/tomcat/bin/
tar -zxvf jsvc.tar.gz
cd jsvc-src
chmod +x configure
./configure --with-java=/usr/local/jdk1.6.0_25
make
cp /usr/local/tomcat/bin/jsvc-src/native/Tomcat5.sh /etc/rc.d/init.d/tomcat
chmod +x /etc/rc.d/init.d/tomcat

vi /etc/rc.d/init.d/tomcat
#chkconfig: 2345 85 15
#description: apache-tomcat.6 #这两行在#!/bin/sh下面添加
JAVA_HOME=/usr/local/jdk1.6.0_25
CATALINA_HOME=/usr/local/tomcat
DAEMON_HOME=/usr/local/tomcat/bin
TOMCAT_USER=tomcat
CATALINA_BASE=/usr/local/tomcat
CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib"
$DAEMON_HOME/jsvc-src/jsvc #有两处需要修改

#添加tomcat用户
groupadd tomcat
useradd -g tomcat -s /sbin/nologin -M tomcat

chown -R tomcat.tomcat /usr/local/tomcat
chkconfig --add tomcat
chkconfig --level 2345 tomcat on

#http://localhost:8080，如果看到猫的页面即tomcat和jdk安装成功
#新建文件目录/data/www为网站存放目录，设置server.xml文件，在Host name="localhost"处将appBase=的指向路径改为/data/www/test,创建index.jsp至/data/www/test/ROOT，内容为：“My web!”
#修改tomcat网站目录
#sed -i 's#appBase="webapps"#appBase="/data/www/test"#' /usr/local/tomcat/conf/server.xml

#报错信息：java.lang.IllegalArgumentException: Document base /data/www/truboshop/host-manager does not exist or is not a readable directory
#解决方法：删除conf\Catalina\localhost目录下面的host-manager.xml、manager.xml两个文件

#============================= 2、安装Nginx =============================
/usr/sbin/groupadd www
/usr/sbin/useradd -g www www -s /sbin/nologin
mkdir -p /data/www
chmod +w /data/www
chown -R www:www /data/www

tar zxvf pcre-8.12.tar.gz
cd pcre-8.12/
./configure
make;make install
cd ../

#使用Tcmalloc 优化nginx、mysql
wget http://google-perftools.googlecode.com/files/google-perftools-1.7.tar.gz
tar -zxvf google-perftools-1.7.tar.gz
cd google-perftools-1.7/
./configure
make;make install

echo "/usr/local/lib" > /etc/ld.so.conf.d/usr_local_lib.conf
/sbin/ldconfig
cd ../

tar zxvf nginx-1.0.0.tar.gz
cd nginx-1.0.0/
./configure --user=www --group=www --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_flv_module --with-http_gzip_static_module --with-google_perftools_module
make;make install
cd ../

mkdir -p /data/logs
chmod +w /data/logs
chown -R www:www /data/logs

#编辑配置文件
#编辑nginx.config
mv /usr/local/nginx/conf/nginx.conf /usr/local/nginx/conf/nginx.conf.bak
cat >>/usr/local/nginx/conf/nginx.conf<<EOF
user www www;
worker_processes 8;
error_log /usr/local/nginx/logs/nginx_error.log crit;
pid /usr/local/nginx/nginx.pid;
#使用Tcmalloc优化nginx性能
google_perftools_profiles /var/tmp/tcmalloc;
#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 65535;
#工作模式及连接数上限
events
{
use epoll;
worker_connections 65535;
}
#设定http服务器，利用它的反向代理功能提供负载均衡支持
http
{
#设定mime类型
include mime.types;
default_type application/octet-stream;
#charset gb2312;
#设定请求缓冲
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
server_tokens off;
client_body_buffer_size 512k;
proxy_connect_timeout 5;
proxy_send_timeout 60;
proxy_read_timeout 5;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
# fastcgi_connect_timeout 300;
# fastcgi_send_timeout 300;
# fastcgi_read_timeout 300;
# fastcgi_buffer_size 64k;
# fastcgi_buffers 4 64k;
# fastcgi_busy_buffers_size 128k;
# fastcgi_temp_file_write_size 128k;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;

#limit_zone crawler $binary_remote_addr 10m;

###禁止通过ip访问站点
server{
server_name _;
return 404;
}
server
{
listen 80;
server_name www.kerry.com;
index index.html index.htm index.jsp index.do;#设定访问的默认首页地址
root /data/www/shopxx;#设定网站的资源存放路径
#limit_conn crawler 20;
if (-d $request_filename)
{
rewrite ^/(.*)([^/])$ http://$host/$1$2/ permanent;
}
#所有jsp的页面均交由tomcat处理
location ~ \.(jsp|jspx|do)?$ {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://127.0.0.1:8080;#转向tomcat处理
}
location ~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc|ppt|pdf|xls|mp3|wma)$ #设定访问静态文件直接读取不经过tomcat
{
expires 30d;
}
location ~ .*\.(js|css)?$
{
expires 1h;
}

#定义访问日志的写入格式
log_format wwwlog '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log /data/logs/www_nginx.log wwwlog;#设定访问日志的存放路径
}
server
{
listen 80;
server_name bbs.kerry.com;
index index.html index.htm index.jsp index.do;#设定访问的默认首页地址
root /data/www/jsprun;#设定网站的资源存放路径
#limit_conn crawler 20;
if (-d $request_filename)
{
rewrite ^/(.*)([^/])$ http://$host/$1$2/ permanent;
}
#所有jsp的页面均交由tomcat处理
location ~ \.(jsp|jspx|do)?$ {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://127.0.0.1:8080;#转向tomcat处理
}
location ~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc|ppt|pdf|xls|mp3|wma)$ #设定访问静态文件直接读取不经过tomcat
{
expires 30d;
}
location ~ .*\.(js|css)?$
{
expires 1h;
}

#定义访问日志的写入格式
log_format bbslog '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log /data/logs/bbs_nginx.log bbslog;#设定访问日志的存放路径
}
server
{
listen 80;
server_name status.www.kerry.com;
location / {
stub_status on;
access_log off;
}
}
}
EOF

#检测配置文件
#/usr/local/nginx/sbin/nginx -t

#启动nginx
/usr/local/nginx/sbin/nginx
echo "/usr/local/nginx/sbin/nginx" >> /etc/rc.local

#查看nginx主进程号
#ps -ef | grep "nginx: master process" | grep -v "grep" | awk -F ' ' '{print $2}'

#停止nginx
#/usr/local/nginx/sbin/nginx -s stop

#平滑重启
#/usr/local/nginx/sbin/nginx -s reload

#============================= 3、配置一套java商城系统，进行测试 =============================
cd /opt
wget http://www.rarlab.com/rar/rarlinux-3.8.0.tar.gz
#rar解压
tar -zxvf rarlinux-3.8.0.tar.gz
cd rar
make install
cd /opt
unrar x jsprun.rar
mv jsprun /data/www/

#7z解压
wget http://sourceforge.net/projects/p7zip/files/p7zip/9.13/p7zip_9.13_src_all.tar.bz2/download
tar -jxvf p7zip_9.13_src_all.tar.bz2
cd p7zip_9.13
make;make install
7za x shopxx.7z
mv shopxx /data/www/

#配置tomcat虚拟主机
vi /usr/local/tomcat/conf/server.xml
<Host name="www.kerry.com" appBase="/data/www/shopxx" unpackWARs="true" autoDeploy="true">
<Context path="" docBase="."/>
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="/data/logs" prefix="www_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
</Host>
<Host name="bbs.kerry.com" appBase="/data/www/jsprun" unpackWARs="true" autoDeploy="true">
<Context path="" docBase="."/>
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="/data/logs" prefix="bbs_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
</Host>

#使用cmake编译安装mysql
cd /opt
wget http://www.cmake.org/files/v2.8/cmake-2.8.4.tar.gz
wget http://mirrors.sohu.com/mysql/MySQL-5.5/mysql-5.5.10.tar.gz
tar -zxvf cmake-2.8.4.tar.gz
cd cmake-2.8.4
./configu
make;make install

/usr/sbin/groupadd mysql
/usr/sbin/useradd -g mysql mysql -s /sbin/nologin
mkdir -p /data/mysql/data
mkdir -p /data/mysql/binlog
mkdir -p /data/mysql/relaylog
mkdir -p /data/mysql/mysql
chown -R mysql:mysql /data/mysql
cd /opt
tar zxvf mysql-5.5.10.tar.gz
cd mysql-5.5.10/
rm -rf CMakeCache.txt
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/mysql/ -DMYSQL_DATADIR=/data/mysql/data -DWITH_INNOBASE_STORAGE_ENGINE=1 -DMYSQL_TCP_PORT=3306 -DEXTRA_CHARSETS=all -DDEFAULT_CHARSET=utf8 -DDEFAULT_COLLATION=utf8_general_ci -DMYSQL_UNIX_ADDR=/data/mysql/mysql.sock -DMYSQL_USER=mysql -DWITH_DEBUG=0
make;make install
chmod +w /usr/local/mysql
chown -R mysql:mysql /usr/local/mysql

cat >>/data/mysql/my.cnf<<EOF
[client]
character-set-server = utf8
port = 3306
socket = /data/mysql/mysql.sock

[mysqld]
character-set-server = utf8
replicate-ignore-db = mysql
replicate-ignore-db = test
replicate-ignore-db = information_schema
user = mysql
port = 3306
socket = /data/mysql/mysql.sock
basedir = /usr/local/mysql
datadir = /data/mysql/data
log-error = /data/mysql/mysql_error.log
pid-file = /data/mysql/mysql.pid
open_files_limit = 10240
back_log = 600
max_connections = 5000
max_connect_errors = 6000
table_cache = 614
external-locking = FALSE
max_allowed_packet = 32M
sort_buffer_size = 1M
join_buffer_size = 1M
thread_cache_size = 300
thread_concurrency = 8
query_cache_size = 512M
query_cache_limit = 2M
query_cache_min_res_unit = 2k
default-storage-engine = MyISAM
thread_stack = 192K
transaction_isolation = READ-COMMITTED
tmp_table_size = 246M
max_heap_table_size = 246M
long_query_time = 3
log-slave-updates
log-bin = /data/mysql/binlog/binlog
binlog_cache_size = 4M
binlog_format = MIXED
max_binlog_cache_size = 8M
max_binlog_size = 100M
relay-log-index = /data/mysql/relaylog/relaylog
relay-log-info-file = /data/mysql/relaylog/relaylog
relay-log = /data/mysql/relaylog/relaylog
expire_logs_days = 30
key_buffer_size = 256M
read_buffer_size = 1M
read_rnd_buffer_size = 16M
bulk_insert_buffer_size = 64M
myisam_sort_buffer_size = 128M
myisam_max_sort_file_size = 10G
myisam_repair_threads = 1
myisam_recover

interactive_timeout = 120
wait_timeout = 120

skip-name-resolve
master-connect-retry = 10
slave-skip-errors = 1032,1062,126,1114,1146,1048,1396

#master-host = 192.168.1.2
#master-user = username
#master-password = password
#master-port = 3306

server-id = 1

innodb_additional_mem_pool_size = 16M
innodb_buffer_pool_size = 512M
innodb_data_file_path = ibdata1:256M:autoextend
innodb_file_io_threads = 4
innodb_thread_concurrency = 8
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 16M
innodb_log_file_size = 128M
innodb_log_files_in_group = 3
innodb_max_dirty_pages_pct = 90
innodb_lock_wait_timeout = 120
innodb_file_per_table = 0

#log-slow-queries = /data/mysql/slow.log
#long_query_time = 10

[mysqldump]
quick
max_allowed_packet = 32M
EOF

#初始化mysql
/usr/local/mysql/scripts/mysql_install_db -�Cdefaults-file=/data/mysql/my.cnf --basedir=/usr/local/mysql --datadir=/data/mysql/data --user=mysql

#利用TCMalloc提高mysql在高并发下的性能
vi /usr/local/mysql/bin/mysqld_safe

#在# executing mysqld_safe的下一行，加上：
export LD_PRELOAD=/usr/local/lib/libtcmalloc.so

#使用lsof命令查看tcmalloc是否起效
/usr/sbin/lsof -n | grep tcmalloc

#设置mysql启动文件
cp support-files/mysql.server /etc/rc.d/init.d/mysqld
vi /etc/rc.d/init.d/mysqld
basedir=/usr/local/mysql
datadir=/data/mysql/data

chmod 700 /etc/rc.d/init.d/mysqld
/etc/rc.d/init.d/mysqld start
/sbin/chkconfig --add mysqld
/sbin/chkconfig --level 2345 mysqld on
ln -s /usr/local/mysql/bin/mysql /sbin/mysql
ln -s /usr/local/mysql/bin/mysqladmin /sbin/mysqladmin

#设置root密码（753951）
/sbin/mysqladmin -u root password 753951

#配置库文件搜索路径
echo "/usr/local/mysql/lib/mysql" >> /etc/ld.so.conf
/sbin/ldconfig

#添加/usr/local/mysql/bin到环境变量PATH中
export PATH=$PATH:/usr/local/mysql/bin

#添加mysql管理帐户
#mysql -h localhost -u root -p753951
#msqyl> use mysql;
#msqyl> grant all on *.* to 'kerry'@'192.168.9.100' identified by '852741';
#msqyl> flush privileges;
#msqyl> exit;

#创建一个shopxx空库
CREATE DATABASE shopxx DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

#=============================== 4、tomcat优化 ============================
#添加tomcat管理员
#Tomcat 6.x不再提供admin管理工具(Tomcat's administration web application is no longer installed by default，Download and install the "admin" package to use it)，要自己安装
cd /opt
wget http://archive.apache.org/dist/tomcat/tomcat-5/v5.5.26/bin/apache-tomcat-5.5.26-admin.tar.gz
wget http://mirror.bjtu.edu.cn/apache//commons/logging/binaries/commons-logging-1.1.1-bin.tar.gz
wget http://labs.renren.com/apache-mirror//commons/modeler/binaries/commons-modeler-2.0.1.tar.gz
tar -zxvf apache-tomcat-5.5.26-admin.tar.gz
cd apache-tomcat-5.5.26
cp -r conf/Catalina /usr/local/tomcat/conf/
cp -r server /usr/local/tomcat/
cd /opt
tar -zxvf commons-modeler-2.0.1.tar.gz
cd commons-modeler-2.0.1
cp -r *.jar /usr/local/tomcat/lib/
cd ../
tar -zxvf commons-logging-1.1.1-bin.tar.gz
cd commons-logging-1.1.1
cp -r *.jar /usr/local/tomcat/lib/

vi /usr/local/tomcat/conf/tomcat-users.xml
#在<tomcat-users> </tomcat-users>中间添加 ,并去掉注释<!-- -->
<role rolename="manager"/>
< role rolename="admin"/>
< user username="admin" password="admin741852" roles="admin,manager"/>

vi /usr/local/tomcat/conf/context.xml
< Manager pathname="/manager" debug="0" privileged="true" docBase="/usr/local/tomcat/webapps/manager" />

#修改Java虚拟机的虚拟内存，tomcat默认的JVM比较小，查看JVM，打开http://127.0.0.1:8080/ ，点击"status"，查看JVM选项
#在修改配置文件的时候不知道怎么把manager/WEB-INF/web.xml搞没见了，导致在访问管理页面的时候出现“The requested resource (/manager/status) is not available.”
Free memory: 3.25 MB Total memory: 7.62 MB Max memory: 121.81 MB

vi /etc/init.d/tomcat
#在CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib"下方添加一句
JAVA_OPTS="-Xms1024m -Xmx1024m -Xmn256m -Djava.awt.headless=true"

#重启一下tomcat，然后再次查看JVM
#Free memory: 953.12 MB Total memory: 998.43 MB Max memory: 998.43 MB

#修改tomcat并发连接数，tomcat6.0 默认并发数为200，查看方法和JVM一样，查看http-8080选项
Max threads: 200

vi /usr/local/tomcat/conf/server.xml
#修改前：
<Connector port="8080" protocol="HTTP/1.1"
redirectPort="8443"
connectionTimeout="20000"
/>
#修改后：
<Connector port="8080" protocol="HTTP/1.1"
maxHttpHeaderSize="8192" useBodyEncodingForURI="true"
maxThreads="1000"
redirectPort="8443"
enableLookups="false"
compression="on"
compressionMinSize="2048"
compressableMimeType="text/html,text/xml,text/javascript,text/css,text/plain"
connectionTimeout="20000"
disableUploadTimeout="true"
/>

port：Tomcat服务器监听的端口号
maxHttpHeaderSize：Http的Header的最大限制
enableLookups="false" 使用允许DNS查询，通常情况下设置为false
redirectPort 服务器正在处理http请求时收到了一个SSL传输请求后重定向的端口号
maxThreads Tomcat使用线程来处理接收的每个请求。这个值表示Tomcat可创建的最大的线程数。
acceptCount 指定当所有可以使用的处理请求的线程数都被使用时，可以放到处理队列中的请求数，超过这个数的请求将不予处理。
connectionTimeout 网络连接超时，单位：毫秒。设置为0表示永不超时，这样设置有隐患的。通常可设置为30000毫秒。
minSpareThreads Tomcat初始化时创建的线程数。
maxSpareThreads 一旦创建的线程超过这个值，Tomcat就会关闭不再需要的socket线程。
compression="on" 打开压缩功能
compressionMinSize 启用压缩的输出内容大小，这里面默认为2KB
compressableMimeType压缩类型

#重启tomcat后再次查看
Max threads: 1000