生产环境WEB服务管理脚本之监控脚本
环境说明：
公司是做在线教育的互联网企业，WEB架构为：前端使用LVS + Heartbeat做负载均衡，后端主要是Apache/Nginx + Tomcat，缓存有redis和Memcached,数据库使用的Oracle和Mysql。
脚本实现目的：
通过在服务器本地运行脚本，当检测到服务不可用时，自动重启相关服务，并发邮件通知管理员。
通过脚本检测服务关键配置文件，若文件丢失，则根据备份自动恢复文件，并重启。
脚本思路：
通过脚本访问事先定义好的监控页面，一共访问三次，每次访问间隔几秒钟，若三次访问都失败，则认为服务故障，重启服务并发邮件。配置文件的备份则是通过重启脚本来完成，所有的服务都是通过配置文件的方式传递给脚本，以方便批量部署。
脚本内容：
#!/bin/bash
#This shell-script is use for check tomcat and apache/nginx alive
#Created in 2012-11-01
#Last changed in 2013-05-25
source ~/.bash_profile &>/dev/null
dir=''
dir=/tol/app
dir2=/tol/script
cd $dir2
dt2=`date +"%Y-%m-%d"`
ls $dir2/logs &>/dev/null || mkdir -p $dir2/logs &>/dev/null
log="$dir2/logs/tomcatwget-$dt2.log"
host=`/sbin/ifconfig |grep "inet addr"|cut -d ':' -f2 |awk '{print $1}'|head -1`
wgetfile=$dir2/tomcat_wgetfile
conf=$dir2/tomcatwget.conf
conf3=$dir2/web.conf
sh_name=$0
function shijian () {
dt=`date +"%Y-%m-%d-%H:%M:%S"`
}
function apache_restart () {
#stop
user=`ps -elf |grep -v grep|grep -v root|grep '/bin/httpd'|grep "$dir"|awk '{print $3}'|head -1`
$dir/apache/bin/httpd -k stop &>/dev/null
sleep 5
opid=''
opid=`ps -elf |grep -v grep|grep "httpd"|grep "$dir"|awk '{print $4}'`
if test -n "$opid" ; then
kill -9 $opid
fi
#start
/usr/bin/ipcs -s | grep "$user"| gawk '{ print $2 }' | xargs -n 1 ipcrm sem &> /dev/null
$dir/apache/bin/httpd -k start &>/dev/null
npid=`ps -elf |grep -v grep|grep "httpd"|grep root |grep "$dir"|awk '{print $4}'`
shijian
if test -z "$npid" ; then
echo "The $dt $host apache restart fail by $sh_name" >> $log
echo "The $dt $host apache restart fail by $sh_name" |mail -s "check $host apache restart fail" jiank
else
echo "The $dt $host apache is restart,the new pid=$npid" >> $log
echo "The $dt $host apache is restart by $sh_name,the new pid=$npid" |mail -s "check $host apache restart" jiank
fi
}
function nginx_restart () {
#stop
opid=''
opid=`cat $dir/nginx/logs/nginx.pid`
kill -QUIT $opid
sleep 3
opid=`ps -elf |grep -v grep|grep process|grep nginx |awk '{print $4}'`
if test ! -z "$opid" ; then
kill -9 $opid
fi
#start
$dir/nginx/sbin/nginx -c $dir/nginx/conf/nginx.conf
npid=''
npid=`ps -elf |grep -v grep|grep process|grep nginx |awk '{print $4}'`
shijian
if test -z "$npid" ; then
echo "The $dt $host nginx restart fail by $sh_name" >> $log
echo "The $dt $host nginx restart fail by $sh_name" |mail -s "check $host nginx restart fail" jiank
else
echo "The $dt $host nginx is restart,the new pid=$npid" >> $log
echo "The $dt $host nginx is restart by $sh_name,the new pid=$npid" |mail -s "check $host nginx restart" jiank
fi
}
function http_restart () {
if test ! -f $conf3 ; then
echo "The $conf3 is not exist" >> $log
apache_restart
sleep 3
nginx_restart
else
while read line
do
dir=`echo $line |awk -F\; '{print $1}'`
softname=`echo $line |awk -F\; '{print $2}'`
$softname\_restart
done < $conf3
fi
}
function web_reboot () {
http_restart
sh $dir2/tomcat-restart.sh $name &>/dev/null
exit 0
}
#conf-check
if test ! -f $conf ; then
echo "The $conf is not exist"
exit 0
fi
shijian
echo "$dt" >> $log
echo "$host" >> $log
##Apache-check
domain=`/usr/bin/head -1 $conf |awk -F\; '{print $1}'`
if test `echo $domain |grep http` ; then
html="$domain"
else
html=http://$domain/t-s.html
fi
cat /dev/null > $wgetfile
/usr/bin/elinks -dump $html >> $wgetfile 2>&1
sleep 3
/usr/bin/elinks -dump $html >> $wgetfile 2>&1
sleep 3
/usr/bin/elinks -dump $html >> $wgetfile 2>&1
num=''
num=`grep -i -c "ok" $wgetfile`
if (( $num < 1 )) ; then
shijian
echo "The $dt $host $html wget error" >> $log
echo "The $dt $host $html wget error" | mail -s "check the $host wget error" jiank
http_restart
elif (( $num < 3&&$num >= 1 )) ; then
echo "The $host $html wget oknumber is $num" |mail -s "$host wget oknumber" jiank
else
echo "The $host $html wget success" >> $log
fi
while read line
do
shijian
file1=''
file1=`echo $line |awk -F\; '{print $2}'`
file2=''
file2=`echo $line |awk -F\; '{print $3}'`
name=''
name=`echo $line |awk -F\; '{print $4}'`
port=''
port=`echo $line |awk -F\; '{print $5}'`
function pid () {
pid=''
pid=`ps -elf|grep java|grep "$dir"|grep $name|awk '{print $4}'`
}
function down () {
`echo $line |awk -F\; '{print $6}'`
sleep 3
pid
if test -n "$pid" ; then
kill -9 $pid
fi
rm -rf $dir/$name/work/Catalina/*
rm -rf $dir/$name/temp/*
}
function up () {
`echo $line |awk -F\; '{print $7}'`
}
#web.xml-check
if test "$file1" ; then
if test ! -f $file1 ; then
shijian
\cp $file1.bak $file1 && chown tomcat.tomcat $file1 &>/dev/null
if test -f $file1 ; then
echo "The $dt $file1 is lost and will be recovery" >> $log
echo "The $dt $file1 is lost and will be recovery" | mail -s "check the $host $file1 lost" jiank
web_reboot
else
echo "The $dt $file1 is lost and recovery fail" >> $log
echo "The $dt $file1 is lost and recovery fail" | mail -s "check the $host $file1 lost" jiank
fi
else
echo "The $dt $file1 is ok" >> $log
fi
fi
#ROOT.xml-check
if test "$file2" ; then
if test ! -f $file2 ; then
shijian
\cp $file2.bak $file2 && chown tomcat.tomcat $file2 &>/dev/null
echo "The $dt $file2 do not exist" >> $log
echo "The $dt $file2 do not exist" | mail -s "check the $host $file2 lost" jiank
down
up
pid
if test -n "$pid" ; then
echo "$dt $host $file2 is lost and restart by $sh_name" |mail -s "$host $file2 is lost" jiank
else
up
echo "The $dt $host $file2 is lost and restart fail by $sh_name" >> $log
echo "The $dt $host $file2 is lost and restart fail by $sh_name" |mail -s "check $host $name restart fail" jiank
fi
else
echo "The $dt $file2 is ok" >> $log
fi
fi
#tomcat-check
if test "$port" ; then
if test `echo $port|grep http` ; then
jsp="$port"
else
jsp=http://127.0.0.1:$port/t-s.jsp
fi
cat /dev/null > $wgetfile
/usr/bin/elinks -dump $jsp >> $wgetfile 2>&1
sleep 5
/usr/bin/elinks -dump $jsp >> $wgetfile 2>&1
sleep 5
/usr/bin/elinks -dump $jsp >> $wgetfile 2>&1
num=''
num=`grep -i -c "ok" $wgetfile`
if (( $num < 1 )) ; then
shijian
echo "The $dt $host $name wget error 3 times and begin to restart" >> $log
down
up
pid
if test -n "$pid" ; then
echo "$dt Restart $name success pid= " $pid >> $log
echo "$dt $host $name wget error and restart by $sh_name" |mail -s "$host $name wget error" jiank
else
up
echo "The $dt $host $name wget error and restart fail by $sh_name" >> $log
echo "The $dt $host $name wget error and restart fail by $sh_name" |mail -s "check $host $name restart fail" jiank
fi
elif (( $num < 3&&$num >= 1 )) ; then
echo "The $host $name wget oknumber is $num" |mail -s "$host tomcat wget oknumber" jiank
else
echo "The $host $name wget success" >> $log
fi
fi
done < $conf
脚本配置文件：
127.0.0.1;/tol/htdocs/tomcata/ROOT/WEB-INF/web.xml;/tol/app/tomcata/conf/Catalina/localhost/ROOT.xml;tomcata;7010;/etc/init.d/tomcata stop;/etc/init.d/tomcata start; ;tol/htdocs/tomcatb/ROOT/WEB-INF/web.xml;/tol/app/tomcatb/conf/Catalina/localhost/ROOT.xml;tomcatb;7011;/etc/init.d/tomcatb stop;/etc/init.d/tomcatb start;