Shell脚本监控Storm集群，邮件报警

这个脚本只是一个临时的方案，在全面的监控，报警系统完成之前为了节省一定的人力而写的

之前写过一个脚本用来监控Storm的Nimbus和Supervisor进程，在检测到进程不存在时会进行重启，在实际使用中发现，该脚本虽然可以不断重启，但某些情况下并不能完成自动重启，人为的介入仍然是必要的，所以对脚本进行了修改，增加检测重启次数，如果检测到重启失败，则会通过邮件通知相关人员。遇到的困难主要在于获取ssh操作的返回值，思路比较简单，脚本内容如下：

#!/bin/bash
dir=`dirname $0`
slaves="cdn36 cdn37 cdn39 cdn21 cdn22 cdn23"
stopnode=""

while [ 1 ]
do
    echo "==========  "`date`"    ==============="
    nid=`jps -l|grep 'nimbus'|awk '{print $1}'`
    if [ "$nid" = "" ]; then
            echo  'storm nimbus is dead!'
            echo  'trying to start nimbus...'
            nohup storm nimbus >nimbus.log &
            echo 'finish starting!'
    else
            echo "storm nimbus id: $nid"
    fi

    uid=`jps -l|grep 'backtype.storm.ui.core'|awk '{print $1}'`
    if [ "$nid" = "" ]; then
        echo  'storm ui process is dead!'
                echo  'trying to start storm ui'
                nohup storm ui >ui.log &
                echo 'finish starting storm ui!'
    else
        echo "storm ui id: $uid"
    fi 

    stopnode=""
    for node in $slaves
    do
       tmp=$(ssh $node 'source /etc/profile; source ~/.bash_profile;sid=`jps |grep supervisor |awk "{print $1}"`;
             if [ "$sid" = "" ]; then
                 echo "supervisor is dead trying to start supervisor!";
                 mkdir -p ~/rzx;
                 #重启前 删除worker文件
                 rm -fr /data/tmp/storm/worker;
                 nohup  storm supervisor >supervisor.log &
             else
                 echo " supervisor is alived,"${sid};
             fi')
    alived="alived"
    if [ "$tmp" = "${tmp//$alived}"  ] ; then
        stopnode=${stopnode}" "${node}
        echo ${node}"'s supervisor is dead!"
        tmp=""
    else
       echo ${node}"'s"${tmp}
       tmp=""
    fi
    done

    #sleep 等待Supervisor重启完成
    sleep 40

    #检测是否有Supervisor dead
    if [ -n "$stopnode" ] ;then
        echo "check dead supervisor!"
        for node in $stopnode
        do
           check=$(ssh  $node 'source /etc/profile; source ~/.bash_profile;sid=`jps |grep supervisor |awk "{print $1}"`;
             if [ "$sid" = "" ]; then
                 echo "supervisor is still dead!";
             else
                 echo " supervisor is alived,"${sid};
             fi')
           if [ "$check" = "${check//$alived}"  ] ; then
                echo ${node}"'s supervisor is still dead, send the email to admin!"
                title="Supervisor--is--dead"
                contxt=${node}"'s--supervisor--is--dead,please--check--the--server!"
                sh /data/www/mail/bin/start.sh  #发送邮件脚本
                slaves1=${slaves/${node}/""}
                slaves=""
                slaves=${slaves1}
                check=""
           else
                echo ${node}"'s"$check
                check=""
           fi
       done
    else
        echo "no dead supervisor!"
    fi  

    if [ "$nid" = "" ]; then
       nid1=`jps -l|grep 'nimbus'|awk '{print $1}'`
       if [ "$nid1" = ""  ]; then
         echo "nimbus is still dead, send the email to admin!"
         title1="Nimbus--is--dead"
         contxt1="Nimbus--is--dead,please--check--the--server!"
         sh /data/www/mail/bin/start.sh  #发送邮件脚本
       else
         echo "Nimbus is restarted!"
       fi       
    else
         echo
    fi
   
  echo "sleeping 20s..."
        sleep 20
done