运维自动化之使用PHP+MYSQL+SHELL打造私有监控系统（三）

（ 3）引擎服务器（以甘肃引擎为例,脚本解释参照甘肃web）

#!/bin/bash
#ip
yq_ip=$(/sbin/ifconfig eth0|grep "inet addr"|cut -d : -f 2|awk '{print $1}')
#name
yq_name=$(hostname)
monitor_name=yinqing
here='gansu'
#mysql info
mysql_ip='1.1.1.1'
mysql_username='root'
mysql_passwd='****'
mysql_database='monitor'
memory_table=''$here'_memory'
load_table=''$here'_load'
io_table=''$here'_io'
hardware_table=''$here'_hardware'
message_table=''$here'_message'
user_table=''$here'_user'
disk_table=''$here'_disk'
cpu_table=''$here'_cpu'
service_table=''$here'_service'
#date and log
day="$(date +%Y%m%d)"
worklog='/usr/local/monitor/logs/all_work_log'
downlog='/usr/local/monitor/logs/all_down_log'
now="$(date +%Y-%m-%d-%T)"
loghere='/usr/local/monitor/logs'
#yinqing service
memcache="$(ps -ef|grep 11211|grep -v grep|wc -l)"
datastorageservice="$(netstat -ant|grep 52010|wc -l)"
dbstatserver="$(ps -ef|grep dbstatserver|grep -v grep|wc -l)"
yq_http="$(netstat -antl|grep 80|wc -l)"
#cpu service
alert_cpu='80'
yq_cpu_idle="$(top -b -n 1 | grep Cpu | awk '{print $5}' | cut -f 1 -d .)"
yq_cpu="$(echo 100 - $yq_cpu_idle|/usr/bin/bc)"
#memory service
alert_mem='100'
yq_mem="$(/usr/bin/free -m|grep Mem|awk '{print $4}')"
yq_swap="$(/usr/bin/free -m|grep Swap|awk '{print $3}')"
alert_swap='0'
#memory log
memory_worklog='/usr/local/monitor/logs/mem_work_log'
memory_downlog='/usr/local/monitor/logs/mem_down_log'
#load service
cpu_count="$(grep -c 'model name' /proc/cpuinfo)"
alert_load="$(echo $cpu_count/2|/usr/bin/bc)"
yq_load="$(uptime|awk '{print $NF}'|cut -f 1 -d .)"
yq_load_15="$(uptime|awk '{print $NF}')"
#load log
load_worklog='/usr/local/monitor/logs/load_work_log'
load_downlog='/usr/local/monitor/logs/load_down_log'
#io service
alert_io='80'
yq_io_idle_back="$(/usr/bin/iostat|awk 'NR==4{print $NF}'|cut -f 1 -d .)"
yq_io_idle="$(echo 100 - $yq_io_idle_back|/usr/bin/bc)"
#io log
io_worklog='/usr/local/monitor/logs/io_work_log'
io_downlog='/usr/local/monitor/logs/io_down_log'
#hardware service
yq_hardware_error="$(dmesg|grep -i error|wc -l)"
yq_info_error="$(dmesg|grep -i error)"
#hardware error log
hard_worklog='/usr/local/monitor/logs/hard_work_log'
hard_downlog='/usr/local/monitor/logs/hard_down_log'
#message service
yq_message_error_count="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error|wc -l)"
yq_message_error="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error)"
#message error log
message_worklog='/usr/local/monitor/logs/message_work_log'
message_downlog='/usr/local/monitor/logs/message_down_log'
#user service
yq_user_count="$(/usr/bin/who|wc -l)"
yq_user_info="$(/usr/bin/who)"
#user  log
user_worklog='/usr/local/monitor/logs/user_work_log'
user_downlog='/usr/local/monitor/logs/user_down_log'
#disk service
alert_disk='80'
yq_disk="$(/bin/df -H|sort -k5nr|grep -vE 'Filesystem|tmpfs|cdrom'|awk '{print $5 " " $1}')"
yq_disk_use=$(echo $yq_disk|awk '{print $1}'|cut -d '%' -f1)
yq_disk_partition=$(echo $yq_disk|awk '{print $2}')
#disk log
disk_worklog='/usr/local/monitor/logs/disk_work_log'
disk_downlog='/usr/local/monitor/logs/disk_down_log'
#cpu log
cpu_worklog='/usr/local/monitor/logs/cpu_work_log'
cpu_downlog='/usr/local/monitor/logs/cpu_down_log'
#notification mail
email='[email protected]'
#monitor
if [ ! -d "$loghere" ];then
    mkdir $loghere
fi
#yinqing check
if [ $memcache -ge 1 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memcache Monitor_Server: $monitor_name is working" >> $worklog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memcache','working',now())";
else
                /opt/memcached-1.4.5/memcached -d -m 1024 -p 11211 -u root
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memcache Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service:  memcache  was a problem" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memcache Monitor_Server: $montior_name is down" >> $downlog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memcache','downing',now())";
fi
if [ $datastorageservice -ge 1 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: datastorageservice Monitor_Server: $monitor_name is working" >> $worklog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','datastorageservice','working',now())";
else
                /usr/local/lbs/bin/DataStorageService_0.sh restart >>/dev/null
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: datastorageservice Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service:  datastorageservice  was a problem" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: datastorageservice Monitor_Server: $montior_name is down" >> $downlog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','datastorageservice','downing',now())";
fi
if [ $dbstatserver -ge 1 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: dbstatserver Monitor_Server: $monitor_name is working" >> $worklog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','dbstatserver','working',now())";
else
                /bin/sh /usr/local/lbs/java/lib/dbstatserver/dbstatserver.sh
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: dbstatserver Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service:  dbstatserver  was a problem" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: dbstatserver Monitor_Server: $montior_name is down" >> $downlog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','dbstatserver','downing',now())";
fi
if [ $yq_http -ge 1 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: http Monitor_Server: $monitor_name is working" >> $worklog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','yq_http','working',now())";
else
                service httpd restart
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: http Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service:  http  was a problem" $email
        echo "$now ShengFen: $here Server: $yq_name Service: http Monitor_Server: $montior_name is down" >> $downlog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','yq_http','downing',now())";
fi
#check cpu_idle
if [ $yq_cpu -ge $alert_cpu ];then
    echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $yq_cpu"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: cpu_use  was Exceed Threshold value: 80%" $email
    echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $yq_cpu" >> $cpu_downlog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$yq_name','$yq_ip','$monitor_name','cpu_use','$alert_cpu','$yq_cpu','abnormal',now())";
else
    echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $yq_cpu" >> $cpu_worklog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$yq_name','$yq_ip','$monitor_name','cpu_use','$alert_cpu','$yq_cpu','normal',now())";
fi
#check memory
if [ $yq_mem -le $alert_mem ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memory Monitor_Server: $monitor_name Free_mem: $yq_mem"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: memory  was Exceed Threshold value" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memory Monitor_Server: $monitor_name Free_mem: $yq_mem" >> $memory_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memory','$alert_mem','$yq_mem','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memory Monitor_Server: $monitor_name Free_mem: $yq_mem" >> $memory_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memory','$alert_mem','$yq_mem','normal',now())";
fi
#check swap
if [ $yq_swap -gt $alert_swap ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: swap Monitor_Server: $monitor_name Swap_yq: $yq_swap"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: swap Ip: $yq_ip was Exceed Threshold value" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: swap Monitor_Server: $monitor_name Swap_yq: $yq_swap" >> $memory_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','swap','$alert_swap','$yq_swap','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: swap Monitor_Server: $monitor_name Swap_yq: $yq_swap" >> $memory_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','swap','$alert_swap','$yq_swap','normal',now())";
fi
#check load_15
if [ $yq_load -ge $alert_load ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $yq_load_15"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: load_15 Ip: $yq_ip was Exceed Threshold value" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $yq_load_15" >> $load_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$yq_name','$yq_ip','$monitor_name','load_15','$alert_load','$yq_load_15','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $server Ip: $ip Service: load Monitor_Server: $monitor_name Load_use: $yq_load_15" >> $load_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$yq_name','$yq_ip','$monitor_name','load_15','$alert_load','$yq_load_15','normal',now())";
fi
#check io_idle
if [ $yq_io_idle -ge $alert_io ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $yq_io_idle"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: io_use  was Exceed Threshold value: 80%" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $yq_io_idle" >> $io_downlog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$yq_name','$yq_ip','$monitor_name','io_use','$alert_io','$yq_io_idle','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $yq_io_idle" >> $io_worklog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$yq_name','$yq_ip','$monitor_name','io_use','$alert_io','$yq_io_idle','normal',now())";
fi
#check hareware error info
if [ $yq_hardware_error -gt 0 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: hardware_error Monitor_Server: $monitor_name Error: $yq_info_error"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: hardware_error  were some hardware imformation error" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: hardware_error Monitor_Server: $monitor_name Error: $yq_info_error" >> $hard_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$yq_name','$yq_ip','$monitor_name','hardware_error','0','$yq_hardware_error','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: hardware_error Monitor_Server: $monitor_name Error: Nothing" >> $hard_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$yq_name','$yq_ip','$monitor_name','hardware_error','0','$yq_hardware_error','normal',now())";
fi
#check message error
if [ $yq_message_error_count -ge 1 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: message_error Monitor_Server: $monitor_name Message_error: $yq_message_error"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: message_error  were some message imformation error" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: message_error Monitor_Server: $monitor_name Message_error: $yq_message_error" >> $message_downlog-$day
        /usr/bin/mysql-h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$yq_name','$yq_ip','$monitor_name','message_error','1','$yq_message_error_count','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: message_error Monitor_Server: $monitor_name Message_error: Nothing" >> $message_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $message_table values ('','$here','$yq_name','$yq_ip','$monitor_name','message_error','1','$yq_message_error_count','normal',now())";
fi
#check user
if [ $yq_user_count -ge 3 ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: user Monitor_Server: $monitor_name User: $yq_user_info"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name  Service: user  was Exceed Threshold value: 3" $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: user Monitor_Server: $monitor_name User: $yq_user_info" >> $user_downlog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$yq_name','$yq_ip','$monitor_name','user','3','$yq_user_count','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: user Monitor_Server: $monitor_name User: normal" >> $user_worklog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$yq_name','$yq_ip','$monitor_name','user','3','$yq_user_count','normal',now())";
fi
#check disk
if [ $yq_disk_use -ge $alert_disk ];then
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $yq_disk_partition ($yq_disk_use%)"|/bin/mail -s "Warning!!! $here Server: $yq_name Service: disk_use  was Exceed Threshold value : $alert_disk% " $email
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $yq_disk_partion ($yq_disk_use%)" >> $disk_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $disk_table values ('','$here','$yq_name','$yq_ip','$monitor_name','disk_use','$alert_disk','$yq_disk_partition','$yq_disk_use','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $yq_disk_partition ($yq_disk_use%)" >> $disk_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $disk_table values ('','$here','$yq_name','$yq_ip','$monitor_name','disk_use','$alert_disk','$yq_disk_partition','$yq_disk_use','normal',now())";
fi

（ 4）备用服务器（以甘肃备用为例，脚本解释参照甘肃web)

#!/bin/bash
#ip
by_ip=$(/sbin/ifconfig eth0|grep "inet addr"|cut -d : -f 2|awk '{print $1}')
#name
by_name=$(hostname)
monitor_name=beiyong
here='gansu'
#mysql info
mysql_ip='1.1.1.1'
mysql_username='root'
mysql_passwd='****'
mysql_database='monitor'
memory_table=''$here'_memory'
load_table=''$here'_load'
io_table=''$here'_io'
hardware_table=''$here'_hardware'
message_table=''$here'_message'
user_table=''$here'_user'
disk_table=''$here'_disk'
cpu_table=''$here'_cpu'
service_table=''$here'_service'
#date and log
day="$(date +%Y%m%d)"
worklog='/usr/local/monitor/logs/all_work_log'
downlog='/usr/local/monitor/logs/all_down_log'
now="$(date +%Y-%m-%d-%T)"
loghere='/usr/local/monitor/logs'
#cpu service
alert_cpu='80'
by_cpu_idle="$(top -b -n 1 | grep Cpu | awk '{print $5}' | cut -f 1 -d .)"
by_cpu="$(echo 100 - $by_cpu_idle|/usr/bin/bc)"
#memory service
alert_mem='100'
by_mem="$(/usr/bin/free -m|grep Mem|awk '{print $4}')"
by_swap="$(/usr/bin/free -m|grep Swap|awk '{print $3}')"
alert_swap='0'
#memory log
memory_worklog='/usr/local/monitor/logs/mem_work_log'
memory_downlog='/usr/local/monitor/logs/mem_down_log'
#load service
cpu_count="$(grep -c 'model name' /proc/cpuinfo)"
alert_load="$(echo $cpu_count/2|/usr/bin/bc)"
by_load="$(uptime|awk '{print $NF}'|cut -f 1 -d .)"
by_load_15="$(uptime|awk '{print $NF}')"
#load log
load_worklog='/usr/local/monitor/logs/load_work_log'
load_downlog='/usr/local/monitor/logs/load_down_log'
#io service
alert_io='80'
by_io_idle_back="$(/usr/bin/iostat|awk 'NR==4{print $NF}'|cut -f 1 -d .)"
by_io_idle="$(echo 100 - $by_io_idle_back|/usr/bin/bc)"
#io log
io_worklog='/usr/local/monitor/logs/io_work_log'
io_downlog='/usr/local/monitor/logs/io_down_log'
#hardware service
by_hardware_error="$(dmesg|grep -i error|wc -l)"
by_info_error="$(dmesg|grep -i error)"
#hardware error log
hard_worklog='/usr/local/monitor/logs/hard_work_log'
hard_downlog='/usr/local/monitor/logs/hard_down_log'
#message service
by_message_error_count="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error|wc -l)"
by_message_error="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error)"
#message error log
message_worklog='/usr/local/monitor/logs/message_work_log'
message_downlog='/usr/local/monitor/logs/message_down_log'
#user service
by_user_count="$(/usr/bin/who|wc -l)"
by_user_info="$(/usr/bin/who)"
#user  log
user_worklog='/usr/local/monitor/logs/user_work_log'
user_downlog='/usr/local/monitor/logs/user_down_log'
#disk service
alert_disk='80'
by_disk="$(/bin/df -H|sort -k5nr|grep -vE 'Filesystem|tmpfs|cdrom'|awk '{print $5 " " $1}')"
by_disk_use=$(echo $by_disk|awk '{print $1}'|cut -d '%' -f1)
by_disk_partition=$(echo $by_disk|awk '{print $2}')
#disk log
disk_worklog='/usr/local/monitor/logs/disk_work_log'
disk_downlog='/usr/local/monitor/logs/disk_down_log'
#cpu log
cpu_worklog='/usr/local/monitor/logs/cpu_work_log'
cpu_downlog='/usr/local/monitor/logs/cpu_down_log'
#notification mail
email='[email protected]'
#monitor
if [ ! -d "$loghere" ];then
    mkdir $loghere
fi
#check cpu_idle
#check cpu_idle
if [ $by_cpu -ge $alert_cpu ];then
    echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $by_cpu"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: cpu_use  was Exceed Threshold value: 80%" $email
    echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $by_cpu" >> $cpu_downlog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$by_name','$by_ip','$monitor_name','cpu_use','$alert_cpu','$by_cpu','abnormal',now())";
else
    echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $by_cpu" >> $cpu_worklog-$day
    /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$by_name','$by_ip','$monitor_name','cpu_use','$alert_cpu','$by_cpu','normal',now())";
fi
#check memory
if [ $by_mem -le $alert_mem ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: memory Monitor_Server: $monitor_name Free_mem: $by_mem"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: memory  was Exceed Threshold value" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: memory Monitor_Server: $monitor_name Free_mem: $by_mem" >> $memory_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','memory','$alert_mem','$by_mem','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: memory Monitor_Server: $monitor_name Free_mem: $by_mem" >> $memory_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','memory','$alert_mem','$by_mem','normal',now())";
fi
#check swap
if [ $by_swap -gt $alert_swap ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: swap Monitor_Server: $monitor_name Swap_by: $by_swap"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: swap Ip: $by_ip was Exceed Threshold value" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: swap Monitor_Server: $monitor_name Swap_by: $by_swap" >> $memory_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','swap','$alert_swap','$by_swap','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: swap Monitor_Server: $monitor_name Swap_by: $by_swap" >> $memory_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','swap','$alert_swap','$by_swap','normal',now())";
fi
#check load_15
if [ $by_load -ge $alert_load ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $by_load_15"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: load_15 Ip: $by_ip was Exceed Threshold value" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $by_load_15" >> $load_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$by_name','$by_ip','$monitor_name','load_15','$alert_load','$by_load_15','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $server Ip: $ip Service: load Monitor_Server: $monitor_name Load_use: $by_load_15" >> $load_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$by_name','$by_ip','$monitor_name','load_15','$alert_load','$by_load_15','normal',now())";
fi
#check io_idle
if [ $by_io_idle -ge $alert_io ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $by_io_idle"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: io_use  was Exceed Threshold value: 80%" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $by_io_idle" >> $io_downlog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$by_name','$by_ip','$monitor_name','io_use','$alert_io','$by_io_idle','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $by_io_idle" >> $io_worklog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$by_name','$by_ip','$monitor_name','io_use','$alert_io','$by_io_idle','normal',now())";
fi
#check hareware error info
if [ $by_hardware_error -gt 0 ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: hardware_error Monitor_Server: $monitor_name Error: $by_info_error"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: hardware_error  were some hardware imformation error" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: hardware_error Monitor_Server: $monitor_name Error: $by_info_error" >> $hard_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$by_name','$by_ip','$monitor_name','hardware_error','0','$by_hardware_error','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: hardware_error Monitor_Server: $monitor_name Error: Nothing" >> $hard_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$by_name','$by_ip','$monitor_name','hardware_error','0','$by_hardware_error','normal',now())";
fi
#check message error
if [ $by_message_error_count -ge 1 ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: message_error Monitor_Server: $monitor_name Message_error: $by_message_error"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: message_error  were some message imformation error" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: message_error Monitor_Server: $monitor_name Message_error: $by_message_error" >> $message_downlog-$day
        /usr/bin/mysql-h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$by_name','$by_ip','$monitor_name','message_error','1','$by_message_error_count','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: message_error Monitor_Server: $monitor_name Message_error: Nothing" >> $message_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $message_table values ('','$here','$by_name','$by_ip','$monitor_name','message_error','1','$by_message_error_count','normal',now())";
fi
#check user
if [ $by_user_count -ge 3 ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: user Monitor_Server: $monitor_name User: $by_user_info"|/bin/mail -s "Notification ShengFen: $here Server: $by_name  Service: user  was Exceed Threshold value: 3" $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: user Monitor_Server: $monitor_name User: $by_user_info" >> $user_downlog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$by_name','$by_ip','$monitor_name','user','3','$by_user_count','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: user Monitor_Server: $monitor_name User: normal" >> $user_worklog-$day
        /usr/bin/mysql  -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$by_name','$by_ip','$monitor_name','user','3','$by_user_count','normal',now())";
fi
#check disk
if [ $by_disk_use -ge $alert_disk ];then
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $by_disk_partition ($by_disk_use%)"|/bin/mail -s "Warning!!! $here Server: $by_name Service: disk_use  was Exceed Threshold value : $alert_disk% " $email
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $by_disk_partion ($by_disk_use%)" >> $disk_downlog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $disk_table values ('','$here','$by_name','$by_ip','$monitor_name','disk_use','$alert_disk','$by_disk_partition','$by_disk_use','abnormal',now())";
else
        echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $by_disk_partition ($by_disk_use%)" >> $disk_worklog-$day
        /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database  -e "insert into $disk_table values ('','$here','$by_name','$by_ip','$monitor_name','disk_use','$alert_disk','$by_disk_partition','$by_disk_use','normal',now())";
fi

 下一篇文章地址：

运维自动化之使用PHP+MYSQL+SHELL打造私有监控系统（四）

http://dl528888.blog.51cto.com/2382721/1035145