目前生产环直要监控指标
1、zabbix_agentd.conf
UserParameter=system1.uname,/bin/uname -r
###Recv-q
#UserParameter=recv-q,ss -nl|awk '{print $2}'|grep -v "Recv-Q"|awk '{if($1>0) {print 1}}'|wc -l
####memory
UserParameter=master.memtotal,echo "scale=2;`/usr/local/zabbix/script/memory.sh Memtotal` /1024" | bc
UserParameter=master.memuse,echo "scale=2;`/usr/local/zabbix/script/memory.sh Memuse` /1024" | bc
UserParameter=master.memfree,echo "scale=2;`/usr/local/zabbix/script/memory.sh Memfree` /1024" | bc
UserParameter=master.bufferuse,echo "scale=2;`/usr/local/zabbix/script/memory.sh Bufferuse` /1024" | bc
UserParameter=master.bufferfree,echo "scale=2;`/usr/local/zabbix/script/memory.sh Bufferfree` /1024" | bc
####disk
#UserParameter=dcserver4.root.total,/bin/df -lh | sed -n 2p | awk '{print $2}' | sed s/G//g
#UserParameter=dcserver4.root.use,/bin/df -lh | sed -n 2p | awk '{print $3}' | sed s/G//g
#UserParameter=dcserver4.root.use_per,/bin/df -lh | sed -n 2p | awk '{print $(NF-1)}' | sed s/%//g
#UserParameter=dcserver4.data0.total,/bin/df -lh | sed -n 5p | awk '{print $2}' | sed s/G//g
#UserParameter=dcserver4.data0.use,/bin/df -lh | sed -n 5p | awk '{print $3}' | sed s/G//g
#UserParameter=dcserver4.data0.use_per,/bin/df -lh | sed -n 5p | awk '{print $(NF-1)}' | sed s/%//g
#UserParameter=dcserver4.data1.total,/bin/df -lh | sed -n 6p | awk '{print $2}' | sed s/G//g
#UserParameter=dcserver4.data1.use,/bin/df -lh | sed -n 6p | awk '{print $3}' | sed s/M//g
#UserParameter=dcserver4.data1.use_per,/bin/df -lh | sed -n 6p | awk '{print $(NF-1)}' | sed s/%//g
###CPU load
UserParameter=master.cpu1,/usr/bin/w | sed -n 1p | awk -F ":" '{print $5}' | awk -F "," '{print $1}'
UserParameter=master.cpu5,/usr/bin/w | sed -n 1p | awk -F ":" '{print $5}' | awk -F "," '{print $2}'
UserParameter=master.cpu15,/usr/bin/w | sed -n 1p | awk -F ":" '{print $5}' | awk -F "," '{print $3}'
###### uptime
UserParameter=master.uptime,/usr/bin/w | sed -n 1p | awk -F "," '{print $1}' | awk -F " " '{print $(NF-1)}'
####DISK IO
UserParameter=disk.pgpgin,cat /proc/vmstat | grep pgpgin | awk -F " " '{print $2}'
UserParameter=disk.pgpgout,cat /proc/vmstat | grep pgpgout| awk -F " " '{print $2}'
UserParameter=disk.pswpin,cat /proc/vmstat | grep pswpin |awk '{print $2}'
UserParameter=disk.pswpout,cat /proc/vmstat | grep pswpout |awk '{print $2}'
###TCP,UDP
UserParameter=passive.tcp.conn,cat /proc/net/snmp | grep Tcp | awk -F: '{print $2}'|awk -F " " '{print $6}'| grep -v PassiveOpens
UserParameter=tcp.conn,cat /proc/net/snmp | grep Tcp | awk -F: '{print $2}'|awk -F " " '{print $9}'| grep -v CurrEstab
UserParameter=udp.receive,cat /proc/net/snmp | grep Udp:|awk -F: '{print $2}'|awk -F " " '{print $1}'|grep -v InDatagrams
UserParameter=udp.send,cat /proc/net/snmp | grep Udp:|awk -F: '{print $2}'|awk -F " " '{print $4}'|grep -v OutDatagrams
UserParameter=custom.vfs.dev.read.ms[*],cat /proc/diskstats | grep $1 | head -1 | awk '{print $$7}'
UserParameter=custom.vfs.dev.io.active[*],cat /proc/diskstats | grep $1 | head -1 | awk '{print $$12}'
UserParameter=custom.vfs.dev.io.ms[*],cat /proc/diskstats | grep $1 | head -1 | awk '{print $$13}'
UserParameter=custom.vfs.dev.read.sectors[*],cat /proc/diskstats | grep $1 | head -1 | awk '{print $$6}'
####Dynamic monitoring of disk performance
UserParameter=io.avgrq-sz[*],/usr/bin/tail /tmp/iostat_output | grep $1 |tail -1|awk '{print $$8}'
UserParameter=io.avgqu-sz[*],/usr/bin/tail /tmp/iostat_output | grep $1 | tail -1 | awk '{print $$9}'
UserParameter=io.util[*],/usr/bin/tail /tmp/iostat_output |grep $1 | tail -1 | awk '{print $$12}'
###vmstat Performance status
####check disk status##
UserParameter=check_disk_status,mount | awk '{print $NF}'|cut -c 2-3|awk '{if($1~/ro/) {print 1}}'|wc -l|awk '{if($1<=0) {print 0 } else {print 1}}'
######check servers status###
UserParameter=check_ip_exsit[*],nmap -sn $1 | grep down | wc -l
####mysql
#UserParameter=zabbix_low_discovery[*],/bin/bash /usr/local/zabbix/script/zabbix_low_discovery.sh $1
#UserParameter=mysql_stats[*],mysql -h 127.0.0.1 -P $1 -uzabbix -phuidong2013 -e "show global status"|grep "\<$2\>"|cut -f2
UserParameter=mysql_stats[*],sudo /usr/local/mysql/bin/mysql --login-path=local -P $1 -e "show global status" |grep "\<$2\>" |cut -f2
UserParameter=mysql_stats_slave[*],sudo /usr/local/mysql/bin/mysql --login-path=local -P $1 -e "show slave global status\G" |grep "\<$2\>" |awk '{if($NF=="Yes"){print 1} else {print 0}}'
#####redis
UserParameter=redis_stats[*],(echo info;sleep 1) | telnet 192.168.1.205 $1 2>&1 | grep $2|cut -d : -f2
#######follow is monitor hardware
UserParameter=hardware_battery,omreport chassis batteries|awk '/^Status/{if($NF=="Ok") {print 1} else {print 0}}'
UserParameter=hardware_cpu_model,awk -v hardware_cpu_crontol=`sudo omreport chassis biossetup|awk '/C State/{if($NF=="Enabled") {print 0} else {print 1}}'` -v hardware_cpu_c1=`sudo omreport chassis biossetup|awk '/C1[-|E]/{if($NF=="Enabled") {print 0} else {print 1}}'` 'BEGIN{if(hardware_cpu_crontol==0 && hardware_cpu_c1==0) {print 0} else {print 1}}'
UserParameter=hardware_fan_health,awk -v hardware_fan_number=`omreport chassis fans|grep -c "^Index"` -v hardware_fan=`omreport chassis fans|awk '/^Status/{if($NF=="Ok") count+=1}END{print count}'` 'BEGIN{if(hardware_fan_number==hardware_fan) {print 1} else {print 0}}'
UserParameter=hardware_memory_health,awk -v hardware_memory=`omreport chassis memory|awk '/^Health/{print $NF}'` 'BEGIN{if(hardware_memory=="Ok") {print 1} else {print 0}}'
UserParameter=hardware_nic_health,awk -v hardware_nic_number=`omreport chassis nics |grep -c "Interface Name"` -v hardware_nic=`omreport chassis nics |awk '/^Connection Status/{print $NF}'|wc -l` 'BEGIN{if(hardware_nic_number==hardware_nic) {print 1} else {print 0}}'
UserParameter=hardware_cpu,omreport chassis processors|awk '/^Health/{if($NF=="Ok") {print 1} else {print 0}}'
UserParameter=hardware_power_health,awk -v hardware_power_number=`omreport chassis pwrsupplies|grep -c "Index"` -v hardware_power=`omreport chassis pwrsupplies|awk '/^Status/{if($NF=="Ok") count+=1}END{print count}'` 'BEGIN{if(hardware_power_number==hardware_power) {print 1} else {print 0}}'
UserParameter=hardware_temp,omreport chassis temps|awk '/^Status/{if($NF=="Ok") {print 1} else {print 0}}'|head -n 1
UserParameter=hardware_physics_health,awk -v hardware_physics_disk_number=`omreport storage pdisk controller=0|grep-c "^ID"` -v hardware_physics_disk=`omreport storage pdisk controller=0|awk '/^Status/{if($NF=="Ok") count+=1}END{print count}'` 'BEGIN{if(hardware_physics_disk_number==hardware_physics_disk) {print 1} else {print 0}}'
UserParameter=hardware_virtual_health,awk -v hardware_virtual_disk_number=`omreport storage vdisk controller=0|grep -c "^ID"` -v hardware_virtual_disk=`omreport storage vdisk controller=0|awk '/^Status/{if($NF=="Ok") count+=1}END{print count}'` 'BEGIN{if(hardware_virtual_disk_number==hardware_virtual_disk) {print 1} else {print 0}}'
2、用到lld发现脚本
[root@dcserver2 script]# vim zabbix_low_discovery.sh
#!/bin/bash
#Fucation:zabbix low-level discovery
memcache() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/[m]emcached/ && /0.0.0.0/ {print $5}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#MEMPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#MEMPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
redis() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/[r]edis/ && /0.0.0.0/ {print $5}'|grep ^7))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#REDISPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#REDISPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
node() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/[n]ode/ && /0.0.0.0/ {print $5}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#NODEPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#NODEPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
mysql() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/[m]ysql/ {print $4}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#MYSQLPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#MYSQLPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
sshd() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/sshd/ && /0.0.0.0/ {print $5}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#SSHDPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#SSHDPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
mongodb() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/mongo/ && /0.0.0.0/ {print $5}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#MONGODBPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#MONGODBPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
partition() {
port=($(grep -E "(vd[a-z]$|sd[a-z]$)" /proc/partitions|awk '{print $4}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#PARTITIONNAME}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#PARTITIONNAME}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
twemproxy() {
port=($(sudo netstat -tpln | awk -F "[ :]+" '/nutcracker/ && /0.0.0.0/ {print $5}'|uniq))
printf '{\n'
printf '\t"data":[\n'
for key in ${!port[@]}
do
if [[ "${#port[@]}" -gt 1 && "${key}" -ne "$((${#port[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#TWEMPROXYPORT}\":\"${port[${key}]}\"},\n"
else [[ "${key}" -eq "((${#port[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#TWEMPROXYPORT}\":\"${port[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
check_ip_exsit() {
ip=($(sudo nmap -sn 192.168.1.162-210|grep "192.168.1"|awk -F " " '{print $NF}'|awk -F "[()]" '{print $2}'))
printf '{\n'
printf '\t"data":[\n'
for key in ${!ip[@]}
do
if [[ "${#ip[@]}" -gt 1 && "${key}" -ne "$((${#ip[@]}-1))" ]];then
printf '\t {\n'
printf "\t\t\t\"{#IPNAME}\":\"${ip[${key}]}\"},\n"
else [[ "${key}" -eq "((${#ip[@]}-1))" ]]
printf '\t {\n'
printf "\t\t\t\"{#IPNAME}\":\"${ip[${key}]}\"}\n"
fi
done
printf '\t ]\n'
printf '}\n'
}
case $1 in
memcache)
memcache
;;
redis)
redis
;;
node)
node
;;
mysql)
mysql
;;
sshd)
sshd
;;
mongodb)
mongodb
;;
partition)
partition
;;
twemproxy)
twemproxy
;;
check_ip_exsit)
check_ip_exsit
;;
*)
echo "Usage:`basename $0` {memcache|redis|node|mysql|sshd|mongodb|partition|twemproxy|check_ip_exsit}"
;;
esac
3、发送邮件脚本
[root@dcserver2 script]# vim zabbix_sendmail
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Zabbix SMTP Alert script from 126.
"""
import sys
import smtplib
from email.mime.text import MIMEText
mail_host="smtp.163.com"
mail_user="******@163.com"
mail_pass="*****"
mail_postfix="pop.163.com"
def send_mail(to_list,sub,content):
'''
to_list:sent to who
sub:subject
content:content
send_mail("[email protected]","sub","content")
'''
address=mail_user+"<"+mail_user+"@"+mail_postfix+">"
msg = MIMEText(content)
msg['Subject'] = sub
msg['From'] = address
msg['To'] =to_list
try:
s = smtplib.SMTP()
s.connect(mail_host)
s.login(mail_user,mail_pass)
s.sendmail(address, to_list, msg.as_string())
s.close()
return True
except Exception, e:
print str(e)
return False
if __name__ == '__main__':
send_mail(sys.argv[1], sys.argv[2], sys.argv[3])