检测脚本自查

#set -o nounset
set -o errexit
set -o pipefail
shopt -s expand_aliases

func() {
echo “Usage:”
echo
echo " $0 [-i UUID]"
echo
echo “Description:”
echo " -i : Get the uuid of instance."
echo " -h : usage message"
echo
echo “Examples:”
echo
echo “./getinfo.sh -i 55b6afc9bd114054a3f1b17bf59f4c71”
echo
exit
}

while getopts ‘:i:h’ OPT; do
case $OPTini)INSTANCEID="${OPTARG}";;
h) func;;
?) func;;
*) func;;
esac
done

if !(set -o nounset; : $INSTANCEID);thenfuncfiecho"取实例UUID:"$INSTANCEID

export PATH=$PATH:/apps/bin/
alias ks=“kubectl -nkube-system”
alias ka=“kubectl -nqfusion-admin”
#export KUBECONFIG=/apps/conf/kubernetes/admin.conf
if [ -r “/apps/conf/kubernetes/admin.conf” ]; then
export KUBECONFIG=/apps/conf/kubernetes/admin.conf
elif [ -r “./admin.conf” ];then
export KUBECONFIG=./admin.conf
else
echo “不存在可读的admin.conf”
fi

export KUBECONFIG=/etc/kubernetes/admin.conf # 这句话是加授权的意思,

export LD_LIBRARY_PATH=./lib/:$LD_LIBRARY_PATH
chmod +x ./nc
chmod +x ./jq
set -o nounset

getEvent() {
podname=$1echo"开始输出pod"${podname}“的Event”
ka get event -o json | ./jq “.items[] | select(.involvedObject.kind==“Pod”) | select(.involvedObject.name==”$podname\ddot{)}|.message"|se{d}^{\prime }s{/}^{/}{/}^{\prime }echo"停止输出pod"${podname}“的Event”
}

podCheck(){

proxy_pod_info=$1
podsinfo=($(echo ${proxy_pod_info} | awk -F ',' '{print $1 " "$2 " "$3}'))
#podsinfo=($(echo ${proxy_pod_infos[$i]} | awk -F ',' '{print $1 " "$2 " "$3}'))
podname=${podsinfo[0]}
podnumofc=${podsinfo[1]}
podstate=${podsinfo[2]}
numofcontainer=${podnumofc#*/}
runningcontainers=${podnumofc%/*}
#numofcontainer=`expr $numofcontainer+0`
#runningcontainers=`expr $runningcontainers+0`

if [ $podstate == "Running" ];
then
    if [ $numofcontainer -eq $runningcontainers ];
    then
        IFS=$'\n'
        echo "    "$podname  "状态是" $podstate "，且pod所有的容器都在运行中"
        if [[ $podname =~ ^slb ]]
        then
            ERRORlogs=($(ks logs --tail=200  $podname  | grep  -E -i  "ERROR|Exception" || echo ""))
        else
            IsMasterLabel=$(ka get pod $podname -o jsonpath="{.metadata.labels.IsMaster}")
            if [ -n "${IsMasterLabel}" ]
            then
                ERRORlogs=($(ka logs --tail=200  $podname  -c leader | grep -E -i "^E|Exception" || echo ""))
            else
                ERRORlogs=($(ka logs --tail=200  $podname  | grep  -i -E "ERROR|Exception" || echo ""))
            fi
        fi
        if [ ${#ERRORlogs[*]} -ne 0 ]
        then
            echo "    打印leader容器最近200行日志中的错误日志如下:"
            for ERRORlog in  ${ERRORlogs[*]}
            do
                echo "    "$ERRORlog
            done
        else
            echo "    未发现错误日志"
        fi
        IFS=$' \t\n'
    else
        echo "    "$podname  "状态是" $podstate "，pod中有"$(($numofcontainer - $runningcontainers))"个容器没在运行"
        getEvent $podname
    fi
else
    echo "    "$podname  "状态异常，状态是" $podstate
    getEvent $podname
fi

}

webserver_pod=$(kubectl -n kube-system get po -l app=webserver -oname | head -1 | awk -F’/’ ‘{print $2}’)

sql_command=“mysql -u$DATABASE_USERNAME -p$DATABASE_PASSWORD -h$REMOTE_HOST -e “use webserver;select app_name from orders where uuid = \”KaTeX parse error: Expected group as argument to '\"' at position 17: …INSTANCEID}\\\" ̲\\G\"" IFS=‘\r’
selectresult=($(kubectl -n kube-system exec -it $webserve{r}_{p}od-cwebserver-db--bash-c"${sql_command}” | grep -v “Using a password on the command line interface can be insecure.” | grep -v “^*” | awk -F “: " '{for (i=2;i<=NF;i++)printf(”%s: ", $i);print “”}’ | sed ‘s/: $//’ || echo “”))

if [ ${#selectresult[]} -gt 1 ]
then
echo " 有多个mysqlcluster,请确认输入"
exit
elif [ ${#selectresult[]} -eq 0 ]
then
echo " 有0个mysqlcluster,确认输入"
exit
fi
#echo KaTeX parse error: Expected '}', got '#' at position 2: {#̲selectresult[*]…’ \t\n’

查mysqlcluster

echo
echo
echo “检查mysqlcluster”
echo “***************************************”
echo mysqlcluster "$selectresult"MYCLUSTER=${selectresult}

customStatus=$(ka get mysqlcluster $MYCLUSTER-ojsonpath=".status.customStatus")fsmStatus=$(ka get mysqlcluster ${MYCLUSTER} -o jsonpath=“{.status.fsmStatus}”)

echo " "customStatus “为” ${customStatus}
echo " "fsmStatus “为” ${fsmStatus}

echo " 检查数据库"

selectfiled=(report_status status deleted_at order_ext_id user_id app_name name op_fields network_type)
sql_command=“mysql -u$DATABASE_USERNAME -p$DATABASE_PASSWORD -h$REMOTE_HOST -e “use webserver;select report_status,status,deleted_at,order_ext_id,user_id,app_name,name,op_fields,network_type from orders where app_name=\”${MYCLUSTER}\” \G""

IFS=KaTeX parse error: Expected group as argument to '\r' at position 4: '\r'̲ declare select…(kubectl -n kube-system exec -it $webserve{r}_{p}od-cwebserver-db--bash-c"${sql_command}" | grep -v “Using a password on the command line interface can be insecure.” | grep -v “^*” | awk -F “: " '{for (i=2;i<=NF;i++)printf(”%s: ", $i);print “”}’ | sed ‘s/: $//’ ))

if [ KaTeX parse error: Expected '}', got '#' at position 2: {#̲selectresult[*]…{#selectfiled[*]};j++))
do
#echo $j " " KaTeX parse error: Expected '}', got 'EOF' at end of input: {selectresult[j]}
declare -r KaTeX parse error: Expected '}', got 'EOF' at end of input: {selectfiled[j]}=“KaTeX parse error: Expected '}', got 'EOF' at end of input: {selectresult[j]}”
done
IFS=$’ \t\n’

arch=$(echo $o{p}_{f}ields|se{d}^{\prime }s/\mathring{:}$//’ | ./jq .arch)
instanceresource=“CPU:$(echo $o{p}_{f}ields|se{d}^{\prime }s/\mathring{:}$//’ | ./jq .resources.cpu) ,Memory:$(echo $o{p}_{f}ields|se{d}^{\prime }s/\mathring{:}$//’ | ./jq .resources.memory)”
if [ $arch{=}^{\prime }"single{"}^{\prime }]thenecho"当前数据库为单机数据库,"$instanceresource
elif [ $arch{=}^{\prime }"ms{"}^{\prime }]thenecho"当前数据库为主备数据库,"$instanceresource
else
echo " 当前数据库为只读数据库,"$instanceresource
fi

echo “”
echo " 开始打印configmap"
echo “”
ka get cm ${MYCLUSTER}“-config” -o json | ./jq -r ‘.data | keys[] as KaTeX parse error: Expected 'EOF', got '#' at position 6: k | "#̲###配置文件: \n \(k) \n####文件内容: \n (.[$k]|.) "’
echo " 结束打印configmap"

检查mysqldatabase

echo
echo
echo “检查mysqldatabase”
echo "**************************************"
mysqldatabase_infos=($(ka get mysqldatabase | grep $MYCLUSTER | awk '{print KaTeX parse error: Expected 'EOF', got '}' at position 2: 1}̲')) for((i=0;i<{#mysqldatabase_infos[]};i++))
do
echo " "mysqldatabase “KaTeX parse error: Expected '}', got 'EOF' at end of input: …database_infos[i]}”
done

检查SLB

echo
echo
echo “检查SLB”
echo “**************************************"
slb_pod_infos=($(ks get pod -LIsMaster | grep slb | awk ‘{print $1",“$2”,"$3}’ || echo ))
if [ KaTeX parse error: Expected '}', got '#' at position 2: {#̲slb_pod_infos[*…{#slb_pod_infos[]};i++))
do
podCheck KaTeX parse error: Expected '}', got 'EOF' at end of input: {slb_pod_infos[i]}
done
else
echo " 未检查到SLB”
fi

echo
echo
echo “检查calico”
echo “**************************************"
if [ -x “/apps/bin/calicoctl” ]; then
INTERNALIPregex=$(sudo /apps/bin/kubectl get nodes -o wide --no-headers | awk ‘{print $6}’ | xargs | sed ‘s/ /|/g’)
#echo $INTERNALIPregexIFS=$‘\n’
calico_conns=($(sudo/apps/bin/calicoctlnodestatus|grep-E"${INTERNALIPregex}" | awk -F ‘|’ ‘{if ($(NF-1)!~/Established/) print $2"is unreachable" }’))
if [ ${#calico_conns[]} -gt 0 ]
then
for calico_conn in $calic{o}_{c}onns[\ast ]doecho""$calico_conn
done
else
echo " calico连通性正常”
fi
IFS=$’ \t\n’
else
echo " 没有/apps/bin/calicoctl无法检查"
fi

检查pod

echo
echo
echo “检查Pod”
echo "**************************************"
pod_infos=($(ka get pod -L IsMaster | grep $MYCLUSTER | awk '{print $1",“$2”,"KaTeX parse error: Expected 'EOF', got '}' at position 2: 3}̲')) for((i=0;i<{#pod_infos[]};i++))
do
podCheck KaTeX parse error: Expected '}', got 'EOF' at end of input: {pod_infos[i]}
done

unset pod_infos
echo
echo
echo “检查备库”
echo “**************************************"
pod_infos=($(ka get pod -l IsMaster=false | grep $MYCLUSTER | awk ‘{print $1",“$2”,"$3}’))
#echo $po{d}_{i}nfos[\ast ]for((i=0;i<${#pod_infos[]};i++))
do
pod_info=KaTeX parse error: Expected '}', got 'EOF' at end of input: {pod_infos[i]}
IFS=‘,’
pod_info_list=($po{d}_{i}nfo)IFS=$’ \t\n’
podname=${pod_info_list[0]}
echo " 检查备库pod:” ${podname}

sql_command="mysql -uroot -p\${MYSQL_ROOT_PASSWORD} -e \"show slave status\\G\""
IFS=$'\n'
declare selectresult=($(ka exec -it ${podname} -c mysql -- bash -c "${sql_command}" |  grep -v "Using a password on the command line interface can be insecure." | grep -v "^\*" | sed    ":a;N;s/,\r\n/,/g;ta" | sed "s/\r//g" || echo ))

if [ ${#selectresult[*]} -ne 0 ]
then
    selectfields=("Slave_IO_Running Slave_SQL_Running Seconds_Behind_Master Executed_Gtid_Set")
    for selectresultele in  ${selectresult[*]}
    do
        IFS=$' '
        for selectfield in ${selectfields[*]}
        do
            if [[ ${selectresultele} =~ "${selectfield}:" ]]
            then
                declare "${selectfield}"=$(echo $selectresultele | awk -F ': ' '{print $2}')
                break
            fi
        done
        IFS=$'\n'
    done

    flag=0
    if [ $Slave_IO_Running != "Yes" ]
    then
        echo "        Slave_IO_Running 状态异常 "${Slave_IO_Running}
        flag=1
    fi

    if [ $Slave_SQL_Running != "Yes" ]
    then
        echo "        Slave_SQL_Running 状态异常 "${Slave_SQL_Running}
        flag=1
    fi

    if [ $Seconds_Behind_Master -gt 0 ]
    then
        echo "        备库落后于主库的时间:"${Seconds_Behind_Master}
        flag=1

        declare selectresult_sleep=($(ka exec -it ${podname} -c mysql -- bash -c "${sql_command}" |  grep -v "Using a password on the command line interface can be insecure." | grep -v "^\*" | sed    ":a;N;s/,\r\n/,/g;ta" | sed "s/\r//g" || echo ))

        for selectresultele in  ${selectresult[*]}
        do
            if [[ ${selectresultele} =~ "Executed_Gtid_Set:" ]]
            then
                Executed_Gtid_Set_sleep=$(echo $selectresultele | awk -F ': ' '{print $2}')
                break
            fi
        done
        if [ $Executed_Gtid_Set != $Executed_Gtid_Set_sleep ]
        then
            echo "        两次Executed_Gtid_Set执行结果不同"
        fi
    fi
    
    if [ $flag -eq 1 ]
    then
        echo "        备库状态异常"
    else
        echo "        备库状态正常"
    fi
else
    echo "        slave status状态为空，无法正常获取状态"
fi

IFS=$' \t\n'

done

ncCheckConnection(){
ncip=$1
ncport=$2set+oerrexitncresult=$(./nc $ncip $ncport -i 2 |& tr -d “\n”)
set -o errexit

if [ "${ncresult}" == "Ncat: Connection refused." ]
then
    echo "    尝试连接$ncip:$ncport,Connection refused，端口似乎没有启动"
elif [ "${ncresult}" == "Ncat: Connection timed out." ]
then
    echo "    尝试连接$ncip:$ncport,Connection timed out.网络似乎不可达"
else
    echo "    连接$ncip:$ncport,端口正常"
fi

}

echo
echo
echo “检查主库和备库”
echo "**************************************"
pod_infos=($(ka get pod -l IsMaster -o wide| grep $MYCLUSTER | awk '{print $1",“$2”,“$3”,"KaTeX parse error: Expected 'EOF', got '}' at position 2: 6}̲')) ncresult=''…{#pod_infos[]};i++))
do
pod_info=KaTeX parse error: Expected '}', got 'EOF' at end of input: {pod_infos[i]}
IFS=‘,’
pod_info_list=($po{d}_{i}nfo)podname=${pod_info_list[0]}
podip=${pod_info_list[3]}
#if [[ $podip={}^{(}[0-9]1,2|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\dot{(}[0-9]1,2|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\dot{(}[0-9]1,2|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\dot{(}[0-9]1,2|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$ ]]
if [ -n “$(echo $podip | grep -P “([0-9]+[.])+[0-9]+”)” ]
then
ncCheckConnection $podip"3306"fiIFS=$’ \t\n’
done

unset podip podname pod_info pod_infos pod_info_list

#检查SERVICE
echo
echo
echo “检查SERVICE”
echo “***************************************”
services=($(ka get svc --no-headers | grep -v ClusterIP |grep $MYCLUSTER | awk ‘{print $2";“$4”;"$5}’))
for service in $services[\ast ]doecho""$service
svctype=$(echo $service|cut-d^{\prime }{;}^{\prime }-f1)externalports=($(echo $service | cut -d ‘;’ -f 3 | grep -oP “[\d]+” | sed -n ‘2~2p’ | xargs))

#svctype="Load"

if [ $svctype == "NodePort" ]
then
    ncip="127.0.0.1"
else
    ncip=$(echo $service | cut -d ';' -f 2)
    if [ -z "$(echo $ncip | grep -P "([0-9]+[.])+[0-9]+")" ] 
    then
        echo "    未找到LoadBlance的 EXTERNAL-IP "
        continue
    fi
fi

for ncport in ${externalports[*]}
do
    ncCheckConnection $ncip $ncport
done

done

#检查数据库的日志
echo
echo
echo “检查errorlog和slowlog”
echo “**************************************"
pod_infos=($(ka get pod -l IsMaster -o wide| grep $MYCLUSTER | awk '{print KaTeX parse error: Expected 'EOF', got '}' at position 2: 1}̲')) ncresult=''…{#pod_infos[]};i++))
do
pod_name=KaTeX parse error: Expected '}', got 'EOF' at end of input: {pod_infos[i]}
echo " 检查pod ${pod_name} 的error.log”
echo " 开始打印ERROR日志"
ka exec -it ${pod_name} -c mysql – bash -c “tail -n 200 /var/log/mysql/error.log | grep -i error || echo " || echo “打印命令错误，请检查命令是否正确或者容器是否正常”
echo " 结束打印ERROR日志”

echo "    检查pod ${pod_name} 的slow.log" 
echo "        开始打印slow-query日志"
ka exec -it ${pod_name} -c mysql --  bash -c "tail -n 200 /var/log/mysql/slow-query.log || echo "  || echo "打印命令错误，请检查命令是否正确或者容器是否正常"
echo "        结束打印slow-query日志"
echo

done