【云原生 | Kubernetes 系列】---AlertManage 无法发送企业微信告警故障排除
【云原生 | Kubernetes 系列】—AlertManage 无法发送企业微信告警故障排除
1. 故障现象
- Prometheus 报警已经产生
- AlertManager接收到报警
- 确认AlertManager的报警也没有错误
# ./amtool check-config alertmanager.yml
- 企业ID,secret,机器人ID也没有搞错
2. 故障排查
编写python文件
root@prometheus-2:/apps/alertmanager# cat wechat.py
#!/usr/bin/env python
#coding=utf_8
#!/root/.virtualenvs/wechat/bin/python
# usage: send message via wechat
import requests, sys, json
import urllib3
urllib3.disable_warnings()
###填写参数###
# Corpid是企业号的标识
Corpid = "ww11e9" ### 修改这里
# Secret是管理组凭证密钥
Secret = "9j4EAng2zKXabEMV" ### 修改这里
# 应用ID
Agentid = "1003" ### 修改这里
# token_config文件放置路径
Token_config = r'/usr/local/test__wechat_config.json'
###下面的代码都不需要动###
def GetTokenFromServer(Corpid, Secret):
"""获取access_token"""
Url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken"
Data = {
"corpid": Corpid,
"corpsecret": Secret
}
r = requests.get(url=Url, params=Data, verify=False)
print(r.json())
if r.json()['errcode'] != 0:
return False
else:
Token = r.json()['access_token']
file = open(Token_config, 'w')
file.write(r.text)
file.close()
return Token
def SendMessage(Partyid, Subject, Content):
"""发送消息"""
# 获取token信息
try:
file = open(Token_config, 'r')
Token = json.load(file)['access_token']
file.close()
except:
Token = GetTokenFromServer(Corpid, Secret)
# 发送消息
# Url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s" % Token
Url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=ACCESS_TOKEN"
Data = {
"toparty": Partyid,
"msgtype": "text",
"agentid": Agentid,
"text": {"content": Subject + '\n' + Content},
"safe": "0"
}
r = requests.post(url=Url, data=json.dumps(Data), verify=False)
# 如果发送失败，将重试三次
n = 1
while r.json()['errcode'] != 0 and n < 4:
n = n + 1
Token = GetTokenFromServer(Corpid, Secret)
if Token:
Url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s" % Token
r = requests.post(url=Url, data=json.dumps(Data), verify=False)
print(r.json())
return r.json()
if __name__ == '__main__':
# 部门id
Partyid = '2'
# 消息标题
Subject = '自应用程序代码测试'
# 消息内容
Content = 'str(sys.argv[3])'
Status = SendMessage(Partyid, Subject, Content)
print(Status)
执行python测试脚本
如果是有这个报错
root@prometheus-2:/apps/alertmanager# python2 wechat.py
Traceback (most recent call last):
File "wechat.py", line 5, in <module>
import requests, sys, json
ImportError: No module named requests
先安装requests包
# pip install requests
DEPRECATION: Python 2.7 reached the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no longer maintained. pip 21.0 will drop support for Python 2.7 in January 2021. More details about Python 2 support in pip can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support pip 21.0 will remove support for this functionality.
Collecting requests
Downloading requests-2.27.1-py2.py3-none-any.whl (63 kB)
|████████████████████████████████| 63 kB 1.7 MB/s
Collecting idna<3,>=2.5; python_version < "3"
Downloading idna-2.10-py2.py3-none-any.whl (58 kB)
|████████████████████████████████| 58 kB 3.0 MB/s
Collecting certifi>=2017.4.17
Downloading certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
|████████████████████████████████| 149 kB 5.7 MB/s
Collecting chardet<5,>=3.0.2; python_version < "3"
Downloading chardet-4.0.0-py2.py3-none-any.whl (178 kB)
|████████████████████████████████| 178 kB 6.1 MB/s
Collecting urllib3<1.27,>=1.21.1
Downloading urllib3-1.26.12-py2.py3-none-any.whl (140 kB)
|████████████████████████████████| 140 kB 6.7 MB/s
Installing collected packages: idna, certifi, chardet, urllib3, requests
Successfully installed certifi-2021.10.8 chardet-4.0.0 idna-2.10 requests-2.27.1 urllib3-1.26.12
执行python测试脚本
root@prometheus-2:/apps/alertmanager# python2 wechat.py
{u'access_token': u'oJrDTOHB-zkbalxAG76yd5mWEq6qLvRvB1haOtHuI7ZD1lvI5qYtahku83Pw1W_WhF12FnpTbxKrSY1s3BZ7c2a23fdiOTuNmA6XUFz0LNRbjXwto6TG73NboPZuZPiUgAMjPnM6FNx3dmbEzQhZBhhlf3qb_RsgjCzKCu1DI4L8YTCUlwV97_idrKr3h9zYY7eJQoRTvpTPosZsLIWKjqRQ', u'expires_in': 7200, u'errcode': 0, u'errmsg': u'ok'}
{u'access_token': u'oJrDTOHB-zkbalxAG76yd5mWEq6qLvRvB1haOtHuI7ZD1lvI5qYtahku83Pw1W_WhF12FnpTbxKrSY1s3BZ7c2a23fdiOTuNmA6XUFz0LNRbjXwto6TG73NboPZuZPiUgAMjPnM6FNx3dmbEzQhZBhhlf3qb_RsgjCzKCu1DI4L8YTCUlwV97_idrKr3h9zYY7eJQoRTvpTPosZsLIWKjqRQ', u'expires_in': 7200, u'errcode': 0, u'errmsg': u'ok'}
{u'errcode': 60020, u'errmsg': u'not allow to access from your ip, hint: [1662508718500612651322560], from ip: 202.96.209.5, more info at https://open.work.weixin.qq.com/devtool/query?e=60020'}
{u'access_token': u'oJrDTOHB-zkbalxAG76yd5mWEq6qLvRvB1haOtHuI7ZD1lvI5qYtahku83Pw1W_WhF12FnpTbxKrSY1s3BZ7c2a23fdiOTuNmA6XUFz0LNRbjXwto6TG73NboPZuZPiUgAMjPnM6FNx3dmbEzQhZBhhlf3qb_RsgjCzKCu1DI4L8YTCUlwV97_idrKr3h9zYY7eJQoRTvpTPosZsLIWKjqRQ', u'expires_in': 7200, u'errcode': 0, u'errmsg': u'ok'}
{u'errcode': 60020, u'errmsg': u'not allow to access from your ip, hint: [1662508718612131773200954], from ip: 202.96.209.5, more info at https://open.work.weixin.qq.com/devtool/query?e=60020'}
{u'access_token': u'oJrDTOHB-zkbalxAG76yd5mWEq6qLvRvB1haOtHuI7ZD1lvI5qYtahku83Pw1W_WhF12FnpTbxKrSY1s3BZ7c2a23fdiOTuNmA6XUFz0LNRbjXwto6TG73NboPZuZPiUgAMjPnM6FNx3dmbEzQhZBhhlf3qb_RsgjCzKCu1DI4L8YTCUlwV97_idrKr3h9zYY7eJQoRTvpTPosZsLIWKjqRQ', u'expires_in': 7200, u'errcode': 0, u'errmsg': u'ok'}
{u'errcode': 60020, u'errmsg': u'not allow to access from your ip, hint: [1662508719480131924910365], from ip: 202.96.209.5, more info at https://open.work.weixin.qq.com/devtool/query?e=60020'}
{u'errcode': 60020, u'errmsg': u'not allow to access from your ip, hint: [1662508719480131924910365], from ip: 202.96.209.5, more info at https://open.work.weixin.qq.com/devtool/query?e=60020'}
可以看有errcode:60020
好了.找到错误代码了,后面就简单了
3. 解决问题
原因就是发送消息的IP不在白名单内.
登录企业微信后台,在应用管理中打开机器人,在下面找到企业可惜IP,添加公网IP.
公网IP其实在上面报错中已经给了,也可以使用下面命令查询.
# curl ifconfig.me
202.96.209.5
此时已经在企业微信客户端收到了告警信息
至此故障排除