# encoding=utf8
import threading
import time
class AutoKnight(object):
def __init__(self, retry_times=5, retry_delay=60, disabled_delay=600, disable_run_times=20, disable_warning=[],
disable_stages=[], warning_handle=None):
"""
①当满足在retry_delay时间内，连续错误retry_times次，
②则自动停用某个stage，之后停用disable_delay秒或者停用disable_run_times次后，
③再继续单次尝试，一旦访问通过，则恢复，否则继续②步骤
__stages = {stage_key: {'retry_times':1, 'first_retry_time':1020004233, 'is_abled': False,
'disabled_times':0, 'disabled_run_times': 0, 'last_failed_msg':''}}
:param retry_times:策略生效的重试次数 <=0 此策略不生效
:param retry_delay:策略生效的时间约束 <=0 此策略不生效
:param disabled_delay:停用时间 <=0 此策略不生效
:param disable_run_times:停用时访问此stage 次数 <=0 此策略不生效
:param disable_warning:一旦命中此此策略，不会将策略降级，而是直接发送waring(一般用于比较重要的stage)
:param disable_stages:永久停用策略 默认为空
"""
self.__stages = {}
self.__retry_times = 0 if retry_times <= 0 else retry_times
self.__retry_delay = 0 if retry_delay <= 0 else retry_delay
self.__disabled_delay = 0 if disabled_delay <= 0 else disabled_delay
self.__disable_run_times = 0 if disable_run_times <= 0 else disable_run_times
self.__disable_stages = disable_stages
self.__disable_warning = disable_warning
self._LOCK = threading.Lock()
self.__handle_warning = warning_handle or (lambda *args: args)
def __is_disabled(self, key, isok):
if key in self.__disable_warning:
if not isok:
self.__handle_warning(key, self.__stages.get(key, {}).get('last_failed_msg'))
return False
else:
return isok
def get_disable_stages(self):
"""
获取当前被disabled的stage
:return: [key]
"""
result = []
with self._LOCK:
for key, val in self.__stages.items():
if val.get('is_abled', True) is False:
result.append(key)
result += self.__disable_stages
return list(set(result))
def get_details(self, keys):
"""
获取keys策略的详细信息
:param keys: string or list
:return: list
"""
if isinstance(keys, basestring):
keys = [keys]
result = []
with self._LOCK:
for key in keys:
result.append([key, self.__stages.get(key, {})])
return result
def is_disable(self, key, skip=True):
"""
检查某个key是否已经被disable
:param key: key
:param skip: 查询某个key，返回是否disable，False：不进行跳过计数，True：进行跳过计数
:return: True or False
"""
def check_disabled_delay(msg):
# 是否需要重试
if self.__disabled_delay <= 0:
return True
if time.time() - msg['disabled_delay'] <= self.__disabled_delay:
return False
else:
msg['disabled_delay'] = time.time()
msg['disabled_run_times'] = 0
return True
def check_disabled_run_times(msg):
# 是否需要重试
if self.__disable_run_times <= 0:
return True
if msg['disabled_run_times'] + 1 <= self.__disable_run_times:
return False
else:
msg['disabled_run_times'] = 0
msg['disabled_delay'] = time.time()
return True
with self._LOCK:
if key in self.__disable_stages:
return self.__is_disabled(key, True)
msg = self.__stages.get(key, {})
if not msg:
msg = self.factory_default_msg(True)
self.__stages.update({key: msg})
if msg['is_abled']:
return self.__is_disabled(key, False)
if not skip:
return self.__is_disabled(key, True)
return self.__is_disabled(key, not (check_disabled_delay(msg) or check_disabled_run_times(msg)))
def factory_default_msg(self, isok, failed_msg=''):
msg = {'retry_times': 0, 'first_retry_time': 0, 'is_abled': True, 'disabled_delay': 0,
'disabled_run_times': 0, 'last_failed_msg': failed_msg}
if isok:
return msg
else:
return msg.update({'retry_times': 1, 'first_retry_time': time.time()})
def set_stage(self, key, isok, failed_msg=''):
"""
设置一个stage的状态
:param key:改stage唯一区别key
:param isok: 本次调用是否ok
:param failed_msg: 失败原因
:return: is_abled
"""
def check_retry_times(msg):
# 是否需要disabled
if self.__retry_times <= 0:
return False
if msg['retry_times'] + 1 >= self.__retry_times:
return True
else:
return False
def check_first_retry_time(msg):
# 是否需要disabled
if self.__retry_delay <= 0:
return False
if time.time() - msg['first_retry_time'] <= self.__retry_delay:
return True
else:
return False
with self._LOCK:
# 如果在被强制命中的list中，直接返回True
if key in self.__disable_stages:
return False
msg = self.__stages.get(key, {})
if not msg:
# 第一次进行存储
self.__stages.update({key: self.factory_default_msg(isok, failed_msg)})
else:
# 不是第一次
is_abled = msg['is_abled']
if isok and is_abled:
# 重置retry_times，first_retry_time
msg['retry_times'] = 0
msg['first_retry_time'] = 0
elif isok and not is_abled:
msg.update({'retry_times': 0, 'first_retry_time': 0, 'is_abled': True, 'disabled_delay': 0,
'disabled_run_times': 0})
elif not isok and is_abled:
if check_retry_times(msg):
if check_first_retry_time(msg):
msg['is_abled'] = False
msg['retry_times'] = 0
msg['first_retry_time'] = 0
msg['disabled_delay'] = time.time()
msg['disabled_run_times'] = 0
else:
msg['retry_times'] = 0
msg['first_retry_time'] = time.time()
else:
if msg['first_retry_time'] <= 0:
msg['first_retry_time'] = time.time()
msg['retry_times'] += 1
else:
# not isok and not is_abled reset disabled_delay, disabled_run_times
msg['disabled_delay'] = time.time()
msg['disabled_run_times'] = 0
return not self.__is_disabled(key, msg['is_abled'])
if __name__ == '__main__':
import random
def test(auto):
stages = ['a', 'b', 'c', 'd']
while True:
for key in stages:
isok = False
if random.random() < 0.5:
isok = True
if not auto.is_disable(key):
print('key = %s, isok=%s' % (key, isok))
auto.set_stage(key, isok)
else:
print('key is skiped %s' % key)
time.sleep(2)
auto = AutoKnight(retry_times=2, disabled_delay=10, disable_warning=['a', 'b', 'c', 'd'])
# test(auto)
for i in range(2):
threading.Thread(target=test, args=[auto]).start()
