Posted on 2020-03-17 15:32
魔のkyo 阅读(1812)
评论(0) 编辑 收藏 引用 所属分类:
Programming 、
Python
监视主线程卡死，发现时进行处理，例如保错再主动崩溃退出，这样再结合监视崩溃自动重启就可以在卡死时实现自动重启。
import threading
import traceback
import time
import sys
import os
from functools import wraps
# from .logger import logger
class WatchDog(threading.Thread):
def __init__(self, timeout=10, echo=False):
super(WatchDog, self).__init__()
self.timeout = timeout
self.echo = echo
self.last_kicked_ts = time.time()
self.lock = threading.Lock()
self.thread_id = threading.currentThread().ident
self.terminated = False
self.setDaemon(True)
self.start()
def terminate(self):
self.terminated = True
self.join(self.timeout)
def kick(self):
self.lock.acquire()
self.last_kicked_ts = time.time()
self.lock.release()
def bark(self):
formated_frame_stack = self._get_formated_frame_stack()
if self.echo:
print("!!!!! WATCH DOG FAILURE TRIGGERED !!!!!\n" + formated_frame_stack)
# logger.fatal("!!!!! WATCH DOG FAILURE TRIGGERED !!!!!\n" + formated_frame_stack)
pid = os.getpid()
os.kill(pid, 2) # 通知进程退出
time.sleep(5) # 等待5秒
os.kill(pid, 9) # 发送强制退出
def run(self):
while not self.terminated:
ts = time.time()
self.lock.acquire()
is_timeout = ts - self.last_kicked_ts > self.timeout
self.lock.release()
if is_timeout:
self.bark()
n = int(max(self.timeout / 3, 1))
for i in range(n*10):
time.sleep(0.1)
if self.terminated:
break
@staticmethod
def _get_thread(tid):
for t in threading.enumerate():
if t.ident == tid:
return t
return None
@staticmethod
def _get_frame_stack(tid):
for thread_id, stack in sys._current_frames().items():
if thread_id == tid:
return stack
return None
def _get_formated_frame_stack(self):
info = []
th = self._get_thread(self.thread_id)
stack = self._get_frame_stack(self.thread_id)
info.append('%s thead_id=%d' % (th.name, self.thread_id))
for filename, lineno, _, line in traceback.extract_stack(stack):
info.append(' at %s(%s:%d)' % (line, filename[filename.rfind(os.path.sep) + 1:], lineno))
return '\n'.join(info)
def watch_dog(timeout=10, echo=False):
def inner(func):
def wrapper(*args, **kw):
dog = WatchDog(timeout=timeout, echo=echo)
ret = func(*args, **kw)
dog.terminate()
return ret
return wrapper
return inner
用法：
dog = WatchDog(echo=True)
while True:
consumer.workex(timeout_ms=100)
dog.kick()