fix: dispatcher 串行上传 + 看门狗自动重启 + 线程状态真实检测

- Dispatcher limit=10→1: 一次只传一个视频,传完再传下一个
- 退避缩短: min(30*(n+1),300)s → 失败后更快重试(原 min(60*(n+1)*2,600)s)
- /api/status: 用 thread.is_alive() 替代 _running 布尔标志
- 三组件(scheduler/dispatcher/poller)添加 is_alive()+check_and_restart()
- app.py 新增 watchdog 线程: 每 60s 检测线程死亡并自动重启
This commit is contained in:
ericwyuan
2026-08-20 13:37:07 +08:00
parent b6c13a9047
commit 881ea3f470
4 changed files with 61 additions and 8 deletions

View File

@@ -9,8 +9,9 @@ Dispatcher - 30s 轮询 PENDING 任务,上传视频至 Edge 异步队列
4. Dispatcher 标记任务为 PROCESSING已派发等待 Poller 拉取结果)
5. Poller 线程定期从 Edge /api/edge/results 拉取结果,写库后标记 SUCCESS
退避重试: min(60 * (retry_count + 1) * 2, 600) 秒
退避重试: min(30 * (retry_count + 1), 300) 秒
分块级重试: 每块最多重试 3 次
看门狗: 线程崩溃后自动重启
"""
import os
import io
@@ -51,8 +52,8 @@ class Dispatcher:
self._thread = None
def _calculate_backoff(self, retry_count):
"""退避策略: min(60 * (retry_count + 1) * 2, 600)"""
return min(60 * (retry_count + 1) * 2, 600)
"""退避策略: min(30 * (retry_count + 1), 300)"""
return min(30 * (retry_count + 1), 300)
def _should_retry(self, task):
"""检查任务是否可以重试"""
@@ -275,7 +276,7 @@ class Dispatcher:
except Exception as e:
logger.error(f"僵尸任务回收失败: {e}", exc_info=True)
tasks = db_layer.get_pending_tasks(limit=10)
tasks = db_layer.get_pending_tasks(limit=1)
for task in tasks:
if self._should_retry(task):
try:
@@ -301,6 +302,17 @@ class Dispatcher:
self._thread = threading.Thread(target=self._run, daemon=True, name='dispatcher')
self._thread.start()
def is_alive(self):
"""线程是否存活"""
return self._thread is not None and self._thread.is_alive()
def check_and_restart(self):
"""看门狗:线程崩溃后自动重启"""
if self._running and not self.is_alive():
logger.warning("Dispatcher 线程已死亡,正在重启...")
self._thread = threading.Thread(target=self._run, daemon=True, name='dispatcher')
self._thread.start()
def stop(self):
"""停止下发线程"""
self._running = False