feat: 异步任务队列架构 - SQLite队列 + 速率限制 + NAS Poller
Edge端: - 新增 SQLite 异步任务队列 (queue_manager + consumer) - 新增 TokenBucket 速率限制器 (Gemini 1000 RPM, NVIDIA 40 RPM, burst 2x) - 新增 /api/edge/video/enqueue + /api/edge/results 端点 - 消费者线程从队列消费任务,按速率限制调用AI模型 - orchestrator 集成 rate_limiter,Gemini优先→NVIDIA兜底 NAS端: - Dispatcher 重构为 enqueue 模式(上传后立即返回,不等结果) - 新增 Poller 线程(定期从Edge拉取结果写 MariaDB) - app.py 启动 Poller,config.yaml 新增 poller 配置 - db_layer 更新 valid_stages 添加 'process'
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
"""
|
||||
Dispatcher - 30s 轮询 PENDING 任务,推送视频至 Edge(推送模式)
|
||||
Dispatcher - 30s 轮询 PENDING 任务,上传视频至 Edge 异步队列
|
||||
|
||||
流程(纯单向通信,NAS → Oracle,无需 Oracle 反向访问 NAS):
|
||||
流程(异步队列模式):
|
||||
1. 读取任务对应的本地视频文件
|
||||
2. multipart 上传至 Edge /api/edge/video/push(附已知成员清单等元数据)
|
||||
3. Edge 同步抽帧+分析,结果直接随 HTTP 响应返回
|
||||
4. Dispatcher 收到响应后直接写 monitor_events/event_details,任务标记 SUCCESS
|
||||
2. multipart 上传至 Edge /api/edge/video/enqueue(附已知成员清单等元数据)
|
||||
3. Edge 保存视频 + 入 SQLite 队列,立即返回 202
|
||||
4. Dispatcher 标记任务为 PROCESSING(已派发,等待 Poller 拉取结果)
|
||||
5. Poller 线程定期从 Edge /api/edge/results 拉取结果,写库后标记 SUCCESS
|
||||
|
||||
退避重试: min(60 * (retry_count + 1) * 2, 600) 秒
|
||||
"""
|
||||
@@ -18,22 +19,23 @@ from datetime import datetime, timedelta
|
||||
from ..logger import setup_logger, log_task
|
||||
from ..config_loader import load_config
|
||||
from .. import db_layer
|
||||
from ..event_receiver.event_receiver import apply_success_event
|
||||
|
||||
logger = setup_logger('fam-core.dispatcher')
|
||||
|
||||
|
||||
class Dispatcher:
|
||||
"""任务下发器,30s 轮询(推送模式)"""
|
||||
"""任务下发器,30s 轮询(异步队列模式)"""
|
||||
|
||||
def __init__(self):
|
||||
cfg = load_config()
|
||||
self.poll_interval = cfg.get('dispatcher', {}).get('poll_interval', 30)
|
||||
self.edge_url = cfg.get('dispatcher', {}).get('edge_url', 'http://localhost:5000/api/edge/video/push')
|
||||
self.edge_url = cfg.get('dispatcher', {}).get('edge_url', 'http://localhost:5000/api/edge/video/enqueue')
|
||||
self.max_retries = cfg.get('dispatcher', {}).get('max_retries', 3)
|
||||
# 推送+分析全程同步,耗时较长(大视频上传 + ARM 多帧分析)
|
||||
self.push_timeout = cfg.get('dispatcher', {}).get('push_timeout', 1800)
|
||||
# enqueue 模式只需上传时间,不含 AI 处理时间
|
||||
self.upload_timeout = cfg.get('dispatcher', {}).get('upload_timeout', 300)
|
||||
self.camera_name = cfg.get('scheduler', {}).get('camera_name', '默认摄像头')
|
||||
# PROCESSING 超时回收:Edge 处理 + 队列等待可能很长
|
||||
self.stale_timeout = cfg.get('dispatcher', {}).get('stale_timeout', 3600)
|
||||
self._running = False
|
||||
self._thread = None
|
||||
|
||||
@@ -61,7 +63,6 @@ class Dispatcher:
|
||||
"event_start_time": "",
|
||||
"event_end_time": "",
|
||||
}
|
||||
# 用文件 mtime 近似事件开始时间
|
||||
try:
|
||||
mtime = os.path.getmtime(video_path)
|
||||
start_dt = datetime.fromtimestamp(mtime)
|
||||
@@ -71,11 +72,10 @@ class Dispatcher:
|
||||
return payload
|
||||
|
||||
def _dispatch_one(self, task):
|
||||
"""推送单个任务:上传视频 → 同步等结果 → 直接写库"""
|
||||
"""上传视频至 Edge 异步队列,立即返回"""
|
||||
task_id = task['task_id']
|
||||
video_path = task['video_path']
|
||||
|
||||
# 本地视频必须存在,否则直接失败(重试无意义)
|
||||
if not video_path or not os.path.isfile(video_path):
|
||||
db_layer.update_task_status(
|
||||
task_id, 'FAILED',
|
||||
@@ -89,7 +89,7 @@ class Dispatcher:
|
||||
size_mb = os.path.getsize(video_path) / (1024 * 1024)
|
||||
db_layer.update_task_status(task_id, 'PROCESSING')
|
||||
log_task(logger, task_id, 'dispatcher',
|
||||
f'推送视频至 Edge: {self.edge_url} ({size_mb:.1f}MB)')
|
||||
f'上传视频至 Edge 队列: {self.edge_url} ({size_mb:.1f}MB)')
|
||||
|
||||
try:
|
||||
with open(video_path, 'rb') as fh:
|
||||
@@ -97,41 +97,29 @@ class Dispatcher:
|
||||
self.edge_url,
|
||||
data=payload,
|
||||
files={'video': (os.path.basename(video_path), fh, 'video/mp4')},
|
||||
timeout=self.push_timeout
|
||||
timeout=self.upload_timeout
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"[task_id={task_id}] 推送失败: {e}")
|
||||
logger.error(f"[task_id={task_id}] 上传失败: {e}")
|
||||
self._schedule_retry(task)
|
||||
return
|
||||
|
||||
if resp.status_code == 202:
|
||||
try:
|
||||
data = resp.json()
|
||||
queue_id = data.get('queue_id', '?')
|
||||
logger.info(f"[task_id={task_id}] 已入 Edge 队列 (queue_id={queue_id}),等待 Poller 拉取结果")
|
||||
except ValueError:
|
||||
logger.info(f"[task_id={task_id}] 已入 Edge 队列,等待 Poller 拉取结果")
|
||||
return
|
||||
|
||||
if resp.status_code == 429:
|
||||
logger.warning(f"[task_id={task_id}] Edge 忙 (429),回到 PENDING 稍后重试")
|
||||
logger.warning(f"[task_id={task_id}] Edge 队列满 (429),回到 PENDING 稍后重试")
|
||||
db_layer.update_task_status(task_id, 'PENDING')
|
||||
return
|
||||
|
||||
if resp.status_code != 200:
|
||||
logger.error(f"[task_id={task_id}] Edge 返回异常状态码: {resp.status_code}")
|
||||
self._schedule_retry(task)
|
||||
return
|
||||
|
||||
try:
|
||||
result = resp.json()
|
||||
except ValueError:
|
||||
result = {}
|
||||
if result.get('status') == 'success':
|
||||
try:
|
||||
event_id = apply_success_event(task_id, result)
|
||||
log_task(logger, task_id, 'dispatcher', f'推送任务完成: event_id={event_id}')
|
||||
except Exception as e:
|
||||
logger.error(f"[task_id={task_id}] 结果落库失败: {e}", exc_info=True)
|
||||
db_layer.update_task_status(
|
||||
task_id, 'FAILED', error_message=str(e), failure_stage='callback')
|
||||
else:
|
||||
error_message = result.get('error_message', 'unknown')
|
||||
failure_stage = result.get('failure_stage', 'edge')
|
||||
logger.error(f"[task_id={task_id}] Edge 分析失败: stage={failure_stage}, error={error_message}")
|
||||
db_layer.update_task_status(
|
||||
task_id, 'FAILED', error_message=error_message, failure_stage=failure_stage)
|
||||
logger.error(f"[task_id={task_id}] Edge 返回异常状态码: {resp.status_code}")
|
||||
self._schedule_retry(task)
|
||||
|
||||
def _schedule_retry(self, task):
|
||||
"""调度重试"""
|
||||
@@ -152,13 +140,11 @@ class Dispatcher:
|
||||
|
||||
def _poll_once(self):
|
||||
"""执行一次轮询"""
|
||||
# 先回收僵尸任务:PROCESSING 超过 push_timeout+缓冲 说明推送进程已丢失
|
||||
# (如 fam-core 重启、Edge 重启掐断 in-flight 连接),重置回 PENDING 走重试
|
||||
stale_timeout = self.push_timeout + 120
|
||||
# 回收僵尸任务:PROCESSING 超过 stale_timeout 说明 Edge 丢失了任务
|
||||
try:
|
||||
stale_ids = db_layer.reclaim_stale_processing(stale_timeout)
|
||||
stale_ids = db_layer.reclaim_stale_processing(self.stale_timeout)
|
||||
for tid in stale_ids:
|
||||
logger.warning(f"[task_id={tid}] PROCESSING 超时 {stale_timeout}s,回收为 PENDING 重试")
|
||||
logger.warning(f"[task_id={tid}] PROCESSING 超时 {self.stale_timeout}s,回收为 PENDING 重试")
|
||||
except Exception as e:
|
||||
logger.error(f"僵尸任务回收失败: {e}", exc_info=True)
|
||||
|
||||
@@ -172,7 +158,7 @@ class Dispatcher:
|
||||
|
||||
def _run(self):
|
||||
"""线程主循环"""
|
||||
logger.info(f"Dispatcher 启动,轮询间隔 {self.poll_interval}s")
|
||||
logger.info(f"Dispatcher 启动 (enqueue 模式),轮询间隔 {self.poll_interval}s")
|
||||
while self._running:
|
||||
try:
|
||||
self._poll_once()
|
||||
|
||||
Reference in New Issue
Block a user