ericwyuan/sentinel-home-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 异步任务队列架构 - SQLite队列 + 速率限制 + NAS Poller

Browse Source 
Edge端:
- 新增 SQLite 异步任务队列 (queue_manager + consumer)
- 新增 TokenBucket 速率限制器 (Gemini 1000 RPM, NVIDIA 40 RPM, burst 2x)
- 新增 /api/edge/video/enqueue + /api/edge/results 端点
- 消费者线程从队列消费任务，按速率限制调用AI模型
- orchestrator 集成 rate_limiter，Gemini优先→NVIDIA兜底

NAS端:
- Dispatcher 重构为 enqueue 模式（上传后立即返回，不等结果）
- 新增 Poller 线程（定期从Edge拉取结果写 MariaDB）
- app.py 启动 Poller，config.yaml 新增 poller 配置
- db_layer 更新 valid_stages 添加 'process'
This commit is contained in:
ericwyuan
2026-08-20 12:07:09 +08:00
parent a4b9178a59
commit 02da23ef42
14 changed files with 686 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fam-core/src/fam_core/app.py
View File

@@ -1,7 +1,7 @@
"""
FAM-Core 主应用 - Flask 单进程
承载: Task-Scheduler / Dispatcher / Event-Receiver / Chat-Handler / Member-Manager / Video-Server
承载: Task-Scheduler / Dispatcher / Poller / Event-Receiver / Chat-Handler / Member-Manager / Video-Server
"""
import os
import sys
@@ -14,6 +14,7 @@ from .config_loader import load_config
from .logger import setup_logger
from .scheduler.scheduler import TaskScheduler
from .dispatcher.dispatcher import Dispatcher
from .poller.poller import Poller
from .event_receiver.event_receiver import event_bp
from .chat_handler.chat_handler import chat_bp
from .member_manager.member_manager import member_bp
@@ -37,6 +38,7 @@ def health():
# 初始化后台线程
_scheduler = None
_dispatcher = None
_poller = None
try:
_scheduler = TaskScheduler()
@@ -52,6 +54,13 @@ try:
except Exception as e:
logger.error(f"Dispatcher 启动失败: {e}")
try:
_poller = Poller()
_poller.start()
logger.info("Poller 已启动")
except Exception as e:
logger.error(f"Poller 启动失败: {e}")
@app.route('/api/status', methods=['GET'])
def status():
@@ -59,6 +68,7 @@ def status():
return jsonify({
"scheduler_running": _scheduler._running if _scheduler else False,
"dispatcher_running": _dispatcher._running if _dispatcher else False,
"poller_running": _poller._running if _poller else False,
}), 200

2
fam-core/src/fam_core/db_layer.py
View File

@@ -92,7 +92,7 @@ def get_tasks_by_status(status: str, limit=10) -> List[Dict]:
def update_task_status(task_id: int, status: str, error_message: str = None,
failure_stage: str = None):
"""更新任务状态"""
valid_stages = {'download', 'extract', 'vlm_visual', 'vlm_fusion', 'callback'}
valid_stages = {'download', 'extract', 'vlm_visual', 'vlm_fusion', 'callback', 'process'}
if failure_stage and failure_stage not in valid_stages:
failure_stage = 'callback'
conn = get_conn()

78
fam-core/src/fam_core/dispatcher/dispatcher.py
View File

@@ -1,11 +1,12 @@
"""
Dispatcher - 30s 轮询 PENDING 任务，推送视频至 Edge（推送模式）
Dispatcher - 30s 轮询 PENDING 任务，上传视频至 Edge 异步队列
流程（纯单向通信，NAS → Oracle，无需 Oracle 反向访问 NAS）:
流程（异步队列模式）:
1. 读取任务对应的本地视频文件
2. multipart 上传至 Edge /api/edge/video/push（附已知成员清单等元数据）
3. Edge 同步抽帧+分析，结果直接随 HTTP 响应返回
4. Dispatcher 收到响应后直接写 monitor_events/event_details，任务标记 SUCCESS
2. multipart 上传至 Edge /api/edge/video/enqueue（附已知成员清单等元数据）
3. Edge 保存视频 + 入 SQLite 队列，立即返回 202
4. Dispatcher 标记任务为 PROCESSING（已派发，等待 Poller 拉取结果）
5. Poller 线程定期从 Edge /api/edge/results 拉取结果，写库后标记 SUCCESS
退避重试: min(60 * (retry_count + 1) * 2, 600) 秒
"""
@@ -18,22 +19,23 @@ from datetime import datetime, timedelta
from ..logger import setup_logger, log_task
from ..config_loader import load_config
from .. import db_layer
from ..event_receiver.event_receiver import apply_success_event
logger = setup_logger('fam-core.dispatcher')
class Dispatcher:
"""任务下发器，30s 轮询（推送模式）"""
"""任务下发器，30s 轮询（异步队列模式）"""
def __init__(self):
cfg = load_config()
self.poll_interval = cfg.get('dispatcher', {}).get('poll_interval', 30)
self.edge_url = cfg.get('dispatcher', {}).get('edge_url', 'http://localhost:5000/api/edge/video/push')
self.edge_url = cfg.get('dispatcher', {}).get('edge_url', 'http://localhost:5000/api/edge/video/enqueue')
self.max_retries = cfg.get('dispatcher', {}).get('max_retries', 3)
# 推送+分析全程同步，耗时较长（大视频上传 + ARM 多帧分析）
self.push_timeout = cfg.get('dispatcher', {}).get('push_timeout', 1800)
# enqueue 模式只需上传时间，不含 AI 处理时间
self.upload_timeout = cfg.get('dispatcher', {}).get('upload_timeout', 300)
self.camera_name = cfg.get('scheduler', {}).get('camera_name', '默认摄像头')
# PROCESSING 超时回收：Edge 处理 + 队列等待可能很长
self.stale_timeout = cfg.get('dispatcher', {}).get('stale_timeout', 3600)
self._running = False
self._thread = None
@@ -61,7 +63,6 @@ class Dispatcher:
"event_start_time": "",
"event_end_time": "",
}
# 用文件 mtime 近似事件开始时间
try:
mtime = os.path.getmtime(video_path)
start_dt = datetime.fromtimestamp(mtime)
@@ -71,11 +72,10 @@ class Dispatcher:
return payload
def _dispatch_one(self, task):
"""推送单个任务：上传视频 → 同步等结果 → 直接写库"""
"""上传视频至 Edge 异步队列，立即返回"""
task_id = task['task_id']
video_path = task['video_path']
# 本地视频必须存在，否则直接失败（重试无意义）
if not video_path or not os.path.isfile(video_path):
db_layer.update_task_status(
task_id, 'FAILED',
@@ -89,7 +89,7 @@ class Dispatcher:
size_mb = os.path.getsize(video_path) / (1024 * 1024)
db_layer.update_task_status(task_id, 'PROCESSING')
log_task(logger, task_id, 'dispatcher',
f'推送视频至 Edge: {self.edge_url} ({size_mb:.1f}MB)')
f'上传视频至 Edge 队列: {self.edge_url} ({size_mb:.1f}MB)')
try:
with open(video_path, 'rb') as fh:
@@ -97,41 +97,29 @@ class Dispatcher:
self.edge_url,
data=payload,
files={'video': (os.path.basename(video_path), fh, 'video/mp4')},
timeout=self.push_timeout
timeout=self.upload_timeout
)
except requests.RequestException as e:
logger.error(f"[task_id={task_id}] 推送失败: {e}")
logger.error(f"[task_id={task_id}] 上传失败: {e}")
self._schedule_retry(task)
return
if resp.status_code == 202:
try:
data = resp.json()
queue_id = data.get('queue_id', '?')
logger.info(f"[task_id={task_id}] 已入 Edge 队列 (queue_id={queue_id})，等待 Poller 拉取结果")
except ValueError:
logger.info(f"[task_id={task_id}] 已入 Edge 队列，等待 Poller 拉取结果")
return
if resp.status_code == 429:
logger.warning(f"[task_id={task_id}] Edge 忙 (429)，回到 PENDING 稍后重试")
logger.warning(f"[task_id={task_id}] Edge 队列满 (429)，回到 PENDING 稍后重试")
db_layer.update_task_status(task_id, 'PENDING')
return
if resp.status_code != 200:
logger.error(f"[task_id={task_id}] Edge 返回异常状态码: {resp.status_code}")
self._schedule_retry(task)
return
try:
result = resp.json()
except ValueError:
result = {}
if result.get('status') == 'success':
try:
event_id = apply_success_event(task_id, result)
log_task(logger, task_id, 'dispatcher', f'推送任务完成: event_id={event_id}')
except Exception as e:
logger.error(f"[task_id={task_id}] 结果落库失败: {e}", exc_info=True)
db_layer.update_task_status(
task_id, 'FAILED', error_message=str(e), failure_stage='callback')
else:
error_message = result.get('error_message', 'unknown')
failure_stage = result.get('failure_stage', 'edge')
logger.error(f"[task_id={task_id}] Edge 分析失败: stage={failure_stage}, error={error_message}")
db_layer.update_task_status(
task_id, 'FAILED', error_message=error_message, failure_stage=failure_stage)
logger.error(f"[task_id={task_id}] Edge 返回异常状态码: {resp.status_code}")
self._schedule_retry(task)
def _schedule_retry(self, task):
"""调度重试"""
@@ -152,13 +140,11 @@ class Dispatcher:
def _poll_once(self):
"""执行一次轮询"""
# 先回收僵尸任务：PROCESSING 超过 push_timeout+缓冲 说明推送进程已丢失
# （如 fam-core 重启、Edge 重启掐断 in-flight 连接），重置回 PENDING 走重试
stale_timeout = self.push_timeout + 120
# 回收僵尸任务：PROCESSING 超过 stale_timeout 说明 Edge 丢失了任务
try:
stale_ids = db_layer.reclaim_stale_processing(stale_timeout)
stale_ids = db_layer.reclaim_stale_processing(self.stale_timeout)
for tid in stale_ids:
logger.warning(f"[task_id={tid}] PROCESSING 超时 {stale_timeout}s，回收为 PENDING 重试")
logger.warning(f"[task_id={tid}] PROCESSING 超时 {self.stale_timeout}s，回收为 PENDING 重试")
except Exception as e:
logger.error(f"僵尸任务回收失败: {e}", exc_info=True)
@@ -172,7 +158,7 @@ class Dispatcher:
def _run(self):
"""线程主循环"""
logger.info(f"Dispatcher 启动，轮询间隔 {self.poll_interval}s")
logger.info(f"Dispatcher 启动 (enqueue 模式)，轮询间隔 {self.poll_interval}s")
while self._running:
try:
self._poll_once()

0
fam-core/src/fam_core/poller/__init__.py Normal file
View File

128
fam-core/src/fam_core/poller/poller.py Normal file
View File

@@ -0,0 +1,128 @@
"""
Poller - 定期从 Edge 拉取已完成的任务结果，写入 MariaDB
流程:
1. 每 N 秒请求 Edge /api/edge/results?limit=10
2. 遍历结果列表，对每个 nas_task_id:
- success: 调用 apply_success_event 写入 monitor_events + event_details，标记 SUCCESS
- failed: 更新任务状态为 FAILED，记录 failure_stage 和 error_message
3. Edge 端自动标记已拉取的结果为 delivered
"""
import time
import threading
import requests
from ..logger import setup_logger, log_task
from ..config_loader import load_config
from .. import db_layer
from ..event_receiver.event_receiver import apply_success_event
logger = setup_logger('fam-core.poller')
class Poller:
"""结果拉取器，定期从 Edge 拉取处理结果"""
def __init__(self):
cfg = load_config()
poller_cfg = cfg.get('poller', {})
self.poll_interval = poller_cfg.get('poll_interval', 30)
self.results_url = poller_cfg.get('results_url', 'http://localhost:5000/api/edge/results')
self.batch_size = poller_cfg.get('batch_size', 10)
self.timeout = poller_cfg.get('timeout', 30)
self._running = False
self._thread = None
def _handle_result(self, item: dict):
"""处理单个结果"""
nas_task_id = item.get('nas_task_id')
result = item.get('result')
if not nas_task_id:
logger.warning(f"结果缺少 nas_task_id，跳过: {item}")
return
if result is None:
logger.error(f"[task_id={nas_task_id}] Edge 返回空结果，标记 FAILED")
db_layer.update_task_status(
nas_task_id, 'FAILED',
error_message='Edge returned empty result',
failure_stage='callback'
)
return
status = result.get('status')
if status == 'success':
try:
event_id = apply_success_event(nas_task_id, result)
log_task(logger, nas_task_id, 'poller', f'结果落库成功: event_id={event_id}')
except Exception as e:
logger.error(f"[task_id={nas_task_id}] 结果落库失败: {e}", exc_info=True)
db_layer.update_task_status(
nas_task_id, 'FAILED', error_message=str(e), failure_stage='callback')
elif status == 'failed':
error_message = result.get('error_message', 'unknown')
failure_stage = result.get('failure_stage', '')
logger.error(f"[task_id={nas_task_id}] Edge 处理失败: stage={failure_stage}, error={error_message}")
db_layer.update_task_status(
nas_task_id, 'FAILED', error_message=error_message, failure_stage=failure_stage)
else:
logger.warning(f"[task_id={nas_task_id}] 未知状态: {status}")
def _poll_once(self):
"""执行一次拉取"""
try:
resp = requests.get(
self.results_url,
params={'limit': self.batch_size},
timeout=self.timeout
)
except requests.RequestException as e:
logger.error(f"拉取结果失败: {e}")
return
if resp.status_code != 200:
logger.warning(f"Edge 返回 {resp.status_code}")
return
try:
data = resp.json()
except ValueError:
logger.error("Edge 返回非 JSON 响应")
return
results = data.get('results', [])
if not results:
return
logger.info(f"拉取到 {len(results)} 条结果")
for item in results:
try:
self._handle_result(item)
except Exception as e:
task_id = item.get('nas_task_id', '?')
logger.error(f"[task_id={task_id}] 处理结果异常: {e}", exc_info=True)
def _run(self):
"""线程主循环"""
logger.info(f"Poller 启动，轮询间隔 {self.poll_interval}s，目标: {self.results_url}")
while self._running:
try:
self._poll_once()
except Exception as e:
logger.error(f"轮询异常: {e}", exc_info=True)
time.sleep(self.poll_interval)
def start(self):
"""启动拉取线程"""
if self._running:
return
self._running = True
self._thread = threading.Thread(target=self._run, daemon=True, name='poller')
self._thread.start()
def stop(self):
"""停止拉取线程"""
self._running = False
if self._thread:
self._thread.join(timeout=5)