feat: 异步任务队列架构 - SQLite队列 + 速率限制 + NAS Poller

Edge端:
- 新增 SQLite 异步任务队列 (queue_manager + consumer)
- 新增 TokenBucket 速率限制器 (Gemini 1000 RPM, NVIDIA 40 RPM, burst 2x)
- 新增 /api/edge/video/enqueue + /api/edge/results 端点
- 消费者线程从队列消费任务,按速率限制调用AI模型
- orchestrator 集成 rate_limiter,Gemini优先→NVIDIA兜底

NAS端:
- Dispatcher 重构为 enqueue 模式(上传后立即返回,不等结果)
- 新增 Poller 线程(定期从Edge拉取结果写 MariaDB)
- app.py 启动 Poller,config.yaml 新增 poller 配置
- db_layer 更新 valid_stages 添加 'process'
This commit is contained in:
ericwyuan
2026-08-20 12:07:09 +08:00
parent a4b9178a59
commit 02da23ef42
14 changed files with 686 additions and 62 deletions

View File

@@ -1,8 +1,12 @@
"""
API-Gateway - Flask 蓝图,接收任务
同时只允许 1 个任务在处理;新任务到达时若当前有任务处理中,返回 429
模式:
1. enqueue (异步): NAS 上传视频 → Edge 入队 → 立即返回 → 消费者异步处理 → NAS 轮询拉取结果
2. push (同步, 兼容保留): NAS 上传 → Edge 同步处理 → 结果随响应返回
3. analyze (旧拉取模式, 兼容保留)
"""
import os
import threading
import requests
from flask import Blueprint, request, jsonify
@@ -10,12 +14,12 @@ from flask import Blueprint, request, jsonify
from ..logger import setup_logger
from ..ai_orchestrator.orchestrator import AIOrchestrator
from ..video_preprocessor.preprocessor import VideoPreprocessor
from ..queue import queue_manager
logger = setup_logger('fam-edge.api_gateway')
api_bp = Blueprint('api_gateway', __name__)
# 并发控制:同时只允许 1 个任务
_current_task_lock = threading.Lock()
_currently_processing = False
@@ -72,6 +76,92 @@ def receive_task():
return jsonify({"status": "accepted", "task_id": task_id}), 202
@api_bp.route('/api/edge/video/enqueue', methods=['POST'])
def enqueue_task():
"""异步模式:接收 multipart 视频上传,入队后立即返回
NAS 上传视频 → Edge 保存到磁盘 + 入 SQLite 队列 → 返回 task_id
消费者线程异步处理NAS 通过 /api/edge/results 拉取结果
"""
task_id_raw = request.form.get('task_id')
file = request.files.get('video')
if not task_id_raw or not file:
return jsonify({"error": "缺少必填字段: task_id, video"}), 400
try:
task_id = int(task_id_raw)
except ValueError:
return jsonify({"error": "task_id 必须是整数"}), 400
camera_name = request.form.get('camera_name', '')
event_start_time = request.form.get('event_start_time', '')
known_members_context = request.form.get('known_members_context', '')
upload_dir = os.environ.get('FAM_UPLOAD_DIR', '/tmp/fam_uploads')
os.makedirs(upload_dir, exist_ok=True)
video_filename = f"task_{task_id}_{file.filename}"
video_path = os.path.join(upload_dir, video_filename)
try:
file.save(video_path)
size_mb = os.path.getsize(video_path) / 1024 / 1024
logger.info(f"[task_id={task_id}] 入队: {file.filename} ({size_mb:.1f}MB)")
queue_id = queue_manager.enqueue(
nas_task_id=task_id,
video_filename=file.filename,
video_path=video_path,
camera_name=camera_name,
event_start_time=event_start_time,
known_members_context=known_members_context,
)
return jsonify({
"status": "queued",
"task_id": task_id,
"queue_id": queue_id,
}), 202
except Exception as e:
logger.error(f"[task_id={task_id}] 入队失败: {e}", exc_info=True)
if os.path.exists(video_path):
os.remove(video_path)
return jsonify({"error": str(e)}), 500
@api_bp.route('/api/edge/results', methods=['GET'])
def get_results():
"""返回已完成但未拉取的结果,标记为已交付"""
limit = int(request.args.get('limit', 10))
results = queue_manager.get_undelivered_results(limit=limit)
import json
payload = []
task_ids = []
for r in results:
try:
result_json = json.loads(r['result_json']) if r['result_json'] else None
except json.JSONDecodeError:
result_json = None
payload.append({
"nas_task_id": r['nas_task_id'],
"result": result_json,
})
task_ids.append(r['id'])
if task_ids:
queue_manager.mark_delivered(task_ids)
return jsonify({"results": payload, "count": len(payload)}), 200
@api_bp.route('/api/edge/queue/stats', methods=['GET'])
def queue_stats():
"""队列状态统计"""
stats = queue_manager.get_queue_stats()
return jsonify(stats), 200
@api_bp.route('/api/edge/video/push', methods=['POST'])
def receive_push_task():
"""推送模式:接收 multipart 视频上传,同步分析,结果随 HTTP 响应返回