feat: 异步任务队列架构 - SQLite队列 + 速率限制 + NAS Poller
Edge端: - 新增 SQLite 异步任务队列 (queue_manager + consumer) - 新增 TokenBucket 速率限制器 (Gemini 1000 RPM, NVIDIA 40 RPM, burst 2x) - 新增 /api/edge/video/enqueue + /api/edge/results 端点 - 消费者线程从队列消费任务,按速率限制调用AI模型 - orchestrator 集成 rate_limiter,Gemini优先→NVIDIA兜底 NAS端: - Dispatcher 重构为 enqueue 模式(上传后立即返回,不等结果) - 新增 Poller 线程(定期从Edge拉取结果写 MariaDB) - app.py 启动 Poller,config.yaml 新增 poller 配置 - db_layer 更新 valid_stages 添加 'process'
This commit is contained in:
@@ -1,8 +1,12 @@
|
||||
"""
|
||||
API-Gateway - Flask 蓝图,接收任务
|
||||
|
||||
同时只允许 1 个任务在处理;新任务到达时若当前有任务处理中,返回 429
|
||||
模式:
|
||||
1. enqueue (异步): NAS 上传视频 → Edge 入队 → 立即返回 → 消费者异步处理 → NAS 轮询拉取结果
|
||||
2. push (同步, 兼容保留): NAS 上传 → Edge 同步处理 → 结果随响应返回
|
||||
3. analyze (旧拉取模式, 兼容保留)
|
||||
"""
|
||||
import os
|
||||
import threading
|
||||
import requests
|
||||
from flask import Blueprint, request, jsonify
|
||||
@@ -10,12 +14,12 @@ from flask import Blueprint, request, jsonify
|
||||
from ..logger import setup_logger
|
||||
from ..ai_orchestrator.orchestrator import AIOrchestrator
|
||||
from ..video_preprocessor.preprocessor import VideoPreprocessor
|
||||
from ..queue import queue_manager
|
||||
|
||||
logger = setup_logger('fam-edge.api_gateway')
|
||||
|
||||
api_bp = Blueprint('api_gateway', __name__)
|
||||
|
||||
# 并发控制:同时只允许 1 个任务
|
||||
_current_task_lock = threading.Lock()
|
||||
_currently_processing = False
|
||||
|
||||
@@ -72,6 +76,92 @@ def receive_task():
|
||||
return jsonify({"status": "accepted", "task_id": task_id}), 202
|
||||
|
||||
|
||||
@api_bp.route('/api/edge/video/enqueue', methods=['POST'])
|
||||
def enqueue_task():
|
||||
"""异步模式:接收 multipart 视频上传,入队后立即返回
|
||||
|
||||
NAS 上传视频 → Edge 保存到磁盘 + 入 SQLite 队列 → 返回 task_id
|
||||
消费者线程异步处理,NAS 通过 /api/edge/results 拉取结果
|
||||
"""
|
||||
task_id_raw = request.form.get('task_id')
|
||||
file = request.files.get('video')
|
||||
if not task_id_raw or not file:
|
||||
return jsonify({"error": "缺少必填字段: task_id, video"}), 400
|
||||
|
||||
try:
|
||||
task_id = int(task_id_raw)
|
||||
except ValueError:
|
||||
return jsonify({"error": "task_id 必须是整数"}), 400
|
||||
|
||||
camera_name = request.form.get('camera_name', '')
|
||||
event_start_time = request.form.get('event_start_time', '')
|
||||
known_members_context = request.form.get('known_members_context', '')
|
||||
|
||||
upload_dir = os.environ.get('FAM_UPLOAD_DIR', '/tmp/fam_uploads')
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
video_filename = f"task_{task_id}_{file.filename}"
|
||||
video_path = os.path.join(upload_dir, video_filename)
|
||||
|
||||
try:
|
||||
file.save(video_path)
|
||||
size_mb = os.path.getsize(video_path) / 1024 / 1024
|
||||
logger.info(f"[task_id={task_id}] 入队: {file.filename} ({size_mb:.1f}MB)")
|
||||
|
||||
queue_id = queue_manager.enqueue(
|
||||
nas_task_id=task_id,
|
||||
video_filename=file.filename,
|
||||
video_path=video_path,
|
||||
camera_name=camera_name,
|
||||
event_start_time=event_start_time,
|
||||
known_members_context=known_members_context,
|
||||
)
|
||||
|
||||
return jsonify({
|
||||
"status": "queued",
|
||||
"task_id": task_id,
|
||||
"queue_id": queue_id,
|
||||
}), 202
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[task_id={task_id}] 入队失败: {e}", exc_info=True)
|
||||
if os.path.exists(video_path):
|
||||
os.remove(video_path)
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@api_bp.route('/api/edge/results', methods=['GET'])
|
||||
def get_results():
|
||||
"""返回已完成但未拉取的结果,标记为已交付"""
|
||||
limit = int(request.args.get('limit', 10))
|
||||
results = queue_manager.get_undelivered_results(limit=limit)
|
||||
|
||||
import json
|
||||
payload = []
|
||||
task_ids = []
|
||||
for r in results:
|
||||
try:
|
||||
result_json = json.loads(r['result_json']) if r['result_json'] else None
|
||||
except json.JSONDecodeError:
|
||||
result_json = None
|
||||
payload.append({
|
||||
"nas_task_id": r['nas_task_id'],
|
||||
"result": result_json,
|
||||
})
|
||||
task_ids.append(r['id'])
|
||||
|
||||
if task_ids:
|
||||
queue_manager.mark_delivered(task_ids)
|
||||
|
||||
return jsonify({"results": payload, "count": len(payload)}), 200
|
||||
|
||||
|
||||
@api_bp.route('/api/edge/queue/stats', methods=['GET'])
|
||||
def queue_stats():
|
||||
"""队列状态统计"""
|
||||
stats = queue_manager.get_queue_stats()
|
||||
return jsonify(stats), 200
|
||||
|
||||
|
||||
@api_bp.route('/api/edge/video/push', methods=['POST'])
|
||||
def receive_push_task():
|
||||
"""推送模式:接收 multipart 视频上传,同步分析,结果随 HTTP 响应返回
|
||||
|
||||
Reference in New Issue
Block a user