feat: 异步任务队列架构 - SQLite队列 + 速率限制 + NAS Poller
Edge端: - 新增 SQLite 异步任务队列 (queue_manager + consumer) - 新增 TokenBucket 速率限制器 (Gemini 1000 RPM, NVIDIA 40 RPM, burst 2x) - 新增 /api/edge/video/enqueue + /api/edge/results 端点 - 消费者线程从队列消费任务，按速率限制调用AI模型 - orchestrator 集成 rate_limiter，Gemini优先→NVIDIA兜底 NAS端: - Dispatcher 重构为 enqueue 模式（上传后立即返回，不等结果） - 新增 Poller 线程（定期从Edge拉取结果写 MariaDB） - app.py 启动 Poller，config.yaml 新增 poller 配置 - db_layer 更新 valid_stages 添加 'process'
This commit is contained in:
ericwyuan
5
fam-edge/src/fam_edge/queue/__init__.py
Normal file
5
fam-edge/src/fam_edge/queue/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""
|
||||
SQLite 异步任务队列 (Oracle 端)
|
||||
"""
|
||||
from . import queue_manager
|
||||
from .consumer import get_consumer
|
||||
128
fam-edge/src/fam_edge/queue/consumer.py
Normal file
128
fam-edge/src/fam_edge/queue/consumer.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
消费者线程 - 从 SQLite 队列消费任务，限速处理
|
||||
|
||||
策略：Gemini 优先 → NVIDIA 兜底（与现有 orchestrator 一致）
|
||||
速率：按 API 限制速度的 2 倍设置突发容量
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from ..logger import setup_logger
|
||||
from ..config_loader import load_config
|
||||
from ..rate_limiter import RateLimiter
|
||||
from ..ai_orchestrator.orchestrator import AIOrchestrator
|
||||
from ..video_preprocessor.preprocessor import VideoPreprocessor
|
||||
from . import queue_manager
|
||||
|
||||
logger = setup_logger('fam-edge.consumer')
|
||||
|
||||
# 从配置加载速率限制参数
|
||||
_cfg = load_config()
|
||||
_queue_cfg = _cfg.get('queue', {})
|
||||
_rate_cfg = _queue_cfg.get('rate_limit', {})
|
||||
GEMINI_RPM = _rate_cfg.get('gemini_rpm', 1000)
|
||||
NVIDIA_RPM = _rate_cfg.get('nvidia_rpm', 40)
|
||||
BURST_FACTOR = _rate_cfg.get('burst_factor', 2)
|
||||
POLL_INTERVAL = _queue_cfg.get('poll_interval', 10)
|
||||
|
||||
# 同步 SQLite DB 路径到环境变量（供 queue_manager 读取）
|
||||
os.environ.setdefault('FAM_QUEUE_DB', _queue_cfg.get('db_path', '/opt/fam-edge/data/fam_queue.db'))
|
||||
os.environ.setdefault('FAM_UPLOAD_DIR', _queue_cfg.get('upload_dir', '/tmp/fam_uploads'))
|
||||
|
||||
|
||||
class Consumer:
|
||||
def __init__(self):
|
||||
self._running = False
|
||||
self._thread = None
|
||||
self._orchestrator = AIOrchestrator()
|
||||
self._rate_limiter = RateLimiter()
|
||||
self._rate_limiter.register('gemini', GEMINI_RPM, burst_factor=BURST_FACTOR)
|
||||
self._rate_limiter.register('nvidia', NVIDIA_RPM, burst_factor=BURST_FACTOR)
|
||||
self._poll_interval = POLL_INTERVAL
|
||||
|
||||
def _process_one(self, task: dict) -> bool:
|
||||
task_id = task['id']
|
||||
nas_task_id = task['nas_task_id']
|
||||
video_path = task['video_path']
|
||||
|
||||
logger.info(f"[nas_task={nas_task_id}] 消费者开始处理")
|
||||
|
||||
preprocessor = None
|
||||
try:
|
||||
preprocessor = VideoPreprocessor(nas_task_id)
|
||||
|
||||
task_data = {
|
||||
"task_id": nas_task_id,
|
||||
"camera_name": task.get('camera_name', ''),
|
||||
"event_start_time": task.get('event_start_time', ''),
|
||||
"event_end_time": "",
|
||||
"known_members_context": task.get('known_members_context', ''),
|
||||
}
|
||||
|
||||
result = self._orchestrator.process_push_task(
|
||||
task_data, video_path, preprocessor, self._rate_limiter
|
||||
)
|
||||
|
||||
if result.get('status') == 'success':
|
||||
import json
|
||||
queue_manager.mark_success(task_id, json.dumps(result, ensure_ascii=False))
|
||||
logger.info(f"[nas_task={nas_task_id}] 消费者处理成功")
|
||||
return True
|
||||
else:
|
||||
error = result.get('error_message', 'unknown')
|
||||
stage = result.get('failure_stage', '')
|
||||
queue_manager.mark_failed(task_id, error, stage)
|
||||
logger.error(f"[nas_task={nas_task_id}] 消费者处理失败: {error}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[nas_task={nas_task_id}] 消费者异常: {e}", exc_info=True)
|
||||
queue_manager.mark_failed(task_id, str(e), 'process')
|
||||
return False
|
||||
finally:
|
||||
if preprocessor is not None:
|
||||
preprocessor.cleanup()
|
||||
try:
|
||||
if video_path and __import__('os').path.exists(video_path):
|
||||
__import__('os').remove(video_path)
|
||||
logger.info(f"[nas_task={nas_task_id}] 清理视频文件: {video_path}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _run(self):
|
||||
logger.info(f"消费者线程启动，轮询间隔 {self._poll_interval}s")
|
||||
logger.info(f"速率限制: Gemini {GEMINI_RPM}RPM x2 burst, NVIDIA {NVIDIA_RPM}RPM x2 burst")
|
||||
while self._running:
|
||||
try:
|
||||
task = queue_manager.claim_next()
|
||||
if task is None:
|
||||
time.sleep(self._poll_interval)
|
||||
continue
|
||||
self._process_one(task)
|
||||
except Exception as e:
|
||||
logger.error(f"消费者循环异常: {e}", exc_info=True)
|
||||
time.sleep(self._poll_interval)
|
||||
|
||||
def start(self):
|
||||
if self._running:
|
||||
return
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._run, daemon=True, name='consumer')
|
||||
self._thread.start()
|
||||
logger.info("消费者线程已启动")
|
||||
|
||||
def stop(self):
|
||||
self._running = False
|
||||
logger.info("消费者线程已停止")
|
||||
|
||||
|
||||
_consumer: Optional[Consumer] = None
|
||||
|
||||
|
||||
def get_consumer() -> Consumer:
|
||||
global _consumer
|
||||
if _consumer is None:
|
||||
_consumer = Consumer()
|
||||
return _consumer
|
||||
174
fam-edge/src/fam_edge/queue/queue_manager.py
Normal file
174
fam-edge/src/fam_edge/queue/queue_manager.py
Normal file
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
SQLite 队列管理器 - Oracle 端异步任务队列
|
||||
|
||||
表结构:
|
||||
- task_queue: 任务队列 (PENDING → PROCESSING → SUCCESS/FAILED)
|
||||
- 元数据: delivered 标记 NAS 是否已拉取结果
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
import json
|
||||
import threading
|
||||
from typing import Optional, List, Dict
|
||||
|
||||
DB_PATH = os.environ.get('FAM_QUEUE_DB', '/opt/fam-edge/data/fam_queue.db')
|
||||
|
||||
_init_lock = threading.Lock()
|
||||
_initialized = False
|
||||
|
||||
|
||||
def _get_conn() -> sqlite3.Connection:
|
||||
global _initialized
|
||||
if not _initialized:
|
||||
with _init_lock:
|
||||
if not _initialized:
|
||||
_init_db()
|
||||
_initialized = True
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
return conn
|
||||
|
||||
|
||||
def _init_db():
|
||||
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS task_queue (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
nas_task_id INTEGER NOT NULL,
|
||||
video_filename TEXT NOT NULL,
|
||||
video_path TEXT NOT NULL,
|
||||
camera_name TEXT DEFAULT '',
|
||||
event_start_time TEXT DEFAULT '',
|
||||
known_members_context TEXT DEFAULT '',
|
||||
status TEXT DEFAULT 'PENDING',
|
||||
result_json TEXT,
|
||||
error_message TEXT,
|
||||
failure_stage TEXT,
|
||||
retry_count INTEGER DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now', 'localtime')),
|
||||
updated_at TEXT DEFAULT (datetime('now', 'localtime')),
|
||||
delivered INTEGER DEFAULT 0,
|
||||
UNIQUE(nas_task_id)
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_status ON task_queue(status)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_delivered ON task_queue(delivered)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def enqueue(nas_task_id: int, video_filename: str, video_path: str,
|
||||
camera_name: str, event_start_time: str,
|
||||
known_members_context: str) -> int:
|
||||
conn = _get_conn()
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"INSERT OR IGNORE INTO task_queue "
|
||||
"(nas_task_id, video_filename, video_path, camera_name, event_start_time, known_members_context) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(nas_task_id, video_filename, video_path, camera_name, event_start_time, known_members_context)
|
||||
)
|
||||
conn.commit()
|
||||
if cur.rowcount == 0:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM task_queue WHERE nas_task_id=?", (nas_task_id,)
|
||||
).fetchone()
|
||||
return row['id'] if row else 0
|
||||
return cur.lastrowid
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def claim_next() -> Optional[Dict]:
|
||||
conn = _get_conn()
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
row = conn.execute(
|
||||
"SELECT * FROM task_queue WHERE status='PENDING' ORDER BY id LIMIT 1"
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE task_queue SET status='PROCESSING', updated_at=datetime('now','localtime') WHERE id=?",
|
||||
(row['id'],)
|
||||
)
|
||||
conn.commit()
|
||||
return dict(row)
|
||||
conn.rollback()
|
||||
return None
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def mark_success(task_id: int, result_json: str):
|
||||
conn = _get_conn()
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE task_queue SET status='SUCCESS', result_json=?, updated_at=datetime('now','localtime') WHERE id=?",
|
||||
(result_json, task_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def mark_failed(task_id: int, error_message: str, failure_stage: str = ''):
|
||||
conn = _get_conn()
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE task_queue SET status='FAILED', error_message=?, failure_stage=?, "
|
||||
"updated_at=datetime('now','localtime') WHERE id=?",
|
||||
(error_message, failure_stage, task_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_undelivered_results(limit: int = 10) -> List[Dict]:
|
||||
conn = _get_conn()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM task_queue WHERE status='SUCCESS' AND delivered=0 "
|
||||
"ORDER BY id LIMIT ?", (limit,)
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def mark_delivered(task_ids: List[int]):
|
||||
if not task_ids:
|
||||
return
|
||||
conn = _get_conn()
|
||||
try:
|
||||
placeholders = ','.join('?' * len(task_ids))
|
||||
conn.execute(
|
||||
f"UPDATE task_queue SET delivered=1, updated_at=datetime('now','localtime') "
|
||||
f"WHERE id IN ({placeholders})", task_ids
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_queue_stats() -> Dict:
|
||||
conn = _get_conn()
|
||||
try:
|
||||
stats = {}
|
||||
for status in ['PENDING', 'PROCESSING', 'SUCCESS', 'FAILED']:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) as cnt FROM task_queue WHERE status=?", (status,)
|
||||
).fetchone()
|
||||
stats[status] = row['cnt']
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) as cnt FROM task_queue WHERE status='SUCCESS' AND delivered=0"
|
||||
).fetchone()
|
||||
stats['UNDELIVERED'] = row['cnt']
|
||||
return stats
|
||||
finally:
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user