feat(fam-edge): 队列健壮性 - ①失败重试间隔 retry_interval_sec=1h(last_fail_at 记录,配额类故障等恢复再试);②max_retries 2→10;③入队前 OpenCV 文件校验(可解码/大小/元数据,失败标 invalid 不入队)+ mtime 稳定窗口防 rclone 半成品 + 处理前二次确认

This commit is contained in:
ericwyuan
2026-08-21 14:09:38 +08:00
parent 92ef9fe163
commit 6afdda582b
4 changed files with 131 additions and 11 deletions

View File

@@ -21,7 +21,7 @@ from typing import List, Optional
from .logger import setup_logger
from .config_loader import load_config
from . import oracle_db
from .video_processor import VideoProcessor
from .video_processor import VideoProcessor, validate_video
logger = setup_logger('fam-edge.video_queue')
@@ -44,7 +44,12 @@ class VideoQueue:
# 模型超时放大倍数:消费时在模型原配置 timeout 上 ×N
self.timeout_multiplier = float(vp.get('timeout_multiplier', 2.0))
# 单视频失败最大重试次数failed 且 retry_count>=max_retries 不再消费)
self.max_retries = int(vp.get('max_retries', 2))
self.max_retries = int(vp.get('max_retries', 10))
# 失败重试最小间隔(秒):配额类瞬时故障(如 429需等其恢复再重试避免短时间重复打爆
self.retry_interval_sec = int(vp.get('retry_interval_sec', 3600))
# 文件校验:入队前用 OpenCV 确认可解码mtime 稳定窗口防 rclone 写入中的半成品
self.file_validate = bool(vp.get('file_validate', True))
self.stable_window_sec = int(vp.get('stable_window_sec', 60))
self._queue: "queue.Queue[int]" = queue.Queue()
self._queued: set = set() # 已在队中的 video_id防重复入队
@@ -75,6 +80,26 @@ class VideoQueue:
fn = os.path.basename(path)
row = self.db.get_video_by_filename(fn)
if row is None:
# 新文件:先过 mtime 稳定窗口 + 可解码校验,通过才登记入队;失败标记 invalid
if self.file_validate:
if time.time() - os.path.getmtime(path) < self.stable_window_sec:
logger.info(f"文件仍在写入mtime 未稳定),跳过本轮: {fn}")
continue
ok, verr, vmeta = validate_video(path)
if not ok:
vid = self.db.ensure_video(fn, path, camera_name=self.camera_name)
self.db.set_video_file_status(vid, False, verr)
self.db._conn.execute(
"UPDATE videos SET status='invalid' WHERE id=?", (vid,))
self.db._conn.commit()
logger.warning(f"文件校验失败,标记 invalid 不入队: {fn} ({verr})")
continue
if vmeta:
vid = self.db.ensure_video(fn, path, camera_name=self.camera_name)
self.db.set_video_file_status(vid, True, '', vmeta)
logger.info(f"登记新视频并入队: {fn} (id={vid}, meta={vmeta})")
self._enqueue(vid)
continue
vid = self.db.ensure_video(fn, path, camera_name=self.camera_name)
logger.info(f"登记新视频并入队: {fn} (id={vid})")
self._enqueue(vid)
@@ -91,10 +116,26 @@ class VideoQueue:
logger.info(f"启动恢复入队 {len(rows)} 个待处理视频")
def _retry_allowed(self, row) -> bool:
if row['status'] == 'pending':
"""是否允许消费/重试该视频pending 直接放行failed 需未超重试上限且距上次失败够久invalid 永不放行"""
status = row['status']
if status == 'pending':
return True
# failed未超重试上限才允许再次消费
return int(row['retry_count'] or 0) < self.max_retries
if status == 'invalid':
return False
# failed未超重试上限且距上次失败 >= retry_interval_sec等配额类故障恢复
if int(row['retry_count'] or 0) >= self.max_retries:
return False
last_fail = row['last_fail_at'] or row['updated_at'] or ''
if last_fail:
try:
from datetime import datetime
lt = datetime.strptime(last_fail[:19], '%Y-%m-%d %H:%M:%S')
elapsed = (datetime.now() - lt).total_seconds()
if elapsed < self.retry_interval_sec:
return False
except ValueError:
pass
return True
def _enqueue(self, video_id: int):
with self._lock: