feat(fam-edge): 队列健壮性 - ①失败重试间隔 retry_interval_sec=1h(last_fail_at 记录,配额类故障等恢复再试);②max_retries 2→10;③入队前 OpenCV 文件校验(可解码/大小/元数据,失败标 invalid 不入队)+ mtime 稳定窗口防 rclone 半成品 + 处理前二次确认
This commit is contained in:
@@ -21,7 +21,7 @@ from typing import List, Optional
|
||||
from .logger import setup_logger
|
||||
from .config_loader import load_config
|
||||
from . import oracle_db
|
||||
from .video_processor import VideoProcessor
|
||||
from .video_processor import VideoProcessor, validate_video
|
||||
|
||||
logger = setup_logger('fam-edge.video_queue')
|
||||
|
||||
@@ -44,7 +44,12 @@ class VideoQueue:
|
||||
# 模型超时放大倍数:消费时在模型原配置 timeout 上 ×N
|
||||
self.timeout_multiplier = float(vp.get('timeout_multiplier', 2.0))
|
||||
# 单视频失败最大重试次数(failed 且 retry_count>=max_retries 不再消费)
|
||||
self.max_retries = int(vp.get('max_retries', 2))
|
||||
self.max_retries = int(vp.get('max_retries', 10))
|
||||
# 失败重试最小间隔(秒):配额类瞬时故障(如 429)需等其恢复再重试,避免短时间重复打爆
|
||||
self.retry_interval_sec = int(vp.get('retry_interval_sec', 3600))
|
||||
# 文件校验:入队前用 OpenCV 确认可解码;mtime 稳定窗口防 rclone 写入中的半成品
|
||||
self.file_validate = bool(vp.get('file_validate', True))
|
||||
self.stable_window_sec = int(vp.get('stable_window_sec', 60))
|
||||
|
||||
self._queue: "queue.Queue[int]" = queue.Queue()
|
||||
self._queued: set = set() # 已在队中的 video_id(防重复入队)
|
||||
@@ -75,6 +80,26 @@ class VideoQueue:
|
||||
fn = os.path.basename(path)
|
||||
row = self.db.get_video_by_filename(fn)
|
||||
if row is None:
|
||||
# 新文件:先过 mtime 稳定窗口 + 可解码校验,通过才登记入队;失败标记 invalid
|
||||
if self.file_validate:
|
||||
if time.time() - os.path.getmtime(path) < self.stable_window_sec:
|
||||
logger.info(f"文件仍在写入(mtime 未稳定),跳过本轮: {fn}")
|
||||
continue
|
||||
ok, verr, vmeta = validate_video(path)
|
||||
if not ok:
|
||||
vid = self.db.ensure_video(fn, path, camera_name=self.camera_name)
|
||||
self.db.set_video_file_status(vid, False, verr)
|
||||
self.db._conn.execute(
|
||||
"UPDATE videos SET status='invalid' WHERE id=?", (vid,))
|
||||
self.db._conn.commit()
|
||||
logger.warning(f"文件校验失败,标记 invalid 不入队: {fn} ({verr})")
|
||||
continue
|
||||
if vmeta:
|
||||
vid = self.db.ensure_video(fn, path, camera_name=self.camera_name)
|
||||
self.db.set_video_file_status(vid, True, '', vmeta)
|
||||
logger.info(f"登记新视频并入队: {fn} (id={vid}, meta={vmeta})")
|
||||
self._enqueue(vid)
|
||||
continue
|
||||
vid = self.db.ensure_video(fn, path, camera_name=self.camera_name)
|
||||
logger.info(f"登记新视频并入队: {fn} (id={vid})")
|
||||
self._enqueue(vid)
|
||||
@@ -91,10 +116,26 @@ class VideoQueue:
|
||||
logger.info(f"启动恢复入队 {len(rows)} 个待处理视频")
|
||||
|
||||
def _retry_allowed(self, row) -> bool:
|
||||
if row['status'] == 'pending':
|
||||
"""是否允许消费/重试该视频:pending 直接放行;failed 需未超重试上限且距上次失败够久;invalid 永不放行"""
|
||||
status = row['status']
|
||||
if status == 'pending':
|
||||
return True
|
||||
# failed:未超重试上限才允许再次消费
|
||||
return int(row['retry_count'] or 0) < self.max_retries
|
||||
if status == 'invalid':
|
||||
return False
|
||||
# failed:未超重试上限,且距上次失败 >= retry_interval_sec(等配额类故障恢复)
|
||||
if int(row['retry_count'] or 0) >= self.max_retries:
|
||||
return False
|
||||
last_fail = row['last_fail_at'] or row['updated_at'] or ''
|
||||
if last_fail:
|
||||
try:
|
||||
from datetime import datetime
|
||||
lt = datetime.strptime(last_fail[:19], '%Y-%m-%d %H:%M:%S')
|
||||
elapsed = (datetime.now() - lt).total_seconds()
|
||||
if elapsed < self.retry_interval_sec:
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
return True
|
||||
|
||||
def _enqueue(self, video_id: int):
|
||||
with self._lock:
|
||||
|
||||
Reference in New Issue
Block a user