refactor(fam-edge): 重构第一阶段 - 人物图片零额外调用 + 运行时稳定性 + 工程质量
人物图片功能重做: bbox 随核心视频分析那一次 Gemini 调用一并产出(prompts.py 加
person_appearances.bbox 字段, [ymin,xmin,ymax,xmax] 0-1000 归一化), frame_service
直接用存好的 bbox 裁剪头像/事件缩略图, 删除原来"展示时额外调用 Gemini 定位人物"的
整套逻辑(locate_person_bbox/VLM 校验/熔断), 从架构上消除与核心视频分析共抢配额的
问题; 用真实数据验证裁剪结果正确框住人物本体。
NVIDIA 模型修复: 实测原配置的 3 个模型均不可用(asset_id 引用 500/400, 不支持视频),
改用 nemotron-3-nano-omni 的 base64 内嵌视频方式(唯一实测打通), 加 max_base64_mb
防止对大文件做注定失败的编码。
Gemini 多 Key 轮换: 支持 extra_api_keys 配置多个独立项目的 key, 配额用尽时依次
换 key 重试(每换 key 需重新上传, Files API 按项目隔离)。
稳定性加固: CircuitBreaker HALF_OPEN 清空旧失败计数(修复探测一失败就重新 OPEN 的
bug); chat() 统一接入熔断器(原来只有视频分析路径检查); NVIDIA 适配器改用共享
json_parser(原来自己重复实现且不做 schema 校验); Gemini Files API 上传超时也尝试
清理远程孤儿文件; video_processor/video_queue 里直接操作 OracleDB._conn 的裸 SQL
改走新增的 set_event_start_time/mark_video_invalid/reset_video_to_pending 方法;
/health 加入队列线程存活状态; 密钥改用 ${ENV_VAR} 引用(.env 已支持自动加载),
不再明文写入 config.yaml。
工程质量: 新增 fam-edge/tests(32 个单元测试, 覆盖熔断器状态机/JSON 解析容错/
时间戳解析/bbox 坐标换算/多 key 解析), 新增 scripts/smoke_test.py(发版前接口
稳定性检查); 清理死代码(OllamaAdapter.analyze_frames、get_sync_delta 死分支、
未使用的 vision_timeout/max_concurrent_tasks 配置项); 修正 get_events_for_label
排序(改最近优先 + 过滤畸形历史时间戳)。
已部署 Oracle 并跑通 smoke test 全部 6 项检查。
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ericwyuan
232
fam-edge/src/fam_edge/frame_service.py
Normal file
232
fam-edge/src/fam_edge/frame_service.py
Normal file
@@ -0,0 +1,232 @@
|
||||
"""
|
||||
Frame-Service - 关键帧抽帧与人物头像裁剪（Oracle 端集中计算）
|
||||
|
||||
新架构 v4（2026-08-21 重构）:
|
||||
- extract_frame: 按 video_id + 绝对时间戳，ffmpeg 精确抽帧，磁盘缓存
|
||||
- build_avatar: 用该人物候选事件里随视频分析一次性产出的 bbox（person_appearances
|
||||
里的 [ymin,xmin,ymax,xmax]）直接裁剪，不再额外调用模型定位人物。
|
||||
|
||||
v3 曾经的做法是在展示缩略图/头像时另外调用一次 Gemini 做人物定位校验，这会和核心
|
||||
视频分析共用同一份 Gemini Key 抢配额（实测两边同时打 429），且当时的坐标解析本身
|
||||
也有 bug（错把 Gemini 原生 [ymin,xmin,ymax,xmax]/1000 当成 [x1,y1,x2,y2]/1）。v4
|
||||
把 bbox 改成随视频分析那一次 Gemini 调用一并产出（见 ai_orchestrator/prompts.py 的
|
||||
person_appearances.bbox 字段），frame_service 只做纯本地的抽帧/裁剪，不再对任何
|
||||
模型发起请求，从架构上消除配额争抢。
|
||||
|
||||
所有产物落到 CACHE_DIR，以 (video_id, offset) 或 label 为 key，避免重复计算。
|
||||
NAS 侧只负责代理与展示，不做任何图像计算。
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
|
||||
from .logger import setup_logger
|
||||
|
||||
logger = setup_logger('fam-edge.frame_service')
|
||||
|
||||
try:
|
||||
from .config_loader import load_config
|
||||
_CFG = load_config().get('frame_service', {})
|
||||
except Exception:
|
||||
_CFG = {}
|
||||
|
||||
CACHE_DIR = _CFG.get('cache_dir', '/opt/fam-edge/frames_cache')
|
||||
FFMPEG = shutil.which('ffmpeg') or 'ffmpeg'
|
||||
FFPROBE = shutil.which('ffprobe') or 'ffprobe'
|
||||
AVATAR_W = int(_CFG.get('avatar_width', 160))
|
||||
FRAME_W = int(_CFG.get('frame_width', 400))
|
||||
|
||||
|
||||
def _ensure_dir():
|
||||
os.makedirs(CACHE_DIR, exist_ok=True)
|
||||
|
||||
|
||||
def _parse_dt(ts_str: str):
|
||||
try:
|
||||
return datetime.strptime(ts_str.strip()[:19], '%Y-%m-%d %H:%M:%S')
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _run_ffmpeg(args, timeout=60) -> bool:
|
||||
try:
|
||||
proc = subprocess.run([FFMPEG, '-hide_banner', '-loglevel', 'error', *args],
|
||||
capture_output=True, timeout=timeout)
|
||||
return proc.returncode == 0
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _out_size(path):
|
||||
"""ffprobe 读图片宽高 -> (w, h) 或 None"""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[FFPROBE, '-v', 'error', '-select_streams', 'v:0',
|
||||
'-show_entries', 'stream=width,height', '-of', 'csv=s=x:p=0', path],
|
||||
capture_output=True, text=True, timeout=20).stdout.strip()
|
||||
w, h = out.split('x')
|
||||
return int(w), int(h)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def extract_frame(db, video_id: int, ts: str, width: int = FRAME_W) -> bytes:
|
||||
"""按 video_id + 绝对时间戳抽帧，返回 jpeg bytes（带磁盘缓存）"""
|
||||
if not FFMPEG:
|
||||
return None
|
||||
row = db.get_video_by_id(video_id)
|
||||
if not row:
|
||||
return None
|
||||
local_path = row['local_path']
|
||||
if not local_path or not os.path.isfile(local_path):
|
||||
logger.warning(f"[video_id={video_id}] 视频文件不存在: {local_path}")
|
||||
return None
|
||||
start = _parse_dt(row['event_start_time'] or '')
|
||||
t = _parse_dt(ts)
|
||||
if t and start:
|
||||
offset = max(0.0, (t - start).total_seconds())
|
||||
else:
|
||||
offset = 0.0
|
||||
|
||||
_ensure_dir()
|
||||
cache = os.path.join(CACHE_DIR, f"frame_{video_id}_{int(offset)}.jpg")
|
||||
if os.path.isfile(cache) and os.path.getsize(cache) > 0:
|
||||
with open(cache, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
fd, tmp = tempfile.mkstemp(suffix='.jpg', dir=CACHE_DIR)
|
||||
os.close(fd)
|
||||
try:
|
||||
# 粗 seek（-i 前，关键帧快进）+ 精 seek（-i 后，逐帧解码）：
|
||||
# 纯输入端 seek 只能跳到最近关键帧，事件按 3s 密度打点时若 GOP 间隔
|
||||
# 大于 3s 会抽到别的关键帧，导致画面与描述对不上。
|
||||
coarse = max(0.0, offset - 5.0)
|
||||
fine = offset - coarse
|
||||
ok = _run_ffmpeg([
|
||||
'-ss', f'{coarse:.3f}', '-i', local_path,
|
||||
'-ss', f'{fine:.3f}',
|
||||
'-frames:v', '1', '-vf', f'scale={width}:-2',
|
||||
'-q:v', '5', '-f', 'image2', '-y', tmp,
|
||||
], timeout=120)
|
||||
if not ok or not os.path.isfile(tmp) or os.path.getsize(tmp) == 0:
|
||||
return None
|
||||
with open(tmp, 'rb') as f:
|
||||
data = f.read()
|
||||
os.replace(tmp, cache) # 原子落缓存
|
||||
return data
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _bbox_to_pixels(bbox, width, height):
|
||||
"""bbox 为 [ymin,xmin,ymax,xmax]，0-1000 归一化 -> 像素 (x1,y1,x2,y2)。"""
|
||||
ymin, xmin, ymax, xmax = bbox
|
||||
x1, y1 = xmin / 1000.0 * width, ymin / 1000.0 * height
|
||||
x2, y2 = xmax / 1000.0 * width, ymax / 1000.0 * height
|
||||
return int(x1), int(y1), int(x2), int(y2)
|
||||
|
||||
|
||||
def _crop_ffmpeg(img_path: str, bbox_px, target_w) -> bool:
|
||||
"""按像素 bbox 裁剪居中并缩小为正方形，覆盖 img_path。失败返回 False。"""
|
||||
size = _out_size(img_path)
|
||||
if not size:
|
||||
return False
|
||||
w, h = size
|
||||
x1, y1, x2, y2 = bbox_px
|
||||
x1, x2 = max(0, x1), min(w, x2)
|
||||
y1, y2 = max(0, y1), min(h, y2)
|
||||
if x2 - x1 <= 0 or y2 - y1 <= 0:
|
||||
return False
|
||||
px, py = int((x2 - x1) * 0.3), int((y2 - y1) * 0.3)
|
||||
x1, y1 = max(0, x1 - px), max(0, y1 - py)
|
||||
x2, y2 = min(w, x2 + px), min(h, y2 + py)
|
||||
cw, ch = x2 - x1, y2 - y1
|
||||
if cw <= 0 or ch <= 0:
|
||||
return False
|
||||
return _apply_filter(img_path, f'crop={cw}:{ch}:{x1}:{y1},scale={target_w}:{target_w}')
|
||||
|
||||
|
||||
def _center_square_ffmpeg(img_path: str, target_w) -> bool:
|
||||
"""整帧居中正方形裁剪缩小，当人物没有 bbox 时兜底"""
|
||||
size = _out_size(img_path)
|
||||
if not size:
|
||||
return False
|
||||
w, h = size
|
||||
side = min(w, h)
|
||||
cx, cy = (w - side) // 2, (h - side) // 2
|
||||
return _apply_filter(img_path, f'crop={side}:{side}:{cx}:{cy},scale={target_w}:{target_w}')
|
||||
|
||||
|
||||
def _apply_filter(img_path: str, vf: str) -> bool:
|
||||
fd, tmp = tempfile.mkstemp(suffix='.jpg', dir=CACHE_DIR)
|
||||
os.close(fd)
|
||||
try:
|
||||
if not _run_ffmpeg(['-i', img_path, '-vf', vf,
|
||||
'-q:v', '5', '-frames:v', '1', '-f', 'image2', '-y', tmp]):
|
||||
return False
|
||||
os.replace(tmp, img_path)
|
||||
return True
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def build_avatar(db, label: str, width: int = AVATAR_W) -> bytes:
|
||||
"""为人物构建头像：用候选事件里已经随视频分析产出的 bbox 直接裁剪；
|
||||
没有任何候选事件带 bbox 时，回退整帧居中裁剪。零额外模型调用。
|
||||
cache key = avatar_{label}。"""
|
||||
_ensure_dir()
|
||||
cache = os.path.join(CACHE_DIR, f"avatar_{label}.jpg")
|
||||
if os.path.isfile(cache) and os.path.getsize(cache) > 0:
|
||||
with open(cache, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
events = db.get_events_for_label(label, limit=6)
|
||||
if not events:
|
||||
return None
|
||||
|
||||
fd, tmp = tempfile.mkstemp(suffix='.jpg', dir=CACHE_DIR)
|
||||
os.close(fd)
|
||||
try:
|
||||
crop_data = None
|
||||
first_good = None
|
||||
for ev in events:
|
||||
src = extract_frame(db, ev['video_id'], ev['ts'], width=600)
|
||||
if src is None:
|
||||
continue
|
||||
with open(tmp, 'wb') as f:
|
||||
f.write(src)
|
||||
if first_good is None:
|
||||
first_good = os.path.getsize(tmp) > 0
|
||||
bbox = ev.get('bbox')
|
||||
if bbox:
|
||||
size = _out_size(tmp)
|
||||
if size and _crop_ffmpeg(tmp, _bbox_to_pixels(bbox, *size), width):
|
||||
crop_data = ('bbox', os.path.getsize(tmp))
|
||||
break
|
||||
# 兜底：首张可用的帧整帧居中（候选事件都没有 bbox 时）
|
||||
if crop_data is None:
|
||||
if first_good and _center_square_ffmpeg(tmp, width):
|
||||
crop_data = ('fallback', os.path.getsize(tmp))
|
||||
if crop_data is None:
|
||||
return None
|
||||
with open(tmp, 'rb') as f:
|
||||
data = f.read()
|
||||
if data:
|
||||
os.replace(tmp, cache)
|
||||
return data
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
Reference in New Issue
Block a user