feat: NVIDIA 切换到原生视频输入 — nemotron-3-nano-omni + 集锦视频单次调用
调研结论: build.nvidia.com 免费托管 API 上 video-llama3-8b 与 qwen2.5-vl-72b 已下线(404),nvidia/nemotron-3-nano-omni-30b-a3b-reasoning 可用(200, 40RPM 免费额度内),原生支持 video_url 输入(MP4 base64)。 实现: 1. nvidia_adapter 新增 analyze_video: 按关键帧时间点截取 ±1.5s 片段 (drawtext 叠加时间戳,连字符避免冒号转义)拼集锦视频,640 宽 CRF28, base64 后经 video_url 单次调用,输出全 schema JSON(frame_details + global_summary + entities_json)并归一化对齐时间戳 2. analyze_frames 保留为无视频文件时的降级路径; chat max_tokens 512→2048(reasoning 模型 token 消耗大); timeout 20→120s 3. orchestrator.run_visual_analysis 增加 video_path 参数,fallback 循环 对支持 analyze_video 的适配器优先走视频模式,失败自动降级逐帧 实测(360MB 测试视频, 3 关键帧): 集锦 107KB, 全程 37s, 动态动作识别准确 (走动→坐沙发→坐餐桌),跨片段综合摘要正常 — 显著优于旧逐帧静态识别。
This commit is contained in:
@@ -53,9 +53,13 @@ class AIOrchestrator:
|
||||
frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str,
|
||||
rate_limiter=None) -> Dict[str, dict]:
|
||||
rate_limiter=None,
|
||||
video_path: str = None) -> Dict[str, dict]:
|
||||
"""视觉分析阶段:仅 role=vision 的适配器参与
|
||||
|
||||
支持 analyze_video 的适配器(如 NVIDIA Omni)优先走原生视频输入,
|
||||
失败自动降级回逐帧图片模式。
|
||||
|
||||
orchestrator.mode:
|
||||
- fallback (默认): 按 config 顺序依次尝试,首个成功即采用(单元素 dict)
|
||||
- ensemble: 并行所有健康 vision 模型,全部成功结果都保留(交叉验证)
|
||||
@@ -88,8 +92,20 @@ class AIOrchestrator:
|
||||
continue
|
||||
start = time.time()
|
||||
try:
|
||||
output = adapter.analyze_frames(
|
||||
frame_paths, frame_timestamps, known_members_context)
|
||||
output = None
|
||||
if video_path and hasattr(adapter, 'analyze_video'):
|
||||
try:
|
||||
logger.info(f"[{adapter.provider_name}] 尝试原生视频输入分析")
|
||||
output = adapter.analyze_video(
|
||||
video_path, frame_timestamps, known_members_context)
|
||||
if not output:
|
||||
logger.warning(f"[{adapter.provider_name}] 视频模式失败,降级逐帧模式")
|
||||
except Exception as ve:
|
||||
logger.warning(f"[{adapter.provider_name}] 视频模式异常: {ve},降级逐帧模式")
|
||||
output = None
|
||||
if not output:
|
||||
output = adapter.analyze_frames(
|
||||
frame_paths, frame_timestamps, known_members_context)
|
||||
duration_ms = int((time.time() - start) * 1000)
|
||||
if output:
|
||||
adapter.get_circuit_breaker().record_success()
|
||||
@@ -354,7 +370,8 @@ class AIOrchestrator:
|
||||
|
||||
# 3. 并行视觉分析
|
||||
model_outputs = self.run_visual_analysis(
|
||||
healthy_adapters, compressed_frames, frame_timestamps, known_members
|
||||
healthy_adapters, compressed_frames, frame_timestamps,
|
||||
known_members, video_path=video_path
|
||||
)
|
||||
|
||||
if not model_outputs:
|
||||
@@ -450,7 +467,7 @@ class AIOrchestrator:
|
||||
# 3. 并行视觉分析
|
||||
model_outputs = self.run_visual_analysis(
|
||||
healthy_adapters, compressed_frames, frame_timestamps,
|
||||
known_members, rate_limiter
|
||||
known_members, rate_limiter, video_path=video_path
|
||||
)
|
||||
if not model_outputs:
|
||||
raise Exception('All models failed in visual analysis')
|
||||
|
||||
Reference in New Issue
Block a user