""" NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器 provider_name = "nvidia" 模型: nvidia/nemotron-3-nano-omni-30b-a3b-reasoning（唯一实测确认可用的视频理解模型） 角色: vision (整视频直出结构化 JSON) + 智能问答 SDK: openai (NIM 兼容 OpenAI API 规范) 整视频分析实测结论（2026-08-21 用真实短视频逐个探测）: - nemotron-3-nano-omni-30b-a3b-reasoning: video_url 只认 base64 data URI （`data:video/mp4;base64,<...>`），Assets API 的 asset_id 引用方式对它直接 500 （报错 "Only base64 data URLs are supported for now"）——所以本适配器不再走 Assets API 上传，直接 base64 内嵌整段视频。 - nemotron-nano-12b-v2-vl: 需要走 NVCF 函数调用协议本身的 NVCF-ASSET-DIR/ NVCF-FUNCTION-ASSET-IDS 请求头，而这两个头的值是 NVCF 服务端按内部路径生成、 不是客户端能自己拼对的（实测传什么都 400 "Invalid NVCF-ASSET-DIR"），标准 OpenAI 兼容 chat.completions 调用打不通，已从模型链移除。 - meta/llama-3.2-11b-vision-instruct: 明确不支持视频输入（"At most 0 video(s) may be provided"），只能单图，已移除。 base64 方案的代价是请求体大小受限（原实现注释称约 25MB 上限），所以本适配器会在 上传前检查文件大小，超过 `max_base64_mb`（默认 20MB）直接放弃，不做注定失败的 慢速编码+上传。真实监控视频压缩后通常在 20MB 上下，属于"够不到就正常降级到失败 重试"，不是本地故意限制过窄。 """ import base64 import os import time from datetime import datetime, timezone, timedelta from typing import Dict, List, Optional from .base_adapter import BaseModelAdapter from .circuit_breaker import CircuitBreaker from ..logger import setup_logger from ..ai_orchestrator.prompts import build_video_prompt from ..ai_orchestrator.json_parser import parse_vlm_json, VLMOutputInvalidError from ..config_loader import load_config logger = setup_logger('fam-edge.nvidia_adapter') try: from openai import OpenAI except ImportError: OpenAI = None class NvidiaVisionAdapter(BaseModelAdapter): """NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答) 多模型降级链（类似 Gemini flash -> flash-lite）: - model_chain = [model_name] + fallback_models - asset 上传一次，遍历模型链逐个调用 video_url 引用同一 assetId - 模型失败/超时 -> 记录统计 -> 间隔 switch_interval_sec 后切换下一模型 - 每个模型可用 model_timeouts 独立设置超时（不参与编排层 ×N 放大） """ def __init__(self, config: dict): super().__init__("nvidia", config) self.model_name = config.get( 'model_name', 'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning') self.model_chain = [self.model_name] + [ m for m in config.get('fallback_models', []) if m and m != self.model_name] self.api_key = self._resolve_key(config.get('api_key', '')) self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1') self.timeout = config.get('timeout', 600) # 问答专用超时，跟视频分析分开——交互式问答不该等到跟视频分析一样久 self.chat_timeout = config.get('chat_timeout', 20) self.max_base64_mb = float(config.get('max_base64_mb', 20)) # 模型级独立超时（最终值，不参与编排层 ×N 放大）: {model_name: seconds} self.model_timeouts = { str(k): int(v) for k, v in (config.get('model_timeouts') or {}).items()} # 模型切换间隔（秒）：一个模型失败后等待再切下一个，避免连续打爆 API self.switch_interval_sec = float(config.get('switch_interval_sec', 5)) cb_cfg = config.get('circuit_breaker', {}) self._cb = CircuitBreaker( threshold=cb_cfg.get('threshold', 3), cooldown=cb_cfg.get('cooldown', 600), enabled=cb_cfg.get('enabled', True) ) self._client = None if OpenAI is not None and self.api_key: try: self._client = OpenAI(base_url=self.base_url, api_key=self.api_key) except Exception as e: logger.error(f"NVIDIA OpenAI 客户端初始化失败: {e}") self._client = None def _resolve_key(self, raw: str) -> str: if raw.startswith('${') and raw.endswith('}'): return os.environ.get(raw[2:-1], '') return raw def health_check(self) -> bool: if self._client is None: logger.warning("NVIDIA OpenAI SDK 未就绪或 Key 未配置，健康检查失败") return False try: self._client.models.list() logger.info("NVIDIA 健康检查通过") return True except Exception as e: logger.warning(f"NVIDIA 健康检查失败: {e}") return False # ------------------------------------------------------------------ # 整视频分析：base64 内嵌 video_url 单次调用（omni 只认 base64，不认 asset_id 引用） # ------------------------------------------------------------------ def analyze_video(self, video_path: str, known_members_context: str, event_start_time: str = '') -> Optional[Dict]: if self._cb.is_open(): logger.warning("NVIDIA 熔断器 OPEN，跳过视频分析") return None if self._client is None: logger.warning("NVIDIA 客户端未初始化，跳过视频分析") return None if not os.path.isfile(video_path): logger.warning(f"NVIDIA 视频文件不存在: {video_path}") return None size_mb = os.path.getsize(video_path) / (1024 * 1024) if size_mb > self.max_base64_mb: logger.warning( f"NVIDIA 视频 {size_mb:.1f}MB 超过 base64 上限 {self.max_base64_mb}MB，" "跳过（不做注定失败的慢速编码）") return None try: with open(video_path, 'rb') as f: video_b64 = base64.b64encode(f.read()).decode() except Exception as e: logger.warning(f"NVIDIA 读取/编码视频失败: {e}") return None prompt = self._build_video_prompt(known_members_context, event_start_time) last_err = "no_model_in_chain" for idx, model in enumerate(self.model_chain): model_timeout = self.model_timeouts.get(model, self.timeout) logger.info(f"NVIDIA 模型链 [{idx+1}/{len(self.model_chain)}] " f"尝试 {model}（超时 {model_timeout}s）") started = datetime.now(timezone(timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S') t0 = time.time() try: resp = self._client.chat.completions.create( model=model, messages=[{"role": "user", "content": [ {"type": "text", "text": prompt}, {"type": "video_url", "video_url": { "url": f"data:video/mp4;base64,{video_b64}"}} ]}], temperature=0.2, max_tokens=16384, # NIM 扩展：控制视频采样帧数（部分模型支持） extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}}, timeout=model_timeout ) duration = time.time() - t0 content = resp.choices[0].message.content if not content: self._emit_model_call(model, started, duration, False, "empty_content") logger.warning(f"NVIDIA [{model}] 返回空 content，切换下一模型") last_err = f"{model}_empty" self._sleep_switch(idx) continue try: data = parse_vlm_json(content) except VLMOutputInvalidError as e: self._emit_model_call(model, started, duration, False, "json_parse_failed") logger.warning(f"NVIDIA [{model}] JSON 解析失败，切换下一模型: {e}") last_err = f"{model}_json" self._sleep_switch(idx) continue self._emit_model_call(model, started, duration, True) self._cb.record_success() logger.info(f"NVIDIA [{model}] 整视频分析完成，events={len(data.get('events', []))}") data['compute_provider'] = f"nvidia:{model}" return data except Exception as e: duration = time.time() - t0 self._emit_model_call(model, started, duration, False, str(e)) last_err = f"{model}_failed" logger.warning(f"NVIDIA [{model}] 视频分析异常，切换下一模型: {str(e)[:150]}") self._sleep_switch(idx) self._cb.record_failure() logger.error(f"NVIDIA 模型链全部失败: {last_err}") return None def _sleep_switch(self, idx: int): """模型切换间隔（最后一个模型失败后无需再等）""" if idx < len(self.model_chain) - 1 and self.switch_interval_sec > 0: logger.info(f"NVIDIA 等待 {self.switch_interval_sec}s 后切换下一模型") time.sleep(self.switch_interval_sec) def _build_video_prompt(self, known_members: str, event_start_time: str) -> str: camera = load_config().get('gdrive_sync', {}).get('camera_name', '') return build_video_prompt(known_members, event_start_time, camera) # ------------------------------------------------------------------ # 智能问答：纯文本 # ------------------------------------------------------------------ def chat(self, prompt: str, max_tokens: int = 2048) -> Optional[str]: if self._cb.is_open(): logger.warning("NVIDIA 熔断器 OPEN，跳过问答") return None if self._client is None: logger.warning("NVIDIA 客户端未初始化，跳过问答") return None for model in self.model_chain: try: resp = self._client.chat.completions.create( model=model, messages=[{"role": "user", "content": prompt}], temperature=0.3, max_tokens=max_tokens, timeout=self.chat_timeout ) content = resp.choices[0].message.content if content: self._cb.record_success() return content.strip() except Exception as e: logger.warning(f"NVIDIA [{model}] 问答异常: {e}") self._cb.record_failure() return None def get_timeout(self) -> int: return self.timeout def get_circuit_breaker(self) -> CircuitBreaker: return self._cb