""" NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器 provider_name = "nvidia" 模型: nvidia/nemotron-nano-12b-v2-vl（NIM 官方支持整视频 video_url 输入，内部自行采样帧） 角色: vision (整视频直出结构化 JSON) + 智能问答 SDK: openai (NIM 兼容 OpenAI API 规范) 整视频分析: 整视频 base64 经 video_url 单次调用 —— 本地不切片、不抽帧 """ import os import base64 import json import re from typing import Dict, List, Optional from .base_adapter import BaseModelAdapter from .circuit_breaker import CircuitBreaker from ..logger import setup_logger logger = setup_logger('fam-edge.nvidia_adapter') try: from openai import OpenAI except ImportError: OpenAI = None class NvidiaVisionAdapter(BaseModelAdapter): """NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)""" def __init__(self, config: dict): super().__init__("nvidia", config) self.model_name = config.get( 'model_name', 'nvidia/nemotron-nano-12b-v2-vl') self.api_key = self._resolve_key(config.get('api_key', '')) self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1') self.timeout = config.get('timeout', 600) cb_cfg = config.get('circuit_breaker', {}) self._cb = CircuitBreaker( threshold=cb_cfg.get('threshold', 3), cooldown=cb_cfg.get('cooldown', 600), enabled=cb_cfg.get('enabled', True) ) self._client = None if OpenAI is not None and self.api_key: try: self._client = OpenAI(base_url=self.base_url, api_key=self.api_key) except Exception as e: logger.error(f"NVIDIA OpenAI 客户端初始化失败: {e}") self._client = None def _resolve_key(self, raw: str) -> str: if raw.startswith('${') and raw.endswith('}'): return os.environ.get(raw[2:-1], '') return raw def health_check(self) -> bool: if self._client is None: logger.warning("NVIDIA OpenAI SDK 未就绪或 Key 未配置，健康检查失败") return False try: self._client.models.list() logger.info("NVIDIA 健康检查通过") return True except Exception as e: logger.warning(f"NVIDIA 健康检查失败: {e}") return False # ------------------------------------------------------------------ # 整视频分析：base64 整视频 -> video_url 单次调用 # ------------------------------------------------------------------ def analyze_video(self, video_path: str, known_members_context: str, event_start_time: str = '') -> Optional[Dict]: if self._cb.is_open(): logger.warning("NVIDIA 熔断器 OPEN，跳过视频分析") return None if self._client is None: logger.warning("NVIDIA 客户端未初始化，跳过视频分析") return None if not os.path.isfile(video_path): logger.warning(f"NVIDIA 视频文件不存在: {video_path}") return None try: with open(video_path, 'rb') as f: b64 = base64.b64encode(f.read()).decode('utf-8') except Exception as e: logger.warning(f"NVIDIA 读取视频失败: {e}") return None prompt = self._build_video_prompt(known_members_context, event_start_time) try: resp = self._client.chat.completions.create( model=self.model_name, messages=[{"role": "user", "content": [ {"type": "text", "text": prompt}, {"type": "video_url", "video_url": { "url": f"data:video/mp4;base64,{b64}"}} ]}], temperature=0.2, max_tokens=4096, # NIM 扩展：控制视频采样帧数（模型上限 128 帧） extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}}, timeout=self.timeout ) content = resp.choices[0].message.content if not content: logger.warning("NVIDIA 视频分析返回空 content") self._cb.record_failure() return None data = self._parse_json(content) if not data or 'events' not in data: logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}") self._cb.record_failure() return None self._cb.record_success() logger.info(f"NVIDIA 整视频分析完成，events={len(data.get('events', []))}") return { "global_summary": str(data.get('global_summary', '')), "events": data.get('events', []), "people_mentioned": data.get('people_mentioned', []), "compute_provider": "nvidia", } except Exception as e: self._cb.record_failure() logger.warning(f"NVIDIA 视频分析异常: {e}") return None @staticmethod def _parse_json(content: str) -> Optional[dict]: content = content.strip() try: return json.loads(content) except json.JSONDecodeError: pass fence = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL) if fence: try: return json.loads(fence.group(1)) except json.JSONDecodeError: pass brace = re.search(r'\{.*\}', content, re.DOTALL) if brace: try: return json.loads(brace.group(0)) except json.JSONDecodeError: pass return None def _build_video_prompt(self, known_members: str, event_start_time: str) -> str: start_hint = "" if event_start_time: start_hint = f"\n视频开始时间（北京时间）约为：{event_start_time}。请据此推算每个事件的绝对时间戳。" return f"""你是家庭监控视频分析助手。下面是一段完整监控录像（已整段上传）。 请观看整段视频，提取其中有用的信息，只输出合法 JSON（不要 markdown、不要解释），结构如下： {{ "global_summary": "整个时段的整体摘要，简体中文，2-4 句", "events": [ {{ "timestamp": "事件发生时的绝对北京时间(YYYY-MM-DD HH:MM:SS)", "description": "该时刻画面/动作信息摘要", "people": ["出现在该时刻的人物，用已知成员真名或'人物A'"], "is_attention_event": false }} ], "people_mentioned": ["本视频出现过的所有人物标识/真名"] }}{start_hint} 规则： 1. 只描述客观画面，不猜测。 2. events 提取有意义的时间点（人物出现/动作变化/异常），timestamp 用绝对北京时间。 3. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）： {known_members or '（暂无已知成员）'} 4. is_attention_event：跌倒、危险、异常哭闹等需关注事件（没有则为 false）。""" # ------------------------------------------------------------------ # 智能问答：纯文本 # ------------------------------------------------------------------ def chat(self, prompt: str, max_tokens: int = 2048) -> Optional[str]: if self._client is None: logger.warning("NVIDIA 客户端未初始化，跳过问答") return None try: resp = self._client.chat.completions.create( model=self.model_name, messages=[{"role": "user", "content": prompt}], temperature=0.3, max_tokens=max_tokens, timeout=self.timeout ) content = resp.choices[0].message.content return content.strip() if content else None except Exception as e: logger.warning(f"NVIDIA 问答异常: {e}") return None def get_timeout(self) -> int: return self.timeout def get_circuit_breaker(self) -> CircuitBreaker: return self._cb