204 lines
8.1 KiB
Python
204 lines
8.1 KiB
Python
"""
|
||
NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器
|
||
|
||
provider_name = "nvidia"
|
||
模型: nvidia/nemotron-nano-12b-v2-vl（NIM 官方支持整视频 video_url 输入，内部自行采样帧）
|
||
角色: vision (整视频直出结构化 JSON) + 智能问答
|
||
SDK: openai (NIM 兼容 OpenAI API 规范)
|
||
整视频分析: 整视频 base64 经 video_url 单次调用 —— 本地不切片、不抽帧
|
||
"""
|
||
import os
|
||
import base64
|
||
import json
|
||
import re
|
||
from typing import Dict, List, Optional
|
||
|
||
from .base_adapter import BaseModelAdapter
|
||
from .circuit_breaker import CircuitBreaker
|
||
from ..logger import setup_logger
|
||
|
||
logger = setup_logger('fam-edge.nvidia_adapter')
|
||
|
||
try:
|
||
from openai import OpenAI
|
||
except ImportError:
|
||
OpenAI = None
|
||
|
||
|
||
class NvidiaVisionAdapter(BaseModelAdapter):
|
||
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)"""
|
||
|
||
def __init__(self, config: dict):
|
||
super().__init__("nvidia", config)
|
||
self.model_name = config.get(
|
||
'model_name', 'nvidia/nemotron-nano-12b-v2-vl')
|
||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
|
||
self.timeout = config.get('timeout', 600)
|
||
cb_cfg = config.get('circuit_breaker', {})
|
||
self._cb = CircuitBreaker(
|
||
threshold=cb_cfg.get('threshold', 3),
|
||
cooldown=cb_cfg.get('cooldown', 600),
|
||
enabled=cb_cfg.get('enabled', True)
|
||
)
|
||
self._client = None
|
||
if OpenAI is not None and self.api_key:
|
||
try:
|
||
self._client = OpenAI(base_url=self.base_url, api_key=self.api_key)
|
||
except Exception as e:
|
||
logger.error(f"NVIDIA OpenAI 客户端初始化失败: {e}")
|
||
self._client = None
|
||
|
||
def _resolve_key(self, raw: str) -> str:
|
||
if raw.startswith('${') and raw.endswith('}'):
|
||
return os.environ.get(raw[2:-1], '')
|
||
return raw
|
||
|
||
def health_check(self) -> bool:
|
||
if self._client is None:
|
||
logger.warning("NVIDIA OpenAI SDK 未就绪或 Key 未配置，健康检查失败")
|
||
return False
|
||
try:
|
||
self._client.models.list()
|
||
logger.info("NVIDIA 健康检查通过")
|
||
return True
|
||
except Exception as e:
|
||
logger.warning(f"NVIDIA 健康检查失败: {e}")
|
||
return False
|
||
|
||
# ------------------------------------------------------------------
|
||
# 整视频分析：base64 整视频 -> video_url 单次调用
|
||
# ------------------------------------------------------------------
|
||
def analyze_video(self, video_path: str,
|
||
known_members_context: str,
|
||
event_start_time: str = '') -> Optional[Dict]:
|
||
if self._cb.is_open():
|
||
logger.warning("NVIDIA 熔断器 OPEN，跳过视频分析")
|
||
return None
|
||
if self._client is None:
|
||
logger.warning("NVIDIA 客户端未初始化，跳过视频分析")
|
||
return None
|
||
if not os.path.isfile(video_path):
|
||
logger.warning(f"NVIDIA 视频文件不存在: {video_path}")
|
||
return None
|
||
|
||
try:
|
||
with open(video_path, 'rb') as f:
|
||
b64 = base64.b64encode(f.read()).decode('utf-8')
|
||
except Exception as e:
|
||
logger.warning(f"NVIDIA 读取视频失败: {e}")
|
||
return None
|
||
|
||
prompt = self._build_video_prompt(known_members_context, event_start_time)
|
||
try:
|
||
resp = self._client.chat.completions.create(
|
||
model=self.model_name,
|
||
messages=[{"role": "user", "content": [
|
||
{"type": "text", "text": prompt},
|
||
{"type": "video_url", "video_url": {
|
||
"url": f"data:video/mp4;base64,{b64}"}}
|
||
]}],
|
||
temperature=0.2,
|
||
max_tokens=4096,
|
||
# NIM 扩展：控制视频采样帧数（模型上限 128 帧）
|
||
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
|
||
timeout=self.timeout
|
||
)
|
||
content = resp.choices[0].message.content
|
||
if not content:
|
||
logger.warning("NVIDIA 视频分析返回空 content")
|
||
self._cb.record_failure()
|
||
return None
|
||
data = self._parse_json(content)
|
||
if not data or 'events' not in data:
|
||
logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}")
|
||
self._cb.record_failure()
|
||
return None
|
||
self._cb.record_success()
|
||
logger.info(f"NVIDIA 整视频分析完成，events={len(data.get('events', []))}")
|
||
return {
|
||
"global_summary": str(data.get('global_summary', '')),
|
||
"events": data.get('events', []),
|
||
"people_mentioned": data.get('people_mentioned', []),
|
||
"compute_provider": "nvidia",
|
||
}
|
||
except Exception as e:
|
||
self._cb.record_failure()
|
||
logger.warning(f"NVIDIA 视频分析异常: {e}")
|
||
return None
|
||
|
||
@staticmethod
|
||
def _parse_json(content: str) -> Optional[dict]:
|
||
content = content.strip()
|
||
try:
|
||
return json.loads(content)
|
||
except json.JSONDecodeError:
|
||
pass
|
||
fence = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL)
|
||
if fence:
|
||
try:
|
||
return json.loads(fence.group(1))
|
||
except json.JSONDecodeError:
|
||
pass
|
||
brace = re.search(r'\{.*\}', content, re.DOTALL)
|
||
if brace:
|
||
try:
|
||
return json.loads(brace.group(0))
|
||
except json.JSONDecodeError:
|
||
pass
|
||
return None
|
||
|
||
def _build_video_prompt(self, known_members: str, event_start_time: str) -> str:
|
||
start_hint = ""
|
||
if event_start_time:
|
||
start_hint = f"\n视频开始时间（北京时间）约为：{event_start_time}。请据此推算每个事件的绝对时间戳。"
|
||
return f"""你是家庭监控视频分析助手。下面是一段完整监控录像（已整段上传）。
|
||
请观看整段视频，提取其中有用的信息，只输出合法 JSON（不要 markdown、不要解释），结构如下：
|
||
|
||
{{
|
||
"global_summary": "整个时段的整体摘要，简体中文，2-4 句",
|
||
"events": [
|
||
{{
|
||
"timestamp": "事件发生时的绝对北京时间(YYYY-MM-DD HH:MM:SS)",
|
||
"description": "该时刻画面/动作信息摘要",
|
||
"people": ["出现在该时刻的人物，用已知成员真名或'人物A'"],
|
||
"is_attention_event": false
|
||
}}
|
||
],
|
||
"people_mentioned": ["本视频出现过的所有人物标识/真名"]
|
||
}}{start_hint}
|
||
|
||
规则：
|
||
1. 只描述客观画面，不猜测。
|
||
2. events 提取有意义的时间点（人物出现/动作变化/异常），timestamp 用绝对北京时间。
|
||
3. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||
{known_members or '（暂无已知成员）'}
|
||
4. is_attention_event：跌倒、危险、异常哭闹等需关注事件（没有则为 false）。"""
|
||
|
||
# ------------------------------------------------------------------
|
||
# 智能问答：纯文本
|
||
# ------------------------------------------------------------------
|
||
def chat(self, prompt: str, max_tokens: int = 2048) -> Optional[str]:
|
||
if self._client is None:
|
||
logger.warning("NVIDIA 客户端未初始化，跳过问答")
|
||
return None
|
||
try:
|
||
resp = self._client.chat.completions.create(
|
||
model=self.model_name,
|
||
messages=[{"role": "user", "content": prompt}],
|
||
temperature=0.3,
|
||
max_tokens=max_tokens,
|
||
timeout=self.timeout
|
||
)
|
||
content = resp.choices[0].message.content
|
||
return content.strip() if content else None
|
||
except Exception as e:
|
||
logger.warning(f"NVIDIA 问答异常: {e}")
|
||
return None
|
||
|
||
def get_timeout(self) -> int:
|
||
return self.timeout
|
||
|
||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||
return self._cb
|