feat(adapter): 云端视觉适配器 + role 角色区分

- base_adapter 增加 role 字段(vision/text)与 get_role()
- gemini_adapter 修复 v1beta 下模型名 404(gemini-1.5-flash→gemini-flash-latest), 改逐帧调用
- 新增 nvidia_adapter(openai SDK, 规避 NIM 单次限 1 图逐帧), 注册 adapter_factory
- 视觉分析仅 vision 角色参与, 文本融合交给 role=text 模型
This commit is contained in:
ericwyuan
2026-08-20 09:11:44 +08:00
parent 0af541097f
commit 99d75a4bda
4 changed files with 211 additions and 89 deletions

View File

@@ -0,0 +1,134 @@
"""
NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器
provider_name = "nvidia"
模型: meta/llama-3.2-11b-vision-instruct
角色: vision (视觉分析)
SDK: openai (NIM 兼容 OpenAI API 规范)
限制: NIM 单次请求最多 1 张图 -> 适配器内部逐帧调用
熔断器: 启用
"""
import os
import base64
from typing import List, Optional
from .base_adapter import BaseModelAdapter
from .circuit_breaker import CircuitBreaker
from ..logger import setup_logger
logger = setup_logger('fam-edge.nvidia_adapter')
try:
from openai import OpenAI
except ImportError:
OpenAI = None
class NvidiaVisionAdapter(BaseModelAdapter):
"""NVIDIA NIM 云端 VLM 适配器 (逐帧)"""
def __init__(self, config: dict):
super().__init__("nvidia", config)
self.model_name = config.get('model_name', 'meta/llama-3.2-11b-vision-instruct')
self.api_key = self._resolve_key(config.get('api_key', ''))
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
self.timeout = config.get('timeout', 20)
cb_cfg = config.get('circuit_breaker', {})
self._cb = CircuitBreaker(
threshold=cb_cfg.get('threshold', 3),
cooldown=cb_cfg.get('cooldown', 600),
enabled=cb_cfg.get('enabled', True)
)
self._client = None
if OpenAI is not None and self.api_key:
try:
self._client = OpenAI(base_url=self.base_url, api_key=self.api_key)
except Exception as e:
logger.error(f"NVIDIA OpenAI 客户端初始化失败: {e}")
self._client = None
def _resolve_key(self, raw: str) -> str:
if raw.startswith('${') and raw.endswith('}'):
return os.environ.get(raw[2:-1], '')
return raw
def health_check(self) -> bool:
if self._client is None:
logger.warning("NVIDIA OpenAI SDK 未就绪或 Key 未配置,健康检查失败")
return False
try:
self._client.models.list()
logger.info("NVIDIA 健康检查通过")
return True
except Exception as e:
logger.warning(f"NVIDIA 健康检查失败: {e}")
return False
def analyze_frames(self, frame_paths: List[str],
frame_timestamps: List[str],
known_members_context: str) -> Optional[str]:
if self._cb.is_open():
logger.warning("NVIDIA 熔断器 OPEN跳过调用")
return None
if self._client is None:
logger.warning("NVIDIA 客户端未初始化,跳过调用")
return None
results = []
for path, ts in zip(frame_paths, frame_timestamps):
desc = self._analyze_one(path, ts, known_members_context)
if desc:
results.append(f"[帧] 时间: {ts}\n{desc}")
if not results:
self._cb.record_failure()
return None
self._cb.record_success()
logger.info(f"NVIDIA 视觉分析完成,{len(results)} 帧有描述")
return "\n".join(results)
def _analyze_one(self, path: str, ts: str,
known_members: str) -> Optional[str]:
try:
with open(path, 'rb') as f:
b64 = base64.b64encode(f.read()).decode('utf-8')
except Exception as e:
logger.error(f"读取图片失败 {path}: {e}")
return None
prompt = self._build_prompt(ts, known_members)
try:
resp = self._client.chat.completions.create(
model=self.model_name,
messages=[{"role": "user", "content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
]}],
temperature=0.2,
max_tokens=512,
timeout=self.timeout
)
content = resp.choices[0].message.content
return content.strip() if content else None
except Exception as e:
logger.warning(f"NVIDIA 单帧异常: {e}")
return None
def get_timeout(self) -> int:
return self.timeout
def get_circuit_breaker(self) -> CircuitBreaker:
return self._cb
def _build_prompt(self, ts: str, known_members: str) -> str:
return f"""你是家庭监控视频分析助手。请看这张监控截图(拍摄时间 {ts}),客观描述画面内容,不要猜测。
需报告:
1. 人物:数量、衣着(颜色+类型)、可见动作
2. 物品:玩具、奶瓶、家具等显眼物体
3. 互动:人与人或人与物体的互动
已知家庭成员(按特征匹配,匹配到用真名,否则用"人物X"
{known_members or '(暂无)'}
要求简洁客观,不要输出 JSON 或 markdown。"""