feat(adapter): 云端视觉适配器 + role 角色区分
- base_adapter 增加 role 字段(vision/text)与 get_role() - gemini_adapter 修复 v1beta 下模型名 404(gemini-1.5-flash→gemini-flash-latest), 改逐帧调用 - 新增 nvidia_adapter(openai SDK, 规避 NIM 单次限 1 图逐帧), 注册 adapter_factory - 视觉分析仅 vision 角色参与, 文本融合交给 role=text 模型
This commit is contained in:
134
fam-edge/src/fam_edge/model_adapters/nvidia_adapter.py
Normal file
134
fam-edge/src/fam_edge/model_adapters/nvidia_adapter.py
Normal file
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器
|
||||
|
||||
provider_name = "nvidia"
|
||||
模型: meta/llama-3.2-11b-vision-instruct
|
||||
角色: vision (视觉分析)
|
||||
SDK: openai (NIM 兼容 OpenAI API 规范)
|
||||
限制: NIM 单次请求最多 1 张图 -> 适配器内部逐帧调用
|
||||
熔断器: 启用
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
from typing import List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
|
||||
logger = setup_logger('fam-edge.nvidia_adapter')
|
||||
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
OpenAI = None
|
||||
|
||||
|
||||
class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (逐帧)"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("nvidia", config)
|
||||
self.model_name = config.get('model_name', 'meta/llama-3.2-11b-vision-instruct')
|
||||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||||
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
|
||||
self.timeout = config.get('timeout', 20)
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 3),
|
||||
cooldown=cb_cfg.get('cooldown', 600),
|
||||
enabled=cb_cfg.get('enabled', True)
|
||||
)
|
||||
self._client = None
|
||||
if OpenAI is not None and self.api_key:
|
||||
try:
|
||||
self._client = OpenAI(base_url=self.base_url, api_key=self.api_key)
|
||||
except Exception as e:
|
||||
logger.error(f"NVIDIA OpenAI 客户端初始化失败: {e}")
|
||||
self._client = None
|
||||
|
||||
def _resolve_key(self, raw: str) -> str:
|
||||
if raw.startswith('${') and raw.endswith('}'):
|
||||
return os.environ.get(raw[2:-1], '')
|
||||
return raw
|
||||
|
||||
def health_check(self) -> bool:
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA OpenAI SDK 未就绪或 Key 未配置,健康检查失败")
|
||||
return False
|
||||
try:
|
||||
self._client.models.list()
|
||||
logger.info("NVIDIA 健康检查通过")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 健康检查失败: {e}")
|
||||
return False
|
||||
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("NVIDIA 熔断器 OPEN,跳过调用")
|
||||
return None
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化,跳过调用")
|
||||
return None
|
||||
|
||||
results = []
|
||||
for path, ts in zip(frame_paths, frame_timestamps):
|
||||
desc = self._analyze_one(path, ts, known_members_context)
|
||||
if desc:
|
||||
results.append(f"[帧] 时间: {ts}\n{desc}")
|
||||
|
||||
if not results:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视觉分析完成,{len(results)} 帧有描述")
|
||||
return "\n".join(results)
|
||||
|
||||
def _analyze_one(self, path: str, ts: str,
|
||||
known_members: str) -> Optional[str]:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(ts, known_members)
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
|
||||
]}],
|
||||
temperature=0.2,
|
||||
max_tokens=512,
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
return content.strip() if content else None
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 单帧异常: {e}")
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
return self.timeout
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_prompt(self, ts: str, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图(拍摄时间 {ts}),客观描述画面内容,不要猜测。
|
||||
|
||||
需报告:
|
||||
1. 人物:数量、衣着(颜色+类型)、可见动作
|
||||
2. 物品:玩具、奶瓶、家具等显眼物体
|
||||
3. 互动:人与人或人与物体的互动
|
||||
|
||||
已知家庭成员(按特征匹配,匹配到用真名,否则用"人物X"):
|
||||
{known_members or '(暂无)'}
|
||||
|
||||
要求简洁客观,不要输出 JSON 或 markdown。"""
|
||||
Reference in New Issue
Block a user