[架构重构] 移除本地Ollama融合，云端直出JSON直存DB，Q&A三模型降级
1. 视频摘要链路：云端VLM直出结构化JSON → Edge format_cloud_result格式化校验 → 直存NAS DB（移除run_text_fusion本地融合） 2. 智能问答链路：Gemini→NVIDIA→Ollama降级，新增chat()纯文本问答方法 3. 适配器重构：base/gemini/nvidia/ollama adapter新增chat()；gemini多图单请求结构化JSON；nvidia逐帧调用聚合 4. 端点变更：/api/edge/chat → /api/edge/chat/ask，调orchestrator.run_qa() 5. chat_handler改经Edge Q&A编排，不再直连Ollama 6. 配置更新：ollama_url → qa_url，Ollama role注释改为Q&A兜底 7. README同步更新架构描述、拓扑图、时序图、模块表
This commit is contained in:
ericwyuan
@@ -3,18 +3,19 @@ NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器
|
||||
|
||||
provider_name = "nvidia"
|
||||
模型: meta/llama-3.2-11b-vision-instruct
|
||||
角色: vision (视觉分析)
|
||||
角色: vision (视觉分析直出结构化 JSON) + 智能问答
|
||||
SDK: openai (NIM 兼容 OpenAI API 规范)
|
||||
限制: NIM 单次请求最多 1 张图 -> 适配器内部逐帧调用
|
||||
限制: NIM 单次请求最多 1 张图 -> 适配器内部逐帧调用，再聚合成 frame_details
|
||||
熔断器: 启用
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
from typing import List, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
from ..ai_orchestrator.json_parser import parse_vlm_json, VLMOutputInvalidError
|
||||
|
||||
logger = setup_logger('fam-edge.nvidia_adapter')
|
||||
|
||||
@@ -25,7 +26,7 @@ except ImportError:
|
||||
|
||||
|
||||
class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (逐帧)"""
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (逐帧结构化 + 聚合; 文本问答)"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("nvidia", config)
|
||||
@@ -64,31 +65,41 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
logger.warning(f"NVIDIA 健康检查失败: {e}")
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 视觉分析：逐帧调用（NIM 限 1 图/请求），聚合为 frame_details
|
||||
# ------------------------------------------------------------------
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
known_members_context: str) -> Optional[Dict]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("NVIDIA 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化，跳过调用")
|
||||
return None
|
||||
if not frame_paths:
|
||||
logger.warning("NVIDIA 无帧可分析")
|
||||
return None
|
||||
|
||||
results = []
|
||||
for path, ts in zip(frame_paths, frame_timestamps):
|
||||
desc = self._analyze_one(path, ts, known_members_context)
|
||||
if desc:
|
||||
results.append(f"[帧] 时间: {ts}\n{desc}")
|
||||
frame_details = []
|
||||
ok = False
|
||||
for i, (path, ts) in enumerate(zip(frame_paths, frame_timestamps), 1):
|
||||
detail = self._analyze_one_structured(path, ts, i, known_members_context)
|
||||
if detail:
|
||||
frame_details.append(detail)
|
||||
ok = True
|
||||
|
||||
if not results:
|
||||
if not ok:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视觉分析完成，{len(results)} 帧有描述")
|
||||
return "\n".join(results)
|
||||
|
||||
def _analyze_one(self, path: str, ts: str,
|
||||
known_members: str) -> Optional[str]:
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视觉分析完成，frame_details={len(frame_details)}")
|
||||
# NVIDIA 单帧无法跨帧综合 global_summary，交由 Edge format_cloud_result 格式化生成
|
||||
return {"frame_details": frame_details}
|
||||
|
||||
def _analyze_one_structured(self, path: str, ts: str, idx: int,
|
||||
known_members: str) -> Optional[Dict]:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('utf-8')
|
||||
@@ -96,7 +107,7 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(ts, known_members)
|
||||
prompt = self._build_structured_prompt(ts, idx, known_members)
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
@@ -109,26 +120,69 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
return content.strip() if content else None
|
||||
if not content:
|
||||
return None
|
||||
try:
|
||||
data = parse_vlm_json(content)
|
||||
except VLMOutputInvalidError:
|
||||
logger.warning(f"NVIDIA 单帧 JSON 解析失败: {content[:120]}")
|
||||
return None
|
||||
# 组装统一字段
|
||||
return {
|
||||
"frame_index": idx,
|
||||
"frame_timestamp": str(data.get("frame_timestamp", ts)),
|
||||
"person": str(data.get("person", "无人")),
|
||||
"action": str(data.get("action", "")),
|
||||
"clothing": str(data.get("clothing", "")),
|
||||
"is_attention_event": bool(data.get("is_attention_event", False)),
|
||||
"source_providers": ["nvidia"],
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 单帧异常: {e}")
|
||||
return None
|
||||
|
||||
def _build_structured_prompt(self, ts: str, idx: int, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}）。
|
||||
只输出合法 JSON（不要 markdown、不要解释），结构如下：
|
||||
|
||||
{{
|
||||
"frame_timestamp": "{ts}",
|
||||
"person": "该帧画面中的人物或'无人'",
|
||||
"action": "该帧可见动作",
|
||||
"clothing": "该帧衣着(颜色+类型)",
|
||||
"is_attention_event": false
|
||||
}}
|
||||
|
||||
规则：
|
||||
1. 只描述客观画面，不猜测。
|
||||
2. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
3. is_attention_event：是否为跌倒、危险、异常哭闹等需关注事件（没有则为 false）。
|
||||
4. 没有人物出现的帧 person 填"无人"，action 填""。"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 智能问答：纯文本
|
||||
# ------------------------------------------------------------------
|
||||
def chat(self, prompt: str, max_tokens: int = 512) -> Optional[str]:
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化，跳过问答")
|
||||
return None
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.3,
|
||||
max_tokens=max_tokens,
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
return content.strip() if content else None
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 问答异常: {e}")
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
return self.timeout
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_prompt(self, ts: str, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}），客观描述画面内容，不要猜测。
|
||||
|
||||
需报告：
|
||||
1. 人物：数量、衣着（颜色+类型）、可见动作
|
||||
2. 物品：玩具、奶瓶、家具等显眼物体
|
||||
3. 互动：人与人或人与物体的互动
|
||||
|
||||
已知家庭成员（按特征匹配，匹配到用真名，否则用"人物X"）：
|
||||
{known_members or '（暂无）'}
|
||||
|
||||
要求简洁客观，不要输出 JSON 或 markdown。"""
|
||||
|
||||
Reference in New Issue
Block a user