[架构重构] 移除本地Ollama融合，云端直出JSON直存DB，Q&A三模型降级
1. 视频摘要链路：云端VLM直出结构化JSON → Edge format_cloud_result格式化校验 → 直存NAS DB（移除run_text_fusion本地融合） 2. 智能问答链路：Gemini→NVIDIA→Ollama降级，新增chat()纯文本问答方法 3. 适配器重构：base/gemini/nvidia/ollama adapter新增chat()；gemini多图单请求结构化JSON；nvidia逐帧调用聚合 4. 端点变更：/api/edge/chat → /api/edge/chat/ask，调orchestrator.run_qa() 5. chat_handler改经Edge Q&A编排，不再直连Ollama 6. 配置更新：ollama_url → qa_url，Ollama role注释改为Q&A兜底 7. README同步更新架构描述、拓扑图、时序图、模块表
This commit is contained in:
ericwyuan
@@ -1,14 +1,16 @@
|
||||
"""
|
||||
AI-Orchestrator - 多模型并行编排
|
||||
AI-Orchestrator - 多模型编排
|
||||
|
||||
流程:
|
||||
视频分析链路（新框架）:
|
||||
1. 加载所有启用的模型适配器
|
||||
2. 健康检查
|
||||
3. 抽帧 + 关键帧筛选 + 压缩
|
||||
4. 并行调用所有健康模型（ThreadPoolExecutor）
|
||||
5. 文本融合（多模型输出平等交叉验证）
|
||||
6. 回调 NAS
|
||||
7. 清理临时文件
|
||||
4. 云端 VLM 视觉分析（Gemini 主 / NVIDIA 兜底），直出结构化 JSON
|
||||
5. format_cloud_result：对云端结果做**格式化/校验**（无本地模型调用，不汇总摘要）
|
||||
6. 同步返回 NAS → 落库
|
||||
|
||||
智能问答链路（新框架）:
|
||||
- run_qa：Gemini → NVIDIA → 本地 Ollama（仅当两云端都失败才用本地兜底）
|
||||
"""
|
||||
import time
|
||||
import json
|
||||
@@ -16,63 +18,17 @@ import base64
|
||||
import requests
|
||||
from datetime import datetime, timedelta
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeout
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from ..logger import setup_logger, log_task
|
||||
from ..config_loader import load_config
|
||||
from ..model_adapters.adapter_factory import build_adapters
|
||||
from ..model_adapters.base_adapter import BaseModelAdapter
|
||||
from ..video_preprocessor.preprocessor import VideoPreprocessor
|
||||
from .json_parser import parse_vlm_json, VLMOutputInvalidError
|
||||
from .json_parser import VLMOutputInvalidError, validate_schema
|
||||
|
||||
logger = setup_logger('fam-edge.orchestrator')
|
||||
|
||||
FUSION_SYSTEM_PROMPT = """你是一个无情的数据提取器。不要输出任何思考过程，只输出合法 JSON。
|
||||
|
||||
输入参数:
|
||||
- 多模型视觉分析日志（每个模型独立输出，平等对待，交叉验证）:
|
||||
{model_outputs}
|
||||
- 已知成员清单: {known_members}
|
||||
|
||||
执行规则:
|
||||
1. 多个模型的输出平等对待，交叉验证：
|
||||
- 多个模型一致描述的内容 → 可信度高，必须纳入 frame_details，source_providers 列出所有一致的模型
|
||||
- 仅单一模型描述的内容 → 纳入 frame_details，source_providers 仅含该模型
|
||||
- 多个模型冲突时（如人物动作描述不一致）→ 以多数模型一致为准，source_providers 列出多数派模型
|
||||
2. 画面人物按特征匹配已知成员清单：
|
||||
- 匹配到已命名成员（real_name 非空）→ person 字段填 real_name
|
||||
- 匹配到未命名成员（real_name 为空）→ person 字段填 abstract_label
|
||||
- 都不匹配 → 按出现顺序赋予新标识"人物B"、"人物C"...
|
||||
3. 提取每张关键帧对应的时间点、人物、动作、衣着，输出到 frame_details 数组。
|
||||
4. frame_details 每条必须包含 source_providers 数组。
|
||||
5. compute_provider 字段填入本次实际成功调用的所有模型标识数组（去重）。
|
||||
6. 仅输出合法 JSON，不输出任何思考过程、markdown 标记或注释。
|
||||
|
||||
输出 JSON 结构:
|
||||
{{
|
||||
"global_summary": "字符串，整个时段的整体摘要，简体中文",
|
||||
"entities_json": [
|
||||
{{
|
||||
"person": "字符串",
|
||||
"action": "字符串",
|
||||
"clothing": "字符串"
|
||||
}}
|
||||
],
|
||||
"frame_details": [
|
||||
{{
|
||||
"frame_index": "数字",
|
||||
"frame_timestamp": "字符串，ISO 8601 格式时间戳",
|
||||
"person": "字符串",
|
||||
"action": "字符串",
|
||||
"clothing": "字符串",
|
||||
"is_attention_event": "布尔值",
|
||||
"source_providers": "数组"
|
||||
}}
|
||||
],
|
||||
"compute_provider": "数组"
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
class AIOrchestrator:
|
||||
"""AI 编排器"""
|
||||
@@ -96,7 +52,7 @@ class AIOrchestrator:
|
||||
def run_visual_analysis(self, adapters: List[BaseModelAdapter],
|
||||
frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Dict[str, str]:
|
||||
known_members_context: str) -> Dict[str, dict]:
|
||||
"""视觉分析阶段：仅 role=vision 的适配器参与
|
||||
|
||||
orchestrator.mode:
|
||||
@@ -141,7 +97,7 @@ class AIOrchestrator:
|
||||
return model_outputs
|
||||
|
||||
def _run_visual_ensemble(self, vision_adapters, frame_paths,
|
||||
frame_timestamps, known_members_context) -> Dict[str, str]:
|
||||
frame_timestamps, known_members_context) -> Dict[str, dict]:
|
||||
"""并行调用所有健康 vision 模型，保留全部成功结果（交叉验证）"""
|
||||
model_outputs = {}
|
||||
max_timeout = max((a.get_timeout() for a in vision_adapters), default=240)
|
||||
@@ -180,68 +136,115 @@ class AIOrchestrator:
|
||||
adapter.get_circuit_breaker().record_failure()
|
||||
return model_outputs
|
||||
|
||||
def run_text_fusion(self, model_outputs: Dict[str, str],
|
||||
known_members_context: str,
|
||||
task_id: int) -> dict:
|
||||
"""文本融合阶段 - 多模型输出平等交叉验证"""
|
||||
# 构建 model_outputs 文本
|
||||
outputs_text = '\n'.join(
|
||||
f" - {provider} 输出: {output}" for provider, output in model_outputs.items()
|
||||
)
|
||||
def format_cloud_result(self, provider: str, raw_result: dict,
|
||||
known_members_context: str = '',
|
||||
task_id: int = 0) -> dict:
|
||||
"""格式化云端 VLM 直出的结构化结果（**无本地模型调用**）。
|
||||
|
||||
prompt = FUSION_SYSTEM_PROMPT.format(
|
||||
model_outputs=outputs_text,
|
||||
known_members=known_members_context or '（暂无已知成员）'
|
||||
)
|
||||
- 云端模型已产出结构化数据（frame_details / 可选 global_summary / entities_json）
|
||||
- 本方法仅做：字段归一化、source_providers 与 compute_provider 填充、
|
||||
entities 推导、global_summary 缺失时格式化生成
|
||||
- 解析/校验失败抛 VLMOutputInvalidError
|
||||
"""
|
||||
if not isinstance(raw_result, dict):
|
||||
raise VLMOutputInvalidError("云端视觉模型未返回结构化数据(dict)")
|
||||
|
||||
# 调用文本角色模型（role=text，默认 ollama / qwen2.5:7b）做融合
|
||||
text_cfg = next(
|
||||
(cfg for cfg in self.config.get('models', []) if cfg.get('role') == 'text'), None
|
||||
) or next(
|
||||
(cfg for cfg in self.config.get('models', []) if cfg.get('provider') == 'ollama'), None
|
||||
)
|
||||
if not text_cfg:
|
||||
raise VLMOutputInvalidError("没有文本角色模型配置，无法执行文本融合")
|
||||
data = dict(raw_result)
|
||||
frame_details = data.get('frame_details')
|
||||
if not isinstance(frame_details, list) or not frame_details:
|
||||
raise VLMOutputInvalidError("云端结果缺少非空的 frame_details")
|
||||
|
||||
base_url = text_cfg.get('base_url', 'http://localhost:11434')
|
||||
model_name = text_cfg.get('model_name', 'qwen2.5:7b')
|
||||
fusion_timeout = self.timeout_cfg.get('vlm_fusion', 300)
|
||||
num_predict = text_cfg.get('num_predict', 1024)
|
||||
# 归一化每条 frame_detail
|
||||
normalized = []
|
||||
for f in frame_details:
|
||||
if not isinstance(f, dict):
|
||||
continue
|
||||
sp = f.get('source_providers')
|
||||
if not isinstance(sp, list) or not sp:
|
||||
sp = [provider]
|
||||
normalized.append({
|
||||
"frame_index": int(f.get("frame_index", len(normalized) + 1)),
|
||||
"frame_timestamp": str(f.get("frame_timestamp", "")),
|
||||
"person": str(f.get("person", "无人")),
|
||||
"action": str(f.get("action", "")),
|
||||
"clothing": str(f.get("clothing", "")),
|
||||
"is_attention_event": bool(f.get("is_attention_event", False)),
|
||||
"source_providers": [str(p) for p in sp],
|
||||
})
|
||||
if not normalized:
|
||||
raise VLMOutputInvalidError("frame_details 解析后为空")
|
||||
data['frame_details'] = normalized
|
||||
|
||||
start = time.time()
|
||||
resp = requests.post(
|
||||
f"{base_url}/api/generate",
|
||||
json={
|
||||
"model": model_name,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"format": "json",
|
||||
"options": {"temperature": 0.0, "num_predict": num_predict}
|
||||
},
|
||||
timeout=fusion_timeout
|
||||
)
|
||||
# compute_provider：本次实际成功的云端模型
|
||||
data['compute_provider'] = [provider]
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise VLMOutputInvalidError(f"融合阶段 Ollama 调用失败: {resp.status_code}")
|
||||
# entities_json：缺失时由 frame_details 推导（按人物去重）
|
||||
if not data.get('entities_json'):
|
||||
seen = set()
|
||||
ents = []
|
||||
for f in normalized:
|
||||
p = f['person']
|
||||
if p and p != '无人' and p not in seen:
|
||||
seen.add(p)
|
||||
ents.append({
|
||||
"person": p,
|
||||
"action": f['action'],
|
||||
"clothing": f['clothing'],
|
||||
})
|
||||
data['entities_json'] = ents
|
||||
|
||||
raw_output = resp.json().get('response', '')
|
||||
duration_ms = int((time.time() - start) * 1000)
|
||||
log_task(logger, task_id, 'vlm_fusion', f'融合完成，原始输出长度={len(raw_output)}', duration_ms=duration_ms)
|
||||
# global_summary：云端未给则格式化生成（非 LLM 汇总，仅拼接事实）
|
||||
if not data.get('global_summary'):
|
||||
data['global_summary'] = self._build_summary_from_frames(normalized)
|
||||
|
||||
# 解析 JSON（三层容错）
|
||||
result = parse_vlm_json(raw_output)
|
||||
return validate_schema(data)
|
||||
|
||||
# 确保 compute_provider 与实际调用的模型一致
|
||||
result['compute_provider'] = list(model_outputs.keys())
|
||||
def _build_summary_from_frames(self, frame_details: List[dict]) -> str:
|
||||
"""当云端模型未提供 global_summary 时，由 frame_details 格式化生成摘要。
|
||||
注意：这是确定性事实拼接，非 LLM 二次汇总。"""
|
||||
persons = {}
|
||||
has_attention = False
|
||||
for f in frame_details:
|
||||
p = f['person']
|
||||
if p and p != '无人':
|
||||
persons.setdefault(p, set()).add(f['action'])
|
||||
if f.get('is_attention_event'):
|
||||
has_attention = True
|
||||
if not persons:
|
||||
summary = "整个时段内画面中未检测到人物出现，主要为环境静态画面。"
|
||||
else:
|
||||
parts = []
|
||||
for p, acts in persons.items():
|
||||
acts_desc = "、".join(sorted(a for a in acts if a)) or "无明显动作"
|
||||
parts.append(f"{p}（{acts_desc}）")
|
||||
summary = f"时段内检测到：{'；'.join(parts)}。"
|
||||
if has_attention:
|
||||
summary += " ⚠️ 存在需关注的异常事件。"
|
||||
return summary
|
||||
|
||||
# 确保 frame_details 的 source_providers 只包含实际成功的模型
|
||||
valid_providers = set(model_outputs.keys())
|
||||
for frame in result.get('frame_details', []):
|
||||
frame['source_providers'] = [
|
||||
p for p in frame.get('source_providers', []) if p in valid_providers
|
||||
] or list(valid_providers)
|
||||
def run_qa(self, prompt: str, max_tokens: int = 512) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""智能问答编排：Gemini → NVIDIA → 本地 Ollama（仅当两云端都失败才用本地兜底）。
|
||||
|
||||
return result
|
||||
返回 (answer, provider)；全部失败返回 (None, None)。
|
||||
"""
|
||||
qa_order = ['gemini', 'nvidia', 'ollama']
|
||||
for name in qa_order:
|
||||
adapter = next((a for a in self.adapters if a.provider_name == name), None)
|
||||
if adapter is None:
|
||||
logger.warning(f"[qa] 未配置模型 {name}，跳过")
|
||||
continue
|
||||
try:
|
||||
if adapter.get_circuit_breaker().is_open():
|
||||
logger.warning(f"[qa] {name} 熔断器 OPEN，跳过")
|
||||
continue
|
||||
answer = adapter.chat(prompt, max_tokens=max_tokens)
|
||||
if answer:
|
||||
logger.info(f"[qa] 由 {name} 回答（长度={len(answer)}）")
|
||||
return answer, name
|
||||
logger.warning(f"[qa] {name} 返回空")
|
||||
except Exception as e:
|
||||
logger.error(f"[qa] {name} 调用异常: {e}")
|
||||
return None, None
|
||||
|
||||
def send_callback(self, webhook_url: str, task_id: int,
|
||||
result: dict, camera_name: str = '',
|
||||
@@ -340,8 +343,10 @@ class AIOrchestrator:
|
||||
if not model_outputs:
|
||||
raise Exception('All models failed in visual analysis')
|
||||
|
||||
# 4. 文本融合
|
||||
fusion_result = self.run_text_fusion(model_outputs, known_members, task_id)
|
||||
# 4. 云端直出结果格式化（无本地融合）
|
||||
provider = next(iter(model_outputs))
|
||||
fusion_result = self.format_cloud_result(
|
||||
provider, model_outputs[provider], known_members, task_id)
|
||||
|
||||
# 5. 回调
|
||||
# 从视频文件名推断 camera_name
|
||||
@@ -359,7 +364,7 @@ class AIOrchestrator:
|
||||
log_task(logger, task_id, 'overall', f'任务完成', duration_ms=total_ms)
|
||||
|
||||
except VLMOutputInvalidError as e:
|
||||
logger.error(f"[task_id={task_id}] VLM 输出解析失败: {e}")
|
||||
logger.error(f"[task_id={task_id}] 云端结果格式化失败: {e}")
|
||||
self.send_failure_callback(webhook_url, task_id, 'vlm_fusion', str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"[task_id={task_id}] 任务处理失败: {e}", exc_info=True)
|
||||
@@ -429,8 +434,10 @@ class AIOrchestrator:
|
||||
if not model_outputs:
|
||||
raise Exception('All models failed in visual analysis')
|
||||
|
||||
# 4. 文本融合
|
||||
fusion_result = self.run_text_fusion(model_outputs, known_members, task_id)
|
||||
# 4. 云端直出结果格式化（无本地融合）
|
||||
provider = next(iter(model_outputs))
|
||||
fusion_result = self.format_cloud_result(
|
||||
provider, model_outputs[provider], known_members, task_id)
|
||||
|
||||
total_ms = int((time.time() - start_time) * 1000)
|
||||
log_task(logger, task_id, 'overall', '推送任务完成', duration_ms=total_ms)
|
||||
|
||||
@@ -150,7 +150,7 @@ def health():
|
||||
|
||||
@api_bp.route('/api/edge/chat', methods=['POST'])
|
||||
def chat_proxy():
|
||||
"""代理转发至本地 Ollama /api/generate（Ollama 未对外暴露）"""
|
||||
"""代理转发至本地 Ollama /api/generate（兼容旧调用，Ollama 未对外暴露）"""
|
||||
data = request.get_json(silent=True)
|
||||
if not data:
|
||||
return jsonify({"error": "Invalid JSON"}), 400
|
||||
@@ -165,3 +165,26 @@ def chat_proxy():
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Chat proxy error: {e}")
|
||||
return jsonify({"error": f"Ollama unreachable: {e}"}), 502
|
||||
|
||||
|
||||
@api_bp.route('/api/edge/chat/ask', methods=['POST'])
|
||||
def chat_ask():
|
||||
"""智能问答编排：Gemini → NVIDIA → 本地 Ollama（两云端都失败才用本地兜底）
|
||||
|
||||
请求: {"prompt": "..."}
|
||||
响应: {"answer": "...", "provider": "gemini"|"nvidia"|"ollama"}
|
||||
"""
|
||||
data = request.get_json(silent=True)
|
||||
if not data or 'prompt' not in data:
|
||||
return jsonify({"error": "缺少必填字段: prompt"}), 400
|
||||
|
||||
prompt = data['prompt']
|
||||
max_tokens = int(data.get('max_tokens', 512))
|
||||
|
||||
answer, provider = get_orchestrator().run_qa(prompt, max_tokens=max_tokens)
|
||||
if answer is None:
|
||||
return jsonify({
|
||||
"error": "所有模型均不可用（Gemini / NVIDIA / Ollama 全部失败）"
|
||||
}), 503
|
||||
|
||||
return jsonify({"answer": answer, "provider": provider}), 200
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
"""
|
||||
模型适配器基类 - 所有模型适配器的抽象基类
|
||||
|
||||
新增模型只需继承此类并实现 4 个方法:
|
||||
新增模型只需继承此类并实现方法:
|
||||
1. health_check() -> bool
|
||||
2. analyze_frames(frame_paths, frame_timestamps, known_members_context) -> Optional[str]
|
||||
3. get_timeout() -> int
|
||||
4. get_circuit_breaker() -> CircuitBreaker
|
||||
2. analyze_frames(frame_paths, frame_timestamps, known_members_context) -> Optional[dict]
|
||||
- 视觉分析：输入帧图片路径 + 时间戳 + 成员清单，直接输出**结构化结果 dict**
|
||||
(含 frame_details 等，详见 format_cloud_result 约定)。
|
||||
- 失败/超时返回 None。
|
||||
3. chat(prompt) -> Optional[str]
|
||||
- 纯文本问答（智能问答场景），返回文本或 None。
|
||||
- 默认实现抛 NotImplementedError；文本/视觉模型按需实现。
|
||||
4. get_timeout() -> int
|
||||
5. get_circuit_breaker() -> CircuitBreaker
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
class BaseModelAdapter(ABC):
|
||||
@@ -17,7 +23,7 @@ class BaseModelAdapter(ABC):
|
||||
def __init__(self, provider_name: str, config: dict):
|
||||
self.provider_name = provider_name # 如 "ollama", "gemini"
|
||||
self.config = config
|
||||
# 角色: vision=视觉分析, text=文本融合/对话; 默认 vision
|
||||
# 角色: vision=视觉分析, text=智能问答兜底(本地模型); 默认 vision
|
||||
self.role = config.get('role', 'vision')
|
||||
|
||||
def get_role(self) -> str:
|
||||
@@ -32,11 +38,28 @@ class BaseModelAdapter(ABC):
|
||||
@abstractmethod
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
"""视觉分析：输入帧图片路径 + 时间戳 + 成员清单，输出自然语言描述。
|
||||
失败/超时返回 None。"""
|
||||
known_members_context: str) -> Optional[Dict]:
|
||||
"""视觉分析：输入帧图片路径 + 时间戳 + 成员清单，
|
||||
直接输出结构化结果 dict（含 frame_details 等）。失败/超时返回 None。
|
||||
|
||||
约定返回结构（云端模型直出，Edge 仅做格式化校验，不再本地融合）:
|
||||
{
|
||||
"global_summary": "整个时段整体摘要(可选，缺失时由 Edge 格式化生成)",
|
||||
"entities_json": [{"person","action","clothing"}] (可选，缺失时由 frame_details 推导),
|
||||
"frame_details": [
|
||||
{"frame_index":int, "frame_timestamp":str, "person":str,
|
||||
"action":str, "clothing":str, "is_attention_event":bool,
|
||||
"source_providers":[provider]}
|
||||
]
|
||||
}
|
||||
"""
|
||||
pass
|
||||
|
||||
def chat(self, prompt: str, max_tokens: int = 512) -> Optional[str]:
|
||||
"""纯文本问答（智能问答场景）。默认不实现。"""
|
||||
raise NotImplementedError(
|
||||
f"{self.provider_name} 适配器未实现 chat()（不参与智能问答）")
|
||||
|
||||
@abstractmethod
|
||||
def get_timeout(self) -> int:
|
||||
"""该模型的调用超时秒数"""
|
||||
|
||||
@@ -3,31 +3,32 @@ GeminiAdapter - Google Gemini 云端 VLM 适配器
|
||||
|
||||
provider_name = "gemini"
|
||||
模型: gemini-flash-latest (v1beta 下 gemini-1.5-flash 会 404，用 flash-latest 别名)
|
||||
角色: vision (视觉分析)
|
||||
角色: vision (视觉分析直出结构化 JSON) + 智能问答
|
||||
健康检查: GET /v1beta/models?key=...
|
||||
熔断器: 启用
|
||||
逐帧分析: 与 NVIDIA 统一流程，逐帧调用（也规避多图返回不稳定）
|
||||
视觉分析: 多图单请求直出结构化 JSON（global_summary/entities_json/frame_details）
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import requests
|
||||
from typing import List, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
from ..ai_orchestrator.json_parser import parse_vlm_json, VLMOutputInvalidError
|
||||
|
||||
logger = setup_logger('fam-edge.gemini_adapter')
|
||||
|
||||
|
||||
class GeminiAdapter(BaseModelAdapter):
|
||||
"""Gemini 云端 VLM 适配器 (逐帧)"""
|
||||
"""Gemini 云端 VLM 适配器 (视觉直出结构化 JSON + 文本问答)"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("gemini", config)
|
||||
self.model_name = config.get('model_name', 'gemini-flash-latest')
|
||||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||||
self.timeout = config.get('timeout', 15)
|
||||
self.timeout = config.get('timeout', 30)
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 3),
|
||||
@@ -62,61 +63,143 @@ class GeminiAdapter(BaseModelAdapter):
|
||||
logger.error(f"Gemini 健康检查异常: {e}")
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 视觉分析：多图单请求，直出结构化 JSON
|
||||
# ------------------------------------------------------------------
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
known_members_context: str) -> Optional[Dict]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("Gemini 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
if not self.api_key:
|
||||
logger.warning("Gemini API Key 未配置，跳过调用")
|
||||
return None
|
||||
|
||||
results = []
|
||||
for path, ts in zip(frame_paths, frame_timestamps):
|
||||
desc = self._analyze_one(path, ts, known_members_context)
|
||||
if desc:
|
||||
results.append(f"[帧] 时间: {ts}\n{desc}")
|
||||
|
||||
if not results:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._cb.record_success()
|
||||
logger.info(f"Gemini 视觉分析完成，{len(results)} 帧有描述")
|
||||
return "\n".join(results)
|
||||
|
||||
def _analyze_one(self, path: str, ts: str,
|
||||
known_members: str) -> Optional[str]:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
img = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
if not frame_paths:
|
||||
logger.warning("Gemini 无帧可分析")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(ts, known_members)
|
||||
parts = []
|
||||
ts_map = {}
|
||||
for i, (path, ts) in enumerate(zip(frame_paths, frame_timestamps), 1):
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
img = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
continue
|
||||
parts.append({"inline_data": {"mime_type": "image/jpeg", "data": img}})
|
||||
parts.append({"text": f"[图片{i}] 时间: {ts}"})
|
||||
ts_map[i] = ts
|
||||
|
||||
if not parts:
|
||||
return None
|
||||
|
||||
parts.insert(0, {"text": self._build_structured_prompt(known_members_context)})
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/models/{self.model_name}:generateContent?key={self.api_key}",
|
||||
json={"contents": [{"parts": [
|
||||
{"text": prompt},
|
||||
{"inline_data": {"mime_type": "image/jpeg", "data": img}}
|
||||
]}], "generationConfig": {"temperature": 0.2, "maxOutputTokens": 300}},
|
||||
json={"contents": [{"parts": parts}],
|
||||
"generationConfig": {"temperature": 0.2, "maxOutputTokens": 2048}},
|
||||
timeout=self.timeout
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
cands = resp.json().get('candidates', [])
|
||||
if cands:
|
||||
parts = cands[0].get('content', {}).get('parts', [])
|
||||
text = ''.join(p.get('text', '') for p in parts).strip()
|
||||
return text or None
|
||||
logger.warning("Gemini 返回空 candidates")
|
||||
text = ''.join(
|
||||
p.get('text', '')
|
||||
for p in cands[0].get('content', {}).get('parts', [])
|
||||
).strip()
|
||||
if not text:
|
||||
logger.warning("Gemini 返回空文本")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
try:
|
||||
result = parse_vlm_json(text)
|
||||
# 确保 frame_details 的 frame_timestamp 与标注一致
|
||||
for f in result.get('frame_details', []):
|
||||
idx = f.get('frame_index')
|
||||
if isinstance(idx, int) and idx in ts_map and not f.get('frame_timestamp'):
|
||||
f['frame_timestamp'] = ts_map[idx]
|
||||
for f in result.get('frame_details', []):
|
||||
if 'source_providers' not in f or not f.get('source_providers'):
|
||||
f['source_providers'] = ['gemini']
|
||||
self._cb.record_success()
|
||||
logger.info(f"Gemini 视觉分析完成，frame_details={len(result.get('frame_details', []))}")
|
||||
return result
|
||||
except VLMOutputInvalidError as e:
|
||||
logger.error(f"Gemini 输出无法解析为 JSON: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
else:
|
||||
logger.warning(f"Gemini 单帧失败 HTTP {resp.status_code}: {resp.text[:150]}")
|
||||
logger.warning(f"Gemini 视觉分析 HTTP {resp.status_code}: {resp.text[:150]}")
|
||||
self._cb.record_failure()
|
||||
except requests.Timeout:
|
||||
logger.warning(f"Gemini 单帧超时 ({self.timeout}s)")
|
||||
logger.warning(f"Gemini 视觉分析超时 ({self.timeout}s)")
|
||||
self._cb.record_failure()
|
||||
except Exception as e:
|
||||
logger.error(f"Gemini 单帧异常: {e}")
|
||||
logger.error(f"Gemini 视觉分析异常: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
def _build_structured_prompt(self, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。下面按时间顺序排列了多张监控截图。
|
||||
请分析整个时段，只输出合法 JSON（不要 markdown、不要任何解释文字），结构如下：
|
||||
|
||||
{{
|
||||
"global_summary": "整个时段的整体摘要，简体中文，2-4 句，客观描述人物与主要活动",
|
||||
"entities_json": [
|
||||
{{"person": "人物标识(匹配已知成员用真名，否则用'人物A'/'人物B'...)", "action": "主要动作", "clothing": "衣着"}}
|
||||
],
|
||||
"frame_details": [
|
||||
{{
|
||||
"frame_index": 图片序号(从1开始，与[图片N]标注对应),
|
||||
"frame_timestamp": "该帧的时间戳(用[图片N]标注里的时间)",
|
||||
"person": "该帧画面中的人物或'无人'",
|
||||
"action": "该帧可见动作",
|
||||
"clothing": "该帧衣着(颜色+类型)",
|
||||
"is_attention_event": false,
|
||||
"source_providers": ["gemini"]
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
规则：
|
||||
1. 只描述客观画面，不要猜测或想象。
|
||||
2. frame_details 每帧一条，frame_index 与上方[图片N]序号对应，frame_timestamp 用标注时间。
|
||||
3. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
4. is_attention_event：是否为跌倒、危险、异常哭闹等需关注事件（没有则为 false）。
|
||||
5. 没有人物出现的帧 person 填"无人"，action 填""。"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 智能问答：纯文本
|
||||
# ------------------------------------------------------------------
|
||||
def chat(self, prompt: str, max_tokens: int = 512) -> Optional[str]:
|
||||
if not self.api_key:
|
||||
logger.warning("Gemini API Key 未配置，跳过问答")
|
||||
return None
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/models/{self.model_name}:generateContent?key={self.api_key}",
|
||||
json={"contents": [{"parts": [{"text": prompt}]}],
|
||||
"generationConfig": {"temperature": 0.3, "maxOutputTokens": max_tokens}},
|
||||
timeout=self.timeout
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
cands = resp.json().get('candidates', [])
|
||||
if cands:
|
||||
text = ''.join(
|
||||
p.get('text', '')
|
||||
for p in cands[0].get('content', {}).get('parts', [])
|
||||
).strip()
|
||||
return text or None
|
||||
logger.warning(f"Gemini 问答 HTTP {resp.status_code}")
|
||||
except requests.Timeout:
|
||||
logger.warning(f"Gemini 问答超时 ({self.timeout}s)")
|
||||
except Exception as e:
|
||||
logger.error(f"Gemini 问答异常: {e}")
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
@@ -124,16 +207,3 @@ class GeminiAdapter(BaseModelAdapter):
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_prompt(self, ts: str, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}），客观描述画面内容，不要猜测。
|
||||
|
||||
需报告：
|
||||
1. 人物：数量、衣着（颜色+类型）、可见动作
|
||||
2. 物品：玩具、奶瓶、家具等显眼物体
|
||||
3. 互动：人与人或人与物体的互动
|
||||
|
||||
已知家庭成员（按特征匹配，匹配到用真名，否则用"人物X"）：
|
||||
{known_members or '（暂无）'}
|
||||
|
||||
要求简洁客观，不要输出 JSON 或 markdown。"""
|
||||
|
||||
@@ -3,18 +3,19 @@ NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器
|
||||
|
||||
provider_name = "nvidia"
|
||||
模型: meta/llama-3.2-11b-vision-instruct
|
||||
角色: vision (视觉分析)
|
||||
角色: vision (视觉分析直出结构化 JSON) + 智能问答
|
||||
SDK: openai (NIM 兼容 OpenAI API 规范)
|
||||
限制: NIM 单次请求最多 1 张图 -> 适配器内部逐帧调用
|
||||
限制: NIM 单次请求最多 1 张图 -> 适配器内部逐帧调用，再聚合成 frame_details
|
||||
熔断器: 启用
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
from typing import List, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
from ..ai_orchestrator.json_parser import parse_vlm_json, VLMOutputInvalidError
|
||||
|
||||
logger = setup_logger('fam-edge.nvidia_adapter')
|
||||
|
||||
@@ -25,7 +26,7 @@ except ImportError:
|
||||
|
||||
|
||||
class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (逐帧)"""
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (逐帧结构化 + 聚合; 文本问答)"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("nvidia", config)
|
||||
@@ -64,31 +65,41 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
logger.warning(f"NVIDIA 健康检查失败: {e}")
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 视觉分析：逐帧调用（NIM 限 1 图/请求），聚合为 frame_details
|
||||
# ------------------------------------------------------------------
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
known_members_context: str) -> Optional[Dict]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("NVIDIA 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化，跳过调用")
|
||||
return None
|
||||
if not frame_paths:
|
||||
logger.warning("NVIDIA 无帧可分析")
|
||||
return None
|
||||
|
||||
results = []
|
||||
for path, ts in zip(frame_paths, frame_timestamps):
|
||||
desc = self._analyze_one(path, ts, known_members_context)
|
||||
if desc:
|
||||
results.append(f"[帧] 时间: {ts}\n{desc}")
|
||||
frame_details = []
|
||||
ok = False
|
||||
for i, (path, ts) in enumerate(zip(frame_paths, frame_timestamps), 1):
|
||||
detail = self._analyze_one_structured(path, ts, i, known_members_context)
|
||||
if detail:
|
||||
frame_details.append(detail)
|
||||
ok = True
|
||||
|
||||
if not results:
|
||||
if not ok:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视觉分析完成，{len(results)} 帧有描述")
|
||||
return "\n".join(results)
|
||||
|
||||
def _analyze_one(self, path: str, ts: str,
|
||||
known_members: str) -> Optional[str]:
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视觉分析完成，frame_details={len(frame_details)}")
|
||||
# NVIDIA 单帧无法跨帧综合 global_summary，交由 Edge format_cloud_result 格式化生成
|
||||
return {"frame_details": frame_details}
|
||||
|
||||
def _analyze_one_structured(self, path: str, ts: str, idx: int,
|
||||
known_members: str) -> Optional[Dict]:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('utf-8')
|
||||
@@ -96,7 +107,7 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(ts, known_members)
|
||||
prompt = self._build_structured_prompt(ts, idx, known_members)
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
@@ -109,26 +120,69 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
return content.strip() if content else None
|
||||
if not content:
|
||||
return None
|
||||
try:
|
||||
data = parse_vlm_json(content)
|
||||
except VLMOutputInvalidError:
|
||||
logger.warning(f"NVIDIA 单帧 JSON 解析失败: {content[:120]}")
|
||||
return None
|
||||
# 组装统一字段
|
||||
return {
|
||||
"frame_index": idx,
|
||||
"frame_timestamp": str(data.get("frame_timestamp", ts)),
|
||||
"person": str(data.get("person", "无人")),
|
||||
"action": str(data.get("action", "")),
|
||||
"clothing": str(data.get("clothing", "")),
|
||||
"is_attention_event": bool(data.get("is_attention_event", False)),
|
||||
"source_providers": ["nvidia"],
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 单帧异常: {e}")
|
||||
return None
|
||||
|
||||
def _build_structured_prompt(self, ts: str, idx: int, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}）。
|
||||
只输出合法 JSON（不要 markdown、不要解释），结构如下：
|
||||
|
||||
{{
|
||||
"frame_timestamp": "{ts}",
|
||||
"person": "该帧画面中的人物或'无人'",
|
||||
"action": "该帧可见动作",
|
||||
"clothing": "该帧衣着(颜色+类型)",
|
||||
"is_attention_event": false
|
||||
}}
|
||||
|
||||
规则：
|
||||
1. 只描述客观画面，不猜测。
|
||||
2. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
3. is_attention_event：是否为跌倒、危险、异常哭闹等需关注事件（没有则为 false）。
|
||||
4. 没有人物出现的帧 person 填"无人"，action 填""。"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 智能问答：纯文本
|
||||
# ------------------------------------------------------------------
|
||||
def chat(self, prompt: str, max_tokens: int = 512) -> Optional[str]:
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化，跳过问答")
|
||||
return None
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.3,
|
||||
max_tokens=max_tokens,
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
return content.strip() if content else None
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 问答异常: {e}")
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
return self.timeout
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_prompt(self, ts: str, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}），客观描述画面内容，不要猜测。
|
||||
|
||||
需报告：
|
||||
1. 人物：数量、衣着（颜色+类型）、可见动作
|
||||
2. 物品：玩具、奶瓶、家具等显眼物体
|
||||
3. 互动：人与人或人与物体的互动
|
||||
|
||||
已知家庭成员（按特征匹配，匹配到用真名，否则用"人物X"）：
|
||||
{known_members or '（暂无）'}
|
||||
|
||||
要求简洁客观，不要输出 JSON 或 markdown。"""
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""
|
||||
OllamaAdapter - 本地 VLM 模型适配器
|
||||
OllamaAdapter - 本地模型适配器（仅智能问答兜底）
|
||||
|
||||
provider_name = "ollama"
|
||||
模型: llava-phi3
|
||||
模型: qwen2.5:7b（纯文本）
|
||||
角色: text（智能问答兜底；Gemini 与 NVIDIA 均失败时启用）
|
||||
健康检查: GET /api/tags
|
||||
不参与视觉分析、不参与视频结构化输出（云端 VLM 直出）
|
||||
"""
|
||||
import base64
|
||||
import requests
|
||||
@@ -115,6 +117,41 @@ class OllamaAdapter(BaseModelAdapter):
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 智能问答：纯文本（本地模型，仅作 Gemini/NVIDIA 全失败时的兜底）
|
||||
# ------------------------------------------------------------------
|
||||
def chat(self, prompt: str, max_tokens: int = 512) -> Optional[str]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("Ollama 熔断器 OPEN，跳过问答")
|
||||
return None
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self.base_url}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.3, "num_predict": max_tokens}
|
||||
},
|
||||
timeout=self.timeout
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
output = resp.json().get('response', '').strip()
|
||||
if output:
|
||||
self._cb.record_success()
|
||||
return output
|
||||
self._cb.record_failure()
|
||||
else:
|
||||
logger.error(f"Ollama 问答失败: {resp.status_code} {resp.text[:200]}")
|
||||
self._cb.record_failure()
|
||||
except requests.Timeout:
|
||||
logger.error(f"Ollama 问答超时 ({self.timeout}s)")
|
||||
self._cb.record_failure()
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama 问答异常: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
def _build_visual_prompt(self, n: int, timestamps: List[str], known_members: str) -> str:
|
||||
"""构建视觉分析 Prompt"""
|
||||
ts_lines = '\n'.join(
|
||||
|
||||
Reference in New Issue
Block a user