[架构重构] 移除本地Ollama融合，云端直出JSON直存DB，Q&A三模型降级
1. 视频摘要链路：云端VLM直出结构化JSON → Edge format_cloud_result格式化校验 → 直存NAS DB（移除run_text_fusion本地融合） 2. 智能问答链路：Gemini→NVIDIA→Ollama降级，新增chat()纯文本问答方法 3. 适配器重构：base/gemini/nvidia/ollama adapter新增chat()；gemini多图单请求结构化JSON；nvidia逐帧调用聚合 4. 端点变更：/api/edge/chat → /api/edge/chat/ask，调orchestrator.run_qa() 5. chat_handler改经Edge Q&A编排，不再直连Ollama 6. 配置更新：ollama_url → qa_url，Ollama role注释改为Q&A兜底 7. README同步更新架构描述、拓扑图、时序图、模块表
This commit is contained in:
ericwyuan
@@ -3,31 +3,32 @@ GeminiAdapter - Google Gemini 云端 VLM 适配器
|
||||
|
||||
provider_name = "gemini"
|
||||
模型: gemini-flash-latest (v1beta 下 gemini-1.5-flash 会 404，用 flash-latest 别名)
|
||||
角色: vision (视觉分析)
|
||||
角色: vision (视觉分析直出结构化 JSON) + 智能问答
|
||||
健康检查: GET /v1beta/models?key=...
|
||||
熔断器: 启用
|
||||
逐帧分析: 与 NVIDIA 统一流程，逐帧调用（也规避多图返回不稳定）
|
||||
视觉分析: 多图单请求直出结构化 JSON（global_summary/entities_json/frame_details）
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import requests
|
||||
from typing import List, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
from ..ai_orchestrator.json_parser import parse_vlm_json, VLMOutputInvalidError
|
||||
|
||||
logger = setup_logger('fam-edge.gemini_adapter')
|
||||
|
||||
|
||||
class GeminiAdapter(BaseModelAdapter):
|
||||
"""Gemini 云端 VLM 适配器 (逐帧)"""
|
||||
"""Gemini 云端 VLM 适配器 (视觉直出结构化 JSON + 文本问答)"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("gemini", config)
|
||||
self.model_name = config.get('model_name', 'gemini-flash-latest')
|
||||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||||
self.timeout = config.get('timeout', 15)
|
||||
self.timeout = config.get('timeout', 30)
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 3),
|
||||
@@ -62,61 +63,143 @@ class GeminiAdapter(BaseModelAdapter):
|
||||
logger.error(f"Gemini 健康检查异常: {e}")
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 视觉分析：多图单请求，直出结构化 JSON
|
||||
# ------------------------------------------------------------------
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
known_members_context: str) -> Optional[Dict]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("Gemini 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
if not self.api_key:
|
||||
logger.warning("Gemini API Key 未配置，跳过调用")
|
||||
return None
|
||||
|
||||
results = []
|
||||
for path, ts in zip(frame_paths, frame_timestamps):
|
||||
desc = self._analyze_one(path, ts, known_members_context)
|
||||
if desc:
|
||||
results.append(f"[帧] 时间: {ts}\n{desc}")
|
||||
|
||||
if not results:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._cb.record_success()
|
||||
logger.info(f"Gemini 视觉分析完成，{len(results)} 帧有描述")
|
||||
return "\n".join(results)
|
||||
|
||||
def _analyze_one(self, path: str, ts: str,
|
||||
known_members: str) -> Optional[str]:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
img = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
if not frame_paths:
|
||||
logger.warning("Gemini 无帧可分析")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(ts, known_members)
|
||||
parts = []
|
||||
ts_map = {}
|
||||
for i, (path, ts) in enumerate(zip(frame_paths, frame_timestamps), 1):
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
img = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
continue
|
||||
parts.append({"inline_data": {"mime_type": "image/jpeg", "data": img}})
|
||||
parts.append({"text": f"[图片{i}] 时间: {ts}"})
|
||||
ts_map[i] = ts
|
||||
|
||||
if not parts:
|
||||
return None
|
||||
|
||||
parts.insert(0, {"text": self._build_structured_prompt(known_members_context)})
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/models/{self.model_name}:generateContent?key={self.api_key}",
|
||||
json={"contents": [{"parts": [
|
||||
{"text": prompt},
|
||||
{"inline_data": {"mime_type": "image/jpeg", "data": img}}
|
||||
]}], "generationConfig": {"temperature": 0.2, "maxOutputTokens": 300}},
|
||||
json={"contents": [{"parts": parts}],
|
||||
"generationConfig": {"temperature": 0.2, "maxOutputTokens": 2048}},
|
||||
timeout=self.timeout
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
cands = resp.json().get('candidates', [])
|
||||
if cands:
|
||||
parts = cands[0].get('content', {}).get('parts', [])
|
||||
text = ''.join(p.get('text', '') for p in parts).strip()
|
||||
return text or None
|
||||
logger.warning("Gemini 返回空 candidates")
|
||||
text = ''.join(
|
||||
p.get('text', '')
|
||||
for p in cands[0].get('content', {}).get('parts', [])
|
||||
).strip()
|
||||
if not text:
|
||||
logger.warning("Gemini 返回空文本")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
try:
|
||||
result = parse_vlm_json(text)
|
||||
# 确保 frame_details 的 frame_timestamp 与标注一致
|
||||
for f in result.get('frame_details', []):
|
||||
idx = f.get('frame_index')
|
||||
if isinstance(idx, int) and idx in ts_map and not f.get('frame_timestamp'):
|
||||
f['frame_timestamp'] = ts_map[idx]
|
||||
for f in result.get('frame_details', []):
|
||||
if 'source_providers' not in f or not f.get('source_providers'):
|
||||
f['source_providers'] = ['gemini']
|
||||
self._cb.record_success()
|
||||
logger.info(f"Gemini 视觉分析完成，frame_details={len(result.get('frame_details', []))}")
|
||||
return result
|
||||
except VLMOutputInvalidError as e:
|
||||
logger.error(f"Gemini 输出无法解析为 JSON: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
else:
|
||||
logger.warning(f"Gemini 单帧失败 HTTP {resp.status_code}: {resp.text[:150]}")
|
||||
logger.warning(f"Gemini 视觉分析 HTTP {resp.status_code}: {resp.text[:150]}")
|
||||
self._cb.record_failure()
|
||||
except requests.Timeout:
|
||||
logger.warning(f"Gemini 单帧超时 ({self.timeout}s)")
|
||||
logger.warning(f"Gemini 视觉分析超时 ({self.timeout}s)")
|
||||
self._cb.record_failure()
|
||||
except Exception as e:
|
||||
logger.error(f"Gemini 单帧异常: {e}")
|
||||
logger.error(f"Gemini 视觉分析异常: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
def _build_structured_prompt(self, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。下面按时间顺序排列了多张监控截图。
|
||||
请分析整个时段，只输出合法 JSON（不要 markdown、不要任何解释文字），结构如下：
|
||||
|
||||
{{
|
||||
"global_summary": "整个时段的整体摘要，简体中文，2-4 句，客观描述人物与主要活动",
|
||||
"entities_json": [
|
||||
{{"person": "人物标识(匹配已知成员用真名，否则用'人物A'/'人物B'...)", "action": "主要动作", "clothing": "衣着"}}
|
||||
],
|
||||
"frame_details": [
|
||||
{{
|
||||
"frame_index": 图片序号(从1开始，与[图片N]标注对应),
|
||||
"frame_timestamp": "该帧的时间戳(用[图片N]标注里的时间)",
|
||||
"person": "该帧画面中的人物或'无人'",
|
||||
"action": "该帧可见动作",
|
||||
"clothing": "该帧衣着(颜色+类型)",
|
||||
"is_attention_event": false,
|
||||
"source_providers": ["gemini"]
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
规则：
|
||||
1. 只描述客观画面，不要猜测或想象。
|
||||
2. frame_details 每帧一条，frame_index 与上方[图片N]序号对应，frame_timestamp 用标注时间。
|
||||
3. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
4. is_attention_event：是否为跌倒、危险、异常哭闹等需关注事件（没有则为 false）。
|
||||
5. 没有人物出现的帧 person 填"无人"，action 填""。"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 智能问答：纯文本
|
||||
# ------------------------------------------------------------------
|
||||
def chat(self, prompt: str, max_tokens: int = 512) -> Optional[str]:
|
||||
if not self.api_key:
|
||||
logger.warning("Gemini API Key 未配置，跳过问答")
|
||||
return None
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/models/{self.model_name}:generateContent?key={self.api_key}",
|
||||
json={"contents": [{"parts": [{"text": prompt}]}],
|
||||
"generationConfig": {"temperature": 0.3, "maxOutputTokens": max_tokens}},
|
||||
timeout=self.timeout
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
cands = resp.json().get('candidates', [])
|
||||
if cands:
|
||||
text = ''.join(
|
||||
p.get('text', '')
|
||||
for p in cands[0].get('content', {}).get('parts', [])
|
||||
).strip()
|
||||
return text or None
|
||||
logger.warning(f"Gemini 问答 HTTP {resp.status_code}")
|
||||
except requests.Timeout:
|
||||
logger.warning(f"Gemini 问答超时 ({self.timeout}s)")
|
||||
except Exception as e:
|
||||
logger.error(f"Gemini 问答异常: {e}")
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
@@ -124,16 +207,3 @@ class GeminiAdapter(BaseModelAdapter):
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_prompt(self, ts: str, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}），客观描述画面内容，不要猜测。
|
||||
|
||||
需报告：
|
||||
1. 人物：数量、衣着（颜色+类型）、可见动作
|
||||
2. 物品：玩具、奶瓶、家具等显眼物体
|
||||
3. 互动：人与人或人与物体的互动
|
||||
|
||||
已知家庭成员（按特征匹配，匹配到用真名，否则用"人物X"）：
|
||||
{known_members or '（暂无）'}
|
||||
|
||||
要求简洁客观，不要输出 JSON 或 markdown。"""
|
||||
|
||||
Reference in New Issue
Block a user