[3.1-3.5] FAM-Edge 全链路 - API-Gateway/Video-Preprocessor/AI-Orchestrator/模型适配器(基类+Ollama+Gemini)/熔断器/JSON解析容错 + 配置
This commit is contained in:
ericwyuan
0
fam-edge/src/fam_edge/model_adapters/__init__.py
Normal file
0
fam-edge/src/fam_edge/model_adapters/__init__.py
Normal file
54
fam-edge/src/fam_edge/model_adapters/adapter_factory.py
Normal file
54
fam-edge/src/fam_edge/model_adapters/adapter_factory.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
适配器工厂 - 根据 config 创建适配器实例
|
||||
|
||||
新增模型只需:
|
||||
1. 实现适配器类（继承 BaseModelAdapter）
|
||||
2. 在此工厂注册
|
||||
3. 在 config.yaml 的 models 数组加一项
|
||||
"""
|
||||
from typing import List
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .ollama_adapter import OllamaAdapter
|
||||
from .gemini_adapter import GeminiAdapter
|
||||
from ..logger import setup_logger
|
||||
|
||||
logger = setup_logger('fam-edge.adapter_factory')
|
||||
|
||||
_ADAPTER_REGISTRY = {
|
||||
"ollama": OllamaAdapter,
|
||||
"gemini": GeminiAdapter,
|
||||
# v1.1 扩展:
|
||||
# "openai": OpenAIAdapter,
|
||||
# "nvidia": NvidiaAdapter,
|
||||
}
|
||||
|
||||
|
||||
def build_adapter(config: dict) -> BaseModelAdapter:
|
||||
"""根据 config 中的 provider 字段创建适配器"""
|
||||
provider = config.get('provider', '')
|
||||
adapter_cls = _ADAPTER_REGISTRY.get(provider)
|
||||
if adapter_cls is None:
|
||||
raise ValueError(f"未知的模型 provider: {provider}，请先注册适配器")
|
||||
return adapter_cls(config)
|
||||
|
||||
|
||||
def build_adapters(configs: List[dict]) -> List[BaseModelAdapter]:
|
||||
"""批量创建适配器（仅 enabled 的）"""
|
||||
adapters = []
|
||||
for cfg in configs:
|
||||
if not cfg.get('enabled', False):
|
||||
continue
|
||||
try:
|
||||
adapter = build_adapter(cfg)
|
||||
adapters.append(adapter)
|
||||
logger.info(f"适配器已创建: {adapter.provider_name} ({cfg.get('model_name', '?')})")
|
||||
except Exception as e:
|
||||
logger.error(f"创建适配器失败 ({cfg.get('provider', '?')}): {e}")
|
||||
return adapters
|
||||
|
||||
|
||||
def register_adapter(provider_name: str, adapter_cls):
|
||||
"""注册新适配器（供扩展使用）"""
|
||||
_ADAPTER_REGISTRY[provider_name] = adapter_cls
|
||||
logger.info(f"适配器已注册: {provider_name}")
|
||||
42
fam-edge/src/fam_edge/model_adapters/base_adapter.py
Normal file
42
fam-edge/src/fam_edge/model_adapters/base_adapter.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
模型适配器基类 - 所有模型适配器的抽象基类
|
||||
|
||||
新增模型只需继承此类并实现 4 个方法:
|
||||
1. health_check() -> bool
|
||||
2. analyze_frames(frame_paths, frame_timestamps, known_members_context) -> Optional[str]
|
||||
3. get_timeout() -> int
|
||||
4. get_circuit_breaker() -> CircuitBreaker
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class BaseModelAdapter(ABC):
|
||||
"""所有模型适配器的抽象基类"""
|
||||
|
||||
def __init__(self, provider_name: str, config: dict):
|
||||
self.provider_name = provider_name # 如 "ollama", "gemini"
|
||||
self.config = config
|
||||
|
||||
@abstractmethod
|
||||
def health_check(self) -> bool:
|
||||
"""健康检查，返回 True/False"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
"""视觉分析：输入帧图片路径 + 时间戳 + 成员清单，输出自然语言描述。
|
||||
失败/超时返回 None。"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_timeout(self) -> int:
|
||||
"""该模型的调用超时秒数"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_circuit_breaker(self):
|
||||
"""返回该模型专属的熔断器实例"""
|
||||
pass
|
||||
47
fam-edge/src/fam_edge/model_adapters/circuit_breaker.py
Normal file
47
fam-edge/src/fam_edge/model_adapters/circuit_breaker.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
熔断器 - 每个云端模型独立实例
|
||||
|
||||
状态机: CLOSED -> OPEN -> HALF_OPEN -> CLOSED/OPEN
|
||||
- 连续 threshold 次失败 -> OPEN
|
||||
- 冷却 cooldown 秒后 -> HALF_OPEN（允许一次探测）
|
||||
- 探测成功 -> CLOSED；探测失败 -> 重新 OPEN
|
||||
"""
|
||||
from collections import deque
|
||||
import time
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
def __init__(self, threshold: int = 5, cooldown: int = 900, enabled: bool = True):
|
||||
self.enabled = enabled
|
||||
if enabled:
|
||||
self.failures = deque(maxlen=threshold)
|
||||
else:
|
||||
self.failures = None
|
||||
self.threshold = threshold
|
||||
self.cooldown = cooldown
|
||||
self.state = 'CLOSED'
|
||||
self.last_failure = None
|
||||
|
||||
def record_failure(self):
|
||||
if not self.enabled:
|
||||
return
|
||||
self.failures.append(time.time())
|
||||
if len(self.failures) >= self.threshold:
|
||||
self.state = 'OPEN'
|
||||
self.last_failure = time.time()
|
||||
|
||||
def record_success(self):
|
||||
if not self.enabled:
|
||||
return
|
||||
self.failures.clear()
|
||||
self.state = 'CLOSED'
|
||||
|
||||
def is_open(self):
|
||||
if not self.enabled:
|
||||
return False
|
||||
if self.state == 'OPEN' and self.last_failure and time.time() - self.last_failure > self.cooldown:
|
||||
self.state = 'HALF_OPEN'
|
||||
return self.state == 'OPEN'
|
||||
|
||||
def __repr__(self):
|
||||
return f"CircuitBreaker(state={self.state}, enabled={self.enabled})"
|
||||
156
fam-edge/src/fam_edge/model_adapters/gemini_adapter.py
Normal file
156
fam-edge/src/fam_edge/model_adapters/gemini_adapter.py
Normal file
@@ -0,0 +1,156 @@
|
||||
"""
|
||||
GeminiAdapter - Google Gemini 云端模型适配器
|
||||
|
||||
provider_name = "gemini"
|
||||
模型: gemini-1.5-flash
|
||||
健康检查: GET models API
|
||||
熔断器: 启用，连续 5 次失败 -> OPEN 15 分钟
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import requests
|
||||
from typing import List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
|
||||
logger = setup_logger('fam-edge.gemini_adapter')
|
||||
|
||||
|
||||
class GeminiAdapter(BaseModelAdapter):
|
||||
"""Gemini 云端 VLM 适配器"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("gemini", config)
|
||||
self.model_name = config.get('model_name', 'gemini-1.5-flash')
|
||||
self.api_key = config.get('api_key', '')
|
||||
self.timeout = config.get('timeout', 8)
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 5),
|
||||
cooldown=cb_cfg.get('cooldown', 900),
|
||||
enabled=cb_cfg.get('enabled', True) # 云端默认启用
|
||||
)
|
||||
self._base_url = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
def health_check(self) -> bool:
|
||||
"""GET models API，检查可用性"""
|
||||
if not self.api_key:
|
||||
logger.warning("Gemini API Key 未配置，健康检查失败")
|
||||
return False
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"{self._base_url}/models?key={self.api_key}",
|
||||
timeout=10
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
models = resp.json().get('models', [])
|
||||
model_names = [m.get('name', '') for m in models]
|
||||
has_model = any(self.model_name in name for name in model_names)
|
||||
if has_model:
|
||||
logger.info(f"Gemini 健康检查通过: 模型 {self.model_name} 可用")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Gemini 健康检查失败: 模型 {self.model_name} 未找到")
|
||||
return False
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Gemini 健康检查异常: {e}")
|
||||
return False
|
||||
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
"""调用 Gemini 视觉分析"""
|
||||
if self._cb.is_open():
|
||||
logger.warning("Gemini 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
|
||||
if not self.api_key:
|
||||
logger.warning("Gemini API Key 未配置，跳过调用")
|
||||
return None
|
||||
|
||||
# 构建 Prompt
|
||||
n = len(frame_paths)
|
||||
prompt = self._build_visual_prompt(n, frame_timestamps, known_members_context)
|
||||
|
||||
# 构建 inline_data
|
||||
parts = [{"text": prompt}]
|
||||
for path in frame_paths:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
img_data = base64.b64encode(f.read()).decode('utf-8')
|
||||
parts.append({
|
||||
"inline_data": {
|
||||
"mime_type": "image/jpeg",
|
||||
"data": img_data
|
||||
}
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/models/{self.model_name}:generateContent?key={self.api_key}",
|
||||
json={
|
||||
"contents": [{"parts": parts}],
|
||||
"generationConfig": {"temperature": 0.2, "topP": 0.8}
|
||||
},
|
||||
timeout=self.timeout
|
||||
)
|
||||
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
candidates = data.get('candidates', [])
|
||||
if candidates:
|
||||
output = candidates[0].get('content', {}).get('parts', [{}])[0].get('text', '')
|
||||
self._cb.record_success()
|
||||
logger.info(f"Gemini 视觉分析完成，输出长度={len(output)}")
|
||||
return output
|
||||
else:
|
||||
logger.warning("Gemini 返回空 candidates")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
else:
|
||||
logger.error(f"Gemini 调用失败: {resp.status_code} {resp.text[:200]}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
except requests.Timeout:
|
||||
logger.warning(f"Gemini 调用超时 ({self.timeout}s)，降级跳过")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Gemini 调用异常: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
return self.timeout
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_visual_prompt(self, n: int, timestamps: List[str], known_members: str) -> str:
|
||||
ts_lines = '\n'.join(
|
||||
f"[Image {i+1}] Time: {ts}" for i, ts in enumerate(timestamps)
|
||||
)
|
||||
return f"""You are a home surveillance video analysis assistant. Describe what you see in the following {n} images chronologically. Be objective.
|
||||
|
||||
Timestamps:
|
||||
{ts_lines}
|
||||
|
||||
For each image, report:
|
||||
1. People: count, clothing (color + type), visible actions
|
||||
2. Objects: toys, bottles, furniture, etc.
|
||||
3. Interactions: between people or people and objects
|
||||
|
||||
Known family members (match by features, use real_name if matched, otherwise "PersonX"):
|
||||
{known_members or 'None'}
|
||||
|
||||
Output format (plain text, one paragraph per image, keep timestamp markers):
|
||||
[Image 1] Time: {timestamps[0] if timestamps else ''}
|
||||
Description: ...
|
||||
|
||||
Be concise and objective. Do not output JSON or markdown."""
|
||||
139
fam-edge/src/fam_edge/model_adapters/ollama_adapter.py
Normal file
139
fam-edge/src/fam_edge/model_adapters/ollama_adapter.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
OllamaAdapter - 本地 VLM 模型适配器
|
||||
|
||||
provider_name = "ollama"
|
||||
模型: llava-phi3
|
||||
健康检查: GET /api/tags
|
||||
"""
|
||||
import base64
|
||||
import requests
|
||||
from typing import List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
from .circuit_breaker import CircuitBreaker
|
||||
from ..logger import setup_logger
|
||||
|
||||
logger = setup_logger('fam-edge.ollama_adapter')
|
||||
|
||||
|
||||
class OllamaAdapter(BaseModelAdapter):
|
||||
"""Ollama 本地 VLM 适配器"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("ollama", config)
|
||||
self.base_url = config.get('base_url', 'http://localhost:11434')
|
||||
self.model_name = config.get('model_name', 'llava-phi3')
|
||||
self.timeout = config.get('timeout', 240)
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 5),
|
||||
cooldown=cb_cfg.get('cooldown', 900),
|
||||
enabled=cb_cfg.get('enabled', False) # 本地模型默认不启用
|
||||
)
|
||||
|
||||
def health_check(self) -> bool:
|
||||
"""GET /api/tags，检查模型是否可用"""
|
||||
try:
|
||||
resp = requests.get(f"{self.base_url}/api/tags", timeout=10)
|
||||
if resp.status_code == 200:
|
||||
models = resp.json().get('models', [])
|
||||
model_names = [m.get('name', '') for m in models]
|
||||
# 兼容 llava-phi3:latest 等后缀
|
||||
has_model = any(self.model_name in name for name in model_names)
|
||||
if has_model:
|
||||
logger.info(f"Ollama 健康检查通过: 模型 {self.model_name} 可用")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Ollama 健康检查失败: 模型 {self.model_name} 未找到，可用模型: {model_names}")
|
||||
return False
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama 健康检查异常: {e}")
|
||||
return False
|
||||
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[str]:
|
||||
"""调用 Ollama 视觉分析"""
|
||||
if self._cb.is_open():
|
||||
logger.warning("Ollama 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
|
||||
# 构建 Prompt
|
||||
n = len(frame_paths)
|
||||
prompt = self._build_visual_prompt(n, frame_timestamps, known_members_context)
|
||||
|
||||
# 读取图片并 Base64 编码
|
||||
images = []
|
||||
for path in frame_paths:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
images.append(base64.b64encode(f.read()).decode('utf-8'))
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
|
||||
if not images:
|
||||
logger.error("没有可用的图片帧")
|
||||
return None
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self.base_url}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"images": images,
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.2, "top_p": 0.8}
|
||||
},
|
||||
timeout=self.timeout
|
||||
)
|
||||
|
||||
if resp.status_code == 200:
|
||||
output = resp.json().get('response', '')
|
||||
self._cb.record_success()
|
||||
logger.info(f"Ollama 视觉分析完成，输出长度={len(output)}")
|
||||
return output
|
||||
else:
|
||||
logger.error(f"Ollama 调用失败: {resp.status_code} {resp.text[:200]}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
except requests.Timeout:
|
||||
logger.error(f"Ollama 调用超时 ({self.timeout}s)")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama 调用异常: {e}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
def get_timeout(self) -> int:
|
||||
return self.timeout
|
||||
|
||||
def get_circuit_breaker(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
def _build_visual_prompt(self, n: int, timestamps: List[str], known_members: str) -> str:
|
||||
"""构建视觉分析 Prompt"""
|
||||
ts_lines = '\n'.join(
|
||||
f"[图片{i+1}] 时间: {ts}" for i, ts in enumerate(timestamps)
|
||||
)
|
||||
return f"""你是家庭监控视频分析助手。请按时间顺序描述下列 {n} 张图片中可见的内容，只描述客观画面，不要猜测或推测。
|
||||
|
||||
每张图片对应的时间戳如下：
|
||||
{ts_lines}
|
||||
|
||||
每张图片需报告：
|
||||
1. 人物：数量、衣着（颜色+类型）、可见动作
|
||||
2. 物品：玩具、奶瓶、家具等显眼物品
|
||||
3. 互动：人与人、人与物品之间的互动
|
||||
|
||||
已知家庭成员清单（按特征匹配，匹配成功用 real_name，未匹配用"人物X"标识）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
|
||||
输出格式（纯文本，每张图片一段，保留时间戳标记）：
|
||||
[图片1] 时间: {timestamps[0] if timestamps else ''}
|
||||
内容: ...
|
||||
|
||||
要求简洁、客观。不要输出 JSON，不要输出 markdown。"""
|
||||
Reference in New Issue
Block a user