[3.1-3.5] FAM-Edge 全链路 - API-Gateway/Video-Preprocessor/AI-Orchestrator/模型适配器(基类+Ollama+Gemini)/熔断器/JSON解析容错 + 配置

fam-edge/src/fam_edge/ai_orchestrator/json_parser.py

@@ -0,0 +1,104 @@
+"""
+VLM JSON 解析容错 + Schema 校验
+
+三层容错策略:
+1. 直接 json.loads
+2. 提取 markdown fence 内容
+3. 贪婪匹配最大的 {...}
+
+validate_schema: 校验 + 脏数据清洗
+"""
+import re
+import json
+from typing import Dict
+
+
+class VLMOutputInvalidError(Exception):
+    """VLM 输出无法解析为合法 JSON"""
+    pass
+
+
+def parse_vlm_json(raw: str) -> dict:
+    """三层容错解析 VLM 输出的 JSON"""
+    # 第 1 层：直接 json.loads
+    try:
+        return validate_schema(json.loads(raw.strip()))
+    except json.JSONDecodeError:
+        pass
+
+    # 第 2 层：提取 markdown fence 内容
+    fence_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', raw, re.DOTALL)
+    if fence_match:
+        try:
+            return validate_schema(json.loads(fence_match.group(1)))
+        except json.JSONDecodeError:
+            pass
+
+    # 第 3 层：贪婪匹配最大的 {...}
+    brace_match = re.search(r'\{.*\}', raw, re.DOTALL)
+    if brace_match:
+        try:
+            return validate_schema(json.loads(brace_match.group(0)))
+        except json.JSONDecodeError:
+            pass
+
+    raise VLMOutputInvalidError(f"无法从 VLM 输出中解析 JSON: {raw[:200]}")
+
+
+def validate_schema(data: dict) -> dict:
+    """Schema 校验 + 脏数据清洗"""
+    required = ["global_summary", "entities_json", "frame_details", "compute_provider"]
+    for k in required:
+        if k not in data:
+            raise VLMOutputInvalidError(f"缺失字段: {k}")
+
+    # entities_json 结构校验
+    if not isinstance(data["entities_json"], list):
+        raise VLMOutputInvalidError("entities_json 必须为数组")
+
+    cleaned_entities = []
+    for ent in data["entities_json"]:
+        if not isinstance(ent, dict):
+            continue
+        if "person" not in ent or "action" not in ent:
+            raise VLMOutputInvalidError("entity 缺少 person 或 action 字段")
+        cleaned_entities.append({
+            "person": str(ent["person"]),
+            "action": str(ent["action"]),
+            "clothing": str(ent.get("clothing", ""))
+        })
+    data["entities_json"] = cleaned_entities
+
+    # frame_details 结构校验
+    if not isinstance(data["frame_details"], list):
+        raise VLMOutputInvalidError("frame_details 必须为数组")
+
+    cleaned_frames = []
+    for frame in data["frame_details"]:
+        if not isinstance(frame, dict):
+            continue
+        for k in ["frame_index", "frame_timestamp", "person", "action", "source_providers"]:
+            if k not in frame:
+                raise VLMOutputInvalidError(f"frame_details 缺少字段: {k}")
+        sp = frame["source_providers"]
+        if not isinstance(sp, list) or len(sp) == 0:
+            raise VLMOutputInvalidError("frame_details.source_providers 必须为非空数组")
+        cleaned_frames.append({
+            "frame_index": int(frame["frame_index"]),
+            "frame_timestamp": str(frame["frame_timestamp"]),
+            "person": str(frame["person"]),
+            "action": str(frame["action"]),
+            "clothing": str(frame.get("clothing", "")),
+            "is_attention_event": bool(frame.get("is_attention_event", False)),
+            "source_providers": [str(p) for p in sp]
+        })
+    data["frame_details"] = cleaned_frames
+
+    # compute_provider 校验为数组
+    if not isinstance(data["compute_provider"], list):
+        raise VLMOutputInvalidError("compute_provider 必须为数组")
+    if len(data["compute_provider"]) == 0:
+        raise VLMOutputInvalidError("compute_provider 不能为空数组")
+    data["compute_provider"] = [str(p) for p in data["compute_provider"]]
+
+    return data

fam-edge/src/fam_edge/ai_orchestrator/orchestrator.py

@@ -0,0 +1,327 @@
+"""
+AI-Orchestrator - 多模型并行编排
+
+流程:
+1. 加载所有启用的模型适配器
+2. 健康检查
+3. 抽帧 + 关键帧筛选 + 压缩
+4. 并行调用所有健康模型（ThreadPoolExecutor）
+5. 文本融合（多模型输出平等交叉验证）
+6. 回调 NAS
+7. 清理临时文件
+"""
+import time
+import json
+import base64
+import requests
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeout
+from typing import Dict, List, Optional
+
+from ..logger import setup_logger, log_task
+from ..config_loader import load_config
+from ..model_adapters.adapter_factory import build_adapters
+from ..model_adapters.base_adapter import BaseModelAdapter
+from ..video_preprocessor.preprocessor import VideoPreprocessor
+from .json_parser import parse_vlm_json, VLMOutputInvalidError
+
+logger = setup_logger('fam-edge.orchestrator')
+
+FUSION_SYSTEM_PROMPT = """你是一个无情的数据提取器。不要输出任何思考过程，只输出合法 JSON。
+
+输入参数:
+- 多模型视觉分析日志（每个模型独立输出，平等对待，交叉验证）:
+{model_outputs}
+- 已知成员清单: {known_members}
+
+执行规则:
+1. 多个模型的输出平等对待，交叉验证：
+   - 多个模型一致描述的内容 → 可信度高，必须纳入 frame_details，source_providers 列出所有一致的模型
+   - 仅单一模型描述的内容 → 纳入 frame_details，source_providers 仅含该模型
+   - 多个模型冲突时（如人物动作描述不一致）→ 以多数模型一致为准，source_providers 列出多数派模型
+2. 画面人物按特征匹配已知成员清单：
+   - 匹配到已命名成员（real_name 非空）→ person 字段填 real_name
+   - 匹配到未命名成员（real_name 为空）→ person 字段填 abstract_label
+   - 都不匹配 → 按出现顺序赋予新标识"人物B"、"人物C"...
+3. 提取每张关键帧对应的时间点、人物、动作、衣着，输出到 frame_details 数组。
+4. frame_details 每条必须包含 source_providers 数组。
+5. compute_provider 字段填入本次实际成功调用的所有模型标识数组（去重）。
+6. 仅输出合法 JSON，不输出任何思考过程、markdown 标记或注释。
+
+输出 JSON 结构:
+{{
+  "global_summary": "字符串，整个时段的整体摘要，简体中文",
+  "entities_json": [
+    {{
+      "person": "字符串",
+      "action": "字符串",
+      "clothing": "字符串"
+    }}
+  ],
+  "frame_details": [
+    {{
+      "frame_index": "数字",
+      "frame_timestamp": "字符串，ISO 8601 格式时间戳",
+      "person": "字符串",
+      "action": "字符串",
+      "clothing": "字符串",
+      "is_attention_event": "布尔值",
+      "source_providers": "数组"
+    }}
+  ],
+  "compute_provider": "数组"
+}}
+"""
+
+
+class AIOrchestrator:
+    """AI 编排器"""
+
+    def __init__(self):
+        self.config = load_config()
+        self.adapters: List[BaseModelAdapter] = build_adapters(self.config.get('models', []))
+        self.timeout_cfg = self.config.get('timeout', {})
+
+    def health_check_all(self) -> List[BaseModelAdapter]:
+        """健康检查，返回健康的适配器列表"""
+        healthy = []
+        for adapter in self.adapters:
+            try:
+                if adapter.health_check():
+                    healthy.append(adapter)
+            except Exception as e:
+                logger.error(f"适配器 {adapter.provider_name} 健康检查异常: {e}")
+        return healthy
+
+    def run_visual_analysis(self, adapters: List[BaseModelAdapter],
+                           frame_paths: List[str],
+                           frame_timestamps: List[str],
+                           known_members_context: str) -> Dict[str, str]:
+        """并行调用所有健康模型进行视觉分析"""
+        model_outputs = {}
+        max_timeout = max((a.get_timeout() for a in adapters), default=240)
+
+        with ThreadPoolExecutor(max_workers=len(adapters)) as pool:
+            futures = {}
+            for adapter in adapters:
+                if adapter.get_circuit_breaker().is_open():
+                    logger.warning(f"[{adapter.provider_name}] 熔断器 OPEN，跳过")
+                    continue
+                future = pool.submit(
+                    adapter.analyze_frames,
+                    frame_paths, frame_timestamps, known_members_context
+                )
+                futures[future] = adapter.provider_name
+
+            for future in as_completed(futures, timeout=max_timeout + 10):
+                provider = futures[future]
+                start = time.time()
+                try:
+                    adapter = next(a for a in adapters if a.provider_name == provider)
+                    output = future.result(timeout=adapter.get_timeout())
+                    duration_ms = int((time.time() - start) * 1000)
+                    if output:
+                        model_outputs[provider] = output
+                        adapter.get_circuit_breaker().record_success()
+                        log_task(logger, 0, f'model_{provider}', f'视觉分析成功，输出长度={len(output)}', duration_ms=duration_ms)
+                    else:
+                        adapter.get_circuit_breaker().record_failure()
+                        logger.warning(f"[{provider}] 视觉分析返回空")
+                except FuturesTimeout:
+                    logger.warning(f"[{provider}] 视觉分析超时")
+                    adapter = next(a for a in adapters if a.provider_name == provider)
+                    adapter.get_circuit_breaker().record_failure()
+                except Exception as e:
+                    logger.error(f"[{provider}] 视觉分析异常: {e}")
+                    adapter = next(a for a in adapters if a.provider_name == provider)
+                    adapter.get_circuit_breaker().record_failure()
+
+        return model_outputs
+
+    def run_text_fusion(self, model_outputs: Dict[str, str],
+                         known_members_context: str,
+                         task_id: int) -> dict:
+        """文本融合阶段 - 多模型输出平等交叉验证"""
+        # 构建 model_outputs 文本
+        outputs_text = '\n'.join(
+            f"  - {provider} 输出: {output}" for provider, output in model_outputs.items()
+        )
+
+        prompt = FUSION_SYSTEM_PROMPT.format(
+            model_outputs=outputs_text,
+            known_members=known_members_context or '（暂无已知成员）'
+        )
+
+        # 调用 Ollama 纯文本模式
+        ollama_cfg = next(
+            (cfg for cfg in self.config.get('models', []) if cfg.get('provider') == 'ollama'),
+            None
+        )
+        if not ollama_cfg:
+            raise VLMOutputInvalidError("没有 Ollama 配置，无法执行文本融合")
+
+        base_url = ollama_cfg.get('base_url', 'http://localhost:11434')
+        model_name = ollama_cfg.get('model_name', 'llava-phi3')
+        fusion_timeout = self.timeout_cfg.get('vlm_fusion', 120)
+
+        start = time.time()
+        resp = requests.post(
+            f"{base_url}/api/generate",
+            json={
+                "model": model_name,
+                "prompt": prompt,
+                "stream": False,
+                "format": "json",
+                "options": {"temperature": 0.0}
+            },
+            timeout=fusion_timeout
+        )
+
+        if resp.status_code != 200:
+            raise VLMOutputInvalidError(f"融合阶段 Ollama 调用失败: {resp.status_code}")
+
+        raw_output = resp.json().get('response', '')
+        duration_ms = int((time.time() - start) * 1000)
+        log_task(logger, task_id, 'vlm_fusion', f'融合完成，原始输出长度={len(raw_output)}', duration_ms=duration_ms)
+
+        # 解析 JSON（三层容错）
+        result = parse_vlm_json(raw_output)
+
+        # 确保 compute_provider 与实际调用的模型一致
+        result['compute_provider'] = list(model_outputs.keys())
+
+        # 确保 frame_details 的 source_providers 只包含实际成功的模型
+        valid_providers = set(model_outputs.keys())
+        for frame in result.get('frame_details', []):
+            frame['source_providers'] = [
+                p for p in frame.get('source_providers', []) if p in valid_providers
+            ] or list(valid_providers)
+
+        return result
+
+    def send_callback(self, webhook_url: str, task_id: int,
+                      result: dict, camera_name: str = '',
+                      event_start_time: str = '', event_end_time: str = ''):
+        """回调 NAS"""
+        payload = {
+            "task_id": task_id,
+            "status": "success",
+            "event_start_time": event_start_time,
+            "event_end_time": event_end_time,
+            "camera_name": camera_name,
+            "global_summary": result.get('global_summary', ''),
+            "entities_json": result.get('entities_json', []),
+            "frame_details": result.get('frame_details', []),
+            "compute_provider": result.get('compute_provider', []),
+            "error_message": None
+        }
+
+        callback_timeout = self.timeout_cfg.get('callback', 30)
+        max_retries = 3
+
+        for attempt in range(max_retries):
+            try:
+                resp = requests.post(webhook_url, json=payload, timeout=callback_timeout)
+                if resp.status_code == 200:
+                    log_task(logger, task_id, 'callback', '回调成功')
+                    return
+                else:
+                    logger.warning(f"[task_id={task_id}] 回调返回 {resp.status_code}，重试 {attempt+1}/{max_retries}")
+            except Exception as e:
+                logger.warning(f"[task_id={task_id}] 回调异常: {e}，重试 {attempt+1}/{max_retries}")
+
+        raise Exception(f"回调失败，已重试 {max_retries} 次")
+
+    def send_failure_callback(self, webhook_url: str, task_id: int,
+                               failure_stage: str, error_message: str):
+        """发送失败回调"""
+        payload = {
+            "task_id": task_id,
+            "status": "failed",
+            "failure_stage": failure_stage,
+            "error_message": error_message
+        }
+        try:
+            requests.post(webhook_url, json=payload, timeout=30)
+        except Exception as e:
+            logger.error(f"[task_id={task_id}] 失败回调也失败: {e}")
+
+    def process_task(self, task_data: dict):
+        """端到端处理任务"""
+        task_id = task_data.get('task_id')
+        video_url = task_data.get('video_url')
+        webhook_url = task_data.get('webhook_url')
+        known_members = task_data.get('known_members_context', '')
+
+        logger.info(f"[task_id={task_id}] ====== 开始处理任务 ======")
+        start_time = time.time()
+
+        # 1. 健康检查
+        healthy_adapters = self.health_check_all()
+        if not healthy_adapters:
+            logger.error(f"[task_id={task_id}] 所有模型不健康，返回 503")
+            self.send_failure_callback(webhook_url, task_id, 'vlm_visual', 'All models unhealthy')
+            return 503
+
+        # 2. 下载 + 抽帧
+        preprocessor = VideoPreprocessor(task_id)
+        try:
+            # 下载
+            video_path = preprocessor.download_video(video_url)
+
+            # 抽帧
+            candidate_frames = preprocessor.extract_candidate_frames(video_path)
+            if not candidate_frames:
+                raise Exception("抽帧失败，无候选帧")
+
+            # 关键帧筛选
+            key_frames = preprocessor.select_key_frames(candidate_frames)
+
+            # 压缩
+            compressed_frames = preprocessor.compress_frames(key_frames)
+            if not compressed_frames:
+                raise Exception("压缩后无可用帧")
+
+            # 计算时间戳
+            event_start_time = task_data.get('event_start_time', '')
+            frame_timestamps = preprocessor.compute_timestamps(
+                video_path, len(compressed_frames), event_start_time
+            )
+
+            # 3. 并行视觉分析
+            model_outputs = self.run_visual_analysis(
+                healthy_adapters, compressed_frames, frame_timestamps, known_members
+            )
+
+            if not model_outputs:
+                raise Exception('All models failed in visual analysis')
+
+            # 4. 文本融合
+            fusion_result = self.run_text_fusion(model_outputs, known_members, task_id)
+
+            # 5. 回调
+            # 从视频文件名推断 camera_name
+            camera_name = task_data.get('camera_name', '')
+            event_end_time = task_data.get('event_end_time', '')
+
+            self.send_callback(
+                webhook_url, task_id, fusion_result,
+                camera_name=camera_name,
+                event_start_time=event_start_time,
+                event_end_time=event_end_time
+            )
+
+            total_ms = int((time.time() - start_time) * 1000)
+            log_task(logger, task_id, 'overall', f'任务完成', duration_ms=total_ms)
+
+        except VLMOutputInvalidError as e:
+            logger.error(f"[task_id={task_id}] VLM 输出解析失败: {e}")
+            self.send_failure_callback(webhook_url, task_id, 'vlm_fusion', str(e))
+        except Exception as e:
+            logger.error(f"[task_id={task_id}] 任务处理失败: {e}", exc_info=True)
+            self.send_failure_callback(webhook_url, task_id, 'download', str(e))
+        finally:
+            # 6. 清理
+            if 'preprocessor' in locals():
+                preprocessor.cleanup()
+
+        return 200