diff --git a/fam-edge/config/config.yaml b/fam-edge/config/config.yaml index d49ff02..f5b8718 100644 --- a/fam-edge/config/config.yaml +++ b/fam-edge/config/config.yaml @@ -67,11 +67,22 @@ models: - provider: "nvidia" role: "vision" enabled: true - # Nemotron Nano 12B v2 VL:NIM 官方支持整视频 video_url 输入(内部自行采样帧) - model_name: "nvidia/nemotron-nano-12b-v2-vl" + # 多模型降级链(实测记录 2026-08-21): + # omni 官方支持视频但 asset_id 引用 500;12b 400;llama-vision 不支持视频; + # cosmos/phi/gemma/kosmos/fuyu/paligemma 均 404 端点不可用。 + # 链机制保留(asset 上传一次,逐个尝试+间隔切换),可用模型出现时自动生效。 + model_name: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning" + fallback_models: + - "nvidia/nemotron-nano-12b-v2-vl" + - "meta/llama-3.2-11b-vision-instruct" base_url: "https://integrate.api.nvidia.com/v1" api_key: "${NVIDIA_API_KEY}" timeout: 600 + switch_interval_sec: 5 # 模型切换间隔:一个失败后等待再试下一个 + model_timeouts: # 模型级独立超时(最终值,不参与 ×2) + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": 300 + "nvidia/nemotron-nano-12b-v2-vl": 300 + "meta/llama-3.2-11b-vision-instruct": 120 circuit_breaker: enabled: true threshold: 5 diff --git a/fam-edge/src/fam_edge/model_adapters/nvidia_adapter.py b/fam-edge/src/fam_edge/model_adapters/nvidia_adapter.py index a505739..b8f181d 100644 --- a/fam-edge/src/fam_edge/model_adapters/nvidia_adapter.py +++ b/fam-edge/src/fam_edge/model_adapters/nvidia_adapter.py @@ -29,15 +29,29 @@ except ImportError: class NvidiaVisionAdapter(BaseModelAdapter): - """NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)""" + """NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答) + + 多模型降级链(类似 Gemini flash -> flash-lite): + - model_chain = [model_name] + fallback_models + - asset 上传一次,遍历模型链逐个调用 video_url 引用同一 assetId + - 模型失败/超时 -> 记录统计 -> 间隔 switch_interval_sec 后切换下一模型 + - 每个模型可用 model_timeouts 独立设置超时(不参与编排层 ×N 放大) + """ def __init__(self, config: dict): super().__init__("nvidia", config) self.model_name = config.get( 'model_name', 'nvidia/nemotron-nano-12b-v2-vl') + self.model_chain = [self.model_name] + [ + m for m in config.get('fallback_models', []) if m and m != self.model_name] self.api_key = self._resolve_key(config.get('api_key', '')) self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1') self.timeout = config.get('timeout', 600) + # 模型级独立超时(最终值,不参与编排层 ×N 放大): {model_name: seconds} + self.model_timeouts = { + str(k): int(v) for k, v in (config.get('model_timeouts') or {}).items()} + # 模型切换间隔(秒):一个模型失败后等待再切下一个,避免连续打爆 API + self.switch_interval_sec = float(config.get('switch_interval_sec', 5)) cb_cfg = config.get('circuit_breaker', {}) self._cb = CircuitBreaker( threshold=cb_cfg.get('threshold', 3), @@ -140,56 +154,73 @@ class NvidiaVisionAdapter(BaseModelAdapter): logger.warning(f"NVIDIA 视频文件不存在: {video_path}") return None + # asset 只上传一次,模型链内复用同一 assetId asset_id = self._upload_asset(video_path) if not asset_id: self._cb.record_failure() return None prompt = self._build_video_prompt(known_members_context, event_start_time) - started = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - t0 = time.time() - try: - resp = self._client.chat.completions.create( - model=self.model_name, - messages=[{"role": "user", "content": [ - {"type": "text", "text": prompt}, - {"type": "video_url", "video_url": { - "url": f"data:video/mp4;asset_id={asset_id}"}} - ]}], - temperature=0.2, - max_tokens=4096, - # NIM 扩展:控制视频采样帧数(模型上限 128 帧) - extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}}, - timeout=self.timeout - ) - duration = time.time() - t0 - content = resp.choices[0].message.content - if not content: - self._emit_model_call(self.model_name, started, duration, False, "empty_content") - logger.warning("NVIDIA 视频分析返回空 content") - self._cb.record_failure() - return None - data = self._parse_json(content) - if not data or 'events' not in data: - self._emit_model_call(self.model_name, started, duration, False, "json_parse_failed") - logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}") - self._cb.record_failure() - return None - self._emit_model_call(self.model_name, started, duration, True) - self._cb.record_success() - logger.info(f"NVIDIA 整视频分析完成,events={len(data.get('events', []))}") - return { - "global_summary": str(data.get('global_summary', '')), - "events": data.get('events', []), - "people_mentioned": data.get('people_mentioned', []), - "compute_provider": "nvidia", - } - except Exception as e: - duration = time.time() - t0 - self._emit_model_call(self.model_name, started, duration, False, str(e)) - self._cb.record_failure() - logger.warning(f"NVIDIA 视频分析异常: {e}") - return None + last_err = "no_model_in_chain" + for idx, model in enumerate(self.model_chain): + model_timeout = self.model_timeouts.get(model, self.timeout) + logger.info(f"NVIDIA 模型链 [{idx+1}/{len(self.model_chain)}] " + f"尝试 {model}(超时 {model_timeout}s)") + started = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + t0 = time.time() + try: + resp = self._client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": [ + {"type": "text", "text": prompt}, + {"type": "video_url", "video_url": { + "url": f"data:video/mp4;asset_id={asset_id}"}} + ]}], + temperature=0.2, + max_tokens=4096, + # NIM 扩展:控制视频采样帧数(部分模型支持) + extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}}, + timeout=model_timeout + ) + duration = time.time() - t0 + content = resp.choices[0].message.content + if not content: + self._emit_model_call(model, started, duration, False, "empty_content") + logger.warning(f"NVIDIA [{model}] 返回空 content,切换下一模型") + last_err = f"{model}_empty" + self._sleep_switch(idx) + continue + data = self._parse_json(content) + if not data or 'events' not in data: + self._emit_model_call(model, started, duration, False, "json_parse_failed") + logger.warning(f"NVIDIA [{model}] JSON 解析失败,切换下一模型: {content[:120]}") + last_err = f"{model}_json" + self._sleep_switch(idx) + continue + self._emit_model_call(model, started, duration, True) + self._cb.record_success() + logger.info(f"NVIDIA [{model}] 整视频分析完成,events={len(data.get('events', []))}") + return { + "global_summary": str(data.get('global_summary', '')), + "events": data.get('events', []), + "people_mentioned": data.get('people_mentioned', []), + "compute_provider": f"nvidia:{model}", + } + except Exception as e: + duration = time.time() - t0 + self._emit_model_call(model, started, duration, False, str(e)) + last_err = f"{model}_failed" + logger.warning(f"NVIDIA [{model}] 视频分析异常,切换下一模型: {str(e)[:150]}") + self._sleep_switch(idx) + self._cb.record_failure() + logger.error(f"NVIDIA 模型链全部失败: {last_err}") + return None + + def _sleep_switch(self, idx: int): + """模型切换间隔(最后一个模型失败后无需再等)""" + if idx < len(self.model_chain) - 1 and self.switch_interval_sec > 0: + logger.info(f"NVIDIA 等待 {self.switch_interval_sec}s 后切换下一模型") + time.sleep(self.switch_interval_sec) @staticmethod def _parse_json(content: str) -> Optional[dict]: