feat(fam-edge): NVIDIA 多模型降级链 - adapter 支持 model_chain(asset 上传一次逐个尝试)+switch_interval_sec 切换间隔+model_timeouts 每模型独立超时;实测全部候选不可用(omni 500/12b 400/其余 404),链机制保留待可用模型

This commit is contained in:
ericwyuan
2026-08-21 13:30:59 +08:00
parent 83eefc194a
commit b7b5fe6f0b
2 changed files with 89 additions and 47 deletions

View File

@@ -67,11 +67,22 @@ models:
- provider: "nvidia" - provider: "nvidia"
role: "vision" role: "vision"
enabled: true enabled: true
# Nemotron Nano 12B v2 VLNIM 官方支持整视频 video_url 输入(内部自行采样帧) # 多模型降级链(实测记录 2026-08-21
model_name: "nvidia/nemotron-nano-12b-v2-vl" # omni 官方支持视频但 asset_id 引用 50012b 400llama-vision 不支持视频;
# cosmos/phi/gemma/kosmos/fuyu/paligemma 均 404 端点不可用。
# 链机制保留asset 上传一次,逐个尝试+间隔切换),可用模型出现时自动生效。
model_name: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"
fallback_models:
- "nvidia/nemotron-nano-12b-v2-vl"
- "meta/llama-3.2-11b-vision-instruct"
base_url: "https://integrate.api.nvidia.com/v1" base_url: "https://integrate.api.nvidia.com/v1"
api_key: "${NVIDIA_API_KEY}" api_key: "${NVIDIA_API_KEY}"
timeout: 600 timeout: 600
switch_interval_sec: 5 # 模型切换间隔:一个失败后等待再试下一个
model_timeouts: # 模型级独立超时(最终值,不参与 ×2
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": 300
"nvidia/nemotron-nano-12b-v2-vl": 300
"meta/llama-3.2-11b-vision-instruct": 120
circuit_breaker: circuit_breaker:
enabled: true enabled: true
threshold: 5 threshold: 5

View File

@@ -29,15 +29,29 @@ except ImportError:
class NvidiaVisionAdapter(BaseModelAdapter): class NvidiaVisionAdapter(BaseModelAdapter):
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)""" """NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)
多模型降级链(类似 Gemini flash -> flash-lite:
- model_chain = [model_name] + fallback_models
- asset 上传一次,遍历模型链逐个调用 video_url 引用同一 assetId
- 模型失败/超时 -> 记录统计 -> 间隔 switch_interval_sec 后切换下一模型
- 每个模型可用 model_timeouts 独立设置超时(不参与编排层 ×N 放大)
"""
def __init__(self, config: dict): def __init__(self, config: dict):
super().__init__("nvidia", config) super().__init__("nvidia", config)
self.model_name = config.get( self.model_name = config.get(
'model_name', 'nvidia/nemotron-nano-12b-v2-vl') 'model_name', 'nvidia/nemotron-nano-12b-v2-vl')
self.model_chain = [self.model_name] + [
m for m in config.get('fallback_models', []) if m and m != self.model_name]
self.api_key = self._resolve_key(config.get('api_key', '')) self.api_key = self._resolve_key(config.get('api_key', ''))
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1') self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
self.timeout = config.get('timeout', 600) self.timeout = config.get('timeout', 600)
# 模型级独立超时(最终值,不参与编排层 ×N 放大): {model_name: seconds}
self.model_timeouts = {
str(k): int(v) for k, v in (config.get('model_timeouts') or {}).items()}
# 模型切换间隔(秒):一个模型失败后等待再切下一个,避免连续打爆 API
self.switch_interval_sec = float(config.get('switch_interval_sec', 5))
cb_cfg = config.get('circuit_breaker', {}) cb_cfg = config.get('circuit_breaker', {})
self._cb = CircuitBreaker( self._cb = CircuitBreaker(
threshold=cb_cfg.get('threshold', 3), threshold=cb_cfg.get('threshold', 3),
@@ -140,17 +154,23 @@ class NvidiaVisionAdapter(BaseModelAdapter):
logger.warning(f"NVIDIA 视频文件不存在: {video_path}") logger.warning(f"NVIDIA 视频文件不存在: {video_path}")
return None return None
# asset 只上传一次,模型链内复用同一 assetId
asset_id = self._upload_asset(video_path) asset_id = self._upload_asset(video_path)
if not asset_id: if not asset_id:
self._cb.record_failure() self._cb.record_failure()
return None return None
prompt = self._build_video_prompt(known_members_context, event_start_time) prompt = self._build_video_prompt(known_members_context, event_start_time)
last_err = "no_model_in_chain"
for idx, model in enumerate(self.model_chain):
model_timeout = self.model_timeouts.get(model, self.timeout)
logger.info(f"NVIDIA 模型链 [{idx+1}/{len(self.model_chain)}] "
f"尝试 {model}(超时 {model_timeout}s")
started = datetime.now().strftime('%Y-%m-%d %H:%M:%S') started = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
t0 = time.time() t0 = time.time()
try: try:
resp = self._client.chat.completions.create( resp = self._client.chat.completions.create(
model=self.model_name, model=model,
messages=[{"role": "user", "content": [ messages=[{"role": "user", "content": [
{"type": "text", "text": prompt}, {"type": "text", "text": prompt},
{"type": "video_url", "video_url": { {"type": "video_url", "video_url": {
@@ -158,39 +178,50 @@ class NvidiaVisionAdapter(BaseModelAdapter):
]}], ]}],
temperature=0.2, temperature=0.2,
max_tokens=4096, max_tokens=4096,
# NIM 扩展:控制视频采样帧数(模型上限 128 帧 # NIM 扩展:控制视频采样帧数(部分模型支持
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}}, extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
timeout=self.timeout timeout=model_timeout
) )
duration = time.time() - t0 duration = time.time() - t0
content = resp.choices[0].message.content content = resp.choices[0].message.content
if not content: if not content:
self._emit_model_call(self.model_name, started, duration, False, "empty_content") self._emit_model_call(model, started, duration, False, "empty_content")
logger.warning("NVIDIA 视频分析返回空 content") logger.warning(f"NVIDIA [{model}] 返回空 content,切换下一模型")
self._cb.record_failure() last_err = f"{model}_empty"
return None self._sleep_switch(idx)
continue
data = self._parse_json(content) data = self._parse_json(content)
if not data or 'events' not in data: if not data or 'events' not in data:
self._emit_model_call(self.model_name, started, duration, False, "json_parse_failed") self._emit_model_call(model, started, duration, False, "json_parse_failed")
logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}") logger.warning(f"NVIDIA [{model}] JSON 解析失败,切换下一模型: {content[:120]}")
self._cb.record_failure() last_err = f"{model}_json"
return None self._sleep_switch(idx)
self._emit_model_call(self.model_name, started, duration, True) continue
self._emit_model_call(model, started, duration, True)
self._cb.record_success() self._cb.record_success()
logger.info(f"NVIDIA 整视频分析完成events={len(data.get('events', []))}") logger.info(f"NVIDIA [{model}] 整视频分析完成events={len(data.get('events', []))}")
return { return {
"global_summary": str(data.get('global_summary', '')), "global_summary": str(data.get('global_summary', '')),
"events": data.get('events', []), "events": data.get('events', []),
"people_mentioned": data.get('people_mentioned', []), "people_mentioned": data.get('people_mentioned', []),
"compute_provider": "nvidia", "compute_provider": f"nvidia:{model}",
} }
except Exception as e: except Exception as e:
duration = time.time() - t0 duration = time.time() - t0
self._emit_model_call(self.model_name, started, duration, False, str(e)) self._emit_model_call(model, started, duration, False, str(e))
last_err = f"{model}_failed"
logger.warning(f"NVIDIA [{model}] 视频分析异常,切换下一模型: {str(e)[:150]}")
self._sleep_switch(idx)
self._cb.record_failure() self._cb.record_failure()
logger.warning(f"NVIDIA 视频分析异常: {e}") logger.error(f"NVIDIA 模型链全部失败: {last_err}")
return None return None
def _sleep_switch(self, idx: int):
"""模型切换间隔(最后一个模型失败后无需再等)"""
if idx < len(self.model_chain) - 1 and self.switch_interval_sec > 0:
logger.info(f"NVIDIA 等待 {self.switch_interval_sec}s 后切换下一模型")
time.sleep(self.switch_interval_sec)
@staticmethod @staticmethod
def _parse_json(content: str) -> Optional[dict]: def _parse_json(content: str) -> Optional[dict]:
content = content.strip() content = content.strip()