feat(fam-edge): NVIDIA 多模型降级链 - adapter 支持 model_chain(asset 上传一次逐个尝试)+switch_interval_sec 切换间隔+model_timeouts 每模型独立超时;实测全部候选不可用(omni 500/12b 400/其余 404),链机制保留待可用模型
This commit is contained in:
@@ -67,11 +67,22 @@ models:
|
|||||||
- provider: "nvidia"
|
- provider: "nvidia"
|
||||||
role: "vision"
|
role: "vision"
|
||||||
enabled: true
|
enabled: true
|
||||||
# Nemotron Nano 12B v2 VL:NIM 官方支持整视频 video_url 输入(内部自行采样帧)
|
# 多模型降级链(实测记录 2026-08-21):
|
||||||
model_name: "nvidia/nemotron-nano-12b-v2-vl"
|
# omni 官方支持视频但 asset_id 引用 500;12b 400;llama-vision 不支持视频;
|
||||||
|
# cosmos/phi/gemma/kosmos/fuyu/paligemma 均 404 端点不可用。
|
||||||
|
# 链机制保留(asset 上传一次,逐个尝试+间隔切换),可用模型出现时自动生效。
|
||||||
|
model_name: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"
|
||||||
|
fallback_models:
|
||||||
|
- "nvidia/nemotron-nano-12b-v2-vl"
|
||||||
|
- "meta/llama-3.2-11b-vision-instruct"
|
||||||
base_url: "https://integrate.api.nvidia.com/v1"
|
base_url: "https://integrate.api.nvidia.com/v1"
|
||||||
api_key: "${NVIDIA_API_KEY}"
|
api_key: "${NVIDIA_API_KEY}"
|
||||||
timeout: 600
|
timeout: 600
|
||||||
|
switch_interval_sec: 5 # 模型切换间隔:一个失败后等待再试下一个
|
||||||
|
model_timeouts: # 模型级独立超时(最终值,不参与 ×2)
|
||||||
|
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": 300
|
||||||
|
"nvidia/nemotron-nano-12b-v2-vl": 300
|
||||||
|
"meta/llama-3.2-11b-vision-instruct": 120
|
||||||
circuit_breaker:
|
circuit_breaker:
|
||||||
enabled: true
|
enabled: true
|
||||||
threshold: 5
|
threshold: 5
|
||||||
|
|||||||
@@ -29,15 +29,29 @@ except ImportError:
|
|||||||
|
|
||||||
|
|
||||||
class NvidiaVisionAdapter(BaseModelAdapter):
|
class NvidiaVisionAdapter(BaseModelAdapter):
|
||||||
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)"""
|
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)
|
||||||
|
|
||||||
|
多模型降级链(类似 Gemini flash -> flash-lite):
|
||||||
|
- model_chain = [model_name] + fallback_models
|
||||||
|
- asset 上传一次,遍历模型链逐个调用 video_url 引用同一 assetId
|
||||||
|
- 模型失败/超时 -> 记录统计 -> 间隔 switch_interval_sec 后切换下一模型
|
||||||
|
- 每个模型可用 model_timeouts 独立设置超时(不参与编排层 ×N 放大)
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, config: dict):
|
def __init__(self, config: dict):
|
||||||
super().__init__("nvidia", config)
|
super().__init__("nvidia", config)
|
||||||
self.model_name = config.get(
|
self.model_name = config.get(
|
||||||
'model_name', 'nvidia/nemotron-nano-12b-v2-vl')
|
'model_name', 'nvidia/nemotron-nano-12b-v2-vl')
|
||||||
|
self.model_chain = [self.model_name] + [
|
||||||
|
m for m in config.get('fallback_models', []) if m and m != self.model_name]
|
||||||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||||||
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
|
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
|
||||||
self.timeout = config.get('timeout', 600)
|
self.timeout = config.get('timeout', 600)
|
||||||
|
# 模型级独立超时(最终值,不参与编排层 ×N 放大): {model_name: seconds}
|
||||||
|
self.model_timeouts = {
|
||||||
|
str(k): int(v) for k, v in (config.get('model_timeouts') or {}).items()}
|
||||||
|
# 模型切换间隔(秒):一个模型失败后等待再切下一个,避免连续打爆 API
|
||||||
|
self.switch_interval_sec = float(config.get('switch_interval_sec', 5))
|
||||||
cb_cfg = config.get('circuit_breaker', {})
|
cb_cfg = config.get('circuit_breaker', {})
|
||||||
self._cb = CircuitBreaker(
|
self._cb = CircuitBreaker(
|
||||||
threshold=cb_cfg.get('threshold', 3),
|
threshold=cb_cfg.get('threshold', 3),
|
||||||
@@ -140,17 +154,23 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
|||||||
logger.warning(f"NVIDIA 视频文件不存在: {video_path}")
|
logger.warning(f"NVIDIA 视频文件不存在: {video_path}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# asset 只上传一次,模型链内复用同一 assetId
|
||||||
asset_id = self._upload_asset(video_path)
|
asset_id = self._upload_asset(video_path)
|
||||||
if not asset_id:
|
if not asset_id:
|
||||||
self._cb.record_failure()
|
self._cb.record_failure()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
prompt = self._build_video_prompt(known_members_context, event_start_time)
|
prompt = self._build_video_prompt(known_members_context, event_start_time)
|
||||||
|
last_err = "no_model_in_chain"
|
||||||
|
for idx, model in enumerate(self.model_chain):
|
||||||
|
model_timeout = self.model_timeouts.get(model, self.timeout)
|
||||||
|
logger.info(f"NVIDIA 模型链 [{idx+1}/{len(self.model_chain)}] "
|
||||||
|
f"尝试 {model}(超时 {model_timeout}s)")
|
||||||
started = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
started = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||||
t0 = time.time()
|
t0 = time.time()
|
||||||
try:
|
try:
|
||||||
resp = self._client.chat.completions.create(
|
resp = self._client.chat.completions.create(
|
||||||
model=self.model_name,
|
model=model,
|
||||||
messages=[{"role": "user", "content": [
|
messages=[{"role": "user", "content": [
|
||||||
{"type": "text", "text": prompt},
|
{"type": "text", "text": prompt},
|
||||||
{"type": "video_url", "video_url": {
|
{"type": "video_url", "video_url": {
|
||||||
@@ -158,39 +178,50 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
|||||||
]}],
|
]}],
|
||||||
temperature=0.2,
|
temperature=0.2,
|
||||||
max_tokens=4096,
|
max_tokens=4096,
|
||||||
# NIM 扩展:控制视频采样帧数(模型上限 128 帧)
|
# NIM 扩展:控制视频采样帧数(部分模型支持)
|
||||||
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
|
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
|
||||||
timeout=self.timeout
|
timeout=model_timeout
|
||||||
)
|
)
|
||||||
duration = time.time() - t0
|
duration = time.time() - t0
|
||||||
content = resp.choices[0].message.content
|
content = resp.choices[0].message.content
|
||||||
if not content:
|
if not content:
|
||||||
self._emit_model_call(self.model_name, started, duration, False, "empty_content")
|
self._emit_model_call(model, started, duration, False, "empty_content")
|
||||||
logger.warning("NVIDIA 视频分析返回空 content")
|
logger.warning(f"NVIDIA [{model}] 返回空 content,切换下一模型")
|
||||||
self._cb.record_failure()
|
last_err = f"{model}_empty"
|
||||||
return None
|
self._sleep_switch(idx)
|
||||||
|
continue
|
||||||
data = self._parse_json(content)
|
data = self._parse_json(content)
|
||||||
if not data or 'events' not in data:
|
if not data or 'events' not in data:
|
||||||
self._emit_model_call(self.model_name, started, duration, False, "json_parse_failed")
|
self._emit_model_call(model, started, duration, False, "json_parse_failed")
|
||||||
logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}")
|
logger.warning(f"NVIDIA [{model}] JSON 解析失败,切换下一模型: {content[:120]}")
|
||||||
self._cb.record_failure()
|
last_err = f"{model}_json"
|
||||||
return None
|
self._sleep_switch(idx)
|
||||||
self._emit_model_call(self.model_name, started, duration, True)
|
continue
|
||||||
|
self._emit_model_call(model, started, duration, True)
|
||||||
self._cb.record_success()
|
self._cb.record_success()
|
||||||
logger.info(f"NVIDIA 整视频分析完成,events={len(data.get('events', []))}")
|
logger.info(f"NVIDIA [{model}] 整视频分析完成,events={len(data.get('events', []))}")
|
||||||
return {
|
return {
|
||||||
"global_summary": str(data.get('global_summary', '')),
|
"global_summary": str(data.get('global_summary', '')),
|
||||||
"events": data.get('events', []),
|
"events": data.get('events', []),
|
||||||
"people_mentioned": data.get('people_mentioned', []),
|
"people_mentioned": data.get('people_mentioned', []),
|
||||||
"compute_provider": "nvidia",
|
"compute_provider": f"nvidia:{model}",
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
duration = time.time() - t0
|
duration = time.time() - t0
|
||||||
self._emit_model_call(self.model_name, started, duration, False, str(e))
|
self._emit_model_call(model, started, duration, False, str(e))
|
||||||
|
last_err = f"{model}_failed"
|
||||||
|
logger.warning(f"NVIDIA [{model}] 视频分析异常,切换下一模型: {str(e)[:150]}")
|
||||||
|
self._sleep_switch(idx)
|
||||||
self._cb.record_failure()
|
self._cb.record_failure()
|
||||||
logger.warning(f"NVIDIA 视频分析异常: {e}")
|
logger.error(f"NVIDIA 模型链全部失败: {last_err}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _sleep_switch(self, idx: int):
|
||||||
|
"""模型切换间隔(最后一个模型失败后无需再等)"""
|
||||||
|
if idx < len(self.model_chain) - 1 and self.switch_interval_sec > 0:
|
||||||
|
logger.info(f"NVIDIA 等待 {self.switch_interval_sec}s 后切换下一模型")
|
||||||
|
time.sleep(self.switch_interval_sec)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_json(content: str) -> Optional[dict]:
|
def _parse_json(content: str) -> Optional[dict]:
|
||||||
content = content.strip()
|
content = content.strip()
|
||||||
|
|||||||
Reference in New Issue
Block a user