feat(fam-edge): NVIDIA 多模型降级链 - adapter 支持 model_chain(asset 上传一次逐个尝试)+switch_interval_sec 切换间隔+model_timeouts 每模型独立超时;实测全部候选不可用(omni 500/12b 400/其余 404),链机制保留待可用模型
This commit is contained in:
@@ -29,15 +29,29 @@ except ImportError:
|
||||
|
||||
|
||||
class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)"""
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)
|
||||
|
||||
多模型降级链(类似 Gemini flash -> flash-lite):
|
||||
- model_chain = [model_name] + fallback_models
|
||||
- asset 上传一次,遍历模型链逐个调用 video_url 引用同一 assetId
|
||||
- 模型失败/超时 -> 记录统计 -> 间隔 switch_interval_sec 后切换下一模型
|
||||
- 每个模型可用 model_timeouts 独立设置超时(不参与编排层 ×N 放大)
|
||||
"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("nvidia", config)
|
||||
self.model_name = config.get(
|
||||
'model_name', 'nvidia/nemotron-nano-12b-v2-vl')
|
||||
self.model_chain = [self.model_name] + [
|
||||
m for m in config.get('fallback_models', []) if m and m != self.model_name]
|
||||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||||
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
|
||||
self.timeout = config.get('timeout', 600)
|
||||
# 模型级独立超时(最终值,不参与编排层 ×N 放大): {model_name: seconds}
|
||||
self.model_timeouts = {
|
||||
str(k): int(v) for k, v in (config.get('model_timeouts') or {}).items()}
|
||||
# 模型切换间隔(秒):一个模型失败后等待再切下一个,避免连续打爆 API
|
||||
self.switch_interval_sec = float(config.get('switch_interval_sec', 5))
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 3),
|
||||
@@ -140,56 +154,73 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
logger.warning(f"NVIDIA 视频文件不存在: {video_path}")
|
||||
return None
|
||||
|
||||
# asset 只上传一次,模型链内复用同一 assetId
|
||||
asset_id = self._upload_asset(video_path)
|
||||
if not asset_id:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
prompt = self._build_video_prompt(known_members_context, event_start_time)
|
||||
started = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "video_url", "video_url": {
|
||||
"url": f"data:video/mp4;asset_id={asset_id}"}}
|
||||
]}],
|
||||
temperature=0.2,
|
||||
max_tokens=4096,
|
||||
# NIM 扩展:控制视频采样帧数(模型上限 128 帧)
|
||||
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
|
||||
timeout=self.timeout
|
||||
)
|
||||
duration = time.time() - t0
|
||||
content = resp.choices[0].message.content
|
||||
if not content:
|
||||
self._emit_model_call(self.model_name, started, duration, False, "empty_content")
|
||||
logger.warning("NVIDIA 视频分析返回空 content")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
data = self._parse_json(content)
|
||||
if not data or 'events' not in data:
|
||||
self._emit_model_call(self.model_name, started, duration, False, "json_parse_failed")
|
||||
logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._emit_model_call(self.model_name, started, duration, True)
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 整视频分析完成,events={len(data.get('events', []))}")
|
||||
return {
|
||||
"global_summary": str(data.get('global_summary', '')),
|
||||
"events": data.get('events', []),
|
||||
"people_mentioned": data.get('people_mentioned', []),
|
||||
"compute_provider": "nvidia",
|
||||
}
|
||||
except Exception as e:
|
||||
duration = time.time() - t0
|
||||
self._emit_model_call(self.model_name, started, duration, False, str(e))
|
||||
self._cb.record_failure()
|
||||
logger.warning(f"NVIDIA 视频分析异常: {e}")
|
||||
return None
|
||||
last_err = "no_model_in_chain"
|
||||
for idx, model in enumerate(self.model_chain):
|
||||
model_timeout = self.model_timeouts.get(model, self.timeout)
|
||||
logger.info(f"NVIDIA 模型链 [{idx+1}/{len(self.model_chain)}] "
|
||||
f"尝试 {model}(超时 {model_timeout}s)")
|
||||
started = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "video_url", "video_url": {
|
||||
"url": f"data:video/mp4;asset_id={asset_id}"}}
|
||||
]}],
|
||||
temperature=0.2,
|
||||
max_tokens=4096,
|
||||
# NIM 扩展:控制视频采样帧数(部分模型支持)
|
||||
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
|
||||
timeout=model_timeout
|
||||
)
|
||||
duration = time.time() - t0
|
||||
content = resp.choices[0].message.content
|
||||
if not content:
|
||||
self._emit_model_call(model, started, duration, False, "empty_content")
|
||||
logger.warning(f"NVIDIA [{model}] 返回空 content,切换下一模型")
|
||||
last_err = f"{model}_empty"
|
||||
self._sleep_switch(idx)
|
||||
continue
|
||||
data = self._parse_json(content)
|
||||
if not data or 'events' not in data:
|
||||
self._emit_model_call(model, started, duration, False, "json_parse_failed")
|
||||
logger.warning(f"NVIDIA [{model}] JSON 解析失败,切换下一模型: {content[:120]}")
|
||||
last_err = f"{model}_json"
|
||||
self._sleep_switch(idx)
|
||||
continue
|
||||
self._emit_model_call(model, started, duration, True)
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA [{model}] 整视频分析完成,events={len(data.get('events', []))}")
|
||||
return {
|
||||
"global_summary": str(data.get('global_summary', '')),
|
||||
"events": data.get('events', []),
|
||||
"people_mentioned": data.get('people_mentioned', []),
|
||||
"compute_provider": f"nvidia:{model}",
|
||||
}
|
||||
except Exception as e:
|
||||
duration = time.time() - t0
|
||||
self._emit_model_call(model, started, duration, False, str(e))
|
||||
last_err = f"{model}_failed"
|
||||
logger.warning(f"NVIDIA [{model}] 视频分析异常,切换下一模型: {str(e)[:150]}")
|
||||
self._sleep_switch(idx)
|
||||
self._cb.record_failure()
|
||||
logger.error(f"NVIDIA 模型链全部失败: {last_err}")
|
||||
return None
|
||||
|
||||
def _sleep_switch(self, idx: int):
|
||||
"""模型切换间隔(最后一个模型失败后无需再等)"""
|
||||
if idx < len(self.model_chain) - 1 and self.switch_interval_sec > 0:
|
||||
logger.info(f"NVIDIA 等待 {self.switch_interval_sec}s 后切换下一模型")
|
||||
time.sleep(self.switch_interval_sec)
|
||||
|
||||
@staticmethod
|
||||
def _parse_json(content: str) -> Optional[dict]:
|
||||
|
||||
Reference in New Issue
Block a user