[阶段2] FAM-Edge 重构为整视频分析+同步接口+人物服务 - 移除切片/抽帧/队列，新增 oracle_db/person_service/qa/watch_processor/video_processor，api_gateway 提供 /api/oracle/sync 与 /api/oracle/people/correct
This commit is contained in:
ericwyuan
@@ -2,23 +2,15 @@
|
||||
NvidiaVisionAdapter - NVIDIA NIM 云端 VLM 适配器
|
||||
|
||||
provider_name = "nvidia"
|
||||
模型: nvidia/nemotron-3-nano-omni-30b-a3b-reasoning (Omni, 原生视频输入)
|
||||
角色: vision (视觉分析直出结构化 JSON) + 智能问答
|
||||
模型: nvidia/nemotron-nano-12b-v2-vl（NIM 官方支持整视频 video_url 输入，内部自行采样帧）
|
||||
角色: vision (整视频直出结构化 JSON) + 智能问答
|
||||
SDK: openai (NIM 兼容 OpenAI API 规范)
|
||||
|
||||
视频模式 (analyze_video): 按关键帧时间点截取 ±1.5s 片段拼接集锦视频
|
||||
(片段左上角叠加原始时间戳)，base64 后经 video_url 单次调用 —
|
||||
模型看到动态画面而非静态帧，动作/轨迹识别显著优于逐帧图片。
|
||||
|
||||
图片模式 (analyze_frames): 逐帧 image_url 调用（无视频文件时的降级路径）。
|
||||
注意: nemotron-omni 是 reasoning 模型，max_tokens 需给足（reasoning 消耗 token）。
|
||||
整视频分析: 整视频 base64 经 video_url 单次调用 —— 本地不切片、不抽帧
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .base_adapter import BaseModelAdapter
|
||||
@@ -32,20 +24,17 @@ try:
|
||||
except ImportError:
|
||||
OpenAI = None
|
||||
|
||||
VIDEO_SEGMENT_PAD = 1.5 # 关键帧前后各截取秒数
|
||||
HIGHLIGHT_WIDTH = 640 # 集锦视频宽度（保持宽高比）
|
||||
|
||||
|
||||
class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (视频集锦单次调用; 逐帧降级; 文本问答)"""
|
||||
"""NVIDIA NIM 云端 VLM 适配器 (整视频单次调用; 文本问答)"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__("nvidia", config)
|
||||
self.model_name = config.get(
|
||||
'model_name', 'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning')
|
||||
'model_name', 'nvidia/nemotron-nano-12b-v2-vl')
|
||||
self.api_key = self._resolve_key(config.get('api_key', ''))
|
||||
self.base_url = config.get('base_url', 'https://integrate.api.nvidia.com/v1')
|
||||
self.timeout = config.get('timeout', 120)
|
||||
self.timeout = config.get('timeout', 600)
|
||||
cb_cfg = config.get('circuit_breaker', {})
|
||||
self._cb = CircuitBreaker(
|
||||
threshold=cb_cfg.get('threshold', 3),
|
||||
@@ -78,127 +67,29 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 视频模式：集锦视频 + video_url 单次调用（主路径）
|
||||
# 整视频分析：base64 整视频 -> video_url 单次调用
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _ts_to_seconds(ts: str) -> float:
|
||||
"""'2026-08-20 04:34:10'（生产格式）/ 'HH:MM:SS' -> 当日秒偏移"""
|
||||
s = str(ts).strip()
|
||||
# 绝对时间格式: 取时间部分（同一天 30min 片段内足够）
|
||||
date_part, _, time_part = s.partition(' ')
|
||||
if time_part and '-' in date_part:
|
||||
s = time_part
|
||||
parts = s.split(':')
|
||||
try:
|
||||
if len(parts) == 3:
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
if len(parts) == 2:
|
||||
return int(parts[0]) * 60 + float(parts[1])
|
||||
return float(ts)
|
||||
except ValueError:
|
||||
return -1.0
|
||||
|
||||
@staticmethod
|
||||
def _probe_duration(video_path: str) -> float:
|
||||
"""ffprobe 解析视频时长，失败返回 0"""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
['ffprobe', '-v', 'quiet', '-show_entries', 'format=duration',
|
||||
'-of', 'csv=p=0', video_path],
|
||||
capture_output=True, timeout=30)
|
||||
return float(r.stdout.decode().strip() or 0)
|
||||
except (subprocess.TimeoutExpired, OSError, ValueError):
|
||||
return 0.0
|
||||
|
||||
def _build_highlight_video(self, video_path: str, frame_timestamps: List[str],
|
||||
event_start_time: str = '') -> Optional[str]:
|
||||
"""按关键帧时间点截取 ±pad 秒片段，叠加时间戳后拼接集锦视频
|
||||
|
||||
时间戳以 2026-08-20 04-34-10 形式叠加（连字符避免 ffmpeg drawtext 冒号转义）。
|
||||
偏移换算: 绝对时间戳 - 视频开始时间（event_start_time 缺失时，
|
||||
时间戳值本身须已是视频内偏移，如 HH:MM:SS 相对时间）。
|
||||
"""
|
||||
start_sec = self._ts_to_seconds(event_start_time) if event_start_time else 0.0
|
||||
duration = self._probe_duration(video_path)
|
||||
clips = []
|
||||
for ts in frame_timestamps:
|
||||
sec = self._ts_to_seconds(ts)
|
||||
if sec < 0:
|
||||
continue
|
||||
if start_sec > 0:
|
||||
sec -= start_sec
|
||||
if sec < 0:
|
||||
sec += 86400 # 跨午夜
|
||||
if duration > 0 and (sec < -VIDEO_SEGMENT_PAD
|
||||
or sec > duration - 0.5):
|
||||
logger.info(f"NVIDIA 跳过超界片段: {ts} -> {sec:.1f}s (视频 {duration:.0f}s)")
|
||||
continue
|
||||
clips.append((sec, str(ts).replace(':', '-')))
|
||||
if not clips:
|
||||
return None
|
||||
|
||||
out_path = os.path.join(
|
||||
tempfile.mkdtemp(prefix='nim_highlight_'), 'highlight.mp4')
|
||||
cmd = ['ffmpeg', '-y', '-loglevel', 'error']
|
||||
for start, _ in clips:
|
||||
cmd += ['-ss', f'{start:.2f}', '-t', f'{VIDEO_SEGMENT_PAD * 2}', '-i', video_path]
|
||||
parts = []
|
||||
for i, (_, label) in enumerate(clips):
|
||||
parts.append(
|
||||
f"[{i}:v]fps=15,scale={HIGHLIGHT_WIDTH}:-2,"
|
||||
f"drawtext=text='ts {label}':x=8:y=8:fontsize=22:"
|
||||
f"fontcolor=white:box=1:boxcolor=black@0.6[v{i}]")
|
||||
concat_in = ''.join(f'[v{i}]' for i in range(len(clips)))
|
||||
parts.append(f'{concat_in}concat=n={len(clips)}:v=1:a=0[out]')
|
||||
cmd += ['-filter_complex', ';'.join(parts), '-map', '[out]',
|
||||
'-r', '15',
|
||||
'-c:v', 'libx264', '-preset', 'veryfast', '-crf', '28',
|
||||
'-an', out_path]
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True, timeout=120)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("NVIDIA 集锦视频生成超时")
|
||||
return None
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f"NVIDIA 集锦视频生成失败: {e.stderr.decode()[:200] if e.stderr else e}")
|
||||
return None
|
||||
size = os.path.getsize(out_path)
|
||||
logger.info(f"NVIDIA 集锦视频生成: {len(clips)} 片段, {size // 1024}KB")
|
||||
if size < 1024:
|
||||
return None
|
||||
return out_path
|
||||
|
||||
def analyze_video(self, video_path: str,
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str,
|
||||
event_start_time: str = '') -> Optional[Dict]:
|
||||
"""原生视频输入分析: 集锦片段 -> video_url 单次调用"""
|
||||
known_members_context: str,
|
||||
event_start_time: str = '') -> Optional[Dict]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("NVIDIA 熔断器 OPEN，跳过视频分析")
|
||||
return None
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化，跳过视频分析")
|
||||
return None
|
||||
|
||||
highlight = self._build_highlight_video(
|
||||
video_path, frame_timestamps, event_start_time)
|
||||
if not highlight:
|
||||
logger.warning("NVIDIA 集锦视频不可用，降级逐帧模式")
|
||||
if not os.path.isfile(video_path):
|
||||
logger.warning(f"NVIDIA 视频文件不存在: {video_path}")
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(highlight, 'rb') as f:
|
||||
with open(video_path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 读取集锦视频失败: {e}")
|
||||
logger.warning(f"NVIDIA 读取视频失败: {e}")
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
os.remove(highlight)
|
||||
os.rmdir(os.path.dirname(highlight))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
prompt = self._build_video_prompt(frame_timestamps, known_members_context)
|
||||
prompt = self._build_video_prompt(known_members_context, event_start_time)
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
@@ -208,134 +99,47 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
"url": f"data:video/mp4;base64,{b64}"}}
|
||||
]}],
|
||||
temperature=0.2,
|
||||
max_tokens=3072,
|
||||
max_tokens=4096,
|
||||
# NIM 扩展：控制视频采样帧数（模型上限 128 帧）
|
||||
extra_body={"media_io_kwargs": {"video": {"num_frames": 128}}},
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
if not content:
|
||||
logger.warning("NVIDIA 视频分析返回空 content")
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
data = self._parse_single_frame_json(content)
|
||||
if not data or 'frame_details' not in data:
|
||||
data = self._parse_json(content)
|
||||
if not data or 'events' not in data:
|
||||
logger.warning(f"NVIDIA 视频 JSON 解析失败: {content[:150]}")
|
||||
return None
|
||||
frame_details = self._normalize_frame_details(data, frame_timestamps)
|
||||
if not frame_details:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视频分析完成，frame_details={len(frame_details)}")
|
||||
result = {"frame_details": frame_details}
|
||||
if data.get('global_summary'):
|
||||
result['global_summary'] = str(data['global_summary'])
|
||||
if data.get('entities_json'):
|
||||
result['entities_json'] = data['entities_json']
|
||||
return result
|
||||
logger.info(f"NVIDIA 整视频分析完成，events={len(data.get('events', []))}")
|
||||
return {
|
||||
"global_summary": str(data.get('global_summary', '')),
|
||||
"events": data.get('events', []),
|
||||
"people_mentioned": data.get('people_mentioned', []),
|
||||
"compute_provider": "nvidia",
|
||||
}
|
||||
except Exception as e:
|
||||
self._cb.record_failure()
|
||||
logger.warning(f"NVIDIA 视频分析异常: {e}")
|
||||
return None
|
||||
|
||||
def _normalize_frame_details(self, data: dict,
|
||||
frame_timestamps: List[str]) -> List[Dict]:
|
||||
"""归一化模型输出的 frame_details，按已知时间戳对齐"""
|
||||
details = []
|
||||
for i, item in enumerate(data.get('frame_details', []), 1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
ts = str(item.get('frame_timestamp',
|
||||
frame_timestamps[i - 1] if i <= len(frame_timestamps) else ''))
|
||||
details.append({
|
||||
"frame_index": i,
|
||||
"frame_timestamp": ts,
|
||||
"person": str(item.get('person', '无人')),
|
||||
"action": str(item.get('action', '')),
|
||||
"clothing": str(item.get('clothing', '')),
|
||||
"is_attention_event": bool(item.get('is_attention_event', False)),
|
||||
"source_providers": ["nvidia"],
|
||||
})
|
||||
return details
|
||||
|
||||
def _build_video_prompt(self, frame_timestamps: List[str],
|
||||
known_members: str) -> str:
|
||||
ts_list = '\n'.join(f' 片段{i}: 原始时间 {ts}' for i, ts in enumerate(frame_timestamps, 1))
|
||||
return f"""你是家庭监控视频分析助手。下面的视频是由一段长时间监控录像中抽取的片段集锦，
|
||||
共 {len(frame_timestamps)} 个片段（每个约 3 秒），按顺序拼接。每个片段左上角叠加了
|
||||
原始时间戳（ts 后的 2026-08-20 04-34-10 表示北京时间 2026年8月20日 04:34:10）。
|
||||
|
||||
片段时间对照：
|
||||
{ts_list}
|
||||
|
||||
只输出合法 JSON（不要 markdown、不要解释），结构如下：
|
||||
{{
|
||||
"frame_details": [
|
||||
{{
|
||||
"frame_timestamp": "<片段原始时间>",
|
||||
"person": "片段中的人物或'无人'",
|
||||
"action": "片段中人物的动作（动态观察，如走动/跑动/坐下）",
|
||||
"clothing": "衣着(颜色+类型)",
|
||||
"is_attention_event": false
|
||||
}}
|
||||
],
|
||||
"global_summary": "整段录像的综合摘要",
|
||||
"entities_json": [{{"person": "人物名或人物X", "action": "行为概括", "clothing": "衣着"}}]
|
||||
}}
|
||||
|
||||
规则：
|
||||
1. 只描述客观画面，不猜测。
|
||||
2. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
3. is_attention_event：跌倒、危险、异常哭闹等需关注事件（没有则为 false）。
|
||||
4. 无人出现的片段 person 填"无人"，action 填""。"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 图片模式：逐帧调用（降级路径，无视频文件时使用）
|
||||
# ------------------------------------------------------------------
|
||||
def analyze_frames(self, frame_paths: List[str],
|
||||
frame_timestamps: List[str],
|
||||
known_members_context: str) -> Optional[Dict]:
|
||||
if self._cb.is_open():
|
||||
logger.warning("NVIDIA 熔断器 OPEN，跳过调用")
|
||||
return None
|
||||
if self._client is None:
|
||||
logger.warning("NVIDIA 客户端未初始化，跳过调用")
|
||||
return None
|
||||
if not frame_paths:
|
||||
logger.warning("NVIDIA 无帧可分析")
|
||||
return None
|
||||
|
||||
frame_details = []
|
||||
ok = False
|
||||
for i, (path, ts) in enumerate(zip(frame_paths, frame_timestamps), 1):
|
||||
detail = self._analyze_one_structured(path, ts, i, known_members_context)
|
||||
if detail:
|
||||
frame_details.append(detail)
|
||||
ok = True
|
||||
|
||||
if not ok:
|
||||
self._cb.record_failure()
|
||||
return None
|
||||
|
||||
self._cb.record_success()
|
||||
logger.info(f"NVIDIA 视觉分析完成，frame_details={len(frame_details)}")
|
||||
# NVIDIA 单帧无法跨帧综合 global_summary，交由 Edge format_cloud_result 格式化生成
|
||||
return {"frame_details": frame_details}
|
||||
|
||||
def _parse_single_frame_json(self, content: str) -> Optional[dict]:
|
||||
"""轻量解析单帧 JSON（不要求全 schema，仅提取字段）"""
|
||||
@staticmethod
|
||||
def _parse_json(content: str) -> Optional[dict]:
|
||||
content = content.strip()
|
||||
# 直接解析
|
||||
try:
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# 提取 markdown fence
|
||||
fence = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL)
|
||||
if fence:
|
||||
try:
|
||||
return json.loads(fence.group(1))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# 贪婪匹配最大 {...}
|
||||
brace = re.search(r'\{.*\}', content, re.DOTALL)
|
||||
if brace:
|
||||
try:
|
||||
@@ -344,68 +148,35 @@ class NvidiaVisionAdapter(BaseModelAdapter):
|
||||
pass
|
||||
return None
|
||||
|
||||
def _analyze_one_structured(self, path: str, ts: str, idx: int,
|
||||
known_members: str) -> Optional[Dict]:
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"读取图片失败 {path}: {e}")
|
||||
return None
|
||||
|
||||
prompt = self._build_structured_prompt(ts, idx, known_members)
|
||||
try:
|
||||
resp = self._client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
|
||||
]}],
|
||||
temperature=0.2,
|
||||
max_tokens=512,
|
||||
timeout=self.timeout
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
if not content:
|
||||
return None
|
||||
data = self._parse_single_frame_json(content)
|
||||
if not data:
|
||||
logger.warning(f"NVIDIA 单帧 JSON 解析失败: {content[:120]}")
|
||||
return None
|
||||
return {
|
||||
"frame_index": idx,
|
||||
"frame_timestamp": str(data.get("frame_timestamp", ts)),
|
||||
"person": str(data.get("person", "无人")),
|
||||
"action": str(data.get("action", "")),
|
||||
"clothing": str(data.get("clothing", "")),
|
||||
"is_attention_event": bool(data.get("is_attention_event", False)),
|
||||
"source_providers": ["nvidia"],
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"NVIDIA 单帧异常: {e}")
|
||||
return None
|
||||
|
||||
def _build_structured_prompt(self, ts: str, idx: int, known_members: str) -> str:
|
||||
return f"""你是家庭监控视频分析助手。请看这张监控截图（拍摄时间 {ts}）。
|
||||
只输出合法 JSON（不要 markdown、不要解释），结构如下：
|
||||
def _build_video_prompt(self, known_members: str, event_start_time: str) -> str:
|
||||
start_hint = ""
|
||||
if event_start_time:
|
||||
start_hint = f"\n视频开始时间（北京时间）约为：{event_start_time}。请据此推算每个事件的绝对时间戳。"
|
||||
return f"""你是家庭监控视频分析助手。下面是一段完整监控录像（已整段上传）。
|
||||
请观看整段视频，提取其中有用的信息，只输出合法 JSON（不要 markdown、不要解释），结构如下：
|
||||
|
||||
{{
|
||||
"frame_timestamp": "{ts}",
|
||||
"person": "该帧画面中的人物或'无人'",
|
||||
"action": "该帧可见动作",
|
||||
"clothing": "该帧衣着(颜色+类型)",
|
||||
"is_attention_event": false
|
||||
}}
|
||||
"global_summary": "整个时段的整体摘要，简体中文，2-4 句",
|
||||
"events": [
|
||||
{{
|
||||
"timestamp": "事件发生时的绝对北京时间(YYYY-MM-DD HH:MM:SS)",
|
||||
"description": "该时刻画面/动作信息摘要",
|
||||
"people": ["出现在该时刻的人物，用已知成员真名或'人物A'"],
|
||||
"is_attention_event": false
|
||||
}}
|
||||
],
|
||||
"people_mentioned": ["本视频出现过的所有人物标识/真名"]
|
||||
}}{start_hint}
|
||||
|
||||
规则：
|
||||
1. 只描述客观画面，不猜测。
|
||||
2. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
2. events 提取有意义的时间点（人物出现/动作变化/异常），timestamp 用绝对北京时间。
|
||||
3. 已知家庭成员（按特征匹配，匹配到用 real_name，否则用"人物X"）：
|
||||
{known_members or '（暂无已知成员）'}
|
||||
3. is_attention_event：是否为跌倒、危险、异常哭闹等需关注事件（没有则为 false）。
|
||||
4. 没有人物出现的帧 person 填"无人"，action 填""。"""
|
||||
4. is_attention_event：跌倒、危险、异常哭闹等需关注事件（没有则为 false）。"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 智能问答：纯文本（reasoning 模型，max_tokens 需给足）
|
||||
# 智能问答：纯文本
|
||||
# ------------------------------------------------------------------
|
||||
def chat(self, prompt: str, max_tokens: int = 2048) -> Optional[str]:
|
||||
if self._client is None:
|
||||
|
||||
Reference in New Issue
Block a user