refactor: 全量迁云——fam-core 直读甲骨文 SQLite，NAS 只剩推送进程
前一天刚把登录迁到甲骨文，隔天 NAS 上的 fam-core 又挂了导致数据接口 502。 盘点后确认：甲骨文的 SQLite 才是权威数据源（videos 3113 / events 16159 / people 60 / model_calls 9876），NAS 的 MariaDB 全是它的镜像——前端读的数据 本来就产自甲骨文，绕了一圈回家又绕回来。 改动： - db_layer.py 从 725 行重写成 377 行：MySQL 镜像查询改为直读 fam-edge 的 SQLite。5 个 upsert_sync_*（约 300 行去重逻辑，8/29 和 9/3 两次 1062 事故的 发源地）连同 oracle_sync.py 整个删除。SQL 方言：JSON_CONTAINS -> json_each （前置 json_valid，历史脏数据不会把查询搞崩）、LEFT() -> substr()、%s -> ?。 函数名 get_sync_* 一并改掉——已经没有 sync 这回事了 - 新增 edge_client.py：写操作（改名/删除）、帧图头像、服务状态都打给同机 fam-edge，全走 127.0.0.1 - 新增 fam-notifier/：motion_notifier 从 fam-core 拆出独立成服务，游标从 MariaDB 换成本地 JSON 文件。NAS 上从此没有 Flask、没有数据库、没有监听端口 - fam-core 移到甲骨文 /opt/fam-core（systemd，gunicorn -w 2，只绑 127.0.0.1:5401——5400 被 chat-relay 占了）。Caddy 的 /api/* 从"frp 隧道 回源 NAS"改成同机反代，forward_auth 闸门不变 - 前端删掉侧边栏同步面板、统计页同步状态、服务状态页的"NAS 同步"卡片与 "立即同步"按钮（背后的镜像层已不存在）；换成"NAS 运动推送"卡片，读 fam-edge activity 新增的 motion 段（心跳年龄 + 最近事件） - 顺带修掉一个隐蔽 bug：镜像表为保外键稳定用的是 NAS 本地自增 id，而帧图接口 要的是甲骨文的 id，两边在 9/3 那次 id 重排后就对不上了。现在只有一套 id 测试：fam-core 21（新增 12 个 db_layer 用例：脏 JSON 不崩、人物精确匹配不误伤 "人物B"、日期过滤、统计口径、chat_history 懒建表）、fam-notifier 6、 fam-edge 157，全绿。 生产验证：甲骨文 /api/ui/stats 返回 videos 2965 / events 16159 / people 59； NAS 侧 fam-notifier 已推送成功（事件 33070-33072 落库，心跳新鲜）； chat_history 19 条经 scripts/import_chat_history.py 迁移完成。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ericwyuan
155
fam-core/tests/test_db_layer.py
Normal file
155
fam-core/tests/test_db_layer.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""db_layer 直读 SQLite 的单测（2026-09-13 迁云重写后新增）。
|
||||
|
||||
重点验证从 MySQL 翻译过来的几处方言：json_each 取代 JSON_CONTAINS、
|
||||
substr 取代 LEFT、以及历史脏数据（person_list_json 不是合法 JSON）不能
|
||||
把整条查询搞崩——这在 MySQL 下 JSON_CONTAINS 会直接报错，SQLite 下
|
||||
json_each 同样会，所以查询里加了 json_valid 前置判断。
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from fam_core import db_layer
|
||||
|
||||
# fam-edge 建表语句的最小子集（列名与 oracle_db.py 保持一致）
|
||||
_SCHEMA = """
|
||||
CREATE TABLE videos (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT UNIQUE, camera_name TEXT,
|
||||
duration_sec REAL, event_start_time TEXT, status TEXT DEFAULT 'pending',
|
||||
summary_json TEXT, events_json TEXT, people_json TEXT, compute_provider TEXT,
|
||||
created_at TEXT, updated_at TEXT, processed_at TEXT);
|
||||
CREATE TABLE events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, video_id INTEGER, ts TEXT, description TEXT,
|
||||
person_list_json TEXT, person_appearances_json TEXT, is_attention_event INTEGER DEFAULT 0);
|
||||
CREATE TABLE people (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT UNIQUE, canonical_name TEXT,
|
||||
first_seen TEXT, appearances INTEGER DEFAULT 0, source TEXT, features_json TEXT,
|
||||
display_uid TEXT, updated_at TEXT);
|
||||
CREATE TABLE model_calls (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, provider TEXT, model TEXT, video_id INTEGER,
|
||||
filename TEXT, started_at TEXT, duration_sec REAL, success INTEGER, error TEXT,
|
||||
created_at TEXT);
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path, monkeypatch):
|
||||
path = tmp_path / "oracle.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript(_SCHEMA)
|
||||
conn.executescript("""
|
||||
INSERT INTO videos (id, filename, camera_name, event_start_time, status, processed_at)
|
||||
VALUES (1, 'motion_20260913_080000.mp4', '客厅', '2026-09-13 08:00:00', 'done', '2026-09-13 08:10:00'),
|
||||
(2, 'whole_20260912_090000.mp4', '客厅', '2026-09-12 09:00:00', 'done', '2026-09-12 09:10:00'),
|
||||
(3, 'whole_20260911_090000.mp4', '客厅', '2026-09-11 09:00:00', 'done', '2026-09-11 09:10:00'),
|
||||
(4, 'motion_20260910_070000.mp4', '客厅', '2026-09-10 07:00:00', 'pending', NULL);
|
||||
-- video 2 有事件（应展示），video 3 没有事件且不是 motion_（应被过滤掉）
|
||||
INSERT INTO events (video_id, ts, description, person_list_json, is_attention_event)
|
||||
VALUES (1, '00:00:05', '汤圆在客厅玩耍', '["汤圆"]', 0),
|
||||
(1, '00:00:20', '有人靠近门口', '["人物B"]', 1),
|
||||
(2, '00:01:00', '汤圆和奶奶', '["汤圆","奶奶"]', 0),
|
||||
(2, '00:02:00', '脏数据事件', '不是合法JSON', 0);
|
||||
INSERT INTO people (id, label, canonical_name, appearances, source)
|
||||
VALUES (1, '人物A', '汤圆', 12, 'manual'),
|
||||
(2, '人物B', NULL, 3, 'llm'),
|
||||
(3, '汤圆', '汤圆', 5, 'manual');
|
||||
INSERT INTO model_calls (provider, model, success, duration_sec, created_at)
|
||||
VALUES ('gemini', 'flash', 1, 10.0, '2026-09-13 08:10:00'),
|
||||
('gemini', 'flash', 0, 20.0, '2026-09-13 08:20:00'),
|
||||
('nvidia', 'vila', 1, 5.0, '2026-09-12 08:10:00');
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
monkeypatch.setattr(db_layer, '_db_path', str(path))
|
||||
monkeypatch.setattr(db_layer, '_chat_schema_ready', False)
|
||||
return path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- videos
|
||||
def test_get_videos_only_returns_content_sessions(db):
|
||||
"""只展示运动片段或含事件的会话：分割 0 段的整段素材不该淹没时间轴。"""
|
||||
ids = [v['id'] for v in db_layer.get_videos()]
|
||||
assert ids == [1, 2] # 3 无事件且非 motion_，4 未完成
|
||||
assert db_layer.get_videos()[0]['event_count'] == 2
|
||||
|
||||
|
||||
def test_get_videos_date_filter_and_paging(db):
|
||||
assert [v['id'] for v in db_layer.get_videos(date_filter='2026-09-12')] == [2]
|
||||
assert [v['id'] for v in db_layer.get_videos(limit=1, offset=1)] == [2]
|
||||
|
||||
|
||||
def test_get_video_and_events(db):
|
||||
assert db_layer.get_video(1)['filename'].startswith('motion_')
|
||||
assert db_layer.get_video(999) is None
|
||||
evs = db_layer.get_events_for_video(1)
|
||||
assert [e['ts'] for e in evs] == ['00:00:05', '00:00:20']
|
||||
|
||||
|
||||
def test_get_attention_events(db):
|
||||
rows = db_layer.get_attention_events()
|
||||
assert len(rows) == 1 and rows[0]['ev_date'].startswith('2026-09-13')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 人物命中（JSON）
|
||||
def test_people_clips_matches_label_and_its_aliases(db):
|
||||
"""'人物A' 的规范名是"汤圆"，而事件里记的是"汤圆"——要能顺着别名找到。"""
|
||||
clips = db_layer.get_people_clips('人物A')
|
||||
assert {c['video_id'] for c in clips} == {1, 2}
|
||||
assert clips[0]['video_id'] == 1 # 按录制时间倒序
|
||||
assert clips[0]['first_ts'] == '00:00:05'
|
||||
assert clips[0]['clip_events'] == 1
|
||||
|
||||
|
||||
def test_person_hit_is_exact_not_substring(db):
|
||||
"""json_each 是精确匹配：查"人物"不该命中"人物B"。"""
|
||||
assert db_layer.get_people_clips('人物') == []
|
||||
assert {c['video_id'] for c in db_layer.get_people_clips('人物B')} == {1}
|
||||
|
||||
|
||||
def test_malformed_person_json_does_not_break_queries(db):
|
||||
"""video 2 里混了一条非 JSON 的脏数据，查询必须照常返回而不是整条报错。"""
|
||||
rows = db_layer.query_events_for_person_date('汤圆', '2026-09-12')
|
||||
assert len(rows) == 1 and rows[0]['ts'] == '00:01:00'
|
||||
|
||||
|
||||
def test_query_events_for_person_date(db):
|
||||
assert db_layer.query_events_for_person_date('汤圆', '2026-09-13')[0]['description'] == '汤圆在客厅玩耍'
|
||||
assert db_layer.query_events_for_person_date('汤圆', '2026-09-01') == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- people / model_calls
|
||||
def test_people_and_named_members(db):
|
||||
assert len(db_layer.get_people()) == 3
|
||||
assert db_layer.get_named_members() == ['汤圆'] # distinct 且非空
|
||||
ctx = db_layer.get_known_members_context()
|
||||
assert '- 汤圆（标识：人物A）' in ctx and '- 人物B' in ctx
|
||||
|
||||
|
||||
def test_model_calls_stats(db):
|
||||
stats = {s['model']: s for s in db_layer.get_model_calls_stats()}
|
||||
assert stats['flash']['ok_cnt'] == 1 and stats['flash']['fail_cnt'] == 1
|
||||
assert stats['flash']['avg_duration'] == 15.0
|
||||
assert len(db_layer.get_model_calls(limit=2)) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 统计
|
||||
def test_stats_全量与按日(db):
|
||||
total = db_layer.get_stats()
|
||||
assert total['videos'] == 2 and total['events'] == 4 and total['attention'] == 1
|
||||
day = db_layer.get_stats('2026-09-13')
|
||||
assert day['videos'] == 1 and day['events'] == 2
|
||||
# 人物数：汤圆（含 label 人物A/汤圆两行都归一到"汤圆"）+ 人物B
|
||||
assert total['people'] >= 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- chat_history
|
||||
def test_chat_history_roundtrip_creates_table_on_demand(db):
|
||||
"""chat_history 是本模块唯一写的表，建表是懒加载的（库文件属于 fam-edge）。"""
|
||||
chat_id = db_layer.insert_chat_history('今天有人来吗', '有，08:00 有人靠近门口',
|
||||
'上下文', '2026-09-13', '汤圆')
|
||||
assert chat_id == 1
|
||||
rows = db_layer.get_chat_history(limit=10)
|
||||
assert rows[0]['user_question'] == '今天有人来吗'
|
||||
assert rows[0]['created_at']
|
||||
assert db_layer.get_chat_history(person_filter='不存在的人') == []
|
||||
assert len(db_layer.get_chat_history(date_filter='2026-09-13')) == 1
|
||||
@@ -1,114 +0,0 @@
|
||||
import time
|
||||
|
||||
from fam_core.motion_notifier.motion_notifier import MotionNotifier
|
||||
|
||||
|
||||
def _cfg(**overrides):
|
||||
base = {
|
||||
"enabled": True,
|
||||
"poll_enabled": False,
|
||||
"dsm_host": "192.168.50.64",
|
||||
"dsm_port": 5000,
|
||||
"dsm_account": "ericwyuan",
|
||||
"dsm_password": "secret",
|
||||
"oracle_base_url": "http://oracle.example:5000",
|
||||
"oracle_token": "tok123",
|
||||
"heartbeat_interval_sec": 300,
|
||||
"timeout_sec": 5,
|
||||
}
|
||||
base.update(overrides)
|
||||
return {"motion_notifier": base}
|
||||
|
||||
|
||||
def _make_notifier(monkeypatch, **cfg_overrides):
|
||||
monkeypatch.setattr(
|
||||
"fam_core.motion_notifier.motion_notifier.load_config",
|
||||
lambda: _cfg(**cfg_overrides))
|
||||
return MotionNotifier()
|
||||
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, status_code=200, payload=None, text=""):
|
||||
self.status_code = status_code
|
||||
self._payload = payload or {}
|
||||
self.text = text
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 心跳（跟轮询无关，enabled=true 就该跑）
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_send_heartbeat_success_updates_state(monkeypatch):
|
||||
n = _make_notifier(monkeypatch)
|
||||
calls = []
|
||||
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
calls.append((url, json))
|
||||
return _FakeResp(200, {"status": "ok", "received": 0, "stored": 0})
|
||||
monkeypatch.setattr("fam_core.motion_notifier.motion_notifier.requests.post", fake_post)
|
||||
ok = n.send_heartbeat()
|
||||
assert ok is True
|
||||
assert n._last_heartbeat_at is not None
|
||||
assert n._last_heartbeat_error is None
|
||||
assert calls[0][0] == "http://oracle.example:5000/api/ss/motion"
|
||||
assert calls[0][1]["events"] == []
|
||||
assert calls[0][1]["token"] == "tok123"
|
||||
|
||||
|
||||
def test_send_heartbeat_http_error_records_failure(monkeypatch):
|
||||
n = _make_notifier(monkeypatch)
|
||||
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
return _FakeResp(500, {}, text="boom")
|
||||
monkeypatch.setattr("fam_core.motion_notifier.motion_notifier.requests.post", fake_post)
|
||||
ok = n.send_heartbeat()
|
||||
assert ok is False
|
||||
assert n._last_heartbeat_error == "HTTP 500"
|
||||
|
||||
|
||||
def test_send_heartbeat_network_exception_records_failure(monkeypatch):
|
||||
import requests as _requests
|
||||
n = _make_notifier(monkeypatch)
|
||||
|
||||
def fake_post(url, json=None, timeout=None):
|
||||
raise _requests.RequestException("connection refused")
|
||||
monkeypatch.setattr("fam_core.motion_notifier.motion_notifier.requests.post", fake_post)
|
||||
ok = n.send_heartbeat()
|
||||
assert ok is False
|
||||
assert "connection refused" in n._last_heartbeat_error
|
||||
|
||||
|
||||
def test_start_runs_heartbeat_thread_even_when_poll_disabled(monkeypatch):
|
||||
"""核心诉求: 心跳跟"是否轮询 SS"是两回事——poll_enabled=false 时轮询线程
|
||||
不应该启动，但心跳线程必须照样跑，否则甲骨文永远收不到心跳，
|
||||
has_motion_in_range_local() 会一直 fail-open，省配额的效果就没了。"""
|
||||
n = _make_notifier(monkeypatch, poll_enabled=False, heartbeat_interval_sec=3600)
|
||||
monkeypatch.setattr(
|
||||
"fam_core.motion_notifier.motion_notifier.requests.post",
|
||||
lambda url, json=None, timeout=None: _FakeResp(200, {"stored": 0}))
|
||||
try:
|
||||
n.start()
|
||||
time.sleep(0.2)
|
||||
assert n.is_alive() is False # 轮询线程未启动
|
||||
assert n._hb_thread is not None and n._hb_thread.is_alive()
|
||||
finally:
|
||||
n.stop()
|
||||
|
||||
|
||||
def test_start_does_nothing_when_disabled(monkeypatch):
|
||||
n = _make_notifier(monkeypatch, enabled=False)
|
||||
n.start()
|
||||
time.sleep(0.1)
|
||||
assert n._hb_thread is None
|
||||
assert n.is_alive() is False
|
||||
|
||||
|
||||
def test_status_includes_heartbeat_fields(monkeypatch):
|
||||
n = _make_notifier(monkeypatch)
|
||||
st = n.status()
|
||||
assert "heartbeat_running" in st
|
||||
assert "last_heartbeat_at" in st
|
||||
assert "last_heartbeat_error" in st
|
||||
Reference in New Issue
Block a user