feat(fam-edge): 新增 DiskGuard 磁盘空间守护,剩余空间不足自动清理旧素材

背景:Oracle 磁盘曾经被写满(gdrive_videos 持续下载新素材、旧文件迟迟没
清理),触发 rclone 的一个安全机制——同步遇到 IO 错误就整体拒绝执行删除,
形成"越满越删不掉,越删不掉越满"的死循环,最终连新视频都下载不了。这次
排查+手动清理已经解决了当次故障,但需要一道独立于 rclone 同步之外的兜底,
防止再次悄悄写满没人发现。

- oracle_db.py 新增 get_oldest_purgeable_material():只挑最旧的、已完成
  分割阶段(status='done')的整段素材(非 motion_ 前缀),绝不碰运动片段
  (事件时间轴/人物头像依赖它)和还在处理中的素材
- disk_guard.py 新增 DiskGuard 后台线程:5 分钟检查一次,剩余空间 <10GB
  触发清理,删到 15GB 水位为止(留缓冲避免刚清完又立刻触发),复用已有的
  delete_video() 完成实际删除
- app.py 启动这个后台服务;/api/oracle/activity 新增 disk 字段(实时剩余
  空间 + 最近一次清理动作),供服务状态页展示
- 新增 9 个单元测试,全部通过(139/139)

已部署 Oracle 验证:DiskGuard 正常启动,配置生效。

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ericwyuan
2026-08-28 12:08:04 +08:00
parent 3ec79de911
commit ef56ae5662
7 changed files with 320 additions and 1 deletions

View File

@@ -0,0 +1,120 @@
import pytest
from fam_edge.disk_guard import DiskGuard
def _cfg(**overrides):
base = {
"enabled": True,
"min_free_gb": 10,
"target_free_gb": 15,
"watch_path": "/opt/fam-edge",
"max_delete_per_round": 50,
"check_interval_sec": 300,
}
base.update(overrides)
return base
class _FakeDB:
def __init__(self, candidates=None):
self._candidates = list(candidates or [])
self.deleted_ids = []
self.activities = []
def get_oldest_purgeable_material(self):
if not self._candidates:
return None
return self._candidates.pop(0)
def delete_video(self, video_id):
self.deleted_ids.append(video_id)
return None
def record_activity(self, service, action, detail=''):
self.activities.append((service, action, detail))
def _guard(monkeypatch, db, free_gb_sequence, **cfg_overrides):
"""free_gb_sequence: 每次调用 _free_gb() 依次返回的值列表(最后一个值
耗尽后保持不变),用来模拟"清理一个文件后空间逐步恢复"的过程。"""
monkeypatch.setattr(
"fam_edge.disk_guard.load_config",
lambda: {"disk_guard": _cfg(**cfg_overrides)})
guard = DiskGuard(db)
seq = list(free_gb_sequence)
def fake_free_gb():
if len(seq) > 1:
return seq.pop(0)
return seq[0]
monkeypatch.setattr(guard, "_free_gb", fake_free_gb)
return guard
def test_check_once_does_nothing_when_space_sufficient(monkeypatch):
db = _FakeDB()
guard = _guard(monkeypatch, db, [20.0])
guard.check_once()
assert db.deleted_ids == []
assert db.activities == []
def test_check_once_cleans_until_target_reached(monkeypatch):
"""核心诉求: 低于 min_free_gb 触发清理,一直清到 target_free_gb 为止,
不是清一个就停(否则马上又会跌破阈值,频繁触发)。"""
db = _FakeDB(candidates=[
{"id": 1, "local_path": "/tmp/a.mp4"},
{"id": 2, "local_path": "/tmp/b.mp4"},
{"id": 3, "local_path": "/tmp/c.mp4"},
])
# 初始 8GB< min_free_gb=10每删一个恢复到 8/12/16GB16 >= target=15 时停)
guard = _guard(monkeypatch, db, [8.0, 8.0, 12.0, 16.0])
guard.check_once()
assert db.deleted_ids == [1, 2]
actions = [a[1] for a in db.activities]
assert "low_space" in actions
assert "cleaned" in actions
def test_check_once_stops_when_no_candidate_left(monkeypatch):
"""核心诉求: 候选清空了但空间依然不足,不能死循环,要停下来并记录一条
"没有可清理素材"的警告,让人能在服务状态页看到这个异常情况——即便如此,
已经发生的清理动作本身也要记录(能看到确实清过、释放了多少),不因为
没完全达标就把 cleaned 记录吞掉。"""
db = _FakeDB(candidates=[{"id": 1, "local_path": "/tmp/a.mp4"}])
guard = _guard(monkeypatch, db, [8.0, 8.0, 9.0]) # 删完仅 1 个后仍然 <15GB
guard.check_once()
assert db.deleted_ids == [1]
actions = [a[1] for a in db.activities]
assert "no_candidate" in actions
assert "cleaned" in actions
def test_check_once_respects_max_delete_per_round(monkeypatch):
"""核心诉求: 单轮清理有上限,防止候选异常多时一次性删太多——下一轮检查
很快就会再触发,没必要在一轮里清空所有候选。"""
candidates = [{"id": i, "local_path": f"/tmp/{i}.mp4"} for i in range(1, 6)]
db = _FakeDB(candidates=candidates)
# 空间一直卡在 8GB 不涨(模拟每个文件都很小,删多少都到不了 target
guard = _guard(monkeypatch, db, [8.0], max_delete_per_round=3)
guard.check_once()
assert db.deleted_ids == [1, 2, 3]
def test_disabled_guard_does_not_start(monkeypatch):
db = _FakeDB()
guard = _guard(monkeypatch, db, [1.0], enabled=False)
guard.start()
assert guard.is_alive() is False

View File

@@ -407,6 +407,45 @@ def test_delete_video_nonexistent_returns_none(tmp_path):
assert db.delete_video(99999) is None
def test_get_oldest_purgeable_material_none_when_empty(tmp_path):
db = _db(tmp_path)
assert db.get_oldest_purgeable_material() is None
def test_get_oldest_purgeable_material_ignores_motion_clips(tmp_path):
"""核心诉求: 运动片段motion_ 前缀)是独立的分析产物,事件时间轴/人物
头像都依赖它,磁盘清理绝不能碰它,只能清理原始整段素材。"""
db = _db(tmp_path)
vid = db.ensure_video("motion_1_1000.mp4", "/tmp/motion_1_1000.mp4",
event_start_time="2026-08-22 10:00:00")
db.mark_video_processed(vid, "摘要", [], [], "gemini")
assert db.get_oldest_purgeable_material() is None
def test_get_oldest_purgeable_material_ignores_non_done_status(tmp_path):
"""核心诉求: 还在 pending/processing 的素材不能被清理,避免删掉还没
来得及处理的数据。"""
db = _db(tmp_path)
db.ensure_video("Generic_ONVIF-001-20260815-000000.mp4",
"/tmp/Generic_ONVIF-001-20260815-000000.mp4")
assert db.get_oldest_purgeable_material() is None
def test_get_oldest_purgeable_material_returns_oldest_done_material(tmp_path):
db = _db(tmp_path)
vid1 = db.ensure_video("Generic_ONVIF-001-20260815-000000.mp4",
"/tmp/Generic_ONVIF-001-20260815-000000.mp4")
db.mark_video_processed(vid1, "(整段素材已分割 0 段运动片段)", [], [], 'motion_segment')
vid2 = db.ensure_video("Generic_ONVIF-001-20260816-000000.mp4",
"/tmp/Generic_ONVIF-001-20260816-000000.mp4")
db.mark_video_processed(vid2, "(整段素材已分割 0 段运动片段)", [], [], 'motion_segment')
candidate = db.get_oldest_purgeable_material()
assert candidate["id"] == vid1
assert candidate["local_path"] == "/tmp/Generic_ONVIF-001-20260815-000000.mp4"
def test_delete_video_does_not_touch_ss_motion_events(tmp_path):
"""核心诉求: ss_motion_events 是运动侦测源事件,跟切出来的视频片段生命周期
独立,删视频不该连带删掉源事件(否则分割逻辑的幂等判断会被破坏)。"""