feat(fam-edge): 新增 DiskGuard 磁盘空间守护,剩余空间不足自动清理旧素材
背景:Oracle 磁盘曾经被写满(gdrive_videos 持续下载新素材、旧文件迟迟没 清理),触发 rclone 的一个安全机制——同步遇到 IO 错误就整体拒绝执行删除, 形成"越满越删不掉,越删不掉越满"的死循环,最终连新视频都下载不了。这次 排查+手动清理已经解决了当次故障,但需要一道独立于 rclone 同步之外的兜底, 防止再次悄悄写满没人发现。 - oracle_db.py 新增 get_oldest_purgeable_material():只挑最旧的、已完成 分割阶段(status='done')的整段素材(非 motion_ 前缀),绝不碰运动片段 (事件时间轴/人物头像依赖它)和还在处理中的素材 - disk_guard.py 新增 DiskGuard 后台线程:5 分钟检查一次,剩余空间 <10GB 触发清理,删到 15GB 水位为止(留缓冲避免刚清完又立刻触发),复用已有的 delete_video() 完成实际删除 - app.py 启动这个后台服务;/api/oracle/activity 新增 disk 字段(实时剩余 空间 + 最近一次清理动作),供服务状态页展示 - 新增 9 个单元测试,全部通过(139/139) 已部署 Oracle 验证:DiskGuard 正常启动,配置生效。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
120
fam-edge/tests/test_disk_guard.py
Normal file
120
fam-edge/tests/test_disk_guard.py
Normal file
@@ -0,0 +1,120 @@
|
||||
import pytest
|
||||
|
||||
from fam_edge.disk_guard import DiskGuard
|
||||
|
||||
|
||||
def _cfg(**overrides):
|
||||
base = {
|
||||
"enabled": True,
|
||||
"min_free_gb": 10,
|
||||
"target_free_gb": 15,
|
||||
"watch_path": "/opt/fam-edge",
|
||||
"max_delete_per_round": 50,
|
||||
"check_interval_sec": 300,
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self, candidates=None):
|
||||
self._candidates = list(candidates or [])
|
||||
self.deleted_ids = []
|
||||
self.activities = []
|
||||
|
||||
def get_oldest_purgeable_material(self):
|
||||
if not self._candidates:
|
||||
return None
|
||||
return self._candidates.pop(0)
|
||||
|
||||
def delete_video(self, video_id):
|
||||
self.deleted_ids.append(video_id)
|
||||
return None
|
||||
|
||||
def record_activity(self, service, action, detail=''):
|
||||
self.activities.append((service, action, detail))
|
||||
|
||||
|
||||
def _guard(monkeypatch, db, free_gb_sequence, **cfg_overrides):
|
||||
"""free_gb_sequence: 每次调用 _free_gb() 依次返回的值列表(最后一个值
|
||||
耗尽后保持不变),用来模拟"清理一个文件后空间逐步恢复"的过程。"""
|
||||
monkeypatch.setattr(
|
||||
"fam_edge.disk_guard.load_config",
|
||||
lambda: {"disk_guard": _cfg(**cfg_overrides)})
|
||||
guard = DiskGuard(db)
|
||||
seq = list(free_gb_sequence)
|
||||
|
||||
def fake_free_gb():
|
||||
if len(seq) > 1:
|
||||
return seq.pop(0)
|
||||
return seq[0]
|
||||
|
||||
monkeypatch.setattr(guard, "_free_gb", fake_free_gb)
|
||||
return guard
|
||||
|
||||
|
||||
def test_check_once_does_nothing_when_space_sufficient(monkeypatch):
|
||||
db = _FakeDB()
|
||||
guard = _guard(monkeypatch, db, [20.0])
|
||||
|
||||
guard.check_once()
|
||||
|
||||
assert db.deleted_ids == []
|
||||
assert db.activities == []
|
||||
|
||||
|
||||
def test_check_once_cleans_until_target_reached(monkeypatch):
|
||||
"""核心诉求: 低于 min_free_gb 触发清理,一直清到 target_free_gb 为止,
|
||||
不是清一个就停(否则马上又会跌破阈值,频繁触发)。"""
|
||||
db = _FakeDB(candidates=[
|
||||
{"id": 1, "local_path": "/tmp/a.mp4"},
|
||||
{"id": 2, "local_path": "/tmp/b.mp4"},
|
||||
{"id": 3, "local_path": "/tmp/c.mp4"},
|
||||
])
|
||||
# 初始 8GB(< min_free_gb=10),每删一个恢复到 8/12/16GB(16 >= target=15 时停)
|
||||
guard = _guard(monkeypatch, db, [8.0, 8.0, 12.0, 16.0])
|
||||
|
||||
guard.check_once()
|
||||
|
||||
assert db.deleted_ids == [1, 2]
|
||||
actions = [a[1] for a in db.activities]
|
||||
assert "low_space" in actions
|
||||
assert "cleaned" in actions
|
||||
|
||||
|
||||
def test_check_once_stops_when_no_candidate_left(monkeypatch):
|
||||
"""核心诉求: 候选清空了但空间依然不足,不能死循环,要停下来并记录一条
|
||||
"没有可清理素材"的警告,让人能在服务状态页看到这个异常情况——即便如此,
|
||||
已经发生的清理动作本身也要记录(能看到确实清过、释放了多少),不因为
|
||||
没完全达标就把 cleaned 记录吞掉。"""
|
||||
db = _FakeDB(candidates=[{"id": 1, "local_path": "/tmp/a.mp4"}])
|
||||
guard = _guard(monkeypatch, db, [8.0, 8.0, 9.0]) # 删完仅 1 个后仍然 <15GB
|
||||
|
||||
guard.check_once()
|
||||
|
||||
assert db.deleted_ids == [1]
|
||||
actions = [a[1] for a in db.activities]
|
||||
assert "no_candidate" in actions
|
||||
assert "cleaned" in actions
|
||||
|
||||
|
||||
def test_check_once_respects_max_delete_per_round(monkeypatch):
|
||||
"""核心诉求: 单轮清理有上限,防止候选异常多时一次性删太多——下一轮检查
|
||||
很快就会再触发,没必要在一轮里清空所有候选。"""
|
||||
candidates = [{"id": i, "local_path": f"/tmp/{i}.mp4"} for i in range(1, 6)]
|
||||
db = _FakeDB(candidates=candidates)
|
||||
# 空间一直卡在 8GB 不涨(模拟每个文件都很小,删多少都到不了 target)
|
||||
guard = _guard(monkeypatch, db, [8.0], max_delete_per_round=3)
|
||||
|
||||
guard.check_once()
|
||||
|
||||
assert db.deleted_ids == [1, 2, 3]
|
||||
|
||||
|
||||
def test_disabled_guard_does_not_start(monkeypatch):
|
||||
db = _FakeDB()
|
||||
guard = _guard(monkeypatch, db, [1.0], enabled=False)
|
||||
|
||||
guard.start()
|
||||
|
||||
assert guard.is_alive() is False
|
||||
Reference in New Issue
Block a user