ef56ae5662
背景：Oracle 磁盘曾经被写满（gdrive_videos 持续下载新素材、旧文件迟迟没 清理），触发 rclone 的一个安全机制——同步遇到 IO 错误就整体拒绝执行删除， 形成"越满越删不掉，越删不掉越满"的死循环，最终连新视频都下载不了。这次 排查+手动清理已经解决了当次故障，但需要一道独立于 rclone 同步之外的兜底， 防止再次悄悄写满没人发现。 - oracle_db.py 新增 get_oldest_purgeable_material()：只挑最旧的、已完成 分割阶段（status='done'）的整段素材（非 motion_ 前缀），绝不碰运动片段 （事件时间轴/人物头像依赖它）和还在处理中的素材 - disk_guard.py 新增 DiskGuard 后台线程：5 分钟检查一次，剩余空间 <10GB 触发清理，删到 15GB 水位为止（留缓冲避免刚清完又立刻触发），复用已有的 delete_video() 完成实际删除 - app.py 启动这个后台服务；/api/oracle/activity 新增 disk 字段（实时剩余 空间 + 最近一次清理动作），供服务状态页展示 - 新增 9 个单元测试，全部通过（139/139） 已部署 Oracle 验证：DiskGuard 正常启动，配置生效。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
121 lines
3.9 KiB
Python
121 lines
3.9 KiB
Python
import pytest
|
||
|
||
from fam_edge.disk_guard import DiskGuard
|
||
|
||
|
||
def _cfg(**overrides):
|
||
base = {
|
||
"enabled": True,
|
||
"min_free_gb": 10,
|
||
"target_free_gb": 15,
|
||
"watch_path": "/opt/fam-edge",
|
||
"max_delete_per_round": 50,
|
||
"check_interval_sec": 300,
|
||
}
|
||
base.update(overrides)
|
||
return base
|
||
|
||
|
||
class _FakeDB:
|
||
def __init__(self, candidates=None):
|
||
self._candidates = list(candidates or [])
|
||
self.deleted_ids = []
|
||
self.activities = []
|
||
|
||
def get_oldest_purgeable_material(self):
|
||
if not self._candidates:
|
||
return None
|
||
return self._candidates.pop(0)
|
||
|
||
def delete_video(self, video_id):
|
||
self.deleted_ids.append(video_id)
|
||
return None
|
||
|
||
def record_activity(self, service, action, detail=''):
|
||
self.activities.append((service, action, detail))
|
||
|
||
|
||
def _guard(monkeypatch, db, free_gb_sequence, **cfg_overrides):
|
||
"""free_gb_sequence: 每次调用 _free_gb() 依次返回的值列表（最后一个值
|
||
耗尽后保持不变），用来模拟"清理一个文件后空间逐步恢复"的过程。"""
|
||
monkeypatch.setattr(
|
||
"fam_edge.disk_guard.load_config",
|
||
lambda: {"disk_guard": _cfg(**cfg_overrides)})
|
||
guard = DiskGuard(db)
|
||
seq = list(free_gb_sequence)
|
||
|
||
def fake_free_gb():
|
||
if len(seq) > 1:
|
||
return seq.pop(0)
|
||
return seq[0]
|
||
|
||
monkeypatch.setattr(guard, "_free_gb", fake_free_gb)
|
||
return guard
|
||
|
||
|
||
def test_check_once_does_nothing_when_space_sufficient(monkeypatch):
|
||
db = _FakeDB()
|
||
guard = _guard(monkeypatch, db, [20.0])
|
||
|
||
guard.check_once()
|
||
|
||
assert db.deleted_ids == []
|
||
assert db.activities == []
|
||
|
||
|
||
def test_check_once_cleans_until_target_reached(monkeypatch):
|
||
"""核心诉求: 低于 min_free_gb 触发清理，一直清到 target_free_gb 为止，
|
||
不是清一个就停（否则马上又会跌破阈值，频繁触发）。"""
|
||
db = _FakeDB(candidates=[
|
||
{"id": 1, "local_path": "/tmp/a.mp4"},
|
||
{"id": 2, "local_path": "/tmp/b.mp4"},
|
||
{"id": 3, "local_path": "/tmp/c.mp4"},
|
||
])
|
||
# 初始 8GB（< min_free_gb=10），每删一个恢复到 8/12/16GB（16 >= target=15 时停）
|
||
guard = _guard(monkeypatch, db, [8.0, 8.0, 12.0, 16.0])
|
||
|
||
guard.check_once()
|
||
|
||
assert db.deleted_ids == [1, 2]
|
||
actions = [a[1] for a in db.activities]
|
||
assert "low_space" in actions
|
||
assert "cleaned" in actions
|
||
|
||
|
||
def test_check_once_stops_when_no_candidate_left(monkeypatch):
|
||
"""核心诉求: 候选清空了但空间依然不足，不能死循环，要停下来并记录一条
|
||
"没有可清理素材"的警告，让人能在服务状态页看到这个异常情况——即便如此，
|
||
已经发生的清理动作本身也要记录（能看到确实清过、释放了多少），不因为
|
||
没完全达标就把 cleaned 记录吞掉。"""
|
||
db = _FakeDB(candidates=[{"id": 1, "local_path": "/tmp/a.mp4"}])
|
||
guard = _guard(monkeypatch, db, [8.0, 8.0, 9.0]) # 删完仅 1 个后仍然 <15GB
|
||
|
||
guard.check_once()
|
||
|
||
assert db.deleted_ids == [1]
|
||
actions = [a[1] for a in db.activities]
|
||
assert "no_candidate" in actions
|
||
assert "cleaned" in actions
|
||
|
||
|
||
def test_check_once_respects_max_delete_per_round(monkeypatch):
|
||
"""核心诉求: 单轮清理有上限，防止候选异常多时一次性删太多——下一轮检查
|
||
很快就会再触发，没必要在一轮里清空所有候选。"""
|
||
candidates = [{"id": i, "local_path": f"/tmp/{i}.mp4"} for i in range(1, 6)]
|
||
db = _FakeDB(candidates=candidates)
|
||
# 空间一直卡在 8GB 不涨（模拟每个文件都很小，删多少都到不了 target）
|
||
guard = _guard(monkeypatch, db, [8.0], max_delete_per_round=3)
|
||
|
||
guard.check_once()
|
||
|
||
assert db.deleted_ids == [1, 2, 3]
|
||
|
||
|
||
def test_disabled_guard_does_not_start(monkeypatch):
|
||
db = _FakeDB()
|
||
guard = _guard(monkeypatch, db, [1.0], enabled=False)
|
||
|
||
guard.start()
|
||
|
||
assert guard.is_alive() is False
|