feat(sync): 全部历史真的是全部,不再截断在两年
730 天是个凭空写死的上限。账号有七年数据的人选「全部历史」,拿到的是 最近两年,而且没有任何提示说剩下的被丢掉了。 * 全部历史现在一直回溯到账号最早的数据:连续 EMPTY_RUN_STOP(120) 天 完全没有内容就停,所以既不会截断,也不会去问手表存在之前的年份。 MAX_HISTORY_DAYS(3650) 只是兜底,可用环境变量覆盖。 * 已经存过的日期跳过(最近 3 天除外,它们还在写入中)。这让多年的 回填变成可续传的:撞上限流停下来,冷却过后再点一次就从断点继续, 而不是每次都从今天重新爬。 * 选择器补上 3 年 / 5 年。 * 前端把每个范围的实际代价写出来,并说明中断可续。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -52,9 +52,9 @@ def sync():
|
|||||||
if days == -1:
|
if days == -1:
|
||||||
days = -1 # 自上次同步(增量)
|
days = -1 # 自上次同步(增量)
|
||||||
elif days == 0:
|
elif days == 0:
|
||||||
days = 730 # 全部历史 → 最大范围
|
days = 0 # 全部历史 — the service walks back to the real end
|
||||||
else:
|
else:
|
||||||
days = max(1, min(days, 730))
|
days = max(1, min(days, garmin_svc.MAX_HISTORY_DAYS))
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
days = None
|
days = None
|
||||||
|
|
||||||
|
|||||||
@@ -931,6 +931,25 @@ def start_backfill(user_id, limit=None):
|
|||||||
return backfill_status(user_id)
|
return backfill_status(user_id)
|
||||||
|
|
||||||
|
|
||||||
|
# What 全部历史 actually means. 730 was a made-up ceiling that silently
|
||||||
|
# truncated anyone with more than two years of Garmin history to two years.
|
||||||
|
# The real end of the data is found by walking back until the days stop
|
||||||
|
# containing anything (EMPTY_RUN_STOP below), so this is only a backstop.
|
||||||
|
MAX_HISTORY_DAYS = int(os.environ.get("GARMIN_MAX_HISTORY_DAYS") or 3650)
|
||||||
|
|
||||||
|
# A long backfill stops once this many consecutive fetched days come back with
|
||||||
|
# nothing at all: that is what reaching the start of the account looks like,
|
||||||
|
# and without it 全部历史 would spend thousands of requests on years that
|
||||||
|
# predate the watch. Long enough to ride out a season of not wearing it.
|
||||||
|
EMPTY_RUN_STOP = 120
|
||||||
|
|
||||||
|
# The most recent days are still being written to, so a backfill refetches
|
||||||
|
# them even when they are already stored. Everything older is skipped if it is
|
||||||
|
# already in the database — that is what makes a multi-year sync resumable
|
||||||
|
# after a rate limit instead of restarting from today every time.
|
||||||
|
ALWAYS_REFETCH_DAYS = 3
|
||||||
|
|
||||||
|
|
||||||
# Up to this many days, a sync also pulls each day's within-day curves inline.
|
# Up to this many days, a sync also pulls each day's within-day curves inline.
|
||||||
# Beyond it the curves are left to the background backfill: five extra calls
|
# Beyond it the curves are left to the background backfill: five extra calls
|
||||||
# per day would turn a year's sync into an hour.
|
# per day would turn a year's sync into an hour.
|
||||||
@@ -981,7 +1000,7 @@ def sync_data(user_id, creds, days=None, client=None):
|
|||||||
"""
|
"""
|
||||||
days = DEFAULT_SYNC_DAYS if days is None else days
|
days = DEFAULT_SYNC_DAYS if days is None else days
|
||||||
if days == 0:
|
if days == 0:
|
||||||
days = 730 # 全部历史 → 最大范围
|
days = MAX_HISTORY_DAYS # 全部历史 → 走到数据尽头为止
|
||||||
now = datetime.datetime.utcnow().isoformat(timespec="seconds")
|
now = datetime.datetime.utcnow().isoformat(timespec="seconds")
|
||||||
until, msg = _rate_limit_block(user_id)
|
until, msg = _rate_limit_block(user_id)
|
||||||
if until:
|
if until:
|
||||||
@@ -1042,11 +1061,26 @@ def sync_data(user_id, creds, days=None, client=None):
|
|||||||
|
|
||||||
start_date = (today - datetime.timedelta(days=days - 1)).isoformat()
|
start_date = (today - datetime.timedelta(days=days - 1)).isoformat()
|
||||||
|
|
||||||
|
# Days already stored, so a resumed backfill does not spend its whole
|
||||||
|
# rate-limit budget re-fetching what it already has.
|
||||||
|
stored = {
|
||||||
|
r["date"] for r in query_all(
|
||||||
|
"SELECT date FROM health_data WHERE user_id = ? AND date >= ?",
|
||||||
|
(user_id, start_date),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
days_synced = 0
|
days_synced = 0
|
||||||
|
days_attempted = 0
|
||||||
|
empty_run = 0
|
||||||
|
reached_the_start = False
|
||||||
day_errors = []
|
day_errors = []
|
||||||
rate_limited_mid_run = None
|
rate_limited_mid_run = None
|
||||||
for i in range(days):
|
for i in range(days):
|
||||||
date_str = (today - datetime.timedelta(days=i)).isoformat()
|
date_str = (today - datetime.timedelta(days=i)).isoformat()
|
||||||
|
if i >= ALWAYS_REFETCH_DAYS and date_str in stored:
|
||||||
|
continue
|
||||||
|
days_attempted += 1
|
||||||
try:
|
try:
|
||||||
record = _extract_daily(client, date_str)
|
record = _extract_daily(client, date_str)
|
||||||
record.update(extras.daily_extras(client, date_str))
|
record.update(extras.daily_extras(client, date_str))
|
||||||
@@ -1066,6 +1100,7 @@ def sync_data(user_id, creds, days=None, client=None):
|
|||||||
if any(record[k] is not None for k in record if k != "date"):
|
if any(record[k] is not None for k in record if k != "date"):
|
||||||
health.upsert_health_daily(user_id, record)
|
health.upsert_health_daily(user_id, record)
|
||||||
days_synced += 1
|
days_synced += 1
|
||||||
|
empty_run = 0
|
||||||
# Within-day curves for short syncs only. A year-long backfill
|
# Within-day curves for short syncs only. A year-long backfill
|
||||||
# would add five calls per day on top of everything else; those
|
# would add five calls per day on top of everything else; those
|
||||||
# days are filled by start_backfill instead.
|
# days are filled by start_backfill instead.
|
||||||
@@ -1075,6 +1110,14 @@ def sync_data(user_id, creds, days=None, client=None):
|
|||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
day_errors.append(f"{date_str} series: {describe(e)}")
|
day_errors.append(f"{date_str} series: {describe(e)}")
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Walked back past the start of the account. Without this, 全部历史
|
||||||
|
# would keep asking Garmin about years before the watch existed.
|
||||||
|
empty_run += 1
|
||||||
|
if days > 90 and empty_run >= EMPTY_RUN_STOP:
|
||||||
|
reached_the_start = True
|
||||||
|
break
|
||||||
|
|
||||||
# A long backfill reports every fifth day — the write is cheap but not
|
# A long backfill reports every fifth day — the write is cheap but not
|
||||||
# free. A short one reports every day: at 7 days a "every 5th" cadence
|
# free. A short one reports every day: at 7 days a "every 5th" cadence
|
||||||
# meant the bar sat at 0 for most of the run and then vanished.
|
# meant the bar sat at 0 for most of the run and then vanished.
|
||||||
@@ -1170,7 +1213,7 @@ def sync_data(user_id, creds, days=None, client=None):
|
|||||||
|
|
||||||
# Every single day failing means something systemic (expired session,
|
# Every single day failing means something systemic (expired session,
|
||||||
# API change) — reporting that as a clean success would hide it.
|
# API change) — reporting that as a clean success would hide it.
|
||||||
if days_synced == 0 and len(day_errors) >= days:
|
if days_synced == 0 and days_attempted > 0 and len(day_errors) >= days_attempted:
|
||||||
message = "; ".join(day_errors[:3])
|
message = "; ".join(day_errors[:3])
|
||||||
_set_sync_status(user_id, "error", now, records_synced=0, last_error=message)
|
_set_sync_status(user_id, "error", now, records_synced=0, last_error=message)
|
||||||
return {"status": "error", "recordsSynced": 0,
|
return {"status": "error", "recordsSynced": 0,
|
||||||
|
|||||||
@@ -43,7 +43,8 @@ SEXES = ("male", "female", "other")
|
|||||||
UNITS = ("metric", "imperial")
|
UNITS = ("metric", "imperial")
|
||||||
# Offered in the UI as a picker; anything else is snapped to the nearest.
|
# Offered in the UI as a picker; anything else is snapped to the nearest.
|
||||||
INTERVALS = (30, 60, 180, 360, 720, 1440)
|
INTERVALS = (30, 60, 180, 360, 720, 1440)
|
||||||
HISTORY = (7, 30, 90, 180, 365, 730, 0, -1)
|
# 0 is 全部历史 (walk back to the start of the account) and -1 自上次同步.
|
||||||
|
HISTORY = (7, 30, 90, 180, 365, 730, 1095, 1825, 0, -1)
|
||||||
|
|
||||||
CAMEL = {
|
CAMEL = {
|
||||||
"height_cm": "heightCm",
|
"height_cm": "heightCm",
|
||||||
|
|||||||
@@ -646,7 +646,7 @@ class TestBackgroundSync:
|
|||||||
lambda uid, creds, days=None: seen.setdefault("days", days) or {"status": "syncing"},
|
lambda uid, creds, days=None: seen.setdefault("days", days) or {"status": "syncing"},
|
||||||
)
|
)
|
||||||
client.post("/api/garmin/sync", headers=auth, json={"days": 99999})
|
client.post("/api/garmin/sync", headers=auth, json={"days": 99999})
|
||||||
assert seen["days"] == 730
|
assert seen["days"] == garmin_svc.MAX_HISTORY_DAYS
|
||||||
|
|
||||||
|
|
||||||
class TestRateLimiting:
|
class TestRateLimiting:
|
||||||
@@ -766,11 +766,49 @@ class TestSyncWindow:
|
|||||||
pulled = {c[1] for c in client.calls if c[0] == "summary"}
|
pulled = {c[1] for c in client.calls if c[0] == "summary"}
|
||||||
assert len(pulled) == garmin_svc.DEFAULT_SYNC_DAYS
|
assert len(pulled) == garmin_svc.DEFAULT_SYNC_DAYS
|
||||||
|
|
||||||
def test_full_history_asks_for_the_maximum_window(self, db, user):
|
def test_full_history_asks_for_the_maximum_window(self, db, user, monkeypatch):
|
||||||
"""`days=0` is 全部历史, not "unset"."""
|
"""`days=0` is 全部历史, not "unset" — and not a fixed two years."""
|
||||||
|
monkeypatch.setattr(garmin_svc, "MAX_HISTORY_DAYS", 50)
|
||||||
client = StubClient()
|
client = StubClient()
|
||||||
garmin_svc.sync_data(user["id"], CREDS, days=0, client=client)
|
garmin_svc.sync_data(user["id"], CREDS, days=0, client=client)
|
||||||
assert len([c for c in client.calls if c[0] == "summary"]) == 730
|
assert len([c for c in client.calls if c[0] == "summary"]) == 50
|
||||||
|
|
||||||
|
def test_days_already_stored_are_not_refetched(self, db, user, seed_health):
|
||||||
|
"""What makes a multi-year backfill resumable: a run that stopped on a
|
||||||
|
429 must continue where it left off, not start over from today."""
|
||||||
|
seed_health([{"date": day(i), "steps": 4000} for i in range(5, 20)])
|
||||||
|
client = StubClient()
|
||||||
|
garmin_svc.sync_data(user["id"], CREDS, days=20, client=client)
|
||||||
|
|
||||||
|
pulled = sorted(c[1] for c in client.calls if c[0] == "summary")
|
||||||
|
assert pulled == sorted(day(i) for i in range(5)), "only the gap"
|
||||||
|
|
||||||
|
def test_the_most_recent_days_are_always_refetched(self, db, user, seed_health):
|
||||||
|
"""Today is still being written to; a stored row for it is not final."""
|
||||||
|
seed_health([{"date": day(i), "steps": 4000} for i in range(0, 10)])
|
||||||
|
client = StubClient()
|
||||||
|
garmin_svc.sync_data(user["id"], CREDS, days=10, client=client)
|
||||||
|
|
||||||
|
pulled = {c[1] for c in client.calls if c[0] == "summary"}
|
||||||
|
assert pulled == {day(i) for i in range(garmin_svc.ALWAYS_REFETCH_DAYS)}
|
||||||
|
|
||||||
|
def test_a_long_backfill_stops_at_the_start_of_the_account(self, db, user,
|
||||||
|
monkeypatch):
|
||||||
|
"""全部历史 must not spend thousands of requests on years that predate
|
||||||
|
the watch."""
|
||||||
|
monkeypatch.setattr(garmin_svc, "EMPTY_RUN_STOP", 10)
|
||||||
|
empty = {"totalSteps": None, "restingHeartRate": None,
|
||||||
|
"averageStressLevel": None, "totalKilocalories": None}
|
||||||
|
client = StubClient(
|
||||||
|
summaries={day(i): empty for i in range(3, 400)},
|
||||||
|
sleeps={day(i): {} for i in range(3, 400)},
|
||||||
|
hrvs={day(i): {} for i in range(3, 400)},
|
||||||
|
)
|
||||||
|
out = garmin_svc.sync_data(user["id"], CREDS, days=365, client=client)
|
||||||
|
|
||||||
|
assert out["recordsSynced"] == 3, "the three days that had data"
|
||||||
|
pulled = len([c for c in client.calls if c[0] == "summary"])
|
||||||
|
assert pulled == 13, f"3 real days + 10 empties, not 365 ({pulled})"
|
||||||
|
|
||||||
def test_the_sync_endpoint_passes_a_zero_through(
|
def test_the_sync_endpoint_passes_a_zero_through(
|
||||||
self, client, auth, user, db, monkeypatch
|
self, client, auth, user, db, monkeypatch
|
||||||
@@ -785,7 +823,9 @@ class TestSyncWindow:
|
|||||||
monkeypatch.setattr(garmin_svc, "start_sync", record)
|
monkeypatch.setattr(garmin_svc, "start_sync", record)
|
||||||
r = client.post("/api/garmin/sync", headers=auth, json={"days": 0})
|
r = client.post("/api/garmin/sync", headers=auth, json={"days": 0})
|
||||||
assert r.status_code == 202
|
assert r.status_code == 202
|
||||||
assert seen["days"] == 730, "全部历史 must not fall back to the default"
|
# 0 travels all the way to the service, which walks back to the real
|
||||||
|
# end of the account rather than to a fixed ceiling.
|
||||||
|
assert seen["days"] == 0, "全部历史 must not fall back to the default"
|
||||||
|
|
||||||
def test_a_429_mid_run_stops_the_sync(self, db, user):
|
def test_a_429_mid_run_stops_the_sync(self, db, user):
|
||||||
"""Filing a 429 as one more skipped day meant a 730-day backfill kept
|
"""Filing a 429 as one more skipped day meant a 730-day backfill kept
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ const historyLabel = (days: number) =>
|
|||||||
/** What each range actually costs, so the choice is made with eyes open. */
|
/** What each range actually costs, so the choice is made with eyes open. */
|
||||||
const rangeHint = (days: number) =>
|
const rangeHint = (days: number) =>
|
||||||
days === -1 ? '只补上次同步之后缺的那几天,最快'
|
days === -1 ? '只补上次同步之后缺的那几天,最快'
|
||||||
: days === 0 ? '约 730 天,20–40 分钟'
|
: days === 0 ? '一直回到账号最早的数据,可能要分几次才拉得完'
|
||||||
: days >= 365 ? `${days} 天,大约 ${Math.ceil((days * 3) / 60)} 分钟`
|
: days >= 365 ? `${days} 天,大约 ${Math.ceil((days * 3) / 60)} 分钟`
|
||||||
: `${days} 天,几分钟`;
|
: `${days} 天,几分钟`;
|
||||||
|
|
||||||
@@ -420,7 +420,8 @@ function SyncPage() {
|
|||||||
</div>
|
</div>
|
||||||
<p className="field-hint">
|
<p className="field-hint">
|
||||||
在后台运行,可以离开本页
|
在后台运行,可以离开本页
|
||||||
{total > 60 ? `,预计还需 ${Math.ceil(((total - current) * 3) / 60)} 分钟` : ''}。
|
{total > 60 ? `,最多还需 ${Math.ceil(((total - current) * 3) / 60)} 分钟` : ''}。
|
||||||
|
{total > 365 && '已经存过的日期会跳过,所以中途被打断也不用从头再来。'}
|
||||||
</p>
|
</p>
|
||||||
</section>
|
</section>
|
||||||
) : (
|
) : (
|
||||||
|
|||||||
Reference in New Issue
Block a user