diff --git a/backend/routes/garmin.py b/backend/routes/garmin.py index a146a47..fc5df60 100644 --- a/backend/routes/garmin.py +++ b/backend/routes/garmin.py @@ -52,9 +52,9 @@ def sync(): if days == -1: days = -1 # 自上次同步（增量） elif days == 0: - days = 730 # 全部历史 → 最大范围 + days = 0 # 全部历史 — the service walks back to the real end else: - days = max(1, min(days, 730)) + days = max(1, min(days, garmin_svc.MAX_HISTORY_DAYS)) except (TypeError, ValueError): days = None diff --git a/backend/services/garmin.py b/backend/services/garmin.py index ec72673..408ed8a 100644 --- a/backend/services/garmin.py +++ b/backend/services/garmin.py @@ -931,6 +931,25 @@ def start_backfill(user_id, limit=None): return backfill_status(user_id) +# What 全部历史 actually means. 730 was a made-up ceiling that silently +# truncated anyone with more than two years of Garmin history to two years. +# The real end of the data is found by walking back until the days stop +# containing anything (EMPTY_RUN_STOP below), so this is only a backstop. +MAX_HISTORY_DAYS = int(os.environ.get("GARMIN_MAX_HISTORY_DAYS") or 3650) + +# A long backfill stops once this many consecutive fetched days come back with +# nothing at all: that is what reaching the start of the account looks like, +# and without it 全部历史 would spend thousands of requests on years that +# predate the watch. Long enough to ride out a season of not wearing it. +EMPTY_RUN_STOP = 120 + +# The most recent days are still being written to, so a backfill refetches +# them even when they are already stored. Everything older is skipped if it is +# already in the database — that is what makes a multi-year sync resumable +# after a rate limit instead of restarting from today every time. +ALWAYS_REFETCH_DAYS = 3 + + # Up to this many days, a sync also pulls each day's within-day curves inline. # Beyond it the curves are left to the background backfill: five extra calls # per day would turn a year's sync into an hour. @@ -981,7 +1000,7 @@ def sync_data(user_id, creds, days=None, client=None): """ days = DEFAULT_SYNC_DAYS if days is None else days if days == 0: - days = 730 # 全部历史 → 最大范围 + days = MAX_HISTORY_DAYS # 全部历史 → 走到数据尽头为止 now = datetime.datetime.utcnow().isoformat(timespec="seconds") until, msg = _rate_limit_block(user_id) if until: @@ -1042,11 +1061,26 @@ def sync_data(user_id, creds, days=None, client=None): start_date = (today - datetime.timedelta(days=days - 1)).isoformat() + # Days already stored, so a resumed backfill does not spend its whole + # rate-limit budget re-fetching what it already has. + stored = { + r["date"] for r in query_all( + "SELECT date FROM health_data WHERE user_id = ? AND date >= ?", + (user_id, start_date), + ) + } + days_synced = 0 + days_attempted = 0 + empty_run = 0 + reached_the_start = False day_errors = [] rate_limited_mid_run = None for i in range(days): date_str = (today - datetime.timedelta(days=i)).isoformat() + if i >= ALWAYS_REFETCH_DAYS and date_str in stored: + continue + days_attempted += 1 try: record = _extract_daily(client, date_str) record.update(extras.daily_extras(client, date_str)) @@ -1066,6 +1100,7 @@ def sync_data(user_id, creds, days=None, client=None): if any(record[k] is not None for k in record if k != "date"): health.upsert_health_daily(user_id, record) days_synced += 1 + empty_run = 0 # Within-day curves for short syncs only. A year-long backfill # would add five calls per day on top of everything else; those # days are filled by start_backfill instead. @@ -1075,6 +1110,14 @@ def sync_data(user_id, creds, days=None, client=None): except Exception as e: # noqa: BLE001 day_errors.append(f"{date_str} series: {describe(e)}") + else: + # Walked back past the start of the account. Without this, 全部历史 + # would keep asking Garmin about years before the watch existed. + empty_run += 1 + if days > 90 and empty_run >= EMPTY_RUN_STOP: + reached_the_start = True + break + # A long backfill reports every fifth day — the write is cheap but not # free. A short one reports every day: at 7 days a "every 5th" cadence # meant the bar sat at 0 for most of the run and then vanished. @@ -1170,7 +1213,7 @@ def sync_data(user_id, creds, days=None, client=None): # Every single day failing means something systemic (expired session, # API change) — reporting that as a clean success would hide it. - if days_synced == 0 and len(day_errors) >= days: + if days_synced == 0 and days_attempted > 0 and len(day_errors) >= days_attempted: message = "; ".join(day_errors[:3]) _set_sync_status(user_id, "error", now, records_synced=0, last_error=message) return {"status": "error", "recordsSynced": 0, diff --git a/backend/services/settings.py b/backend/services/settings.py index a304baa..cf8e818 100644 --- a/backend/services/settings.py +++ b/backend/services/settings.py @@ -43,7 +43,8 @@ SEXES = ("male", "female", "other") UNITS = ("metric", "imperial") # Offered in the UI as a picker; anything else is snapped to the nearest. INTERVALS = (30, 60, 180, 360, 720, 1440) -HISTORY = (7, 30, 90, 180, 365, 730, 0, -1) +# 0 is 全部历史 (walk back to the start of the account) and -1 自上次同步. +HISTORY = (7, 30, 90, 180, 365, 730, 1095, 1825, 0, -1) CAMEL = { "height_cm": "heightCm", diff --git a/backend/tests/test_garmin_sync.py b/backend/tests/test_garmin_sync.py index 36ac3d6..a5964bc 100644 --- a/backend/tests/test_garmin_sync.py +++ b/backend/tests/test_garmin_sync.py @@ -646,7 +646,7 @@ class TestBackgroundSync: lambda uid, creds, days=None: seen.setdefault("days", days) or {"status": "syncing"}, ) client.post("/api/garmin/sync", headers=auth, json={"days": 99999}) - assert seen["days"] == 730 + assert seen["days"] == garmin_svc.MAX_HISTORY_DAYS class TestRateLimiting: @@ -766,11 +766,49 @@ class TestSyncWindow: pulled = {c[1] for c in client.calls if c[0] == "summary"} assert len(pulled) == garmin_svc.DEFAULT_SYNC_DAYS - def test_full_history_asks_for_the_maximum_window(self, db, user): - """`days=0` is 全部历史, not "unset".""" + def test_full_history_asks_for_the_maximum_window(self, db, user, monkeypatch): + """`days=0` is 全部历史, not "unset" — and not a fixed two years.""" + monkeypatch.setattr(garmin_svc, "MAX_HISTORY_DAYS", 50) client = StubClient() garmin_svc.sync_data(user["id"], CREDS, days=0, client=client) - assert len([c for c in client.calls if c[0] == "summary"]) == 730 + assert len([c for c in client.calls if c[0] == "summary"]) == 50 + + def test_days_already_stored_are_not_refetched(self, db, user, seed_health): + """What makes a multi-year backfill resumable: a run that stopped on a + 429 must continue where it left off, not start over from today.""" + seed_health([{"date": day(i), "steps": 4000} for i in range(5, 20)]) + client = StubClient() + garmin_svc.sync_data(user["id"], CREDS, days=20, client=client) + + pulled = sorted(c[1] for c in client.calls if c[0] == "summary") + assert pulled == sorted(day(i) for i in range(5)), "only the gap" + + def test_the_most_recent_days_are_always_refetched(self, db, user, seed_health): + """Today is still being written to; a stored row for it is not final.""" + seed_health([{"date": day(i), "steps": 4000} for i in range(0, 10)]) + client = StubClient() + garmin_svc.sync_data(user["id"], CREDS, days=10, client=client) + + pulled = {c[1] for c in client.calls if c[0] == "summary"} + assert pulled == {day(i) for i in range(garmin_svc.ALWAYS_REFETCH_DAYS)} + + def test_a_long_backfill_stops_at_the_start_of_the_account(self, db, user, + monkeypatch): + """全部历史 must not spend thousands of requests on years that predate + the watch.""" + monkeypatch.setattr(garmin_svc, "EMPTY_RUN_STOP", 10) + empty = {"totalSteps": None, "restingHeartRate": None, + "averageStressLevel": None, "totalKilocalories": None} + client = StubClient( + summaries={day(i): empty for i in range(3, 400)}, + sleeps={day(i): {} for i in range(3, 400)}, + hrvs={day(i): {} for i in range(3, 400)}, + ) + out = garmin_svc.sync_data(user["id"], CREDS, days=365, client=client) + + assert out["recordsSynced"] == 3, "the three days that had data" + pulled = len([c for c in client.calls if c[0] == "summary"]) + assert pulled == 13, f"3 real days + 10 empties, not 365 ({pulled})" def test_the_sync_endpoint_passes_a_zero_through( self, client, auth, user, db, monkeypatch @@ -785,7 +823,9 @@ class TestSyncWindow: monkeypatch.setattr(garmin_svc, "start_sync", record) r = client.post("/api/garmin/sync", headers=auth, json={"days": 0}) assert r.status_code == 202 - assert seen["days"] == 730, "全部历史 must not fall back to the default" + # 0 travels all the way to the service, which walks back to the real + # end of the account rather than to a fixed ceiling. + assert seen["days"] == 0, "全部历史 must not fall back to the default" def test_a_429_mid_run_stops_the_sync(self, db, user): """Filing a 429 as one more skipped day meant a 730-day backfill kept diff --git a/client/src/pages/SyncPage.tsx b/client/src/pages/SyncPage.tsx index 100d4ee..28ec0d9 100644 --- a/client/src/pages/SyncPage.tsx +++ b/client/src/pages/SyncPage.tsx @@ -28,7 +28,7 @@ const historyLabel = (days: number) => /** What each range actually costs, so the choice is made with eyes open. */ const rangeHint = (days: number) => days === -1 ? '只补上次同步之后缺的那几天，最快' - : days === 0 ? '约 730 天，20–40 分钟' + : days === 0 ? '一直回到账号最早的数据，可能要分几次才拉得完' : days >= 365 ? `${days} 天，大约 ${Math.ceil((days * 3) / 60)} 分钟` : `${days} 天，几分钟`; @@ -420,7 +420,8 @@ function SyncPage() {

在后台运行，可以离开本页 - {total > 60 ? `，预计还需 ${Math.ceil(((total - current) * 3) / 60)} 分钟` : ''}。 + {total > 60 ? `，最多还需 ${Math.ceil(((total - current) * 3) / 60)} 分钟` : ''}。 + {total > 365 && '已经存过的日期会跳过，所以中途被打断也不用从头再来。'}

) : (