feat(sync): 全部历史真的是全部,不再截断在两年
730 天是个凭空写死的上限。账号有七年数据的人选「全部历史」,拿到的是 最近两年,而且没有任何提示说剩下的被丢掉了。 * 全部历史现在一直回溯到账号最早的数据:连续 EMPTY_RUN_STOP(120) 天 完全没有内容就停,所以既不会截断,也不会去问手表存在之前的年份。 MAX_HISTORY_DAYS(3650) 只是兜底,可用环境变量覆盖。 * 已经存过的日期跳过(最近 3 天除外,它们还在写入中)。这让多年的 回填变成可续传的:撞上限流停下来,冷却过后再点一次就从断点继续, 而不是每次都从今天重新爬。 * 选择器补上 3 年 / 5 年。 * 前端把每个范围的实际代价写出来,并说明中断可续。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -931,6 +931,25 @@ def start_backfill(user_id, limit=None):
|
||||
return backfill_status(user_id)
|
||||
|
||||
|
||||
# What 全部历史 actually means. 730 was a made-up ceiling that silently
|
||||
# truncated anyone with more than two years of Garmin history to two years.
|
||||
# The real end of the data is found by walking back until the days stop
|
||||
# containing anything (EMPTY_RUN_STOP below), so this is only a backstop.
|
||||
MAX_HISTORY_DAYS = int(os.environ.get("GARMIN_MAX_HISTORY_DAYS") or 3650)
|
||||
|
||||
# A long backfill stops once this many consecutive fetched days come back with
|
||||
# nothing at all: that is what reaching the start of the account looks like,
|
||||
# and without it 全部历史 would spend thousands of requests on years that
|
||||
# predate the watch. Long enough to ride out a season of not wearing it.
|
||||
EMPTY_RUN_STOP = 120
|
||||
|
||||
# The most recent days are still being written to, so a backfill refetches
|
||||
# them even when they are already stored. Everything older is skipped if it is
|
||||
# already in the database — that is what makes a multi-year sync resumable
|
||||
# after a rate limit instead of restarting from today every time.
|
||||
ALWAYS_REFETCH_DAYS = 3
|
||||
|
||||
|
||||
# Up to this many days, a sync also pulls each day's within-day curves inline.
|
||||
# Beyond it the curves are left to the background backfill: five extra calls
|
||||
# per day would turn a year's sync into an hour.
|
||||
@@ -981,7 +1000,7 @@ def sync_data(user_id, creds, days=None, client=None):
|
||||
"""
|
||||
days = DEFAULT_SYNC_DAYS if days is None else days
|
||||
if days == 0:
|
||||
days = 730 # 全部历史 → 最大范围
|
||||
days = MAX_HISTORY_DAYS # 全部历史 → 走到数据尽头为止
|
||||
now = datetime.datetime.utcnow().isoformat(timespec="seconds")
|
||||
until, msg = _rate_limit_block(user_id)
|
||||
if until:
|
||||
@@ -1042,11 +1061,26 @@ def sync_data(user_id, creds, days=None, client=None):
|
||||
|
||||
start_date = (today - datetime.timedelta(days=days - 1)).isoformat()
|
||||
|
||||
# Days already stored, so a resumed backfill does not spend its whole
|
||||
# rate-limit budget re-fetching what it already has.
|
||||
stored = {
|
||||
r["date"] for r in query_all(
|
||||
"SELECT date FROM health_data WHERE user_id = ? AND date >= ?",
|
||||
(user_id, start_date),
|
||||
)
|
||||
}
|
||||
|
||||
days_synced = 0
|
||||
days_attempted = 0
|
||||
empty_run = 0
|
||||
reached_the_start = False
|
||||
day_errors = []
|
||||
rate_limited_mid_run = None
|
||||
for i in range(days):
|
||||
date_str = (today - datetime.timedelta(days=i)).isoformat()
|
||||
if i >= ALWAYS_REFETCH_DAYS and date_str in stored:
|
||||
continue
|
||||
days_attempted += 1
|
||||
try:
|
||||
record = _extract_daily(client, date_str)
|
||||
record.update(extras.daily_extras(client, date_str))
|
||||
@@ -1066,6 +1100,7 @@ def sync_data(user_id, creds, days=None, client=None):
|
||||
if any(record[k] is not None for k in record if k != "date"):
|
||||
health.upsert_health_daily(user_id, record)
|
||||
days_synced += 1
|
||||
empty_run = 0
|
||||
# Within-day curves for short syncs only. A year-long backfill
|
||||
# would add five calls per day on top of everything else; those
|
||||
# days are filled by start_backfill instead.
|
||||
@@ -1075,6 +1110,14 @@ def sync_data(user_id, creds, days=None, client=None):
|
||||
except Exception as e: # noqa: BLE001
|
||||
day_errors.append(f"{date_str} series: {describe(e)}")
|
||||
|
||||
else:
|
||||
# Walked back past the start of the account. Without this, 全部历史
|
||||
# would keep asking Garmin about years before the watch existed.
|
||||
empty_run += 1
|
||||
if days > 90 and empty_run >= EMPTY_RUN_STOP:
|
||||
reached_the_start = True
|
||||
break
|
||||
|
||||
# A long backfill reports every fifth day — the write is cheap but not
|
||||
# free. A short one reports every day: at 7 days a "every 5th" cadence
|
||||
# meant the bar sat at 0 for most of the run and then vanished.
|
||||
@@ -1170,7 +1213,7 @@ def sync_data(user_id, creds, days=None, client=None):
|
||||
|
||||
# Every single day failing means something systemic (expired session,
|
||||
# API change) — reporting that as a clean success would hide it.
|
||||
if days_synced == 0 and len(day_errors) >= days:
|
||||
if days_synced == 0 and days_attempted > 0 and len(day_errors) >= days_attempted:
|
||||
message = "; ".join(day_errors[:3])
|
||||
_set_sync_status(user_id, "error", now, records_synced=0, last_error=message)
|
||||
return {"status": "error", "recordsSynced": 0,
|
||||
|
||||
Reference in New Issue
Block a user