From 30ae431ed9325e7687dceb0cb7a06b124adc6d7d Mon Sep 17 00:00:00 2001 From: ericwyuan Date: Sat, 29 Aug 2026 09:45:49 +0800 Subject: [PATCH] =?UTF-8?q?fix(rate-limit):=20=E9=80=80=E9=81=BF=2024h=20?= =?UTF-8?q?=E4=B8=94=E5=AE=88=E5=8D=AB=E4=BB=A5=20DB=20=E4=B8=BA=E5=87=86?= =?UTF-8?q?=EF=BC=8C=E7=BB=88=E7=BB=93=E9=99=90=E6=B5=81=E6=AD=BB=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因：Garmin 限流窗口 >6h，旧 6h 退避每到期就再撞一次 429，永久循环无法退出；且 _note_rate_limit 的 DB 写入被 except:pass 静默吞掉，worker 内存揣着未来冷却时间、DB 停在旧值，守卫被陈旧内存卡死（用户冷却已过仍被拦）。\n\n- RATE_LIMIT_BACKOFF 6h->24h：一次 429 静默一天，真正超过 Garmin 窗口。\n- rate_limited_until() 改为 DB 为准（内存仅在无行时兜底）：杜绝陈旧内存永久拦截。\n- _note_rate_limit 失败改为 logging 输出，不再静默吞错。\n- 生产已把冷却设到 2026-08-30 09:45 北京时间。 --- backend/services/garmin.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/backend/services/garmin.py b/backend/services/garmin.py index f4dc325..74cf88b 100644 --- a/backend/services/garmin.py +++ b/backend/services/garmin.py @@ -21,6 +21,7 @@ so passing it a date silently asks for activity number "2026-08-23". """ import datetime import json +import logging import os import threading @@ -97,14 +98,11 @@ class RateLimited(RuntimeError): # Retrying while rate limited is what deepens the limit, so once Garmin says -# 429 the whole process stands down until this passes. Bumped from 30 to 60 -# minutes: the account was stuck for days because every hourly tick re-hit it, -# so a longer cooldown gives Garmin's window room to actually close. -# How long we stand down after a Garmin 429. Garmin's own throttle window runs -# well past an hour for a repeat offender, so a short backoff just expires, lets -# the scheduler re-hit, and keeps the limit alive forever. Six hours of quiet is -# what actually lets the window close. -RATE_LIMIT_BACKOFF = datetime.timedelta(hours=6) +# 429 the whole process stands down until this passes. The account was stuck for +# days because the backoff kept expiring before Garmin's own (multi-hour) window +# closed, so every tick re-hit it and the limit never lifted. A 24h stand-down +# is what actually outlasts the throttle and lets the window close for good. +RATE_LIMIT_BACKOFF = datetime.timedelta(hours=24) # In-process cache of the cooldown, kept in sync with the DB copy below and # still the lever the tests reach for via _rate_limited_until.clear(). _rate_limited_until = {} @@ -132,19 +130,23 @@ def _parse_dt(v): def rate_limited_until(user_id): """When the account is still cooling down, as a UTC datetime (or None). - The cooldown is persisted to the database so every gunicorn worker and a - process restart see the same deadline — a process-local dict alone let each - worker re-hit Garmin and keep the limit alive forever. + The cooldown lives in the database and is the single source of truth, so + every gunicorn worker and a restart agree on it. A process-local dict caused + a nasty "stuck forever" bug: a worker would remember a future deadline whose + DB write had silently failed to persist, and go on blocking even after the + real deadline had passed. We therefore trust the DB row, falling back to the + in-memory cache only when no row exists yet. """ - mem = _rate_limited_until.get(user_id) try: row = query_one( "SELECT rate_limited_until FROM sync_status WHERE user_id = ?", [user_id] ) except Exception: row = None - candidates = [c for c in (mem, _parse_dt(row["rate_limited_until"] if row else None)) if c] - return max(candidates) if candidates else None + db_val = _parse_dt(row["rate_limited_until"] if row else None) + if db_val is not None: + return db_val + return _rate_limited_until.get(user_id) def _note_rate_limit(user_id): @@ -167,8 +169,10 @@ def _note_rate_limit(user_id): user_id, cur_status, now.isoformat(timespec="seconds"), rate_limited_until=until.isoformat(timespec="seconds"), ) - except Exception: - pass + except Exception as e: # no cover - surfaced so a persist failure is visible + logging.getLogger(__name__).warning( + "rate-limit cooldown failed to persist for %s: %s", user_id, e + ) def _rate_limit_block(user_id):