fix(rate-limit): 退避 24h 且守卫以 DB 为准,终结限流死循环
根因:Garmin 限流窗口 >6h,旧 6h 退避每到期就再撞一次 429,永久循环无法退出;且 _note_rate_limit 的 DB 写入被 except:pass 静默吞掉,worker 内存揣着未来冷却时间、DB 停在旧值,守卫被陈旧内存卡死(用户冷却已过仍被拦)。\n\n- RATE_LIMIT_BACKOFF 6h->24h:一次 429 静默一天,真正超过 Garmin 窗口。\n- rate_limited_until() 改为 DB 为准(内存仅在无行时兜底):杜绝陈旧内存永久拦截。\n- _note_rate_limit 失败改为 logging 输出,不再静默吞错。\n- 生产已把冷却设到 2026-08-30 09:45 北京时间。
This commit is contained in:
@@ -21,6 +21,7 @@ so passing it a date silently asks for activity number "2026-08-23".
|
|||||||
"""
|
"""
|
||||||
import datetime
|
import datetime
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
@@ -97,14 +98,11 @@ class RateLimited(RuntimeError):
|
|||||||
|
|
||||||
|
|
||||||
# Retrying while rate limited is what deepens the limit, so once Garmin says
|
# Retrying while rate limited is what deepens the limit, so once Garmin says
|
||||||
# 429 the whole process stands down until this passes. Bumped from 30 to 60
|
# 429 the whole process stands down until this passes. The account was stuck for
|
||||||
# minutes: the account was stuck for days because every hourly tick re-hit it,
|
# days because the backoff kept expiring before Garmin's own (multi-hour) window
|
||||||
# so a longer cooldown gives Garmin's window room to actually close.
|
# closed, so every tick re-hit it and the limit never lifted. A 24h stand-down
|
||||||
# How long we stand down after a Garmin 429. Garmin's own throttle window runs
|
# is what actually outlasts the throttle and lets the window close for good.
|
||||||
# well past an hour for a repeat offender, so a short backoff just expires, lets
|
RATE_LIMIT_BACKOFF = datetime.timedelta(hours=24)
|
||||||
# the scheduler re-hit, and keeps the limit alive forever. Six hours of quiet is
|
|
||||||
# what actually lets the window close.
|
|
||||||
RATE_LIMIT_BACKOFF = datetime.timedelta(hours=6)
|
|
||||||
# In-process cache of the cooldown, kept in sync with the DB copy below and
|
# In-process cache of the cooldown, kept in sync with the DB copy below and
|
||||||
# still the lever the tests reach for via _rate_limited_until.clear().
|
# still the lever the tests reach for via _rate_limited_until.clear().
|
||||||
_rate_limited_until = {}
|
_rate_limited_until = {}
|
||||||
@@ -132,19 +130,23 @@ def _parse_dt(v):
|
|||||||
def rate_limited_until(user_id):
|
def rate_limited_until(user_id):
|
||||||
"""When the account is still cooling down, as a UTC datetime (or None).
|
"""When the account is still cooling down, as a UTC datetime (or None).
|
||||||
|
|
||||||
The cooldown is persisted to the database so every gunicorn worker and a
|
The cooldown lives in the database and is the single source of truth, so
|
||||||
process restart see the same deadline — a process-local dict alone let each
|
every gunicorn worker and a restart agree on it. A process-local dict caused
|
||||||
worker re-hit Garmin and keep the limit alive forever.
|
a nasty "stuck forever" bug: a worker would remember a future deadline whose
|
||||||
|
DB write had silently failed to persist, and go on blocking even after the
|
||||||
|
real deadline had passed. We therefore trust the DB row, falling back to the
|
||||||
|
in-memory cache only when no row exists yet.
|
||||||
"""
|
"""
|
||||||
mem = _rate_limited_until.get(user_id)
|
|
||||||
try:
|
try:
|
||||||
row = query_one(
|
row = query_one(
|
||||||
"SELECT rate_limited_until FROM sync_status WHERE user_id = ?", [user_id]
|
"SELECT rate_limited_until FROM sync_status WHERE user_id = ?", [user_id]
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
row = None
|
row = None
|
||||||
candidates = [c for c in (mem, _parse_dt(row["rate_limited_until"] if row else None)) if c]
|
db_val = _parse_dt(row["rate_limited_until"] if row else None)
|
||||||
return max(candidates) if candidates else None
|
if db_val is not None:
|
||||||
|
return db_val
|
||||||
|
return _rate_limited_until.get(user_id)
|
||||||
|
|
||||||
|
|
||||||
def _note_rate_limit(user_id):
|
def _note_rate_limit(user_id):
|
||||||
@@ -167,8 +169,10 @@ def _note_rate_limit(user_id):
|
|||||||
user_id, cur_status, now.isoformat(timespec="seconds"),
|
user_id, cur_status, now.isoformat(timespec="seconds"),
|
||||||
rate_limited_until=until.isoformat(timespec="seconds"),
|
rate_limited_until=until.isoformat(timespec="seconds"),
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception as e: # no cover - surfaced so a persist failure is visible
|
||||||
pass
|
logging.getLogger(__name__).warning(
|
||||||
|
"rate-limit cooldown failed to persist for %s: %s", user_id, e
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _rate_limit_block(user_id):
|
def _rate_limit_block(user_id):
|
||||||
|
|||||||
Reference in New Issue
Block a user