fix(garmin): 登录端点的 429 在封锁期内再试会延长封锁,唯独这一处要停手

实测记录:昨天为了验证令牌写回的修复,我发了一次 1 天同步,把 rate_limited_until
从 09-04T00:41 推到了 09-04T15:26——一次尝试,延长十五小时。

Garmin 的登录/换令牌端点和数据端点是两套规则:
- 数据端点的 429:本地冷却只是估算,过一会儿发个真实请求正是确认它有没有解除
  的唯一办法,没解除也不吃亏
- 登录端点的 429:窗口内每次尝试都把窗口往后推。这也是这个账号一直卡着出不来
  的原因——每次同步都去换一次令牌,每次都把封锁续上

所以:
- sync_status 加 rate_limit_source 列,记住 429 是哪个端点给的
- 只有 source=sso 且窗口未过时,才拒绝**刷新令牌**这一个动作

这个闸门刻意做得很窄,因为上一次的教训是「一刀切的闸门会让健康账号白白闲置」:
- 只拦刷新,不拦同步。库里的令牌只要还没过期,冷却期内照常同步
- 数据端点的 429 不参与判断——为了一次指标调用把账号锁一天,比问题本身更糟
- 用户可以推翻它:同步请求带 force 就先清掉冷却记录。那个截止时间是我们自己
  猜的 24 小时,不是佳明说的,所以必须能被推翻

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ericwyuan
2026-09-04 06:13:52 +08:00
parent 682936b0b6
commit fd5e3363fd
5 changed files with 204 additions and 5 deletions

View File

@@ -495,6 +495,12 @@ MIGRATIONS = {
("progress_current", "INT"),
("progress_total", "INT"),
("started_at", "DATETIME"),
# Which endpoint answered 429. The two need opposite handling: a data
# 429 clears on its own and retrying is harmless, while Garmin's SSO
# limit is *extended* by every attempt made inside its window — so a
# token refresh must stand down and a data sync need not. Without this
# column the cooldown cannot tell the caller which rule applies.
("rate_limit_source", "VARCHAR(16)"),
# Which part of the sync is running. "0 / 730 天" says nothing about
# what is actually happening for the several minutes of it.
("stage", "VARCHAR(64)"),

View File

@@ -58,6 +58,14 @@ def sync():
except (TypeError, ValueError):
days = None
# `force` overrules the recorded SSO cooldown. That deadline is our own
# 24h guess rather than something Garmin told us, so the user has to be
# able to say "it has cleared, try anyway" — otherwise a wrong estimate
# strands the account for a day, which is the failure that got a blanket
# rate-limit gate removed once before.
if data.get("force"):
garmin_svc.clear_rate_limit(g.user_id)
# Always run in the background: even a week takes ~20s, and a full
# backfill runs for many minutes. Progress is polled via /status.
result = garmin_svc.start_sync(g.user_id, creds, days)

View File

@@ -242,14 +242,24 @@ def rate_limited_until(user_id):
return _rate_limited_until.get(user_id)
def _note_rate_limit(user_id):
"""Record a rate-limit cooldown.
def _note_rate_limit(user_id, source="data"):
"""Record a rate-limit cooldown, and which endpoint caused it.
A 429 means Garmin is throttling this account/IP, and every further request
A 429 means Garmin is throttling this account, and every further request
just extends the window — so we stand down for a long, fixed stretch
(RATE_LIMIT_BACKOFF) rather than a short one that expires before Garmin's own
throttle clears. The cooldown is persisted so every gunicorn worker and a
restart agree on it.
`source` matters because the two kinds need opposite handling:
* `"data"` — the metric endpoints. The cooldown is an estimate; issuing a
real request later is how we find out whether it has cleared, and costs
nothing if it has not.
* `"sso"` — the login/token endpoint. Attempting it *inside* the window
pushes the window out: on 2026-09-03 a single test sync moved a recorded
deadline from 00:41 to 15:26, fifteen hours later. That one must not be
retried until the window has passed.
"""
now = datetime.datetime.utcnow()
until = now + RATE_LIMIT_BACKOFF
@@ -261,6 +271,7 @@ def _note_rate_limit(user_id):
_set_sync_status(
user_id, cur_status, now.isoformat(timespec="seconds"),
rate_limited_until=until.isoformat(timespec="seconds"),
rate_limit_source=source,
)
except Exception as e: # no cover - surfaced so a persist failure is visible
logging.getLogger(__name__).warning(
@@ -285,6 +296,44 @@ def _clear_rate_limit(user_id):
pass
def sso_cooldown(user_id):
"""Remaining SSO login cooldown, or None.
Only reports a cooldown that a *login* 429 wrote. A data-endpoint 429 is
deliberately not reported here: retrying those is how we discover the
limit has cleared, and letting one keep a token refresh from happening
would strand the account for a day over a metric call.
"""
try:
row = query_one(
"SELECT rate_limited_until, rate_limit_source FROM sync_status "
"WHERE user_id = ?", [user_id],
)
except Exception:
return None
if not row or (row.get("rate_limit_source") or "") != "sso":
return None
until = _parse_dt(row.get("rate_limited_until"))
if until and until > datetime.datetime.utcnow():
return until
return None
def clear_rate_limit(user_id):
"""Public: forget the recorded cooldown so the next attempt goes through.
For the "我确认已恢复,立即重试" path. The recorded deadline is our own
24h guess, not something Garmin told us, so the user has to be able to
overrule it — that is exactly why a blanket gate was removed once before.
"""
_clear_rate_limit(user_id)
try:
execute("UPDATE sync_status SET rate_limit_source = NULL WHERE user_id = ?",
[user_id])
except Exception: # noqa: BLE001 - best-effort
pass
def _rate_limit_block(user_id):
"""Human-readable recovery estimate from the *recorded* cooldown.
@@ -529,11 +578,36 @@ def _connect(creds, user_id=None):
client.garth.loads(stored)
oauth2 = getattr(client.garth, "oauth2_token", None)
if oauth2 is None or getattr(oauth2, "expired", True):
# The one place a cooldown is allowed to refuse an
# attempt. Garmin's login limit is *extended* by every
# request made inside its window — measured, not assumed:
# on 2026-09-03 one test sync moved the deadline from
# 00:41 to 15:26. So while that window is open we do not
# touch the endpoint.
#
# Narrow on purpose. It gates only the refresh, and only
# when a refresh is actually needed: a stored token whose
# OAuth2 is still valid syncs normally whatever the
# cooldown says, which is what a blanket gate got wrong
# (it kept healthy accounts idle). And the user can
# overrule it — see clear_rate_limit / force.
blocked = sso_cooldown(user_id) if user_id else None
if blocked:
minutes = int(
(blocked - datetime.datetime.utcnow()).total_seconds() // 60
)
raise RateLimited(
"Garmin 正在限制该账号的登录请求,且本地令牌已过期,"
f"需要重新换取。预计 {blocked.isoformat(timespec='minutes')} "
f"UTC 之后恢复(约 {minutes} 分钟)。"
"此时再试会延长封锁时间,因此本次不发请求。"
"确认已恢复可在同步页选择强制重试。"
)
try:
client.garth.refresh_oauth2()
except Exception as e: # noqa: BLE001 - re-raised, named better
if _is_rate_limited(e):
_note_rate_limit(user_id)
_note_rate_limit(user_id, "sso")
raise RateLimited(
"Garmin 暂时限制了请求频率。这通常是短时间内连接过于频繁,"
"等待约半小时后会自动恢复,令牌本身没有失效。"

View File

@@ -119,7 +119,7 @@ def _run_login(session_id, user_id, garmin_email, password, is_cn, import_garmin
# so the scheduler and future logins back off instead of re-hammering and
# keeping Garmin's throttle alive forever.
if garmin_svc._is_rate_limited(e):
garmin_svc._note_rate_limit(user_id)
garmin_svc._note_rate_limit(user_id, "sso")
error = (
"Garmin 返回 429 限流(登录接口)。已自动退避 24 小时,"
"请等待冷却窗口结束后再试——反复尝试会越撞越久。"

View File

@@ -1128,3 +1128,114 @@ class TestRefreshedTokenIsKept:
with pytest.raises(garmin_svc.RateLimited):
garmin_svc._connect({}, user["id"])
assert garmin_svc.rate_limited_until(user["id"]) is not None
class TestSsoCooldownIsRespected:
"""Garmin's *login* limit is extended by every attempt inside its window.
Measured on 2026-09-03: one test sync moved a recorded deadline from
00:41 to 15:26. That is why this one cooldown refuses rather than probes —
and why the refusal is as narrow as it can be.
"""
@staticmethod
def _garmin(refreshes, oauth2_expired=True):
class StubOAuth2:
expired = oauth2_expired
class StubGarth:
profile = {"displayName": "Tester"}
def __init__(self):
self.oauth2_token = StubOAuth2()
def configure(self, **kwargs):
pass
def loads(self, token):
pass
def refresh_oauth2(self):
refreshes.append(1)
self.oauth2_token = None
def dumps(self):
return "fresh"
class StubGarmin:
def __init__(self, is_cn=False):
self.garth = StubGarth()
return StubGarmin
def test_no_login_request_is_made_inside_the_window(
self, db, user, monkeypatch
):
refreshes = []
monkeypatch.setattr(garmin_svc, "_import_garmin",
lambda: self._garmin(refreshes))
garmin_svc.save_token(user["id"], "stored", "a@example.com")
garmin_svc._note_rate_limit(user["id"], "sso")
with pytest.raises(garmin_svc.RateLimited):
garmin_svc._connect({}, user["id"])
assert refreshes == [], "every attempt inside the window extends it"
def test_a_data_429_does_not_block_a_token_refresh(self, db, user, monkeypatch):
"""Stranding an account for a day over a metric call would be worse
than the problem."""
refreshes = []
monkeypatch.setattr(garmin_svc, "_import_garmin",
lambda: self._garmin(refreshes))
garmin_svc.save_token(user["id"], "stored", "a@example.com")
garmin_svc._note_rate_limit(user["id"], "data")
garmin_svc._connect({}, user["id"])
assert refreshes == [1]
def test_a_valid_token_syncs_whatever_the_cooldown_says(
self, db, user, monkeypatch
):
"""The gate covers the refresh, not the account: a stored token that
has not expired needs no login request at all."""
refreshes = []
monkeypatch.setattr(
garmin_svc, "_import_garmin",
lambda: self._garmin(refreshes, oauth2_expired=False))
garmin_svc.save_token(user["id"], "stored", "a@example.com")
garmin_svc._note_rate_limit(user["id"], "sso")
garmin_svc._connect({}, user["id"]) # must not raise
assert refreshes == []
def test_the_message_says_when_and_why(self, db, user, monkeypatch):
monkeypatch.setattr(garmin_svc, "_import_garmin",
lambda: self._garmin([]))
garmin_svc.save_token(user["id"], "stored", "a@example.com")
garmin_svc._note_rate_limit(user["id"], "sso")
with pytest.raises(garmin_svc.RateLimited) as caught:
garmin_svc._connect({}, user["id"])
text = str(caught.value)
assert "延长封锁" in text, "the reason for not retrying must be stated"
assert "强制重试" in text, "and the way out of it"
def test_the_user_can_overrule_the_estimate(self, db, user, monkeypatch):
refreshes = []
monkeypatch.setattr(garmin_svc, "_import_garmin",
lambda: self._garmin(refreshes))
garmin_svc.save_token(user["id"], "stored", "a@example.com")
garmin_svc._note_rate_limit(user["id"], "sso")
garmin_svc.clear_rate_limit(user["id"])
garmin_svc._connect({}, user["id"])
assert refreshes == [1]
def test_sync_force_clears_the_cooldown(self, client, auth, db, user, monkeypatch):
garmin_svc.save_token(user["id"], "stored", "a@example.com")
garmin_svc._note_rate_limit(user["id"], "sso")
monkeypatch.setattr(garmin_svc, "start_sync",
lambda *a, **k: {"status": "syncing"})
client.post("/api/garmin/sync", json={"force": True}, headers=auth)
assert garmin_svc.sso_cooldown(user["id"]) is None