fix(garmin): 登录端点的 429 在封锁期内再试会延长封锁,唯独这一处要停手
实测记录:昨天为了验证令牌写回的修复,我发了一次 1 天同步,把 rate_limited_until 从 09-04T00:41 推到了 09-04T15:26——一次尝试,延长十五小时。 Garmin 的登录/换令牌端点和数据端点是两套规则: - 数据端点的 429:本地冷却只是估算,过一会儿发个真实请求正是确认它有没有解除 的唯一办法,没解除也不吃亏 - 登录端点的 429:窗口内每次尝试都把窗口往后推。这也是这个账号一直卡着出不来 的原因——每次同步都去换一次令牌,每次都把封锁续上 所以: - sync_status 加 rate_limit_source 列,记住 429 是哪个端点给的 - 只有 source=sso 且窗口未过时,才拒绝**刷新令牌**这一个动作 这个闸门刻意做得很窄,因为上一次的教训是「一刀切的闸门会让健康账号白白闲置」: - 只拦刷新,不拦同步。库里的令牌只要还没过期,冷却期内照常同步 - 数据端点的 429 不参与判断——为了一次指标调用把账号锁一天,比问题本身更糟 - 用户可以推翻它:同步请求带 force 就先清掉冷却记录。那个截止时间是我们自己 猜的 24 小时,不是佳明说的,所以必须能被推翻 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -495,6 +495,12 @@ MIGRATIONS = {
|
||||
("progress_current", "INT"),
|
||||
("progress_total", "INT"),
|
||||
("started_at", "DATETIME"),
|
||||
# Which endpoint answered 429. The two need opposite handling: a data
|
||||
# 429 clears on its own and retrying is harmless, while Garmin's SSO
|
||||
# limit is *extended* by every attempt made inside its window — so a
|
||||
# token refresh must stand down and a data sync need not. Without this
|
||||
# column the cooldown cannot tell the caller which rule applies.
|
||||
("rate_limit_source", "VARCHAR(16)"),
|
||||
# Which part of the sync is running. "0 / 730 天" says nothing about
|
||||
# what is actually happening for the several minutes of it.
|
||||
("stage", "VARCHAR(64)"),
|
||||
|
||||
@@ -58,6 +58,14 @@ def sync():
|
||||
except (TypeError, ValueError):
|
||||
days = None
|
||||
|
||||
# `force` overrules the recorded SSO cooldown. That deadline is our own
|
||||
# 24h guess rather than something Garmin told us, so the user has to be
|
||||
# able to say "it has cleared, try anyway" — otherwise a wrong estimate
|
||||
# strands the account for a day, which is the failure that got a blanket
|
||||
# rate-limit gate removed once before.
|
||||
if data.get("force"):
|
||||
garmin_svc.clear_rate_limit(g.user_id)
|
||||
|
||||
# Always run in the background: even a week takes ~20s, and a full
|
||||
# backfill runs for many minutes. Progress is polled via /status.
|
||||
result = garmin_svc.start_sync(g.user_id, creds, days)
|
||||
|
||||
@@ -242,14 +242,24 @@ def rate_limited_until(user_id):
|
||||
return _rate_limited_until.get(user_id)
|
||||
|
||||
|
||||
def _note_rate_limit(user_id):
|
||||
"""Record a rate-limit cooldown.
|
||||
def _note_rate_limit(user_id, source="data"):
|
||||
"""Record a rate-limit cooldown, and which endpoint caused it.
|
||||
|
||||
A 429 means Garmin is throttling this account/IP, and every further request
|
||||
A 429 means Garmin is throttling this account, and every further request
|
||||
just extends the window — so we stand down for a long, fixed stretch
|
||||
(RATE_LIMIT_BACKOFF) rather than a short one that expires before Garmin's own
|
||||
throttle clears. The cooldown is persisted so every gunicorn worker and a
|
||||
restart agree on it.
|
||||
|
||||
`source` matters because the two kinds need opposite handling:
|
||||
|
||||
* `"data"` — the metric endpoints. The cooldown is an estimate; issuing a
|
||||
real request later is how we find out whether it has cleared, and costs
|
||||
nothing if it has not.
|
||||
* `"sso"` — the login/token endpoint. Attempting it *inside* the window
|
||||
pushes the window out: on 2026-09-03 a single test sync moved a recorded
|
||||
deadline from 00:41 to 15:26, fifteen hours later. That one must not be
|
||||
retried until the window has passed.
|
||||
"""
|
||||
now = datetime.datetime.utcnow()
|
||||
until = now + RATE_LIMIT_BACKOFF
|
||||
@@ -261,6 +271,7 @@ def _note_rate_limit(user_id):
|
||||
_set_sync_status(
|
||||
user_id, cur_status, now.isoformat(timespec="seconds"),
|
||||
rate_limited_until=until.isoformat(timespec="seconds"),
|
||||
rate_limit_source=source,
|
||||
)
|
||||
except Exception as e: # no cover - surfaced so a persist failure is visible
|
||||
logging.getLogger(__name__).warning(
|
||||
@@ -285,6 +296,44 @@ def _clear_rate_limit(user_id):
|
||||
pass
|
||||
|
||||
|
||||
def sso_cooldown(user_id):
|
||||
"""Remaining SSO login cooldown, or None.
|
||||
|
||||
Only reports a cooldown that a *login* 429 wrote. A data-endpoint 429 is
|
||||
deliberately not reported here: retrying those is how we discover the
|
||||
limit has cleared, and letting one keep a token refresh from happening
|
||||
would strand the account for a day over a metric call.
|
||||
"""
|
||||
try:
|
||||
row = query_one(
|
||||
"SELECT rate_limited_until, rate_limit_source FROM sync_status "
|
||||
"WHERE user_id = ?", [user_id],
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
if not row or (row.get("rate_limit_source") or "") != "sso":
|
||||
return None
|
||||
until = _parse_dt(row.get("rate_limited_until"))
|
||||
if until and until > datetime.datetime.utcnow():
|
||||
return until
|
||||
return None
|
||||
|
||||
|
||||
def clear_rate_limit(user_id):
|
||||
"""Public: forget the recorded cooldown so the next attempt goes through.
|
||||
|
||||
For the "我确认已恢复,立即重试" path. The recorded deadline is our own
|
||||
24h guess, not something Garmin told us, so the user has to be able to
|
||||
overrule it — that is exactly why a blanket gate was removed once before.
|
||||
"""
|
||||
_clear_rate_limit(user_id)
|
||||
try:
|
||||
execute("UPDATE sync_status SET rate_limit_source = NULL WHERE user_id = ?",
|
||||
[user_id])
|
||||
except Exception: # noqa: BLE001 - best-effort
|
||||
pass
|
||||
|
||||
|
||||
def _rate_limit_block(user_id):
|
||||
"""Human-readable recovery estimate from the *recorded* cooldown.
|
||||
|
||||
@@ -529,11 +578,36 @@ def _connect(creds, user_id=None):
|
||||
client.garth.loads(stored)
|
||||
oauth2 = getattr(client.garth, "oauth2_token", None)
|
||||
if oauth2 is None or getattr(oauth2, "expired", True):
|
||||
# The one place a cooldown is allowed to refuse an
|
||||
# attempt. Garmin's login limit is *extended* by every
|
||||
# request made inside its window — measured, not assumed:
|
||||
# on 2026-09-03 one test sync moved the deadline from
|
||||
# 00:41 to 15:26. So while that window is open we do not
|
||||
# touch the endpoint.
|
||||
#
|
||||
# Narrow on purpose. It gates only the refresh, and only
|
||||
# when a refresh is actually needed: a stored token whose
|
||||
# OAuth2 is still valid syncs normally whatever the
|
||||
# cooldown says, which is what a blanket gate got wrong
|
||||
# (it kept healthy accounts idle). And the user can
|
||||
# overrule it — see clear_rate_limit / force.
|
||||
blocked = sso_cooldown(user_id) if user_id else None
|
||||
if blocked:
|
||||
minutes = int(
|
||||
(blocked - datetime.datetime.utcnow()).total_seconds() // 60
|
||||
)
|
||||
raise RateLimited(
|
||||
"Garmin 正在限制该账号的登录请求,且本地令牌已过期,"
|
||||
f"需要重新换取。预计 {blocked.isoformat(timespec='minutes')} "
|
||||
f"UTC 之后恢复(约 {minutes} 分钟)。"
|
||||
"此时再试会延长封锁时间,因此本次不发请求。"
|
||||
"确认已恢复可在同步页选择强制重试。"
|
||||
)
|
||||
try:
|
||||
client.garth.refresh_oauth2()
|
||||
except Exception as e: # noqa: BLE001 - re-raised, named better
|
||||
if _is_rate_limited(e):
|
||||
_note_rate_limit(user_id)
|
||||
_note_rate_limit(user_id, "sso")
|
||||
raise RateLimited(
|
||||
"Garmin 暂时限制了请求频率。这通常是短时间内连接过于频繁,"
|
||||
"等待约半小时后会自动恢复,令牌本身没有失效。"
|
||||
|
||||
@@ -119,7 +119,7 @@ def _run_login(session_id, user_id, garmin_email, password, is_cn, import_garmin
|
||||
# so the scheduler and future logins back off instead of re-hammering and
|
||||
# keeping Garmin's throttle alive forever.
|
||||
if garmin_svc._is_rate_limited(e):
|
||||
garmin_svc._note_rate_limit(user_id)
|
||||
garmin_svc._note_rate_limit(user_id, "sso")
|
||||
error = (
|
||||
"Garmin 返回 429 限流(登录接口)。已自动退避 24 小时,"
|
||||
"请等待冷却窗口结束后再试——反复尝试会越撞越久。"
|
||||
|
||||
@@ -1128,3 +1128,114 @@ class TestRefreshedTokenIsKept:
|
||||
with pytest.raises(garmin_svc.RateLimited):
|
||||
garmin_svc._connect({}, user["id"])
|
||||
assert garmin_svc.rate_limited_until(user["id"]) is not None
|
||||
|
||||
|
||||
class TestSsoCooldownIsRespected:
|
||||
"""Garmin's *login* limit is extended by every attempt inside its window.
|
||||
|
||||
Measured on 2026-09-03: one test sync moved a recorded deadline from
|
||||
00:41 to 15:26. That is why this one cooldown refuses rather than probes —
|
||||
and why the refusal is as narrow as it can be.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _garmin(refreshes, oauth2_expired=True):
|
||||
class StubOAuth2:
|
||||
expired = oauth2_expired
|
||||
|
||||
class StubGarth:
|
||||
profile = {"displayName": "Tester"}
|
||||
|
||||
def __init__(self):
|
||||
self.oauth2_token = StubOAuth2()
|
||||
|
||||
def configure(self, **kwargs):
|
||||
pass
|
||||
|
||||
def loads(self, token):
|
||||
pass
|
||||
|
||||
def refresh_oauth2(self):
|
||||
refreshes.append(1)
|
||||
self.oauth2_token = None
|
||||
|
||||
def dumps(self):
|
||||
return "fresh"
|
||||
|
||||
class StubGarmin:
|
||||
def __init__(self, is_cn=False):
|
||||
self.garth = StubGarth()
|
||||
|
||||
return StubGarmin
|
||||
|
||||
def test_no_login_request_is_made_inside_the_window(
|
||||
self, db, user, monkeypatch
|
||||
):
|
||||
refreshes = []
|
||||
monkeypatch.setattr(garmin_svc, "_import_garmin",
|
||||
lambda: self._garmin(refreshes))
|
||||
garmin_svc.save_token(user["id"], "stored", "a@example.com")
|
||||
garmin_svc._note_rate_limit(user["id"], "sso")
|
||||
|
||||
with pytest.raises(garmin_svc.RateLimited):
|
||||
garmin_svc._connect({}, user["id"])
|
||||
assert refreshes == [], "every attempt inside the window extends it"
|
||||
|
||||
def test_a_data_429_does_not_block_a_token_refresh(self, db, user, monkeypatch):
|
||||
"""Stranding an account for a day over a metric call would be worse
|
||||
than the problem."""
|
||||
refreshes = []
|
||||
monkeypatch.setattr(garmin_svc, "_import_garmin",
|
||||
lambda: self._garmin(refreshes))
|
||||
garmin_svc.save_token(user["id"], "stored", "a@example.com")
|
||||
garmin_svc._note_rate_limit(user["id"], "data")
|
||||
|
||||
garmin_svc._connect({}, user["id"])
|
||||
assert refreshes == [1]
|
||||
|
||||
def test_a_valid_token_syncs_whatever_the_cooldown_says(
|
||||
self, db, user, monkeypatch
|
||||
):
|
||||
"""The gate covers the refresh, not the account: a stored token that
|
||||
has not expired needs no login request at all."""
|
||||
refreshes = []
|
||||
monkeypatch.setattr(
|
||||
garmin_svc, "_import_garmin",
|
||||
lambda: self._garmin(refreshes, oauth2_expired=False))
|
||||
garmin_svc.save_token(user["id"], "stored", "a@example.com")
|
||||
garmin_svc._note_rate_limit(user["id"], "sso")
|
||||
|
||||
garmin_svc._connect({}, user["id"]) # must not raise
|
||||
assert refreshes == []
|
||||
|
||||
def test_the_message_says_when_and_why(self, db, user, monkeypatch):
|
||||
monkeypatch.setattr(garmin_svc, "_import_garmin",
|
||||
lambda: self._garmin([]))
|
||||
garmin_svc.save_token(user["id"], "stored", "a@example.com")
|
||||
garmin_svc._note_rate_limit(user["id"], "sso")
|
||||
|
||||
with pytest.raises(garmin_svc.RateLimited) as caught:
|
||||
garmin_svc._connect({}, user["id"])
|
||||
text = str(caught.value)
|
||||
assert "延长封锁" in text, "the reason for not retrying must be stated"
|
||||
assert "强制重试" in text, "and the way out of it"
|
||||
|
||||
def test_the_user_can_overrule_the_estimate(self, db, user, monkeypatch):
|
||||
refreshes = []
|
||||
monkeypatch.setattr(garmin_svc, "_import_garmin",
|
||||
lambda: self._garmin(refreshes))
|
||||
garmin_svc.save_token(user["id"], "stored", "a@example.com")
|
||||
garmin_svc._note_rate_limit(user["id"], "sso")
|
||||
|
||||
garmin_svc.clear_rate_limit(user["id"])
|
||||
garmin_svc._connect({}, user["id"])
|
||||
assert refreshes == [1]
|
||||
|
||||
def test_sync_force_clears_the_cooldown(self, client, auth, db, user, monkeypatch):
|
||||
garmin_svc.save_token(user["id"], "stored", "a@example.com")
|
||||
garmin_svc._note_rate_limit(user["id"], "sso")
|
||||
monkeypatch.setattr(garmin_svc, "start_sync",
|
||||
lambda *a, **k: {"status": "syncing"})
|
||||
|
||||
client.post("/api/garmin/sync", json={"force": True}, headers=auth)
|
||||
assert garmin_svc.sso_cooldown(user["id"]) is None
|
||||
|
||||
Reference in New Issue
Block a user