ericwyuan/GarminHealthLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(garmin): 刷新出来的令牌从来没写回库，于是每次连接都重换一次

Browse Source 
「又被限流了」的根因找到了，不是请求量，是令牌。

`_connect` 里 `refresh_oauth2()` 换来的新 OAuth2 令牌只活在进程内存里——
`save_token` 只在绑定账号时调用过一次。于是每次客户端缓存过期（15 分钟）、
每个 gunicorn worker、每次部署重启，都从库里读回**同一个已过期的令牌**，
然后再做一次真实 SSO 换令牌。而 SSO 端点是按账号限流最狠的那个，社区报告能
封 48 小时（garth #217、python-garminconnect #337）。我今天为了部署重启了
八次服务，每次都清掉缓存。

- `refresh_oauth2()` 成功后 `_persist_token()` 写回。拆出这个函数是因为它和
  `save_token` 想要的正好相反：重新绑定要作废现有会话，持久化刷新结果必须
  保住刚刚产出它的那个会话
- 写回时不带 garmin_email，否则 upsert 会把绑定邮箱刷成 NULL，数据同步页会
  忘记绑的是哪个账号
- 刷新加进程内锁，并在拿到锁后重读一次库：另一个线程刚换过就直接用它的，
  不再自己去换一次
- 五条测试盯住这个不变量，包括「冷缓存不该再换一次」（这条如果回归，就是同一
  个 bug 再来一遍）

顺带把数据端点也节流了——那是另外一半问题，不是这次的病因，但一天历史要 9 次
调用，730 天全历史 6600 个请求全速打出去，不该指望佳明一直容忍：

- services/garmin_throttle.py：代理包住 client，所有调用（含以后新加的）都经
  同一个收口，按间隔排队并计数
- 0.5s 是查过的：garmin-data-export 默认 0.15s、garmin-connect-scraper 默认
  3s、官方合作方 API 100 次/分钟（0.6s）。依据写在文件顶部
- 单次同步 1200 个请求预算，跑满就干净收尾、下次接着跑（已存的天数本来就跳过）
- 运动详情每次最多补 40 条——新账号几百条，不限量就是一次性打光预算

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ericwyuan
2026-09-03 23:24:48 +08:00
parent 57c236ba16
commit 682936b0b6
7 changed files with 606 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
backend/services/garmin.py
View File

@@ -30,6 +30,7 @@ from db import execute, query_one, query_all
from config import DB_TYPE
from services import health
from services import garmin_extras as extras
from services import garmin_throttle as throttle
# How many days back a sync reaches.
DEFAULT_SYNC_DAYS = int(os.environ.get("GARMIN_SYNC_DAYS") or 7)
@@ -372,7 +373,13 @@ def load_token(user_id):
return row["token"] if row else None
def save_token(user_id, token, garmin_email=None):
def _persist_token(user_id, token, garmin_email=None):
"""Write the token row. Does **not** drop the cached session.
Separate from `save_token` because the two callers want opposite things: a
re-bind must invalidate the live session, while persisting a refreshed
OAuth2 token must keep the very session that just produced it.
"""
cols = ["user_id", "token", "garmin_email", "updated_at"]
placeholders = ", ".join(["?"] * len(cols))
if DB_TYPE == "mariadb":
@@ -383,8 +390,33 @@ def save_token(user_id, token, garmin_email=None):
updates = ", ".join(f"{c}=excluded.{c}" for c in cols if c != "user_id")
sql = (f"INSERT INTO garmin_tokens ({', '.join(cols)}) VALUES ({placeholders}) "
f"ON CONFLICT(user_id) DO UPDATE SET {updates}")
execute(sql, [user_id, token, garmin_email,
datetime.datetime.utcnow().isoformat(timespec="seconds")])
params = [user_id, token, garmin_email,
datetime.datetime.utcnow().isoformat(timespec="seconds")]
if garmin_email is None:
# A refresh must not blank the remembered email: the column is part of
# the upsert, so passing None would overwrite it with NULL and the
# 数据同步 page would forget which account is bound.
cols_kept = [c for c in cols if c != "garmin_email"]
placeholders_kept = ", ".join(["?"] * len(cols_kept))
if DB_TYPE == "mariadb":
updates_kept = ", ".join(
f"{c}=VALUES({c})" for c in cols_kept if c != "user_id")
sql = (f"INSERT INTO garmin_tokens ({', '.join(cols_kept)}) "
f"VALUES ({placeholders_kept}) "
f"ON DUPLICATE KEY UPDATE {updates_kept}")
else:
updates_kept = ", ".join(
f"{c}=excluded.{c}" for c in cols_kept if c != "user_id")
sql = (f"INSERT INTO garmin_tokens ({', '.join(cols_kept)}) "
f"VALUES ({placeholders_kept}) "
f"ON CONFLICT(user_id) DO UPDATE SET {updates_kept}")
params = [user_id, token, params[-1]]
execute(sql, params)
def save_token(user_id, token, garmin_email=None):
"""Store a token from a fresh login, and stand the old session down."""
_persist_token(user_id, token, garmin_email)
# A re-bind means the old session is stale; the next call must build a
# fresh one rather than keep using the session the old token minted.
forget_client(user_id)
@@ -433,6 +465,8 @@ def delete_token(user_id):
CLIENT_TTL_SECONDS = 900
_clients = {}
_clients_lock = threading.Lock()
# Held only around an OAuth2 refresh — see _connect.
_refresh_lock = threading.Lock()
def _cached_client(user_id):
@@ -483,16 +517,45 @@ def _connect(creds, user_id=None):
# quota for nothing, and that is what walked the account into a 429.
oauth2 = getattr(client.garth, "oauth2_token", None)
if oauth2 is None or getattr(oauth2, "expired", True):
try:
client.garth.refresh_oauth2()
except Exception as e: # noqa: BLE001 - re-raised, just named better
if _is_rate_limited(e):
_note_rate_limit(user_id)
raise RateLimited(
"Garmin 暂时限制了请求频率。这通常是短时间内连接过于频繁，"
"等待约半小时后会自动恢复，令牌本身没有失效。"
) from e
raise
# One refresh at a time per process. Two threads reaching a cold
# client cache together would otherwise each mint a token against
# the endpoint that rate-limits hardest.
with _refresh_lock:
oauth2 = getattr(client.garth, "oauth2_token", None)
stored = load_token(user_id) if user_id else None
if stored and stored != token:
# Another thread refreshed while we waited: adopt its
# token instead of spending an SSO call of our own.
client.garth.loads(stored)
oauth2 = getattr(client.garth, "oauth2_token", None)
if oauth2 is None or getattr(oauth2, "expired", True):
try:
client.garth.refresh_oauth2()
except Exception as e: # noqa: BLE001 - re-raised, named better
if _is_rate_limited(e):
_note_rate_limit(user_id)
raise RateLimited(
"Garmin 暂时限制了请求频率。这通常是短时间内连接过于频繁，"
"等待约半小时后会自动恢复，令牌本身没有失效。"
) from e
raise
# Keep it. This line is the whole reason the account kept
# getting throttled: the refreshed OAuth2 token used to
# live only in this process's memory, so every cold client
# cache — a 15-minute timeout, a second gunicorn worker, a
# deploy restart — re-read the same expired token from the
# database and minted another one against Garmin's SSO
# endpoint. That endpoint limits per account and blocks for
# hours. Persisting it turns "an SSO call per connect" into
# "one per token lifetime" (about an hour).
if user_id:
try:
_persist_token(user_id, client.garth.dumps())
except Exception as e: # noqa: BLE001 - never fail a sync
logging.getLogger(__name__).warning(
"refreshed Garmin token failed to persist "
"for %s: %s", user_id, e
)
# garminconnect builds most of its URLs from display_name, so leaving
# it unset sends every request to ".../None".
client.display_name = client.garth.profile["displayName"]
@@ -1051,6 +1114,12 @@ ALWAYS_REFETCH_DAYS = 3
# per day would turn a year's sync into an hour.
SERIES_INLINE_DAYS = 14
# Activity details fetched per sync run. One request each, and a fresh account
# has hundreds — fetching them all in one run is the single biggest burst a
# sync can produce. The rest are picked up by the next run and by
# start_backfill, so nothing is lost by spreading them out.
DETAILS_PER_SYNC = int(os.environ.get("GARMIN_DETAILS_PER_SYNC") or 40)
# Above this many days a sync is long enough that the caller must not block
# on it — a year takes roughly 20 minutes at ~3s per day.
@@ -1136,6 +1205,14 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
"lastSyncTime": now,
})
# Every Garmin call from here on is spaced and counted. Applied here
# rather than in `_connect` so an injected client is paced identically —
# the pacing is a property of a sync run, not of how the session was
# obtained. See services/garmin_throttle.py for why this is necessary at
# all: a day of history costs nine requests, and a full backfill used to
# fire six thousand of them flat out.
client = throttle.pace(client, user_id)
today = datetime.date.today()
# -1 means "incremental sync": pick up from the latest date already in the
@@ -1177,6 +1254,7 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
reached_the_start = False
day_errors = []
rate_limited_mid_run = None
budget_spent = False
for i in range(days):
date_str = (today - datetime.timedelta(days=i)).isoformat()
if i >= ALWAYS_REFETCH_DAYS and date_str in stored:
@@ -1185,6 +1263,12 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
try:
record = _extract_daily(client, date_str)
record.update(extras.daily_extras(client, date_str))
except throttle.BudgetExhausted:
# Not a failure: this run has issued as many requests as it is
# allowed. Keep what is stored and stop — the next run skips the
# days already written and carries on from here.
budget_spent = True
break
except Exception as e:
# A 429 inside the day loop used to be filed as one more skipped
# day, so a 730-day backfill kept hammering Garmin for another
@@ -1252,9 +1336,12 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
progress_current=days, progress_total=days,
stage="运动记录")
try:
activities_synced = _sync_activities(
client, user_id, start_date, today.isoformat()
)
if not budget_spent:
activities_synced = _sync_activities(
client, user_id, start_date, today.isoformat()
)
except throttle.BudgetExhausted:
budget_spent = True
except Exception as e:
day_errors.append(f"activities: {describe(e)}")
@@ -1262,13 +1349,19 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
# opening one later is a local read.
details_synced = 0
try:
details_synced = sync_activity_details(
client, user_id,
on_progress=lambda d, n: _set_sync_status(
user_id, "syncing", now, records_synced=days_synced,
progress_current=days, progress_total=days,
stage=f"运动详情 {d}/{n}"),
)
if not budget_spent:
# Capped per run. On a fresh account this is one request for each
# of a couple of hundred activities; unbounded, it spends the whole
# budget (and used to spend the account's goodwill) in one burst.
details_synced = sync_activity_details(
client, user_id, limit=DETAILS_PER_SYNC,
on_progress=lambda d, n: _set_sync_status(
user_id, "syncing", now, records_synced=days_synced,
progress_current=days, progress_total=days,
stage=f"运动详情 {d}/{n}"),
)
except throttle.BudgetExhausted:
budget_spent = True
except Exception as e:
day_errors.append(f"activity_details: {describe(e)}")
@@ -1335,12 +1428,19 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
)
if day_errors:
message += f"（{len(day_errors)} 项跳过）"
if budget_spent:
# Said plainly: an unexplained "only 120 of 730 days" reads as a bug,
# and the next run picking up where this one stopped is the design.
message += "；本次已达请求预算上限，剩余部分下次同步继续"
requests_used = getattr(getattr(client, "limiter", None), "used", None)
return finish({
"status": "success",
"recordsSynced": days_synced,
"activitiesSynced": activities_synced,
"badgesSynced": badges_synced,
"personalRecordsSynced": records_synced_pr,
"requestsUsed": requests_used,
"partial": budget_spent,
"message": message,
"lastSyncTime": now,
})

202
backend/services/garmin_throttle.py Normal file
View File

@@ -0,0 +1,202 @@
"""
Client-side pacing for Garmin Connect.
Why this exists
---------------
A sync is not a handful of requests. Each day costs **nine** API calls
(`_extract_daily` makes six, `daily_extras` three), short syncs add five more
for the within-day curves, every activity without a stored detail costs one,
and five account-wide calls finish the run. So:
7 days ~ 100 requests
30 days ~ 300 requests
730 days ~ 6600 requests
and until now they were fired as fast as the network allowed. That is what
walks the account into a 429 — repeatedly, because the only defence was a 24h
cooldown *after* Garmin had already punished us. Backing off after the fact
does not stop it happening again; spacing the requests does.
Garmin publishes no limit for these endpoints, so the default comes from what
other consumers actually get away with (checked 2026-09-03):
* `sirredbeard/garmin-data-export` paces at **0.15s** by default, backs off on
a 429, ramps back up, and pauses every 250 calls. Its README suggests
`--delay 1.0` "if you want to play it safe".
* `evg656e/garmin-connect-scraper` defaults to **3s**, warning that smaller
values "may result in timeout penalties from Garmin".
* Garmin's *official* (partner) API allows **100 requests/minute** — 0.6s
each — which is the closest thing to a number Garmin itself stands behind.
0.5s sits between them: slower than the official allowance, three times more
cautious than the aggressive scraper, six times faster than the timid one.
Worth being clear about what this does and does not fix. The brutal
multi-hour blocks reported across those projects are on the **SSO/login**
endpoint and are keyed per account; this app's own lockout came from there,
not from data volume (it was re-minting a token on every connect — see
`_connect` in garmin.py). Pacing the data calls is the other half: a full
backfill firing 6600 requests flat out is not something to rely on Garmin
tolerating.
What it does
------------
* **Spacing.** Every call waits until at least `min_interval` has passed since
the previous one, per account.
* **A per-run budget.** A run may issue at most N requests; the sync then stops
cleanly and resumes next time (it already skips days it has stored). Without
this, "全部历史" is a single unbounded burst no matter how well spaced.
Both apply at one choke point — a proxy around the client object — so every
call site is covered, including ones added later. Anything that reaches Garmin
without going through the client is not paced, which today means only the
login/token calls on `client.garth`: a couple per sync, not thousands.
"""
import functools
import os
import threading
import time
class BudgetExhausted(Exception):
"""This run has issued as many requests as it is allowed.
Not an error: the caller keeps what it stored and continues next run.
"""
def min_interval():
"""Seconds between requests. Read per call so it is tunable without a
restart, and so tests can set it to 0."""
return float(os.environ.get("GARMIN_MIN_INTERVAL_SECONDS") or 0.5)
def default_budget():
"""Requests one sync run may issue.
1200 at 0.5s is about 10 minutes of wall clock — long enough to cover four
months of history in one go (9 requests per day), short enough that a full
backfill is a handful of background runs rather than one long hammering.
"""
return int(os.environ.get("GARMIN_REQUEST_BUDGET") or 1200)
class Limiter:
"""Per-account spacing and per-run request accounting."""
def __init__(self, interval=None):
self._interval = interval
self._lock = threading.Lock()
self._next_at = 0.0
self.used = 0
self.budget = None
@property
def interval(self):
return min_interval() if self._interval is None else self._interval
def start_run(self, budget=None):
"""Begin a run. `budget` None means unlimited (a short, known-size job)."""
with self._lock:
self.used = 0
self.budget = budget
@property
def remaining(self):
if self.budget is None:
return None
return max(0, self.budget - self.used)
def acquire(self):
"""Claim a slot, waiting for it if necessary."""
with self._lock:
if self.budget is not None and self.used >= self.budget:
raise BudgetExhausted(
f"本次同步已达请求预算上限（{self.budget} 次）"
)
self.used += 1
now = time.monotonic()
wait = self._next_at - now
# Reserve this call's slot before releasing the lock, so two
# threads get consecutive slots rather than the same one.
self._next_at = max(now, self._next_at) + self.interval
# Slept outside the lock: two callers should wait out their own
# (non-overlapping) slots concurrently, not queue behind each other's.
if wait > 0:
time.sleep(wait)
_limiters = {}
_limiters_lock = threading.Lock()
def limiter_for(user_id):
"""The limiter for one account, created on first use.
Per account and per process. Two Gunicorn workers therefore pace
independently — acceptable because a given account syncs from one place at
a time (the scheduler claims its tick through the database, and a manual
sync sets the status to `syncing`), and the budget bounds the damage if
that ever stops holding.
"""
key = user_id or "_anon"
with _limiters_lock:
found = _limiters.get(key)
if found is None:
found = Limiter()
_limiters[key] = found
return found
class PacedClient:
"""Attribute proxy that paces every public method call.
Non-callables (`display_name`, `garth`) pass through untouched, as do
private names, and attribute writes reach the wrapped client — `_connect`
sets `client.display_name` and the library reads it back on every request.
"""
def __init__(self, inner, limiter):
object.__setattr__(self, "_inner", inner)
object.__setattr__(self, "_limiter", limiter)
@property
def limiter(self):
return object.__getattribute__(self, "_limiter")
@property
def inner(self):
return object.__getattribute__(self, "_inner")
def __getattr__(self, name):
attr = getattr(object.__getattribute__(self, "_inner"), name)
if name.startswith("_") or not callable(attr):
return attr
limiter = object.__getattribute__(self, "_limiter")
@functools.wraps(attr)
def paced(*args, **kwargs):
limiter.acquire()
return attr(*args, **kwargs)
return paced
def __setattr__(self, name, value):
setattr(object.__getattribute__(self, "_inner"), name, value)
def __repr__(self):
inner = object.__getattribute__(self, "_inner")
return f"<PacedClient {inner!r}>"
def pace(client, user_id, budget=None):
"""Wrap `client` so its calls are spaced, and start a run.
Idempotent: wrapping an already-wrapped client returns it (with the run
restarted) rather than stacking two layers of waiting.
"""
limiter = limiter_for(user_id)
limiter.start_run(default_budget() if budget is None else budget)
if isinstance(client, PacedClient):
return client
return PacedClient(client, limiter)