ericwyuan/GarminHealthLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(garmin): 刷新出来的令牌从来没写回库，于是每次连接都重换一次

Browse Source 
「又被限流了」的根因找到了，不是请求量，是令牌。

`_connect` 里 `refresh_oauth2()` 换来的新 OAuth2 令牌只活在进程内存里——
`save_token` 只在绑定账号时调用过一次。于是每次客户端缓存过期（15 分钟）、
每个 gunicorn worker、每次部署重启，都从库里读回**同一个已过期的令牌**，
然后再做一次真实 SSO 换令牌。而 SSO 端点是按账号限流最狠的那个，社区报告能
封 48 小时（garth #217、python-garminconnect #337）。我今天为了部署重启了
八次服务，每次都清掉缓存。

- `refresh_oauth2()` 成功后 `_persist_token()` 写回。拆出这个函数是因为它和
  `save_token` 想要的正好相反：重新绑定要作废现有会话，持久化刷新结果必须
  保住刚刚产出它的那个会话
- 写回时不带 garmin_email，否则 upsert 会把绑定邮箱刷成 NULL，数据同步页会
  忘记绑的是哪个账号
- 刷新加进程内锁，并在拿到锁后重读一次库：另一个线程刚换过就直接用它的，
  不再自己去换一次
- 五条测试盯住这个不变量，包括「冷缓存不该再换一次」（这条如果回归，就是同一
  个 bug 再来一遍）

顺带把数据端点也节流了——那是另外一半问题，不是这次的病因，但一天历史要 9 次
调用，730 天全历史 6600 个请求全速打出去，不该指望佳明一直容忍：

- services/garmin_throttle.py：代理包住 client，所有调用（含以后新加的）都经
  同一个收口，按间隔排队并计数
- 0.5s 是查过的：garmin-data-export 默认 0.15s、garmin-connect-scraper 默认
  3s、官方合作方 API 100 次/分钟（0.6s）。依据写在文件顶部
- 单次同步 1200 个请求预算，跑满就干净收尾、下次接着跑（已存的天数本来就跳过）
- 运动详情每次最多补 40 条——新账号几百条，不限量就是一次性打光预算

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ericwyuan
2026-09-03 23:24:48 +08:00
parent 57c236ba16
commit 682936b0b6
7 changed files with 606 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
backend/services/garmin.py
View File

@@ -30,6 +30,7 @@ from db import execute, query_one, query_all
from config import DB_TYPE
from services import health
from services import garmin_extras as extras
from services import garmin_throttle as throttle
# How many days back a sync reaches.
DEFAULT_SYNC_DAYS = int(os.environ.get("GARMIN_SYNC_DAYS") or 7)
@@ -372,7 +373,13 @@ def load_token(user_id):
return row["token"] if row else None
def save_token(user_id, token, garmin_email=None):
def _persist_token(user_id, token, garmin_email=None):
"""Write the token row. Does **not** drop the cached session.
Separate from `save_token` because the two callers want opposite things: a
re-bind must invalidate the live session, while persisting a refreshed
OAuth2 token must keep the very session that just produced it.
"""
cols = ["user_id", "token", "garmin_email", "updated_at"]
placeholders = ", ".join(["?"] * len(cols))
if DB_TYPE == "mariadb":
@@ -383,8 +390,33 @@ def save_token(user_id, token, garmin_email=None):
updates = ", ".join(f"{c}=excluded.{c}" for c in cols if c != "user_id")
sql = (f"INSERT INTO garmin_tokens ({', '.join(cols)}) VALUES ({placeholders}) "
f"ON CONFLICT(user_id) DO UPDATE SET {updates}")
execute(sql, [user_id, token, garmin_email,
datetime.datetime.utcnow().isoformat(timespec="seconds")])
params = [user_id, token, garmin_email,
datetime.datetime.utcnow().isoformat(timespec="seconds")]
if garmin_email is None:
# A refresh must not blank the remembered email: the column is part of
# the upsert, so passing None would overwrite it with NULL and the
# 数据同步 page would forget which account is bound.
cols_kept = [c for c in cols if c != "garmin_email"]
placeholders_kept = ", ".join(["?"] * len(cols_kept))
if DB_TYPE == "mariadb":
updates_kept = ", ".join(
f"{c}=VALUES({c})" for c in cols_kept if c != "user_id")
sql = (f"INSERT INTO garmin_tokens ({', '.join(cols_kept)}) "
f"VALUES ({placeholders_kept}) "
f"ON DUPLICATE KEY UPDATE {updates_kept}")
else:
updates_kept = ", ".join(
f"{c}=excluded.{c}" for c in cols_kept if c != "user_id")
sql = (f"INSERT INTO garmin_tokens ({', '.join(cols_kept)}) "
f"VALUES ({placeholders_kept}) "
f"ON CONFLICT(user_id) DO UPDATE SET {updates_kept}")
params = [user_id, token, params[-1]]
execute(sql, params)
def save_token(user_id, token, garmin_email=None):
"""Store a token from a fresh login, and stand the old session down."""
_persist_token(user_id, token, garmin_email)
# A re-bind means the old session is stale; the next call must build a
# fresh one rather than keep using the session the old token minted.
forget_client(user_id)
@@ -433,6 +465,8 @@ def delete_token(user_id):
CLIENT_TTL_SECONDS = 900
_clients = {}
_clients_lock = threading.Lock()
# Held only around an OAuth2 refresh — see _connect.
_refresh_lock = threading.Lock()
def _cached_client(user_id):
@@ -483,16 +517,45 @@ def _connect(creds, user_id=None):
# quota for nothing, and that is what walked the account into a 429.
oauth2 = getattr(client.garth, "oauth2_token", None)
if oauth2 is None or getattr(oauth2, "expired", True):
try:
client.garth.refresh_oauth2()
except Exception as e: # noqa: BLE001 - re-raised, just named better
if _is_rate_limited(e):
_note_rate_limit(user_id)
raise RateLimited(
"Garmin 暂时限制了请求频率。这通常是短时间内连接过于频繁，"
"等待约半小时后会自动恢复，令牌本身没有失效。"
) from e
raise
# One refresh at a time per process. Two threads reaching a cold
# client cache together would otherwise each mint a token against
# the endpoint that rate-limits hardest.
with _refresh_lock:
oauth2 = getattr(client.garth, "oauth2_token", None)
stored = load_token(user_id) if user_id else None
if stored and stored != token:
# Another thread refreshed while we waited: adopt its
# token instead of spending an SSO call of our own.
client.garth.loads(stored)
oauth2 = getattr(client.garth, "oauth2_token", None)
if oauth2 is None or getattr(oauth2, "expired", True):
try:
client.garth.refresh_oauth2()
except Exception as e: # noqa: BLE001 - re-raised, named better
if _is_rate_limited(e):
_note_rate_limit(user_id)
raise RateLimited(
"Garmin 暂时限制了请求频率。这通常是短时间内连接过于频繁，"
"等待约半小时后会自动恢复，令牌本身没有失效。"
) from e
raise
# Keep it. This line is the whole reason the account kept
# getting throttled: the refreshed OAuth2 token used to
# live only in this process's memory, so every cold client
# cache — a 15-minute timeout, a second gunicorn worker, a
# deploy restart — re-read the same expired token from the
# database and minted another one against Garmin's SSO
# endpoint. That endpoint limits per account and blocks for
# hours. Persisting it turns "an SSO call per connect" into
# "one per token lifetime" (about an hour).
if user_id:
try:
_persist_token(user_id, client.garth.dumps())
except Exception as e: # noqa: BLE001 - never fail a sync
logging.getLogger(__name__).warning(
"refreshed Garmin token failed to persist "
"for %s: %s", user_id, e
)
# garminconnect builds most of its URLs from display_name, so leaving
# it unset sends every request to ".../None".
client.display_name = client.garth.profile["displayName"]
@@ -1051,6 +1114,12 @@ ALWAYS_REFETCH_DAYS = 3
# per day would turn a year's sync into an hour.
SERIES_INLINE_DAYS = 14
# Activity details fetched per sync run. One request each, and a fresh account
# has hundreds — fetching them all in one run is the single biggest burst a
# sync can produce. The rest are picked up by the next run and by
# start_backfill, so nothing is lost by spreading them out.
DETAILS_PER_SYNC = int(os.environ.get("GARMIN_DETAILS_PER_SYNC") or 40)
# Above this many days a sync is long enough that the caller must not block
# on it — a year takes roughly 20 minutes at ~3s per day.
@@ -1136,6 +1205,14 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
"lastSyncTime": now,
})
# Every Garmin call from here on is spaced and counted. Applied here
# rather than in `_connect` so an injected client is paced identically —
# the pacing is a property of a sync run, not of how the session was
# obtained. See services/garmin_throttle.py for why this is necessary at
# all: a day of history costs nine requests, and a full backfill used to
# fire six thousand of them flat out.
client = throttle.pace(client, user_id)
today = datetime.date.today()
# -1 means "incremental sync": pick up from the latest date already in the
@@ -1177,6 +1254,7 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
reached_the_start = False
day_errors = []
rate_limited_mid_run = None
budget_spent = False
for i in range(days):
date_str = (today - datetime.timedelta(days=i)).isoformat()
if i >= ALWAYS_REFETCH_DAYS and date_str in stored:
@@ -1185,6 +1263,12 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
try:
record = _extract_daily(client, date_str)
record.update(extras.daily_extras(client, date_str))
except throttle.BudgetExhausted:
# Not a failure: this run has issued as many requests as it is
# allowed. Keep what is stored and stop — the next run skips the
# days already written and carries on from here.
budget_spent = True
break
except Exception as e:
# A 429 inside the day loop used to be filed as one more skipped
# day, so a 730-day backfill kept hammering Garmin for another
@@ -1252,9 +1336,12 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
progress_current=days, progress_total=days,
stage="运动记录")
try:
activities_synced = _sync_activities(
client, user_id, start_date, today.isoformat()
)
if not budget_spent:
activities_synced = _sync_activities(
client, user_id, start_date, today.isoformat()
)
except throttle.BudgetExhausted:
budget_spent = True
except Exception as e:
day_errors.append(f"activities: {describe(e)}")
@@ -1262,13 +1349,19 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
# opening one later is a local read.
details_synced = 0
try:
details_synced = sync_activity_details(
client, user_id,
on_progress=lambda d, n: _set_sync_status(
user_id, "syncing", now, records_synced=days_synced,
progress_current=days, progress_total=days,
stage=f"运动详情 {d}/{n}"),
)
if not budget_spent:
# Capped per run. On a fresh account this is one request for each
# of a couple of hundred activities; unbounded, it spends the whole
# budget (and used to spend the account's goodwill) in one burst.
details_synced = sync_activity_details(
client, user_id, limit=DETAILS_PER_SYNC,
on_progress=lambda d, n: _set_sync_status(
user_id, "syncing", now, records_synced=days_synced,
progress_current=days, progress_total=days,
stage=f"运动详情 {d}/{n}"),
)
except throttle.BudgetExhausted:
budget_spent = True
except Exception as e:
day_errors.append(f"activity_details: {describe(e)}")
@@ -1335,12 +1428,19 @@ def sync_data(user_id, creds, days=None, client=None, trigger="manual"):
)
if day_errors:
message += f"（{len(day_errors)} 项跳过）"
if budget_spent:
# Said plainly: an unexplained "only 120 of 730 days" reads as a bug,
# and the next run picking up where this one stopped is the design.
message += "；本次已达请求预算上限，剩余部分下次同步继续"
requests_used = getattr(getattr(client, "limiter", None), "used", None)
return finish({
"status": "success",
"recordsSynced": days_synced,
"activitiesSynced": activities_synced,
"badgesSynced": badges_synced,
"personalRecordsSynced": records_synced_pr,
"requestsUsed": requests_used,
"partial": budget_spent,
"message": message,
"lastSyncTime": now,
})