fix(db): 数据库晚一步启动就会让服务永久挂掉
今天 11:33 站点整个不可用。不是代码的问题,是启动顺序: NAS 上 MariaDB 在 11:34 才起来,而应用 11:33 就尝试连接, init_db() 抛异常 → gunicorn 报 Worker failed to boot → master 退出。 一分钟后数据库好了,但已经没有进程在跑,没人会去重试—— 站点就一直躺到有人手工重启为止。 init_db 改为在 DB_INIT_RETRY_SECONDS(默认 120 秒)内重试等待数据库, 超时仍然如实抛错,不会假装启动成功。NAS 重启时应用和数据库一起起来, 这个竞争是常态而不是意外。 Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ MariaDB automatically. Upserts must use backend-specific SQL (see services).
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
import threading
|
||||
import queue
|
||||
import datetime
|
||||
@@ -511,8 +512,35 @@ def _statements(schema):
|
||||
yield stmt
|
||||
|
||||
|
||||
# How long to keep waiting for the database at startup.
|
||||
#
|
||||
# On a NAS reboot the app and MariaDB come up together and the app usually
|
||||
# wins the race. Without this it raised, gunicorn reported "Worker failed to
|
||||
# boot", the master shut down — and when MariaDB appeared seconds later there
|
||||
# was nothing left running to notice. The site stayed down until someone
|
||||
# restarted it by hand.
|
||||
INIT_RETRY_SECONDS = int(os.environ.get("DB_INIT_RETRY_SECONDS") or 120)
|
||||
INIT_RETRY_INTERVAL = 3
|
||||
|
||||
|
||||
def init_db():
|
||||
conn = _connect()
|
||||
deadline = time.monotonic() + INIT_RETRY_SECONDS
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
try:
|
||||
conn = _connect()
|
||||
break
|
||||
except Exception as e: # noqa: BLE001 - any connection failure is worth retrying
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
if attempt == 1:
|
||||
print(f"[db] 数据库还没准备好,重试中:{e}")
|
||||
time.sleep(INIT_RETRY_INTERVAL)
|
||||
|
||||
if attempt > 1:
|
||||
print(f"[db] 第 {attempt} 次尝试后连上数据库")
|
||||
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
for stmt in _statements(SCHEMA):
|
||||
|
||||
Reference in New Issue
Block a user