fix(db): 数据库晚一步启动就会让服务永久挂掉
今天 11:33 站点整个不可用。不是代码的问题,是启动顺序: NAS 上 MariaDB 在 11:34 才起来,而应用 11:33 就尝试连接, init_db() 抛异常 → gunicorn 报 Worker failed to boot → master 退出。 一分钟后数据库好了,但已经没有进程在跑,没人会去重试—— 站点就一直躺到有人手工重启为止。 init_db 改为在 DB_INIT_RETRY_SECONDS(默认 120 秒)内重试等待数据库, 超时仍然如实抛错,不会假装启动成功。NAS 重启时应用和数据库一起起来, 这个竞争是常态而不是意外。 Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ MariaDB automatically. Upserts must use backend-specific SQL (see services).
|
|||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
import time
|
||||||
import threading
|
import threading
|
||||||
import queue
|
import queue
|
||||||
import datetime
|
import datetime
|
||||||
@@ -511,8 +512,35 @@ def _statements(schema):
|
|||||||
yield stmt
|
yield stmt
|
||||||
|
|
||||||
|
|
||||||
|
# How long to keep waiting for the database at startup.
|
||||||
|
#
|
||||||
|
# On a NAS reboot the app and MariaDB come up together and the app usually
|
||||||
|
# wins the race. Without this it raised, gunicorn reported "Worker failed to
|
||||||
|
# boot", the master shut down — and when MariaDB appeared seconds later there
|
||||||
|
# was nothing left running to notice. The site stayed down until someone
|
||||||
|
# restarted it by hand.
|
||||||
|
INIT_RETRY_SECONDS = int(os.environ.get("DB_INIT_RETRY_SECONDS") or 120)
|
||||||
|
INIT_RETRY_INTERVAL = 3
|
||||||
|
|
||||||
|
|
||||||
def init_db():
|
def init_db():
|
||||||
|
deadline = time.monotonic() + INIT_RETRY_SECONDS
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
|
attempt += 1
|
||||||
|
try:
|
||||||
conn = _connect()
|
conn = _connect()
|
||||||
|
break
|
||||||
|
except Exception as e: # noqa: BLE001 - any connection failure is worth retrying
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise
|
||||||
|
if attempt == 1:
|
||||||
|
print(f"[db] 数据库还没准备好,重试中:{e}")
|
||||||
|
time.sleep(INIT_RETRY_INTERVAL)
|
||||||
|
|
||||||
|
if attempt > 1:
|
||||||
|
print(f"[db] 第 {attempt} 次尝试后连上数据库")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
for stmt in _statements(SCHEMA):
|
for stmt in _statements(SCHEMA):
|
||||||
|
|||||||
Reference in New Issue
Block a user