fix(ai): 队列会把共用的网关打到 502,加并发上限和间隔
排查生产上一直生成不出来,发现网关在返 502。上去看:机器好好的、systemd 说 active、5100 端口在监听——但它是 `gunicorn -w 1 --threads 4`,全部并发 就四个,而且 fam-edge 和摄像头项目也在用同一个。 我们这边一个请求占一个线程 2~5 分钟,NVIDIA 链重试起来最坏十七分钟(它自己 README 已知问题 #3)。而我写的 worker 是跑完一个立刻拉下一个,同步后还有八 个 scope 排队——等于拿满线程不撒手。这个 502 大概率是我打出来的,而且顺带 把另外两个项目也打下线了。 - 并发按整个部署计算,不是每个 gunicorn worker 一个:claim 前先数全局 running(两个 worker 各跑「一个」就是两个并发) - 每跑完一个任务停 20 秒,不只是空闲时才停 - 两个都可用环境变量调,注释里写清楚调大的代价是什么 补齐的历史数据晚二十分钟到没有任何人受影响;网关不响应是三个项目一起受影响。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1073,3 +1073,35 @@ class TestHighlightsReadAlone:
|
||||
f"{name}: 折叠态只显示 detail,但它没提到 {title!r}: "
|
||||
f"{first['detail']!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestGatewayCourtesy:
|
||||
"""The gateway runs one worker with four threads and is shared with two
|
||||
other projects. This consumer must not be able to saturate it."""
|
||||
|
||||
def test_only_one_job_runs_at_a_time_across_the_deployment(self, db, user):
|
||||
jobs.enqueue(user["id"], "health", "a")
|
||||
jobs.enqueue(user["id"], "sleep", "b")
|
||||
assert jobs._claim_next() is not None
|
||||
assert jobs._claim_next() is None, \
|
||||
"a second Gunicorn worker must not start a second gateway call"
|
||||
|
||||
def test_a_finished_job_frees_the_slot(self, db, user):
|
||||
jobs.enqueue(user["id"], "health", "a")
|
||||
jobs.enqueue(user["id"], "sleep", "b")
|
||||
first = jobs._claim_next()
|
||||
jobs._finish(first["id"])
|
||||
assert jobs._claim_next() is not None
|
||||
|
||||
def test_an_abandoned_claim_does_not_block_the_queue_forever(self, db, user):
|
||||
jobs.enqueue(user["id"], "health", "a")
|
||||
jobs.enqueue(user["id"], "sleep", "b")
|
||||
jobs._claim_next()
|
||||
stale = jobs._now() - jobs.datetime.timedelta(
|
||||
seconds=jobs.CLAIM_TIMEOUT_SECONDS + 60)
|
||||
db.execute("UPDATE ai_jobs SET claimed_at = ?", [jobs._iso(stale)])
|
||||
assert jobs._claim_next() is not None
|
||||
|
||||
def test_there_is_a_gap_between_jobs(self):
|
||||
"""Back-to-back is what saturates a four-thread box."""
|
||||
assert jobs.GAP_SECONDS > 0
|
||||
|
||||
Reference in New Issue
Block a user