From 28a06aa7779e5ed0ef0f78e67baf30ee69f3a98f Mon Sep 17 00:00:00 2001 From: ajayr Date: Wed, 14 Oct 2026 08:59:08 +0100 Subject: [PATCH] fix(deploy): exec the gateway server so SIGTERM reaches uvicorn (#6836) * fix(deploy): exec the gateway server so SIGTERM reaches uvicorn The production compose command, both CMDs in backend/Dockerfile and the Helm chart start the Gateway as `sh -c "cd backend && ... uv run ..."`. dash forks uv and stays PID 1 without a SIGTERM handler, so the kernel drops the stop signal: uvicorn never shuts down, the lifespan teardown (memory flush, run drain, service stops) never runs, and every stop waits out the grace period and ends in SIGKILL (exit 137). exec the server in every production launch path, as docker/dev-entrypoint.sh already does, and give the compose gateway `init: true` so orphaned grandchildren are still reaped once sh is no longer PID 1. A regression test pins exec in each launch path. Fixes #6835 Co-Authored-By: Claude Opus 5.5 * docs(changelog): link the gateway SIGTERM entry to #6836 Co-Authored-By: Claude Opus 5.5 * test(gateway): decouple dependency sync from exec launch form * docs: compact startup guidance to pass inherited size limits * docs: trim inherited startup guidance after main merge --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Willem Jiang --- AGENTS.md | 12 ++-- CHANGELOG.md | 12 ++++ backend/Dockerfile | 6 +- .../tests/test_gateway_sigterm_delivery.py | 68 +++++++++++++++++++ backend/tests/test_gateway_startup.py | 6 +- .../templates/gateway-deployment.yaml | 8 ++- docker/docker-compose.yaml | 10 ++- 7 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_gateway_sigterm_delivery.py diff --git a/AGENTS.md b/AGENTS.md index 423db758f0..911762d7e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -138,14 +138,12 @@ make up / down # Build/stop the production Docker stack (browser at localhost: make docker-start / docker-stop / docker-logs # Docker development environment ``` -Production startup uses the image's pre-built Python environment with `uv run ---no-sync`, gives the Gateway a real `/health` probe, and makes `make up` wait -for that probe before printing its success banner. A readiness failure must -surface Compose status and recent Gateway logs instead of claiming the stack is -running. +Production uses image Python (`uv run --no-sync`) and `exec` for SIGTERM; +Compose sets `init: true` (`backend/tests/test_gateway_sigterm_delivery.py`). +`make up` waits for Gateway `/health`; failures show Compose status and Gateway logs. -Docker log and restart commands resolve `DEER_FLOW_ROOT` from the current -checkout before invoking Compose, matching the start and stop commands. +Docker logs/restarts resolve `DEER_FLOW_ROOT` from this checkout before Compose, +as start/stop do. Run `make help` for the full list. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dab182319..9cf6f5198e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1247,6 +1247,17 @@ This release closes that milestone with **621 merged pull requests**. ### Fixed +- **deploy:** The Gateway container now shuts down gracefully. The production + compose `command`, both `CMD`s in `backend/Dockerfile` and the Helm chart + started it as `sh -c "cd backend && … uv run … uvicorn …"`, which leaves `sh` + as PID 1 with no SIGTERM handler, so the kernel dropped the stop signal: + uvicorn never shut down, the lifespan teardown (memory queue flush, run + drain, channel, scheduler and MCP shutdown) never ran, and every stop waited + out the grace period and ended in SIGKILL (exit 137). Every production launch + path now `exec`s the server, as `docker/dev-entrypoint.sh` already did, and + the compose gateway sets `init: true` so orphaned grandchildren are still + reaped. (issue #6835, [#6836]) + - **channels:** `FeishuChannel.stop()` and `DingTalkChannel.stop()` now end their SDK threads instead of timing out. Both SDKs run on a thread-private event loop that only returned on a fatal error, so every stop joined for @@ -11686,6 +11697,7 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#6781]: https://github.com/bytedance/deer-flow/pull/6781 [#6785]: https://github.com/bytedance/deer-flow/pull/6785 [#6798]: https://github.com/bytedance/deer-flow/pull/6798 +[#6836]: https://github.com/bytedance/deer-flow/pull/6836 [#6843]: https://github.com/bytedance/deer-flow/pull/6843 [#6844]: https://github.com/bytedance/deer-flow/pull/6844 [#6852]: https://github.com/bytedance/deer-flow/pull/6852 diff --git a/backend/Dockerfile b/backend/Dockerfile index b1b17595b4..3418c0dad4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -89,7 +89,8 @@ COPY --from=docker:cli /usr/local/bin/docker /usr/local/bin/docker EXPOSE 8001 -CMD ["sh", "-c", "cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] +# exec: uv must replace sh as PID 1 or `docker stop`'s SIGTERM never reaches uvicorn (#6835). +CMD ["sh", "-c", "cd backend && PYTHONPATH=. exec uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] # ── Stage 3: Runtime ────────────────────────────────────────────────────────── # Clean image without build-essential — reduces size (~200 MB) and attack surface. @@ -124,4 +125,5 @@ COPY --from=builder /app/backend ./backend EXPOSE 8001 # Default command (can be overridden in docker-compose) -CMD ["sh", "-c", "cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] +# exec: uv must replace sh as PID 1 or `docker stop`'s SIGTERM never reaches uvicorn (#6835). +CMD ["sh", "-c", "cd backend && PYTHONPATH=. exec uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] diff --git a/backend/tests/test_gateway_sigterm_delivery.py b/backend/tests/test_gateway_sigterm_delivery.py new file mode 100644 index 0000000000..4d9fdfd14e --- /dev/null +++ b/backend/tests/test_gateway_sigterm_delivery.py @@ -0,0 +1,68 @@ +"""The Gateway must receive the stop signal in every production launch path (#6835). + +Container runtimes signal PID 1. A ``sh -c "cd backend && … uv run …"`` command +forks uv and leaves the shell as PID 1 with no SIGTERM handler, so the kernel +drops the signal: uvicorn never runs its lifespan teardown and every stop ends +in SIGKILL after the grace period. Each production launch path must therefore +``exec`` the server, and the compose gateway must also run an init process: +``exec`` removes the orphan reaping the shell provided as a side effect. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMPOSE_PATH = REPO_ROOT / "docker" / "docker-compose.yaml" +DOCKERFILE_PATH = REPO_ROOT / "backend" / "Dockerfile" +HELM_GATEWAY_PATH = REPO_ROOT / "deploy" / "helm" / "deer-flow" / "templates" / "gateway-deployment.yaml" +DEV_ENTRYPOINT_PATH = REPO_ROOT / "docker" / "dev-entrypoint.sh" + +# ``exec`` makes uv replace the shell as PID 1, so the runtime's stop signal +# reaches uvicorn instead of being dropped. +EXEC_SERVER = "exec uv run --no-sync uvicorn app.gateway.app:app" + + +def _compose_gateway() -> dict: + compose = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8")) + return compose["services"]["gateway"] + + +def test_compose_gateway_execs_the_server() -> None: + """A forked server under ``sh -c`` never sees ``docker stop``'s SIGTERM (#6835).""" + command = _compose_gateway()["command"] + # ``command`` may load as a scalar string or a list depending on YAML style. + if isinstance(command, list): + command = " ".join(str(part) for part in command) + assert EXEC_SERVER in command, f"docker/docker-compose.yaml must exec the Gateway so SIGTERM reaches uvicorn (#6835); got: {command}" + + +def test_compose_gateway_runs_an_init_process() -> None: + """With the shell replaced, orphaned sandbox children need Docker's init to reap them (#6835).""" + init = _compose_gateway().get("init") + assert init is True, f"docker/docker-compose.yaml must set init: true so orphaned grandchildren are still reaped (#6835); got: {init!r}" + + +def test_dockerfile_cmds_exec_the_server() -> None: + """Both image stages are launch paths: a bare ``sh -c`` makes ``docker stop`` wait out the grace period (#6835).""" + lines = DOCKERFILE_PATH.read_text(encoding="utf-8").splitlines() + cmds = [line for line in lines if line.startswith("CMD") and "uvicorn app.gateway.app:app" in line] + assert len(cmds) == 2, f"backend/Dockerfile must keep one Gateway CMD per stage (#6835); got {len(cmds)}: {cmds}" + for line in cmds: + assert EXEC_SERVER in line, f"backend/Dockerfile CMD must exec the Gateway so SIGTERM reaches uvicorn (#6835); got: {line}" + + +def test_helm_gateway_args_exec_the_server() -> None: + """Without ``exec`` the kubelet's SIGTERM is dropped and the chart's graceful-shutdown budget never starts (#6835).""" + lines = HELM_GATEWAY_PATH.read_text(encoding="utf-8").splitlines() + launches = [line for line in lines if "uv run --no-sync uvicorn app.gateway.app:app" in line] + assert len(launches) == 1, f"expected exactly one Gateway launch in the Helm chart (#6835); got {len(launches)}: {launches}" + assert EXEC_SERVER in launches[0], f"deploy/helm/deer-flow/templates/gateway-deployment.yaml must exec the Gateway so the kubelet's SIGTERM reaches uvicorn (#6835); got: {launches[0]}" + + +def test_dev_entrypoint_execs_the_server() -> None: + """The dev entrypoint already execs; a regression reintroduces the dropped SIGTERM on every dev-container restart (#6835).""" + entrypoint = DEV_ENTRYPOINT_PATH.read_text(encoding="utf-8") + assert f"PYTHONPATH=. {EXEC_SERVER}" in entrypoint, "docker/dev-entrypoint.sh must exec the Gateway so the container's stop signal reaches uvicorn (#6835)" diff --git a/backend/tests/test_gateway_startup.py b/backend/tests/test_gateway_startup.py index 2e7dfdc5db..ca8d3bba64 100644 --- a/backend/tests/test_gateway_startup.py +++ b/backend/tests/test_gateway_startup.py @@ -49,8 +49,10 @@ def test_gateway_runtime_commands_never_sync_dependencies() -> None: compose = _read("docker/docker-compose.yaml") serve = _read("scripts/serve.sh") - assert "PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app" in compose - assert "PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app" in serve + # Dependency syncing is independent of the surrounding shell/exec form. + # Exact container launch forms belong to test_gateway_sigterm_delivery.py. + assert "uv run --no-sync uvicorn app.gateway.app:app" in compose + assert "uv run --no-sync uvicorn app.gateway.app:app" in serve def test_local_frontend_ignores_public_docker_port() -> None: diff --git a/deploy/helm/deer-flow/templates/gateway-deployment.yaml b/deploy/helm/deer-flow/templates/gateway-deployment.yaml index 43da44f65c..d0cb228ac1 100644 --- a/deploy/helm/deer-flow/templates/gateway-deployment.yaml +++ b/deploy/helm/deer-flow/templates/gateway-deployment.yaml @@ -101,8 +101,12 @@ spec: external provisioner's key is never touched and the gateway boots as it did before the key existed, failing sandbox calls at use time. With the bundled provisioner the key is injected below and - required, so no default is needed. */}} - - {{ if not .Values.provisioner.enabled }}export PROVISIONER_API_KEY="${PROVISIONER_API_KEY:-}" && {{ end }}cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers 1 --timeout-graceful-shutdown {{ .Values.gateway.uvicornGracefulShutdownSeconds | int }} + required, so no default is needed. + The command execs uv so it replaces sh as the container's PID 1: + a forking `sh -c` has no SIGTERM handler, so the kubelet's + SIGTERM would be dropped and the graceful-shutdown budget + below would never start (#6835). */}} + - {{ if not .Values.provisioner.enabled }}export PROVISIONER_API_KEY="${PROVISIONER_API_KEY:-}" && {{ end }}cd backend && PYTHONPATH=. exec uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers 1 --timeout-graceful-shutdown {{ .Values.gateway.uvicornGracefulShutdownSeconds | int }} env: - name: CI value: "true" diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 0fc74a41fa..b5ee1a56a0 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -132,10 +132,18 @@ services: # run drain and the 30s memory flush (memory.shutdown_flush_timeout_seconds) # -- about 71s -- plus slack. backend/tests/_gateway_shutdown_budget.py # pins the arithmetic. - command: sh -c "cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers ${GATEWAY_WORKERS:-1} --timeout-graceful-shutdown 10" + command: sh -c "cd backend && PYTHONPATH=. exec uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers ${GATEWAY_WORKERS:-1} --timeout-graceful-shutdown 10" # Docker's 10s default SIGKILLed the 30s memory queue flush # (memory.shutdown_flush_timeout_seconds) on every restart. stop_grace_period: 90s + # `exec` in the command above lets the stop signal reach uvicorn: a `sh -c` + # that forks the server stays PID 1 with no SIGTERM handler, the kernel + # drops the signal, and the lifespan shutdown never runs; every stop then + # waits out stop_grace_period and ends in SIGKILL (#6835). `init` runs + # Docker's tini as PID 1: it forwards the signal to uv, which forwards it + # to uvicorn, and it reaps orphaned grandchildren, which `sh` used to do + # as a side effect of waiting. + init: true volumes: - ${DEER_FLOW_CONFIG_PATH}:/app/backend/config.yaml:ro # Writable on purpose: the Gateway edits this file at runtime (MCP