mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-05-13 04:03:41 +00:00
94da8f67d7
`make dev` ran `uv sync` unconditionally on every restart, wiping any
optional extras the user had installed manually with
`uv sync --all-packages --extra postgres`. The Docker image-build path
already solved this via the `UV_EXTRAS` build-arg in backend/Dockerfile;
the local serve.sh path and the docker-compose-dev startup command
were the remaining outliers.
`scripts/serve.sh` now resolves extras before `uv sync`:
1. honors `UV_EXTRAS` (parity with backend/Dockerfile and
docker/docker-compose.yaml — no new convention introduced);
2. falls back to parsing config.yaml — `database.backend: postgres`
or legacy `checkpointer.type: postgres` auto-pins
`--extra postgres`, so the common case needs zero extra config.
3. detector stderr is no longer suppressed, so whitelist warnings or
crashes surface to the dev terminal (review feedback).
Detection lives in `scripts/detect_uv_extras.py` (stdlib-only — has to
run before the venv exists). Extra names are validated against
`^[A-Za-z][A-Za-z0-9_-]*$` so a stray shell metacharacter in `.env`
cannot reach `uv sync` downstream (defense in depth).
`docker/docker-compose-dev.yaml`'s startup command is now extracted to
`docker/dev-entrypoint.sh` (review feedback — the inline command had
grown to a ~350-char one-liner). The script:
- parses comma/whitespace-separated UV_EXTRAS, applying the same
`^[A-Za-z][A-Za-z0-9_-]*$` whitelist as the local detector;
- emits one `--extra X` flag per token, so `UV_EXTRAS=postgres,ollama`
works in Docker dev too (harmonized with local — review feedback);
- calls `uv sync --all-packages` (PR #2584) so workspace member
extras (deerflow-harness's postgres extra) are installed;
- keeps the existing self-heal `(uv sync || (recreate venv && retry))`
branch;
- exposes `--print-extras` for dry-run testing.
The compose file mounts the script read-only at runtime, so script
edits take effect on `make docker-restart` without an image rebuild.
The `--no-sync` alternative (a separate suggestion in the issue thread)
was considered but rejected for dev paths because it would drop the
self-heal branch and the auto-pickup of new pyproject deps. `--no-sync`
is already in use for the production CMD (`backend/Dockerfile:101`)
where it's appropriate.
Updates the asyncpg-missing error message to include the
`--all-packages` flag (matching #2584) plus the persistent install flow,
and expands `config.example.yaml` so all three install paths
(local / docker dev / docker image build) are documented with their
multi-extra capabilities.
Tests:
- `tests/test_detect_uv_extras.py` (21 tests) — local-path env parsing,
YAML edge cases, env-vs-config precedence, whitelist rejection of
shell metacharacters.
- `tests/test_dev_entrypoint.py` (15 tests) — docker-path validation
via `--print-extras`, multi-extra parsing, metacharacter abort.
- `tests/test_persistence_scaffold.py` (22 tests, unchanged) — passes
with the merged `--all-packages --extra postgres` error message.
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
103 lines
3.1 KiB
Python
103 lines
3.1 KiB
Python
"""Unit tests for docker/dev-entrypoint.sh (UV_EXTRAS validation + parsing).
|
|
|
|
Exercises the script via its `--print-extras` dry-run hook so we don't actually
|
|
launch uvicorn or hit /app/logs. Together with test_detect_uv_extras.py these
|
|
cover both the local make-dev path and the docker-compose-dev path with the
|
|
same shape — see PR #2767 / Issue #2754.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
ENTRYPOINT = REPO_ROOT / "docker" / "dev-entrypoint.sh"
|
|
|
|
|
|
def _run(uv_extras: str | None) -> subprocess.CompletedProcess[str]:
|
|
"""Invoke `dev-entrypoint.sh --print-extras` with UV_EXTRAS set."""
|
|
env = os.environ.copy()
|
|
env.pop("UV_EXTRAS", None)
|
|
if uv_extras is not None:
|
|
env["UV_EXTRAS"] = uv_extras
|
|
return subprocess.run(
|
|
["sh", str(ENTRYPOINT), "--print-extras"],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def test_entrypoint_script_exists_and_is_posix_sh():
|
|
assert ENTRYPOINT.is_file()
|
|
# Catch syntax errors before runtime — `sh -n` is a parse-only check.
|
|
proc = subprocess.run(["sh", "-n", str(ENTRYPOINT)], capture_output=True, text=True, check=False)
|
|
assert proc.returncode == 0, proc.stderr
|
|
|
|
|
|
def test_no_uv_extras_yields_empty_flags():
|
|
proc = _run(None)
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == ""
|
|
|
|
|
|
def test_single_extra():
|
|
proc = _run("postgres")
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == "--extra postgres"
|
|
|
|
|
|
def test_multi_extra_comma_separated():
|
|
proc = _run("postgres,ollama")
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == "--extra postgres --extra ollama"
|
|
|
|
|
|
def test_multi_extra_whitespace_separated():
|
|
proc = _run("postgres ollama")
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == "--extra postgres --extra ollama"
|
|
|
|
|
|
def test_multi_extra_mixed_separators():
|
|
proc = _run(" postgres , ollama ,")
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == "--extra postgres --extra ollama"
|
|
|
|
|
|
def test_empty_string_yields_empty_flags():
|
|
proc = _run("")
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == ""
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"bad_value",
|
|
[
|
|
"; rm -rf /", # the canonical injection attempt
|
|
"$(whoami)", # command substitution
|
|
"`echo bad`", # backticks
|
|
"postgres;evil", # mixed legal+illegal in a single token
|
|
"1postgres", # leading digit
|
|
"-postgres", # leading hyphen
|
|
"post gres extra/path", # contains slash
|
|
],
|
|
)
|
|
def test_metacharacters_abort_with_nonzero_exit(bad_value):
|
|
proc = _run(bad_value)
|
|
assert proc.returncode != 0, f"expected abort for {bad_value!r}, got 0"
|
|
assert "is invalid" in proc.stderr
|
|
assert proc.stdout.strip() == ""
|
|
|
|
|
|
def test_underscores_and_hyphens_in_name_are_allowed():
|
|
"""Mirrors uv's accepted shape for `[project.optional-dependencies]` keys."""
|
|
proc = _run("post_gres,post-gres")
|
|
assert proc.returncode == 0
|
|
assert proc.stdout.strip() == "--extra post_gres --extra post-gres"
|