diff --git a/README.md b/README.md index 965e230f9c..4951832e5a 100644 --- a/README.md +++ b/README.md @@ -802,7 +802,15 @@ error even when the referenced sandbox is cached. When host Bash is enabled for Local Execution, DeerFlow starts OS detection with `uname -s`, then uses `sw_vers` on Darwin. On Linux, it reads host system files such as `/etc/os-release` only when the active sandbox policy permits it. Host filesystem path checks still apply; after a blocked path, the agent is directed to use a permitted command-only probe or virtual path instead of repeating the rejected command. -For Docker development, service startup follows `config.yaml` sandbox mode. In Local/Docker modes, `provisioner` is not started. +For Docker development and production, service startup follows the configured +sandbox mode. With `AioSandboxProvider`, an omitted `provisioner_url`, an empty +string (`""` or `''`), or YAML null (`null`, `~`, or a blank value), including +trailing comments, selects `aio`: `provisioner` is not started and the Docker +socket overlay is enabled. This grants the gateway host root-equivalent Docker +access; see the [sandbox security notes](backend/docs/CONFIGURATION.md#sandbox-isolation-and-the-docker-socket-dood). +A nonempty URL selects provisioner mode without that overlay. Quoted `"null"` +and `"~"` are nonempty strings, not YAML nulls; mode detection does not validate +URLs. The default local provider never enables the socket overlay. Local AIO sandbox port allocation stops at TCP port 65535. If the remaining ports from the configured starting port are occupied, allocation reports diff --git a/backend/tests/test_docker_sandbox_mode_detection.py b/backend/tests/test_docker_sandbox_mode_detection.py index 0e9049d067..a8408197a3 100644 --- a/backend/tests/test_docker_sandbox_mode_detection.py +++ b/backend/tests/test_docker_sandbox_mode_detection.py @@ -3,6 +3,8 @@ from __future__ import annotations import os +import re +import shlex import shutil import subprocess import tempfile @@ -15,6 +17,195 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SCRIPT_PATH = REPO_ROOT / "scripts" / "docker.sh" BASH_EXECUTABLE = find_script_bash() +# These fixtures are YAML source text, not Python values: quoting must remain +# observable (e.g. null is empty, but "null" is a nonempty string). +PROVISIONER_VALUES = [ + pytest.param(None, "aio", id="omitted"), + pytest.param('""', "aio", id="double-quoted-empty"), + pytest.param("''", "aio", id="single-quoted-empty"), + pytest.param("null", "aio", id="null"), + pytest.param("Null", "aio", id="title-case-null"), + pytest.param("NULL", "aio", id="upper-case-null"), + pytest.param("~", "aio", id="tilde"), + pytest.param("", "aio", id="blank"), + pytest.param(" ", "aio", id="whitespace"), + pytest.param("# not configured", "aio", id="comment-only"), + pytest.param('"" # not configured', "aio", id="double-empty-comment"), + pytest.param("'' # not configured", "aio", id="single-empty-comment"), + pytest.param("null # not configured", "aio", id="null-comment"), + pytest.param("~ # not configured", "aio", id="tilde-comment"), + pytest.param("http://provisioner:8002", "provisioner", id="url"), + pytest.param('"http://provisioner:8002" # configured', "provisioner", id="quoted-url-comment"), + pytest.param('"null"', "provisioner", id="quoted-null"), + pytest.param("'null'", "provisioner", id="single-quoted-null"), + pytest.param('"~"', "provisioner", id="double-quoted-tilde"), + pytest.param("'~'", "provisioner", id="quoted-tilde"), + pytest.param('"null" # literal', "provisioner", id="quoted-null-comment"), + pytest.param('" # not a comment"', "provisioner", id="quoted-hash"), + pytest.param('" "', "provisioner", id="quoted-space"), + pytest.param("nullish", "provisioner", id="nonempty-string"), +] + + +def _sandbox_config(value: str | None, provider: str = "deerflow.community.aio_sandbox:AioSandboxProvider") -> str: + config = f"sandbox:\n use: {provider}\n" + return config if value is None else config + f" provisioner_url: {value}\n" + + +def _isolated_shell_env(tmp_path: Path) -> dict[str, str]: + # Do not inherit developer credentials, config selectors or shell startup + # hooks. Keep only the platform variables needed to find/run Git Bash. + env = {key: os.environ[key] for key in ("PATH", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT") if key in os.environ} + env.update(HOME=str(tmp_path), TMPDIR=str(tmp_path), TEMP=str(tmp_path), TMP=str(tmp_path)) + return env + + +def _run_detector(script_name: str, tmp_path: Path, config: str | None) -> str: + """Execute the real function, never deploy.sh's top-level setup.""" + config_path = tmp_path / "config.yaml" + if config is not None: + config_path.write_text(config, encoding="utf-8") + script = (REPO_ROOT / "scripts" / script_name).read_text(encoding="utf-8") + match = re.search(r"^detect_sandbox_mode\(\) \{\n.*?^\}", script, re.M | re.S) + assert match is not None, f"{script_name} has no detector" + command = f""" +set -eu +SCRIPT_DIR={shlex.quote(str(REPO_ROOT / "scripts"))} +REPO_ROOT={shlex.quote(str(REPO_ROOT))} +PROJECT_ROOT={shlex.quote(str(tmp_path))} +DEER_FLOW_CONFIG_PATH={shlex.quote(str(config_path))} +{match.group()} +detect_sandbox_mode +""" + result = subprocess.run([BASH_EXECUTABLE, "-c", command], cwd=tmp_path, env=_isolated_shell_env(tmp_path), capture_output=True, text=True, encoding="utf-8", timeout=10) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +@pytest.mark.parametrize("script_name", ["docker.sh", "deploy.sh"]) +@pytest.mark.parametrize(("value", "expected"), PROVISIONER_VALUES) +def test_provisioner_scalar_detection(script_name, value, expected, tmp_path): + assert _run_detector(script_name, tmp_path, _sandbox_config(value)) == expected + + +@pytest.mark.parametrize(("value", "expected"), PROVISIONER_VALUES) +def test_provisioner_helper_scalar_detection(value, expected, tmp_path): + config_path = tmp_path / "synthetic config.yaml" + config_path.write_text(_sandbox_config(value), encoding="utf-8") + result = subprocess.run( + [BASH_EXECUTABLE, str(REPO_ROOT / "scripts" / "sandbox-mode.sh"), str(config_path)], + cwd=tmp_path, + env=_isolated_shell_env(tmp_path), + capture_output=True, + text=True, + encoding="utf-8", + timeout=10, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + + +@pytest.mark.parametrize("script_name", ["docker.sh", "deploy.sh"]) +@pytest.mark.parametrize( + "config", + [ + None, + _sandbox_config("null", "deerflow.sandbox.local:LocalSandboxProvider"), + _sandbox_config("http://provisioner:8002", "deerflow.sandbox.local:LocalSandboxProvider"), + _sandbox_config("http://provisioner:8002", "custom:UnknownProvider"), + ], +) +def test_provisioner_detection_local_fallback(script_name, config, tmp_path): + assert _run_detector(script_name, tmp_path, config) == "local" + + +@pytest.mark.parametrize("script_name", ["docker.sh", "deploy.sh"]) +@pytest.mark.parametrize( + "config", + [ + _sandbox_config(None) + " # provisioner_url: http://ignored:8002\n", + _sandbox_config(None) + "other:\n provisioner_url: http://ignored:8002\n", + "other:\n provisioner_url: http://ignored:8002\n" + _sandbox_config("null"), + _sandbox_config("'' # empty\t").replace("\n", "\r\n"), + ], + ids=["commented-key", "following-section", "preceding-section", "crlf"], +) +def test_provisioner_detection_preserves_section_boundaries(script_name, config, tmp_path): + assert _run_detector(script_name, tmp_path, config) == "aio" + + +@pytest.mark.parametrize("script_name", ["docker.sh", "deploy.sh"]) +@pytest.mark.parametrize(("value", "expected"), [*PROVISIONER_VALUES, pytest.param(None, "local", id="local-provider")]) +def test_provisioner_start_selects_services_and_overlay(script_name, value, expected, tmp_path): + """Run both launchers in a synthetic checkout; all external commands are doubles.""" + checkout = tmp_path / "synthetic checkout" + shutil.copytree(REPO_ROOT / "scripts", checkout / "scripts") + docker_dir = checkout / "docker" + docker_dir.mkdir() + for filename in ("docker-compose-dev.yaml", "docker-compose.yaml", "docker-compose.dood.yaml"): + (docker_dir / filename).write_text("services: {}\n", encoding="utf-8") + (checkout / "frontend").mkdir() + for filename in (".env", "frontend/.env"): + (checkout / filename).write_text("# synthetic, no credentials\n", encoding="utf-8") + extensions = checkout / "extensions_config.json" + extensions.write_text("{}\n", encoding="utf-8") + config = _sandbox_config(value, "deerflow.sandbox.local:LocalSandboxProvider") if expected == "local" else _sandbox_config(value) + root_config = checkout / "config.yaml" + selected_config = checkout / "selected config.yaml" + # Dev must keep using PROJECT_ROOT/config.yaml; prod must honor its selector. + decoy = _sandbox_config("http://decoy:8002") if expected != "provisioner" else _sandbox_config(None) + root_config.write_text(config if script_name == "docker.sh" else decoy, encoding="utf-8") + selected_config.write_text(config if script_name == "deploy.sh" else decoy, encoding="utf-8") + capture = tmp_path / "compose-args" + shell_env = tmp_path / "stubs.sh" + shell_env.write_text( + """docker() { + if [ "$*" = 'compose version --short' ]; then echo '2.24.0'; return; fi + if [ "$*" = info ]; then return 0; fi + if [ "$1" = compose ]; then + printf '%s\\n' "$@" >> "$CAPTURE_COMPOSE_ARGS" + return 0 + fi + echo "Unexpected Docker call: $*" >&2; return 97 +} +# Exercise the existing Windows socket preflight without requiring a host socket. +uname() { echo MINGW64_NT; } +python() { return 97; } +python3() { return 97; } +uv() { return 97; } +pnpm() { return 97; } +kubectl() { return 97; } +openssl() { return 97; } +""", + encoding="utf-8", + ) + env = _isolated_shell_env(tmp_path) + env.update( + BASH_ENV=str(shell_env), + CAPTURE_COMPOSE_ARGS=str(capture), + DEER_FLOW_HOME=str(tmp_path / "runtime"), + DEER_FLOW_CONFIG_PATH=str(selected_config), + DEER_FLOW_EXTENSIONS_CONFIG_PATH=str(extensions), + BETTER_AUTH_SECRET="synthetic-test-secret", + DEER_FLOW_INTERNAL_AUTH_TOKEN="synthetic-test-token", + DEER_FLOW_CREDENTIALS_KEY="synthetic-test-key", + UV_EXTRAS="redis", + ) + result = subprocess.run([BASH_EXECUTABLE, str(checkout / "scripts" / script_name), "start"], cwd=tmp_path, env=env, capture_output=True, text=True, encoding="utf-8", timeout=15) + assert result.returncode == 0, result.stdout + result.stderr + args = capture.read_text(encoding="utf-8").splitlines() + assert args.count("up") == 1, args + assert ("provisioner" in args) == (expected == "provisioner"), args + files = [Path(args[index + 1]).name for index, arg in enumerate(args) if arg == "-f"] + base = "docker-compose-dev.yaml" if script_name == "docker.sh" else "docker-compose.yaml" + assert files == [base] + (["docker-compose.dood.yaml"] if expected == "aio" else []) + assert root_config.read_text(encoding="utf-8") == (config if script_name == "docker.sh" else decoy) + assert selected_config.read_text(encoding="utf-8") == (config if script_name == "deploy.sh" else decoy) + assert not any((tmp_path / "runtime").glob(".*secret*")) + assert not (tmp_path / "runtime" / ".internal-auth-token").exists() + assert not (tmp_path / "runtime" / ".credentials_key").exists() + + if BASH_EXECUTABLE is None: pytestmark = pytest.mark.skip(reason="Git Bash is required for docker.sh detection tests") diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 4ffef036a7..d56179eb73 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -46,6 +46,14 @@ trimming its contents. Offline CLI coverage: ## Service Startup Contracts +Both launchers call `sandbox-mode.sh` via Bash; keep it Bash/awk-only. +Dev passes `$PROJECT_ROOT/config.yaml`; prod passes `$DEER_FLOW_CONFIG_PATH`. +For AIO, omitted/blank/empty-quoted/YAML-null URLs (including comments) select +`aio`; preserve quoting so `"null"` and `"~"` remain nonempty strings. +This is not a general YAML parser or URL validator; aliases, multiline values +and environment expansion are out of scope. Offline helper and launcher tests: +`backend/tests/test_docker_sandbox_mode_detection.py`. + The setup wizard offers Webz.io as a news-only `web_search` provider using `deerflow.community.webz.tools:web_search_tool` and `WEBZ_API_KEY`. Keep its entry aligned with the credential check in `doctor.py` and the example config. diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d029142279..322a815046 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -386,36 +386,7 @@ fi # ── detect_sandbox_mode ─────────────────────────────────────────────────────── detect_sandbox_mode() { - local sandbox_use="" - local provisioner_url="" - - [ -f "$DEER_FLOW_CONFIG_PATH" ] || { echo "local"; return; } - - sandbox_use=$(awk ' - /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } - in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } - in_sandbox && /^[[:space:]]*use:[[:space:]]*/ { - line=$0; sub(/^[[:space:]]*use:[[:space:]]*/, "", line); print line; exit - } - ' "$DEER_FLOW_CONFIG_PATH") - - provisioner_url=$(awk ' - /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } - in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } - in_sandbox && /^[[:space:]]*provisioner_url:[[:space:]]*/ { - line=$0; sub(/^[[:space:]]*provisioner_url:[[:space:]]*/, "", line); print line; exit - } - ' "$DEER_FLOW_CONFIG_PATH") - - if [[ "$sandbox_use" == *"deerflow.community.aio_sandbox:AioSandboxProvider"* ]]; then - if [ -n "$provisioner_url" ]; then - echo "provisioner" - else - echo "aio" - fi - else - echo "local" - fi + bash "$REPO_ROOT/scripts/sandbox-mode.sh" "$DEER_FLOW_CONFIG_PATH" } # ── down ────────────────────────────────────────────────────────────────────── diff --git a/scripts/docker.sh b/scripts/docker.sh index 02a155a1db..dee4c3939b 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -181,48 +181,7 @@ load_proxy_env_from_dotenv() { } detect_sandbox_mode() { - local config_file="$PROJECT_ROOT/config.yaml" - local sandbox_use="" - local provisioner_url="" - - if [ ! -f "$config_file" ]; then - echo "local" - return - fi - - sandbox_use=$(awk ' - /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } - in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } - in_sandbox && /^[[:space:]]*use:[[:space:]]*/ { - line=$0 - sub(/^[[:space:]]*use:[[:space:]]*/, "", line) - print line - exit - } - ' "$config_file") - - provisioner_url=$(awk ' - /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } - in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } - in_sandbox && /^[[:space:]]*provisioner_url:[[:space:]]*/ { - line=$0 - sub(/^[[:space:]]*provisioner_url:[[:space:]]*/, "", line) - print line - exit - } - ' "$config_file") - - if [[ "$sandbox_use" == *"deerflow.sandbox.local:LocalSandboxProvider"* ]]; then - echo "local" - elif [[ "$sandbox_use" == *"deerflow.community.aio_sandbox:AioSandboxProvider"* ]]; then - if [ -n "$provisioner_url" ]; then - echo "provisioner" - else - echo "aio" - fi - else - echo "local" - fi + bash "$SCRIPT_DIR/sandbox-mode.sh" "$PROJECT_ROOT/config.yaml" } # Cleanup function for Ctrl+C diff --git a/scripts/sandbox-mode.sh b/scripts/sandbox-mode.sh new file mode 100755 index 0000000000..30694c75b6 --- /dev/null +++ b/scripts/sandbox-mode.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Shared, read-only mode detection for the Docker dev and production launchers. +# Callers own config-path resolution. Use only Bash/awk: Docker-only hosts do +# not need Python or backend dependencies installed. +set -e + +config_file="$1" +if [ ! -f "$config_file" ]; then + echo "local" + exit 0 +fi + +# Keep the existing block-mapping lookup, not a general YAML parser. Recognize +# empty single-line scalars before discarding quote information: quoted "null" +# and "~" are nonempty strings. URL validation, aliases, multiline scalars and +# environment-variable expansion remain outside this lightweight detector. +awk ' + function empty_scalar(value) { + sub(/[[:space:]]+$/, "", value) + return value == "" || value ~ /^#/ || value ~ /^(null|Null|NULL|~|""|\047\047)([[:space:]]+#.*)?$/ + } + /^[[:space:]]*sandbox:[[:space:]]*$/ { in_sandbox=1; next } + in_sandbox && /^[^[:space:]#]/ { in_sandbox=0 } + in_sandbox && !seen_use && /^[[:space:]]*use:[[:space:]]*/ { + sandbox_use=$0 + sub(/^[[:space:]]*use:[[:space:]]*/, "", sandbox_use) + seen_use=1 + } + in_sandbox && !seen_url && /^[[:space:]]*provisioner_url:[[:space:]]*/ { + provisioner_url=$0 + sub(/^[[:space:]]*provisioner_url:[[:space:]]*/, "", provisioner_url) + seen_url=1 + } + END { + if (index(sandbox_use, "deerflow.sandbox.local:LocalSandboxProvider")) { + print "local" + } else if (index(sandbox_use, "deerflow.community.aio_sandbox:AioSandboxProvider")) { + print empty_scalar(provisioner_url) ? "aio" : "provisioner" + } else { + print "local" + } + } +' "$config_file"