From db06b3966bb3fcf7b32b31a1ee09c42c50935063 Mon Sep 17 00:00:00 2001 From: haochen <3033940900@qq.com> Date: Thu, 8 Oct 2026 20:47:59 +0800 Subject: [PATCH] fix(support-bundle): locate threads in configured runtime home (#6486) * fix(support-bundle): locate threads in configured runtime home * chore: preserve upstream LF line endings * fix(support-bundle): align path settings with launcher and dotenv * fix(support-bundle): match launcher runtime path settings * fix(support-bundle): expand launcher runtime path variables safely --------- Co-authored-by: Zhangchenhao-111 <285911435+Zhangchenhao-111@users.noreply.github.com> Co-authored-by: Willem Jiang --- README.md | 15 ++ backend/tests/test_support_bundle.py | 283 +++++++++++++++++++++++++++ scripts/AGENTS.md | 23 +++ scripts/support_bundle.py | 83 +++++++- 4 files changed, 397 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0fb635fea8..b89c200cd4 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,24 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta only, and does not include `.env`, raw conversation messages, or user file contents. Subprocess diagnostics are captured as UTF-8, with Python helpers emitting UTF-8 even on non-UTF-8 hosts and escaping unencodable characters. + Thread manifests follow the local launcher's runtime paths: checkout `.env` + values override shell exports, and a project-root override alone still uses + `backend/.deer-flow` first. Simple variable references such as + `DEER_FLOW_HOME="$PWD/backend/.deer-flow"` use the checkout as `PWD`; + single-quoted references remain literal. For standalone Gateway launches using `backend/.env` + or `DEER_FLOW_ENV_FILE`, export the effective `DEER_FLOW_HOME` when collecting + the bundle and ensure the checkout `.env` does not override it. Doctor's internal tool probes also decode UTF-8 with replacement for invalid bytes so the remaining diagnostic output stays available. + When a thread manifest is requested, a nonempty `DEER_FLOW_HOME` selects the + runtime data directory instead of stale checkout data. External data paths + appear as `{DEER_FLOW_HOME}` in the report; file contents remain excluded. + Runtime path settings also come from the checkout `.env`, with shell exports + taking precedence. Relative paths resolve from the checkout as in `make dev`. + Without an explicit home, a configured `DEER_FLOW_PROJECT_ROOT` selects its + `.deer-flow` directory; external paths use that variable name in the report. + > **Advanced / manual configuration**: If you prefer to edit `config.yaml` directly, run `make config` instead to copy the full template. Optional dependency auto-detection accepts UTF-8 configuration files with or without a byte-order mark (BOM). See `config.example.yaml` for the complete reference including CLI-backed providers (Codex CLI, Claude Code OAuth), OpenRouter, Responses API, subagent runtime caps such as `subagents.max_total_per_run`, and more. Optional per-model pricing must use one currency across all priced models. diff --git a/backend/tests/test_support_bundle.py b/backend/tests/test_support_bundle.py index 8938254b6a..7184058175 100644 --- a/backend/tests/test_support_bundle.py +++ b/backend/tests/test_support_bundle.py @@ -4,18 +4,73 @@ from __future__ import annotations import json import os +import shutil +import subprocess import sys import zipfile +from pathlib import Path import pytest import support_bundle +@pytest.fixture(autouse=True) +def _isolate_runtime_paths(monkeypatch): + for name in ("DEER_FLOW_HOME", "DEER_FLOW_PROJECT_ROOT", "DEER_FLOW_ENV_FILE"): + monkeypatch.delenv(name, raising=False) + + def _zip_text(zip_path, name: str) -> str: with zipfile.ZipFile(zip_path) as zf: return zf.read(name).decode("utf-8") +@pytest.mark.parametrize("layout", ["threads/thread-home", "users/alice/threads/thread-home"]) +def test_thread_summary_uses_configured_runtime_home(tmp_path, monkeypatch, layout): + project = tmp_path / "checkout" + project.mkdir() + home = tmp_path / "runtime-data" + outputs = home / layout / "user-data" / "outputs" + outputs.mkdir(parents=True) + (outputs / "report.txt").write_text("private document contents", encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_HOME", str(home)) + + summary = support_bundle.collect_thread_summary(project, "thread-home") + + assert summary["found"] is True + assert summary["outputs"][0]["path"] == "report.txt" + assert "private document contents" not in json.dumps(summary) + assert str(home) not in json.dumps(summary) + assert summary["layout"].startswith("{DEER_FLOW_HOME}/") + + +def test_configured_home_does_not_fall_back_to_stale_checkout_thread(tmp_path, monkeypatch): + project = tmp_path / "checkout" + (project / ".deer-flow" / "threads" / "thread-home" / "user-data").mkdir(parents=True) + home = tmp_path / "runtime-data" + monkeypatch.setenv("DEER_FLOW_HOME", str(home)) + + summary = support_bundle.collect_thread_summary(project, "thread-home") + + assert summary["found"] is False + assert summary["checked_layouts"] == ["{DEER_FLOW_HOME}/threads/thread-home/user-data"] + + +def test_relative_runtime_home_is_resolved_from_current_directory(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEER_FLOW_HOME", "runtime-data") + data = tmp_path / "runtime-data" / "users" / "alice" / "threads" / "thread-home" / "user-data" + data.mkdir(parents=True) + assert support_bundle.collect_thread_summary(tmp_path, "thread-home")["found"] is True + + +def test_empty_runtime_home_keeps_legacy_lookup(tmp_path, monkeypatch): + monkeypatch.setenv("DEER_FLOW_HOME", "") + data = tmp_path / "backend" / ".deer-flow" / "threads" / "thread-home" / "user-data" + data.mkdir(parents=True) + assert support_bundle.collect_thread_summary(tmp_path, "thread-home")["found"] is True + + @pytest.mark.parametrize("host_encoding", ["ascii", "cp936"]) @pytest.mark.parametrize("returncode", [0, 7]) def test_run_command_decodes_utf8_independently_of_host_locale(tmp_path, monkeypatch, host_encoding, returncode): @@ -841,3 +896,231 @@ def test_main_prints_reporter_next_steps_and_optional_upload(tmp_path, capsys): assert "Suggested next steps:" in captured.out assert "If an AI assistant files the issue, start from the issue draft" in captured.out assert "Attach the zip if a maintainer asks" in captured.out + + +@pytest.mark.parametrize("invocation_dir", [".", "backend"]) +def test_relative_home_matches_launcher_from_both_directories(tmp_path, monkeypatch, invocation_dir): + project = tmp_path / "checkout" + (project / "backend").mkdir(parents=True) + (project / ".deer-flow" / "users" / "alice" / "threads" / "relative" / "user-data").mkdir(parents=True) + monkeypatch.setenv("DEER_FLOW_HOME", ".deer-flow") + monkeypatch.chdir(project / invocation_dir) + assert support_bundle.collect_thread_summary(project, "relative")["found"] is True + + +def test_thread_home_from_dotenv_does_not_mutate_environment(tmp_path, monkeypatch): + project = tmp_path / "checkout" + project.mkdir() + home = tmp_path / "data" + (home / "users" / "alice" / "threads" / "dotenv" / "user-data").mkdir(parents=True) + (project / ".env").write_text(f'DEER_FLOW_HOME="{home.as_posix()}"\nPRIVATE_KEY=private-value\n', encoding="utf-8") + monkeypatch.delenv("DEER_FLOW_HOME", raising=False) + original = dict(os.environ) + summary = support_bundle.collect_thread_summary(project, "dotenv") + assert summary["found"] is True + assert dict(os.environ) == original + assert "private-value" not in json.dumps(summary) + assert str(home) not in json.dumps(summary) + + +def test_dotenv_home_wins_over_shell_without_exporting_secrets(tmp_path, monkeypatch): + (tmp_path / ".env").write_text("DEER_FLOW_HOME=dotenv-data\nPRIVATE_KEY=private-value\n", encoding="utf-8") + home = tmp_path / "shell-data" + (home / "threads" / "conflict" / "user-data").mkdir(parents=True) + (tmp_path / "dotenv-data" / "threads" / "conflict" / "user-data").mkdir(parents=True) + monkeypatch.setenv("DEER_FLOW_HOME", str(home)) + original = dict(os.environ) + summary = support_bundle.collect_thread_summary(tmp_path, "conflict") + assert summary["layout"] == "dotenv-data/threads/conflict/user-data" + assert dict(os.environ) == original + assert "private-value" not in json.dumps(summary) + + +@pytest.mark.parametrize("source", ["shell", "dotenv"]) +def test_thread_summary_honors_project_root_override(tmp_path, monkeypatch, source): + project = tmp_path / "checkout" + project.mkdir() + runtime_project = tmp_path / "other-project" + (runtime_project / ".deer-flow" / "threads" / "custom-root" / "user-data").mkdir(parents=True) + monkeypatch.delenv("DEER_FLOW_HOME", raising=False) + monkeypatch.delenv("DEER_FLOW_PROJECT_ROOT", raising=False) + if source == "shell": + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(runtime_project)) + else: + (project / ".env").write_text(f'DEER_FLOW_PROJECT_ROOT="{runtime_project.as_posix()}"\n', encoding="utf-8") + summary = support_bundle.collect_thread_summary(project, "custom-root") + assert summary["found"] is True + assert str(runtime_project) not in json.dumps(summary) + + +def _local_launcher_home(project): + """Execute the launcher's actual dotenv/default blocks without starting services.""" + bash = shutil.which("bash") + if bash is None: + pytest.skip("Bash is required to exercise the local launcher") + serve = (Path(support_bundle.__file__).parent / "serve.sh").read_text(encoding="utf-8") + dotenv_block = serve.split("# ── Load .env", 1)[1].split("\n\n", 1)[1].split("_pick_python()", 1)[0] + defaults = serve[serve.index("# Runtime path defaults.") :].split("# Extra flags", 1)[0] + result = subprocess.run( + [ + bash, + "-c", + 'set -e\ncd "$REPO_ROOT"\n' + dotenv_block + defaults + '\nprintf "%s" "$DEER_FLOW_HOME"', + ], + cwd=project, + env={**os.environ, "REPO_ROOT": str(project)}, + capture_output=True, + text=True, + encoding="utf-8", + check=True, + ) + return Path(result.stdout) + + +@pytest.mark.parametrize("invocation_dir", [".", "backend"]) +@pytest.mark.parametrize("source", ["shell", "dotenv"]) +def test_project_root_only_finds_launchers_default_home(tmp_path, monkeypatch, invocation_dir, source): + project = tmp_path / "checkout" + (project / "backend").mkdir(parents=True) + runtime_project = tmp_path / "custom-project" + # A stale standalone-harness thread must not shadow the local launcher. + (runtime_project / ".deer-flow" / "threads" / "launcher" / "user-data").mkdir(parents=True) + if source == "shell": + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(runtime_project)) + else: + (project / ".env").write_text(f'DEER_FLOW_PROJECT_ROOT="{runtime_project.as_posix()}"\n', encoding="utf-8") + home = _local_launcher_home(project) + (home / "users" / "alice" / "threads" / "launcher" / "user-data").mkdir(parents=True) + monkeypatch.chdir(project / invocation_dir) + + summary = support_bundle.collect_thread_summary(project, "launcher") + + assert home == project / "backend" / ".deer-flow" + assert summary["found"] is True + assert summary["layout"] == "backend/.deer-flow/users/alice/threads/launcher/user-data" + assert str(runtime_project) not in json.dumps(summary) + + +@pytest.mark.parametrize("value", ["~/runtime-data", "'~/runtime-data'", '"~/runtime-data"']) +@pytest.mark.parametrize("variable", ["DEER_FLOW_HOME", "DEER_FLOW_PROJECT_ROOT"]) +def test_dotenv_tilde_matches_launcher_quote_semantics(tmp_path, monkeypatch, value, variable): + project = tmp_path / "checkout" + project.mkdir() + user_home = tmp_path / "user-home" + user_home.mkdir() + monkeypatch.setenv("HOME", str(user_home)) + (project / ".env").write_text(f"{variable}={value}\n", encoding="utf-8") + if variable == "DEER_FLOW_HOME": + home = _local_launcher_home(project) + else: + home = (project / "~/runtime-data" if value.startswith(("'", '"')) else user_home / "runtime-data") / ".deer-flow" + (home / "threads" / "tilde" / "user-data").mkdir(parents=True) + original = dict(os.environ) + + summary = support_bundle.collect_thread_summary(project, "tilde") + + assert summary["found"] is True + assert dict(os.environ) == original + assert str(user_home) not in json.dumps(summary) + + +@pytest.mark.parametrize("dotenv_home", ["", "stale-data"]) +def test_dotenv_empty_home_also_overrides_shell(tmp_path, monkeypatch, dotenv_home): + project = tmp_path / "checkout" + project.mkdir() + (project / ".env").write_text(f"DEER_FLOW_HOME={dotenv_home}\n", encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "shell-data")) + home = _local_launcher_home(project) + (home / "threads" / "empty-home" / "user-data").mkdir(parents=True) + + summary = support_bundle.collect_thread_summary(project, "empty-home") + + assert summary["found"] is True + assert summary["layout"] == home.relative_to(project).as_posix() + "/threads/empty-home/user-data" + + +@pytest.mark.parametrize("source", ["shell", "legacy"]) +def test_missing_dotenv_keeps_support_bundle_available(tmp_path, monkeypatch, source): + (tmp_path / ".env").write_text("DEER_FLOW_HOME=unavailable-data\nPRIVATE_KEY=private-value\n", encoding="utf-8") + import builtins + + real_import = builtins.__import__ + + def without_dotenv(name, *args, **kwargs): + if name == "dotenv" or name.startswith("dotenv."): + raise ModuleNotFoundError("No module named dotenv") + return real_import(name, *args, **kwargs) + + if source == "shell": + home = tmp_path / "shell-data" + monkeypatch.setenv("DEER_FLOW_HOME", str(home)) + else: + home = tmp_path / "backend" / ".deer-flow" + (home / "threads" / "no-dotenv" / "user-data").mkdir(parents=True) + original = dict(os.environ) + with monkeypatch.context() as context: + context.setattr(builtins, "__import__", without_dotenv) + summary = support_bundle.collect_thread_summary(tmp_path, "no-dotenv") + + assert summary["found"] is True + assert dict(os.environ) == original + assert "private-value" not in json.dumps(summary) + + +@pytest.mark.parametrize("invocation_dir", [".", "backend"]) +@pytest.mark.parametrize( + ("setup", "value"), + [ + ("", "$PWD/backend/.deer-flow"), + ("", '"$PWD/backend/.deer-flow"'), + ("", '"${PWD}/backend/.deer-flow"'), + ("", "'$PWD/backend/.deer-flow'"), + ("", "'${PWD}/backend/.deer-flow'"), + ("", r'"\$PWD/backend/.deer-flow"'), + ('RUNTIME_DATA="$PWD/runtime-data"\n', '"$RUNTIME_DATA"'), + ('RUNTIME_DATA="$PWD/old"\nRUNTIME_DATA="$PWD/new"\n', '"${RUNTIME_DATA}"'), + ('RUNTIME_DATA="$PWD/first"\nRUNTIME_DATA="$RUNTIME_DATA/second"\n', '"$RUNTIME_DATA"'), + ("", '"${RUNTIME_DATA:-runtime-data}"'), + ], +) +def test_dotenv_variable_paths_match_launcher(tmp_path, monkeypatch, invocation_dir, setup, value): + project = tmp_path / "checkout" + (project / "backend").mkdir(parents=True) + monkeypatch.setenv("PWD", str(tmp_path / "other-cwd")) + monkeypatch.setenv("RUNTIME_DATA", "") + (project / ".env").write_text(setup + f"DEER_FLOW_HOME={value}\nPRIVATE_KEY=private-value\n", encoding="utf-8") + home = _local_launcher_home(project) + outputs = home / "users" / "alice" / "threads" / "variables" / "user-data" / "outputs" + outputs.mkdir(parents=True) + (outputs / "report.txt").write_text("private contents", encoding="utf-8") + monkeypatch.chdir(project / invocation_dir) + original = dict(os.environ) + + summary = support_bundle.collect_thread_summary(project, "variables") + + assert summary["found"] is True + assert summary["outputs"][0]["path"] == "report.txt" + assert dict(os.environ) == original + assert "private-value" not in json.dumps(summary) + assert "private contents" not in json.dumps(summary) + + +def test_dotenv_shell_variable_project_root_is_resolved(tmp_path, monkeypatch): + project = tmp_path / "checkout" + project.mkdir() + (project / ".env").write_text('DEER_FLOW_PROJECT_ROOT="$PWD/standalone"\n', encoding="utf-8") + home = project / "standalone" / ".deer-flow" + (home / "threads" / "variables" / "user-data").mkdir(parents=True) + monkeypatch.setenv("PWD", str(tmp_path / "other-cwd")) + + assert support_bundle.collect_thread_summary(project, "variables")["found"] is True + + +def test_dotenv_path_lookup_does_not_execute_shell_commands(tmp_path): + marker = tmp_path / "shell-executed" + (tmp_path / ".env").write_text(f'DEER_FLOW_HOME="$(touch {marker.as_posix()})"\n', encoding="utf-8") + + summary = support_bundle.collect_thread_summary(tmp_path, "variables") + + assert summary["found"] is False + assert not marker.exists() diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 298c2b5240..82d80e6560 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -1,3 +1,26 @@ +## Support Bundle Runtime Home + +Thread manifests use a nonempty `DEER_FLOW_HOME` exclusively, resolving relative +values from the checkout like the local launcher. Read root `.env` path settings +without exporting secrets; dotenv overrides shell exports, including empty values. +Expand unquoted leading tildes, preserving quoted literals. If python-dotenv is +unavailable, retain shell/legacy lookup so troubleshooting remains usable. +Resolve `$NAME`, `${NAME}` and `${NAME:-literal}` in unquoted/double-quoted +values using a private environment with checkout `PWD` and earlier dotenv +assignments. Single-quoted values and escaped dollars stay literal. Parse the +file without sourcing it or executing command substitutions. +When home is unset and `DEER_FLOW_PROJECT_ROOT` is configured, search the launcher's +`backend/.deer-flow` first, then the standalone harness project root. Scan both +legacy threads and user-scoped threads in that root. With no override, retain +the two checkout layouts. Display an external home as `{DEER_FLOW_HOME}` rather +than its absolute host path, and never include file contents in the manifest. +Coverage lives in `backend/tests/test_support_bundle.py`. +This lookup follows the local launcher's root `.env`; it does not discover +`backend/.env` or `DEER_FLOW_ENV_FILE` used by standalone Gateway launches. +For those launches, export the effective `DEER_FLOW_HOME` when collecting a bundle +and ensure the checkout `.env` does not override it. Tests clear all three runtime +path variables and compare storage defaults with the launcher's actual shell blocks. + ## Dependency Check Diagnostics `check.py` captures tool output as UTF-8 with replacement for malformed bytes, diff --git a/scripts/support_bundle.py b/scripts/support_bundle.py index c70567c93c..4aaac50bfe 100644 --- a/scripts/support_bundle.py +++ b/scripts/support_bundle.py @@ -262,13 +262,75 @@ def _validate_thread_id(thread_id: str) -> None: raise ValueError(f"Invalid thread_id: {thread_id!r}") +def _expand_launcher_variables(value: str, environment: dict[str, str]) -> str: + """Expand simple shell references in one pass, without executing shell syntax.""" + + def replace(match: re.Match[str]) -> str: + if match[0].startswith("\\"): + return match[0][1:] + name = match[1] or match[3] + resolved = environment.get(name, "") + return match[2] if match[2] is not None and not resolved else resolved + + return re.sub(r"\\?\$(?:\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}|([A-Za-z_][A-Za-z0-9_]*))", replace, value) + + +def _thread_data_roots(project_root: Path) -> list[tuple[Path, str]]: + """Resolve launcher-relative runtime paths without exporting .env secrets.""" + env_file = project_root / ".env" + names = ("DEER_FLOW_HOME", "DEER_FLOW_PROJECT_ROOT") + values = {name: os.environ.get(name) for name in names} + if env_file.is_file(): + try: + from dotenv.parser import parse_stream + except ImportError: + # Diagnostics must still work in an incomplete backend environment. + pass + else: + # serve.sh changes to the checkout before sourcing .env. Keep its + # assignment order in a private mapping; never export parsed secrets. + environment = {**os.environ, "PWD": str(project_root.resolve())} + with env_file.open(encoding="utf-8-sig") as stream: + for binding in parse_stream(stream): + if not binding.key or binding.value is None or binding.error: + continue + quote = binding.original.string.split("=", 1)[-1].lstrip()[:1] + value = binding.value + if quote != "'": + if quote != '"': + value = os.path.expanduser(value) + value = _expand_launcher_variables(value, environment) + environment[binding.key] = value + if binding.key in names: + # Even an empty .env assignment overrides inherited exports. + values[binding.key] = value + home = values["DEER_FLOW_HOME"] + runtime_project = values["DEER_FLOW_PROJECT_ROOT"] + if home: + return [((project_root / home).resolve(), "{DEER_FLOW_HOME}")] + if runtime_project: + # The local launcher pins this home independently of the project root. + # Keep the standalone harness location as a second candidate. + return [ + (project_root / "backend" / ".deer-flow", ""), + ( + (project_root / runtime_project).resolve() / ".deer-flow", + "{DEER_FLOW_PROJECT_ROOT}/.deer-flow", + ), + ] + return [ + (project_root / ".deer-flow", ""), + (project_root / "backend" / ".deer-flow", ""), + ] + + def _candidate_thread_data_dirs(project_root: Path, thread_id: str) -> list[Path]: _validate_thread_id(thread_id) - candidates = [ - project_root / ".deer-flow" / "threads" / thread_id / "user-data", - project_root / "backend" / ".deer-flow" / "threads" / thread_id / "user-data", - ] - for base in (project_root / ".deer-flow" / "users", project_root / "backend" / ".deer-flow" / "users"): + roots = _thread_data_roots(project_root) + candidates = [] + for root, _ in roots: + candidates.append(root / "threads" / thread_id / "user-data") + base = root / "users" if base.exists(): candidates.extend(user_dir / "threads" / thread_id / "user-data" for user_dir in base.iterdir() if user_dir.is_dir()) return candidates @@ -278,6 +340,12 @@ def _display_path(path: Path, project_root: Path) -> str: try: return path.resolve().relative_to(project_root.resolve()).as_posix() except (OSError, ValueError): + for root, marker in _thread_data_roots(project_root): + if marker: + try: + return marker + "/" + path.resolve().relative_to(root.resolve()).as_posix() + except (OSError, ValueError): + pass return redact_text(path.as_posix()) @@ -311,7 +379,8 @@ def _file_manifest(root: Path, *, max_files: int = 500) -> list[dict[str, Any]]: def collect_thread_summary(project_root: Path, thread_id: str) -> dict[str, Any]: """Collect a thread file manifest without reading user file contents.""" - for data_dir in _candidate_thread_data_dirs(project_root, thread_id): + candidates = _candidate_thread_data_dirs(project_root, thread_id) + for data_dir in candidates: if data_dir.exists(): return { "thread_id": thread_id, @@ -324,7 +393,7 @@ def collect_thread_summary(project_root: Path, thread_id: str) -> dict[str, Any] return { "thread_id": thread_id, "found": False, - "checked_layouts": [_display_path(path, project_root) for path in _candidate_thread_data_dirs(project_root, thread_id)], + "checked_layouts": [_display_path(path, project_root) for path in candidates], }