From 6b5f5e789a51fb0175b474ef7d5978013745a947 Mon Sep 17 00:00:00 2001 From: ShitK <80999801+ShitK@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:16:48 +0800 Subject: [PATCH] fix(tests): require explicit opt-in for live client tests (#4482) * fix(tests): require explicit opt-in for live client tests * test: align live target with marker --- CONTRIBUTING.md | 11 +- README.md | 7 + backend/AGENTS.md | 23 +++- backend/Makefile | 5 +- backend/README.md | 12 +- backend/pyproject.toml | 1 + backend/tests/test_client_live.py | 19 ++- backend/tests/test_client_live_policy.py | 161 +++++++++++++++++++++++ 8 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 backend/tests/test_client_live_policy.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07442c630..9501df45c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -305,10 +305,14 @@ before review. ## Testing ```bash -# Backend tests +# Backend tests (offline by default; excludes live external-API tests) cd backend make test +# Live DeerFlowClient integration tests (explicit opt-in) +# Requires a valid root config.yaml and API credentials. +make test-live + # Frontend unit tests cd frontend make test @@ -318,6 +322,11 @@ cd frontend make test-e2e ``` +`make test-live` calls real external APIs and may incur API costs or create +local sandboxes, artifacts, and files. It is never run by the default backend +test command or CI. Direct pytest invocations of `tests/test_client_live.py` +must also set `DEER_FLOW_RUN_LIVE_TESTS=1`. + ### PR Regression Checks Every pull request triggers the following CI workflows: diff --git a/README.md b/README.md index 79f20cb13..e968daeee 100644 --- a/README.md +++ b/README.md @@ -1126,6 +1126,13 @@ DeerFlow has key high-privilege capabilities including **system command executio We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, workflow, and guidelines. +Backend `make test` is offline by default and excludes live external-API +coverage. Maintainers can explicitly run the real `DeerFlowClient` integration +suite with `cd backend && make test-live` after providing a valid root +`config.yaml` and API credentials; this may incur API costs and create local +sandboxes, artifacts, or files. Direct pytest runs additionally require +`DEER_FLOW_RUN_LIVE_TESTS=1`. + Regression coverage includes Docker sandbox mode detection and provisioner kubeconfig-path handling tests in `backend/tests/`. Backend blocking-IO diagnostics are available from the repository root with `make detect-blocking-io`: it statically scans backend business code for diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 4bc61f7dc..94387a82c 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -96,7 +96,8 @@ make stop # Stop all services make install # Install backend dependencies make dev # Run Gateway API with reload (port 8001) make gateway # Run Gateway API only (port 8001) -make test # Run all backend tests +make test # Run offline backend tests (excludes live external-API tests) +make test-live # Explicitly run live DeerFlowClient tests with real APIs make test-blocking-io # Run strict Blockbuster runtime gate on tests/blocking_io/ make lint # Lint with ruff make format # Format code with ruff @@ -1219,7 +1220,13 @@ Gateway API endpoints and `DeerFlowClient` methods can modify MCP servers and sk **Key difference from Gateway**: Upload accepts local `Path` objects instead of HTTP `UploadFile`, rejects directory paths before copying, and reuses a single worker when document conversion must run inside an active event loop. Artifact returns `(bytes, mime_type)` instead of HTTP Response. The new Gateway-only thread cleanup route deletes `.deer-flow/threads/{thread_id}` after LangGraph thread deletion; there is no matching `DeerFlowClient` method yet. `update_mcp_config()` and `update_skill()` automatically invalidate the cached agent. -**Tests**: `tests/test_client.py` (77 unit tests including `TestGatewayConformance`), `tests/test_client_live.py` (live integration tests, requires config.yaml) +**Tests**: `tests/test_client.py` (offline unit tests including +`TestGatewayConformance`), `tests/test_client_live.py` (live integration tests, +requires a root `config.yaml`, valid API credentials, and explicit opt-in via +`make test-live` or `DEER_FLOW_RUN_LIVE_TESTS=1`). The live suite calls real +external APIs and may incur API costs or create local sandboxes, artifacts, and +files. It is marked `live`, excluded from `make test`, and skipped in default +CI. **Gateway Conformance Tests** (`TestGatewayConformance`): Validate that every dict-returning client method conforms to the corresponding Gateway Pydantic response model. Each test parses the client output through the Gateway model — if Gateway adds a required field that the client doesn't provide, Pydantic raises `ValidationError` and CI catches the drift. Covers: `ModelsListResponse`, `ModelResponse`, `SkillsListResponse`, `SkillResponse`, `SkillInstallResponse`, `McpConfigResponse`, `UploadResponse`, `MemoryConfigResponse`, `MemoryStatusResponse`. @@ -1230,19 +1237,27 @@ Gateway API endpoints and `DeerFlowClient` methods can modify MCP servers and sk **Every new feature or bug fix MUST be accompanied by unit tests. No exceptions.** - Write tests in `backend/tests/` following the existing naming convention `test_.py` -- Run the full suite before and after your change: `make test` +- Run the full offline suite before and after your change: `make test` - Tests must pass before a feature is considered complete - For lightweight config/utility modules, prefer pure unit tests with no external dependencies - If a module causes circular import issues in tests, add a `sys.modules` mock in `tests/conftest.py` (see existing example for `deerflow.subagents.executor`) ```bash -# Run all tests +# Run all offline tests make test +# Explicit live integration tests (requires config.yaml and credentials; +# calls real APIs and may create local side effects) +make test-live + # Run a specific test file PYTHONPATH=. uv run pytest tests/test_.py -v ``` +Direct pytest collection or execution of `tests/test_client_live.py` remains +skipped unless `DEER_FLOW_RUN_LIVE_TESTS=1` is set. Do not add that opt-in to +default CI workflows. + ### Running the Full Application From the **project root** directory: diff --git a/backend/Makefile b/backend/Makefile index 8f0e891ea..27cb31953 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -8,7 +8,10 @@ gateway: PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 test: - PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest tests/ -v + PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m "not live" tests/ -v + +test-live: + DEER_FLOW_RUN_LIVE_TESTS=1 PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m live tests/ -v -s test-blocking-io: PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest tests/blocking_io -q --tb=short diff --git a/backend/README.md b/backend/README.md index 6e9818673..4e5c4b947 100644 --- a/backend/README.md +++ b/backend/README.md @@ -454,9 +454,19 @@ the only execution path, which keeps operational mistakes off the table. See ### Testing ```bash -uv run pytest +# Offline backend suite (live external-API tests are excluded) +make test + +# Explicit real-API DeerFlowClient integration suite +make test-live ``` +The live suite requires a valid root `config.yaml` and API credentials. It may +incur API costs or create local sandboxes, artifacts, and files, so it is not +part of default test runs or CI. Direct pytest invocation of +`tests/test_client_live.py` also requires +`DEER_FLOW_RUN_LIVE_TESTS=1`. + `make detect-blocking-io` statically scans backend business code for blocking IO that may run on the backend event loop and is not test-coverage-bound. It prints a concise summary for human review and writes complete JSON findings to diff --git a/backend/pyproject.toml b/backend/pyproject.toml index e87936e5d..1963b5d22 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -58,6 +58,7 @@ markers = [ "no_auto_user: disable the conftest autouse contextvar fixture for this test", "allow_blocking_io: opt out of the strict Blockbuster gate in tests/blocking_io/", "integration: tests that require an external service (e.g. Redis); skipped when unavailable", + "live: tests that call real external APIs and require explicit opt-in", ] [tool.uv] diff --git a/backend/tests/test_client_live.py b/backend/tests/test_client_live.py index 0271ebf21..4c4de8104 100644 --- a/backend/tests/test_client_live.py +++ b/backend/tests/test_client_live.py @@ -1,9 +1,14 @@ -"""Live integration tests for DeerFlowClient with real API. +"""Live integration tests for DeerFlowClient with real external APIs. These tests require a working config.yaml with valid API credentials. -They are skipped in CI and must be run explicitly: +They can incur API costs and create local sandboxes, artifacts, or files. +They are skipped in CI and default test runs and must be run explicitly: - PYTHONPATH=. uv run pytest tests/test_client_live.py -v -s + make test-live + +For direct pytest invocation, set the explicit opt-in flag: + + DEER_FLOW_RUN_LIVE_TESTS=1 PYTHONPATH=. uv run pytest tests/test_client_live.py -v -s """ import json @@ -16,10 +21,16 @@ from deerflow.client import DeerFlowClient, StreamEvent from deerflow.sandbox.security import is_host_bash_allowed from deerflow.uploads.manager import PathTraversalError -# Skip entire module in CI or when no config.yaml exists +pytestmark = pytest.mark.live + +_LIVE_TEST_OPT_IN = "DEER_FLOW_RUN_LIVE_TESTS" + +# Skip the entire module unless every live-test precondition is satisfied. _skip_reason = None if os.environ.get("CI"): _skip_reason = "Live tests skipped in CI" +elif os.environ.get(_LIVE_TEST_OPT_IN) != "1": + _skip_reason = f"Set {_LIVE_TEST_OPT_IN}=1 to run live tests with real external APIs" elif not Path(__file__).resolve().parents[2].joinpath("config.yaml").exists(): _skip_reason = "No config.yaml found — live tests require valid API credentials" diff --git a/backend/tests/test_client_live_policy.py b/backend/tests/test_client_live_policy.py new file mode 100644 index 000000000..a63db634f --- /dev/null +++ b/backend/tests/test_client_live_policy.py @@ -0,0 +1,161 @@ +"""Regression tests for the explicit opt-in policy of live client tests.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +REPO_ROOT = BACKEND_ROOT.parent +LIVE_TEST_PATH = BACKEND_ROOT / "tests" / "test_client_live.py" +LIVE_OPT_IN = "DEER_FLOW_RUN_LIVE_TESTS" +REPRESENTATIVE_LIVE_NODE_IDS = ( + "::TestLiveBasicChat::test_chat_returns_nonempty_string", + "::TestLiveStreaming::test_stream_yields_messages_tuple_and_end", +) + + +def _collect_live_tests( + tmp_path: Path, + *, + config_exists: bool, + opt_in: bool, + ci: bool, +) -> subprocess.CompletedProcess[str]: + """Collect a temporary copy without inheriting credentials or a real .env.""" + temp_repo = tmp_path / "repo" + temp_tests = temp_repo / "backend" / "tests" + temp_tests.mkdir(parents=True) + (temp_tests / "test_client_live.py").write_text( + LIVE_TEST_PATH.read_text(encoding="utf-8"), + encoding="utf-8", + ) + if config_exists: + (temp_repo / "config.yaml").write_text("models: []\n", encoding="utf-8") + + env = { + "PATH": os.environ.get("PATH", ""), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONUTF8": "1", + "PYTHONPATH": os.pathsep.join( + [ + str(BACKEND_ROOT), + str(BACKEND_ROOT / "packages" / "harness"), + ] + ), + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + } + if opt_in: + env[LIVE_OPT_IN] = "1" + if ci: + env["CI"] = "1" + + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-c", + str(BACKEND_ROOT / "pyproject.toml"), + "--collect-only", + "-q", + "-rs", + "-p", + "no:cacheprovider", + str(temp_tests / "test_client_live.py"), + ], + cwd=temp_repo / "backend", + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def _assert_collection_skipped(result: subprocess.CompletedProcess[str], reason: str) -> None: + output = result.stdout + result.stderr + assert result.returncode in {0, pytest.ExitCode.NO_TESTS_COLLECTED}, output + assert reason in output + assert "no tests collected" in output + assert all(node_id not in output for node_id in REPRESENTATIVE_LIVE_NODE_IDS) + + +def _dry_run_make_target(target: str) -> str: + result = subprocess.run( + ["make", "-n", target], + cwd=BACKEND_ROOT, + capture_output=True, + text=True, + check=False, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + return output + + +def test_config_alone_does_not_enable_live_collection(tmp_path: Path) -> None: + result = _collect_live_tests(tmp_path, config_exists=True, opt_in=False, ci=False) + + _assert_collection_skipped(result, f"Set {LIVE_OPT_IN}=1") + + +def test_explicit_opt_in_collects_live_tests(tmp_path: Path) -> None: + result = _collect_live_tests(tmp_path, config_exists=True, opt_in=True, ci=False) + + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert all(node_id in output for node_id in REPRESENTATIVE_LIVE_NODE_IDS) + + +def test_ci_blocks_live_collection_even_with_opt_in(tmp_path: Path) -> None: + result = _collect_live_tests(tmp_path, config_exists=True, opt_in=True, ci=True) + + _assert_collection_skipped(result, "Live tests skipped in CI") + + +def test_opt_in_without_config_reports_missing_config(tmp_path: Path) -> None: + result = _collect_live_tests(tmp_path, config_exists=False, opt_in=True, ci=False) + + _assert_collection_skipped(result, "No config.yaml found") + + +def test_make_targets_keep_default_tests_offline_and_support_live_opt_in() -> None: + default_command = _dry_run_make_target("test") + live_command = _dry_run_make_target("test-live") + + assert 'pytest -m "not live"' in default_command + assert "tests/" in default_command + assert LIVE_OPT_IN not in default_command + + assert f"{LIVE_OPT_IN}=1" in live_command + assert "pytest -m live" in live_command + assert "tests/" in live_command + assert "tests/test_client_live.py" not in live_command + + +def test_live_marker_is_registered() -> None: + pyproject = (BACKEND_ROOT / "pyproject.toml").read_text(encoding="utf-8") + + assert '"live: tests that call real external APIs and require explicit opt-in"' in pyproject + + +def test_documentation_matches_live_test_commands() -> None: + contributor_docs = (REPO_ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") + backend_agent_docs = (BACKEND_ROOT / "AGENTS.md").read_text(encoding="utf-8") + + for docs in (contributor_docs, backend_agent_docs): + assert "make test-live" in docs + assert LIVE_OPT_IN in docs + assert "API" in docs + + +def test_default_ci_workflow_does_not_opt_in_to_live_tests() -> None: + workflow = (REPO_ROOT / ".github" / "workflows" / "backend-unit-tests.yml").read_text(encoding="utf-8") + + assert "make test" in workflow + assert LIVE_OPT_IN not in workflow