From 3b8141aa43e1e3911ca8725c9533823ce3ca1c22 Mon Sep 17 00:00:00 2001 From: yangfucheng123 <140314128+yangfucheng123@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:45:58 +0800 Subject: [PATCH] fix(community): Fail closed on delegated fetch backends to close SSRF gap (#6531) * Fail closed on delegated fetch backends to close SSRF gap * fix(community): resolve delegated-fetch credentials from one config snapshot Address review feedback on the fail-closed SSRF guard: - firecrawl/fastcrw client construction no longer re-reads get_app_config(), so the API key and endpoint come from the same revision the endpoint screen validated (prevents cross-revision credential misrouting). - fix the browserless test double to accept the new two-argument _get_browserless_client signature. - document the check-only (DNS rebinding) caveat on validate_delegated_backend_url. - add a CHANGELOG upgrade note for the default self-hosted backend break. * fix(doctor): consolidate delegated fetch backend checks and deployment guidance --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 18 ++ README.md | 2 + backend/docs/CONFIGURATION.md | 43 ++++- .../deerflow/community/browserless/tools.py | 44 ++++- .../deerflow/community/crawl4ai/tools.py | 24 ++- .../deerflow/community/fastcrw/tools.py | 61 +++++-- .../deerflow/community/firecrawl/tools.py | 70 +++++++- .../harness/deerflow/community/url_safety.py | 55 +++++- .../packages/harness/deerflow/tools/AGENTS.md | 2 +- backend/tests/test_browserless_client.py | 105 ++++++++++++ backend/tests/test_crawl4ai_tools.py | 76 +++++++++ backend/tests/test_doctor.py | 159 +++++++++++++++++- backend/tests/test_fastcrw_tools.py | 114 +++++++++++++ backend/tests/test_firecrawl_tools.py | 110 +++++++++++- backend/tests/test_url_safety.py | 40 ++++- .../tests/test_web_fetch_error_shell_meta.py | 2 + .../tests/test_web_fetch_relative_links.py | 3 +- config.example.yaml | 22 +++ scripts/AGENTS.md | 8 + scripts/doctor.py | 47 ++++++ 20 files changed, 968 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f499b2091..79012c935d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4337,6 +4337,24 @@ This release closes that milestone with **439 merged pull requests**. package. Previously a `hooks/install.jse` carrying one stray byte received no static analysis and no executable review. ([#6321]) +- **community:** Delegated `web_fetch`/`web_capture` backends fail closed by + default. Browserless, Crawl4AI, Firecrawl, and fastCRW resolve the target URL + in the backend's own network namespace, so the target-URL SSRF screen cannot + be enforced end-to-end. Each entry point now screens its resolved backend + base URL (config key, env fallback, and default) before delegating: a public + backend still works unchanged, but a loopback, private, or unverifiable + backend is refused unless the operator confirms its egress isolation. + **Upgrade note:** a deployment using the documented defaults (Browserless at + `http://localhost:3032`, Crawl4AI at `http://localhost:11235`, or a + self-hosted Firecrawl/fastCRW via `base_url`/`CRW_API_URL`) now sees a + delegation error on `web_fetch`/`web_capture` until it either points the + backend at its public address or sets `network_isolation_confirmed: true` in + each tool config after isolating the backend's egress. `make doctor` warns + about refused backend configurations, including `CRW_API_URL` overrides. + `allow_private_addresses` still controls target URLs only. See the + [deployment guidance](backend/docs/CONFIGURATION.md#delegated-fetch-backend-isolation). + ([#6531]) + - **scripts:** Bind local `make dev` / `make start` to loopback. `serve.sh` and `backend/Makefile` started the Gateway with `--host 0.0.0.0`, `nginx.local.conf` listened on every interface, and Next.js kept its diff --git a/README.md b/README.md index f7924ab9eb..757111aa75 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,8 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta Jina, Browserless, and InfoQuest web fetches resolve relative links and image sources using the requested page URL (or a usable HTML base URL), so returned Markdown includes complete destinations. Link resolution preserves the surrounding HTML source, including malformed-page formatting. + Self-hosted Browserless, Crawl4AI, Firecrawl, and fastCRW fetch backends require outbound isolation from private, loopback, link-local, and cloud-metadata networks before setting `network_isolation_confirmed: true` on each tool. DeerFlow rejects private or unverifiable backends by default because those services handle redirects, DNS resolution, and subresources themselves. `allow_private_addresses` permits intentional internal target URLs and does not bypass the backend check. See [delegated fetch deployment](backend/docs/CONFIGURATION.md#delegated-fetch-backend-isolation); `make doctor` reports backend configurations that would be refused. + Jina fetches support opt-in bounded retries via `max_retries` (default `0`) and `retry_budget_seconds` (default `30`) in the tool configuration. Valid `Retry-After` hints set a minimum wait for HTTP 429/503; 429 without a valid hint stays terminal. Hints that cannot fit the remaining budget stop retries. Local backoff remains randomized. Retries may increase upstream requests and cost; see [Jina fetch retries](backend/docs/CONFIGURATION.md#jina-fetch-retries). Jina also accepts an opt-in `max_response_bytes` tool setting (positive integer; omitted/null disables it). It stops oversized decoded responses before extraction, with no partial success or retry. This leaves the 4096-character output cap unchanged and does not bound HTTPX decompressor allocations or wire bandwidth; see [response budget](backend/docs/CONFIGURATION.md#jina-response-byte-budget). diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index af6f8bc4ca..1495347eb6 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -885,7 +885,41 @@ retrieval behavior. No paid API calls are needed for the regression suite. - `str_replace` - String replacement in files - `bash` - Execute bash commands -Browserless can be configured as an opt-in visual capture tool: +#### Delegated Fetch Backend Isolation + +Browserless, Crawl4AI, Firecrawl, and fastCRW perform navigation, redirects, +DNS resolution, and subresource loading in the backend's own network namespace. +Checking the submitted URL in the Gateway cannot constrain these later requests. +For `web_fetch`, and Browserless `web_capture`, DeerFlow therefore refuses to +delegate to a private, loopback, or otherwise unverifiable backend by default. +HTTP(S) backend endpoints resolving only to public addresses do not require an +isolation acknowledgement. + +The backend endpoint is the tool's `base_url`. Browserless defaults to +`http://localhost:3032`, Crawl4AI to `http://localhost:11235`, and Firecrawl to +its SDK's public cloud endpoint when `base_url` is unset. fastCRW uses +`base_url`, then `CRW_API_URL`, then `https://fastcrw.com/api`. The guard checks +the effective endpoint, including environment references in the config. + +Before enabling a self-hosted backend, enforce an outbound policy on that +service that blocks private, loopback, link-local, shared (`100.64.0.0/10`), +other non-global, and cloud-metadata destinations, including redirects and +subresources. Then set `network_isolation_confirmed: true` on each tool using +that backend. Browserless fetch and capture are separate tool entries and each +needs the setting. Run `make doctor` to check for configurations that would be +refused. Existing localhost deployments need this migration before fetching or +capturing pages after an upgrade. + +`network_isolation_confirmed` is an operator acknowledgement; it does not +install or verify an egress policy. `allow_private_addresses: true` controls +the submitted target URL only and never bypasses backend screening. Reserve it +for intentional internal targets and explicitly account for those destinations +in the backend's outbound policy. Neither setting relaxes the HTTP(S) requirement. +Backend DNS screening happens at validation time and does not pin the subsequent +client connection, so it cannot by itself close a DNS-rebinding window. + +Browserless can be configured as an opt-in visual capture tool after its egress +policy is in place: ```yaml tools: @@ -894,6 +928,7 @@ tools: use: deerflow.community.browserless.tools:web_capture_tool base_url: http://localhost:3032 # token: $BROWSERLESS_TOKEN + network_isolation_confirmed: true # Only after isolating Browserless egress output_format: png full_page: true viewport_width: 1280 @@ -917,9 +952,13 @@ Browserless instance. You can point `base_url` at [Browserless Cloud](https://ww # match the default base_url (http://localhost:3032). Recent Browserless # images always require a token — if you don't pass one, a random token is # generated and requests without it are rejected — so set it explicitly. -docker run -d --name browserless -p 3032:3000 -e "TOKEN=local-dev-token" ghcr.io/browserless/chromium +docker run -d --name browserless -p 127.0.0.1:3032:3000 -e "TOKEN=local-dev-token" ghcr.io/browserless/chromium ``` +Publishing a port does not isolate outbound traffic. Apply the egress policy +appropriate to your container platform before setting +`network_isolation_confirmed: true`; the command above only starts the service. + Then set the same token so the tool sends it (uncomment `token: $BROWSERLESS_TOKEN` in the config above): diff --git a/backend/packages/harness/deerflow/community/browserless/tools.py b/backend/packages/harness/deerflow/community/browserless/tools.py index e713e3aa68..9ec9ecf82a 100644 --- a/backend/packages/harness/deerflow/community/browserless/tools.py +++ b/backend/packages/harness/deerflow/community/browserless/tools.py @@ -12,7 +12,7 @@ from langchain_core.messages import ToolMessage from langgraph.types import Command from deerflow.community.url_safety import resolve_host_addresses as _resolve_host_addresses -from deerflow.community.url_safety import validate_public_http_url +from deerflow.community.url_safety import validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config from deerflow.config.paths import VIRTUAL_PATH_PREFIX from deerflow.tools.types import Runtime @@ -33,6 +33,8 @@ _OUTPUT_FORMAT_TO_EXTENSION = { _SAFE_FILENAME_RE = re.compile(r"[^A-Za-z0-9._-]+") # Cap collision-suffix probing so a saturated outputs directory cannot spin forever. _MAX_FILENAME_COLLISION_PROBES = 1000 +DEFAULT_BASE_URL = "http://localhost:3032" +_UNSET = object() def _get_tool_config(tool_name: str) -> dict | None: @@ -79,9 +81,10 @@ def _resolve_timeout(cfg: dict, default: float) -> float: return default -def _get_browserless_client(tool_name: str = "web_fetch") -> BrowserlessClient: - cfg = _get_tool_config(tool_name) - base_url = "http://localhost:3032" +def _get_browserless_client(tool_name: str = "web_fetch", cfg: dict | None = _UNSET) -> BrowserlessClient: + if cfg is _UNSET: + cfg = _get_tool_config(tool_name) + base_url = DEFAULT_BASE_URL token = os.getenv("BROWSERLESS_TOKEN", "") timeout_s = 30.0 if cfg is not None: @@ -103,6 +106,27 @@ def _as_bool(value: object, default: bool) -> bool: return default +def _validate_backend_base_url(cfg: dict | None) -> str | None: + """Refuse delegation to a self-hosted backend unless its egress is isolated. + + Browserless resolves the target URL, follows redirects, and loads subresources + in the Browserless service's own network namespace, so the target-URL screen + cannot be enforced end-to-end. Delegation is only safe when the backend's + outbound network is isolated from private and metadata networks, which the + operator confirms via ``network_isolation_confirmed``. A public backend + (Browserless Cloud) needs no confirmation. Blocking; call via + ``asyncio.to_thread``. + """ + cfg = cfg or {} + base_url = cfg.get("base_url", DEFAULT_BASE_URL) + network_isolation_confirmed = _as_bool(cfg.get("network_isolation_confirmed"), False) + return validate_delegated_backend_url( + base_url, + network_isolation_confirmed=network_isolation_confirmed, + resolver=_resolve_host_addresses, + ) + + def _as_int(value: object, default: int) -> int: if isinstance(value, int) and not isinstance(value, bool): return value @@ -266,6 +290,10 @@ async def web_fetch_tool(url: str) -> str: if url_error: return url_error + backend_error = await asyncio.to_thread(_validate_backend_base_url, cfg) + if backend_error: + return backend_error + wait_for_event = "" wait_for_timeout_ms = 0 wait_for_selector = "" @@ -278,7 +306,7 @@ async def web_fetch_tool(url: str) -> str: wait_for_selector = cfg.get("wait_for_selector", wait_for_selector) wait_for_selector_timeout_ms = _as_int(cfg.get("wait_for_selector_timeout_ms"), wait_for_selector_timeout_ms) - client = _get_browserless_client("web_fetch") + client = _get_browserless_client("web_fetch", cfg) result = await client.fetch_html_with_status( url=url, wait_for_event=wait_for_event, @@ -333,6 +361,10 @@ async def web_capture_tool( if url_error: return _tool_message(url_error, tool_call_id) + backend_error = await asyncio.to_thread(_validate_backend_base_url, cfg) + if backend_error: + return _tool_message(backend_error, tool_call_id) + outputs_path = _thread_outputs_path(runtime) if isinstance(outputs_path, str): return _tool_message(outputs_path, tool_call_id) @@ -349,7 +381,7 @@ async def web_capture_tool( output_name = _safe_capture_filename(filename, url, final_format) - client = _get_browserless_client("web_capture") + client = _get_browserless_client("web_capture", cfg) result = await client.capture_screenshot( url=url, full_page=final_full_page, diff --git a/backend/packages/harness/deerflow/community/crawl4ai/tools.py b/backend/packages/harness/deerflow/community/crawl4ai/tools.py index 917dbb2689..0148ca0389 100644 --- a/backend/packages/harness/deerflow/community/crawl4ai/tools.py +++ b/backend/packages/harness/deerflow/community/crawl4ai/tools.py @@ -3,7 +3,7 @@ import logging from langchain.tools import tool -from deerflow.community.url_safety import validate_public_http_url +from deerflow.community.url_safety import validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config from .crawl4ai_client import Crawl4AiClient @@ -87,6 +87,25 @@ def _build_client(cfg: dict | None) -> Crawl4AiClient: return Crawl4AiClient(base_url=base_url, token=token, timeout_s=timeout_s) +def _validate_backend_base_url(cfg: dict | None) -> str | None: + """Refuse delegation to a self-hosted backend unless its egress is isolated. + + Crawl4AI resolves the target URL, follows redirects, and loads subresources + in the Crawl4AI service's own network namespace, so the target-URL screen + cannot be enforced end-to-end. Delegation is only safe when the backend's + outbound network is isolated from private and metadata networks, which the + operator confirms via ``network_isolation_confirmed``. Blocking; call via + ``asyncio.to_thread``. + """ + cfg = cfg or {} + base_url = cfg.get("base_url", DEFAULT_BASE_URL) + network_isolation_confirmed = _coerce_bool(cfg.get("network_isolation_confirmed"), False) + return validate_delegated_backend_url( + base_url, + network_isolation_confirmed=network_isolation_confirmed, + ) + + @tool("web_fetch", parse_docstring=True) async def web_fetch_tool(url: str) -> str: """Fetch the contents of a web page at a given URL. @@ -104,6 +123,9 @@ async def web_fetch_tool(url: str) -> str: url_error = await asyncio.to_thread(validate_public_http_url, url, allow_private_addresses=allow_private_addresses) if url_error: return url_error + backend_error = await asyncio.to_thread(_validate_backend_base_url, cfg) + if backend_error: + return backend_error filter_mode = _coerce_filter(cfg.get("filter") if cfg is not None else None) client = _build_client(cfg) markdown = await client.fetch_markdown(url, filter_mode=filter_mode) diff --git a/backend/packages/harness/deerflow/community/fastcrw/tools.py b/backend/packages/harness/deerflow/community/fastcrw/tools.py index 0bc09d8566..279907f423 100644 --- a/backend/packages/harness/deerflow/community/fastcrw/tools.py +++ b/backend/packages/harness/deerflow/community/fastcrw/tools.py @@ -6,7 +6,7 @@ from firecrawl import FirecrawlApp from langchain.tools import tool from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results -from deerflow.community.url_safety import validate_public_http_url +from deerflow.community.url_safety import validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config logger = logging.getLogger(__name__) @@ -18,15 +18,24 @@ logger = logging.getLogger(__name__) DEFAULT_BASE_URL = "https://fastcrw.com/api" -def _get_fastcrw_client(tool_name: str = "web_search") -> FirecrawlApp: - config = get_app_config().get_tool_config(tool_name) - api_key = None - base_url = None - if config is not None: - if "api_key" in config.model_extra: - api_key = config.model_extra.get("api_key") - if "base_url" in config.model_extra: - base_url = config.model_extra.get("base_url") +def _get_fastcrw_client( + tool_name: str = "web_search", + *, + cfg: dict | None = None, + base_url: str | None = None, +) -> FirecrawlApp: + """Build a fastCRW client from one configuration snapshot. + + ``cfg`` is the tool config extras already read by the caller. Resolving the + API key and endpoint from that same snapshot keeps a backend change made + between the URL screen and client construction from pairing one revision's + endpoint with another revision's key. + """ + if cfg is None: + cfg = _get_tool_config_extra(tool_name) + api_key = cfg.get("api_key") + if base_url is None: + base_url = cfg.get("base_url") if api_key is None: api_key = os.getenv("CRW_API_KEY") if base_url is None: @@ -34,6 +43,19 @@ def _get_fastcrw_client(tool_name: str = "web_search") -> FirecrawlApp: return FirecrawlApp(api_key=api_key, api_url=base_url) # type: ignore[arg-type] +def _resolve_fastcrw_base_url(cfg: dict | None) -> str: + """Resolve the fastCRW base URL from config, then the ``CRW_API_URL`` env var. + + The backend screen must run on this resolved value (config key first, then the + env fallback), not on the raw config key, or the ``CRW_API_URL`` path would + bypass the gate. + """ + base_url = cfg.get("base_url") if cfg is not None else None + if base_url is None: + base_url = os.getenv("CRW_API_URL", DEFAULT_BASE_URL) + return base_url + + def _get_tool_config_extra(tool_name: str) -> dict: config = get_app_config().get_tool_config(tool_name) return dict(config.model_extra or {}) if config is not None else {} @@ -51,6 +73,19 @@ def _coerce_bool(value: object, default: bool) -> bool: return default +def _validate_backend_base_url(cfg: dict, base_url: str) -> str | None: + """Refuse delegation to a self-hosted fastCRW backend unless its egress is isolated. + + fastCRW resolves the target URL, follows redirects, and loads subresources in + the fastCRW service's own network namespace, so the target-URL screen cannot be + enforced end-to-end. Delegation is only safe when the backend's outbound network + is isolated from private and metadata networks, which the operator confirms via + ``network_isolation_confirmed``. Blocking; fastCRW's ``web_fetch_tool`` is sync. + """ + network_isolation_confirmed = _coerce_bool(cfg.get("network_isolation_confirmed"), False) + return validate_delegated_backend_url(base_url, network_isolation_confirmed=network_isolation_confirmed) + + @tool("web_search", parse_docstring=True) def web_search_tool(query: str) -> str: """Search the web. @@ -100,7 +135,11 @@ def web_fetch_tool(url: str) -> str: url_error = validate_public_http_url(url, allow_private_addresses=allow_private_addresses) if url_error: return url_error - client = _get_fastcrw_client("web_fetch") + base_url = _resolve_fastcrw_base_url(cfg) + backend_error = _validate_backend_base_url(cfg, base_url) + if backend_error: + return backend_error + client = _get_fastcrw_client("web_fetch", cfg=cfg, base_url=base_url) result = client.scrape(url, formats=["markdown"]) markdown_content = result.markdown or "" diff --git a/backend/packages/harness/deerflow/community/firecrawl/tools.py b/backend/packages/harness/deerflow/community/firecrawl/tools.py index 9f741fbfd1..173b350ea8 100644 --- a/backend/packages/harness/deerflow/community/firecrawl/tools.py +++ b/backend/packages/harness/deerflow/community/firecrawl/tools.py @@ -1,3 +1,4 @@ +import asyncio import inspect import json import logging @@ -6,6 +7,7 @@ from firecrawl import AsyncFirecrawlApp from langchain.tools import tool from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results +from deerflow.community.url_safety import validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config logger = logging.getLogger(__name__) @@ -35,15 +37,55 @@ async def _aclose_firecrawl_client(client: AsyncFirecrawlApp) -> None: logger.warning("Failed to close the Firecrawl async HTTP pool", exc_info=True) -def _get_firecrawl_client(tool_name: str = "web_search") -> AsyncFirecrawlApp: +def _get_tool_config_extra(tool_name: str) -> dict: config = get_app_config().get_tool_config(tool_name) - api_key = None - api_url = None - if config is not None: - if "api_key" in config.model_extra: - api_key = config.model_extra.get("api_key") - if "base_url" in config.model_extra: - api_url = config.model_extra.get("base_url") + return dict(config.model_extra or {}) if config is not None else {} + + +def _coerce_bool(value: object, default: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return default + + +def _validate_backend_base_url(cfg: dict, base_url: str) -> str | None: + """Refuse delegation to a self-hosted Firecrawl backend unless its egress is isolated. + + Firecrawl resolves the target URL, follows redirects, and loads subresources + in the Firecrawl service's own network namespace, so the target-URL screen + cannot be enforced end-to-end. Delegation is only safe when the backend's + outbound network is isolated from private and metadata networks, which the + operator confirms via ``network_isolation_confirmed``. Blocking; call via + ``asyncio.to_thread``. + """ + network_isolation_confirmed = _coerce_bool(cfg.get("network_isolation_confirmed"), False) + return validate_delegated_backend_url(base_url, network_isolation_confirmed=network_isolation_confirmed) + + +def _get_firecrawl_client( + tool_name: str = "web_search", + *, + cfg: dict | None = None, + api_url: str | None = None, +) -> AsyncFirecrawlApp: + """Build a Firecrawl app from one configuration snapshot. + + ``cfg`` is the tool config extras already read by the caller. Resolving the + API key and endpoint from that same snapshot keeps a backend change made + between the URL screen and client construction from pairing one revision's + endpoint with another revision's key. + """ + if cfg is None: + cfg = _get_tool_config_extra(tool_name) + api_key = cfg.get("api_key") + if api_url is None: + api_url = cfg.get("base_url") kwargs = {"api_key": api_key} if api_url: kwargs["api_url"] = api_url @@ -99,7 +141,17 @@ async def web_fetch_tool(url: str) -> str: """ client: AsyncFirecrawlApp | None = None try: - client = _get_firecrawl_client("web_fetch") + cfg = _get_tool_config_extra("web_fetch") + allow_private_addresses = _coerce_bool(cfg.get("allow_private_addresses"), False) + url_error = await asyncio.to_thread(validate_public_http_url, url, allow_private_addresses=allow_private_addresses) + if url_error: + return url_error + api_url = cfg.get("base_url") + if api_url is not None: + backend_error = await asyncio.to_thread(_validate_backend_base_url, cfg, api_url) + if backend_error: + return backend_error + client = _get_firecrawl_client("web_fetch", cfg=cfg, api_url=api_url) result = await client.scrape(url, formats=["markdown"]) markdown_content = result.markdown or "" diff --git a/backend/packages/harness/deerflow/community/url_safety.py b/backend/packages/harness/deerflow/community/url_safety.py index c6e1b36d72..ad88a97c6e 100644 --- a/backend/packages/harness/deerflow/community/url_safety.py +++ b/backend/packages/harness/deerflow/community/url_safety.py @@ -99,9 +99,9 @@ def validate_public_http_url( resolves the name again, so a rebinding DNS server can still hand that connect a private address unless the connection is pinned to the vetted IPs (:func:`resolve_public_addresses`), as ``deerflow.mcp.personal_network`` - and the browser egress proxy do. Delegated fetch services (crawl4ai, - Browserless, fastcrw) resolve on their own side and cannot be pinned from - here. + and the browser egress proxy do. Delegated fetch services (Browserless, + crawl4ai, fastcrw, firecrawl) resolve on their own side and cannot be pinned + from here. """ parsed = urlparse(url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: @@ -119,3 +119,52 @@ def validate_public_http_url( except ValueError as exc: return str(exc) return None + + +def validate_delegated_backend_url( + base_url: str, + *, + network_isolation_confirmed: bool = False, + resolver: Callable[[str], list[ipaddress._BaseAddress]] | None = None, +) -> str | None: + """Validate a delegated fetch backend's base URL before delegating navigation. + + Browserless, Crawl4AI, fastCRW, and Firecrawl resolve the target URL, follow + redirects, and load subresources in the backend's own network namespace, so + the target-URL screen in :func:`validate_public_http_url` cannot be enforced + end-to-end. The only safe way to delegate is to ensure the backend's outbound + network is isolated from the deployment's private and metadata networks. + + A backend reachable at a public address (for example Browserless Cloud) runs + outside the deployment network, so delegation needs no extra confirmation. A + backend at a loopback, private, or unverifiable address is self-hosted inside + the deployment network, so this fails closed unless the operator confirms the + backend's egress is isolated via ``network_isolation_confirmed=True``. + + The check resolves the backend hostname once, at validation time only. A + backend client built afterwards connects by hostname and resolves the name + again, so a DNS name that answers with a public address during screening and + a private one at connect time still passes. Pin the connection to the vetted + addresses (:func:`resolve_public_addresses`) to close that rebinding window. + + Returns an ``"Error: ..."`` string when delegation must be refused, or + ``None`` when the caller may proceed. Blocking like + :func:`resolve_public_addresses`; from a coroutine, call it via + ``asyncio.to_thread``. + """ + parsed = urlparse(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + return "Error: Only http:// and https:// backend URLs are supported" + + if network_isolation_confirmed: + return None + + hostname = parsed.hostname + if not hostname: + return "Error: Backend URL host could not be parsed" + + try: + resolve_public_addresses(hostname, action="delegate to", resolver=resolver) + except ValueError as exc: + return f"{exc}. To delegate to a self-hosted backend, set network_isolation_confirmed: true." + return None diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index ecf6bfb232..fe83a28852 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -80,7 +80,7 @@ Durable MCP task-management tools are added only while the process-local task su - `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest). Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction. -The shared SSRF guard (`community/url_safety.py`) resolves hostnames with blocking `socket.getaddrinfo`, so async tools, Gateway routes, and the Playwright request guard (`BrowserSession._install_request_guard`, which runs on the shared browser loop) call `validate_public_http_url` (or a wrapper such as `validate_browser_url`) through `asyncio.to_thread`. The guard checks at validation time only, so a launched browser sends every TCP connection (HTTP, HTTPS, WebSocket) through its per-session loopback SOCKS5 proxy (`browser_automation/egress.py`, `bypass: <-loopback>`): Chromium hands it every hostname, and it resolves once via `resolve_browser_egress` → `url_safety.resolve_public_addresses` and connects to exactly those addresses, closing the DNS-rebinding window (`mcp/personal_network.py` pins the same way). Every `acquire_session`/`get_session` caller must pass `egress_resolver`, because a thread's first caller fixes its launch options. WebRTC UDP does not traverse a SOCKS proxy and is not pinned (out of scope, so this is not a complete egress guarantee). Delegated fetch services (crawl4ai, Browserless, fastcrw) resolve on their own side and cannot be pinned; CDP-attached Chrome has neither guard. The strict blocking-IO gate carries a `socket.getaddrinfo` rule for this; `tests/blocking_io/test_web_tool_url_validation.py` drives each web tool entry point, the request guard, and the egress proxy; `tests/test_browser_egress.py` pins the proxy protocol, the session/manager/router wiring, and (with Playwright installed) a real-Chromium rebinding case. +The shared SSRF guard (`community/url_safety.py`) resolves hostnames with blocking `socket.getaddrinfo`, so async tools, Gateway routes, and the Playwright request guard (`BrowserSession._install_request_guard`, which runs on the shared browser loop) call `validate_public_http_url` (or a wrapper such as `validate_browser_url`) through `asyncio.to_thread`. The guard checks at validation time only, so a launched browser sends every TCP connection (HTTP, HTTPS, WebSocket) through its per-session loopback SOCKS5 proxy (`browser_automation/egress.py`, `bypass: <-loopback>`): Chromium hands it every hostname, and it resolves once via `resolve_browser_egress` → `url_safety.resolve_public_addresses` and connects to exactly those addresses, closing the DNS-rebinding window (`mcp/personal_network.py` pins the same way). Every `acquire_session`/`get_session` caller must pass `egress_resolver`, because a thread's first caller fixes its launch options. WebRTC UDP does not traverse a SOCKS proxy and is not pinned (out of scope, so this is not a complete egress guarantee). Delegated fetch services (Browserless, crawl4ai, fastcrw, firecrawl) resolve on their own side and cannot be pinned, so each `web_fetch`/`web_capture` entry point instead screens its own backend base URL with `validate_delegated_backend_url` and fails closed on a self-hosted backend unless the operator confirms the backend's egress is isolated via `network_isolation_confirmed`; CDP-attached Chrome has neither guard. The strict blocking-IO gate carries a `socket.getaddrinfo` rule for this; `tests/blocking_io/test_web_tool_url_validation.py` drives each web tool entry point, the request guard, and the egress proxy; `tests/test_browser_egress.py` pins the proxy protocol, the session/manager/router wiring, and (with Playwright installed) a real-Chromium rebinding case. Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`); see each subpackage for specifics. Browserless authenticates only from the `token` query parameter or `Authorization` header and rejects unknown body keys, so `BrowserlessClient._auth_params()` carries the token for every endpoint; never add it to the JSON body. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. diff --git a/backend/tests/test_browserless_client.py b/backend/tests/test_browserless_client.py index e72aae3dfe..d116999404 100644 --- a/backend/tests/test_browserless_client.py +++ b/backend/tests/test_browserless_client.py @@ -368,6 +368,16 @@ class TestBrowserlessTools: ): yield + @pytest.fixture(autouse=True) + def _backend_isolation_confirmed(self): + """Assume the delegated backend's egress is isolated for these tests. + + These tests exercise fetch/capture behavior, not backend isolation; the + fail-closed backend guard is covered in TestBrowserlessBackendIsolation. + """ + with patch("deerflow.community.browserless.tools._validate_backend_base_url", return_value=None): + yield + async def test_get_browserless_client_uses_env_token_fallback(self): """Browserless tools use BROWSERLESS_TOKEN when config omits token.""" with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg: @@ -983,3 +993,98 @@ class TestBrowserlessTools: assert "Thread outputs path is not available" in result.update["messages"][0].content assert "artifacts" not in result.update mock_get_client.assert_not_called() + + +@pytest.mark.asyncio +class TestBrowserlessBackendIsolation: + """Fail-closed delegated-backend SSRF guard (issue #5970).""" + + @pytest.fixture(autouse=True) + def _resolve_host_addresses(self): + """Resolve hostnames to a public address to keep target validation offline.""" + with patch( + "deerflow.community.browserless.tools._resolve_host_addresses", + return_value=[ipaddress.ip_address("93.184.216.34")], + ): + yield + + @patch("deerflow.community.browserless.tools._get_browserless_client") + async def test_web_fetch_tool_fails_closed_on_default_backend(self, mock_get_client): + """A default (localhost) backend fails closed without isolation confirmation.""" + with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert "network_isolation_confirmed" in result + mock_get_client.assert_not_called() + + @patch("deerflow.community.browserless.tools._get_browserless_client") + async def test_web_fetch_tool_fails_closed_even_with_allow_private(self, mock_get_client): + """allow_private_addresses governs the target, not the backend's egress.""" + with patch( + "deerflow.community.browserless.tools._get_tool_config", + return_value={"allow_private_addresses": True}, + ): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert "network_isolation_confirmed" in result + mock_get_client.assert_not_called() + + @patch("deerflow.community.browserless.tools._get_browserless_client") + async def test_web_fetch_tool_allows_self_hosted_backend_when_confirmed(self, mock_get_client): + """network_isolation_confirmed opts into a self-hosted backend.""" + mock_client = MagicMock() + mock_client.fetch_html_with_status = AsyncMock( + return_value=BrowserlessFetchResult( + html="

ok

", + target_status_code="200", + target_status="OK", + ) + ) + mock_get_client.return_value = mock_client + + with patch( + "deerflow.community.browserless.tools._get_tool_config", + return_value={"network_isolation_confirmed": True}, + ): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert "Error:" not in result + mock_client.fetch_html_with_status.assert_called_once() + + @patch("deerflow.community.browserless.tools._get_browserless_client") + async def test_web_fetch_tool_allows_public_backend(self, mock_get_client): + """A public backend (Browserless Cloud) needs no isolation confirmation.""" + mock_client = MagicMock() + mock_client.fetch_html_with_status = AsyncMock( + return_value=BrowserlessFetchResult( + html="

ok

", + target_status_code="200", + target_status="OK", + ) + ) + mock_get_client.return_value = mock_client + + with patch( + "deerflow.community.browserless.tools._get_tool_config", + return_value={"base_url": "https://production-sfo.browserless.io"}, + ): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert "Error:" not in result + mock_client.fetch_html_with_status.assert_called_once() + + @patch("deerflow.community.browserless.tools._get_browserless_client") + async def test_web_capture_tool_fails_closed_on_default_backend(self, mock_get_client): + """web_capture_tool applies the same fail-closed backend guard.""" + runtime = SimpleNamespace(state={"thread_data": {}}) + + with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None): + result = await tools.web_capture_tool.coroutine( + runtime=runtime, + url="https://example.com", + tool_call_id="tool-1", + ) + + assert "network_isolation_confirmed" in result.update["messages"][0].content + assert "artifacts" not in result.update + mock_get_client.assert_not_called() diff --git a/backend/tests/test_crawl4ai_tools.py b/backend/tests/test_crawl4ai_tools.py index 615d89e43a..2e02422a32 100644 --- a/backend/tests/test_crawl4ai_tools.py +++ b/backend/tests/test_crawl4ai_tools.py @@ -173,6 +173,16 @@ class TestCrawl4AiClient: class TestCrawl4AiTools: """Tests for the Crawl4AI tool functions.""" + @pytest.fixture(autouse=True) + def _backend_isolation_confirmed(self): + """Assume the delegated backend's egress is isolated for these tests. + + These tests exercise fetch behavior, not backend isolation; the + fail-closed backend guard is covered in TestCrawl4AiBackendIsolation. + """ + with patch("deerflow.community.crawl4ai.tools._validate_backend_base_url", return_value=None): + yield + @patch("deerflow.community.crawl4ai.tools._build_client") @pytest.mark.usefixtures("public_dns") async def test_web_fetch_tool_success(self, mock_build): @@ -356,3 +366,69 @@ class TestCrawl4AiTools: assert tools._coerce_filter(" FIt ") == "fit" assert tools._coerce_filter("bogus") == "fit" assert tools._coerce_filter(None) == "fit" + + +@pytest.mark.asyncio +class TestCrawl4AiBackendIsolation: + """Fail-closed delegated-backend SSRF guard (issue #5970).""" + + @patch("deerflow.community.crawl4ai.tools._build_client") + @pytest.mark.usefixtures("public_dns") + async def test_web_fetch_tool_fails_closed_on_default_backend(self, mock_build): + from deerflow.community.crawl4ai import tools + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value=None): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert "network_isolation_confirmed" in result + mock_build.assert_not_called() + + @patch("deerflow.community.crawl4ai.tools._build_client") + @pytest.mark.usefixtures("public_dns") + async def test_web_fetch_tool_fails_closed_even_with_allow_private(self, mock_build): + from deerflow.community.crawl4ai import tools + + with patch( + "deerflow.community.crawl4ai.tools._get_tool_config", + return_value={"allow_private_addresses": True}, + ): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert "network_isolation_confirmed" in result + mock_build.assert_not_called() + + @patch("deerflow.community.crawl4ai.tools._build_client") + @pytest.mark.usefixtures("public_dns") + async def test_web_fetch_tool_allows_self_hosted_backend_when_confirmed(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="# ok") + mock_build.return_value = mock_client + + with patch( + "deerflow.community.crawl4ai.tools._get_tool_config", + return_value={"network_isolation_confirmed": True}, + ): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert result == "# ok" + mock_client.fetch_markdown.assert_called_once() + + @patch("deerflow.community.crawl4ai.tools._build_client") + @pytest.mark.usefixtures("public_dns") + async def test_web_fetch_tool_allows_public_backend(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="# ok") + mock_build.return_value = mock_client + + with patch( + "deerflow.community.crawl4ai.tools._get_tool_config", + return_value={"base_url": "https://crawl4ai.example.com"}, + ): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert result == "# ok" + mock_client.fetch_markdown.assert_called_once() diff --git a/backend/tests/test_doctor.py b/backend/tests/test_doctor.py index b7e68e7ad2..f3b019e6da 100644 --- a/backend/tests/test_doctor.py +++ b/backend/tests/test_doctor.py @@ -8,6 +8,7 @@ from __future__ import annotations import builtins import importlib.util +import ipaddress import json import sys from pathlib import Path @@ -803,17 +804,173 @@ class TestCheckWebFetch: # --------------------------------------------------------------------------- +@pytest.fixture +def delegated_backend_dns(monkeypatch): + """Keep backend screening real while replacing external DNS.""" + from deerflow.community import url_safety + + def resolve(host): + if host == "missing.invalid": + return [] + if host == "mixed.invalid": + return [ipaddress.ip_address("93.184.216.34"), ipaddress.ip_address("10.0.0.5")] + return [ipaddress.ip_address("93.184.216.34")] + + monkeypatch.setattr(url_safety, "resolve_host_addresses", resolve) + + +@pytest.mark.usefixtures("delegated_backend_dns") +class TestCheckDelegatedBackends: + @pytest.fixture(autouse=True) + def provider_keys(self, monkeypatch): + monkeypatch.setenv("FIRECRAWL_API_KEY", "test-key") + monkeypatch.setenv("CRW_API_KEY", "test-key") + monkeypatch.delenv("CRW_API_URL", raising=False) + + def _config(self, tmp_path, provider, *, tool_name="web_fetch", **extra): + cfg = tmp_path / "config.yaml" + tool = {"name": tool_name, "use": f"deerflow.community.{provider}.tools:{tool_name}_tool", **extra} + cfg.write_text(json.dumps({"config_version": 5, "tools": [tool]}), encoding="utf-8") + return cfg + + @pytest.mark.parametrize("provider", ["browserless", "crawl4ai", "firecrawl", "fastcrw"]) + @pytest.mark.parametrize("base_url", ["http://10.0.0.5:3000", "http://missing.invalid:3000", "http://mixed.invalid:3000"]) + def test_private_or_unverifiable_backend_warns(self, tmp_path, provider, base_url): + result = doctor.check_web_fetch(self._config(tmp_path, provider, base_url=base_url)) + + assert result.status == "warn" + assert "backend" in result.detail + assert "network_isolation_confirmed" in result.detail + assert "CONFIGURATION.md#delegated-fetch-backend-isolation" in result.fix + + @pytest.mark.parametrize("provider", ["browserless", "crawl4ai"]) + def test_localhost_default_warns(self, tmp_path, provider): + result = doctor.check_web_fetch(self._config(tmp_path, provider)) + + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + + @pytest.mark.parametrize("provider", ["browserless", "crawl4ai", "firecrawl", "fastcrw"]) + def test_isolated_private_backend_ok(self, tmp_path, provider): + result = doctor.check_web_fetch(self._config(tmp_path, provider, base_url="http://10.0.0.5:3000", network_isolation_confirmed=True)) + + assert result.status == "ok" + + @pytest.mark.parametrize("provider", ["browserless", "crawl4ai", "firecrawl", "fastcrw"]) + def test_public_backend_needs_no_acknowledgement(self, tmp_path, provider): + result = doctor.check_web_fetch(self._config(tmp_path, provider, base_url="https://public.invalid")) + + assert result.status == "ok" + + @pytest.mark.parametrize("provider", ["browserless", "crawl4ai", "firecrawl", "fastcrw"]) + def test_private_target_opt_in_does_not_authorize_backend(self, tmp_path, provider): + result = doctor.check_web_fetch(self._config(tmp_path, provider, base_url="http://10.0.0.5:3000", allow_private_addresses=True)) + + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + + @pytest.mark.parametrize("provider", ["browserless", "crawl4ai", "firecrawl", "fastcrw"]) + def test_invalid_scheme_warns_even_when_isolation_confirmed(self, tmp_path, provider): + result = doctor.check_web_fetch(self._config(tmp_path, provider, base_url="ftp://public.invalid", network_isolation_confirmed=True)) + + assert result.status == "warn" + assert "Only http:// and https://" in result.detail + + def test_fastcrw_env_backend_is_screened(self, tmp_path, monkeypatch): + monkeypatch.setenv("CRW_API_URL", "http://127.0.0.1:3000") + + result = doctor.check_web_fetch(self._config(tmp_path, "fastcrw")) + + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + + def test_fastcrw_config_endpoint_takes_precedence_over_env(self, tmp_path, monkeypatch): + monkeypatch.setenv("CRW_API_URL", "http://127.0.0.1:3000") + + result = doctor.check_web_fetch(self._config(tmp_path, "fastcrw", base_url="https://public.invalid")) + + assert result.status == "ok" + + @pytest.mark.parametrize("provider", ["firecrawl", "fastcrw"]) + def test_cloud_defaults_need_no_acknowledgement(self, tmp_path, provider): + result = doctor.check_web_fetch(self._config(tmp_path, provider)) + + assert result.status == "ok" + + def test_env_endpoint_and_acknowledgement_are_resolved(self, tmp_path, monkeypatch): + monkeypatch.setenv("FETCH_BACKEND", "http://10.0.0.5:3000") + monkeypatch.setenv("FETCH_ISOLATED", "false") + cfg = self._config(tmp_path, "crawl4ai", base_url="$FETCH_BACKEND", network_isolation_confirmed="$FETCH_ISOLATED") + + assert doctor.check_web_fetch(cfg).status == "warn" + monkeypatch.setenv("FETCH_ISOLATED", "true") + assert doctor.check_web_fetch(cfg).status == "ok" + + def test_dotenv_backend_is_screened(self, tmp_path, monkeypatch): + monkeypatch.setenv("FETCH_BACKEND", "") # Restore the environment after load_dotenv. + monkeypatch.delenv("FETCH_BACKEND") + (tmp_path / ".env").write_text("FETCH_BACKEND=http://10.0.0.5:3000\n", encoding="utf-8") + + result = doctor.check_web_fetch(self._config(tmp_path, "crawl4ai", base_url="$FETCH_BACKEND")) + + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + + def test_other_provider_does_not_mask_unsafe_backend(self, tmp_path): + cfg = self._config(tmp_path, "crawl4ai") + data = json.loads(cfg.read_text(encoding="utf-8")) + data["tools"].insert(0, {"name": "web_fetch", "use": "deerflow.community.jina_ai.tools:web_fetch_tool"}) + cfg.write_text(json.dumps(data), encoding="utf-8") + + result = doctor.check_web_fetch(cfg) + + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + + def test_search_policy_is_unchanged(self, tmp_path): + result = doctor.check_web_search(self._config(tmp_path, "fastcrw", tool_name="web_search", base_url="http://10.0.0.5:3000")) + + assert result.status == "ok" + + +@pytest.mark.usefixtures("delegated_backend_dns") class TestCheckWebCapture: - def test_browserless_self_host_without_token_ok(self, tmp_path, monkeypatch): + def test_browserless_self_host_without_confirmation_warns(self, tmp_path, monkeypatch): monkeypatch.delenv("BROWSERLESS_TOKEN", raising=False) cfg = tmp_path / "config.yaml" cfg.write_text("config_version: 5\ntools:\n - name: web_capture\n use: deerflow.community.browserless.tools:web_capture_tool\n base_url: http://localhost:3032\n") result = doctor.check_web_capture(cfg) + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + assert "CONFIGURATION.md#delegated-fetch-backend-isolation" in result.fix + + def test_browserless_isolated_self_host_without_token_ok(self, tmp_path, monkeypatch): + monkeypatch.delenv("BROWSERLESS_TOKEN", raising=False) + cfg = tmp_path / "config.yaml" + cfg.write_text( + "config_version: 5\ntools:\n - name: web_capture\n use: deerflow.community.browserless.tools:web_capture_tool\n base_url: http://localhost:3032\n network_isolation_confirmed: true\n", + encoding="utf-8", + ) + + result = doctor.check_web_capture(cfg) + assert result.status == "ok" assert "self-hosted" in result.detail + def test_private_target_opt_in_does_not_authorize_capture_backend(self, tmp_path): + cfg = tmp_path / "config.yaml" + cfg.write_text( + "config_version: 5\ntools:\n - name: web_capture\n use: deerflow.community.browserless.tools:web_capture_tool\n allow_private_addresses: true\n", + encoding="utf-8", + ) + + result = doctor.check_web_capture(cfg) + + assert result.status == "warn" + assert "network_isolation_confirmed" in result.detail + def test_browserless_token_env_ref_ok(self, tmp_path, monkeypatch): monkeypatch.setenv("BROWSERLESS_TOKEN", "browserless-test") cfg = tmp_path / "config.yaml" diff --git a/backend/tests/test_fastcrw_tools.py b/backend/tests/test_fastcrw_tools.py index 0aeb4c98b3..c2bd06d5c5 100644 --- a/backend/tests/test_fastcrw_tools.py +++ b/backend/tests/test_fastcrw_tools.py @@ -79,6 +79,16 @@ class TestWebSearchTool: class TestWebFetchTool: + @pytest.fixture(autouse=True) + def _backend_isolation_confirmed(self): + """Assume the delegated backend's egress is isolated for these tests. + + These tests exercise fetch behavior, not backend isolation; the + fail-closed backend guard is covered in TestFastcrwBackendIsolation. + """ + with patch("deerflow.community.fastcrw.tools._validate_backend_base_url", return_value=None): + yield + @patch.dict("os.environ", {}, clear=True) @patch("deerflow.community.fastcrw.tools.FirecrawlApp") @patch("deerflow.community.fastcrw.tools.get_app_config") @@ -192,6 +202,110 @@ class TestWebFetchTool: formats=["markdown"], ) + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + def test_fetch_builds_client_from_the_validated_config_snapshot(self, mock_get_app_config, mock_fastcrw_cls): + # The endpoint screen reads the config once; a later read returning a + # different revision must not leak its key into the screened client. + config_a = MagicMock() + config_a.model_extra = { + "base_url": "http://192.168.0.47:3002", + "api_key": "key-a", + } + config_b = MagicMock() + config_b.model_extra = { + "base_url": "http://192.168.0.99:3002", + "api_key": "key-b", + } + mock_get_app_config.return_value.get_tool_config.side_effect = [config_a, config_b] + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "Fetched markdown" + mock_scrape_result.metadata = MagicMock(title="Fetched Page") + mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result + + from deerflow.community.fastcrw.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# Fetched Page\n\nFetched markdown" + mock_fastcrw_cls.assert_called_once_with(api_key="key-a", api_url="http://192.168.0.47:3002") + + +class TestFastcrwBackendIsolation: + """Fail-closed delegated-backend SSRF guard (issue #5970).""" + + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + def test_fetch_fails_closed_on_self_hosted_backend(self, mock_get_app_config, mock_fastcrw_cls): + fetch_config = MagicMock() + fetch_config.model_extra = {"base_url": "http://localhost:3000"} + mock_get_app_config.return_value.get_tool_config.return_value = fetch_config + + from deerflow.community.fastcrw.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert "network_isolation_confirmed" in result + mock_fastcrw_cls.assert_not_called() + + @patch.dict("os.environ", {"CRW_API_URL": "http://localhost:3000"}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + def test_fetch_fails_closed_on_env_backend(self, mock_get_app_config, mock_fastcrw_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + + from deerflow.community.fastcrw.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert "network_isolation_confirmed" in result + mock_fastcrw_cls.assert_not_called() + + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + def test_fetch_allows_self_hosted_backend_when_confirmed(self, mock_get_app_config, mock_fastcrw_cls): + fetch_config = MagicMock() + fetch_config.model_extra = {"base_url": "http://localhost:3000", "network_isolation_confirmed": True} + mock_get_app_config.return_value.get_tool_config.return_value = fetch_config + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "ok" + mock_scrape_result.metadata = MagicMock(title="T") + mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result + + from deerflow.community.fastcrw.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# T\n\nok" + mock_fastcrw_cls.return_value.scrape.assert_called_once_with("https://example.com", formats=["markdown"]) + + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + def test_fetch_allows_public_backend(self, mock_get_app_config, mock_fastcrw_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "ok" + mock_scrape_result.metadata = MagicMock(title="T") + mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result + + from deerflow.community.fastcrw.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# T\n\nok" + # `None` is itself a configured value (`max_results:` with nothing after it in YAML), so an absent key # needs its own sentinel — otherwise the key-present case is never built and never tested. diff --git a/backend/tests/test_firecrawl_tools.py b/backend/tests/test_firecrawl_tools.py index cf8850f71e..abbedc213d 100644 --- a/backend/tests/test_firecrawl_tools.py +++ b/backend/tests/test_firecrawl_tools.py @@ -1,5 +1,6 @@ """Unit tests for the Firecrawl community tools.""" +import ipaddress import json import logging from types import SimpleNamespace @@ -8,6 +9,16 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest +@pytest.fixture +def public_dns(): + """Keep mocked tool calls offline while exercising the real URL safety check.""" + with patch( + "deerflow.community.url_safety.resolve_host_addresses", + return_value=[ipaddress.ip_address("93.184.216.34")], + ): + yield + + class TestWebSearchTool: @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") @@ -42,6 +53,7 @@ class TestWebSearchTool: class TestWebFetchTool: @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") @pytest.mark.anyio async def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls): fetch_config = MagicMock() @@ -71,14 +83,48 @@ class TestWebFetchTool: formats=["markdown"], ) + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") + @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + @pytest.mark.anyio + async def test_fetch_builds_client_from_the_validated_config_snapshot(self, mock_get_app_config, mock_firecrawl_cls): + # The endpoint screen reads the config once; a later read returning a + # different revision must not leak its key into the screened client. + config_a = MagicMock() + config_a.model_extra = { + "base_url": "http://192.168.0.47:3002", + "network_isolation_confirmed": True, + "api_key": "key-a", + } + config_b = MagicMock() + config_b.model_extra = { + "base_url": "http://192.168.0.99:3002", + "network_isolation_confirmed": True, + "api_key": "key-b", + } + mock_get_app_config.return_value.get_tool_config.side_effect = [config_a, config_b] + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "Fetched markdown" + mock_scrape_result.metadata = MagicMock(title="Fetched Page") + mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result) + + from deerflow.community.firecrawl.tools import web_fetch_tool + + result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) + + assert result == "# Fetched Page\n\nFetched markdown" + mock_firecrawl_cls.assert_called_once_with(api_key="key-a", api_url="http://192.168.0.47:3002") + class TestFirecrawlBaseUrl: @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") @pytest.mark.anyio async def test_fetch_passes_base_url_as_api_url(self, mock_get_app_config, mock_firecrawl_cls): fetch_config = MagicMock() - fetch_config.model_extra = {"base_url": "http://192.168.0.47:3002"} + fetch_config.model_extra = {"base_url": "http://192.168.0.47:3002", "network_isolation_confirmed": True} def get_tool_config(name): if name == "web_fetch": @@ -146,6 +192,7 @@ class TestPerCallClientTeardown: @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") @pytest.mark.anyio async def test_fetch_closes_the_per_call_client(self, mock_get_app_config, mock_firecrawl_cls): fetch_config = MagicMock() @@ -182,6 +229,7 @@ class TestPerCallClientTeardown: @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") @pytest.mark.anyio async def test_fetch_closes_the_per_call_client_when_scrape_raises(self, mock_get_app_config, mock_firecrawl_cls): mock_get_app_config.return_value.get_tool_config.return_value = None @@ -198,6 +246,7 @@ class TestPerCallClientTeardown: @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") @pytest.mark.anyio async def test_close_failure_does_not_hide_scrape_error(self, mock_get_app_config, mock_firecrawl_cls, caplog): mock_get_app_config.return_value.get_tool_config.return_value = None @@ -281,6 +330,65 @@ class TestPerCallClientTeardown: assert result == "[]" +class TestFirecrawlBackendIsolation: + """Fail-closed delegated-backend SSRF guard (issue #5970).""" + + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") + @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + @pytest.mark.anyio + async def test_fetch_fails_closed_on_self_hosted_backend(self, mock_get_app_config, mock_firecrawl_cls): + fetch_config = MagicMock() + fetch_config.model_extra = {"base_url": "http://localhost:3000"} + mock_get_app_config.return_value.get_tool_config.return_value = fetch_config + + from deerflow.community.firecrawl.tools import web_fetch_tool + + result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) + + assert "network_isolation_confirmed" in result + mock_firecrawl_cls.assert_not_called() + + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") + @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + @pytest.mark.anyio + async def test_fetch_allows_self_hosted_backend_when_confirmed(self, mock_get_app_config, mock_firecrawl_cls): + fetch_config = MagicMock() + fetch_config.model_extra = {"base_url": "http://localhost:3000", "network_isolation_confirmed": True} + mock_get_app_config.return_value.get_tool_config.return_value = fetch_config + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "ok" + mock_scrape_result.metadata = MagicMock(title="T") + mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result) + + from deerflow.community.firecrawl.tools import web_fetch_tool + + result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) + + assert result == "# T\n\nok" + mock_firecrawl_cls.return_value.scrape.assert_called_once_with("https://example.com", formats=["markdown"]) + + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") + @patch("deerflow.community.firecrawl.tools.get_app_config") + @pytest.mark.usefixtures("public_dns") + @pytest.mark.anyio + async def test_fetch_allows_public_backend(self, mock_get_app_config, mock_firecrawl_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "ok" + mock_scrape_result.metadata = MagicMock(title="T") + mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result) + + from deerflow.community.firecrawl.tools import web_fetch_tool + + result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) + + assert result == "# T\n\nok" + + # `None` is itself a configured value (`max_results:` with nothing after it in YAML), so an absent key # needs its own sentinel — otherwise the key-present case is never built and never tested. _OMITTED = object() diff --git a/backend/tests/test_url_safety.py b/backend/tests/test_url_safety.py index c7bad100c9..6f119059f6 100644 --- a/backend/tests/test_url_safety.py +++ b/backend/tests/test_url_safety.py @@ -4,7 +4,7 @@ import ipaddress import pytest -from deerflow.community.url_safety import is_blocked_address, validate_public_http_url +from deerflow.community.url_safety import is_blocked_address, validate_delegated_backend_url, validate_public_http_url @pytest.mark.parametrize( @@ -51,3 +51,41 @@ def test_validator_refuses_a_hostname_resolving_into_shared_address_space(): error = validate_public_http_url("http://metadata.example/latest/meta-data/", resolver=lambda _host: [ipaddress.ip_address("100.100.100.200")]) assert error == "Error: Refusing to fetch a private, loopback, or metadata address" + + +@pytest.mark.parametrize( + "base_url", + [ + "http://localhost:3032", + "http://127.0.0.1:3032", + "http://10.0.0.5:3032", + "http://169.254.169.254:3032", + "http://[::1]:3032", + ], +) +def test_delegated_backend_fails_closed_on_private_addresses(base_url): + error = validate_delegated_backend_url(base_url) + + assert error is not None + assert "network_isolation_confirmed" in error + + +def test_delegated_backend_fails_closed_when_unresolvable(): + error = validate_delegated_backend_url("http://browserless:3000", resolver=lambda _host: []) + + assert error is not None + assert "network_isolation_confirmed" in error + + +def test_delegated_backend_allows_public_address(): + error = validate_delegated_backend_url("https://production-sfo.browserless.io", resolver=lambda _host: [ipaddress.ip_address("93.184.216.34")]) + + assert error is None + + +def test_delegated_backend_allows_private_when_isolation_confirmed(): + assert validate_delegated_backend_url("http://10.0.0.5:3032", network_isolation_confirmed=True) is None + + +def test_delegated_backend_rejects_non_http_scheme(): + assert validate_delegated_backend_url("ftp://example.com") == "Error: Only http:// and https:// backend URLs are supported" diff --git a/backend/tests/test_web_fetch_error_shell_meta.py b/backend/tests/test_web_fetch_error_shell_meta.py index d1b2d0078a..b3c6cf0d21 100644 --- a/backend/tests/test_web_fetch_error_shell_meta.py +++ b/backend/tests/test_web_fetch_error_shell_meta.py @@ -231,6 +231,7 @@ def _render(html: str, code: str, status: str) -> str: patch.object(browserless_tools, "_get_browserless_client", return_value=client), patch.object(browserless_tools, "_get_tool_config", return_value=None), patch.object(browserless_tools, "validate_public_http_url", return_value=None), + patch.object(browserless_tools, "_validate_backend_base_url", return_value=None), ): return asyncio.run(browserless_tools.web_fetch_tool.ainvoke("https://example.org/x")) @@ -333,6 +334,7 @@ def _render_crawl4ai(markdown: str) -> str: patch.object(crawl4ai_tools, "_build_client", return_value=client), patch.object(crawl4ai_tools, "_get_tool_config", return_value=None), patch.object(crawl4ai_tools, "validate_public_http_url", return_value=None), + patch.object(crawl4ai_tools, "_validate_backend_base_url", return_value=None), ): return asyncio.run(crawl4ai_tools.web_fetch_tool.ainvoke("https://example.org/x")) diff --git a/backend/tests/test_web_fetch_relative_links.py b/backend/tests/test_web_fetch_relative_links.py index 2a67b5a71b..787e13453c 100644 --- a/backend/tests/test_web_fetch_relative_links.py +++ b/backend/tests/test_web_fetch_relative_links.py @@ -30,8 +30,9 @@ async def test_web_fetch_resolves_relative_links_through_real_extraction(monkeyp monkeypatch.setattr(module, "_get_infoquest_client", lambda: SimpleNamespace(fetch=AsyncMock(return_value=html))) else: client = SimpleNamespace(fetch_html_with_status=AsyncMock(return_value=BrowserlessFetchResult(html, "200", "OK"))) - monkeypatch.setattr(module, "_get_browserless_client", lambda name: client) + monkeypatch.setattr(module, "_get_browserless_client", lambda name, cfg: client) monkeypatch.setattr(module, "_resolve_host_addresses", lambda host: [ip_address("93.184.216.34")]) + monkeypatch.setattr(module, "_validate_backend_base_url", lambda cfg: None) result = await module.web_fetch_tool.ainvoke({"url": PAGE_URL}) assert "[Next](https://example.com/next)" in result assert "[Reference](https://example.com/reference)" in result diff --git a/config.example.yaml b/config.example.yaml index 91b8b01423..87e331dd4a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1059,6 +1059,10 @@ tools: # # token: $BROWSERLESS_TOKEN # API token (required for Browserless Cloud; optional for self-hosted) # timeout_s: 30 # Request timeout in seconds # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets. + # # network_isolation_confirmed: false # SSRF guard: the Browserless server resolves URLs, follows redirects, + # # # and loads subresources on its own side, so the target-URL guard above + # # # cannot be enforced end-to-end. Set true ONLY when its outbound network + # # # is isolated from private and cloud-metadata networks. # # wait_for_event: "networkidle" # Wait for a page event before returning (e.g. "load", "networkidle") # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds # # wait_for_selector: "article" # CSS selector to wait for before returning @@ -1089,6 +1093,10 @@ tools: # token: $CRAWL4AI_TOKEN # Bearer token; required by Crawl4AI >= 0.9 # timeout: 30 # Request timeout in seconds # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets. + # # network_isolation_confirmed: false # SSRF guard: the Crawl4AI server resolves URLs, follows redirects, + # # # and loads subresources on its own side, so the target-URL guard above + # # # cannot be enforced end-to-end. Set true ONLY when its outbound network + # # # is isolated from private and cloud-metadata networks. # # filter: fit # Markdown filter: fit (default) | raw | bm25 | llm # Web capture tool (uses Browserless /screenshot to render a page as an artifact) @@ -1113,6 +1121,11 @@ tools: # # best_attempt: true # Continue with current page state if waits time out # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY to # # # capture internal/private targets (loopback, RFC1918, etc.) + # # network_isolation_confirmed: false # SSRF guard: the Browserless server resolves URLs, follows + # # # redirects, and loads subresources on its own side, so the + # # # target-URL guard above cannot be enforced end-to-end. Set true + # # # ONLY when its outbound network is isolated from private and + # # # cloud-metadata networks. # Agentic browser control (stateful navigate → observe → click/type loop). # Unlike web_fetch/web_capture (read-only, stateless), these tools keep a live @@ -1214,6 +1227,11 @@ tools: # use: deerflow.community.firecrawl.tools:web_fetch_tool # # api_key: $FIRECRAWL_API_KEY # # base_url: http://:3002 # optional: self-hosted Firecrawl; no cloud API key needed + # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets. + # # network_isolation_confirmed: false # SSRF guard: the Firecrawl server resolves URLs, follows redirects, + # # # and loads subresources on its own side, so the target-URL guard above + # # # cannot be enforced end-to-end. Set true ONLY when its outbound network + # # # is isolated from private and cloud-metadata networks. # Web fetch tool (uses GroundRoute, requires GROUNDROUTE_API_KEY) # Fetches a page's extracted text via GroundRoute mode=page. @@ -1234,6 +1252,10 @@ tools: # # api_key: $CRW_API_KEY # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets. + # # network_isolation_confirmed: false # SSRF guard: the fastCRW server resolves URLs, follows redirects, + # # # and loads subresources on its own side, so the target-URL guard above + # # # cannot be enforced end-to-end. Set true ONLY when its outbound network + # # # is isolated from private and cloud-metadata networks. # Web fetch tool (uses Sofya, requires SOFYA_API_KEY) # Returns the page as markdown. Also handles PDF, DOCX and other documents. diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 30fcd5a63c..f93131833a 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -124,6 +124,14 @@ Gateway's error, and the config-dependent checks skip. Any failure to import the harness is reported, never raised: doctor diagnoses broken environments. Pinned by `backend/tests/test_doctor.py::TestMainConfigResolution`. +Doctor screens Browserless fetch/capture and Crawl4AI, Firecrawl, and fastCRW +fetch backends with the runtime's `validate_delegated_backend_url`, before +provider success shortcuts. Keep endpoint defaults, `CRW_API_URL` precedence, +config environment resolution, and isolation acknowledgement coercion aligned +with those tools. `allow_private_addresses` affects targets only. Doctor reports +refused delegation with the deployment guide; it does not verify egress policies. +Offline coverage lives in `backend/tests/test_doctor.py`. + CLI credential JSON checks accept UTF-8 with or without a leading BOM, matching the runtime credential loader. Keep `_load_json_object` on `utf-8-sig`; malformed JSON and invalid encoding remain missing/invalid sources without exposing tokens. diff --git a/scripts/doctor.py b/scripts/doctor.py index 3b51416847..131602c417 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -650,6 +650,53 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes base_url = str(tool.get("base_url") or "http://localhost:3032").lower() return "browserless.io" not in base_url + def _delegated_backend_error(tool: dict) -> tuple[str, str] | None: + if tool_name not in {"web_fetch", "web_capture"}: + return None + providers = { + "browserless": ("Browserless", "http://localhost:3032"), + "crawl4ai": ("Crawl4AI", "http://localhost:11235"), + "firecrawl": ("Firecrawl", None), + "fastcrw": ("fastCRW", "https://fastcrw.com/api"), + } + use = str(tool.get("use") or "") + for provider, (provider_label, default_url) in providers.items(): + if not use.startswith(f"deerflow.community.{provider}."): + continue + if tool_name == "web_capture" and provider != "browserless": + return None + + from deerflow.community.url_safety import validate_delegated_backend_url + from deerflow.config.app_config import AppConfig + + # Resolve only backend settings: credential diagnostics below + # still need the literal values and environment references. + cfg = AppConfig.resolve_env_variables({key: tool[key] for key in ("base_url", "network_isolation_confirmed") if key in tool}) + base_url = cfg.get("base_url", default_url) + if provider == "fastcrw" and cfg.get("base_url") is None: + base_url = os.environ.get("CRW_API_URL", default_url) + if provider == "firecrawl" and base_url is None: + # The SDK uses its public cloud endpoint when unset. + return None + confirmed = cfg.get("network_isolation_confirmed") + confirmed = confirmed if isinstance(confirmed, bool) else isinstance(confirmed, str) and confirmed.strip().lower() in {"1", "true", "yes", "on"} + error = validate_delegated_backend_url(base_url, network_isolation_confirmed=confirmed) + return (provider_label, error) if error else None + return None + + # Check every delegated backend before any provider's early success. + # allow_private_addresses permits targets, not backend delegation. + for tool in tool_entries: + backend_error = _delegated_backend_error(tool) + if backend_error: + provider, error = backend_error + return CheckResult( + label, + "warn", + f"{provider} backend: {error}", + fix="Configure an HTTP(S) backend URL; for a private or unverifiable backend, isolate its egress before setting network_isolation_confirmed: true. See backend/docs/CONFIGURATION.md#delegated-fetch-backend-isolation", + ) + for tool in tool_entries: use = tool.get("use", "") for provider, detail in free_providers.get(tool_name, {}).items():