diff --git a/.env.example b/.env.example index 87520bcd64..d4ce890df8 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,13 @@ SERPER_API_KEY=your-serper-api-key # The configured API key is sent in X-API-KEY to this host; use a trusted provider. SERPER_BASE_URL=https://google.serper.dev +# Exa API Key - https://exa.ai +EXA_API_KEY=your-exa-api-key + +# Optional operator-controlled Exa-compatible API base URL for the Exa web_search and web_fetch tools. +# The configured API key is sent in x-api-key to this host; use a trusted provider. +EXA_BASE_URL=https://api.exa.ai + # Serply API Key (Google Search, News and Scholar) - https://serply.io SERPLY_API_KEY=your-serply-api-key diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index c0b0839597..61187d0ea3 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -1126,6 +1126,34 @@ is resolved once before transport so it can remain constant across retry attempt Override debug diagnostics show the effective endpoint without URL credentials, query or fragment; result-URL guards do not restrict the operator's API host. +#### Exa endpoints + +To use an Exa-compatible provider for the Exa `web_search` and `web_fetch` +tools, set `EXA_BASE_URL` to its base URL (without `/search` or `/contents`) +and `EXA_API_KEY` to that provider's key. Each tool entry can instead set its +own `base_url`; a non-blank string there takes precedence over `EXA_BASE_URL`. +When neither is set, the SDK default `https://api.exa.ai` is used. + +```yaml +tools: + - name: web_search + group: web + use: deerflow.community.exa.tools:web_search_tool + base_url: https://exa-compatible.example + api_key: $EXA_COMPATIBLE_API_KEY +``` + +Overrides must be absolute `http://` or `https://` URLs with a host and valid +port, without a query string or fragment. Surrounding whitespace and trailing +slashes are removed. An invalid override returns a configuration error that +does not echo the configured value and does not fall back to another host. The +key is sent to the configured host in the `x-api-key` header. + +The endpoint fetches pages for `web_fetch` in its own network, so `web_fetch` +screens a configured base URL like the other delegated fetch backends: a host +that resolves to a loopback, private or unverifiable address is refused unless +the tool entry sets `network_isolation_confirmed: true`. + #### Serper source filters ```yaml @@ -1946,6 +1974,8 @@ models: - `BRAVE_SEARCH_API_KEY` - Brave Search API key for `web_search` and `image_search` - `SERPER_API_KEY` - Serper (Google Search/Images API) key for `web_search` and `image_search` - `SERPER_BASE_URL` - Optional operator-controlled Serper-compatible base URL for both tools; each tool's `base_url` takes precedence. The provider key is sent to the configured host in `X-API-KEY`, never a query parameter. See [Serper endpoints](#serper-endpoints). +- `EXA_API_KEY` - [Exa](https://exa.ai) key for `web_search` and `web_fetch` +- `EXA_BASE_URL` - Optional operator-controlled Exa-compatible base URL for both Exa tools; each tool's `base_url` takes precedence. See [Exa endpoints](#exa-endpoints). - `SERPLY_API_KEY` - [Serply](https://serply.io) key for `web_search` (Google Search, plus Google News and Google Scholar via `vertical`) - `GROUNDROUTE_API_KEY` - GroundRoute meta-search API key for `web_search` and `web_fetch` (routes across Serper, Brave, Exa, Tavily, Firecrawl, Perplexity with gain-share pricing) - `SOFYA_API_KEY` - [Sofya](https://sofya.co) key for `web_search` and `web_fetch` diff --git a/backend/packages/harness/deerflow/community/exa/tools.py b/backend/packages/harness/deerflow/community/exa/tools.py index de0c8f0a5d..9ac4f0899b 100644 --- a/backend/packages/harness/deerflow/community/exa/tools.py +++ b/backend/packages/harness/deerflow/community/exa/tools.py @@ -1,10 +1,12 @@ import json import logging +import os +from urllib.parse import urlparse from exa_py import Exa from langchain.tools import tool -from deerflow.community.url_safety import safe_public_result_url +from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url from deerflow.config import get_app_config logger = logging.getLogger(__name__) @@ -35,12 +37,55 @@ def _coerce_positive_int(value: object, default: int, option: str) -> int: return count +def _coerce_bool(value: object, default: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return default + + +def _resolve_base_url(extra: dict) -> str | None: + """Return the configured Exa-compatible base URL, or ``None`` for the SDK default. + + A non-blank string ``base_url`` in the tool entry takes precedence over + ``EXA_BASE_URL``. Overrides must be absolute HTTP(S) URLs with a host, a + valid port, and no query or fragment; an invalid one raises without echoing + the configured value and never falls back to another host. + """ + base_url = extra.get("base_url") + if not isinstance(base_url, str) or not base_url.strip(): + base_url = os.getenv("EXA_BASE_URL") or "" + base_url = base_url.strip() + if not base_url: + return None + try: + parsed = urlparse(base_url) + valid = parsed.scheme in {"http", "https"} and bool(parsed.netloc and parsed.hostname) and "?" not in base_url and "#" not in base_url + # Accessing port also rejects invalid and out-of-range ports. + parsed.port + except ValueError: + valid = False + if not valid: + raise ValueError("Invalid Exa base_url/EXA_BASE_URL: use an absolute HTTP(S) URL with a host, a valid port, and no query or fragment") + return base_url.rstrip("/") + + def _get_exa_client(tool_name: str = "web_search") -> Exa: config = get_app_config().get_tool_config(tool_name) - api_key = None - if config is not None and "api_key" in config.model_extra: - api_key = config.model_extra.get("api_key") - return Exa(api_key=api_key) + extra = config.model_extra if config is not None else {} + return _build_exa_client(extra, _resolve_base_url(extra)) + + +def _build_exa_client(extra: dict, base_url: str | None) -> Exa: + kwargs = {"api_key": extra.get("api_key")} + if base_url: + kwargs["base_url"] = base_url + return Exa(**kwargs) @tool("web_search", parse_docstring=True) @@ -98,7 +143,18 @@ def web_fetch_tool(url: str) -> str: url: The URL to fetch the contents of. """ try: - client = _get_exa_client("web_fetch") + config = get_app_config().get_tool_config("web_fetch") + extra = config.model_extra if config is not None else {} + base_url = _resolve_base_url(extra) + if base_url: + # A custom backend fetches the URL in its own network, like other delegated fetch services. + backend_error = validate_delegated_backend_url( + base_url, + network_isolation_confirmed=_coerce_bool(extra.get("network_isolation_confirmed"), False), + ) + if backend_error: + return backend_error + client = _build_exa_client(extra, base_url) res = client.get_contents([url], text={"max_characters": 4096}) if res.results: diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index 28a14659e9..a4fea22582 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -101,7 +101,7 @@ Durable MCP task-management tools are added only while the process-local task su - `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest). Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction. -The shared SSRF guard (`community/url_safety.py`) resolves hostnames with blocking `socket.getaddrinfo`, so async tools, Gateway routes, and the Playwright request guard (`BrowserSession._install_request_guard`, which runs on the shared browser loop) call `validate_public_http_url` (or a wrapper such as `validate_browser_url`) through `asyncio.to_thread`. The guard checks at validation time only, so a launched browser sends every TCP connection (HTTP, HTTPS, WebSocket) through its per-session loopback SOCKS5 proxy (`browser_automation/egress.py`, `bypass: <-loopback>`): Chromium hands it every hostname, and it resolves once via `resolve_browser_egress` → `url_safety.resolve_public_addresses` and connects to exactly those addresses, closing the DNS-rebinding window (`mcp/personal_network.py` pins the same way). Every `acquire_session`/`get_session` caller must pass `egress_resolver`, because a thread's first caller fixes its launch options. WebRTC UDP does not traverse a SOCKS proxy and is not pinned (out of scope, so this is not a complete egress guarantee). Delegated fetch services (Browserless, crawl4ai, fastcrw, firecrawl) resolve on their own side and cannot be pinned, so each `web_fetch`/`web_capture` entry point instead screens its own backend base URL with `validate_delegated_backend_url` and fails closed on a self-hosted backend unless the operator confirms the backend's egress is isolated via `network_isolation_confirmed`; CDP-attached Chrome has neither guard. The strict blocking-IO gate carries a `socket.getaddrinfo` rule for this; `tests/blocking_io/test_web_tool_url_validation.py` drives each web tool entry point, the request guard, and the egress proxy; `tests/test_browser_egress.py` pins the proxy protocol, the session/manager/router wiring, and (with Playwright installed) a real-Chromium rebinding case. +The shared SSRF guard (`community/url_safety.py`) resolves hostnames with blocking `socket.getaddrinfo`, so async tools, Gateway routes, and the Playwright request guard (`BrowserSession._install_request_guard`, which runs on the shared browser loop) call `validate_public_http_url` (or a wrapper such as `validate_browser_url`) through `asyncio.to_thread`. The guard checks at validation time only, so a launched browser sends every TCP connection (HTTP, HTTPS, WebSocket) through its per-session loopback SOCKS5 proxy (`browser_automation/egress.py`, `bypass: <-loopback>`): Chromium hands it every hostname, and it resolves once via `resolve_browser_egress` → `url_safety.resolve_public_addresses` and connects to exactly those addresses, closing the DNS-rebinding window (`mcp/personal_network.py` pins the same way). Every `acquire_session`/`get_session` caller must pass `egress_resolver`, because a thread's first caller fixes its launch options. WebRTC UDP does not traverse a SOCKS proxy and is not pinned (out of scope, so this is not a complete egress guarantee). Delegated fetch services (Browserless, crawl4ai, fastcrw, firecrawl, and Exa with a configured `base_url`) resolve on their own side and cannot be pinned, so each `web_fetch`/`web_capture` entry point instead screens its own backend base URL with `validate_delegated_backend_url` and fails closed on a self-hosted backend unless the operator confirms the backend's egress is isolated via `network_isolation_confirmed`; CDP-attached Chrome has neither guard. The strict blocking-IO gate carries a `socket.getaddrinfo` rule for this; `tests/blocking_io/test_web_tool_url_validation.py` drives each web tool entry point, the request guard, and the egress proxy; `tests/test_browser_egress.py` pins the proxy protocol, the session/manager/router wiring, and (with Playwright installed) a real-Chromium rebinding case. Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`, `youcom`); see each subpackage for specifics. Browserless authenticates only from the `token` query parameter or `Authorization` header and rejects unknown body keys, so `BrowserlessClient._auth_params()` carries the token for every endpoint; never add it to the JSON body. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. diff --git a/backend/tests/test_exa_tools.py b/backend/tests/test_exa_tools.py index bd1778cbc5..300db87c37 100644 --- a/backend/tests/test_exa_tools.py +++ b/backend/tests/test_exa_tools.py @@ -304,3 +304,88 @@ class TestWebFetchTool: # "# Long Page\n\n" is 14 chars, content truncated to 4096 content_after_header = result.split("\n\n", 1)[1] assert len(content_after_header) == 4096 + + +@pytest.fixture(autouse=True) +def _no_exa_base_url_env(monkeypatch): + monkeypatch.delenv("EXA_BASE_URL", raising=False) + + +def _patch_tool_config(extra): + patcher = patch("deerflow.community.exa.tools.get_app_config") + mock_config = patcher.start() + tool_config = MagicMock() + tool_config.model_extra = extra + mock_config.return_value.get_tool_config.return_value = tool_config + return patcher + + +class TestBaseUrl: + def _search(self, extra): + patcher = _patch_tool_config(extra) + try: + with patch("deerflow.community.exa.tools.Exa") as mock_exa_cls: + mock_exa_cls.return_value.search.return_value = MagicMock(results=[]) + from deerflow.community.exa.tools import web_search_tool + + result = web_search_tool.invoke({"query": "q"}) + return result, mock_exa_cls + finally: + patcher.stop() + + def test_default_endpoint_when_unset(self): + result, mock_exa_cls = self._search({"api_key": "k"}) + assert result == "[]" + mock_exa_cls.assert_called_once_with(api_key="k") + + def test_tool_base_url_is_passed_to_sdk_without_trailing_slash(self, monkeypatch): + monkeypatch.setenv("EXA_BASE_URL", "https://env.example") + result, mock_exa_cls = self._search({"api_key": "k", "base_url": " https://exa-compatible.example/api/ "}) + assert result == "[]" + mock_exa_cls.assert_called_once_with(api_key="k", base_url="https://exa-compatible.example/api") + + @pytest.mark.parametrize("raw", [None, "", " ", 123]) + def test_env_base_url_is_used_when_tool_value_missing(self, monkeypatch, raw): + monkeypatch.setenv("EXA_BASE_URL", "https://env.example/") + extra = {"api_key": "k"} if raw is None else {"api_key": "k", "base_url": raw} + result, mock_exa_cls = self._search(extra) + assert result == "[]" + mock_exa_cls.assert_called_once_with(api_key="k", base_url="https://env.example") + + @pytest.mark.parametrize( + "raw", + ["api.example.com", "ftp://api.example.com", "https://", "https://api.example.com?x=1", "https://api.example.com/#", "https://api.example.com:99999"], + ) + def test_invalid_base_url_returns_error_without_value(self, raw): + result, mock_exa_cls = self._search({"api_key": "k", "base_url": raw}) + assert result.startswith("Error: Invalid Exa base_url/EXA_BASE_URL") + assert raw not in result + mock_exa_cls.assert_not_called() + + def _fetch(self, extra): + patcher = _patch_tool_config(extra) + try: + with patch("deerflow.community.exa.tools.Exa") as mock_exa_cls: + mock_exa_cls.return_value.get_contents.return_value = MagicMock(results=[MagicMock(title="Page", text="Body")]) + from deerflow.community.exa.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + return result, mock_exa_cls + finally: + patcher.stop() + + def test_fetch_refuses_private_backend_without_isolation_confirmation(self): + result, mock_exa_cls = self._fetch({"api_key": "k", "base_url": "http://127.0.0.1:8080"}) + assert result.startswith("Error:") + assert "network_isolation_confirmed" in result + mock_exa_cls.assert_not_called() + + def test_fetch_allows_private_backend_with_isolation_confirmed(self): + result, mock_exa_cls = self._fetch({"api_key": "k", "base_url": "http://127.0.0.1:8080", "network_isolation_confirmed": "true"}) + assert result == "# Page\n\nBody" + mock_exa_cls.assert_called_once_with(api_key="k", base_url="http://127.0.0.1:8080") + + def test_fetch_allows_public_backend(self): + result, mock_exa_cls = self._fetch({"api_key": "k", "base_url": "https://93.184.216.34"}) + assert result == "# Page\n\nBody" + mock_exa_cls.assert_called_once_with(api_key="k", base_url="https://93.184.216.34") diff --git a/config.example.yaml b/config.example.yaml index d34c1c2513..89774dc823 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1026,6 +1026,7 @@ tools: # search_type: auto # Options: auto, neural, keyword # contents_max_characters: 1000 # # api_key: $EXA_API_KEY + # # base_url: https://api.exa.ai # Exa-compatible endpoint; overrides EXA_BASE_URL # Web search tool (uses Firecrawl. Cloud requires FIRECRAWL_API_KEY; self-host may need no key.) # - name: web_search @@ -1231,6 +1232,7 @@ tools: # group: web # use: deerflow.community.exa.tools:web_fetch_tool # # api_key: $EXA_API_KEY + # # base_url: https://api.exa.ai # Exa-compatible endpoint; overrides EXA_BASE_URL # Web fetch tool (uses Jina AI reader) - name: web_fetch diff --git a/frontend/src/content/en/harness/tools.mdx b/frontend/src/content/en/harness/tools.mdx index 2589287be7..a6ab242872 100644 --- a/frontend/src/content/en/harness/tools.mdx +++ b/frontend/src/content/en/harness/tools.mdx @@ -161,7 +161,7 @@ tools: - use: deerflow.community.exa.tools:web_search_tool api_key: $EXA_API_KEY ``` -Semantic search with neural retrieval. Requires an [Exa](https://exa.ai) API key. +Semantic search with neural retrieval. Requires an [Exa](https://exa.ai) API key. Set `base_url` (or `EXA_BASE_URL`) to use an Exa-compatible endpoint. No extra dependency required. diff --git a/frontend/src/content/zh/harness/tools.mdx b/frontend/src/content/zh/harness/tools.mdx index ec7cfc66c1..771439284b 100644 --- a/frontend/src/content/zh/harness/tools.mdx +++ b/frontend/src/content/zh/harness/tools.mdx @@ -158,7 +158,7 @@ tools: - use: deerflow.community.exa.tools:web_search_tool api_key: $EXA_API_KEY ``` -带神经检索的语义搜索。需要 [Exa](https://exa.ai) API Key。 +带神经检索的语义搜索。需要 [Exa](https://exa.ai) API Key。设置 `base_url`（或 `EXA_BASE_URL`）可使用兼容 Exa 的端点。 无需额外依赖。