fix(tools): bound deferred catalog regex execution (#6676)

* fix(tools): bound deferred catalog regex execution

* fix(tools): clarify deferred regex limit diagnostics

README.md

@@ -856,6 +856,8 @@ When deferred MCP tool discovery is enabled, `tool_search` supports opt-in liter
 
 MCP routing hints can also prefer a specific MCP tool for matching requests without forbidding other tools. When `tool_search` defers MCP schemas, matching routing metadata can auto-promote up to `tool_search.auto_promote_top_k` deferred schemas before the model call.
 
+Deferred `tool_search` regex queries share a 100 ms execution budget across the catalog, including `+required` ranking. Patterns are limited to 256 characters and searchable names/descriptions to 65,536 characters each. Queries that exceed a limit return a retry hint without promoting partial results; use a simpler pattern, `keywords:`, or `select:` with exact names. Oversized catalog fields are reported with the tool name, field and length, and logged with the MCP server name when available; use `keywords:` or `select:` because simplifying the pattern cannot shrink a catalog field. Exact selection remains uncapped and bypasses regex limits; the literal `keywords:` mode also bypasses regex limits. Invalid regex syntax still falls back to a literal substring match.
+
 OpenViking users can register the official Streamable HTTP endpoint at `/mcp`
 with an owner-bound USER API key. The native `forget` tool is exposed for
 capability parity; deletion is irreversible, so it should be called only after

backend/packages/harness/deerflow/tools/AGENTS.md

@@ -7,6 +7,21 @@ closers paired with openers inside the detected reference (including literal
 glob directory names). Structured references remain literal. See
 [tool artifact middleware](../agents/middlewares/TOOL_ARTIFACTS.md).
 
+Deferred-tool regex discovery uses `regex.VERSION0` with a shared 100 ms
+deadline per ranked query, including required-name scoring. Validate the bounded
+pattern with stdlib `re` first to preserve the invalid-regex literal fallback
+and avoid admitting engine-specific recursion syntax. Compile once per query;
+bound each searchable field before concatenation and count matches with an
+iterator. Budget failure raises `ToolSearchLimitError`; the tool returns a retry
+hint and no promotions, never a partial ranking. `select:` and the literal
+`keywords:` path bypass these regex limits.
+Field-limit errors identify the tool (name preview capped at 128 characters),
+field and length, and direct the model to `keywords:` or `select:` rather than
+query simplification. Log the same bounded identity with the logical MCP server
+name when available; never log the oversized description. Keep the model-visible
+tool description aligned with these limits. Pathological-regex subprocess tests
+allow 30 seconds for cold startup while matching retains its 100 ms budget.
+
 `task` and `batch_task` opt into JSON checks with `file:<path> json-valid`.
 See [subagents/AGENTS.md](../subagents/AGENTS.md) for read limits and UNVERIFIED semantics.
 Batch readers bound by assembly use its AppConfig output budget. Fit the complete escaped response under 10K and active per-tool/fallback limits; too-small envelopes stop reading. Numeric reader inputs are strict integers. Escape `<` in outer JSON to preserve untrusted data without exposing framework tags; decoding restores the document unchanged.

backend/packages/harness/deerflow/tools/builtins/tool_search.py

@@ -24,15 +24,17 @@ import re
 from collections.abc import Iterable
 from dataclasses import dataclass
 from functools import cached_property
+from time import monotonic
 from typing import TYPE_CHECKING, Annotated, Any
 
+import regex
 from langchain.tools import BaseTool
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import InjectedToolCallId, tool
 from langchain_core.utils.function_calling import convert_to_openai_function
 from langgraph.types import Command
 
-from deerflow.tools.mcp_metadata import get_mcp_routing, is_mcp_tool
+from deerflow.tools.mcp_metadata import get_mcp_routing, get_mcp_source, is_mcp_tool
 
 if TYPE_CHECKING:
     from langchain.agents.middleware import AgentMiddleware
@@ -40,20 +42,65 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 MAX_RESULTS = 5  # Max tools returned per search
+MAX_REGEX_PATTERN_CHARS = 256
+MAX_REGEX_FIELD_CHARS = 65_536
+REGEX_QUERY_TIMEOUT_SECONDS = 0.1
 MAX_KEYWORD_QUERY_CHARS = 256
 MAX_KEYWORD_QUERY_TERMS = 16
 
 
-def _compile_catalog_regex(pattern: str) -> re.Pattern[str]:
+class ToolSearchLimitError(ValueError):
+    """The query exceeded the discovery budget; no partial ranking is valid."""
+
+    def __init__(self, message: str, *, retry_hint: str = "Use a simpler query, keywords: for literal search, or select: with exact tool names."):
+        super().__init__(message)
+        self.retry_hint = retry_hint
+
+
+def _compile_catalog_regex(pattern: str) -> regex.Pattern[str]:
     """Compile ``pattern`` case-insensitively, falling back to a literal match.
 
     Search queries come from the model, so an invalid regex (e.g. an unbalanced
     paren) must degrade to a literal substring match rather than raise.
     """
+    if len(pattern) > MAX_REGEX_PATTERN_CHARS:
+        raise ToolSearchLimitError(f"Regex pattern exceeds the {MAX_REGEX_PATTERN_CHARS}-character budget.")
     try:
-        return re.compile(pattern, re.IGNORECASE)
+        # Keep Python re's accepted syntax and invalid-pattern literal fallback;
+        # do not accidentally enable regex-only features such as recursion.
+        re.compile(pattern, re.IGNORECASE)
     except re.error:
-        return re.compile(re.escape(pattern), re.IGNORECASE)
+        pattern = re.escape(pattern)
+    except (RecursionError, OverflowError) as exc:
+        raise ToolSearchLimitError("Regex compilation exceeds the supported query budget.") from exc
+    try:
+        return regex.compile(pattern, regex.IGNORECASE | regex.VERSION0)
+    except (regex.error, RecursionError, OverflowError) as exc:
+        raise ToolSearchLimitError("Regex compilation exceeds the supported query budget.") from exc
+
+
+def _remaining_regex_time(deadline: float) -> float:
+    remaining = deadline - monotonic()
+    if remaining <= 0:
+        raise ToolSearchLimitError("Regex search exceeded its execution budget.")
+    return remaining
+
+
+def _catalog_searchable(t: BaseTool) -> str:
+    description = t.description or ""
+    for field, value in (("name", t.name), ("description", description)):
+        if len(value) > MAX_REGEX_FIELD_CHARS:
+            # Names and source labels are external data; keep diagnostics bounded
+            # even when the name itself is the oversized field.
+            tool_label = repr(t.name[:128]) + (" (truncated)" if len(t.name) > 128 else "")
+            source = get_mcp_source(t)
+            server_name = source["server_name"][:128] if source else None
+            logger.warning("Deferred tool regex field limit: tool=%s field=%s length=%d limit=%d server=%r", tool_label, field, len(value), MAX_REGEX_FIELD_CHARS, server_name)
+            raise ToolSearchLimitError(
+                f"Tool {tool_label} {field} has {len(value)} characters, exceeding the {MAX_REGEX_FIELD_CHARS}-character regex search budget.",
+                retry_hint="Use keywords: for literal search or select: with exact tool names from <available-deferred-tools>.",
+            )
+    return f"{t.name} {description}"
 
 
 # ── Catalog ──
@@ -101,16 +148,27 @@ class DeferredToolCatalog:
             required = parts[0].lower()
             candidates = [t for t in self.tools if required in t.name.lower()]
             if len(parts) > 1:
-                candidates.sort(key=lambda t: _catalog_regex_score(parts[1], t), reverse=True)
+                deadline = monotonic() + REGEX_QUERY_TIMEOUT_SECONDS
+                compiled = _compile_catalog_regex(parts[1])
+                try:
+                    candidates.sort(key=lambda t: _catalog_regex_score(compiled, t, deadline), reverse=True)
+                except TimeoutError as exc:
+                    raise ToolSearchLimitError("Regex search exceeded its execution budget.") from exc
+                _remaining_regex_time(deadline)
             return candidates[:MAX_RESULTS]
 
-        regex = _compile_catalog_regex(query)
+        deadline = monotonic() + REGEX_QUERY_TIMEOUT_SECONDS
+        compiled = _compile_catalog_regex(query)
         scored: list[tuple[int, BaseTool]] = []
-        for t in self.tools:
-            searchable = f"{t.name} {t.description or ''}"
-            if regex.search(searchable):
-                scored.append((2 if regex.search(t.name) else 1, t))
+        try:
+            for t in self.tools:
+                searchable = _catalog_searchable(t)
+                if compiled.search(searchable, timeout=_remaining_regex_time(deadline)):
+                    scored.append((2 if compiled.search(t.name, timeout=_remaining_regex_time(deadline)) else 1, t))
+        except TimeoutError as exc:
+            raise ToolSearchLimitError("Regex search exceeded its execution budget.") from exc
         scored.sort(key=lambda x: x[0], reverse=True)
+        _remaining_regex_time(deadline)
         return [t for _, t in scored][:MAX_RESULTS]
 
 
@@ -136,9 +194,15 @@ def _rank_catalog_keywords(query: str, tools: tuple[BaseTool, ...]) -> list[Base
     return [candidate for _, candidate in scored[:MAX_RESULTS]]
 
 
-def _catalog_regex_score(pattern: str, t: BaseTool) -> int:
-    regex = _compile_catalog_regex(pattern)
-    return len(regex.findall(f"{t.name} {t.description or ''}"))
+def _catalog_regex_score(compiled: regex.Pattern[str], t: BaseTool, deadline: float) -> int:
+    # Count without materializing every captured substring in findall(). The
+    # same deadline covers every candidate and every match, including empty ones.
+    matches = compiled.finditer(_catalog_searchable(t), timeout=_remaining_regex_time(deadline))
+    count = 0
+    for _ in matches:
+        _remaining_regex_time(deadline)
+        count += 1
+    return count
 
 
 # ── Setup / tool ──
@@ -187,13 +251,20 @@ def build_tool_search_tool(catalog: DeferredToolCatalog) -> BaseTool:
           - "jupyter.*notebook" -- regex on name and description, name matches first
           - "+slack send" -- require "slack" in the name, rank by the remaining regex
         Queries without a prefix keep regex semantics (invalid regex is literal).
+        Regex patterns are capped at 256 characters; each name or description
+        at 65,536 characters. Regex searches share a 100 ms budget across candidates
+        and ranking. Prefer keywords: or select: for broad catalogs.
         """
-        matched = catalog.search(query)
-        if not matched:
-            content, names = f"No tools found matching: {query}", []
+        try:
+            matched = catalog.search(query)
+        except ToolSearchLimitError as exc:
+            content, names = f"Tool search stopped: {exc} {exc.retry_hint}", []
         else:
-            content = json.dumps([convert_to_openai_function(t) for t in matched], indent=2, ensure_ascii=False)
-            names = [t.name for t in matched]
+            if matched:
+                content = json.dumps([convert_to_openai_function(t) for t in matched], indent=2, ensure_ascii=False)
+                names = [t.name for t in matched]
+            else:
+                content, names = f"No tools found matching: {query}", []
         return Command(
             update={
                 "promoted": {"catalog_hash": catalog_hash, "names": names},

backend/packages/harness/pyproject.toml

@@ -40,6 +40,7 @@ dependencies = [
     "pydantic>=2.12.5",
     "pyyaml>=6.0.3",
     "readabilipy>=0.3.0",
+    "regex>=2026.4.4",
     "tavily-python>=0.7.17",
     "firecrawl-py>=1.15.0",
     "tiktoken>=0.8.0",

backend/tests/test_deferred_catalog_regex_limits.py

@@ -0,0 +1,144 @@
+"""Untrusted regex queries must not monopolize deferred-tool discovery."""
+
+import re
+import subprocess
+import sys
+
+import pytest
+from langchain_core.tools import StructuredTool
+
+from deerflow.tools.builtins import tool_search as search_module
+from deerflow.tools.builtins.tool_search import MAX_REGEX_FIELD_CHARS, MAX_REGEX_PATTERN_CHARS, DeferredToolCatalog, ToolSearchLimitError, build_tool_search_tool
+from deerflow.tools.mcp_metadata import tag_mcp_tool
+
+
+def make_tool(name="aaa_tool", description="ordinary"):
+    return StructuredTool.from_function(lambda: None, name=name, description=description)
+
+
+@pytest.mark.parametrize("mode", ["name", "valid_name", "description", "required"])
+def test_pathological_regex_finishes_in_isolated_process(mode):
+    # Allow cold imports on shared runners; the regex work still has a 100 ms
+    # budget. A process deadline protects pytest if the engine regresses.
+    script = """
+import sys
+from langchain_core.tools import StructuredTool
+from deerflow.tools.builtins.tool_search import DeferredToolCatalog
+mode = sys.argv[1]
+name = 'a' * 10000 + ('_' if mode == 'valid_name' else '!') if mode in ('name', 'valid_name') else 'aaa_tool'
+description = 'a' * 10000 + '!' if mode in ('description', 'required') else 'ordinary'
+tool = StructuredTool.from_function(lambda: None, name=name, description=description)
+catalog = DeferredToolCatalog((tool,))
+query = '+aaa (a+)+$' if mode == 'required' else '(a+)+$'
+try:
+    catalog.search(query)
+except ValueError as exc:
+    assert 'budget' in str(exc).lower(), str(exc)
+print('finished')
+"""
+    result = subprocess.run([sys.executable, "-c", script, mode], capture_output=True, text=True, timeout=30, check=False)
+    assert result.returncode == 0, result.stderr
+    assert "finished" in result.stdout
+
+
+@pytest.mark.parametrize("query", ["a" * (MAX_REGEX_PATTERN_CHARS + 1), "+aaa " + "a" * (MAX_REGEX_PATTERN_CHARS + 1)])
+def test_oversized_pattern_rejected_without_truncation(query):
+    with pytest.raises(ToolSearchLimitError, match="character budget"):
+        DeferredToolCatalog((make_tool(),)).search(query)
+
+
+@pytest.mark.parametrize("field", ["name", "description"])
+@pytest.mark.parametrize("query", ["a", "+aaa a"])
+def test_oversized_field_rejected_but_exact_selection_still_works(field, query, caplog):
+    tool = make_tool(**{field: "a" * (MAX_REGEX_FIELD_CHARS + 1)})
+    catalog = DeferredToolCatalog((tool,))
+    with pytest.raises(ToolSearchLimitError, match="regex search budget") as caught:
+        catalog.search(query)
+    assert repr(tool.name[:128]) in str(caught.value)
+    assert field in str(caught.value)
+    assert str(MAX_REGEX_FIELD_CHARS + 1) in str(caught.value)
+    assert len(str(caught.value)) < 512
+    assert len(caplog.text) < 1024
+    assert tool.name[:128] in caplog.text
+    assert field in caplog.text
+    assert catalog.search("select:" + tool.name) == [tool]
+
+
+def test_keyword_search_bypasses_regex_limits(monkeypatch):
+    def reject_regex_path(*args):
+        pytest.fail("Literal keyword search must not enter the regex budget path.")
+
+    monkeypatch.setattr(search_module, "_compile_catalog_regex", reject_regex_path)
+    monkeypatch.setattr(search_module, "monotonic", reject_regex_path)
+    tool = make_tool(description="a" * (MAX_REGEX_FIELD_CHARS + 1))
+    assert DeferredToolCatalog((tool,)).search("keywords:aaa") == [tool]
+
+
+@pytest.mark.parametrize("query", ["a", "+aaa a", "+aaa (?=a)"])
+def test_query_uses_one_budget_across_candidates(monkeypatch, query):
+    # Deterministic elapsed time: the second candidate crosses the deadline.
+    ticks = iter([0, 0.01, 0.02, 0.03, 0.2] + [0.2] * 20)
+    monkeypatch.setattr(search_module, "monotonic", lambda: next(ticks))
+    catalog = DeferredToolCatalog(tuple(make_tool(f"aaa_{i}") for i in range(10)))
+    with pytest.raises(ToolSearchLimitError, match="execution budget"):
+        catalog.search(query)
+
+
+def test_tool_reports_limit_without_promoting_partial_matches():
+    catalog = DeferredToolCatalog((make_tool(), make_tool(description="a" * (MAX_REGEX_FIELD_CHARS + 1))))
+    tool = build_tool_search_tool(catalog)
+    result = tool.invoke({"type": "tool_call", "name": "tool_search", "args": {"query": "aaa"}, "id": "tc-limit"})
+    assert result.update["promoted"] == {"catalog_hash": catalog.hash, "names": []}
+    message = result.update["messages"][0]
+    assert "Tool search stopped:" in message.content
+    assert "select:" in message.content
+    assert message.tool_call_id == "tc-limit"
+
+
+def test_oversized_description_reports_source_and_actionable_retry(caplog):
+    oversized = make_tool("mcp_catalog_lookup", "a" * (MAX_REGEX_FIELD_CHARS + 1))
+    tag_mcp_tool(oversized, server_name="catalog-server", transport="stdio")
+    catalog = DeferredToolCatalog((make_tool(), oversized))
+    tool = build_tool_search_tool(catalog)
+    result = tool.invoke({"type": "tool_call", "name": "tool_search", "args": {"query": "aaa"}, "id": "tc-field-limit"})
+    assert result.update["promoted"]["names"] == []
+    content = result.update["messages"][0].content
+    assert "mcp_catalog_lookup" in content
+    assert "description" in content
+    assert "keywords:" in content
+    assert "select:" in content
+    assert "simpler" not in content
+    assert "catalog-server" in caplog.text
+    assert "mcp_catalog_lookup" in caplog.text
+    assert "description" in caplog.text
+    assert str(MAX_REGEX_FIELD_CHARS + 1) in caplog.text
+    assert str(MAX_REGEX_FIELD_CHARS) in caplog.text
+    assert "a" * 1000 not in caplog.text
+    assert catalog.search("keywords:lookup") == [oversized]
+    assert catalog.search("select:mcp_catalog_lookup") == [oversized]
+
+
+def test_tool_description_exposes_regex_limits_and_alternatives():
+    description = build_tool_search_tool(DeferredToolCatalog((make_tool(),))).description
+    assert "256" in description
+    assert "100 ms" in description
+    assert "65,536" in description
+    assert "keywords:" in description
+    assert "select:" in description
+
+
+@pytest.mark.parametrize("pattern", [r"^aaa", r"ordinary$", r"aaa|bbb", r"(?<=aaa)_", r"(a)\1", r"(?i:AAA)", r"[A-Z]+", r"(?=a)", "sum(", "(?R)"])
+def test_regex_compatibility_matches_stdlib(pattern):
+    tools = (make_tool(), make_tool("bbb", "sum(a)"), make_tool("ccc", "(?R)"))
+    try:
+        compiled = re.compile(pattern, re.IGNORECASE)
+    except re.error:
+        compiled = re.compile(re.escape(pattern), re.IGNORECASE)
+    expected = [(2 if compiled.search(t.name) else 1, t) for t in tools if compiled.search(f"{t.name} {t.description}")]
+    expected.sort(key=lambda item: item[0], reverse=True)
+    assert DeferredToolCatalog(tools).search(pattern) == [t for _, t in expected]
+
+
+def test_required_ranking_preserves_occurrence_counts_and_stable_ties():
+    tools = (make_tool("aaa_first", "bbb"), make_tool("aaa_second", "bbb bbb"), make_tool("aaa_third", "bbb"))
+    assert DeferredToolCatalog(tools).search("+aaa (bbb)") == [tools[1], tools[0], tools[2]]

backend/uv.lock

@@ -969,6 +969,7 @@ dependencies = [
     { name = "pydantic" },
     { name = "pyyaml" },
     { name = "readabilipy" },
+    { name = "regex" },
     { name = "sqlalchemy", extra = ["asyncio"] },
     { name = "tavily-python" },
     { name = "tiktoken" },
@@ -1066,6 +1067,7 @@ requires-dist = [
     { name = "pyyaml", specifier = ">=6.0.3" },
     { name = "readabilipy", specifier = ">=0.3.0" },
     { name = "redis", marker = "extra == 'redis'", specifier = ">=5.0.0" },
+    { name = "regex", specifier = ">=2026.4.4" },
     { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0,<3.0" },
     { name = "tavily-python", specifier = ">=0.7.17" },
     { name = "tenki", marker = "extra == 'tenki'", specifier = ">=1.4.0" },