fix(subagents): build the chat model off the event loop (#7967)

* fix(subagents): build the chat model off the event loop

`SubagentExecutor._create_agent` is awaited from `_aexecute` on the shared
event loop, and it constructed the chat model inline. Provider construction can
read local credentials -- the Codex provider loads the Codex CLI `auth.json`
while it is built -- so every delegated task did synchronous file IO on the loop
before its first model call, which stalls every concurrent turn sharing that
loop.

Build the model through `run_assembly`, the dedicated assembly pool the other
provider-construction sites already use, and leave the rest of the middleware
and graph assembly where it is. The helper preserves request context, and a
cancelled construction still never starts an inference.

`tests/blocking_io/test_subagent_model_offloop.py` drives the real
`_create_agent` with a real `CodexChatModel` under the strict Blockbuster gate:
on `main` the credential load fails with `BlockingError(os.stat)`, and reverting
the `run_assembly` call reproduces the same failure.

* docs(subagents): name the isolated subagent loop in the off-loop notes

The blocked loop is the persistent isolated subagent loop every concurrent
delegated task shares, not the Gateway request loop; say so in the boundary
audit, the regression docstring, and the assertion message.

---------

Co-authored-by: Totoro <msy626836554@gmail.com>

backend/packages/harness/deerflow/subagents/executor.py

@@ -61,6 +61,7 @@ from deerflow.tracing import (
     build_tracing_callbacks,
     inject_langfuse_metadata,
 )
+from deerflow.utils.assembly_io import run_assembly
 from deerflow.utils.bash_exit import BASH_TOOL_NAMES, bash_exit_status
 from deerflow.utils.file_io import await_drained
 from deerflow.utils.messages import message_content_to_text
@@ -1141,7 +1142,10 @@ class SubagentExecutor:
         except ReasoningPolicyError as exc:
             raise ValueError(f"Model {self.model_name}: {exc}") from exc
         # Preserve the original request for the factory's validation and logs.
-        model = create_chat_model(
+        # Provider construction can read local credentials (the Codex provider
+        # loads the CLI auth.json), so keep the build off the event loop.
+        model = await run_assembly(
+            create_chat_model,
             name=self.model_name,
             thinking_enabled=self.thinking_enabled,
             reasoning_effort=self.reasoning_effort,

backend/packages/harness/deerflow/utils/AGENTS.md

@@ -132,13 +132,18 @@ construction must not start inference. A scan without `app_config`
 (in-graph `skill_manage`) reads the live config in the same pool, and its
 fail-closed fallback reuses that config rather than reading it again on the
 loop. Manual compaction without `app_config` (the `/compact` route) reads
-the live config in that pool as well. Subagent construction inside graph runs
-is a separate boundary.
+the live config in that pool as well. A subagent builds its chat model there
+too: `SubagentExecutor._create_agent` is awaited from `_aexecute` on the
+persistent isolated subagent loop every concurrent delegated task shares (not
+the Gateway request loop), so its `create_chat_model` call goes through
+`run_assembly`; the rest of its middleware and graph assembly is a separate
+boundary.
 Coverage includes `tests/blocking_io/test_oneshot_llm_offloop.py`,
 `tests/blocking_io/test_system_model_offloop.py`,
 `tests/blocking_io/test_title_model_offloop.py`,
-`tests/blocking_io/test_summarization_model_offloop.py` and
-`tests/blocking_io/test_compaction_model_offloop.py`.
+`tests/blocking_io/test_summarization_model_offloop.py`,
+`tests/blocking_io/test_compaction_model_offloop.py` and
+`tests/blocking_io/test_subagent_model_offloop.py`.
 
 `start_run` also resolves the factory through `run_assembly` before admission:
 cold imports cannot occupy the loop or its default pool, and import errors

backend/tests/blocking_io/test_subagent_model_offloop.py

@@ -0,0 +1,121 @@
+"""A subagent must build its chat model off the loop it is executed on.
+
+``SubagentExecutor._create_agent`` is awaited from ``_aexecute`` on the
+persistent isolated subagent loop that every concurrent delegated task shares --
+not the Gateway request loop. It constructed the chat model inline, and
+provider construction can read local credentials -- the Codex provider loads the
+Codex CLI ``auth.json`` while it is built -- so every delegated task did
+synchronous file IO on that loop before its first model call. The real Codex
+provider is built here so that IO stays inside the strict gate; inference never
+runs.
+"""
+
+from __future__ import annotations
+
+import importlib
+import json
+import sys
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from types import SimpleNamespace
+
+import pytest
+
+from deerflow.agents.middlewares import tool_error_handling_middleware
+from deerflow.models.openai_codex_provider import CodexChatModel
+
+pytestmark = pytest.mark.asyncio
+
+_MISSING = object()
+
+
+@contextmanager
+def _real_subagent_executor() -> Iterator[SimpleNamespace]:
+    """Import the real executor despite the suite-level circular-import mock.
+
+    ``deerflow.models`` and ``deerflow.agents`` stay real here: this test builds
+    an actual Codex provider and patches the real middleware builder.
+    """
+    original_executor = sys.modules.get("deerflow.subagents.executor", _MISSING)
+    parent_module = sys.modules.get("deerflow.subagents")
+    original_parent_executor = getattr(parent_module, "executor", _MISSING) if parent_module is not None else _MISSING
+
+    sys.modules.pop("deerflow.subagents.executor", None)
+    try:
+        module = importlib.import_module("deerflow.subagents.executor")
+        yield module
+    finally:
+        if original_executor is _MISSING:
+            sys.modules.pop("deerflow.subagents.executor", None)
+        else:
+            sys.modules["deerflow.subagents.executor"] = original_executor
+
+        if parent_module is not None:
+            if original_parent_executor is _MISSING:
+                try:
+                    delattr(parent_module, "executor")
+                except AttributeError:
+                    pass
+            else:
+                parent_module.executor = original_parent_executor
+
+
+def _codex_config():
+    from deerflow.config.app_config import AppConfig
+
+    return AppConfig.model_validate(
+        {
+            "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
+            "models": [
+                {
+                    "name": "fixture-codex",
+                    "model": "fixture-model",
+                    "use": "deerflow.models.openai_codex_provider:CodexChatModel",
+                }
+            ],
+            "memory": {"enabled": False},
+        }
+    )
+
+
+@pytest.fixture
+def credential_reads(tmp_path, monkeypatch):
+    """Real Codex credential loading, with the thread of every read recorded."""
+    credentials = tmp_path / "auth.json"
+    credentials.write_text(json.dumps({"access_token": "fixture-token", "account_id": "fixture-account"}), encoding="utf-8")
+    monkeypatch.setenv("CODEX_AUTH_PATH", str(credentials))
+    reads = []
+    original_load = CodexChatModel._load_codex_auth
+
+    def record(self):
+        reads.append(threading.get_ident())
+        return original_load(self)
+
+    monkeypatch.setattr(CodexChatModel, "_load_codex_auth", record)
+    return reads
+
+
+async def test_subagent_builds_its_chat_model_off_the_loop(credential_reads, monkeypatch):
+    from deerflow.subagents.config import SubagentConfig
+
+    loop_thread = threading.get_ident()
+    with _real_subagent_executor() as executor_module:
+        # The anchoring line is the model build; keep the middleware chain and
+        # the compiled graph out of the test.
+        monkeypatch.setattr(tool_error_handling_middleware, "build_subagent_runtime_middlewares", lambda **kwargs: [])
+        monkeypatch.setattr(
+            executor_module,
+            "create_agent",
+            lambda **kwargs: SimpleNamespace(get_graph=lambda: SimpleNamespace(nodes={})),
+        )
+        executor = executor_module.SubagentExecutor(
+            config=SubagentConfig(name="probe", description="probe", model="fixture-codex"),
+            tools=[],
+            app_config=_codex_config(),
+        )
+
+        await executor._create_agent(tools=[])
+
+    assert credential_reads, "the Codex provider was never constructed"
+    assert all(ident != loop_thread for ident in credential_reads), "provider construction blocks the subagent loop"