From 0cdecf7b30bf5d369f4b6b24ab6fba4093955ee2 Mon Sep 17 00:00:00 2001 From: AochenShen99 <142667174+ShenAC-SAC@users.noreply.github.com> Date: Wed, 1 Apr 2026 16:45:29 +0800 Subject: [PATCH 01/47] feat(memory): structured reflection + correction detection in MemoryMiddleware (#1620) (#1668) * feat(memory): add structured reflection and correction detection * fix(memory): align sourceError schema and prompt guidance --------- Co-authored-by: Willem Jiang --- backend/app/gateway/routers/memory.py | 10 ++ .../harness/deerflow/agents/memory/prompt.py | 25 ++++- .../harness/deerflow/agents/memory/queue.py | 29 ++++-- .../harness/deerflow/agents/memory/updater.py | 36 ++++++- .../agents/middlewares/memory_middleware.py | 65 +++++++++++-- backend/tests/test_memory_prompt_injection.py | 35 +++++++ backend/tests/test_memory_queue.py | 50 ++++++++++ backend/tests/test_memory_router.py | 50 ++++++++++ backend/tests/test_memory_updater.py | 97 +++++++++++++++++++ backend/tests/test_memory_upload_filtering.py | 60 +++++++++++- 10 files changed, 436 insertions(+), 21 deletions(-) create mode 100644 backend/tests/test_memory_queue.py diff --git a/backend/app/gateway/routers/memory.py b/backend/app/gateway/routers/memory.py index 7a074100a..6ee546924 100644 --- a/backend/app/gateway/routers/memory.py +++ b/backend/app/gateway/routers/memory.py @@ -49,6 +49,7 @@ class Fact(BaseModel): confidence: float = Field(default=0.5, description="Confidence score (0-1)") createdAt: str = Field(default="", description="Creation timestamp") source: str = Field(default="unknown", description="Source thread ID") + sourceError: str | None = Field(default=None, description="Optional description of the prior mistake or wrong approach") class MemoryResponse(BaseModel): @@ -108,6 +109,7 @@ class MemoryStatusResponse(BaseModel): @router.get( "/memory", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Get Memory Data", description="Retrieve the current global memory data including user context, history, and facts.", ) @@ -152,6 +154,7 @@ async def get_memory() -> MemoryResponse: @router.post( "/memory/reload", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Reload Memory Data", description="Reload memory data from the storage file, refreshing the in-memory cache.", ) @@ -171,6 +174,7 @@ async def reload_memory() -> MemoryResponse: @router.delete( "/memory", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Clear All Memory Data", description="Delete all saved memory data and reset the memory structure to an empty state.", ) @@ -187,6 +191,7 @@ async def clear_memory() -> MemoryResponse: @router.post( "/memory/facts", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Create Memory Fact", description="Create a single saved memory fact manually.", ) @@ -209,6 +214,7 @@ async def create_memory_fact_endpoint(request: FactCreateRequest) -> MemoryRespo @router.delete( "/memory/facts/{fact_id}", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Delete Memory Fact", description="Delete a single saved memory fact by its fact id.", ) @@ -227,6 +233,7 @@ async def delete_memory_fact_endpoint(fact_id: str) -> MemoryResponse: @router.patch( "/memory/facts/{fact_id}", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Patch Memory Fact", description="Partially update a single saved memory fact by its fact id while preserving omitted fields.", ) @@ -252,6 +259,7 @@ async def update_memory_fact_endpoint(fact_id: str, request: FactPatchRequest) - @router.get( "/memory/export", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Export Memory Data", description="Export the current global memory data as JSON for backup or transfer.", ) @@ -264,6 +272,7 @@ async def export_memory() -> MemoryResponse: @router.post( "/memory/import", response_model=MemoryResponse, + response_model_exclude_none=True, summary="Import Memory Data", description="Import and overwrite the current global memory data from a JSON payload.", ) @@ -317,6 +326,7 @@ async def get_memory_config_endpoint() -> MemoryConfigResponse: @router.get( "/memory/status", response_model=MemoryStatusResponse, + response_model_exclude_none=True, summary="Get Memory Status", description="Retrieve both memory configuration and current data in a single request.", ) diff --git a/backend/packages/harness/deerflow/agents/memory/prompt.py b/backend/packages/harness/deerflow/agents/memory/prompt.py index 0d4e86d97..e0c04b77e 100644 --- a/backend/packages/harness/deerflow/agents/memory/prompt.py +++ b/backend/packages/harness/deerflow/agents/memory/prompt.py @@ -29,6 +29,17 @@ Instructions: 2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies) 3. Update the memory sections as needed following the detailed length guidelines below +Before extracting facts, perform a structured reflection on the conversation: +1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results? + If yes, record the root cause and correct approach as a high-confidence fact with category "correction". +2. User Correction Detection: Did the user correct the agent's direction, understanding, or output? + If yes, record the correct interpretation or approach as a high-confidence fact with category "correction". + Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation. +3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation? + If yes, record them as facts with the most appropriate category and confidence. + +{correction_hint} + Memory Section Guidelines: **User Context** (Current state - concise summaries): @@ -62,6 +73,7 @@ Memory Section Guidelines: * context: Background facts (job title, projects, locations, languages) * behavior: Working patterns, communication habits, problem-solving approaches * goal: Stated objectives, learning targets, project ambitions + * correction: Explicit agent mistakes or user corrections, including the correct approach - Confidence levels: * 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y") * 0.7-0.8: Strongly implied from actions/discussions @@ -94,7 +106,7 @@ Output Format (JSON): "longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }} }}, "newFacts": [ - {{ "content": "...", "category": "preference|knowledge|context|behavior|goal", "confidence": 0.0-1.0 }} + {{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }} ], "factsToRemove": ["fact_id_1", "fact_id_2"] }} @@ -104,6 +116,8 @@ Important Rules: - Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs) - Include specific metrics, version numbers, and proper nouns in facts - Only add facts that are clearly stated (0.9+) or strongly implied (0.7+) +- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit +- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise - Remove facts that are contradicted by new information - When updating topOfMind, integrate new focus areas while removing completed/abandoned ones Keep 3-5 concurrent focus themes that are still active and relevant @@ -126,7 +140,7 @@ Message: Extract facts in this JSON format: {{ "facts": [ - {{ "content": "...", "category": "preference|knowledge|context|behavior|goal", "confidence": 0.0-1.0 }} + {{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }} ] }} @@ -136,6 +150,7 @@ Categories: - context: Background context (location, job, projects) - behavior: Behavioral patterns - goal: User's goals or objectives +- correction: Explicit corrections or mistakes to avoid repeating Rules: - Only extract clear, specific facts @@ -262,7 +277,11 @@ def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2 continue category = str(fact.get("category", "context")).strip() or "context" confidence = _coerce_confidence(fact.get("confidence"), default=0.0) - line = f"- [{category} | {confidence:.2f}] {content}" + source_error = fact.get("sourceError") + if category == "correction" and isinstance(source_error, str) and source_error.strip(): + line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})" + else: + line = f"- [{category} | {confidence:.2f}] {content}" # Each additional line is preceded by a newline (except the first). line_text = ("\n" + line) if fact_lines else line diff --git a/backend/packages/harness/deerflow/agents/memory/queue.py b/backend/packages/harness/deerflow/agents/memory/queue.py index a9a683860..6d777a67e 100644 --- a/backend/packages/harness/deerflow/agents/memory/queue.py +++ b/backend/packages/harness/deerflow/agents/memory/queue.py @@ -20,6 +20,7 @@ class ConversationContext: messages: list[Any] timestamp: datetime = field(default_factory=datetime.utcnow) agent_name: str | None = None + correction_detected: bool = False class MemoryUpdateQueue: @@ -37,25 +38,38 @@ class MemoryUpdateQueue: self._timer: threading.Timer | None = None self._processing = False - def add(self, thread_id: str, messages: list[Any], agent_name: str | None = None) -> None: + def add( + self, + thread_id: str, + messages: list[Any], + agent_name: str | None = None, + correction_detected: bool = False, + ) -> None: """Add a conversation to the update queue. Args: thread_id: The thread ID. messages: The conversation messages. agent_name: If provided, memory is stored per-agent. If None, uses global memory. + correction_detected: Whether recent turns include an explicit correction signal. """ config = get_memory_config() if not config.enabled: return - context = ConversationContext( - thread_id=thread_id, - messages=messages, - agent_name=agent_name, - ) - with self._lock: + existing_context = next( + (context for context in self._queue if context.thread_id == thread_id), + None, + ) + merged_correction_detected = correction_detected or (existing_context.correction_detected if existing_context is not None else False) + context = ConversationContext( + thread_id=thread_id, + messages=messages, + agent_name=agent_name, + correction_detected=merged_correction_detected, + ) + # Check if this thread already has a pending update # If so, replace it with the newer one self._queue = [c for c in self._queue if c.thread_id != thread_id] @@ -115,6 +129,7 @@ class MemoryUpdateQueue: messages=context.messages, thread_id=context.thread_id, agent_name=context.agent_name, + correction_detected=context.correction_detected, ) if success: logger.info("Memory updated successfully for thread %s", context.thread_id) diff --git a/backend/packages/harness/deerflow/agents/memory/updater.py b/backend/packages/harness/deerflow/agents/memory/updater.py index e8c8b5898..c59749d7b 100644 --- a/backend/packages/harness/deerflow/agents/memory/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/updater.py @@ -266,13 +266,20 @@ class MemoryUpdater: model_name = self._model_name or config.model_name return create_chat_model(name=model_name, thinking_enabled=False) - def update_memory(self, messages: list[Any], thread_id: str | None = None, agent_name: str | None = None) -> bool: + def update_memory( + self, + messages: list[Any], + thread_id: str | None = None, + agent_name: str | None = None, + correction_detected: bool = False, + ) -> bool: """Update memory based on conversation messages. Args: messages: List of conversation messages. thread_id: Optional thread ID for tracking source. agent_name: If provided, updates per-agent memory. If None, updates global memory. + correction_detected: Whether recent turns include an explicit correction signal. Returns: True if update was successful, False otherwise. @@ -295,9 +302,19 @@ class MemoryUpdater: return False # Build prompt + correction_hint = "" + if correction_detected: + correction_hint = ( + "IMPORTANT: Explicit correction signals were detected in this conversation. " + "Pay special attention to what the agent got wrong, what the user corrected, " + "and record the correct approach as a fact with category " + '"correction" and confidence >= 0.95 when appropriate.' + ) + prompt = MEMORY_UPDATE_PROMPT.format( current_memory=json.dumps(current_memory, indent=2), conversation=conversation_text, + correction_hint=correction_hint, ) # Call LLM @@ -383,6 +400,8 @@ class MemoryUpdater: confidence = fact.get("confidence", 0.5) if confidence >= config.fact_confidence_threshold: raw_content = fact.get("content", "") + if not isinstance(raw_content, str): + continue normalized_content = raw_content.strip() fact_key = _fact_content_key(normalized_content) if fact_key is not None and fact_key in existing_fact_keys: @@ -396,6 +415,11 @@ class MemoryUpdater: "createdAt": now, "source": thread_id or "unknown", } + source_error = fact.get("sourceError") + if isinstance(source_error, str): + normalized_source_error = source_error.strip() + if normalized_source_error: + fact_entry["sourceError"] = normalized_source_error current_memory["facts"].append(fact_entry) if fact_key is not None: existing_fact_keys.add(fact_key) @@ -412,16 +436,22 @@ class MemoryUpdater: return current_memory -def update_memory_from_conversation(messages: list[Any], thread_id: str | None = None, agent_name: str | None = None) -> bool: +def update_memory_from_conversation( + messages: list[Any], + thread_id: str | None = None, + agent_name: str | None = None, + correction_detected: bool = False, +) -> bool: """Convenience function to update memory from a conversation. Args: messages: List of conversation messages. thread_id: Optional thread ID. agent_name: If provided, updates per-agent memory. If None, updates global memory. + correction_detected: Whether recent turns include an explicit correction signal. Returns: True if successful, False otherwise. """ updater = MemoryUpdater() - return updater.update_memory(messages, thread_id, agent_name) + return updater.update_memory(messages, thread_id, agent_name, correction_detected) diff --git a/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py index 90907b6e9..6215a2957 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py @@ -14,6 +14,21 @@ from deerflow.config.memory_config import get_memory_config logger = logging.getLogger(__name__) +_UPLOAD_BLOCK_RE = re.compile(r"[\s\S]*?\n*", re.IGNORECASE) +_CORRECTION_PATTERNS = ( + re.compile(r"\bthat(?:'s| is) (?:wrong|incorrect)\b", re.IGNORECASE), + re.compile(r"\byou misunderstood\b", re.IGNORECASE), + re.compile(r"\btry again\b", re.IGNORECASE), + re.compile(r"\bredo\b", re.IGNORECASE), + re.compile(r"不对"), + re.compile(r"你理解错了"), + re.compile(r"你理解有误"), + re.compile(r"重试"), + re.compile(r"重新来"), + re.compile(r"换一种"), + re.compile(r"改用"), +) + class MemoryMiddlewareState(AgentState): """Compatible with the `ThreadState` schema.""" @@ -21,6 +36,22 @@ class MemoryMiddlewareState(AgentState): pass +def _extract_message_text(message: Any) -> str: + """Extract plain text from message content for filtering and signal detection.""" + content = getattr(message, "content", "") + if isinstance(content, list): + text_parts: list[str] = [] + for part in content: + if isinstance(part, str): + text_parts.append(part) + elif isinstance(part, dict): + text_val = part.get("text") + if isinstance(text_val, str): + text_parts.append(text_val) + return " ".join(text_parts) + return str(content) + + def _filter_messages_for_memory(messages: list[Any]) -> list[Any]: """Filter messages to keep only user inputs and final assistant responses. @@ -44,18 +75,13 @@ def _filter_messages_for_memory(messages: list[Any]) -> list[Any]: Returns: Filtered list containing only user inputs and final assistant responses. """ - _UPLOAD_BLOCK_RE = re.compile(r"[\s\S]*?\n*", re.IGNORECASE) - filtered = [] skip_next_ai = False for msg in messages: msg_type = getattr(msg, "type", None) if msg_type == "human": - content = getattr(msg, "content", "") - if isinstance(content, list): - content = " ".join(p.get("text", "") for p in content if isinstance(p, dict)) - content_str = str(content) + content_str = _extract_message_text(msg) if "" in content_str: # Strip the ephemeral upload block; keep the user's real question. stripped = _UPLOAD_BLOCK_RE.sub("", content_str).strip() @@ -87,6 +113,25 @@ def _filter_messages_for_memory(messages: list[Any]) -> list[Any]: return filtered +def detect_correction(messages: list[Any]) -> bool: + """Detect explicit user corrections in recent conversation turns. + + The queue keeps only one pending context per thread, so callers pass the + latest filtered message list. Checking only recent user turns keeps signal + detection conservative while avoiding stale corrections from long histories. + """ + recent_user_msgs = [msg for msg in messages[-6:] if getattr(msg, "type", None) == "human"] + + for msg in recent_user_msgs: + content = _extract_message_text(msg).strip() + if not content: + continue + if any(pattern.search(content) for pattern in _CORRECTION_PATTERNS): + return True + + return False + + class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]): """Middleware that queues conversation for memory update after agent execution. @@ -150,7 +195,13 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]): return None # Queue the filtered conversation for memory update + correction_detected = detect_correction(filtered_messages) queue = get_memory_queue() - queue.add(thread_id=thread_id, messages=filtered_messages, agent_name=self._agent_name) + queue.add( + thread_id=thread_id, + messages=filtered_messages, + agent_name=self._agent_name, + correction_detected=correction_detected, + ) return None diff --git a/backend/tests/test_memory_prompt_injection.py b/backend/tests/test_memory_prompt_injection.py index ab1f0a783..d33b69a92 100644 --- a/backend/tests/test_memory_prompt_injection.py +++ b/backend/tests/test_memory_prompt_injection.py @@ -119,3 +119,38 @@ def test_format_memory_skips_non_string_content_facts() -> None: # The formatted line for a list content would be "- [knowledge | 0.85] ['list']". assert "| 0.85]" not in result assert "Valid fact" in result + + +def test_format_memory_renders_correction_source_error() -> None: + memory_data = { + "facts": [ + { + "content": "Use make dev for local development.", + "category": "correction", + "confidence": 0.95, + "sourceError": "The agent previously suggested npm start.", + } + ] + } + + result = format_memory_for_injection(memory_data, max_tokens=2000) + + assert "Use make dev for local development." in result + assert "avoid: The agent previously suggested npm start." in result + + +def test_format_memory_renders_correction_without_source_error_normally() -> None: + memory_data = { + "facts": [ + { + "content": "Use make dev for local development.", + "category": "correction", + "confidence": 0.95, + } + ] + } + + result = format_memory_for_injection(memory_data, max_tokens=2000) + + assert "Use make dev for local development." in result + assert "avoid:" not in result diff --git a/backend/tests/test_memory_queue.py b/backend/tests/test_memory_queue.py new file mode 100644 index 000000000..6ef91a142 --- /dev/null +++ b/backend/tests/test_memory_queue.py @@ -0,0 +1,50 @@ +from unittest.mock import MagicMock, patch + +from deerflow.agents.memory.queue import ConversationContext, MemoryUpdateQueue +from deerflow.config.memory_config import MemoryConfig + + +def _memory_config(**overrides: object) -> MemoryConfig: + config = MemoryConfig() + for key, value in overrides.items(): + setattr(config, key, value) + return config + + +def test_queue_add_preserves_existing_correction_flag_for_same_thread() -> None: + queue = MemoryUpdateQueue() + + with ( + patch("deerflow.agents.memory.queue.get_memory_config", return_value=_memory_config(enabled=True)), + patch.object(queue, "_reset_timer"), + ): + queue.add(thread_id="thread-1", messages=["first"], correction_detected=True) + queue.add(thread_id="thread-1", messages=["second"], correction_detected=False) + + assert len(queue._queue) == 1 + assert queue._queue[0].messages == ["second"] + assert queue._queue[0].correction_detected is True + + +def test_process_queue_forwards_correction_flag_to_updater() -> None: + queue = MemoryUpdateQueue() + queue._queue = [ + ConversationContext( + thread_id="thread-1", + messages=["conversation"], + agent_name="lead_agent", + correction_detected=True, + ) + ] + mock_updater = MagicMock() + mock_updater.update_memory.return_value = True + + with patch("deerflow.agents.memory.updater.MemoryUpdater", return_value=mock_updater): + queue._process_queue() + + mock_updater.update_memory.assert_called_once_with( + messages=["conversation"], + thread_id="thread-1", + agent_name="lead_agent", + correction_detected=True, + ) diff --git a/backend/tests/test_memory_router.py b/backend/tests/test_memory_router.py index 39134c61d..23a4f30fe 100644 --- a/backend/tests/test_memory_router.py +++ b/backend/tests/test_memory_router.py @@ -72,6 +72,56 @@ def test_import_memory_route_returns_imported_memory() -> None: assert response.json()["facts"] == imported_memory["facts"] +def test_export_memory_route_preserves_source_error() -> None: + app = FastAPI() + app.include_router(memory.router) + exported_memory = _sample_memory( + facts=[ + { + "id": "fact_correction", + "content": "Use make dev for local development.", + "category": "correction", + "confidence": 0.95, + "createdAt": "2026-03-20T00:00:00Z", + "source": "thread-1", + "sourceError": "The agent previously suggested npm start.", + } + ] + ) + + with patch("app.gateway.routers.memory.get_memory_data", return_value=exported_memory): + with TestClient(app) as client: + response = client.get("/api/memory/export") + + assert response.status_code == 200 + assert response.json()["facts"][0]["sourceError"] == "The agent previously suggested npm start." + + +def test_import_memory_route_preserves_source_error() -> None: + app = FastAPI() + app.include_router(memory.router) + imported_memory = _sample_memory( + facts=[ + { + "id": "fact_correction", + "content": "Use make dev for local development.", + "category": "correction", + "confidence": 0.95, + "createdAt": "2026-03-20T00:00:00Z", + "source": "thread-1", + "sourceError": "The agent previously suggested npm start.", + } + ] + ) + + with patch("app.gateway.routers.memory.import_memory_data", return_value=imported_memory): + with TestClient(app) as client: + response = client.post("/api/memory/import", json=imported_memory) + + assert response.status_code == 200 + assert response.json()["facts"][0]["sourceError"] == "The agent previously suggested npm start." + + def test_clear_memory_route_returns_cleared_memory() -> None: app = FastAPI() app.include_router(memory.router) diff --git a/backend/tests/test_memory_updater.py b/backend/tests/test_memory_updater.py index f7b48228a..6309cf9f6 100644 --- a/backend/tests/test_memory_updater.py +++ b/backend/tests/test_memory_updater.py @@ -146,6 +146,53 @@ def test_apply_updates_preserves_threshold_and_max_facts_trimming() -> None: assert result["facts"][1]["source"] == "thread-9" +def test_apply_updates_preserves_source_error() -> None: + updater = MemoryUpdater() + current_memory = _make_memory() + update_data = { + "newFacts": [ + { + "content": "Use make dev for local development.", + "category": "correction", + "confidence": 0.95, + "sourceError": "The agent previously suggested npm start.", + } + ] + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100, fact_confidence_threshold=0.7), + ): + result = updater._apply_updates(current_memory, update_data, thread_id="thread-correction") + + assert result["facts"][0]["sourceError"] == "The agent previously suggested npm start." + assert result["facts"][0]["category"] == "correction" + + +def test_apply_updates_ignores_empty_source_error() -> None: + updater = MemoryUpdater() + current_memory = _make_memory() + update_data = { + "newFacts": [ + { + "content": "Use make dev for local development.", + "category": "correction", + "confidence": 0.95, + "sourceError": " ", + } + ] + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100, fact_confidence_threshold=0.7), + ): + result = updater._apply_updates(current_memory, update_data, thread_id="thread-correction") + + assert "sourceError" not in result["facts"][0] + + def test_clear_memory_data_resets_all_sections() -> None: with patch("deerflow.agents.memory.updater._save_memory_to_file", return_value=True): result = clear_memory_data() @@ -522,3 +569,53 @@ class TestUpdateMemoryStructuredResponse: result = updater.update_memory([msg, ai_msg]) assert result is True + + def test_correction_hint_injected_when_detected(self): + updater = MemoryUpdater() + valid_json = '{"user": {}, "history": {}, "newFacts": [], "factsToRemove": []}' + model = self._make_mock_model(valid_json) + + with ( + patch.object(updater, "_get_model", return_value=model), + patch("deerflow.agents.memory.updater.get_memory_config", return_value=_memory_config(enabled=True)), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=_make_memory()), + patch("deerflow.agents.memory.updater.get_memory_storage", return_value=MagicMock(save=MagicMock(return_value=True))), + ): + msg = MagicMock() + msg.type = "human" + msg.content = "No, that's wrong." + ai_msg = MagicMock() + ai_msg.type = "ai" + ai_msg.content = "Understood" + ai_msg.tool_calls = [] + + result = updater.update_memory([msg, ai_msg], correction_detected=True) + + assert result is True + prompt = model.invoke.call_args[0][0] + assert "Explicit correction signals were detected" in prompt + + def test_correction_hint_empty_when_not_detected(self): + updater = MemoryUpdater() + valid_json = '{"user": {}, "history": {}, "newFacts": [], "factsToRemove": []}' + model = self._make_mock_model(valid_json) + + with ( + patch.object(updater, "_get_model", return_value=model), + patch("deerflow.agents.memory.updater.get_memory_config", return_value=_memory_config(enabled=True)), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=_make_memory()), + patch("deerflow.agents.memory.updater.get_memory_storage", return_value=MagicMock(save=MagicMock(return_value=True))), + ): + msg = MagicMock() + msg.type = "human" + msg.content = "Let's talk about memory." + ai_msg = MagicMock() + ai_msg.type = "ai" + ai_msg.content = "Sure" + ai_msg.tool_calls = [] + + result = updater.update_memory([msg, ai_msg], correction_detected=False) + + assert result is True + prompt = model.invoke.call_args[0][0] + assert "Explicit correction signals were detected" not in prompt diff --git a/backend/tests/test_memory_upload_filtering.py b/backend/tests/test_memory_upload_filtering.py index 45d0dbf4e..1ff0aa3b6 100644 --- a/backend/tests/test_memory_upload_filtering.py +++ b/backend/tests/test_memory_upload_filtering.py @@ -10,7 +10,7 @@ persisting in long-term memory: from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from deerflow.agents.memory.updater import _strip_upload_mentions_from_memory -from deerflow.agents.middlewares.memory_middleware import _filter_messages_for_memory +from deerflow.agents.middlewares.memory_middleware import _filter_messages_for_memory, detect_correction # --------------------------------------------------------------------------- # Helpers @@ -134,6 +134,64 @@ class TestFilterMessagesForMemory: assert "" not in all_content +# =========================================================================== +# detect_correction +# =========================================================================== + + +class TestDetectCorrection: + def test_detects_english_correction_signal(self): + msgs = [ + _human("Please help me run the project."), + _ai("Use npm start."), + _human("That's wrong, use make dev instead."), + _ai("Understood."), + ] + + assert detect_correction(msgs) is True + + def test_detects_chinese_correction_signal(self): + msgs = [ + _human("帮我启动项目"), + _ai("用 npm start"), + _human("不对，改用 make dev"), + _ai("明白了"), + ] + + assert detect_correction(msgs) is True + + def test_returns_false_without_signal(self): + msgs = [ + _human("Please explain the build setup."), + _ai("Here is the build setup."), + _human("Thanks, that makes sense."), + ] + + assert detect_correction(msgs) is False + + def test_only_checks_recent_messages(self): + msgs = [ + _human("That is wrong, use make dev instead."), + _ai("Noted."), + _human("Let's discuss tests."), + _ai("Sure."), + _human("What about linting?"), + _ai("Use ruff."), + _human("And formatting?"), + _ai("Use make format."), + ] + + assert detect_correction(msgs) is False + + def test_handles_list_content(self): + msgs = [ + HumanMessage(content=["That is wrong,", {"type": "text", "text": "use make dev instead."}]), + _ai("Updated."), + ] + + assert detect_correction(msgs) is True + + # =========================================================================== # _strip_upload_mentions_from_memory # =========================================================================== From 52c8c06cf27406b91e888d35b27c0f091d004660 Mon Sep 17 00:00:00 2001 From: Llugaes <43494187+Llugaes@users.noreply.github.com> Date: Wed, 1 Apr 2026 16:45:51 +0800 Subject: [PATCH 02/47] fix: add --n-jobs-per-worker 10 to local dev Makefile (#1694) #1623 added this flag to both Docker Compose files but missed the backend Makefile used by `make dev`. Without it `langgraph dev` defaults to n_jobs_per_worker=1, so all conversation runs are serialised and concurrent requests block. This mirrors the Docker configuration. --- backend/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Makefile b/backend/Makefile index 8fe28c985..e16359f73 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -2,7 +2,7 @@ install: uv sync dev: - uv run langgraph dev --no-browser --allow-blocking --no-reload + uv run langgraph dev --no-browser --allow-blocking --no-reload --n-jobs-per-worker 10 gateway: PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 From 2f3744f80721aa86c941197126d47a50f0df8a7a Mon Sep 17 00:00:00 2001 From: Shengyuan Wang <78770936+ShengyuanWang@users.noreply.github.com> Date: Wed, 1 Apr 2026 02:02:39 -0700 Subject: [PATCH 03/47] refactor: replace sync requests with async httpx in Jina AI client (#1603) * refactor: replace sync requests with async httpx in Jina AI client Replace synchronous `requests.post()` with `httpx.AsyncClient` in JinaClient.crawl() and make web_fetch_tool async. This is part of the planned async concurrency optimization for the agent hot path (see docs/TODO.md). * fix: address Copilot review feedback on async Jina client - Short-circuit error strings in web_fetch_tool before passing to ReadabilityExtractor, preventing misleading extraction results - Log missing JINA_API_KEY warning only once per process to reduce noise under concurrent async fetching - Use logger.exception instead of logger.error in crawl exception handler to preserve stack traces for debugging - Add async web_fetch_tool tests and warn-once coverage * fix: mock get_app_config in web_fetch_tool tests for CI The web_fetch_tool tests failed in CI because get_app_config requires a config.yaml file that isn't present in the test environment. Mock the config loader to remove the filesystem dependency. --------- Co-authored-by: Willem Jiang --- .../deerflow/community/jina_ai/jina_client.py | 15 +- .../deerflow/community/jina_ai/tools.py | 6 +- backend/tests/test_jina_client.py | 177 ++++++++++++++++++ 3 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 backend/tests/test_jina_client.py diff --git a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py index 3b1a219e3..3adc5458a 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py +++ b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py @@ -1,13 +1,16 @@ import logging import os -import requests +import httpx logger = logging.getLogger(__name__) +_api_key_warned = False + class JinaClient: - def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str: + async def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str: + global _api_key_warned headers = { "Content-Type": "application/json", "X-Return-Format": return_format, @@ -15,11 +18,13 @@ class JinaClient: } if os.getenv("JINA_API_KEY"): headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}" - else: + elif not _api_key_warned: + _api_key_warned = True logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.") data = {"url": url} try: - response = requests.post("https://r.jina.ai/", headers=headers, json=data) + async with httpx.AsyncClient() as client: + response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=timeout) if response.status_code != 200: error_message = f"Jina API returned status {response.status_code}: {response.text}" @@ -34,5 +39,5 @@ class JinaClient: return response.text except Exception as e: error_message = f"Request to Jina API failed: {str(e)}" - logger.error(error_message) + logger.exception(error_message) return f"Error: {error_message}" diff --git a/backend/packages/harness/deerflow/community/jina_ai/tools.py b/backend/packages/harness/deerflow/community/jina_ai/tools.py index 0bde35a63..9f243ecd9 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/tools.py +++ b/backend/packages/harness/deerflow/community/jina_ai/tools.py @@ -8,7 +8,7 @@ readability_extractor = ReadabilityExtractor() @tool("web_fetch", parse_docstring=True) -def web_fetch_tool(url: str) -> str: +async def web_fetch_tool(url: str) -> str: """Fetch the contents of a web page at a given URL. Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools. This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls. @@ -23,6 +23,8 @@ def web_fetch_tool(url: str) -> str: config = get_app_config().get_tool_config("web_fetch") if config is not None and "timeout" in config.model_extra: timeout = config.model_extra.get("timeout") - html_content = jina_client.crawl(url, return_format="html", timeout=timeout) + html_content = await jina_client.crawl(url, return_format="html", timeout=timeout) + if isinstance(html_content, str) and html_content.startswith("Error:"): + return html_content article = readability_extractor.extract_article(html_content) return article.to_markdown()[:4096] diff --git a/backend/tests/test_jina_client.py b/backend/tests/test_jina_client.py new file mode 100644 index 000000000..037436f73 --- /dev/null +++ b/backend/tests/test_jina_client.py @@ -0,0 +1,177 @@ +"""Tests for JinaClient async crawl method.""" + +import logging +from unittest.mock import MagicMock + +import httpx +import pytest + +import deerflow.community.jina_ai.jina_client as jina_client_module +from deerflow.community.jina_ai.jina_client import JinaClient +from deerflow.community.jina_ai.tools import web_fetch_tool + + +@pytest.fixture +def jina_client(): + return JinaClient() + + +@pytest.mark.anyio +async def test_crawl_success(jina_client, monkeypatch): + """Test successful crawl returns response text.""" + + async def mock_post(self, url, **kwargs): + return httpx.Response(200, text="Hello", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + result = await jina_client.crawl("https://example.com") + assert result == "Hello" + + +@pytest.mark.anyio +async def test_crawl_non_200_status(jina_client, monkeypatch): + """Test that non-200 status returns error message.""" + + async def mock_post(self, url, **kwargs): + return httpx.Response(429, text="Rate limited", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + result = await jina_client.crawl("https://example.com") + assert result.startswith("Error:") + assert "429" in result + + +@pytest.mark.anyio +async def test_crawl_empty_response(jina_client, monkeypatch): + """Test that empty response returns error message.""" + + async def mock_post(self, url, **kwargs): + return httpx.Response(200, text="", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + result = await jina_client.crawl("https://example.com") + assert result.startswith("Error:") + assert "empty" in result.lower() + + +@pytest.mark.anyio +async def test_crawl_whitespace_only_response(jina_client, monkeypatch): + """Test that whitespace-only response returns error message.""" + + async def mock_post(self, url, **kwargs): + return httpx.Response(200, text=" \n ", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + result = await jina_client.crawl("https://example.com") + assert result.startswith("Error:") + assert "empty" in result.lower() + + +@pytest.mark.anyio +async def test_crawl_network_error(jina_client, monkeypatch): + """Test that network errors are handled gracefully.""" + + async def mock_post(self, url, **kwargs): + raise httpx.ConnectError("Connection refused") + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + result = await jina_client.crawl("https://example.com") + assert result.startswith("Error:") + assert "failed" in result.lower() + + +@pytest.mark.anyio +async def test_crawl_passes_headers(jina_client, monkeypatch): + """Test that correct headers are sent.""" + captured_headers = {} + + async def mock_post(self, url, **kwargs): + captured_headers.update(kwargs.get("headers", {})) + return httpx.Response(200, text="ok", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + await jina_client.crawl("https://example.com", return_format="markdown", timeout=30) + assert captured_headers["X-Return-Format"] == "markdown" + assert captured_headers["X-Timeout"] == "30" + + +@pytest.mark.anyio +async def test_crawl_includes_api_key_when_set(jina_client, monkeypatch): + """Test that Authorization header is set when JINA_API_KEY is available.""" + captured_headers = {} + + async def mock_post(self, url, **kwargs): + captured_headers.update(kwargs.get("headers", {})) + return httpx.Response(200, text="ok", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + monkeypatch.setenv("JINA_API_KEY", "test-key-123") + await jina_client.crawl("https://example.com") + assert captured_headers["Authorization"] == "Bearer test-key-123" + + +@pytest.mark.anyio +async def test_crawl_warns_once_when_api_key_missing(jina_client, monkeypatch, caplog): + """Test that the missing API key warning is logged only once.""" + jina_client_module._api_key_warned = False + + async def mock_post(self, url, **kwargs): + return httpx.Response(200, text="ok", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + monkeypatch.delenv("JINA_API_KEY", raising=False) + + with caplog.at_level(logging.WARNING, logger="deerflow.community.jina_ai.jina_client"): + await jina_client.crawl("https://example.com") + await jina_client.crawl("https://example.com") + + warning_count = sum(1 for record in caplog.records if "Jina API key is not set" in record.message) + assert warning_count == 1 + + +@pytest.mark.anyio +async def test_crawl_no_auth_header_without_api_key(jina_client, monkeypatch): + """Test that no Authorization header is set when JINA_API_KEY is not available.""" + jina_client_module._api_key_warned = False + captured_headers = {} + + async def mock_post(self, url, **kwargs): + captured_headers.update(kwargs.get("headers", {})) + return httpx.Response(200, text="ok", request=httpx.Request("POST", url)) + + monkeypatch.setattr(httpx.AsyncClient, "post", mock_post) + monkeypatch.delenv("JINA_API_KEY", raising=False) + await jina_client.crawl("https://example.com") + assert "Authorization" not in captured_headers + + +@pytest.mark.anyio +async def test_web_fetch_tool_returns_error_on_crawl_failure(monkeypatch): + """Test that web_fetch_tool short-circuits and returns the error string when crawl fails.""" + + async def mock_crawl(self, url, **kwargs): + return "Error: Jina API returned status 429: Rate limited" + + mock_config = MagicMock() + mock_config.get_tool_config.return_value = None + monkeypatch.setattr("deerflow.community.jina_ai.tools.get_app_config", lambda: mock_config) + monkeypatch.setattr(JinaClient, "crawl", mock_crawl) + result = await web_fetch_tool.ainvoke("https://example.com") + assert result.startswith("Error:") + assert "429" in result + + +@pytest.mark.anyio +async def test_web_fetch_tool_returns_markdown_on_success(monkeypatch): + """Test that web_fetch_tool returns extracted markdown on successful crawl.""" + + async def mock_crawl(self, url, **kwargs): + return "

Hello world

" + + mock_config = MagicMock() + mock_config.get_tool_config.return_value = None + monkeypatch.setattr("deerflow.community.jina_ai.tools.get_app_config", lambda: mock_config) + monkeypatch.setattr(JinaClient, "crawl", mock_crawl) + result = await web_fetch_tool.ainvoke("https://example.com") + assert "Hello world" in result + assert not result.startswith("Error:") From c2ff59a5b172049202a6d4fadd5d4e911c108334 Mon Sep 17 00:00:00 2001 From: rayhpeng Date: Wed, 1 Apr 2026 17:17:09 +0800 Subject: [PATCH 04/47] fix(gateway): merge context field into configurable for langgraph-compat runs (#1699) (#1707) The langgraph-compat layer dropped the DeerFlow-specific `context` field from run requests, causing agent config (subagent_enabled, is_plan_mode, thinking_enabled, etc.) to fall back to defaults. Add `context` to RunCreateRequest and merge allowlisted keys into config.configurable in start_run, with existing configurable values taking precedence. Co-authored-by: Claude Opus 4.6 (1M context) --- backend/app/gateway/routers/thread_runs.py | 1 + backend/app/gateway/services.py | 21 ++++ backend/tests/test_gateway_services.py | 124 +++++++++++++++++++++ 3 files changed, 146 insertions(+) diff --git a/backend/app/gateway/routers/thread_runs.py b/backend/app/gateway/routers/thread_runs.py index 217605685..d29786edd 100644 --- a/backend/app/gateway/routers/thread_runs.py +++ b/backend/app/gateway/routers/thread_runs.py @@ -38,6 +38,7 @@ class RunCreateRequest(BaseModel): command: dict[str, Any] | None = Field(default=None, description="LangGraph Command") metadata: dict[str, Any] | None = Field(default=None, description="Run metadata") config: dict[str, Any] | None = Field(default=None, description="RunnableConfig overrides") + context: dict[str, Any] | None = Field(default=None, description="DeerFlow context overrides (model_name, thinking_enabled, etc.)") webhook: str | None = Field(default=None, description="Completion callback URL") checkpoint_id: str | None = Field(default=None, description="Resume from checkpoint") checkpoint: dict[str, Any] | None = Field(default=None, description="Full checkpoint object") diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index ea9e8662b..dffedce1c 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -271,6 +271,27 @@ async def start_run( agent_factory = resolve_agent_factory(body.assistant_id) graph_input = normalize_input(body.input) config = build_run_config(thread_id, body.config, body.metadata, assistant_id=body.assistant_id) + + # Merge DeerFlow-specific context overrides into configurable. + # The ``context`` field is a custom extension for the langgraph-compat layer + # that carries agent configuration (model_name, thinking_enabled, etc.). + # Only agent-relevant keys are forwarded; unknown keys (e.g. thread_id) are ignored. + context = getattr(body, "context", None) + if context: + _CONTEXT_CONFIGURABLE_KEYS = { + "model_name", + "mode", + "thinking_enabled", + "reasoning_effort", + "is_plan_mode", + "subagent_enabled", + "max_concurrent_subagents", + } + configurable = config.setdefault("configurable", {}) + for key in _CONTEXT_CONFIGURABLE_KEYS: + if key in context: + configurable.setdefault(key, context[key]) + stream_modes = normalize_stream_modes(body.stream_mode) task = asyncio.create_task( diff --git a/backend/tests/test_gateway_services.py b/backend/tests/test_gateway_services.py index 3921a2b34..6eab20a9f 100644 --- a/backend/tests/test_gateway_services.py +++ b/backend/tests/test_gateway_services.py @@ -160,3 +160,127 @@ def test_resolve_agent_factory_returns_make_lead_agent(): assert resolve_agent_factory("lead_agent") is make_lead_agent assert resolve_agent_factory("finalis") is make_lead_agent assert resolve_agent_factory("custom-agent-123") is make_lead_agent + + +# --------------------------------------------------------------------------- +# Regression tests for issue #1699: +# context field in langgraph-compat requests not merged into configurable +# --------------------------------------------------------------------------- + + +def test_run_create_request_accepts_context(): + """RunCreateRequest must accept the ``context`` field without dropping it.""" + from app.gateway.routers.thread_runs import RunCreateRequest + + body = RunCreateRequest( + input={"messages": [{"role": "user", "content": "hi"}]}, + context={ + "model_name": "deepseek-v3", + "thinking_enabled": True, + "is_plan_mode": True, + "subagent_enabled": True, + "thread_id": "some-thread-id", + }, + ) + assert body.context is not None + assert body.context["model_name"] == "deepseek-v3" + assert body.context["is_plan_mode"] is True + assert body.context["subagent_enabled"] is True + + +def test_run_create_request_context_defaults_to_none(): + """RunCreateRequest without context should default to None (backward compat).""" + from app.gateway.routers.thread_runs import RunCreateRequest + + body = RunCreateRequest(input=None) + assert body.context is None + + +def test_context_merges_into_configurable(): + """Context values must be merged into config['configurable'] by start_run. + + Since start_run is async and requires many dependencies, we test the + merging logic directly by simulating what start_run does. + """ + from app.gateway.services import build_run_config + + # Simulate the context merging logic from start_run + config = build_run_config("thread-1", None, None) + + context = { + "model_name": "deepseek-v3", + "mode": "ultra", + "reasoning_effort": "high", + "thinking_enabled": True, + "is_plan_mode": True, + "subagent_enabled": True, + "max_concurrent_subagents": 5, + "thread_id": "should-be-ignored", + } + + _CONTEXT_CONFIGURABLE_KEYS = { + "model_name", + "mode", + "thinking_enabled", + "reasoning_effort", + "is_plan_mode", + "subagent_enabled", + "max_concurrent_subagents", + } + configurable = config.setdefault("configurable", {}) + for key in _CONTEXT_CONFIGURABLE_KEYS: + if key in context: + configurable.setdefault(key, context[key]) + + assert config["configurable"]["model_name"] == "deepseek-v3" + assert config["configurable"]["thinking_enabled"] is True + assert config["configurable"]["is_plan_mode"] is True + assert config["configurable"]["subagent_enabled"] is True + assert config["configurable"]["max_concurrent_subagents"] == 5 + assert config["configurable"]["reasoning_effort"] == "high" + assert config["configurable"]["mode"] == "ultra" + # thread_id from context should NOT override the one from build_run_config + assert config["configurable"]["thread_id"] == "thread-1" + # Non-allowlisted keys should not appear + assert "thread_id" not in {k for k in context if k in _CONTEXT_CONFIGURABLE_KEYS} + + +def test_context_does_not_override_existing_configurable(): + """Values already in config.configurable must NOT be overridden by context. + + This ensures that explicit configurable values from the ``config`` field + take precedence over the ``context`` field. + """ + from app.gateway.services import build_run_config + + config = build_run_config( + "thread-1", + {"configurable": {"model_name": "gpt-4", "is_plan_mode": False}}, + None, + ) + + context = { + "model_name": "deepseek-v3", + "is_plan_mode": True, + "subagent_enabled": True, + } + + _CONTEXT_CONFIGURABLE_KEYS = { + "model_name", + "mode", + "thinking_enabled", + "reasoning_effort", + "is_plan_mode", + "subagent_enabled", + "max_concurrent_subagents", + } + configurable = config.setdefault("configurable", {}) + for key in _CONTEXT_CONFIGURABLE_KEYS: + if key in context: + configurable.setdefault(key, context[key]) + + # Existing values must NOT be overridden + assert config["configurable"]["model_name"] == "gpt-4" + assert config["configurable"]["is_plan_mode"] is False + # New values should be added + assert config["configurable"]["subagent_enabled"] is True From 68d44f6755b94047177311c29ad6da0e0ee8ca9d Mon Sep 17 00:00:00 2001 From: DanielWalnut <45447813+hetaoBackend@users.noreply.github.com> Date: Wed, 1 Apr 2026 21:04:13 +0800 Subject: [PATCH 05/47] fix: share .deer-flow in docker-compose-dev for uploads (#1718) * fix: share dev thread data between gateway and langgraph * refactor: drop redundant dev .deer-flow bind mounts --- docker/docker-compose-dev.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index 0bf6c5fb6..a77a957fa 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -149,6 +149,7 @@ services: working_dir: /app environment: - CI=true + - DEER_FLOW_HOME=/app/backend/.deer-flow - DEER_FLOW_CHANNELS_LANGGRAPH_URL=${DEER_FLOW_CHANNELS_LANGGRAPH_URL:-http://langgraph:2024} - DEER_FLOW_CHANNELS_GATEWAY_URL=${DEER_FLOW_CHANNELS_GATEWAY_URL:-http://gateway:8001} - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow @@ -204,6 +205,7 @@ services: working_dir: /app environment: - CI=true + - DEER_FLOW_HOME=/app/backend/.deer-flow - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow - DEER_FLOW_HOST_SKILLS_PATH=${DEER_FLOW_ROOT}/skills - DEER_FLOW_SANDBOX_HOST=host.docker.internal From e97c8c99431ab4f71fe0dde4fd427b14deb1c545 Mon Sep 17 00:00:00 2001 From: Alian Date: Wed, 1 Apr 2026 23:08:30 +0800 Subject: [PATCH 06/47] fix(skills): support parsing multiline YAML strings in SKILL.md frontmatter (#1703) * fix(skills): support parsing multiline YAML strings in SKILL.md frontmatter * test(skills): add tests for multiline YAML descriptions --- .../harness/deerflow/skills/parser.py | 67 +++++++++++++++++-- backend/tests/test_skills_parser.py | 21 ++++++ 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/backend/packages/harness/deerflow/skills/parser.py b/backend/packages/harness/deerflow/skills/parser.py index b7cb89761..d2a3af67b 100644 --- a/backend/packages/harness/deerflow/skills/parser.py +++ b/backend/packages/harness/deerflow/skills/parser.py @@ -33,15 +33,72 @@ def parse_skill_file(skill_file: Path, category: str, relative_path: Path | None front_matter = front_matter_match.group(1) - # Parse YAML front matter (simple key-value parsing) + # Parse YAML front matter with basic multiline string support metadata = {} - for line in front_matter.split("\n"): - line = line.strip() - if not line: + lines = front_matter.split("\n") + current_key = None + current_value = [] + is_multiline = False + multiline_style = None + indent_level = None + + for line in lines: + if is_multiline: + if not line.strip(): + current_value.append("") + continue + + current_indent = len(line) - len(line.lstrip()) + + if indent_level is None: + if current_indent > 0: + indent_level = current_indent + current_value.append(line[indent_level:]) + continue + elif current_indent >= indent_level: + current_value.append(line[indent_level:]) + continue + + # If we reach here, it's either a new key or the end of multiline + if current_key and is_multiline: + if multiline_style == "|": + metadata[current_key] = "\n".join(current_value).rstrip() + else: + text = "\n".join(current_value).rstrip() + # Replace single newlines with spaces for folded blocks + metadata[current_key] = re.sub(r"(?", "|"): + current_key = key + is_multiline = True + multiline_style = value + current_value = [] + indent_level = None + else: + metadata[key] = value + + if current_key and is_multiline: + if multiline_style == "|": + metadata[current_key] = "\n".join(current_value).rstrip() + else: + text = "\n".join(current_value).rstrip() + metadata[current_key] = re.sub(r"(?\n This is a multiline\n description for a skill.\n\n It spans multiple lines.\nlicense: MIT\n---\n\nBody\n", + ) + result = parse_skill_file(skill_file, "public") + assert result is not None + assert result.name == "multiline-skill" + assert result.description == "This is a multiline description for a skill.\n\nIt spans multiple lines." + assert result.license == "MIT" + + def test_multiline_yaml_literal_description(self, tmp_path): + skill_file = _write_skill( + tmp_path, + "---\nname: pipe-skill\ndescription: |\n First line.\n Second line.\n---\n\nBody\n", + ) + result = parse_skill_file(skill_file, "public") + assert result is not None + assert result.name == "pipe-skill" + assert result.description == "First line.\nSecond line." + def test_empty_front_matter_returns_none(self, tmp_path): skill_file = _write_skill(tmp_path, "---\n\n---\n\nBody\n") assert parse_skill_file(skill_file, "public") is None From 82c3dbbc6bb6c7a8e5349144ffd77125d22618b2 Mon Sep 17 00:00:00 2001 From: Admire <64821731+LittleChenLiya@users.noreply.github.com> Date: Wed, 1 Apr 2026 23:13:00 +0800 Subject: [PATCH 07/47] Fix Windows startup and dependency checks (#1709) * windows check and dev fixes * fix windows startup scripts * fix windows startup scripts --------- Co-authored-by: Willem Jiang --- Makefile | 11 +++++++- README.md | 1 + README_zh.md | 1 + scripts/check.py | 55 +++++++++++++++++++++++++-------------- scripts/config-upgrade.sh | 25 ++++++++++++------ scripts/serve.sh | 11 +++++++- scripts/start-daemon.sh | 1 + 7 files changed, 76 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 550e7cd41..e74a02db3 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ .PHONY: help config config-upgrade check install dev dev-daemon start stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway -PYTHON ?= python BASH ?= bash # Detect OS for Windows compatibility ifeq ($(OS),Windows_NT) SHELL := cmd.exe + PYTHON ?= python +else + PYTHON ?= python3 endif help: @@ -96,6 +98,7 @@ setup-sandbox: # Start all services in development mode (with hot-reloading) dev: + @$(PYTHON) ./scripts/check.py ifeq ($(OS),Windows_NT) @call scripts\run-with-git-bash.cmd ./scripts/serve.sh --dev else @@ -104,6 +107,7 @@ endif # Start all services in production mode (with optimizations) start: + @$(PYTHON) ./scripts/check.py ifeq ($(OS),Windows_NT) @call scripts\run-with-git-bash.cmd ./scripts/serve.sh --prod else @@ -112,7 +116,12 @@ endif # Start all services in daemon mode (background) dev-daemon: + @$(PYTHON) ./scripts/check.py +ifeq ($(OS),Windows_NT) + @call scripts\run-with-git-bash.cmd ./scripts/start-daemon.sh +else @./scripts/start-daemon.sh +endif # Stop all services stop: diff --git a/README.md b/README.md index 317a43f29..a527acd7c 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed Docker development guide. If you prefer running services locally: Prerequisite: complete the "Configuration" steps above first (`make config` and model API keys). `make dev` requires a valid configuration file (defaults to `config.yaml` in the project root; can be overridden via `DEER_FLOW_CONFIG_PATH`). +On Windows, run the local development flow from Git Bash. Native `cmd.exe` and PowerShell shells are not supported for the bash-based service scripts, and WSL is not guaranteed because some scripts rely on Git for Windows utilities such as `cygpath`. 1. **Check prerequisites**: ```bash diff --git a/README_zh.md b/README_zh.md index 3a838e72c..cbb3b5601 100644 --- a/README_zh.md +++ b/README_zh.md @@ -180,6 +180,7 @@ make down # 停止并移除容器 如果你更希望直接在本地启动各个服务： 前提：先完成上面的“配置”步骤（`make config` 和模型 API key 配置）。`make dev` 需要有效配置文件，默认读取项目根目录下的 `config.yaml`，也可以通过 `DEER_FLOW_CONFIG_PATH` 覆盖。 +在 Windows 上，请使用 Git Bash 运行本地开发流程。基于 bash 的服务脚本不支持直接在原生 `cmd.exe` 或 PowerShell 中执行，且 WSL 也不保证可用，因为部分脚本依赖 Git for Windows 的 `cygpath` 等工具。 1. **检查依赖环境**： ```bash diff --git a/scripts/check.py b/scripts/check.py index 650932376..d358e3086 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -29,6 +29,18 @@ def run_command(command: list[str]) -> Optional[str]: return result.stdout.strip() or result.stderr.strip() +def find_pnpm_command() -> Optional[list[str]]: + """Return a pnpm-compatible command that exists on this machine.""" + candidates = [["pnpm"], ["pnpm.cmd"]] + if shutil.which("corepack"): + candidates.append(["corepack", "pnpm"]) + + for command in candidates: + if shutil.which(command[0]): + return command + return None + + def parse_node_major(version_text: str) -> Optional[int]: version = version_text.strip() if version.startswith("v"): @@ -55,35 +67,39 @@ def main() -> int: if node_version: major = parse_node_major(node_version) if major is not None and major >= 22: - print(f" ✓ Node.js {node_version.lstrip('v')} (>= 22 required)") + print(f" OK Node.js {node_version.lstrip('v')} (>= 22 required)") else: print( - f" ✗ Node.js {node_version.lstrip('v')} found, but version 22+ is required" + f" FAIL Node.js {node_version.lstrip('v')} found, but version 22+ is required" ) print(" Install from: https://nodejs.org/") failed = True else: - print(" ✗ Unable to determine Node.js version") + print(" INFO Unable to determine Node.js version") print(" Install from: https://nodejs.org/") failed = True else: - print(" ✗ Node.js not found (version 22+ required)") + print(" FAIL Node.js not found (version 22+ required)") print(" Install from: https://nodejs.org/") failed = True print() print("Checking pnpm...") - pnpm_executable = shutil.which("pnpm.cmd") or shutil.which("pnpm") - if pnpm_executable: - pnpm_version = run_command([pnpm_executable, "-v"]) + pnpm_command = find_pnpm_command() + if pnpm_command: + pnpm_version = run_command([*pnpm_command, "-v"]) if pnpm_version: - print(f" ✓ pnpm {pnpm_version}") + if pnpm_command[0] == "corepack": + print(f" OK pnpm {pnpm_version} (via Corepack)") + else: + print(f" OK pnpm {pnpm_version}") else: - print(" ✗ Unable to determine pnpm version") + print(" INFO Unable to determine pnpm version") failed = True else: - print(" ✗ pnpm not found") + print(" FAIL pnpm not found") print(" Install: npm install -g pnpm") + print(" Or enable Corepack: corepack enable") print(" Or visit: https://pnpm.io/installation") failed = True @@ -92,13 +108,14 @@ def main() -> int: if shutil.which("uv"): uv_version_text = run_command(["uv", "--version"]) if uv_version_text: - uv_version = uv_version_text.split()[-1] - print(f" ✓ uv {uv_version}") + uv_version_parts = uv_version_text.split() + uv_version = uv_version_parts[1] if len(uv_version_parts) > 1 else uv_version_text + print(f" OK uv {uv_version}") else: - print(" ✗ Unable to determine uv version") + print(" INFO Unable to determine uv version") failed = True else: - print(" ✗ uv not found") + print(" FAIL uv not found") print(" Visit the official installation guide for your platform:") print(" https://docs.astral.sh/uv/getting-started/installation/") failed = True @@ -109,11 +126,11 @@ def main() -> int: nginx_version_text = run_command(["nginx", "-v"]) if nginx_version_text and "/" in nginx_version_text: nginx_version = nginx_version_text.split("/", 1)[1] - print(f" ✓ nginx {nginx_version}") + print(f" OK nginx {nginx_version}") else: - print(" ✓ nginx (version unknown)") + print(" INFO nginx (version unknown)") else: - print(" ✗ nginx not found") + print(" FAIL nginx not found") print(" macOS: brew install nginx") print(" Ubuntu: sudo apt install nginx") print(" Windows: use WSL for local mode or use Docker mode") @@ -123,7 +140,7 @@ def main() -> int: print() if not failed: print("==========================================") - print(" ✓ All dependencies are installed!") + print(" OK All dependencies are installed!") print("==========================================") print() print("You can now run:") @@ -134,7 +151,7 @@ def main() -> int: return 0 print("==========================================") - print(" ✗ Some dependencies are missing") + print(" FAIL Some dependencies are missing") print("==========================================") print() print("Please install the missing tools and run 'make check' again.") diff --git a/scripts/config-upgrade.sh b/scripts/config-upgrade.sh index 02d592608..96b0ceaf9 100755 --- a/scripts/config-upgrade.sh +++ b/scripts/config-upgrade.sh @@ -30,19 +30,28 @@ fi if [ -z "$CONFIG" ]; then echo "No config.yaml found — creating from example..." cp "$EXAMPLE" "$REPO_ROOT/config.yaml" - echo "✓ config.yaml created. Please review and set your API keys." + echo "OK config.yaml created. Please review and set your API keys." exit 0 fi # Use inline Python to do migrations + recursive merge with PyYAML -cd "$REPO_ROOT/backend" && uv run python3 -c " +if command -v cygpath >/dev/null 2>&1; then + CONFIG_WIN="$(cygpath -w "$CONFIG")" + EXAMPLE_WIN="$(cygpath -w "$EXAMPLE")" +else + CONFIG_WIN="$CONFIG" + EXAMPLE_WIN="$EXAMPLE" +fi + +cd "$REPO_ROOT/backend" && CONFIG_WIN_PATH="$CONFIG_WIN" EXAMPLE_WIN_PATH="$EXAMPLE_WIN" uv run python -c " +import os import sys, shutil, copy, re from pathlib import Path import yaml -config_path = Path('$CONFIG') -example_path = Path('$EXAMPLE') +config_path = Path(os.environ['CONFIG_WIN_PATH']) +example_path = Path(os.environ['EXAMPLE_WIN_PATH']) with open(config_path, encoding='utf-8') as f: raw_text = f.read() @@ -55,10 +64,10 @@ user_version = user.get('config_version', 0) example_version = example.get('config_version', 0) if user_version >= example_version: - print(f'✓ config.yaml is already up to date (version {user_version}).') + print(f'OK config.yaml is already up to date (version {user_version}).') sys.exit(0) -print(f'Upgrading config.yaml: version {user_version} → {example_version}') +print(f'Upgrading config.yaml: version {user_version} -> {example_version}') print() # ── Migrations ─────────────────────────────────────────────────────────── @@ -93,7 +102,7 @@ for version in range(user_version + 1, example_version + 1): for old, new in migration.get('replacements', []): if old in raw_text: raw_text = raw_text.replace(old, new) - migrated.append(f'{old} → {new}') + migrated.append(f'{old} -> {new}') # Re-parse after text migrations user = yaml.safe_load(raw_text) or {} @@ -141,6 +150,6 @@ if not migrated and not added: print('No changes needed (version bumped only).') print() -print(f'✓ config.yaml upgraded to version {example_version}.') +print(f'OK config.yaml upgraded to version {example_version}.') print(' Please review the changes and set any new required values.') " diff --git a/scripts/serve.sh b/scripts/serve.sh index d5d3b42cf..ae1c153ac 100755 --- a/scripts/serve.sh +++ b/scripts/serve.sh @@ -30,7 +30,15 @@ done if $DEV_MODE; then FRONTEND_CMD="pnpm run dev" else - FRONTEND_CMD="env BETTER_AUTH_SECRET=$(python3 -c 'import secrets; print(secrets.token_hex(16))') pnpm run preview" + if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN="python3" + elif command -v python >/dev/null 2>&1; then + PYTHON_BIN="python" + else + echo "Python is required to generate BETTER_AUTH_SECRET, but neither python3 nor python was found." + exit 1 + fi + FRONTEND_CMD="env BETTER_AUTH_SECRET=$($PYTHON_BIN -c 'import secrets; print(secrets.token_hex(16))') pnpm run preview" fi # ── Stop existing services ──────────────────────────────────────────────────── @@ -121,6 +129,7 @@ trap cleanup INT TERM # ── Start services ──────────────────────────────────────────────────────────── mkdir -p logs +mkdir -p temp/client_body_temp temp/proxy_temp temp/fastcgi_temp temp/uwsgi_temp temp/scgi_temp if $DEV_MODE; then LANGGRAPH_EXTRA_FLAGS="--no-reload" diff --git a/scripts/start-daemon.sh b/scripts/start-daemon.sh index a60653dbc..96ee788e1 100755 --- a/scripts/start-daemon.sh +++ b/scripts/start-daemon.sh @@ -71,6 +71,7 @@ trap cleanup_on_failure INT TERM # ── Start services ──────────────────────────────────────────────────────────── mkdir -p logs +mkdir -p temp/client_body_temp temp/proxy_temp temp/fastcgi_temp temp/uwsgi_temp temp/scgi_temp echo "Starting LangGraph server..." nohup sh -c 'cd backend && NO_COLOR=1 uv run langgraph dev --no-browser --allow-blocking --no-reload > ../logs/langgraph.log 2>&1' & From 1fb5acee3956338f4844f51afb6e30a79219a14f Mon Sep 17 00:00:00 2001 From: Jason <101583541+JasonOA888@users.noreply.github.com> Date: Wed, 1 Apr 2026 23:21:32 +0800 Subject: [PATCH 08/47] fix(gateway): prevent 400 error when client sends context with configurable (#1660) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(gateway): prevent 400 error when client sends context with configurable Fixes #1290 LangGraph >= 0.6.0 rejects requests that include both 'configurable' and 'context' in the run config. If the client (e.g. useStream hook) sends a 'context' key, we now honour it and skip creating our own 'configurable' dict to avoid the conflict. When no 'context' is provided, we fall back to the existing 'configurable' behaviour with thread_id. * fix(gateway): address review feedback — warn on dual keys, fix runtime injection, add tests - Log a warning when client sends both 'context' and 'configurable' so it's no longer silently dropped (reviewer feedback) - Ensure thread_id is available in config['context'] when present so middlewares can find it there too - Add test coverage for the context path, the both-keys-present case, passthrough of other keys, and the no-config fallback * style: ruff format services.py --------- Co-authored-by: JasonOA888 Co-authored-by: Willem Jiang --- backend/app/gateway/services.py | 44 ++++++---- .../harness/deerflow/runtime/runs/worker.py | 5 ++ backend/tests/test_gateway_services.py | 84 +++++++++++++++---- 3 files changed, 103 insertions(+), 30 deletions(-) diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index dffedce1c..272801b6a 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -129,26 +129,38 @@ def build_run_config( the LangGraph Platform-compatible HTTP API and the IM channel path behave identically. """ - configurable: dict[str, Any] = {"thread_id": thread_id} + config: dict[str, Any] = {"recursion_limit": 100} if request_config: - configurable.update(request_config.get("configurable", {})) + # LangGraph >= 0.6.0 introduced ``context`` as the preferred way to + # pass thread-level data and rejects requests that include both + # ``configurable`` and ``context``. If the caller already sends + # ``context``, honour it and skip our own ``configurable`` dict. + if "context" in request_config: + if "configurable" in request_config: + logger.warning( + "build_run_config: client sent both 'context' and 'configurable'; preferring 'context' (LangGraph >= 0.6.0). thread_id=%s, caller_configurable keys=%s", + thread_id, + list(request_config.get("configurable", {}).keys()), + ) + config["context"] = request_config["context"] + else: + configurable = {"thread_id": thread_id} + configurable.update(request_config.get("configurable", {})) + config["configurable"] = configurable + for k, v in request_config.items(): + if k not in ("configurable", "context"): + config[k] = v + else: + config["configurable"] = {"thread_id": thread_id} # Inject custom agent name when the caller specified a non-default assistant. # Honour an explicit configurable["agent_name"] in the request if already set. - if assistant_id and assistant_id != _DEFAULT_ASSISTANT_ID and "agent_name" not in configurable: - # Normalize the same way ChannelManager does: strip, lowercase, - # replace underscores with hyphens, then validate to prevent path - # traversal and invalid agent directory lookups. - normalized = assistant_id.strip().lower().replace("_", "-") - if not normalized or not re.fullmatch(r"[a-z0-9-]+", normalized): - raise ValueError(f"Invalid assistant_id {assistant_id!r}: must contain only letters, digits, and hyphens after normalization.") - configurable["agent_name"] = normalized - - config: dict[str, Any] = {"configurable": configurable, "recursion_limit": 100} - if request_config: - for k, v in request_config.items(): - if k != "configurable": - config[k] = v + if assistant_id and assistant_id != _DEFAULT_ASSISTANT_ID and "configurable" in config: + if "agent_name" not in config["configurable"]: + normalized = assistant_id.strip().lower().replace("_", "-") + if not normalized or not re.fullmatch(r"[a-z0-9-]+", normalized): + raise ValueError(f"Invalid assistant_id {assistant_id!r}: must contain only letters, digits, and hyphens after normalization.") + config["configurable"]["agent_name"] = normalized if metadata: config.setdefault("metadata", {}).update(metadata) return config diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index deaec055a..2d67ecb27 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -90,6 +90,11 @@ async def run_agent( # Inject runtime context so middlewares can access thread_id # (langgraph-cli does this automatically; we must do it manually) runtime = Runtime(context={"thread_id": thread_id}, store=store) + # If the caller already set a ``context`` key (LangGraph >= 0.6.0 + # prefers it over ``configurable`` for thread-level data), make + # sure ``thread_id`` is available there too. + if "context" in config and isinstance(config["context"], dict): + config["context"].setdefault("thread_id", thread_id) config.setdefault("configurable", {})["__pregel_runtime"] = runtime runnable_config = RunnableConfig(**config) diff --git a/backend/tests/test_gateway_services.py b/backend/tests/test_gateway_services.py index 6eab20a9f..782306e38 100644 --- a/backend/tests/test_gateway_services.py +++ b/backend/tests/test_gateway_services.py @@ -109,17 +109,11 @@ def test_build_run_config_with_overrides(): def test_build_run_config_custom_agent_injects_agent_name(): - """Custom assistant_id must be forwarded as configurable['agent_name']. - - Regression test for #1644: when the LangGraph Platform-compatible - /runs endpoint receives a custom assistant_id (e.g. 'finalis'), the - Gateway must inject configurable['agent_name'] so that make_lead_agent - loads the correct agents/finalis/SOUL.md. - """ + """Custom assistant_id must be forwarded as configurable['agent_name'].""" from app.gateway.services import build_run_config config = build_run_config("thread-1", None, None, assistant_id="finalis") - assert config["configurable"]["agent_name"] == "finalis", "Custom assistant_id must be forwarded as configurable['agent_name'] so that make_lead_agent loads the correct SOUL.md" + assert config["configurable"]["agent_name"] == "finalis" def test_build_run_config_lead_agent_no_agent_name(): @@ -148,7 +142,7 @@ def test_build_run_config_explicit_agent_name_not_overwritten(): None, assistant_id="other-agent", ) - assert config["configurable"]["agent_name"] == "explicit-agent", "An explicit configurable['agent_name'] in the request body must not be overwritten by the assistant_id mapping" + assert config["configurable"]["agent_name"] == "explicit-agent" def test_resolve_agent_factory_returns_make_lead_agent(): @@ -162,6 +156,8 @@ def test_resolve_agent_factory_returns_make_lead_agent(): assert resolve_agent_factory("custom-agent-123") is make_lead_agent +# --------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # Regression tests for issue #1699: # context field in langgraph-compat requests not merged into configurable @@ -246,11 +242,7 @@ def test_context_merges_into_configurable(): def test_context_does_not_override_existing_configurable(): - """Values already in config.configurable must NOT be overridden by context. - - This ensures that explicit configurable values from the ``config`` field - take precedence over the ``context`` field. - """ + """Values already in config.configurable must NOT be overridden by context.""" from app.gateway.services import build_run_config config = build_run_config( @@ -284,3 +276,67 @@ def test_context_does_not_override_existing_configurable(): assert config["configurable"]["is_plan_mode"] is False # New values should be added assert config["configurable"]["subagent_enabled"] is True + + +# --------------------------------------------------------------------------- +# build_run_config — context / configurable precedence (LangGraph >= 0.6.0) +# --------------------------------------------------------------------------- + + +def test_build_run_config_with_context(): + """When caller sends 'context', prefer it over 'configurable'.""" + from app.gateway.services import build_run_config + + config = build_run_config( + "thread-1", + {"context": {"user_id": "u-42", "thread_id": "thread-1"}}, + None, + ) + assert "context" in config + assert config["context"]["user_id"] == "u-42" + assert "configurable" not in config + assert config["recursion_limit"] == 100 + + +def test_build_run_config_context_plus_configurable_warns(caplog): + """When caller sends both 'context' and 'configurable', prefer 'context' and log a warning.""" + import logging + + from app.gateway.services import build_run_config + + with caplog.at_level(logging.WARNING, logger="app.gateway.services"): + config = build_run_config( + "thread-1", + { + "context": {"user_id": "u-42"}, + "configurable": {"model_name": "gpt-4"}, + }, + None, + ) + assert "context" in config + assert config["context"]["user_id"] == "u-42" + assert "configurable" not in config + assert any("both 'context' and 'configurable'" in r.message for r in caplog.records) + + +def test_build_run_config_context_passthrough_other_keys(): + """Non-conflicting keys from request_config are still passed through when context is used.""" + from app.gateway.services import build_run_config + + config = build_run_config( + "thread-1", + {"context": {"thread_id": "thread-1"}, "tags": ["prod"]}, + None, + ) + assert config["context"]["thread_id"] == "thread-1" + assert "configurable" not in config + assert config["tags"] == ["prod"] + + +def test_build_run_config_no_request_config(): + """When request_config is None, fall back to basic configurable with thread_id.""" + from app.gateway.services import build_run_config + + config = build_run_config("thread-abc", None, None) + assert config["configurable"] == {"thread_id": "thread-abc"} + assert "context" not in config From 0a379602b80b45a1f91f5815e7171fa5d465fe63 Mon Sep 17 00:00:00 2001 From: LYU Yichen <1836114995@qq.com> Date: Wed, 1 Apr 2026 23:23:00 +0800 Subject: [PATCH 09/47] fix: avoid treating Feishu file paths as commands (#1654) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feishu channel classified any slash-prefixed text (including absolute paths such as /mnt/user-data/...) as a COMMAND, causing them to be misrouted through the command pipeline instead of the chat pipeline. Fix by introducing a shared KNOWN_CHANNEL_COMMANDS frozenset in app/channels/commands.py — the single authoritative source for the set of supported slash commands. Both the Feishu inbound parser and the ChannelManager's unknown-command reply now derive from it, so adding or removing a command requires only one edit. Changes: - app/channels/commands.py (new): defines KNOWN_CHANNEL_COMMANDS - app/channels/feishu.py: replace local KNOWN_FEISHU_COMMANDS with the shared constant; _is_feishu_command() now gates on it - app/channels/manager.py: import KNOWN_CHANNEL_COMMANDS and use it in the unknown-command fallback reply so the displayed list stays in sync - tests/test_feishu_parser.py: parametrize over every entry in KNOWN_CHANNEL_COMMANDS (each must yield msg_type=command) and add parametrized chat cases for /unknown, absolute paths, etc. Made with Cursor Made-with: Cursor Co-authored-by: Willem Jiang --- backend/app/channels/commands.py | 20 +++++++++++ backend/app/channels/feishu.py | 12 +++++-- backend/app/channels/manager.py | 4 ++- backend/tests/test_feishu_parser.py | 55 +++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 backend/app/channels/commands.py diff --git a/backend/app/channels/commands.py b/backend/app/channels/commands.py new file mode 100644 index 000000000..704330410 --- /dev/null +++ b/backend/app/channels/commands.py @@ -0,0 +1,20 @@ +"""Shared command definitions used by all channel implementations. + +Keeping the authoritative command set in one place ensures that channel +parsers (e.g. Feishu) and the ChannelManager dispatcher stay in sync +automatically — adding or removing a command here is the single edit +required. +""" + +from __future__ import annotations + +KNOWN_CHANNEL_COMMANDS: frozenset[str] = frozenset( + { + "/bootstrap", + "/new", + "/status", + "/models", + "/memory", + "/help", + } +) diff --git a/backend/app/channels/feishu.py b/backend/app/channels/feishu.py index 2001a87de..32f71f1f6 100644 --- a/backend/app/channels/feishu.py +++ b/backend/app/channels/feishu.py @@ -9,11 +9,18 @@ import threading from typing import Any from app.channels.base import Channel +from app.channels.commands import KNOWN_CHANNEL_COMMANDS from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment logger = logging.getLogger(__name__) +def _is_feishu_command(text: str) -> bool: + if not text.startswith("/"): + return False + return text.split(maxsplit=1)[0].lower() in KNOWN_CHANNEL_COMMANDS + + class FeishuChannel(Channel): """Feishu/Lark IM channel using the ``lark-oapi`` WebSocket client. @@ -509,8 +516,9 @@ class FeishuChannel(Channel): logger.info("[Feishu] empty text, ignoring message") return - # Check if it's a command - if text.startswith("/"): + # Only treat known slash commands as commands; absolute paths and + # other slash-prefixed text should be handled as normal chat. + if _is_feishu_command(text): msg_type = InboundMessageType.COMMAND else: msg_type = InboundMessageType.CHAT diff --git a/backend/app/channels/manager.py b/backend/app/channels/manager.py index 3e3fa17d4..ab63100fa 100644 --- a/backend/app/channels/manager.py +++ b/backend/app/channels/manager.py @@ -12,6 +12,7 @@ from typing import Any from langgraph_sdk.errors import ConflictError +from app.channels.commands import KNOWN_CHANNEL_COMMANDS from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment from app.channels.store import ChannelStore @@ -735,7 +736,8 @@ class ChannelManager: "/help — Show this help" ) else: - reply = f"Unknown command: /{command}. Type /help for available commands." + available = " | ".join(sorted(KNOWN_CHANNEL_COMMANDS)) + reply = f"Unknown command: /{command}. Available commands: {available}" outbound = OutboundMessage( channel_name=msg.channel_name, diff --git a/backend/tests/test_feishu_parser.py b/backend/tests/test_feishu_parser.py index 39dc6e450..7a1fd9fc7 100644 --- a/backend/tests/test_feishu_parser.py +++ b/backend/tests/test_feishu_parser.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import pytest +from app.channels.commands import KNOWN_CHANNEL_COMMANDS from app.channels.feishu import FeishuChannel from app.channels.message_bus import MessageBus @@ -68,3 +69,57 @@ def test_feishu_on_message_rich_text(): assert "Paragraph 1, part 1. Paragraph 1, part 2." in parsed_text assert "@bot Paragraph 2." in parsed_text assert "\n\n" in parsed_text + + +@pytest.mark.parametrize("command", sorted(KNOWN_CHANNEL_COMMANDS)) +def test_feishu_recognizes_all_known_slash_commands(command): + """Every entry in KNOWN_CHANNEL_COMMANDS must be classified as a command.""" + bus = MessageBus() + config = {"app_id": "test", "app_secret": "test"} + channel = FeishuChannel(bus, config) + + event = MagicMock() + event.event.message.chat_id = "chat_1" + event.event.message.message_id = "msg_1" + event.event.message.root_id = None + event.event.sender.sender_id.open_id = "user_1" + event.event.message.content = json.dumps({"text": command}) + + with pytest.MonkeyPatch.context() as m: + mock_make_inbound = MagicMock() + m.setattr(channel, "_make_inbound", mock_make_inbound) + channel._on_message(event) + + mock_make_inbound.assert_called_once() + assert mock_make_inbound.call_args[1]["msg_type"].value == "command", f"{command!r} should be classified as COMMAND" + + +@pytest.mark.parametrize( + "text", + [ + "/unknown", + "/mnt/user-data/outputs/prd/technical-design.md", + "/etc/passwd", + "/not-a-command at all", + ], +) +def test_feishu_treats_unknown_slash_text_as_chat(text): + """Slash-prefixed text that is not a known command must be classified as CHAT.""" + bus = MessageBus() + config = {"app_id": "test", "app_secret": "test"} + channel = FeishuChannel(bus, config) + + event = MagicMock() + event.event.message.chat_id = "chat_1" + event.event.message.message_id = "msg_1" + event.event.message.root_id = None + event.event.sender.sender_id.open_id = "user_1" + event.event.message.content = json.dumps({"text": text}) + + with pytest.MonkeyPatch.context() as m: + mock_make_inbound = MagicMock() + m.setattr(channel, "_make_inbound", mock_make_inbound) + channel._on_message(event) + + mock_make_inbound.assert_called_once() + assert mock_make_inbound.call_args[1]["msg_type"].value == "chat", f"{text!r} should be classified as CHAT" From 0eb6550cf4c3b853056de404f96937bc121f4ae9 Mon Sep 17 00:00:00 2001 From: Admire <64821731+LittleChenLiya@users.noreply.github.com> Date: Wed, 1 Apr 2026 23:27:03 +0800 Subject: [PATCH 10/47] fix(frontend): persist model selection per thread (#1553) * fix(frontend): persist model selection per thread * fix(frontend): apply thread model override on fallback * refactor(frontend): split thread settings hook * fix frontend local storage guards --- .../[agent_name]/chats/[thread_id]/page.tsx | 4 +- .../app/workspace/chats/[thread_id]/page.tsx | 5 +- frontend/src/core/settings/hooks.ts | 58 +++++---- frontend/src/core/settings/local.ts | 110 +++++++++++++++--- 4 files changed, 133 insertions(+), 44 deletions(-) diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx index 12ba855e3..482f25c74 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx @@ -20,7 +20,7 @@ import { Tooltip } from "@/components/workspace/tooltip"; import { useAgent } from "@/core/agents"; import { useI18n } from "@/core/i18n/hooks"; import { useNotification } from "@/core/notification/hooks"; -import { useLocalSettings } from "@/core/settings"; +import { useThreadSettings } from "@/core/settings"; import { useThreadStream } from "@/core/threads/hooks"; import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; @@ -28,7 +28,6 @@ import { cn } from "@/lib/utils"; export default function AgentChatPage() { const { t } = useI18n(); - const [settings, setSettings] = useLocalSettings(); const router = useRouter(); const { agent_name } = useParams<{ @@ -38,6 +37,7 @@ export default function AgentChatPage() { const { agent } = useAgent(agent_name); const { threadId, isNewThread, setIsNewThread } = useThreadChat(); + const [settings, setSettings] = useThreadSettings(threadId); const { showNotification } = useNotification(); const [thread, sendMessage] = useThreadStream({ diff --git a/frontend/src/app/workspace/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/chats/[thread_id]/page.tsx index 8805522ad..0cff87d0d 100644 --- a/frontend/src/app/workspace/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/chats/[thread_id]/page.tsx @@ -19,7 +19,7 @@ import { TokenUsageIndicator } from "@/components/workspace/token-usage-indicato import { Welcome } from "@/components/workspace/welcome"; import { useI18n } from "@/core/i18n/hooks"; import { useNotification } from "@/core/notification/hooks"; -import { useLocalSettings } from "@/core/settings"; +import { useThreadSettings } from "@/core/settings"; import { useThreadStream } from "@/core/threads/hooks"; import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; @@ -27,9 +27,8 @@ import { cn } from "@/lib/utils"; export default function ChatPage() { const { t } = useI18n(); - const [settings, setSettings] = useLocalSettings(); - const { threadId, isNewThread, setIsNewThread, isMock } = useThreadChat(); + const [settings, setSettings] = useThreadSettings(threadId); useSpecificChatMode(); const { showNotification } = useNotification(); diff --git a/frontend/src/core/settings/hooks.ts b/frontend/src/core/settings/hooks.ts index 47797d489..f5ec0d2a6 100644 --- a/frontend/src/core/settings/hooks.ts +++ b/frontend/src/core/settings/hooks.ts @@ -3,44 +3,62 @@ import { useCallback, useLayoutEffect, useState } from "react"; import { DEFAULT_LOCAL_SETTINGS, getLocalSettings, + getThreadLocalSettings, saveLocalSettings, + saveThreadLocalSettings, type LocalSettings, } from "./local"; -export function useLocalSettings(): [ - LocalSettings, - ( - key: keyof LocalSettings, - value: Partial, - ) => void, -] { - const [mounted, setMounted] = useState(false); +type LocalSettingsSetter = ( + key: keyof LocalSettings, + value: Partial, +) => void; + +function useSettingsState( + getSettings: () => LocalSettings, + saveSettings: (settings: LocalSettings) => void, +): [LocalSettings, LocalSettingsSetter] { const [state, setState] = useState(DEFAULT_LOCAL_SETTINGS); + + const [mounted, setMounted] = useState(false); useLayoutEffect(() => { - if (!mounted) { - setState(getLocalSettings()); - } + setState(getSettings()); setMounted(true); - }, [mounted]); - const setter = useCallback( - ( - key: keyof LocalSettings, - value: Partial, - ) => { + }, [getSettings]); + + const setter = useCallback( + (key, value) => { if (!mounted) return; setState((prev) => { - const newState = { + const newState: LocalSettings = { ...prev, [key]: { ...prev[key], ...value, }, }; - saveLocalSettings(newState); + saveSettings(newState); return newState; }); }, - [mounted], + [mounted, saveSettings], ); + return [state, setter]; } + +export function useLocalSettings(): [LocalSettings, LocalSettingsSetter] { + return useSettingsState(getLocalSettings, saveLocalSettings); +} + +export function useThreadSettings( + threadId: string, +): [LocalSettings, LocalSettingsSetter] { + return useSettingsState( + useCallback(() => getThreadLocalSettings(threadId), [threadId]), + useCallback( + (settings: LocalSettings) => saveThreadLocalSettings(threadId, settings), + [threadId], + ), + ); +} diff --git a/frontend/src/core/settings/local.ts b/frontend/src/core/settings/local.ts index d560e7b8d..562ed7046 100644 --- a/frontend/src/core/settings/local.ts +++ b/frontend/src/core/settings/local.ts @@ -15,6 +15,11 @@ export const DEFAULT_LOCAL_SETTINGS: LocalSettings = { }; const LOCAL_SETTINGS_KEY = "deerflow.local-settings"; +const THREAD_MODEL_KEY_PREFIX = "deerflow.thread-model."; + +function isBrowser(): boolean { + return typeof window !== "undefined"; +} export interface LocalSettings { notification: { @@ -22,8 +27,14 @@ export interface LocalSettings { }; context: Omit< AgentThreadContext, - "thread_id" | "is_plan_mode" | "thinking_enabled" | "subagent_enabled" + | "thread_id" + | "is_plan_mode" + | "thinking_enabled" + | "subagent_enabled" + | "model_name" + | "reasoning_effort" > & { + model_name?: string | undefined; mode: "flash" | "thinking" | "pro" | "ultra" | undefined; reasoning_effort?: "minimal" | "low" | "medium" | "high"; }; @@ -32,35 +43,96 @@ export interface LocalSettings { }; } +function mergeLocalSettings(settings?: Partial): LocalSettings { + return { + ...DEFAULT_LOCAL_SETTINGS, + context: { + ...DEFAULT_LOCAL_SETTINGS.context, + ...settings?.context, + }, + layout: { + ...DEFAULT_LOCAL_SETTINGS.layout, + ...settings?.layout, + }, + notification: { + ...DEFAULT_LOCAL_SETTINGS.notification, + ...settings?.notification, + }, + }; +} + +function getThreadModelStorageKey(threadId: string): string { + return `${THREAD_MODEL_KEY_PREFIX}${threadId}`; +} + +export function getThreadModelName(threadId: string): string | undefined { + if (!isBrowser()) { + return undefined; + } + return localStorage.getItem(getThreadModelStorageKey(threadId)) ?? undefined; +} + +export function saveThreadModelName( + threadId: string, + modelName: string | undefined, +) { + if (!isBrowser()) { + return; + } + const key = getThreadModelStorageKey(threadId); + if (!modelName) { + localStorage.removeItem(key); + return; + } + localStorage.setItem(key, modelName); +} + +function applyThreadModelOverride( + settings: LocalSettings, + threadId?: string, +): LocalSettings { + const threadModelName = threadId ? getThreadModelName(threadId) : undefined; + if (!threadModelName) { + return settings; + } + return { + ...settings, + context: { + ...settings.context, + model_name: threadModelName, + }, + }; +} + export function getLocalSettings(): LocalSettings { - if (typeof window === "undefined") { + if (!isBrowser()) { return DEFAULT_LOCAL_SETTINGS; } const json = localStorage.getItem(LOCAL_SETTINGS_KEY); try { if (json) { - const settings = JSON.parse(json); - const mergedSettings = { - ...DEFAULT_LOCAL_SETTINGS, - context: { - ...DEFAULT_LOCAL_SETTINGS.context, - ...settings.context, - }, - layout: { - ...DEFAULT_LOCAL_SETTINGS.layout, - ...settings.layout, - }, - notification: { - ...DEFAULT_LOCAL_SETTINGS.notification, - ...settings.notification, - }, - }; - return mergedSettings; + const settings = JSON.parse(json) as Partial; + return mergeLocalSettings(settings); } } catch {} return DEFAULT_LOCAL_SETTINGS; } +export function getThreadLocalSettings(threadId: string): LocalSettings { + return applyThreadModelOverride(getLocalSettings(), threadId); +} + export function saveLocalSettings(settings: LocalSettings) { + if (!isBrowser()) { + return; + } localStorage.setItem(LOCAL_SETTINGS_KEY, JSON.stringify(settings)); } + +export function saveThreadLocalSettings( + threadId: string, + settings: LocalSettings, +) { + saveLocalSettings(settings); + saveThreadModelName(threadId, settings.context.model_name); +} From df5339b5d076fc71dd4e54d3f69c396fc2d63944 Mon Sep 17 00:00:00 2001 From: SHIYAO ZHANG <834247613@qq.com> Date: Thu, 2 Apr 2026 09:22:41 +0800 Subject: [PATCH 11/47] feat(sandbox): truncate oversized bash and read_file tool outputs (#1677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sandbox): truncate oversized bash and read_file tool outputs Long tool outputs (large directory listings, multi-MB source files) can overflow the model's context window. Two new configurable limits: - bash_output_max_chars (default 20000): middle-truncates bash output, preserving both head and tail so stderr at the end is not lost - read_file_output_max_chars (default 50000): head-truncates file output with a hint to use start_line/end_line for targeted reads Both limits are enforced at the tool layer (sandbox/tools.py) rather than middleware, so truncation is guaranteed regardless of call path. Setting either limit to 0 disables truncation entirely. Measured: read_file on a 250KB source file drops from 63,698 tokens to 19,927 tokens (69% reduction) with the default limit. Co-Authored-By: Claude Sonnet 4.6 * fix(tests): remove unused pytest import and fix import sort order * style: apply ruff format to sandbox/tools.py * refactor(sandbox): address Copilot review feedback on truncation feature - strict hard cap: while-loop ensures result (including marker) ≤ max_chars - max_chars=0 now returns "" instead of original output - get_app_config() wrapped in try/except with fallback to defaults - sandbox_config.py: add ge=0 validation on truncation limit fields - config.example.yaml: bump config_version 4→5 - tests: add len(result) <= max_chars assertions, edge-case (max=0, small max, various sizes) tests; fix skipped-count test for strict hard cap * refactor(sandbox): replace while-loop truncation with fixed marker budget Use a pre-allocated constant (_MARKER_MAX_LEN) instead of a convergence loop to ensure result <= max_chars. Simpler, safer, and skipped-char count in the marker is now an exact predictable value. * refactor(sandbox): compute marker budget dynamically instead of hardcoding * fix(sandbox): make max_chars=0 disable truncation instead of returning empty string --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: JeffJiang --- .../harness/deerflow/config/sandbox_config.py | 11 ++ .../harness/deerflow/sandbox/tools.py | 80 ++++++++- backend/tests/test_tool_output_truncation.py | 161 ++++++++++++++++++ config.example.yaml | 9 +- 4 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_tool_output_truncation.py diff --git a/backend/packages/harness/deerflow/config/sandbox_config.py b/backend/packages/harness/deerflow/config/sandbox_config.py index d5447f3dc..0634ce7b9 100644 --- a/backend/packages/harness/deerflow/config/sandbox_config.py +++ b/backend/packages/harness/deerflow/config/sandbox_config.py @@ -64,4 +64,15 @@ class SandboxConfig(BaseModel): description="Environment variables to inject into the sandbox container. Values starting with $ will be resolved from host environment variables.", ) + bash_output_max_chars: int = Field( + default=20000, + ge=0, + description="Maximum characters to keep from bash tool output. Output exceeding this limit is middle-truncated (head + tail), preserving the first and last half. Set to 0 to disable truncation.", + ) + read_file_output_max_chars: int = Field( + default=50000, + ge=0, + description="Maximum characters to keep from read_file tool output. Output exceeding this limit is head-truncated. Set to 0 to disable truncation.", + ) + model_config = ConfigDict(extra="allow") diff --git a/backend/packages/harness/deerflow/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py index 432669b65..7cdaa26da 100644 --- a/backend/packages/harness/deerflow/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -757,6 +757,59 @@ def ensure_thread_directories_exist(runtime: ToolRuntime[ContextT, ThreadState] runtime.state["thread_directories_created"] = True +def _truncate_bash_output(output: str, max_chars: int) -> str: + """Middle-truncate bash output, preserving head and tail (50/50 split). + + bash output may have errors at either end (stderr/stdout ordering is + non-deterministic), so both ends are preserved equally. + + The returned string (including the truncation marker) is guaranteed to be + no longer than max_chars characters. Pass max_chars=0 to disable truncation + and return the full output unchanged. + """ + if max_chars == 0: + return output + if len(output) <= max_chars: + return output + total_len = len(output) + # Compute the exact worst-case marker length: skipped chars is at most + # total_len, so this is a tight upper bound. + marker_max_len = len(f"\n... [middle truncated: {total_len} chars skipped] ...\n") + kept = max(0, max_chars - marker_max_len) + if kept == 0: + return output[:max_chars] + head_len = kept // 2 + tail_len = kept - head_len + skipped = total_len - kept + marker = f"\n... [middle truncated: {skipped} chars skipped] ...\n" + return f"{output[:head_len]}{marker}{output[-tail_len:] if tail_len > 0 else ''}" + + +def _truncate_read_file_output(output: str, max_chars: int) -> str: + """Head-truncate read_file output, preserving the beginning of the file. + + Source code and documents are read top-to-bottom; the head contains the + most context (imports, class definitions, function signatures). + + The returned string (including the truncation marker) is guaranteed to be + no longer than max_chars characters. Pass max_chars=0 to disable truncation + and return the full output unchanged. + """ + if max_chars == 0: + return output + if len(output) <= max_chars: + return output + total = len(output) + # Compute the exact worst-case marker length: both numeric fields are at + # their maximum (total chars), so this is a tight upper bound. + marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars. Use start_line/end_line to read a specific range] ...") + kept = max(0, max_chars - marker_max_len) + if kept == 0: + return output[:max_chars] + marker = f"\n... [truncated: showing first {kept} of {total} chars. Use start_line/end_line to read a specific range] ..." + return f"{output[:kept]}{marker}" + + @tool("bash", parse_docstring=True) def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, command: str) -> str: """Execute a bash command in a Linux environment. @@ -781,9 +834,23 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com command = replace_virtual_paths_in_command(command, thread_data) command = _apply_cwd_prefix(command, thread_data) output = sandbox.execute_command(command) - return mask_local_paths_in_output(output, thread_data) + try: + from deerflow.config.app_config import get_app_config + + sandbox_cfg = get_app_config().sandbox + max_chars = sandbox_cfg.bash_output_max_chars if sandbox_cfg else 20000 + except Exception: + max_chars = 20000 + return _truncate_bash_output(mask_local_paths_in_output(output, thread_data), max_chars) ensure_thread_directories_exist(runtime) - return sandbox.execute_command(command) + try: + from deerflow.config.app_config import get_app_config + + sandbox_cfg = get_app_config().sandbox + max_chars = sandbox_cfg.bash_output_max_chars if sandbox_cfg else 20000 + except Exception: + max_chars = 20000 + return _truncate_bash_output(sandbox.execute_command(command), max_chars) except SandboxError as e: return f"Error: {e}" except PermissionError as e: @@ -861,7 +928,14 @@ def read_file_tool( return "(empty)" if start_line is not None and end_line is not None: content = "\n".join(content.splitlines()[start_line - 1 : end_line]) - return content + try: + from deerflow.config.app_config import get_app_config + + sandbox_cfg = get_app_config().sandbox + max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000 + except Exception: + max_chars = 50000 + return _truncate_read_file_output(content, max_chars) except SandboxError as e: return f"Error: {e}" except FileNotFoundError: diff --git a/backend/tests/test_tool_output_truncation.py b/backend/tests/test_tool_output_truncation.py new file mode 100644 index 000000000..e76bb20e2 --- /dev/null +++ b/backend/tests/test_tool_output_truncation.py @@ -0,0 +1,161 @@ +"""Unit tests for tool output truncation functions. + +These functions truncate long tool outputs to prevent context window overflow. +- _truncate_bash_output: middle-truncation (head + tail), for bash tool +- _truncate_read_file_output: head-truncation, for read_file tool +""" + +from deerflow.sandbox.tools import _truncate_bash_output, _truncate_read_file_output + +# --------------------------------------------------------------------------- +# _truncate_bash_output +# --------------------------------------------------------------------------- + + +class TestTruncateBashOutput: + def test_short_output_returned_unchanged(self): + output = "hello world" + assert _truncate_bash_output(output, 20000) == output + + def test_output_equal_to_limit_returned_unchanged(self): + output = "A" * 20000 + assert _truncate_bash_output(output, 20000) == output + + def test_long_output_is_truncated(self): + output = "A" * 30000 + result = _truncate_bash_output(output, 20000) + assert len(result) < len(output) + + def test_result_never_exceeds_max_chars(self): + output = "A" * 30000 + max_chars = 20000 + result = _truncate_bash_output(output, max_chars) + assert len(result) <= max_chars + + def test_head_is_preserved(self): + head = "HEAD_CONTENT" + output = head + "M" * 30000 + result = _truncate_bash_output(output, 20000) + assert result.startswith(head) + + def test_tail_is_preserved(self): + tail = "TAIL_CONTENT" + output = "M" * 30000 + tail + result = _truncate_bash_output(output, 20000) + assert result.endswith(tail) + + def test_middle_truncation_marker_present(self): + output = "A" * 30000 + result = _truncate_bash_output(output, 20000) + assert "[middle truncated:" in result + assert "chars skipped" in result + + def test_skipped_chars_count_is_correct(self): + output = "A" * 25000 + result = _truncate_bash_output(output, 20000) + # Extract the reported skipped count and verify it equals len(output) - kept. + # (kept = max_chars - marker_max_len, where marker_max_len is computed from + # the worst-case marker string — so the exact value is implementation-defined, + # but it must equal len(output) minus the chars actually preserved.) + import re + + m = re.search(r"(\d+) chars skipped", result) + assert m is not None + reported_skipped = int(m.group(1)) + # Verify the number is self-consistent: head + skipped + tail == total + assert reported_skipped > 0 + # The marker reports exactly the chars between head and tail + head_and_tail = len(output) - reported_skipped + assert result.startswith(output[: head_and_tail // 2]) + + def test_max_chars_zero_disables_truncation(self): + output = "A" * 100000 + assert _truncate_bash_output(output, 0) == output + + def test_50_50_split(self): + # head and tail should each be roughly max_chars // 2 + output = "H" * 20000 + "M" * 10000 + "T" * 20000 + result = _truncate_bash_output(output, 20000) + assert result[:100] == "H" * 100 + assert result[-100:] == "T" * 100 + + def test_small_max_chars_does_not_crash(self): + output = "A" * 1000 + result = _truncate_bash_output(output, 10) + assert len(result) <= 10 + + def test_result_never_exceeds_max_chars_various_sizes(self): + output = "X" * 50000 + for max_chars in [100, 1000, 5000, 20000, 49999]: + result = _truncate_bash_output(output, max_chars) + assert len(result) <= max_chars, f"failed for max_chars={max_chars}" + + +# --------------------------------------------------------------------------- +# _truncate_read_file_output +# --------------------------------------------------------------------------- + + +class TestTruncateReadFileOutput: + def test_short_output_returned_unchanged(self): + output = "def foo():\n pass\n" + assert _truncate_read_file_output(output, 50000) == output + + def test_output_equal_to_limit_returned_unchanged(self): + output = "X" * 50000 + assert _truncate_read_file_output(output, 50000) == output + + def test_long_output_is_truncated(self): + output = "X" * 60000 + result = _truncate_read_file_output(output, 50000) + assert len(result) < len(output) + + def test_result_never_exceeds_max_chars(self): + output = "X" * 60000 + max_chars = 50000 + result = _truncate_read_file_output(output, max_chars) + assert len(result) <= max_chars + + def test_head_is_preserved(self): + head = "import os\nimport sys\n" + output = head + "X" * 60000 + result = _truncate_read_file_output(output, 50000) + assert result.startswith(head) + + def test_truncation_marker_present(self): + output = "X" * 60000 + result = _truncate_read_file_output(output, 50000) + assert "[truncated:" in result + assert "showing first" in result + + def test_total_chars_reported_correctly(self): + output = "X" * 60000 + result = _truncate_read_file_output(output, 50000) + assert "of 60000 chars" in result + + def test_start_line_hint_present(self): + output = "X" * 60000 + result = _truncate_read_file_output(output, 50000) + assert "start_line" in result + assert "end_line" in result + + def test_max_chars_zero_disables_truncation(self): + output = "X" * 100000 + assert _truncate_read_file_output(output, 0) == output + + def test_tail_is_not_preserved(self): + # head-truncation: tail should be cut off + output = "H" * 50000 + "TAIL_SHOULD_NOT_APPEAR" + result = _truncate_read_file_output(output, 50000) + assert "TAIL_SHOULD_NOT_APPEAR" not in result + + def test_small_max_chars_does_not_crash(self): + output = "X" * 1000 + result = _truncate_read_file_output(output, 10) + assert len(result) <= 10 + + def test_result_never_exceeds_max_chars_various_sizes(self): + output = "X" * 50000 + for max_chars in [100, 1000, 5000, 20000, 49999]: + result = _truncate_read_file_output(output, max_chars) + assert len(result) <= max_chars, f"failed for max_chars={max_chars}" diff --git a/config.example.yaml b/config.example.yaml index b2ccfd4cc..0bff5d6ad 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -12,7 +12,7 @@ # ============================================================================ # Bump this number when the config schema changes. # Run `make config-upgrade` to merge new fields into your local config.yaml. -config_version: 4 +config_version: 5 # ============================================================================ # Logging @@ -366,6 +366,13 @@ sandbox: # trusted, single-user local workflows. allow_host_bash: false + # Tool output truncation limits (characters). + # bash uses middle-truncation (head + tail) since errors can appear anywhere in the output. + # read_file uses head-truncation since source code context is front-loaded. + # Set to 0 to disable truncation. + bash_output_max_chars: 20000 + read_file_output_max_chars: 50000 + # Option 2: Container-based AIO Sandbox # Executes commands in isolated containers (Docker or Apple Container) # On macOS: Automatically prefers Apple Container if available, falls back to Docker From 3a672b39c798604abc5911371c56745260186324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=82=96?= <168966994+luoxiao6645@users.noreply.github.com> Date: Thu, 2 Apr 2026 10:12:17 +0800 Subject: [PATCH 12/47] Fix/1681 llm call retry handling (#1683) * fix(runtime): handle llm call errors gracefully * fix(runtime): preserve graph control flow in llm retry middleware --------- Co-authored-by: luoxiao6645 --- .../llm_error_handling_middleware.py | 275 ++++++++++++++++++ .../tool_error_handling_middleware.py | 3 + .../test_llm_error_handling_middleware.py | 136 +++++++++ frontend/src/core/threads/hooks.ts | 14 + 4 files changed, 428 insertions(+) create mode 100644 backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py create mode 100644 backend/tests/test_llm_error_handling_middleware.py diff --git a/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py new file mode 100644 index 000000000..e1a3af714 --- /dev/null +++ b/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py @@ -0,0 +1,275 @@ +"""LLM error handling middleware with retry/backoff and user-facing fallbacks.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Awaitable, Callable +from email.utils import parsedate_to_datetime +from typing import Any, override + +from langchain.agents import AgentState +from langchain.agents.middleware import AgentMiddleware +from langchain.agents.middleware.types import ( + ModelCallResult, + ModelRequest, + ModelResponse, +) +from langchain_core.messages import AIMessage +from langgraph.errors import GraphBubbleUp + +logger = logging.getLogger(__name__) + +_RETRIABLE_STATUS_CODES = {408, 409, 425, 429, 500, 502, 503, 504} +_BUSY_PATTERNS = ( + "server busy", + "temporarily unavailable", + "try again later", + "please retry", + "please try again", + "overloaded", + "high demand", + "rate limit", + "负载较高", + "服务繁忙", + "稍后重试", + "请稍后重试", +) +_QUOTA_PATTERNS = ( + "insufficient_quota", + "quota", + "billing", + "credit", + "payment", + "余额不足", + "超出限额", + "额度不足", + "欠费", +) +_AUTH_PATTERNS = ( + "authentication", + "unauthorized", + "invalid api key", + "invalid_api_key", + "permission", + "forbidden", + "access denied", + "无权", + "未授权", +) + + +class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]): + """Retry transient LLM errors and surface graceful assistant messages.""" + + retry_max_attempts: int = 3 + retry_base_delay_ms: int = 1000 + retry_cap_delay_ms: int = 8000 + + def _classify_error(self, exc: BaseException) -> tuple[bool, str]: + detail = _extract_error_detail(exc) + lowered = detail.lower() + error_code = _extract_error_code(exc) + status_code = _extract_status_code(exc) + + if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(str(error_code).lower(), _QUOTA_PATTERNS): + return False, "quota" + if _matches_any(lowered, _AUTH_PATTERNS): + return False, "auth" + + exc_name = exc.__class__.__name__ + if exc_name in { + "APITimeoutError", + "APIConnectionError", + "InternalServerError", + }: + return True, "transient" + if status_code in _RETRIABLE_STATUS_CODES: + return True, "transient" + if _matches_any(lowered, _BUSY_PATTERNS): + return True, "busy" + + return False, "generic" + + def _build_retry_delay_ms(self, attempt: int, exc: BaseException) -> int: + retry_after = _extract_retry_after_ms(exc) + if retry_after is not None: + return retry_after + backoff = self.retry_base_delay_ms * (2 ** max(0, attempt - 1)) + return min(backoff, self.retry_cap_delay_ms) + + def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str: + seconds = max(1, round(wait_ms / 1000)) + reason_text = "provider is busy" if reason == "busy" else "provider request failed temporarily" + return f"LLM request retry {attempt}/{self.retry_max_attempts}: {reason_text}. Retrying in {seconds}s." + + def _build_user_message(self, exc: BaseException, reason: str) -> str: + detail = _extract_error_detail(exc) + if reason == "quota": + return "The configured LLM provider rejected the request because the account is out of quota, billing is unavailable, or usage is restricted. Please fix the provider account and try again." + if reason == "auth": + return "The configured LLM provider rejected the request because authentication or access is invalid. Please check the provider credentials and try again." + if reason in {"busy", "transient"}: + return "The configured LLM provider is temporarily unavailable after multiple retries. Please wait a moment and continue the conversation." + return f"LLM request failed: {detail}" + + def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None: + try: + from langgraph.config import get_stream_writer + + writer = get_stream_writer() + writer( + { + "type": "llm_retry", + "attempt": attempt, + "max_attempts": self.retry_max_attempts, + "wait_ms": wait_ms, + "reason": reason, + "message": self._build_retry_message(attempt, wait_ms, reason), + } + ) + except Exception: + logger.debug("Failed to emit llm_retry event", exc_info=True) + + @override + def wrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], ModelResponse], + ) -> ModelCallResult: + attempt = 1 + while True: + try: + return handler(request) + except GraphBubbleUp: + # Preserve LangGraph control-flow signals (interrupt/pause/resume). + raise + except Exception as exc: + retriable, reason = self._classify_error(exc) + if retriable and attempt < self.retry_max_attempts: + wait_ms = self._build_retry_delay_ms(attempt, exc) + logger.warning( + "Transient LLM error on attempt %d/%d; retrying in %dms: %s", + attempt, + self.retry_max_attempts, + wait_ms, + _extract_error_detail(exc), + ) + self._emit_retry_event(attempt, wait_ms, reason) + time.sleep(wait_ms / 1000) + attempt += 1 + continue + logger.warning( + "LLM call failed after %d attempt(s): %s", + attempt, + _extract_error_detail(exc), + exc_info=exc, + ) + return AIMessage(content=self._build_user_message(exc, reason)) + + @override + async def awrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], Awaitable[ModelResponse]], + ) -> ModelCallResult: + attempt = 1 + while True: + try: + return await handler(request) + except GraphBubbleUp: + # Preserve LangGraph control-flow signals (interrupt/pause/resume). + raise + except Exception as exc: + retriable, reason = self._classify_error(exc) + if retriable and attempt < self.retry_max_attempts: + wait_ms = self._build_retry_delay_ms(attempt, exc) + logger.warning( + "Transient LLM error on attempt %d/%d; retrying in %dms: %s", + attempt, + self.retry_max_attempts, + wait_ms, + _extract_error_detail(exc), + ) + self._emit_retry_event(attempt, wait_ms, reason) + await asyncio.sleep(wait_ms / 1000) + attempt += 1 + continue + logger.warning( + "LLM call failed after %d attempt(s): %s", + attempt, + _extract_error_detail(exc), + exc_info=exc, + ) + return AIMessage(content=self._build_user_message(exc, reason)) + + +def _matches_any(detail: str, patterns: tuple[str, ...]) -> bool: + return any(pattern in detail for pattern in patterns) + + +def _extract_error_code(exc: BaseException) -> Any: + for attr in ("code", "error_code"): + value = getattr(exc, attr, None) + if value not in (None, ""): + return value + + body = getattr(exc, "body", None) + if isinstance(body, dict): + error = body.get("error") + if isinstance(error, dict): + for key in ("code", "type"): + value = error.get(key) + if value not in (None, ""): + return value + return None + + +def _extract_status_code(exc: BaseException) -> int | None: + for attr in ("status_code", "status"): + value = getattr(exc, attr, None) + if isinstance(value, int): + return value + response = getattr(exc, "response", None) + status = getattr(response, "status_code", None) + return status if isinstance(status, int) else None + + +def _extract_retry_after_ms(exc: BaseException) -> int | None: + response = getattr(exc, "response", None) + headers = getattr(response, "headers", None) + if headers is None: + return None + + raw = None + header_name = "" + for key in ("retry-after-ms", "Retry-After-Ms", "retry-after", "Retry-After"): + header_name = key + if hasattr(headers, "get"): + raw = headers.get(key) + if raw: + break + if not raw: + return None + + try: + multiplier = 1 if "ms" in header_name.lower() else 1000 + return max(0, int(float(raw) * multiplier)) + except (TypeError, ValueError): + try: + target = parsedate_to_datetime(str(raw)) + delta = target.timestamp() - time.time() + return max(0, int(delta * 1000)) + except (TypeError, ValueError, OverflowError): + return None + + +def _extract_error_detail(exc: BaseException) -> str: + detail = str(exc).strip() + if detail: + return detail + message = getattr(exc, "message", None) + if isinstance(message, str) and message.strip(): + return message.strip() + return exc.__class__.__name__ diff --git a/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py index 35a37f852..c3acd86cc 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py @@ -72,6 +72,7 @@ def _build_runtime_middlewares( lazy_init: bool = True, ) -> list[AgentMiddleware]: """Build shared base middlewares for agent execution.""" + from deerflow.agents.middlewares.llm_error_handling_middleware import LLMErrorHandlingMiddleware from deerflow.agents.middlewares.thread_data_middleware import ThreadDataMiddleware from deerflow.sandbox.middleware import SandboxMiddleware @@ -90,6 +91,8 @@ def _build_runtime_middlewares( middlewares.append(DanglingToolCallMiddleware()) + middlewares.append(LLMErrorHandlingMiddleware()) + # Guardrail middleware (if configured) from deerflow.config.guardrails_config import get_guardrails_config diff --git a/backend/tests/test_llm_error_handling_middleware.py b/backend/tests/test_llm_error_handling_middleware.py new file mode 100644 index 000000000..9c3077e31 --- /dev/null +++ b/backend/tests/test_llm_error_handling_middleware.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest +from langchain_core.messages import AIMessage +from langgraph.errors import GraphBubbleUp + +from deerflow.agents.middlewares.llm_error_handling_middleware import ( + LLMErrorHandlingMiddleware, +) + + +class FakeError(Exception): + def __init__( + self, + message: str, + *, + status_code: int | None = None, + code: str | None = None, + headers: dict[str, str] | None = None, + body: dict | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.code = code + self.body = body + self.response = SimpleNamespace(status_code=status_code, headers=headers or {}) if status_code is not None or headers else None + + +def _build_middleware(**attrs: int) -> LLMErrorHandlingMiddleware: + middleware = LLMErrorHandlingMiddleware() + for key, value in attrs.items(): + setattr(middleware, key, value) + return middleware + + +def test_async_model_call_retries_busy_provider_then_succeeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = _build_middleware(retry_max_attempts=3, retry_base_delay_ms=25, retry_cap_delay_ms=25) + attempts = 0 + waits: list[float] = [] + events: list[dict] = [] + + async def fake_sleep(delay: float) -> None: + waits.append(delay) + + def fake_writer(): + return events.append + + async def handler(_request) -> AIMessage: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise FakeError("当前服务集群负载较高，请稍后重试，感谢您的耐心等待。 (2064)") + return AIMessage(content="ok") + + monkeypatch.setattr("asyncio.sleep", fake_sleep) + monkeypatch.setattr( + "langgraph.config.get_stream_writer", + fake_writer, + ) + + result = asyncio.run(middleware.awrap_model_call(SimpleNamespace(), handler)) + + assert isinstance(result, AIMessage) + assert result.content == "ok" + assert attempts == 3 + assert waits == [0.025, 0.025] + assert [event["type"] for event in events] == ["llm_retry", "llm_retry"] + + +def test_async_model_call_returns_user_message_for_quota_errors() -> None: + middleware = _build_middleware(retry_max_attempts=3) + + async def handler(_request) -> AIMessage: + raise FakeError( + "insufficient_quota: account balance is empty", + status_code=429, + code="insufficient_quota", + ) + + result = asyncio.run(middleware.awrap_model_call(SimpleNamespace(), handler)) + + assert isinstance(result, AIMessage) + assert "out of quota" in str(result.content) + + +def test_sync_model_call_uses_retry_after_header(monkeypatch: pytest.MonkeyPatch) -> None: + middleware = _build_middleware(retry_max_attempts=2, retry_base_delay_ms=10, retry_cap_delay_ms=10) + waits: list[float] = [] + attempts = 0 + + def fake_sleep(delay: float) -> None: + waits.append(delay) + + def handler(_request) -> AIMessage: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise FakeError( + "server busy", + status_code=503, + headers={"Retry-After": "2"}, + ) + return AIMessage(content="ok") + + monkeypatch.setattr("time.sleep", fake_sleep) + + result = middleware.wrap_model_call(SimpleNamespace(), handler) + + assert isinstance(result, AIMessage) + assert result.content == "ok" + assert waits == [2.0] + + +def test_sync_model_call_propagates_graph_bubble_up() -> None: + middleware = _build_middleware() + + def handler(_request) -> AIMessage: + raise GraphBubbleUp() + + with pytest.raises(GraphBubbleUp): + middleware.wrap_model_call(SimpleNamespace(), handler) + + +def test_async_model_call_propagates_graph_bubble_up() -> None: + middleware = _build_middleware() + + async def handler(_request) -> AIMessage: + raise GraphBubbleUp() + + with pytest.raises(GraphBubbleUp): + asyncio.run(middleware.awrap_model_call(SimpleNamespace(), handler)) diff --git a/frontend/src/core/threads/hooks.ts b/frontend/src/core/threads/hooks.ts index 99cfda94f..305879045 100644 --- a/frontend/src/core/threads/hooks.ts +++ b/frontend/src/core/threads/hooks.ts @@ -170,6 +170,20 @@ export function useThreadStream({ message: AIMessage; }; updateSubtask({ id: e.task_id, latestMessage: e.message }); + return; + } + + if ( + typeof event === "object" && + event !== null && + "type" in event && + event.type === "llm_retry" && + "message" in event && + typeof event.message === "string" && + event.message.trim() + ) { + const e = event as { type: "llm_retry"; message: string }; + toast(e.message); } }, onError(error) { From 2d1f90d5dc0b0c992cb421428d33b431be5a5a17 Mon Sep 17 00:00:00 2001 From: totoyang <34332614+totoyang@users.noreply.github.com> Date: Thu, 2 Apr 2026 13:06:10 +0800 Subject: [PATCH 13/47] feat(tracing): add optional Langfuse support (#1717) * feat(tracing): add optional Langfuse support * Fix tracing fail-fast behavior for explicitly enabled providers * fix(lint) --- README.md | 21 +++ backend/README.md | 23 ++- .../harness/deerflow/config/__init__.py | 11 +- .../harness/deerflow/config/tracing_config.py | 131 +++++++++---- .../harness/deerflow/models/factory.py | 21 +-- .../harness/deerflow/tracing/__init__.py | 3 + .../harness/deerflow/tracing/factory.py | 54 ++++++ backend/packages/harness/pyproject.toml | 1 + backend/tests/test_model_factory.py | 15 +- backend/tests/test_tracing_config.py | 95 +++++++++- backend/tests/test_tracing_factory.py | 173 ++++++++++++++++++ backend/uv.lock | 81 +++++++- docs/plans/2026-04-01-langfuse-tracing.md | 105 +++++++++++ 13 files changed, 667 insertions(+), 67 deletions(-) create mode 100644 backend/packages/harness/deerflow/tracing/__init__.py create mode 100644 backend/packages/harness/deerflow/tracing/factory.py create mode 100644 backend/tests/test_tracing_factory.py create mode 100644 docs/plans/2026-04-01-langfuse-tracing.md diff --git a/README.md b/README.md index a527acd7c..41ab57cc7 100644 --- a/README.md +++ b/README.md @@ -423,6 +423,27 @@ LANGSMITH_API_KEY=lsv2_pt_xxxxxxxxxxxxxxxx LANGSMITH_PROJECT=xxx ``` +#### Langfuse Tracing + +DeerFlow also supports [Langfuse](https://langfuse.com) observability for LangChain-compatible runs. + +Add the following to your `.env` file: + +```bash +LANGFUSE_TRACING=true +LANGFUSE_PUBLIC_KEY=pk-lf-xxxxxxxxxxxxxxxx +LANGFUSE_SECRET_KEY=sk-lf-xxxxxxxxxxxxxxxx +LANGFUSE_BASE_URL=https://cloud.langfuse.com +``` + +If you are using a self-hosted Langfuse instance, set `LANGFUSE_BASE_URL` to your deployment URL. + +#### Using Both Providers + +If both LangSmith and Langfuse are enabled, DeerFlow attaches both tracing callbacks and reports the same model activity to both systems. + +If a provider is explicitly enabled but missing required credentials, or if its callback fails to initialize, DeerFlow fails fast when tracing is initialized during model creation and the error message names the provider that caused the failure. + For Docker deployments, tracing is disabled by default. Set `LANGSMITH_TRACING=true` and `LANGSMITH_API_KEY` in your `.env` to enable it. ## From Deep Research to Super Agent Harness diff --git a/backend/README.md b/backend/README.md index f2903c3bb..2296fcfe2 100644 --- a/backend/README.md +++ b/backend/README.md @@ -330,7 +330,28 @@ LANGSMITH_PROJECT=xxx **Legacy variables:** The `LANGCHAIN_TRACING_V2`, `LANGCHAIN_API_KEY`, `LANGCHAIN_PROJECT`, and `LANGCHAIN_ENDPOINT` variables are also supported for backward compatibility. `LANGSMITH_*` variables take precedence when both are set. -**Docker:** In `docker-compose.yaml`, tracing is disabled by default (`LANGSMITH_TRACING=false`). Set `LANGSMITH_TRACING=true` and provide `LANGSMITH_API_KEY` in your `.env` to enable it in containerized deployments. +### Langfuse Tracing + +DeerFlow also supports [Langfuse](https://langfuse.com) observability for LangChain-compatible runs. + +Add the following to your `.env` file: + +```bash +LANGFUSE_TRACING=true +LANGFUSE_PUBLIC_KEY=pk-lf-xxxxxxxxxxxxxxxx +LANGFUSE_SECRET_KEY=sk-lf-xxxxxxxxxxxxxxxx +LANGFUSE_BASE_URL=https://cloud.langfuse.com +``` + +If you are using a self-hosted Langfuse deployment, set `LANGFUSE_BASE_URL` to your Langfuse host. + +### Dual Provider Behavior + +If both LangSmith and Langfuse are enabled, DeerFlow initializes and attaches both callbacks so the same run data is reported to both systems. + +If a provider is explicitly enabled but required credentials are missing, or the provider callback cannot be initialized, DeerFlow raises an error when tracing is initialized during model creation instead of silently disabling tracing. + +**Docker:** In `docker-compose.yaml`, tracing is disabled by default (`LANGSMITH_TRACING=false`). Set `LANGSMITH_TRACING=true` and/or `LANGFUSE_TRACING=true` in your `.env`, together with the required credentials, to enable tracing in containerized deployments. --- diff --git a/backend/packages/harness/deerflow/config/__init__.py b/backend/packages/harness/deerflow/config/__init__.py index 7cba105a7..aa379f2aa 100644 --- a/backend/packages/harness/deerflow/config/__init__.py +++ b/backend/packages/harness/deerflow/config/__init__.py @@ -3,7 +3,13 @@ from .extensions_config import ExtensionsConfig, get_extensions_config from .memory_config import MemoryConfig, get_memory_config from .paths import Paths, get_paths from .skills_config import SkillsConfig -from .tracing_config import get_tracing_config, is_tracing_enabled +from .tracing_config import ( + get_enabled_tracing_providers, + get_explicitly_enabled_tracing_providers, + get_tracing_config, + is_tracing_enabled, + validate_enabled_tracing_providers, +) __all__ = [ "get_app_config", @@ -15,5 +21,8 @@ __all__ = [ "MemoryConfig", "get_memory_config", "get_tracing_config", + "get_explicitly_enabled_tracing_providers", + "get_enabled_tracing_providers", "is_tracing_enabled", + "validate_enabled_tracing_providers", ] diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index 6de9505ff..1ef5ebeb4 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -1,14 +1,12 @@ -import logging import os import threading from pydantic import BaseModel, Field -logger = logging.getLogger(__name__) _config_lock = threading.Lock() -class TracingConfig(BaseModel): +class LangSmithTracingConfig(BaseModel): """Configuration for LangSmith tracing.""" enabled: bool = Field(...) @@ -18,9 +16,69 @@ class TracingConfig(BaseModel): @property def is_configured(self) -> bool: - """Check if tracing is fully configured (enabled and has API key).""" return self.enabled and bool(self.api_key) + def validate(self) -> None: + if self.enabled and not self.api_key: + raise ValueError("LangSmith tracing is enabled but LANGSMITH_API_KEY (or LANGCHAIN_API_KEY) is not set.") + + +class LangfuseTracingConfig(BaseModel): + """Configuration for Langfuse tracing.""" + + enabled: bool = Field(...) + public_key: str | None = Field(...) + secret_key: str | None = Field(...) + host: str = Field(...) + + @property + def is_configured(self) -> bool: + return self.enabled and bool(self.public_key) and bool(self.secret_key) + + def validate(self) -> None: + if not self.enabled: + return + missing: list[str] = [] + if not self.public_key: + missing.append("LANGFUSE_PUBLIC_KEY") + if not self.secret_key: + missing.append("LANGFUSE_SECRET_KEY") + if missing: + raise ValueError(f"Langfuse tracing is enabled but required settings are missing: {', '.join(missing)}") + + +class TracingConfig(BaseModel): + """Tracing configuration for supported providers.""" + + langsmith: LangSmithTracingConfig = Field(...) + langfuse: LangfuseTracingConfig = Field(...) + + @property + def is_configured(self) -> bool: + return bool(self.enabled_providers) + + @property + def explicitly_enabled_providers(self) -> list[str]: + enabled: list[str] = [] + if self.langsmith.enabled: + enabled.append("langsmith") + if self.langfuse.enabled: + enabled.append("langfuse") + return enabled + + @property + def enabled_providers(self) -> list[str]: + enabled: list[str] = [] + if self.langsmith.is_configured: + enabled.append("langsmith") + if self.langfuse.is_configured: + enabled.append("langfuse") + return enabled + + def validate_enabled(self) -> None: + self.langsmith.validate() + self.langfuse.validate() + _tracing_config: TracingConfig | None = None @@ -29,12 +87,7 @@ _TRUTHY_VALUES = {"1", "true", "yes", "on"} def _env_flag_preferred(*names: str) -> bool: - """Return the boolean value of the first env var that is present and non-empty. - - Accepted truthy values (case-insensitive): ``1``, ``true``, ``yes``, ``on``. - Any other non-empty value is treated as falsy. If none of the named - variables is set, returns ``False``. - """ + """Return the boolean value of the first env var that is present and non-empty.""" for name in names: value = os.environ.get(name) if value is not None and value.strip(): @@ -52,43 +105,45 @@ def _first_env_value(*names: str) -> str | None: def get_tracing_config() -> TracingConfig: - """Get the current tracing configuration from environment variables. - - ``LANGSMITH_*`` variables take precedence over their legacy ``LANGCHAIN_*`` - counterparts. For boolean flags (``enabled``), the *first* variable that is - present and non-empty in the priority list is the sole authority – its value - is parsed and returned without consulting the remaining candidates. Accepted - truthy values are ``1``, ``true``, ``yes``, and ``on`` (case-insensitive); - any other non-empty value is treated as falsy. - - Priority order: - enabled : LANGSMITH_TRACING > LANGCHAIN_TRACING_V2 > LANGCHAIN_TRACING - api_key : LANGSMITH_API_KEY > LANGCHAIN_API_KEY - project : LANGSMITH_PROJECT > LANGCHAIN_PROJECT (default: "deer-flow") - endpoint : LANGSMITH_ENDPOINT > LANGCHAIN_ENDPOINT (default: https://api.smith.langchain.com) - - Returns: - TracingConfig with current settings. - """ + """Get the current tracing configuration from environment variables.""" global _tracing_config if _tracing_config is not None: return _tracing_config with _config_lock: - if _tracing_config is not None: # Double-check after acquiring lock + if _tracing_config is not None: return _tracing_config _tracing_config = TracingConfig( - # Keep compatibility with both legacy LANGCHAIN_* and newer LANGSMITH_* variables. - enabled=_env_flag_preferred("LANGSMITH_TRACING", "LANGCHAIN_TRACING_V2", "LANGCHAIN_TRACING"), - api_key=_first_env_value("LANGSMITH_API_KEY", "LANGCHAIN_API_KEY"), - project=_first_env_value("LANGSMITH_PROJECT", "LANGCHAIN_PROJECT") or "deer-flow", - endpoint=_first_env_value("LANGSMITH_ENDPOINT", "LANGCHAIN_ENDPOINT") or "https://api.smith.langchain.com", + langsmith=LangSmithTracingConfig( + enabled=_env_flag_preferred("LANGSMITH_TRACING", "LANGCHAIN_TRACING_V2", "LANGCHAIN_TRACING"), + api_key=_first_env_value("LANGSMITH_API_KEY", "LANGCHAIN_API_KEY"), + project=_first_env_value("LANGSMITH_PROJECT", "LANGCHAIN_PROJECT") or "deer-flow", + endpoint=_first_env_value("LANGSMITH_ENDPOINT", "LANGCHAIN_ENDPOINT") or "https://api.smith.langchain.com", + ), + langfuse=LangfuseTracingConfig( + enabled=_env_flag_preferred("LANGFUSE_TRACING"), + public_key=_first_env_value("LANGFUSE_PUBLIC_KEY"), + secret_key=_first_env_value("LANGFUSE_SECRET_KEY"), + host=_first_env_value("LANGFUSE_BASE_URL") or "https://cloud.langfuse.com", + ), ) return _tracing_config +def get_enabled_tracing_providers() -> list[str]: + """Return the configured tracing providers that are enabled and complete.""" + return get_tracing_config().enabled_providers + + +def get_explicitly_enabled_tracing_providers() -> list[str]: + """Return tracing providers explicitly enabled by config, even if incomplete.""" + return get_tracing_config().explicitly_enabled_providers + + +def validate_enabled_tracing_providers() -> None: + """Validate that any explicitly enabled providers are fully configured.""" + get_tracing_config().validate_enabled() + + def is_tracing_enabled() -> bool: - """Check if LangSmith tracing is enabled and configured. - Returns: - True if tracing is enabled and has an API key. - """ + """Check if any tracing provider is enabled and fully configured.""" return get_tracing_config().is_configured diff --git a/backend/packages/harness/deerflow/models/factory.py b/backend/packages/harness/deerflow/models/factory.py index 6f7a69a5d..51332c5e5 100644 --- a/backend/packages/harness/deerflow/models/factory.py +++ b/backend/packages/harness/deerflow/models/factory.py @@ -2,8 +2,9 @@ import logging from langchain.chat_models import BaseChatModel -from deerflow.config import get_app_config, get_tracing_config, is_tracing_enabled +from deerflow.config import get_app_config from deerflow.reflection import resolve_class +from deerflow.tracing import build_tracing_callbacks logger = logging.getLogger(__name__) @@ -79,17 +80,9 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, * model_instance = model_class(**kwargs, **model_settings_from_config) - if is_tracing_enabled(): - try: - from langchain_core.tracers.langchain import LangChainTracer - - tracing_config = get_tracing_config() - tracer = LangChainTracer( - project_name=tracing_config.project, - ) - existing_callbacks = model_instance.callbacks or [] - model_instance.callbacks = [*existing_callbacks, tracer] - logger.debug(f"LangSmith tracing attached to model '{name}' (project='{tracing_config.project}')") - except Exception as e: - logger.warning(f"Failed to attach LangSmith tracing to model '{name}': {e}") + callbacks = build_tracing_callbacks() + if callbacks: + existing_callbacks = model_instance.callbacks or [] + model_instance.callbacks = [*existing_callbacks, *callbacks] + logger.debug(f"Tracing attached to model '{name}' with providers={len(callbacks)}") return model_instance diff --git a/backend/packages/harness/deerflow/tracing/__init__.py b/backend/packages/harness/deerflow/tracing/__init__.py new file mode 100644 index 000000000..f132815fb --- /dev/null +++ b/backend/packages/harness/deerflow/tracing/__init__.py @@ -0,0 +1,3 @@ +from .factory import build_tracing_callbacks + +__all__ = ["build_tracing_callbacks"] diff --git a/backend/packages/harness/deerflow/tracing/factory.py b/backend/packages/harness/deerflow/tracing/factory.py new file mode 100644 index 000000000..a8ef85721 --- /dev/null +++ b/backend/packages/harness/deerflow/tracing/factory.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import Any + +from deerflow.config import ( + get_enabled_tracing_providers, + get_tracing_config, + validate_enabled_tracing_providers, +) + + +def _create_langsmith_tracer(config) -> Any: + from langchain_core.tracers.langchain import LangChainTracer + + return LangChainTracer(project_name=config.project) + + +def _create_langfuse_handler(config) -> Any: + from langfuse import Langfuse + from langfuse.langchain import CallbackHandler as LangfuseCallbackHandler + + # langfuse>=4 initializes project-specific credentials through the client + # singleton; the LangChain callback then attaches to that configured client. + Langfuse( + secret_key=config.secret_key, + public_key=config.public_key, + host=config.host, + ) + return LangfuseCallbackHandler(public_key=config.public_key) + + +def build_tracing_callbacks() -> list[Any]: + """Build callbacks for all explicitly enabled tracing providers.""" + validate_enabled_tracing_providers() + enabled_providers = get_enabled_tracing_providers() + if not enabled_providers: + return [] + + tracing_config = get_tracing_config() + callbacks: list[Any] = [] + + for provider in enabled_providers: + if provider == "langsmith": + try: + callbacks.append(_create_langsmith_tracer(tracing_config.langsmith)) + except Exception as exc: # pragma: no cover - exercised via tests with monkeypatch + raise RuntimeError(f"LangSmith tracing initialization failed: {exc}") from exc + elif provider == "langfuse": + try: + callbacks.append(_create_langfuse_handler(tracing_config.langfuse)) + except Exception as exc: # pragma: no cover - exercised via tests with monkeypatch + raise RuntimeError(f"Langfuse tracing initialization failed: {exc}") from exc + + return callbacks diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index 9315d883a..c0c37e3d2 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "langchain-deepseek>=1.0.1", "langchain-mcp-adapters>=0.1.0", "langchain-openai>=1.1.7", + "langfuse>=3.4.1", "langgraph>=1.0.6,<1.0.10", "langgraph-api>=0.7.0,<0.8.0", "langgraph-cli>=0.4.14", diff --git a/backend/tests/test_model_factory.py b/backend/tests/test_model_factory.py index 9d8157483..9ae2c726a 100644 --- a/backend/tests/test_model_factory.py +++ b/backend/tests/test_model_factory.py @@ -73,7 +73,7 @@ def _patch_factory(monkeypatch, app_config: AppConfig, model_class=FakeChatModel """Patch get_app_config, resolve_class, and tracing for isolated unit tests.""" monkeypatch.setattr(factory_module, "get_app_config", lambda: app_config) monkeypatch.setattr(factory_module, "resolve_class", lambda path, base: model_class) - monkeypatch.setattr(factory_module, "is_tracing_enabled", lambda: False) + monkeypatch.setattr(factory_module, "build_tracing_callbacks", lambda: []) # --------------------------------------------------------------------------- @@ -95,12 +95,23 @@ def test_uses_first_model_when_name_is_none(monkeypatch): def test_raises_when_model_not_found(monkeypatch): cfg = _make_app_config([_make_model("only-model")]) monkeypatch.setattr(factory_module, "get_app_config", lambda: cfg) - monkeypatch.setattr(factory_module, "is_tracing_enabled", lambda: False) + monkeypatch.setattr(factory_module, "build_tracing_callbacks", lambda: []) with pytest.raises(ValueError, match="ghost-model"): factory_module.create_chat_model(name="ghost-model") +def test_appends_all_tracing_callbacks(monkeypatch): + cfg = _make_app_config([_make_model("alpha")]) + _patch_factory(monkeypatch, cfg) + monkeypatch.setattr(factory_module, "build_tracing_callbacks", lambda: ["smith-callback", "langfuse-callback"]) + + FakeChatModel.captured_kwargs = {} + model = factory_module.create_chat_model(name="alpha") + + assert model.callbacks == ["smith-callback", "langfuse-callback"] + + # --------------------------------------------------------------------------- # thinking_enabled=True # --------------------------------------------------------------------------- diff --git a/backend/tests/test_tracing_config.py b/backend/tests/test_tracing_config.py index b638d39f6..a13be516d 100644 --- a/backend/tests/test_tracing_config.py +++ b/backend/tests/test_tracing_config.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from deerflow.config import tracing_config as tracing_module @@ -9,6 +11,29 @@ def _reset_tracing_cache() -> None: tracing_module._tracing_config = None +@pytest.fixture(autouse=True) +def clear_tracing_env(monkeypatch): + for name in ( + "LANGSMITH_TRACING", + "LANGCHAIN_TRACING_V2", + "LANGCHAIN_TRACING", + "LANGSMITH_API_KEY", + "LANGCHAIN_API_KEY", + "LANGSMITH_PROJECT", + "LANGCHAIN_PROJECT", + "LANGSMITH_ENDPOINT", + "LANGCHAIN_ENDPOINT", + "LANGFUSE_TRACING", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_BASE_URL", + ): + monkeypatch.delenv(name, raising=False) + _reset_tracing_cache() + yield + _reset_tracing_cache() + + def test_prefers_langsmith_env_names(monkeypatch): monkeypatch.setenv("LANGSMITH_TRACING", "true") monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_key") @@ -18,11 +43,12 @@ def test_prefers_langsmith_env_names(monkeypatch): _reset_tracing_cache() cfg = tracing_module.get_tracing_config() - assert cfg.enabled is True - assert cfg.api_key == "lsv2_key" - assert cfg.project == "smith-project" - assert cfg.endpoint == "https://smith.example.com" + assert cfg.langsmith.enabled is True + assert cfg.langsmith.api_key == "lsv2_key" + assert cfg.langsmith.project == "smith-project" + assert cfg.langsmith.endpoint == "https://smith.example.com" assert tracing_module.is_tracing_enabled() is True + assert tracing_module.get_enabled_tracing_providers() == ["langsmith"] def test_falls_back_to_langchain_env_names(monkeypatch): @@ -39,11 +65,12 @@ def test_falls_back_to_langchain_env_names(monkeypatch): _reset_tracing_cache() cfg = tracing_module.get_tracing_config() - assert cfg.enabled is True - assert cfg.api_key == "legacy-key" - assert cfg.project == "legacy-project" - assert cfg.endpoint == "https://legacy.example.com" + assert cfg.langsmith.enabled is True + assert cfg.langsmith.api_key == "legacy-key" + assert cfg.langsmith.project == "legacy-project" + assert cfg.langsmith.endpoint == "https://legacy.example.com" assert tracing_module.is_tracing_enabled() is True + assert tracing_module.get_enabled_tracing_providers() == ["langsmith"] def test_langsmith_tracing_false_overrides_langchain_tracing_v2_true(monkeypatch): @@ -55,8 +82,9 @@ def test_langsmith_tracing_false_overrides_langchain_tracing_v2_true(monkeypatch _reset_tracing_cache() cfg = tracing_module.get_tracing_config() - assert cfg.enabled is False + assert cfg.langsmith.enabled is False assert tracing_module.is_tracing_enabled() is False + assert tracing_module.get_enabled_tracing_providers() == [] def test_defaults_when_project_not_set(monkeypatch): @@ -68,4 +96,51 @@ def test_defaults_when_project_not_set(monkeypatch): _reset_tracing_cache() cfg = tracing_module.get_tracing_config() - assert cfg.project == "deer-flow" + assert cfg.langsmith.project == "deer-flow" + + +def test_langfuse_config_is_loaded(monkeypatch): + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.com") + + _reset_tracing_cache() + cfg = tracing_module.get_tracing_config() + + assert cfg.langfuse.enabled is True + assert cfg.langfuse.public_key == "pk-lf-test" + assert cfg.langfuse.secret_key == "sk-lf-test" + assert cfg.langfuse.host == "https://langfuse.example.com" + assert tracing_module.get_enabled_tracing_providers() == ["langfuse"] + + +def test_dual_provider_config_is_loaded(monkeypatch): + monkeypatch.setenv("LANGSMITH_TRACING", "true") + monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_key") + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + + _reset_tracing_cache() + cfg = tracing_module.get_tracing_config() + + assert cfg.langsmith.is_configured is True + assert cfg.langfuse.is_configured is True + assert tracing_module.is_tracing_enabled() is True + assert tracing_module.get_enabled_tracing_providers() == ["langsmith", "langfuse"] + + +def test_langfuse_enabled_requires_public_and_secret_keys(monkeypatch): + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + + _reset_tracing_cache() + + assert tracing_module.get_tracing_config().is_configured is False + assert tracing_module.get_enabled_tracing_providers() == [] + assert tracing_module.get_tracing_config().explicitly_enabled_providers == ["langfuse"] + + with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): + tracing_module.validate_enabled_tracing_providers() diff --git a/backend/tests/test_tracing_factory.py b/backend/tests/test_tracing_factory.py new file mode 100644 index 000000000..b3e77935f --- /dev/null +++ b/backend/tests/test_tracing_factory.py @@ -0,0 +1,173 @@ +"""Tests for deerflow.tracing.factory.""" + +from __future__ import annotations + +import sys +import types + +import pytest + +from deerflow.tracing import factory as tracing_factory + + +@pytest.fixture(autouse=True) +def clear_tracing_env(monkeypatch): + from deerflow.config import tracing_config as tracing_module + + for name in ( + "LANGSMITH_TRACING", + "LANGCHAIN_TRACING_V2", + "LANGCHAIN_TRACING", + "LANGSMITH_API_KEY", + "LANGCHAIN_API_KEY", + "LANGSMITH_PROJECT", + "LANGCHAIN_PROJECT", + "LANGSMITH_ENDPOINT", + "LANGCHAIN_ENDPOINT", + "LANGFUSE_TRACING", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_BASE_URL", + ): + monkeypatch.delenv(name, raising=False) + tracing_module._tracing_config = None + yield + tracing_module._tracing_config = None + + +def test_build_tracing_callbacks_returns_empty_list_when_disabled(monkeypatch): + monkeypatch.setattr(tracing_factory, "validate_enabled_tracing_providers", lambda: None) + monkeypatch.setattr(tracing_factory, "get_enabled_tracing_providers", lambda: []) + + callbacks = tracing_factory.build_tracing_callbacks() + + assert callbacks == [] + + +def test_build_tracing_callbacks_creates_langsmith_and_langfuse(monkeypatch): + class FakeLangSmithTracer: + def __init__(self, *, project_name: str): + self.project_name = project_name + + class FakeLangfuseHandler: + def __init__(self, *, public_key: str): + self.public_key = public_key + + monkeypatch.setattr(tracing_factory, "get_enabled_tracing_providers", lambda: ["langsmith", "langfuse"]) + monkeypatch.setattr(tracing_factory, "validate_enabled_tracing_providers", lambda: None) + monkeypatch.setattr( + tracing_factory, + "get_tracing_config", + lambda: type( + "Cfg", + (), + { + "langsmith": type("LangSmithCfg", (), {"project": "smith-project"})(), + "langfuse": type( + "LangfuseCfg", + (), + { + "secret_key": "sk-lf-test", + "public_key": "pk-lf-test", + "host": "https://langfuse.example.com", + }, + )(), + }, + )(), + ) + monkeypatch.setattr(tracing_factory, "_create_langsmith_tracer", lambda cfg: FakeLangSmithTracer(project_name=cfg.project)) + monkeypatch.setattr( + tracing_factory, + "_create_langfuse_handler", + lambda cfg: FakeLangfuseHandler(public_key=cfg.public_key), + ) + + callbacks = tracing_factory.build_tracing_callbacks() + + assert len(callbacks) == 2 + assert callbacks[0].project_name == "smith-project" + assert callbacks[1].public_key == "pk-lf-test" + + +def test_build_tracing_callbacks_raises_when_enabled_provider_fails(monkeypatch): + monkeypatch.setattr(tracing_factory, "get_enabled_tracing_providers", lambda: ["langfuse"]) + monkeypatch.setattr(tracing_factory, "validate_enabled_tracing_providers", lambda: None) + monkeypatch.setattr( + tracing_factory, + "get_tracing_config", + lambda: type( + "Cfg", + (), + { + "langfuse": type( + "LangfuseCfg", + (), + {"secret_key": "sk-lf-test", "public_key": "pk-lf-test", "host": "https://langfuse.example.com"}, + )(), + }, + )(), + ) + monkeypatch.setattr(tracing_factory, "_create_langfuse_handler", lambda cfg: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="Langfuse tracing initialization failed"): + tracing_factory.build_tracing_callbacks() + + +def test_build_tracing_callbacks_raises_for_explicitly_enabled_misconfigured_provider(monkeypatch): + from deerflow.config import tracing_config as tracing_module + + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + tracing_module._tracing_config = None + + with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): + tracing_factory.build_tracing_callbacks() + + +def test_create_langfuse_handler_initializes_client_before_handler(monkeypatch): + calls: list[tuple[str, dict]] = [] + + class FakeLangfuse: + def __init__(self, **kwargs): + calls.append(("client", kwargs)) + + class FakeCallbackHandler: + def __init__(self, **kwargs): + calls.append(("handler", kwargs)) + + fake_langfuse_module = types.ModuleType("langfuse") + fake_langfuse_module.Langfuse = FakeLangfuse + fake_langfuse_langchain_module = types.ModuleType("langfuse.langchain") + fake_langfuse_langchain_module.CallbackHandler = FakeCallbackHandler + monkeypatch.setitem(sys.modules, "langfuse", fake_langfuse_module) + monkeypatch.setitem(sys.modules, "langfuse.langchain", fake_langfuse_langchain_module) + + cfg = type( + "LangfuseCfg", + (), + { + "secret_key": "sk-lf-test", + "public_key": "pk-lf-test", + "host": "https://langfuse.example.com", + }, + )() + + tracing_factory._create_langfuse_handler(cfg) + + assert calls == [ + ( + "client", + { + "secret_key": "sk-lf-test", + "public_key": "pk-lf-test", + "host": "https://langfuse.example.com", + }, + ), + ( + "handler", + { + "public_key": "pk-lf-test", + }, + ), + ] diff --git a/backend/uv.lock b/backend/uv.lock index af1aab013..a7fb93eaf 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -316,6 +316,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/7b/5652771e24fff12da9dde4c20ecf4682e606b104f26419d139758cc935a6/azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651", size = 191317, upload-time = "2025-10-06T20:30:04.251Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -720,6 +729,7 @@ dependencies = [ { name = "langchain-google-genai" }, { name = "langchain-mcp-adapters" }, { name = "langchain-openai" }, + { name = "langfuse" }, { name = "langgraph" }, { name = "langgraph-api" }, { name = "langgraph-checkpoint-sqlite" }, @@ -751,6 +761,7 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.1" }, { name = "langchain-mcp-adapters", specifier = ">=0.1.0" }, { name = "langchain-openai", specifier = ">=1.1.7" }, + { name = "langfuse", specifier = ">=3.4.1" }, { name = "langgraph", specifier = ">=1.0.6,<1.0.10" }, { name = "langgraph-api", specifier = ">=0.7.0,<0.8.0" }, { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.3" }, @@ -1597,6 +1608,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/a1/50e7596aca775d8c3883eceeaf47489fac26c57c1abe243c00174f715a8a/langchain_openai-1.1.7-py3-none-any.whl", hash = "sha256:34e9cd686aac1a120d6472804422792bf8080a2103b5d21ee450c9e42d053815", size = 84753, upload-time = "2026-01-07T19:44:58.629Z" }, ] +[[package]] +name = "langfuse" +version = "4.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/de/b319a127e231e6ac10fad7a75e040b0c961669d9aa1f372f131d48ee4835/langfuse-4.0.5.tar.gz", hash = "sha256:f07fc88526d0699b3696df6ff606bc3c509c86419b5f551dea3d95ed31b4b7f8", size = 273892, upload-time = "2026-04-01T11:05:48.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/92/b4699c9ce5f2e1ab04e7fc1c656cc14a522f10f2c7170d6e427013ce0d37/langfuse-4.0.5-py3-none-any.whl", hash = "sha256:48ef89fec839b40f0f0e68b26c160e7bc0178cf10c8e53932895f4aed428b4df", size = 472730, upload-time = "2026-04-01T11:05:46.948Z" }, +] + [[package]] name = "langgraph" version = "1.0.9" @@ -3854,6 +3884,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, ] +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", size = 54003, upload-time = "2025-08-12T05:51:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", size = 39025, upload-time = "2025-08-12T05:51:37.156Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", size = 39108, upload-time = "2025-08-12T05:51:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, + { url = "https://files.pythonhosted.org/packages/e8/cf/7d848740203c7b4b27eb55dbfede11aca974a51c3d894f6cc4b865f42f58/wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", size = 36711, upload-time = "2025-08-12T05:53:10.074Z" }, + { url = "https://files.pythonhosted.org/packages/57/54/35a84d0a4d23ea675994104e667ceff49227ce473ba6a59ba2c84f250b74/wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", size = 38885, upload-time = "2025-08-12T05:53:08.695Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/66e54407c59d7b02a3c4e0af3783168fff8e5d61def52cda8728439d86bc/wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", size = 36896, upload-time = "2025-08-12T05:52:55.34Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/cd864b2a14f20d14f4c496fab97802001560f9f41554eef6df201cd7f76c/wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", size = 54132, upload-time = "2025-08-12T05:51:49.864Z" }, + { url = "https://files.pythonhosted.org/packages/d5/46/d011725b0c89e853dc44cceb738a307cde5d240d023d6d40a82d1b4e1182/wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", size = 39091, upload-time = "2025-08-12T05:51:38.935Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9e/3ad852d77c35aae7ddebdbc3b6d35ec8013af7d7dddad0ad911f3d891dae/wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", size = 39172, upload-time = "2025-08-12T05:51:59.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, + { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/da8560695e9284810b8d3df8a19396a6e40e7518059584a1a394a2b35e0a/wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", size = 37178, upload-time = "2025-08-12T05:53:12.605Z" }, + { url = "https://files.pythonhosted.org/packages/db/c8/b71eeb192c440d67a5a0449aaee2310a1a1e8eca41676046f99ed2487e9f/wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", size = 39310, upload-time = "2025-08-12T05:53:11.106Z" }, + { url = "https://files.pythonhosted.org/packages/45/20/2cda20fd4865fa40f86f6c46ed37a2a8356a7a2fde0773269311f2af56c7/wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", size = 37266, upload-time = "2025-08-12T05:52:56.531Z" }, + { url = "https://files.pythonhosted.org/packages/77/ed/dd5cf21aec36c80443c6f900449260b80e2a65cf963668eaef3b9accce36/wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", size = 56544, upload-time = "2025-08-12T05:51:51.109Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/450c651cc753877ad100c7949ab4d2e2ecc4d97157e00fa8f45df682456a/wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", size = 40283, upload-time = "2025-08-12T05:51:39.912Z" }, + { url = "https://files.pythonhosted.org/packages/d1/86/2fcad95994d9b572db57632acb6f900695a648c3e063f2cd344b3f5c5a37/wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", size = 40366, upload-time = "2025-08-12T05:52:00.693Z" }, + { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, + { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, + { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, + { url = "https://files.pythonhosted.org/packages/be/44/a1bd64b723d13bb151d6cc91b986146a1952385e0392a78567e12149c7b4/wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", size = 38717, upload-time = "2025-08-12T05:53:15.214Z" }, + { url = "https://files.pythonhosted.org/packages/79/d9/7cfd5a312760ac4dd8bf0184a6ee9e43c33e47f3dadc303032ce012b8fa3/wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", size = 41334, upload-time = "2025-08-12T05:53:14.178Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/10ad9781128ed2f99dbc474f43283b13fea8ba58723e98844367531c18e9/wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", size = 38471, upload-time = "2025-08-12T05:52:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + [[package]] name = "xlrd" version = "2.0.2" diff --git a/docs/plans/2026-04-01-langfuse-tracing.md b/docs/plans/2026-04-01-langfuse-tracing.md new file mode 100644 index 000000000..b8f42a04e --- /dev/null +++ b/docs/plans/2026-04-01-langfuse-tracing.md @@ -0,0 +1,105 @@ +# Langfuse Tracing Implementation Plan + +**Goal:** Add optional Langfuse observability support to DeerFlow while preserving existing LangSmith tracing and allowing both providers to be enabled at the same time. + +**Architecture:** Extend tracing configuration from a single LangSmith-only shape to a multi-provider config, add a tracing callback factory that builds zero, one, or two callbacks based on environment variables, and update model creation to attach those callbacks. If a provider is explicitly enabled but misconfigured or fails to initialize, tracing initialization during model creation should fail with a clear error naming that provider. + +**Tech Stack:** Python 3.12, Pydantic, LangChain callbacks, LangSmith, Langfuse, pytest + +--- + +### Task 1: Add failing tracing config tests + +**Files:** +- Modify: `backend/tests/test_tracing_config.py` + +**Step 1: Write the failing tests** + +Add tests covering: +- Langfuse-only config parsing +- dual-provider parsing +- explicit enable with missing required Langfuse fields +- provider enable detection without relying on LangSmith-only helpers + +**Step 2: Run tests to verify they fail** + +Run: `cd backend && uv run pytest tests/test_tracing_config.py -q` +Expected: FAIL because tracing config only supports LangSmith today. + +**Step 3: Write minimal implementation** + +Update tracing config code to represent multiple providers and expose helper functions needed by the tests. + +**Step 4: Run tests to verify they pass** + +Run: `cd backend && uv run pytest tests/test_tracing_config.py -q` +Expected: PASS + +### Task 2: Add failing callback factory and model attachment tests + +**Files:** +- Modify: `backend/tests/test_model_factory.py` +- Create: `backend/tests/test_tracing_factory.py` + +**Step 1: Write the failing tests** + +Add tests covering: +- LangSmith callback creation +- Langfuse callback creation +- dual callback creation +- startup failure when an explicitly enabled provider cannot initialize +- model factory appends all tracing callbacks to model callbacks + +**Step 2: Run tests to verify they fail** + +Run: `cd backend && uv run pytest tests/test_model_factory.py tests/test_tracing_factory.py -q` +Expected: FAIL because there is no provider factory and model creation only attaches LangSmith. + +**Step 3: Write minimal implementation** + +Create tracing callback factory module and update model factory to use it. + +**Step 4: Run tests to verify they pass** + +Run: `cd backend && uv run pytest tests/test_model_factory.py tests/test_tracing_factory.py -q` +Expected: PASS + +### Task 3: Wire dependency and docs + +**Files:** +- Modify: `backend/packages/harness/pyproject.toml` +- Modify: `README.md` +- Modify: `backend/README.md` + +**Step 1: Update dependency** + +Add `langfuse` to the harness dependencies. + +**Step 2: Update docs** + +Document: +- Langfuse environment variables +- dual-provider behavior +- failure behavior for explicitly enabled providers + +**Step 3: Run targeted verification** + +Run: `cd backend && uv run pytest tests/test_tracing_config.py tests/test_model_factory.py tests/test_tracing_factory.py -q` +Expected: PASS + +### Task 4: Run broader regression checks + +**Files:** +- No code changes required + +**Step 1: Run relevant suite** + +Run: `cd backend && uv run pytest tests/test_tracing_config.py tests/test_model_factory.py tests/test_tracing_factory.py -q` + +**Step 2: Run lint if needed** + +Run: `cd backend && uv run ruff check packages/harness/deerflow/config/tracing_config.py packages/harness/deerflow/models/factory.py packages/harness/deerflow/tracing` + +**Step 3: Review diff** + +Run: `git diff -- backend/packages/harness backend/tests README.md backend/README.md` From f8fb8d6fb129a38049e0e86dd099093a45498a37 Mon Sep 17 00:00:00 2001 From: knukn Date: Thu, 2 Apr 2026 15:02:09 +0800 Subject: [PATCH 14/47] feat/per agent skill filter (#1650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(agent): 为AgentConfig添加skills字段并更新lead_agent系统提示 在AgentConfig中添加skills字段以支持配置agent可用技能 更新lead_agent的系统提示模板以包含可用技能信息 * fix: resolve agent skill configuration edge cases and add tests * Update backend/packages/harness/deerflow/agents/lead_agent/prompt.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(agent): address PR review comments for skills configuration - Add detailed docstring to `skills` field in `AgentConfig` to clarify the semantics of `None` vs `[]`. - Add unit tests in `test_custom_agent.py` to verify `load_agent_config()` correctly parses omitted skills and explicit empty lists. - Fix `test_make_lead_agent_empty_skills_passed_correctly` to include `agent_name` in the runtime config, ensuring it exercises the real code path. * docs: 添加关于按代理过滤技能的配置说明 在配置示例文件和文档中添加说明，解释如何通过代理的config.yaml文件限制加载的技能 --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- backend/docs/CONFIGURATION.md | 6 ++ .../deerflow/agents/lead_agent/agent.py | 4 +- .../deerflow/agents/lead_agent/prompt.py | 4 + .../harness/deerflow/config/agents_config.py | 5 + backend/tests/test_custom_agent.py | 22 +++++ backend/tests/test_lead_agent_skills.py | 96 +++++++++++++++++++ config.example.yaml | 6 ++ 7 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_lead_agent_skills.py diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 63ccc8d28..63791b820 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -278,6 +278,12 @@ skills: - Skills are automatically discovered and loaded - Available in both local and Docker sandbox via path mapping +**Per-Agent Skill Filtering**: +Custom agents can restrict which skills they load by defining a `skills` field in their `config.yaml` (located at `workspace/agents//config.yaml`): +- **Omitted or `null`**: Loads all globally enabled skills (default fallback). +- **`[]` (empty list)**: Disables all skills for this specific agent. +- **`["skill-name"]`**: Loads only the explicitly specified skills. + ### Title Generation Automatic conversation title generation: diff --git a/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/backend/packages/harness/deerflow/agents/lead_agent/agent.py index fe743e448..c7e9d77b1 100644 --- a/backend/packages/harness/deerflow/agents/lead_agent/agent.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/agent.py @@ -343,6 +343,8 @@ def make_lead_agent(config: RunnableConfig): model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort), tools=get_available_tools(model_name=model_name, groups=agent_config.tool_groups if agent_config else None, subagent_enabled=subagent_enabled), middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_name), - system_prompt=apply_prompt_template(subagent_enabled=subagent_enabled, max_concurrent_subagents=max_concurrent_subagents, agent_name=agent_name), + system_prompt=apply_prompt_template( + subagent_enabled=subagent_enabled, max_concurrent_subagents=max_concurrent_subagents, agent_name=agent_name, available_skills=set(agent_config.skills) if agent_config and agent_config.skills is not None else None + ), state_schema=ThreadState, ) diff --git a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py index 6329fd193..0ce6ff899 100644 --- a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py @@ -402,6 +402,10 @@ def get_skills_prompt_section(available_skills: set[str] | None = None) -> str: if available_skills is not None: skills = [skill for skill in skills if skill.name in available_skills] + # Check again after filtering + if not skills: + return "" + skill_items = "\n".join( f" \n {skill.name}\n {skill.description}\n {skill.get_container_file_path(container_base_path)}\n " for skill in skills ) diff --git a/backend/packages/harness/deerflow/config/agents_config.py b/backend/packages/harness/deerflow/config/agents_config.py index c35308df8..baf47fc6b 100644 --- a/backend/packages/harness/deerflow/config/agents_config.py +++ b/backend/packages/harness/deerflow/config/agents_config.py @@ -22,6 +22,11 @@ class AgentConfig(BaseModel): description: str = "" model: str | None = None tool_groups: list[str] | None = None + # skills controls which skills are loaded into the agent's prompt: + # - None (or omitted): load all enabled skills (default fallback behavior) + # - [] (explicit empty list): disable all skills + # - ["skill1", "skill2"]: load only the specified skills + skills: list[str] | None = None def load_agent_config(name: str | None) -> AgentConfig | None: diff --git a/backend/tests/test_custom_agent.py b/backend/tests/test_custom_agent.py index e2b4b631e..c97cb4789 100644 --- a/backend/tests/test_custom_agent.py +++ b/backend/tests/test_custom_agent.py @@ -164,6 +164,28 @@ class TestLoadAgentConfig: assert cfg.tool_groups == ["file:read", "file:write"] + def test_load_config_with_skills_empty_list(self, tmp_path): + config_dict = {"name": "no-skills-agent", "skills": []} + _write_agent(tmp_path, "no-skills-agent", config_dict) + + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config + + cfg = load_agent_config("no-skills-agent") + + assert cfg.skills == [] + + def test_load_config_with_skills_omitted(self, tmp_path): + config_dict = {"name": "default-skills-agent"} + _write_agent(tmp_path, "default-skills-agent", config_dict) + + with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)): + from deerflow.config.agents_config import load_agent_config + + cfg = load_agent_config("default-skills-agent") + + assert cfg.skills is None + def test_legacy_prompt_file_field_ignored(self, tmp_path): """Unknown fields like the old prompt_file should be silently ignored.""" agent_dir = tmp_path / "agents" / "legacy-agent" diff --git a/backend/tests/test_lead_agent_skills.py b/backend/tests/test_lead_agent_skills.py new file mode 100644 index 000000000..37a6dbff8 --- /dev/null +++ b/backend/tests/test_lead_agent_skills.py @@ -0,0 +1,96 @@ +from pathlib import Path + +from deerflow.agents.lead_agent.prompt import get_skills_prompt_section +from deerflow.config.agents_config import AgentConfig +from deerflow.skills.types import Skill + + +def _make_skill(name: str) -> Skill: + return Skill( + name=name, + description=f"Description for {name}", + license="MIT", + skill_dir=Path(f"/tmp/{name}"), + skill_file=Path(f"/tmp/{name}/SKILL.md"), + relative_path=Path(name), + category="public", + enabled=True, + ) + + +def test_get_skills_prompt_section_returns_empty_when_no_skills_match(monkeypatch): + skills = [_make_skill("skill1"), _make_skill("skill2")] + monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: skills) + + result = get_skills_prompt_section(available_skills={"non_existent_skill"}) + assert result == "" + + +def test_get_skills_prompt_section_returns_empty_when_available_skills_empty(monkeypatch): + skills = [_make_skill("skill1"), _make_skill("skill2")] + monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: skills) + + result = get_skills_prompt_section(available_skills=set()) + assert result == "" + + +def test_get_skills_prompt_section_returns_skills(monkeypatch): + skills = [_make_skill("skill1"), _make_skill("skill2")] + monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: skills) + + result = get_skills_prompt_section(available_skills={"skill1"}) + assert "skill1" in result + assert "skill2" not in result + + +def test_get_skills_prompt_section_returns_all_when_available_skills_is_none(monkeypatch): + skills = [_make_skill("skill1"), _make_skill("skill2")] + monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: skills) + + result = get_skills_prompt_section(available_skills=None) + assert "skill1" in result + assert "skill2" in result + + +def test_make_lead_agent_empty_skills_passed_correctly(monkeypatch): + from unittest.mock import MagicMock + + from deerflow.agents.lead_agent import agent as lead_agent_module + + # Mock dependencies + monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: MagicMock()) + monkeypatch.setattr(lead_agent_module, "_resolve_model_name", lambda x=None: "default-model") + monkeypatch.setattr(lead_agent_module, "create_chat_model", lambda **kwargs: "model") + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) + monkeypatch.setattr(lead_agent_module, "_build_middlewares", lambda *args, **kwargs: []) + monkeypatch.setattr(lead_agent_module, "create_agent", lambda **kwargs: kwargs) + + class MockModelConfig: + supports_thinking = False + + mock_app_config = MagicMock() + mock_app_config.get_model_config.return_value = MockModelConfig() + monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: mock_app_config) + + captured_skills = [] + + def mock_apply_prompt_template(**kwargs): + captured_skills.append(kwargs.get("available_skills")) + return "mock_prompt" + + monkeypatch.setattr(lead_agent_module, "apply_prompt_template", mock_apply_prompt_template) + + # Case 1: Empty skills list + monkeypatch.setattr(lead_agent_module, "load_agent_config", lambda x: AgentConfig(name="test", skills=[])) + lead_agent_module.make_lead_agent({"configurable": {"agent_name": "test"}}) + assert captured_skills[-1] == set() + + # Case 2: None skills list + monkeypatch.setattr(lead_agent_module, "load_agent_config", lambda x: AgentConfig(name="test", skills=None)) + lead_agent_module.make_lead_agent({"configurable": {"agent_name": "test"}}) + assert captured_skills[-1] is None + + # Case 3: Some skills list + monkeypatch.setattr(lead_agent_module, "load_agent_config", lambda x: AgentConfig(name="test", skills=["skill1"])) + lead_agent_module.make_lead_agent({"configurable": {"agent_name": "test"}}) + assert captured_skills[-1] == {"skill1"} diff --git a/config.example.yaml b/config.example.yaml index 0bff5d6ad..813da9749 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -484,6 +484,12 @@ skills: # Default: /mnt/skills container_path: /mnt/skills +# Note: To restrict which skills are loaded for a specific custom agent, +# define a `skills` list in that agent's `config.yaml` (e.g. `agents/my-agent/config.yaml`): +# - Omitted or null: load all globally enabled skills (default) +# - []: disable all skills for this agent +# - ["skill-name"]: load only specific skills + # ============================================================================ # Title Generation Configuration # ============================================================================ From 3aab2445a60ddae5a31b0958e857114a33c64eb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Apr 2026 15:26:51 +0800 Subject: [PATCH 15/47] build(deps): bump aiohttp from 3.13.3 to 3.13.4 in /backend (#1750) --- updated-dependencies: - dependency-name: aiohttp dependency-version: 3.13.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/uv.lock | 142 ++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/backend/uv.lock b/backend/uv.lock index a7fb93eaf..73e4d9694 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -53,7 +53,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.13.3" +version = "3.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -64,76 +64,76 @@ dependencies = [ { name = "propcache" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/50/42/32cf8e7704ceb4481406eb87161349abb46a57fee3f008ba9cb610968646/aiohttp-3.13.3.tar.gz", hash = "sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88", size = 7844556, upload-time = "2026-01-03T17:33:05.204Z" } +sdist = { url = "https://files.pythonhosted.org/packages/45/4a/064321452809dae953c1ed6e017504e72551a26b6f5708a5a80e4bf556ff/aiohttp-3.13.4.tar.gz", hash = "sha256:d97a6d09c66087890c2ab5d49069e1e570583f7ac0314ecf98294c1b6aaebd38", size = 7859748, upload-time = "2026-03-28T17:19:40.6Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/be/4fc11f202955a69e0db803a12a062b8379c970c7c84f4882b6da17337cc1/aiohttp-3.13.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c", size = 739732, upload-time = "2026-01-03T17:30:14.23Z" }, - { url = "https://files.pythonhosted.org/packages/97/2c/621d5b851f94fa0bb7430d6089b3aa970a9d9b75196bc93bb624b0db237a/aiohttp-3.13.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168", size = 494293, upload-time = "2026-01-03T17:30:15.96Z" }, - { url = "https://files.pythonhosted.org/packages/5d/43/4be01406b78e1be8320bb8316dc9c42dbab553d281c40364e0f862d5661c/aiohttp-3.13.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d", size = 493533, upload-time = "2026-01-03T17:30:17.431Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a8/5a35dc56a06a2c90d4742cbf35294396907027f80eea696637945a106f25/aiohttp-3.13.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29", size = 1737839, upload-time = "2026-01-03T17:30:19.422Z" }, - { url = "https://files.pythonhosted.org/packages/bf/62/4b9eeb331da56530bf2e198a297e5303e1c1ebdceeb00fe9b568a65c5a0c/aiohttp-3.13.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3", size = 1703932, upload-time = "2026-01-03T17:30:21.756Z" }, - { url = "https://files.pythonhosted.org/packages/7c/f6/af16887b5d419e6a367095994c0b1332d154f647e7dc2bd50e61876e8e3d/aiohttp-3.13.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d", size = 1771906, upload-time = "2026-01-03T17:30:23.932Z" }, - { url = "https://files.pythonhosted.org/packages/ce/83/397c634b1bcc24292fa1e0c7822800f9f6569e32934bdeef09dae7992dfb/aiohttp-3.13.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463", size = 1871020, upload-time = "2026-01-03T17:30:26Z" }, - { url = "https://files.pythonhosted.org/packages/86/f6/a62cbbf13f0ac80a70f71b1672feba90fdb21fd7abd8dbf25c0105fb6fa3/aiohttp-3.13.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc", size = 1755181, upload-time = "2026-01-03T17:30:27.554Z" }, - { url = "https://files.pythonhosted.org/packages/0a/87/20a35ad487efdd3fba93d5843efdfaa62d2f1479eaafa7453398a44faf13/aiohttp-3.13.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf", size = 1561794, upload-time = "2026-01-03T17:30:29.254Z" }, - { url = "https://files.pythonhosted.org/packages/de/95/8fd69a66682012f6716e1bc09ef8a1a2a91922c5725cb904689f112309c4/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033", size = 1697900, upload-time = "2026-01-03T17:30:31.033Z" }, - { url = "https://files.pythonhosted.org/packages/e5/66/7b94b3b5ba70e955ff597672dad1691333080e37f50280178967aff68657/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f", size = 1728239, upload-time = "2026-01-03T17:30:32.703Z" }, - { url = "https://files.pythonhosted.org/packages/47/71/6f72f77f9f7d74719692ab65a2a0252584bf8d5f301e2ecb4c0da734530a/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679", size = 1740527, upload-time = "2026-01-03T17:30:34.695Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b4/75ec16cbbd5c01bdaf4a05b19e103e78d7ce1ef7c80867eb0ace42ff4488/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423", size = 1554489, upload-time = "2026-01-03T17:30:36.864Z" }, - { url = "https://files.pythonhosted.org/packages/52/8f/bc518c0eea29f8406dcf7ed1f96c9b48e3bc3995a96159b3fc11f9e08321/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce", size = 1767852, upload-time = "2026-01-03T17:30:39.433Z" }, - { url = "https://files.pythonhosted.org/packages/9d/f2/a07a75173124f31f11ea6f863dc44e6f09afe2bca45dd4e64979490deab1/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a", size = 1722379, upload-time = "2026-01-03T17:30:41.081Z" }, - { url = "https://files.pythonhosted.org/packages/3c/4a/1a3fee7c21350cac78e5c5cef711bac1b94feca07399f3d406972e2d8fcd/aiohttp-3.13.3-cp312-cp312-win32.whl", hash = "sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046", size = 428253, upload-time = "2026-01-03T17:30:42.644Z" }, - { url = "https://files.pythonhosted.org/packages/d9/b7/76175c7cb4eb73d91ad63c34e29fc4f77c9386bba4a65b53ba8e05ee3c39/aiohttp-3.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57", size = 455407, upload-time = "2026-01-03T17:30:44.195Z" }, - { url = "https://files.pythonhosted.org/packages/97/8a/12ca489246ca1faaf5432844adbfce7ff2cc4997733e0af120869345643a/aiohttp-3.13.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c", size = 734190, upload-time = "2026-01-03T17:30:45.832Z" }, - { url = "https://files.pythonhosted.org/packages/32/08/de43984c74ed1fca5c014808963cc83cb00d7bb06af228f132d33862ca76/aiohttp-3.13.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9", size = 491783, upload-time = "2026-01-03T17:30:47.466Z" }, - { url = "https://files.pythonhosted.org/packages/17/f8/8dd2cf6112a5a76f81f81a5130c57ca829d101ad583ce57f889179accdda/aiohttp-3.13.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3", size = 490704, upload-time = "2026-01-03T17:30:49.373Z" }, - { url = "https://files.pythonhosted.org/packages/6d/40/a46b03ca03936f832bc7eaa47cfbb1ad012ba1be4790122ee4f4f8cba074/aiohttp-3.13.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf", size = 1720652, upload-time = "2026-01-03T17:30:50.974Z" }, - { url = "https://files.pythonhosted.org/packages/f7/7e/917fe18e3607af92657e4285498f500dca797ff8c918bd7d90b05abf6c2a/aiohttp-3.13.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6", size = 1692014, upload-time = "2026-01-03T17:30:52.729Z" }, - { url = "https://files.pythonhosted.org/packages/71/b6/cefa4cbc00d315d68973b671cf105b21a609c12b82d52e5d0c9ae61d2a09/aiohttp-3.13.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d", size = 1759777, upload-time = "2026-01-03T17:30:54.537Z" }, - { url = "https://files.pythonhosted.org/packages/fb/e3/e06ee07b45e59e6d81498b591fc589629be1553abb2a82ce33efe2a7b068/aiohttp-3.13.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261", size = 1861276, upload-time = "2026-01-03T17:30:56.512Z" }, - { url = "https://files.pythonhosted.org/packages/7c/24/75d274228acf35ceeb2850b8ce04de9dd7355ff7a0b49d607ee60c29c518/aiohttp-3.13.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0", size = 1743131, upload-time = "2026-01-03T17:30:58.256Z" }, - { url = "https://files.pythonhosted.org/packages/04/98/3d21dde21889b17ca2eea54fdcff21b27b93f45b7bb94ca029c31ab59dc3/aiohttp-3.13.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730", size = 1556863, upload-time = "2026-01-03T17:31:00.445Z" }, - { url = "https://files.pythonhosted.org/packages/9e/84/da0c3ab1192eaf64782b03971ab4055b475d0db07b17eff925e8c93b3aa5/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91", size = 1682793, upload-time = "2026-01-03T17:31:03.024Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0f/5802ada182f575afa02cbd0ec5180d7e13a402afb7c2c03a9aa5e5d49060/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3", size = 1716676, upload-time = "2026-01-03T17:31:04.842Z" }, - { url = "https://files.pythonhosted.org/packages/3f/8c/714d53bd8b5a4560667f7bbbb06b20c2382f9c7847d198370ec6526af39c/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4", size = 1733217, upload-time = "2026-01-03T17:31:06.868Z" }, - { url = "https://files.pythonhosted.org/packages/7d/79/e2176f46d2e963facea939f5be2d26368ce543622be6f00a12844d3c991f/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998", size = 1552303, upload-time = "2026-01-03T17:31:08.958Z" }, - { url = "https://files.pythonhosted.org/packages/ab/6a/28ed4dea1759916090587d1fe57087b03e6c784a642b85ef48217b0277ae/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0", size = 1763673, upload-time = "2026-01-03T17:31:10.676Z" }, - { url = "https://files.pythonhosted.org/packages/e8/35/4a3daeb8b9fab49240d21c04d50732313295e4bd813a465d840236dd0ce1/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591", size = 1721120, upload-time = "2026-01-03T17:31:12.575Z" }, - { url = "https://files.pythonhosted.org/packages/bc/9f/d643bb3c5fb99547323e635e251c609fbbc660d983144cfebec529e09264/aiohttp-3.13.3-cp313-cp313-win32.whl", hash = "sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf", size = 427383, upload-time = "2026-01-03T17:31:14.382Z" }, - { url = "https://files.pythonhosted.org/packages/4e/f1/ab0395f8a79933577cdd996dd2f9aa6014af9535f65dddcf88204682fe62/aiohttp-3.13.3-cp313-cp313-win_amd64.whl", hash = "sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e", size = 453899, upload-time = "2026-01-03T17:31:15.958Z" }, - { url = "https://files.pythonhosted.org/packages/99/36/5b6514a9f5d66f4e2597e40dea2e3db271e023eb7a5d22defe96ba560996/aiohttp-3.13.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808", size = 737238, upload-time = "2026-01-03T17:31:17.909Z" }, - { url = "https://files.pythonhosted.org/packages/f7/49/459327f0d5bcd8c6c9ca69e60fdeebc3622861e696490d8674a6d0cb90a6/aiohttp-3.13.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415", size = 492292, upload-time = "2026-01-03T17:31:19.919Z" }, - { url = "https://files.pythonhosted.org/packages/e8/0b/b97660c5fd05d3495b4eb27f2d0ef18dc1dc4eff7511a9bf371397ff0264/aiohttp-3.13.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f", size = 493021, upload-time = "2026-01-03T17:31:21.636Z" }, - { url = "https://files.pythonhosted.org/packages/54/d4/438efabdf74e30aeceb890c3290bbaa449780583b1270b00661126b8aae4/aiohttp-3.13.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6", size = 1717263, upload-time = "2026-01-03T17:31:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/71/f2/7bddc7fd612367d1459c5bcf598a9e8f7092d6580d98de0e057eb42697ad/aiohttp-3.13.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687", size = 1669107, upload-time = "2026-01-03T17:31:25.334Z" }, - { url = "https://files.pythonhosted.org/packages/00/5a/1aeaecca40e22560f97610a329e0e5efef5e0b5afdf9f857f0d93839ab2e/aiohttp-3.13.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26", size = 1760196, upload-time = "2026-01-03T17:31:27.394Z" }, - { url = "https://files.pythonhosted.org/packages/f8/f8/0ff6992bea7bd560fc510ea1c815f87eedd745fe035589c71ce05612a19a/aiohttp-3.13.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a", size = 1843591, upload-time = "2026-01-03T17:31:29.238Z" }, - { url = "https://files.pythonhosted.org/packages/e3/d1/e30e537a15f53485b61f5be525f2157da719819e8377298502aebac45536/aiohttp-3.13.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1", size = 1720277, upload-time = "2026-01-03T17:31:31.053Z" }, - { url = "https://files.pythonhosted.org/packages/84/45/23f4c451d8192f553d38d838831ebbc156907ea6e05557f39563101b7717/aiohttp-3.13.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25", size = 1548575, upload-time = "2026-01-03T17:31:32.87Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ed/0a42b127a43712eda7807e7892c083eadfaf8429ca8fb619662a530a3aab/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603", size = 1679455, upload-time = "2026-01-03T17:31:34.76Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b5/c05f0c2b4b4fe2c9d55e73b6d3ed4fd6c9dc2684b1d81cbdf77e7fad9adb/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a", size = 1687417, upload-time = "2026-01-03T17:31:36.699Z" }, - { url = "https://files.pythonhosted.org/packages/c9/6b/915bc5dad66aef602b9e459b5a973529304d4e89ca86999d9d75d80cbd0b/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926", size = 1729968, upload-time = "2026-01-03T17:31:38.622Z" }, - { url = "https://files.pythonhosted.org/packages/11/3b/e84581290a9520024a08640b63d07673057aec5ca548177a82026187ba73/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba", size = 1545690, upload-time = "2026-01-03T17:31:40.57Z" }, - { url = "https://files.pythonhosted.org/packages/f5/04/0c3655a566c43fd647c81b895dfe361b9f9ad6d58c19309d45cff52d6c3b/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c", size = 1746390, upload-time = "2026-01-03T17:31:42.857Z" }, - { url = "https://files.pythonhosted.org/packages/1f/53/71165b26978f719c3419381514c9690bd5980e764a09440a10bb816ea4ab/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43", size = 1702188, upload-time = "2026-01-03T17:31:44.984Z" }, - { url = "https://files.pythonhosted.org/packages/29/a7/cbe6c9e8e136314fa1980da388a59d2f35f35395948a08b6747baebb6aa6/aiohttp-3.13.3-cp314-cp314-win32.whl", hash = "sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1", size = 433126, upload-time = "2026-01-03T17:31:47.463Z" }, - { url = "https://files.pythonhosted.org/packages/de/56/982704adea7d3b16614fc5936014e9af85c0e34b58f9046655817f04306e/aiohttp-3.13.3-cp314-cp314-win_amd64.whl", hash = "sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984", size = 459128, upload-time = "2026-01-03T17:31:49.2Z" }, - { url = "https://files.pythonhosted.org/packages/6c/2a/3c79b638a9c3d4658d345339d22070241ea341ed4e07b5ac60fb0f418003/aiohttp-3.13.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c", size = 769512, upload-time = "2026-01-03T17:31:51.134Z" }, - { url = "https://files.pythonhosted.org/packages/29/b9/3e5014d46c0ab0db8707e0ac2711ed28c4da0218c358a4e7c17bae0d8722/aiohttp-3.13.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592", size = 506444, upload-time = "2026-01-03T17:31:52.85Z" }, - { url = "https://files.pythonhosted.org/packages/90/03/c1d4ef9a054e151cd7839cdc497f2638f00b93cbe8043983986630d7a80c/aiohttp-3.13.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f", size = 510798, upload-time = "2026-01-03T17:31:54.91Z" }, - { url = "https://files.pythonhosted.org/packages/ea/76/8c1e5abbfe8e127c893fe7ead569148a4d5a799f7cf958d8c09f3eedf097/aiohttp-3.13.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29", size = 1868835, upload-time = "2026-01-03T17:31:56.733Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ac/984c5a6f74c363b01ff97adc96a3976d9c98940b8969a1881575b279ac5d/aiohttp-3.13.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc", size = 1720486, upload-time = "2026-01-03T17:31:58.65Z" }, - { url = "https://files.pythonhosted.org/packages/b2/9a/b7039c5f099c4eb632138728828b33428585031a1e658d693d41d07d89d1/aiohttp-3.13.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2", size = 1847951, upload-time = "2026-01-03T17:32:00.989Z" }, - { url = "https://files.pythonhosted.org/packages/3c/02/3bec2b9a1ba3c19ff89a43a19324202b8eb187ca1e928d8bdac9bbdddebd/aiohttp-3.13.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587", size = 1941001, upload-time = "2026-01-03T17:32:03.122Z" }, - { url = "https://files.pythonhosted.org/packages/37/df/d879401cedeef27ac4717f6426c8c36c3091c6e9f08a9178cc87549c537f/aiohttp-3.13.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8", size = 1797246, upload-time = "2026-01-03T17:32:05.255Z" }, - { url = "https://files.pythonhosted.org/packages/8d/15/be122de1f67e6953add23335c8ece6d314ab67c8bebb3f181063010795a7/aiohttp-3.13.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632", size = 1627131, upload-time = "2026-01-03T17:32:07.607Z" }, - { url = "https://files.pythonhosted.org/packages/12/12/70eedcac9134cfa3219ab7af31ea56bc877395b1ac30d65b1bc4b27d0438/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64", size = 1795196, upload-time = "2026-01-03T17:32:09.59Z" }, - { url = "https://files.pythonhosted.org/packages/32/11/b30e1b1cd1f3054af86ebe60df96989c6a414dd87e27ad16950eee420bea/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0", size = 1782841, upload-time = "2026-01-03T17:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/88/0d/d98a9367b38912384a17e287850f5695c528cff0f14f791ce8ee2e4f7796/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56", size = 1795193, upload-time = "2026-01-03T17:32:13.705Z" }, - { url = "https://files.pythonhosted.org/packages/43/a5/a2dfd1f5ff5581632c7f6a30e1744deda03808974f94f6534241ef60c751/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72", size = 1621979, upload-time = "2026-01-03T17:32:15.965Z" }, - { url = "https://files.pythonhosted.org/packages/fa/f0/12973c382ae7c1cccbc4417e129c5bf54c374dfb85af70893646e1f0e749/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df", size = 1822193, upload-time = "2026-01-03T17:32:18.219Z" }, - { url = "https://files.pythonhosted.org/packages/3c/5f/24155e30ba7f8c96918af1350eb0663e2430aad9e001c0489d89cd708ab1/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa", size = 1769801, upload-time = "2026-01-03T17:32:20.25Z" }, - { url = "https://files.pythonhosted.org/packages/eb/f8/7314031ff5c10e6ece114da79b338ec17eeff3a079e53151f7e9f43c4723/aiohttp-3.13.3-cp314-cp314t-win32.whl", hash = "sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767", size = 466523, upload-time = "2026-01-03T17:32:22.215Z" }, - { url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bd/ede278648914cabbabfdf95e436679b5d4156e417896a9b9f4587169e376/aiohttp-3.13.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ee62d4471ce86b108b19c3364db4b91180d13fe3510144872d6bad5401957360", size = 752158, upload-time = "2026-03-28T17:16:06.901Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/581c053253c07b480b03785196ca5335e3c606a37dc73e95f6527f1591fe/aiohttp-3.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c0fd8f41b54b58636402eb493afd512c23580456f022c1ba2db0f810c959ed0d", size = 501037, upload-time = "2026-03-28T17:16:08.82Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f9/a5ede193c08f13cc42c0a5b50d1e246ecee9115e4cf6e900d8dbd8fd6acb/aiohttp-3.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4baa48ce49efd82d6b1a0be12d6a36b35e5594d1dd42f8bfba96ea9f8678b88c", size = 501556, upload-time = "2026-03-28T17:16:10.63Z" }, + { url = "https://files.pythonhosted.org/packages/d6/10/88ff67cd48a6ec36335b63a640abe86135791544863e0cfe1f065d6cef7a/aiohttp-3.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d738ebab9f71ee652d9dbd0211057690022201b11197f9a7324fd4dba128aa97", size = 1757314, upload-time = "2026-03-28T17:16:12.498Z" }, + { url = "https://files.pythonhosted.org/packages/8b/15/fdb90a5cf5a1f52845c276e76298c75fbbcc0ac2b4a86551906d54529965/aiohttp-3.13.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0ce692c3468fa831af7dceed52edf51ac348cebfc8d3feb935927b63bd3e8576", size = 1731819, upload-time = "2026-03-28T17:16:14.558Z" }, + { url = "https://files.pythonhosted.org/packages/ec/df/28146785a007f7820416be05d4f28cc207493efd1e8c6c1068e9bdc29198/aiohttp-3.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e08abcfe752a454d2cb89ff0c08f2d1ecd057ae3e8cc6d84638de853530ebab", size = 1793279, upload-time = "2026-03-28T17:16:16.594Z" }, + { url = "https://files.pythonhosted.org/packages/10/47/689c743abf62ea7a77774d5722f220e2c912a77d65d368b884d9779ef41b/aiohttp-3.13.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5977f701b3fff36367a11087f30ea73c212e686d41cd363c50c022d48b011d8d", size = 1891082, upload-time = "2026-03-28T17:16:18.71Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/f7f4f318c7e58c23b761c9b13b9a3c9b394e0f9d5d76fbc6622fa98509f6/aiohttp-3.13.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54203e10405c06f8b6020bd1e076ae0fe6c194adcee12a5a78af3ffa3c57025e", size = 1773938, upload-time = "2026-03-28T17:16:21.125Z" }, + { url = "https://files.pythonhosted.org/packages/aa/06/f207cb3121852c989586a6fc16ff854c4fcc8651b86c5d3bd1fc83057650/aiohttp-3.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:358a6af0145bc4dda037f13167bef3cce54b132087acc4c295c739d05d16b1c3", size = 1579548, upload-time = "2026-03-28T17:16:23.588Z" }, + { url = "https://files.pythonhosted.org/packages/6c/58/e1289661a32161e24c1fe479711d783067210d266842523752869cc1d9c2/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:898ea1850656d7d61832ef06aa9846ab3ddb1621b74f46de78fbc5e1a586ba83", size = 1714669, upload-time = "2026-03-28T17:16:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/96/0a/3e86d039438a74a86e6a948a9119b22540bae037d6ba317a042ae3c22711/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7bc30cceb710cf6a44e9617e43eebb6e3e43ad855a34da7b4b6a73537d8a6763", size = 1754175, upload-time = "2026-03-28T17:16:28.18Z" }, + { url = "https://files.pythonhosted.org/packages/f4/30/e717fc5df83133ba467a560b6d8ef20197037b4bb5d7075b90037de1018e/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4a31c0c587a8a038f19a4c7e60654a6c899c9de9174593a13e7cc6e15ff271f9", size = 1762049, upload-time = "2026-03-28T17:16:30.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/28/8f7a2d4492e336e40005151bdd94baf344880a4707573378579f833a64c1/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2062f675f3fe6e06d6113eb74a157fb9df58953ffed0cdb4182554b116545758", size = 1570861, upload-time = "2026-03-28T17:16:32.953Z" }, + { url = "https://files.pythonhosted.org/packages/78/45/12e1a3d0645968b1c38de4b23fdf270b8637735ea057d4f84482ff918ad9/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d1ba8afb847ff80626d5e408c1fdc99f942acc877d0702fe137015903a220a9", size = 1790003, upload-time = "2026-03-28T17:16:35.468Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/60374e18d590de16dcb39d6ff62f39c096c1b958e6f37727b5870026ea30/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b08149419994cdd4d5eecf7fd4bc5986b5a9380285bcd01ab4c0d6bfca47b79d", size = 1737289, upload-time = "2026-03-28T17:16:38.187Z" }, + { url = "https://files.pythonhosted.org/packages/02/bf/535e58d886cfbc40a8b0013c974afad24ef7632d645bca0b678b70033a60/aiohttp-3.13.4-cp312-cp312-win32.whl", hash = "sha256:fc432f6a2c4f720180959bc19aa37259651c1a4ed8af8afc84dd41c60f15f791", size = 434185, upload-time = "2026-03-28T17:16:40.735Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1a/d92e3325134ebfff6f4069f270d3aac770d63320bd1fcd0eca023e74d9a8/aiohttp-3.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:6148c9ae97a3e8bff9a1fc9c757fa164116f86c100468339730e717590a3fb77", size = 461285, upload-time = "2026-03-28T17:16:42.713Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ac/892f4162df9b115b4758d615f32ec63d00f3084c705ff5526630887b9b42/aiohttp-3.13.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:63dd5e5b1e43b8fb1e91b79b7ceba1feba588b317d1edff385084fcc7a0a4538", size = 745744, upload-time = "2026-03-28T17:16:44.67Z" }, + { url = "https://files.pythonhosted.org/packages/97/a9/c5b87e4443a2f0ea88cb3000c93a8fdad1ee63bffc9ded8d8c8e0d66efc6/aiohttp-3.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:746ac3cc00b5baea424dacddea3ec2c2702f9590de27d837aa67004db1eebc6e", size = 498178, upload-time = "2026-03-28T17:16:46.766Z" }, + { url = "https://files.pythonhosted.org/packages/94/42/07e1b543a61250783650df13da8ddcdc0d0a5538b2bd15cef6e042aefc61/aiohttp-3.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bda8f16ea99d6a6705e5946732e48487a448be874e54a4f73d514660ff7c05d3", size = 498331, upload-time = "2026-03-28T17:16:48.9Z" }, + { url = "https://files.pythonhosted.org/packages/20/d6/492f46bf0328534124772d0cf58570acae5b286ea25006900650f69dae0e/aiohttp-3.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b061e7b5f840391e3f64d0ddf672973e45c4cfff7a0feea425ea24e51530fc2", size = 1744414, upload-time = "2026-03-28T17:16:50.968Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4d/e02627b2683f68051246215d2d62b2d2f249ff7a285e7a858dc47d6b6a14/aiohttp-3.13.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b252e8d5cd66184b570d0d010de742736e8a4fab22c58299772b0c5a466d4b21", size = 1719226, upload-time = "2026-03-28T17:16:53.173Z" }, + { url = "https://files.pythonhosted.org/packages/7b/6c/5d0a3394dd2b9f9aeba6e1b6065d0439e4b75d41f1fb09a3ec010b43552b/aiohttp-3.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20af8aad61d1803ff11152a26146d8d81c266aa8c5aa9b4504432abb965c36a0", size = 1782110, upload-time = "2026-03-28T17:16:55.362Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2d/c20791e3437700a7441a7edfb59731150322424f5aadf635602d1d326101/aiohttp-3.13.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:13a5cc924b59859ad2adb1478e31f410a7ed46e92a2a619d6d1dd1a63c1a855e", size = 1884809, upload-time = "2026-03-28T17:16:57.734Z" }, + { url = "https://files.pythonhosted.org/packages/c8/94/d99dbfbd1924a87ef643833932eb2a3d9e5eee87656efea7d78058539eff/aiohttp-3.13.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:534913dfb0a644d537aebb4123e7d466d94e3be5549205e6a31f72368980a81a", size = 1764938, upload-time = "2026-03-28T17:17:00.221Z" }, + { url = "https://files.pythonhosted.org/packages/49/61/3ce326a1538781deb89f6cf5e094e2029cd308ed1e21b2ba2278b08426f6/aiohttp-3.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:320e40192a2dcc1cf4b5576936e9652981ab596bf81eb309535db7e2f5b5672f", size = 1570697, upload-time = "2026-03-28T17:17:02.985Z" }, + { url = "https://files.pythonhosted.org/packages/b6/77/4ab5a546857bb3028fbaf34d6eea180267bdab022ee8b1168b1fcde4bfdd/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9e587fcfce2bcf06526a43cb705bdee21ac089096f2e271d75de9c339db3100c", size = 1702258, upload-time = "2026-03-28T17:17:05.28Z" }, + { url = "https://files.pythonhosted.org/packages/79/63/d8f29021e39bc5af8e5d5e9da1b07976fb9846487a784e11e4f4eeda4666/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9eb9c2eea7278206b5c6c1441fdd9dc420c278ead3f3b2cc87f9b693698cc500", size = 1740287, upload-time = "2026-03-28T17:17:07.712Z" }, + { url = "https://files.pythonhosted.org/packages/55/3a/cbc6b3b124859a11bc8055d3682c26999b393531ef926754a3445b99dfef/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:29be00c51972b04bf9d5c8f2d7f7314f48f96070ca40a873a53056e652e805f7", size = 1753011, upload-time = "2026-03-28T17:17:10.053Z" }, + { url = "https://files.pythonhosted.org/packages/e0/30/836278675205d58c1368b21520eab9572457cf19afd23759216c04483048/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90c06228a6c3a7c9f776fe4fc0b7ff647fffd3bed93779a6913c804ae00c1073", size = 1566359, upload-time = "2026-03-28T17:17:12.433Z" }, + { url = "https://files.pythonhosted.org/packages/50/b4/8032cc9b82d17e4277704ba30509eaccb39329dc18d6a35f05e424439e32/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a533ec132f05fd9a1d959e7f34184cd7d5e8511584848dab85faefbaac573069", size = 1785537, upload-time = "2026-03-28T17:17:14.721Z" }, + { url = "https://files.pythonhosted.org/packages/17/7d/5873e98230bde59f493bf1f7c3e327486a4b5653fa401144704df5d00211/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1c946f10f413836f82ea4cfb90200d2a59578c549f00857e03111cf45ad01ca5", size = 1740752, upload-time = "2026-03-28T17:17:17.387Z" }, + { url = "https://files.pythonhosted.org/packages/7b/f2/13e46e0df051494d7d3c68b7f72d071f48c384c12716fc294f75d5b1a064/aiohttp-3.13.4-cp313-cp313-win32.whl", hash = "sha256:48708e2706106da6967eff5908c78ca3943f005ed6bcb75da2a7e4da94ef8c70", size = 433187, upload-time = "2026-03-28T17:17:19.523Z" }, + { url = "https://files.pythonhosted.org/packages/ea/c0/649856ee655a843c8f8664592cfccb73ac80ede6a8c8db33a25d810c12db/aiohttp-3.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:74a2eb058da44fa3a877a49e2095b591d4913308bb424c418b77beb160c55ce3", size = 459778, upload-time = "2026-03-28T17:17:21.964Z" }, + { url = "https://files.pythonhosted.org/packages/6d/29/6657cc37ae04cacc2dbf53fb730a06b6091cc4cbe745028e047c53e6d840/aiohttp-3.13.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:e0a2c961fc92abeff61d6444f2ce6ad35bb982db9fc8ff8a47455beacf454a57", size = 749363, upload-time = "2026-03-28T17:17:24.044Z" }, + { url = "https://files.pythonhosted.org/packages/90/7f/30ccdf67ca3d24b610067dc63d64dcb91e5d88e27667811640644aa4a85d/aiohttp-3.13.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:153274535985a0ff2bff1fb6c104ed547cec898a09213d21b0f791a44b14d933", size = 499317, upload-time = "2026-03-28T17:17:26.199Z" }, + { url = "https://files.pythonhosted.org/packages/93/13/e372dd4e68ad04ee25dafb050c7f98b0d91ea643f7352757e87231102555/aiohttp-3.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:351f3171e2458da3d731ce83f9e6b9619e325c45cbd534c7759750cabf453ad7", size = 500477, upload-time = "2026-03-28T17:17:28.279Z" }, + { url = "https://files.pythonhosted.org/packages/e5/fe/ee6298e8e586096fb6f5eddd31393d8544f33ae0792c71ecbb4c2bef98ac/aiohttp-3.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f989ac8bc5595ff761a5ccd32bdb0768a117f36dd1504b1c2c074ed5d3f4df9c", size = 1737227, upload-time = "2026-03-28T17:17:30.587Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b9/a7a0463a09e1a3fe35100f74324f23644bfc3383ac5fd5effe0722a5f0b7/aiohttp-3.13.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d36fc1709110ec1e87a229b201dd3ddc32aa01e98e7868083a794609b081c349", size = 1694036, upload-time = "2026-03-28T17:17:33.29Z" }, + { url = "https://files.pythonhosted.org/packages/57/7c/8972ae3fb7be00a91aee6b644b2a6a909aedb2c425269a3bfd90115e6f8f/aiohttp-3.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42adaeea83cbdf069ab94f5103ce0787c21fb1a0153270da76b59d5578302329", size = 1786814, upload-time = "2026-03-28T17:17:36.035Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/c81e97e85c774decbaf0d577de7d848934e8166a3a14ad9f8aa5be329d28/aiohttp-3.13.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:92deb95469928cc41fd4b42a95d8012fa6df93f6b1c0a83af0ffbc4a5e218cde", size = 1866676, upload-time = "2026-03-28T17:17:38.441Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5f/5b46fe8694a639ddea2cd035bf5729e4677ea882cb251396637e2ef1590d/aiohttp-3.13.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0c7c07c4257ef3a1df355f840bc62d133bcdef5c1c5ba75add3c08553e2eed", size = 1740842, upload-time = "2026-03-28T17:17:40.783Z" }, + { url = "https://files.pythonhosted.org/packages/20/a2/0d4b03d011cca6b6b0acba8433193c1e484efa8d705ea58295590fe24203/aiohttp-3.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f062c45de8a1098cb137a1898819796a2491aec4e637a06b03f149315dff4d8f", size = 1566508, upload-time = "2026-03-28T17:17:43.235Z" }, + { url = "https://files.pythonhosted.org/packages/98/17/e689fd500da52488ec5f889effd6404dece6a59de301e380f3c64f167beb/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:76093107c531517001114f0ebdb4f46858ce818590363e3e99a4a2280334454a", size = 1700569, upload-time = "2026-03-28T17:17:46.165Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0d/66402894dbcf470ef7db99449e436105ea862c24f7ea4c95c683e635af35/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6f6ec32162d293b82f8b63a16edc80769662fbd5ae6fbd4936d3206a2c2cc63b", size = 1707407, upload-time = "2026-03-28T17:17:48.825Z" }, + { url = "https://files.pythonhosted.org/packages/2f/eb/af0ab1a3650092cbd8e14ef29e4ab0209e1460e1c299996c3f8288b3f1ff/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5903e2db3d202a00ad9f0ec35a122c005e85d90c9836ab4cda628f01edf425e2", size = 1752214, upload-time = "2026-03-28T17:17:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/5a/bf/72326f8a98e4c666f292f03c385545963cc65e358835d2a7375037a97b57/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2d5bea57be7aca98dbbac8da046d99b5557c5cf4e28538c4c786313078aca09e", size = 1562162, upload-time = "2026-03-28T17:17:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/67/9f/13b72435f99151dd9a5469c96b3b5f86aa29b7e785ca7f35cf5e538f74c0/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:bcf0c9902085976edc0232b75006ef38f89686901249ce14226b6877f88464fb", size = 1768904, upload-time = "2026-03-28T17:17:55.991Z" }, + { url = "https://files.pythonhosted.org/packages/18/bc/28d4970e7d5452ac7776cdb5431a1164a0d9cf8bd2fffd67b4fb463aa56d/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3295f98bfeed2e867cab588f2a146a9db37a85e3ae9062abf46ba062bd29165", size = 1723378, upload-time = "2026-03-28T17:17:58.348Z" }, + { url = "https://files.pythonhosted.org/packages/53/74/b32458ca1a7f34d65bdee7aef2036adbe0438123d3d53e2b083c453c24dd/aiohttp-3.13.4-cp314-cp314-win32.whl", hash = "sha256:a598a5c5767e1369d8f5b08695cab1d8160040f796c4416af76fd773d229b3c9", size = 438711, upload-time = "2026-03-28T17:18:00.728Z" }, + { url = "https://files.pythonhosted.org/packages/40/b2/54b487316c2df3e03a8f3435e9636f8a81a42a69d942164830d193beb56a/aiohttp-3.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:c555db4bc7a264bead5a7d63d92d41a1122fcd39cc62a4db815f45ad46f9c2c8", size = 464977, upload-time = "2026-03-28T17:18:03.367Z" }, + { url = "https://files.pythonhosted.org/packages/47/fb/e41b63c6ce71b07a59243bb8f3b457ee0c3402a619acb9d2c0d21ef0e647/aiohttp-3.13.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:45abbbf09a129825d13c18c7d3182fecd46d9da3cfc383756145394013604ac1", size = 781549, upload-time = "2026-03-28T17:18:05.779Z" }, + { url = "https://files.pythonhosted.org/packages/97/53/532b8d28df1e17e44c4d9a9368b78dcb6bf0b51037522136eced13afa9e8/aiohttp-3.13.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:74c80b2bc2c2adb7b3d1941b2b60701ee2af8296fc8aad8b8bc48bc25767266c", size = 514383, upload-time = "2026-03-28T17:18:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/1b/1f/62e5d400603e8468cd635812d99cb81cfdc08127a3dc474c647615f31339/aiohttp-3.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c97989ae40a9746650fa196894f317dafc12227c808c774929dda0ff873a5954", size = 518304, upload-time = "2026-03-28T17:18:10.642Z" }, + { url = "https://files.pythonhosted.org/packages/90/57/2326b37b10896447e3c6e0cbef4fe2486d30913639a5cfd1332b5d870f82/aiohttp-3.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dae86be9811493f9990ef44fff1685f5c1a3192e9061a71a109d527944eed551", size = 1893433, upload-time = "2026-03-28T17:18:13.121Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b4/a24d82112c304afdb650167ef2fe190957d81cbddac7460bedd245f765aa/aiohttp-3.13.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1db491abe852ca2fa6cc48a3341985b0174b3741838e1341b82ac82c8bd9e871", size = 1755901, upload-time = "2026-03-28T17:18:16.21Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2d/0883ef9d878d7846287f036c162a951968f22aabeef3ac97b0bea6f76d5d/aiohttp-3.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e5d701c0aad02a7dce72eef6b93226cf3734330f1a31d69ebbf69f33b86666e", size = 1876093, upload-time = "2026-03-28T17:18:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/ad/52/9204bb59c014869b71971addad6778f005daa72a96eed652c496789d7468/aiohttp-3.13.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8ac32a189081ae0a10ba18993f10f338ec94341f0d5df8fff348043962f3c6f8", size = 1970815, upload-time = "2026-03-28T17:18:21.858Z" }, + { url = "https://files.pythonhosted.org/packages/d6/b5/e4eb20275a866dde0f570f411b36c6b48f7b53edfe4f4071aa1b0728098a/aiohttp-3.13.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98e968cdaba43e45c73c3f306fca418c8009a957733bac85937c9f9cf3f4de27", size = 1816223, upload-time = "2026-03-28T17:18:24.729Z" }, + { url = "https://files.pythonhosted.org/packages/d8/23/e98075c5bb146aa61a1239ee1ac7714c85e814838d6cebbe37d3fe19214a/aiohttp-3.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca114790c9144c335d538852612d3e43ea0f075288f4849cf4b05d6cd2238ce7", size = 1649145, upload-time = "2026-03-28T17:18:27.269Z" }, + { url = "https://files.pythonhosted.org/packages/d6/c1/7bad8be33bb06c2bb224b6468874346026092762cbec388c3bdb65a368ee/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ea2e071661ba9cfe11eabbc81ac5376eaeb3061f6e72ec4cc86d7cdd1ffbdbbb", size = 1816562, upload-time = "2026-03-28T17:18:29.847Z" }, + { url = "https://files.pythonhosted.org/packages/5c/10/c00323348695e9a5e316825969c88463dcc24c7e9d443244b8a2c9cf2eae/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:34e89912b6c20e0fd80e07fa401fd218a410aa1ce9f1c2f1dad6db1bd0ce0927", size = 1800333, upload-time = "2026-03-28T17:18:32.269Z" }, + { url = "https://files.pythonhosted.org/packages/84/43/9b2147a1df3559f49bd723e22905b46a46c068a53adb54abdca32c4de180/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0e217cf9f6a42908c52b46e42c568bd57adc39c9286ced31aaace614b6087965", size = 1820617, upload-time = "2026-03-28T17:18:35.238Z" }, + { url = "https://files.pythonhosted.org/packages/a9/7f/b3481a81e7a586d02e99387b18c6dafff41285f6efd3daa2124c01f87eae/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:0c296f1221e21ba979f5ac1964c3b78cfde15c5c5f855ffd2caab337e9cd9182", size = 1643417, upload-time = "2026-03-28T17:18:37.949Z" }, + { url = "https://files.pythonhosted.org/packages/8f/72/07181226bc99ce1124e0f89280f5221a82d3ae6a6d9d1973ce429d48e52b/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:d99a9d168ebaffb74f36d011750e490085ac418f4db926cce3989c8fe6cb6b1b", size = 1849286, upload-time = "2026-03-28T17:18:40.534Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e6/1b3566e103eca6da5be4ae6713e112a053725c584e96574caf117568ffef/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cb19177205d93b881f3f89e6081593676043a6828f59c78c17a0fd6c1fbed2ba", size = 1782635, upload-time = "2026-03-28T17:18:43.073Z" }, + { url = "https://files.pythonhosted.org/packages/37/58/1b11c71904b8d079eb0c39fe664180dd1e14bebe5608e235d8bfbadc8929/aiohttp-3.13.4-cp314-cp314t-win32.whl", hash = "sha256:c606aa5656dab6552e52ca368e43869c916338346bfaf6304e15c58fb113ea30", size = 472537, upload-time = "2026-03-28T17:18:46.286Z" }, + { url = "https://files.pythonhosted.org/packages/bc/8f/87c56a1a1977d7dddea5b31e12189665a140fdb48a71e9038ff90bb564ec/aiohttp-3.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:014dcc10ec8ab8db681f0d68e939d1e9286a5aa2b993cbbdb0db130853e02144", size = 506381, upload-time = "2026-03-28T17:18:48.74Z" }, ] [[package]] From a2cb38f62bbd1b6d018a3368acd4bfeeff9e2fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Varian=5F=E7=B1=B3=E6=B3=BD?= Date: Thu, 2 Apr 2026 15:39:41 +0800 Subject: [PATCH 16/47] fix: prevent concurrent subagent file write conflicts in sandbox tools (#1714) * fix: prevent concurrent subagent file write conflicts Serialize same-path str_replace operations in sandbox tools Guard AioSandbox write_file/update_file with the existing sandbox lock Add regression tests for concurrent str_replace and append races Verify with backend full tests and ruff lint checks * fix(sandbox): Fix the concurrency issue of file operations on the same path in isolated sandboxes. Ensure that different sandbox instances use independent locks for file operations on the same virtual path to avoid concurrency conflicts. Change the lock key from a single path to a composite key of (sandbox.id, path), and add tests to verify the concurrent safety of isolated sandboxes. * feat(sandbox): Extract file operation lock logic to standalone module and fix concurrency issues Extract file operation lock related logic from tools.py into a separate file_operation_lock.py module. Fix data race issues during concurrent str_replace and write_file operations. --- backend/CLAUDE.md | 2 +- backend/README.md | 1 + .../community/aio_sandbox/aio_sandbox.py | 33 +-- .../deerflow/sandbox/file_operation_lock.py | 23 ++ .../harness/deerflow/sandbox/tools.py | 25 +- backend/tests/test_aio_sandbox.py | 50 ++++ backend/tests/test_sandbox_tools_security.py | 221 ++++++++++++++++++ 7 files changed, 327 insertions(+), 28 deletions(-) create mode 100644 backend/packages/harness/deerflow/sandbox/file_operation_lock.py diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 12fd14983..a45b14253 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -232,7 +232,7 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → - `ls` - Directory listing (tree format, max 2 levels) - `read_file` - Read file contents with optional line range - `write_file` - Write/append to files, creates directories -- `str_replace` - Substring replacement (single or all occurrences) +- `str_replace` - Substring replacement (single or all occurrences); same-path serialization is scoped to `(sandbox.id, path)` so isolated sandboxes do not contend on identical virtual paths inside one process ### Subagent System (`packages/harness/deerflow/subagents/`) diff --git a/backend/README.md b/backend/README.md index 2296fcfe2..158540946 100644 --- a/backend/README.md +++ b/backend/README.md @@ -78,6 +78,7 @@ Per-thread isolated execution with virtual path translation: - **Virtual paths**: `/mnt/user-data/{workspace,uploads,outputs}` → thread-specific physical directories - **Skills path**: `/mnt/skills` → `deer-flow/skills/` directory - **Skills loading**: Recursively discovers nested `SKILL.md` files under `skills/{public,custom}` and preserves nested container paths +- **File-write safety**: `str_replace` serializes read-modify-write per `(sandbox.id, path)` so isolated sandboxes keep concurrency even when virtual paths match - **Tools**: `bash`, `ls`, `read_file`, `write_file`, `str_replace` (`bash` is disabled by default when using `LocalSandboxProvider`; use `AioSandboxProvider` for isolated shell access) ### Subagent System diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py index 599462da7..7f8784492 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py @@ -124,16 +124,16 @@ class AioSandbox(Sandbox): content: The text content to write to the file. append: Whether to append the content to the file. """ - try: - if append: - # Read existing content first and append - existing = self.read_file(path) - if not existing.startswith("Error:"): - content = existing + content - self._client.file.write_file(file=path, content=content) - except Exception as e: - logger.error(f"Failed to write file in sandbox: {e}") - raise + with self._lock: + try: + if append: + existing = self.read_file(path) + if not existing.startswith("Error:"): + content = existing + content + self._client.file.write_file(file=path, content=content) + except Exception as e: + logger.error(f"Failed to write file in sandbox: {e}") + raise def update_file(self, path: str, content: bytes) -> None: """Update a file with binary content in the sandbox. @@ -142,9 +142,10 @@ class AioSandbox(Sandbox): path: The absolute path of the file to update. content: The binary content to write to the file. """ - try: - base64_content = base64.b64encode(content).decode("utf-8") - self._client.file.write_file(file=path, content=base64_content, encoding="base64") - except Exception as e: - logger.error(f"Failed to update file in sandbox: {e}") - raise + with self._lock: + try: + base64_content = base64.b64encode(content).decode("utf-8") + self._client.file.write_file(file=path, content=base64_content, encoding="base64") + except Exception as e: + logger.error(f"Failed to update file in sandbox: {e}") + raise diff --git a/backend/packages/harness/deerflow/sandbox/file_operation_lock.py b/backend/packages/harness/deerflow/sandbox/file_operation_lock.py new file mode 100644 index 000000000..2464015c0 --- /dev/null +++ b/backend/packages/harness/deerflow/sandbox/file_operation_lock.py @@ -0,0 +1,23 @@ +import threading + +from deerflow.sandbox.sandbox import Sandbox + +_FILE_OPERATION_LOCKS: dict[tuple[str, str], threading.Lock] = {} +_FILE_OPERATION_LOCKS_GUARD = threading.Lock() + + +def get_file_operation_lock_key(sandbox: Sandbox, path: str) -> tuple[str, str]: + sandbox_id = getattr(sandbox, "id", None) + if not sandbox_id: + sandbox_id = f"instance:{id(sandbox)}" + return sandbox_id, path + + +def get_file_operation_lock(sandbox: Sandbox, path: str) -> threading.Lock: + lock_key = get_file_operation_lock_key(sandbox, path) + with _FILE_OPERATION_LOCKS_GUARD: + lock = _FILE_OPERATION_LOCKS.get(lock_key) + if lock is None: + lock = threading.Lock() + _FILE_OPERATION_LOCKS[lock_key] = lock + return lock diff --git a/backend/packages/harness/deerflow/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py index 7cdaa26da..f7c079322 100644 --- a/backend/packages/harness/deerflow/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -13,6 +13,7 @@ from deerflow.sandbox.exceptions import ( SandboxNotFoundError, SandboxRuntimeError, ) +from deerflow.sandbox.file_operation_lock import get_file_operation_lock from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.sandbox_provider import get_sandbox_provider from deerflow.sandbox.security import LOCAL_HOST_BASH_DISABLED_MESSAGE, is_host_bash_allowed @@ -971,7 +972,8 @@ def write_file_tool( thread_data = get_thread_data(runtime) validate_local_tool_path(path, thread_data) path = _resolve_and_validate_user_data_path(path, thread_data) - sandbox.write_file(path, content, append) + with get_file_operation_lock(sandbox, path): + sandbox.write_file(path, content, append) return "OK" except SandboxError as e: return f"Error: {e}" @@ -1012,16 +1014,17 @@ def str_replace_tool( thread_data = get_thread_data(runtime) validate_local_tool_path(path, thread_data) path = _resolve_and_validate_user_data_path(path, thread_data) - content = sandbox.read_file(path) - if not content: - return "OK" - if old_str not in content: - return f"Error: String to replace not found in file: {requested_path}" - if replace_all: - content = content.replace(old_str, new_str) - else: - content = content.replace(old_str, new_str, 1) - sandbox.write_file(path, content) + with get_file_operation_lock(sandbox, path): + content = sandbox.read_file(path) + if not content: + return "OK" + if old_str not in content: + return f"Error: String to replace not found in file: {requested_path}" + if replace_all: + content = content.replace(old_str, new_str) + else: + content = content.replace(old_str, new_str, 1) + sandbox.write_file(path, content) return "OK" except SandboxError as e: return f"Error: {e}" diff --git a/backend/tests/test_aio_sandbox.py b/backend/tests/test_aio_sandbox.py index c86259432..789fbde20 100644 --- a/backend/tests/test_aio_sandbox.py +++ b/backend/tests/test_aio_sandbox.py @@ -131,3 +131,53 @@ class TestListDirSerialization: result = sandbox.list_dir("/test") assert result == ["/a", "/b"] assert lock_was_held == [True], "list_dir must hold the lock during exec_command" + + +class TestConcurrentFileWrites: + """Verify file write paths do not lose concurrent updates.""" + + def test_append_should_preserve_both_parallel_writes(self, sandbox): + storage = {"content": "seed\n"} + active_reads = 0 + state_lock = threading.Lock() + overlap_detected = threading.Event() + + def overlapping_read_file(path): + nonlocal active_reads + with state_lock: + active_reads += 1 + snapshot = storage["content"] + if active_reads == 2: + overlap_detected.set() + + overlap_detected.wait(0.05) + + with state_lock: + active_reads -= 1 + + return snapshot + + def write_back(*, file, content, **kwargs): + storage["content"] = content + return SimpleNamespace(data=SimpleNamespace()) + + sandbox.read_file = overlapping_read_file + sandbox._client.file.write_file = write_back + + barrier = threading.Barrier(2) + + def writer(payload: str): + barrier.wait() + sandbox.write_file("/tmp/shared.log", payload, append=True) + + threads = [ + threading.Thread(target=writer, args=("A\n",)), + threading.Thread(target=writer, args=("B\n",)), + ] + + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert storage["content"] in {"seed\nA\nB\n", "seed\nB\nA\n"} diff --git a/backend/tests/test_sandbox_tools_security.py b/backend/tests/test_sandbox_tools_security.py index bfcff5d72..982b5d389 100644 --- a/backend/tests/test_sandbox_tools_security.py +++ b/backend/tests/test_sandbox_tools_security.py @@ -1,3 +1,4 @@ +import threading from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -17,8 +18,10 @@ from deerflow.sandbox.tools import ( mask_local_paths_in_output, replace_virtual_path, replace_virtual_paths_in_command, + str_replace_tool, validate_local_bash_command_paths, validate_local_tool_path, + write_file_tool, ) _THREAD_DATA = { @@ -512,3 +515,221 @@ def test_validate_local_bash_command_paths_allows_mcp_filesystem_paths() -> None with patch("deerflow.config.extensions_config.get_extensions_config", return_value=disabled_config): with pytest.raises(PermissionError, match="Unsafe absolute paths"): validate_local_bash_command_paths("ls /mnt/d/workspace", _THREAD_DATA) + + +def test_str_replace_parallel_updates_should_preserve_both_edits(monkeypatch) -> None: + class SharedSandbox: + def __init__(self) -> None: + self.content = "alpha\nbeta\n" + self._active_reads = 0 + self._state_lock = threading.Lock() + self._overlap_detected = threading.Event() + + def read_file(self, path: str) -> str: + with self._state_lock: + self._active_reads += 1 + snapshot = self.content + if self._active_reads == 2: + self._overlap_detected.set() + + self._overlap_detected.wait(0.05) + + with self._state_lock: + self._active_reads -= 1 + + return snapshot + + def write_file(self, path: str, content: str, append: bool = False) -> None: + self.content = content + + sandbox = SharedSandbox() + runtimes = [ + SimpleNamespace(state={}, context={"thread_id": "thread-1"}, config={}), + SimpleNamespace(state={}, context={"thread_id": "thread-1"}, config={}), + ] + failures: list[BaseException] = [] + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: sandbox) + monkeypatch.setattr("deerflow.sandbox.tools.ensure_thread_directories_exist", lambda runtime: None) + monkeypatch.setattr("deerflow.sandbox.tools.is_local_sandbox", lambda runtime: False) + + def worker(runtime: SimpleNamespace, old_str: str, new_str: str) -> None: + try: + result = str_replace_tool.func( + runtime=runtime, + description="并发替换同一文件", + path="/mnt/user-data/workspace/shared.txt", + old_str=old_str, + new_str=new_str, + ) + assert result == "OK" + except BaseException as exc: # pragma: no cover - failure is asserted below + failures.append(exc) + + threads = [ + threading.Thread(target=worker, args=(runtimes[0], "alpha", "ALPHA")), + threading.Thread(target=worker, args=(runtimes[1], "beta", "BETA")), + ] + + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert failures == [] + assert "ALPHA" in sandbox.content + assert "BETA" in sandbox.content + + +def test_str_replace_parallel_updates_in_isolated_sandboxes_should_not_share_path_lock(monkeypatch) -> None: + class IsolatedSandbox: + def __init__(self, sandbox_id: str, shared_state: dict[str, object]) -> None: + self.id = sandbox_id + self.content = "alpha\nbeta\n" + self._shared_state = shared_state + + def read_file(self, path: str) -> str: + state_lock = self._shared_state["state_lock"] + with state_lock: + active_reads = self._shared_state["active_reads"] + self._shared_state["active_reads"] = active_reads + 1 + snapshot = self.content + if self._shared_state["active_reads"] == 2: + overlap_detected = self._shared_state["overlap_detected"] + overlap_detected.set() + + overlap_detected = self._shared_state["overlap_detected"] + overlap_detected.wait(0.05) + + with state_lock: + active_reads = self._shared_state["active_reads"] + self._shared_state["active_reads"] = active_reads - 1 + + return snapshot + + def write_file(self, path: str, content: str, append: bool = False) -> None: + self.content = content + + shared_state: dict[str, object] = { + "active_reads": 0, + "state_lock": threading.Lock(), + "overlap_detected": threading.Event(), + } + sandboxes = { + "sandbox-a": IsolatedSandbox("sandbox-a", shared_state), + "sandbox-b": IsolatedSandbox("sandbox-b", shared_state), + } + runtimes = [ + SimpleNamespace(state={}, context={"thread_id": "thread-1", "sandbox_key": "sandbox-a"}, config={}), + SimpleNamespace(state={}, context={"thread_id": "thread-2", "sandbox_key": "sandbox-b"}, config={}), + ] + failures: list[BaseException] = [] + + monkeypatch.setattr( + "deerflow.sandbox.tools.ensure_sandbox_initialized", + lambda runtime: sandboxes[runtime.context["sandbox_key"]], + ) + monkeypatch.setattr("deerflow.sandbox.tools.ensure_thread_directories_exist", lambda runtime: None) + monkeypatch.setattr("deerflow.sandbox.tools.is_local_sandbox", lambda runtime: False) + + def worker(runtime: SimpleNamespace, old_str: str, new_str: str) -> None: + try: + result = str_replace_tool.func( + runtime=runtime, + description="隔离 sandbox 并发替换同一路径", + path="/mnt/user-data/workspace/shared.txt", + old_str=old_str, + new_str=new_str, + ) + assert result == "OK" + except BaseException as exc: # pragma: no cover - failure is asserted below + failures.append(exc) + + threads = [ + threading.Thread(target=worker, args=(runtimes[0], "alpha", "ALPHA")), + threading.Thread(target=worker, args=(runtimes[1], "beta", "BETA")), + ] + + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert failures == [] + assert sandboxes["sandbox-a"].content == "ALPHA\nbeta\n" + assert sandboxes["sandbox-b"].content == "alpha\nBETA\n" + assert shared_state["overlap_detected"].is_set() + + +def test_str_replace_and_append_on_same_path_should_preserve_both_updates(monkeypatch) -> None: + class SharedSandbox: + def __init__(self) -> None: + self.id = "sandbox-1" + self.content = "alpha\n" + self.state_lock = threading.Lock() + self.str_replace_has_snapshot = threading.Event() + self.append_finished = threading.Event() + + def read_file(self, path: str) -> str: + with self.state_lock: + snapshot = self.content + self.str_replace_has_snapshot.set() + self.append_finished.wait(0.05) + return snapshot + + def write_file(self, path: str, content: str, append: bool = False) -> None: + with self.state_lock: + if append: + self.content += content + self.append_finished.set() + else: + self.content = content + + sandbox = SharedSandbox() + runtimes = [ + SimpleNamespace(state={}, context={"thread_id": "thread-1"}, config={}), + SimpleNamespace(state={}, context={"thread_id": "thread-1"}, config={}), + ] + failures: list[BaseException] = [] + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: sandbox) + monkeypatch.setattr("deerflow.sandbox.tools.ensure_thread_directories_exist", lambda runtime: None) + monkeypatch.setattr("deerflow.sandbox.tools.is_local_sandbox", lambda runtime: False) + + def replace_worker() -> None: + try: + result = str_replace_tool.func( + runtime=runtimes[0], + description="替换旧内容", + path="/mnt/user-data/workspace/shared.txt", + old_str="alpha", + new_str="ALPHA", + ) + assert result == "OK" + except BaseException as exc: # pragma: no cover - failure is asserted below + failures.append(exc) + + def append_worker() -> None: + try: + sandbox.str_replace_has_snapshot.wait(0.05) + result = write_file_tool.func( + runtime=runtimes[1], + description="追加新内容", + path="/mnt/user-data/workspace/shared.txt", + content="tail\n", + append=True, + ) + assert result == "OK" + except BaseException as exc: # pragma: no cover - failure is asserted below + failures.append(exc) + + replace_thread = threading.Thread(target=replace_worker) + append_thread = threading.Thread(target=append_worker) + + replace_thread.start() + append_thread.start() + replace_thread.join() + append_thread.join() + + assert failures == [] + assert sandbox.content == "ALPHA\ntail\n" From f56d0b4869722111fb4d8ae8ed79e59699acb9e2 Mon Sep 17 00:00:00 2001 From: moose-lab Date: Thu, 2 Apr 2026 16:09:14 +0800 Subject: [PATCH 17/47] fix(sandbox): exclude URL paths from absolute path validation (#1385) (#1419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): URL路径被误判为不安全绝对路径 (#1385) 在本地沙箱模式下，bash工具对命令做绝对路径安全校验时，会把curl命令中的 HTTPS URL（如 https://example.com/api/v1/check）误识别为本地绝对路径并拦截。 根因：_ABSOLUTE_PATH_PATTERN 正则的负向后行断言 (? * fix(sandbox): refine absolute path regex to preserve file:// defense-in-depth Change lookbehind from (? --------- Signed-off-by: moose-lab Co-authored-by: moose-lab Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Willem Jiang --- .../harness/deerflow/sandbox/tools.py | 8 ++- backend/tests/test_sandbox_tools_security.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/backend/packages/harness/deerflow/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py index f7c079322..40d176a26 100644 --- a/backend/packages/harness/deerflow/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -18,7 +18,8 @@ from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.sandbox_provider import get_sandbox_provider from deerflow.sandbox.security import LOCAL_HOST_BASH_DISABLED_MESSAGE, is_host_bash_allowed -_ABSOLUTE_PATH_PATTERN = re.compile(r"(?()]+)") +_ABSOLUTE_PATH_PATTERN = re.compile(r"(?()]+)") +_FILE_URL_PATTERN = re.compile(r"\bfile://\S+", re.IGNORECASE) _LOCAL_BASH_SYSTEM_PATH_PREFIXES = ( "/bin/", "/usr/bin/", @@ -516,6 +517,11 @@ def validate_local_bash_command_paths(command: str, thread_data: ThreadDataState if thread_data is None: raise SandboxRuntimeError("Thread data not available for local sandbox") + # Block file:// URLs which bypass the absolute-path regex but allow local file exfiltration + file_url_match = _FILE_URL_PATTERN.search(command) + if file_url_match: + raise PermissionError(f"Unsafe file:// URL in command: {file_url_match.group()}. Use paths under {VIRTUAL_PATH_PREFIX}") + unsafe_paths: list[str] = [] allowed_paths = _get_mcp_allowed_paths() diff --git a/backend/tests/test_sandbox_tools_security.py b/backend/tests/test_sandbox_tools_security.py index 982b5d389..02d1b27ce 100644 --- a/backend/tests/test_sandbox_tools_security.py +++ b/backend/tests/test_sandbox_tools_security.py @@ -324,6 +324,56 @@ def test_validate_local_bash_command_paths_allows_skills_path() -> None: ) +def test_validate_local_bash_command_paths_allows_urls() -> None: + """URLs in bash commands should not be mistaken for absolute paths (issue #1385).""" + # HTTPS URLs + validate_local_bash_command_paths( + "curl -X POST https://example.com/api/v1/risk/check", + _THREAD_DATA, + ) + # HTTP URLs + validate_local_bash_command_paths( + "curl http://localhost:8080/health", + _THREAD_DATA, + ) + # URLs with query strings + validate_local_bash_command_paths( + "curl https://api.example.com/v2/search?q=test", + _THREAD_DATA, + ) + # FTP URLs + validate_local_bash_command_paths( + "curl ftp://ftp.example.com/pub/file.tar.gz", + _THREAD_DATA, + ) + # URL mixed with valid virtual path + validate_local_bash_command_paths( + "curl https://example.com/data -o /mnt/user-data/workspace/data.json", + _THREAD_DATA, + ) + + +def test_validate_local_bash_command_paths_blocks_file_urls() -> None: + """file:// URLs should be treated as unsafe and blocked.""" + with pytest.raises(PermissionError): + validate_local_bash_command_paths("curl file:///etc/passwd", _THREAD_DATA) + + +def test_validate_local_bash_command_paths_blocks_file_urls_case_insensitive() -> None: + """file:// URL detection should be case-insensitive.""" + with pytest.raises(PermissionError): + validate_local_bash_command_paths("curl FILE:///etc/shadow", _THREAD_DATA) + + +def test_validate_local_bash_command_paths_blocks_file_urls_mixed_with_valid() -> None: + """file:// URLs should be blocked even when mixed with valid paths.""" + with pytest.raises(PermissionError): + validate_local_bash_command_paths( + "curl file:///etc/passwd -o /mnt/user-data/workspace/out.txt", + _THREAD_DATA, + ) + + def test_validate_local_bash_command_paths_still_blocks_other_paths() -> None: """Paths outside virtual and system prefixes must still be blocked.""" with patch("deerflow.sandbox.tools._get_skills_container_path", return_value="/mnt/skills"): From 636053fb6da8a61681ab2290ef25e32d9e3f4916 Mon Sep 17 00:00:00 2001 From: yangzheli <43645580+yangzheli@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:32:52 +0800 Subject: [PATCH 18/47] fix(frontend): add missing rel="noopener noreferrer" to target="_blank" links (#1741) * fix(frontend): add missing rel="noopener noreferrer" to target="_blank" links Prevent tabnabbing attacks and referrer leakage by ensuring all external links with target="_blank" include both noopener and noreferrer in the rel attribute. Made-with: Cursor * style: fix code formatting --- frontend/src/components/ai-elements/open-in-chat.tsx | 12 ++++++------ frontend/src/components/ai-elements/sources.tsx | 2 +- frontend/src/components/landing/header.tsx | 12 ++++++++++-- .../landing/sections/case-study-section.tsx | 1 + .../landing/sections/community-section.tsx | 6 +++++- .../workspace/artifacts/artifact-file-detail.tsx | 7 ++++++- .../workspace/artifacts/artifact-file-list.tsx | 1 + .../components/workspace/messages/message-group.tsx | 6 +++--- 8 files changed, 33 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/ai-elements/open-in-chat.tsx b/frontend/src/components/ai-elements/open-in-chat.tsx index 0c62a6ac4..b86e93db5 100644 --- a/frontend/src/components/ai-elements/open-in-chat.tsx +++ b/frontend/src/components/ai-elements/open-in-chat.tsx @@ -253,7 +253,7 @@ export const OpenInChatGPT = (props: OpenInChatGPTProps) => { {providers.chatgpt.icon} @@ -273,7 +273,7 @@ export const OpenInClaude = (props: OpenInClaudeProps) => { {providers.claude.icon} @@ -293,7 +293,7 @@ export const OpenInT3 = (props: OpenInT3Props) => { {providers.t3.icon} @@ -313,7 +313,7 @@ export const OpenInScira = (props: OpenInSciraProps) => { {providers.scira.icon} @@ -333,7 +333,7 @@ export const OpenInv0 = (props: OpenInv0Props) => { {providers.v0.icon} @@ -353,7 +353,7 @@ export const OpenInCursor = (props: OpenInCursorProps) => { {providers.cursor.icon} diff --git a/frontend/src/components/ai-elements/sources.tsx b/frontend/src/components/ai-elements/sources.tsx index f3570f9b2..dd0aa623c 100644 --- a/frontend/src/components/ai-elements/sources.tsx +++ b/frontend/src/components/ai-elements/sources.tsx @@ -63,7 +63,7 @@ export const Source = ({ href, title, children, ...props }: SourceProps) => ( diff --git a/frontend/src/components/landing/header.tsx b/frontend/src/components/landing/header.tsx index 7e4afa435..39e40d106 100644 --- a/frontend/src/components/landing/header.tsx +++ b/frontend/src/components/landing/header.tsx @@ -8,7 +8,11 @@ export function Header() { return (
@@ -26,7 +30,11 @@ export function Header() { asChild className="group relative z-10" > - + Star on GitHub {env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" && diff --git a/frontend/src/components/landing/sections/case-study-section.tsx b/frontend/src/components/landing/sections/case-study-section.tsx index 0ae2f667f..6a7cc4956 100644 --- a/frontend/src/components/landing/sections/case-study-section.tsx +++ b/frontend/src/components/landing/sections/case-study-section.tsx @@ -57,6 +57,7 @@ export function CaseStudySection({ className }: { className?: string }) { key={caseStudy.title} href={pathOfThread(caseStudy.threadId) + "?mock=true"} target="_blank" + rel="noopener noreferrer" >
- - )} -
+ {showFollowups && ( +
+
+ {followupsLoading ? ( +
+ {t.inputBox.followupLoading} +
+ ) : ( + + {followups.map((s) => ( + handleFollowupClick(s)} + /> + ))} + + + )}
- )} +
+ )} diff --git a/frontend/src/components/workspace/messages/message-list.tsx b/frontend/src/components/workspace/messages/message-list.tsx index 8d5e0f6b9..354a27f04 100644 --- a/frontend/src/components/workspace/messages/message-list.tsx +++ b/frontend/src/components/workspace/messages/message-list.tsx @@ -29,11 +29,14 @@ import { MessageListItem } from "./message-list-item"; import { MessageListSkeleton } from "./skeleton"; import { SubtaskCard } from "./subtask-card"; +export const MESSAGE_LIST_DEFAULT_PADDING_BOTTOM = 160; +export const MESSAGE_LIST_FOLLOWUPS_EXTRA_PADDING_BOTTOM = 80; + export function MessageList({ className, threadId, thread, - paddingBottom = 160, + paddingBottom = MESSAGE_LIST_DEFAULT_PADDING_BOTTOM, }: { className?: string; threadId: string; From c6cdf200ceae043d9c14982d1da1549fc3fb806b Mon Sep 17 00:00:00 2001 From: DanielWalnut <45447813+hetaoBackend@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:03:06 +0800 Subject: [PATCH 28/47] feat(sandbox): add built-in grep and glob tools (#1784) * feat(sandbox): add grep and glob tools * refactor(aio-sandbox): use native file search APIs * fix(sandbox): address review issues in grep/glob tools - aio_sandbox: use should_ignore_path() instead of should_ignore_name() for include_dirs=True branch to filter nested ignored paths correctly - aio_sandbox: add early exit when max_results reached in glob loop - aio_sandbox: guard entry.path.startswith(path) before stripping prefix - aio_sandbox: validate regex locally before sending to remote API - search: skip lines exceeding max_line_chars to prevent ReDoS - search: remove resolve() syscall in os.walk loop - tools: avoid double get_thread_data() call in glob_tool/grep_tool - tests: add 6 new cases covering the above code paths - tests: patch get_app_config in truncation test to isolate config * Fix sandbox grep/glob review feedback * Remove unrelated Langfuse RFC from PR --- .gitignore | 3 +- backend/docs/rfc-grep-glob-tools.md | 446 ++++++++++++++++++ .../community/aio_sandbox/aio_sandbox.py | 81 ++++ .../deerflow/sandbox/local/list_dir.py | 70 +-- .../deerflow/sandbox/local/local_sandbox.py | 34 ++ .../harness/deerflow/sandbox/sandbox.py | 21 + .../harness/deerflow/sandbox/search.py | 210 +++++++++ .../harness/deerflow/sandbox/tools.py | 189 ++++++++ backend/tests/test_sandbox_search_tools.py | 393 +++++++++++++++ config.example.yaml | 10 + 10 files changed, 1388 insertions(+), 69 deletions(-) create mode 100644 backend/docs/rfc-grep-glob-tools.md create mode 100644 backend/packages/harness/deerflow/sandbox/search.py create mode 100644 backend/tests/test_sandbox_search_tools.py diff --git a/.gitignore b/.gitignore index 9f1ab8ce5..97cb8ed9b 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,5 @@ web/ # Deployment artifacts backend/Dockerfile.langgraph config.yaml.bak -.playwright-mcp \ No newline at end of file +.playwright-mcp +.gstack/ diff --git a/backend/docs/rfc-grep-glob-tools.md b/backend/docs/rfc-grep-glob-tools.md new file mode 100644 index 000000000..f4defca98 --- /dev/null +++ b/backend/docs/rfc-grep-glob-tools.md @@ -0,0 +1,446 @@ +# [RFC] 在 DeerFlow 中增加 `grep` 与 `glob` 文件搜索工具 + +## Summary + +我认为这个方向是对的，而且值得做。 + +如果 DeerFlow 想更接近 Claude Code 这类 coding agent 的实际工作流，仅有 `ls` / `read_file` / `write_file` / `str_replace` 还不够。模型在进入修改前，通常还需要两类能力： + +- `glob`: 快速按路径模式找文件 +- `grep`: 快速按内容模式找候选位置 + +这两类工具的价值，不是“功能上 bash 也能做”，而是它们能以更低 token 成本、更强约束、更稳定的输出格式，替代模型频繁走 `bash find` / `bash grep` / `rg` 的习惯。 + +但前提是实现方式要对：**它们应该是只读、结构化、受限、可审计的原生工具，而不是对 shell 命令的简单包装。** + +## Problem + +当前 DeerFlow 的文件工具层主要覆盖： + +- `ls`: 浏览目录结构 +- `read_file`: 读取文件内容 +- `write_file`: 写文件 +- `str_replace`: 做局部字符串替换 +- `bash`: 兜底执行命令 + +这套能力能完成任务，但在代码库探索阶段效率不高。 + +典型问题： + +1. 模型想找 “所有 `*.tsx` 的 page 文件” 时，只能反复 `ls` 多层目录，或者退回 `bash find` +2. 模型想找 “某个 symbol / 文案 / 配置键在哪里出现” 时，只能逐文件 `read_file`，或者退回 `bash grep` / `rg` +3. 一旦退回 `bash`，工具调用就失去结构化输出，结果也更难做裁剪、分页、审计和跨 sandbox 一致化 +4. 对没有开启 host bash 的本地模式，`bash` 甚至可能不可用，此时缺少足够强的只读检索能力 + +结论：DeerFlow 现在缺的不是“再多一个 shell 命令”，而是**文件系统检索层**。 + +## Goals + +- 为 agent 提供稳定的路径搜索和内容搜索能力 +- 减少对 `bash` 的依赖，特别是在仓库探索阶段 +- 保持与现有 sandbox 安全模型一致 +- 输出格式结构化，便于模型后续串联 `read_file` / `str_replace` +- 让本地 sandbox、容器 sandbox、未来 MCP 文件系统工具都能遵守同一语义 + +## Non-Goals + +- 不做通用 shell 兼容层 +- 不暴露完整 grep/find/rg CLI 语法 +- 不在第一版支持二进制检索、复杂 PCRE 特性、上下文窗口高亮渲染等重功能 +- 不把它做成“任意磁盘搜索”，仍然只允许在 DeerFlow 已授权的路径内执行 + +## Why This Is Worth Doing + +参考 Claude Code 这一类 agent 的设计思路，`glob` 和 `grep` 的核心价值不是新能力本身，而是把“探索代码库”的常见动作从开放式 shell 降到受控工具层。 + +这样有几个直接收益： + +1. **更低的模型负担** + 模型不需要自己拼 `find`, `grep`, `rg`, `xargs`, quoting 等命令细节。 + +2. **更稳定的跨环境行为** + 本地、Docker、AIO sandbox 不必依赖容器里是否装了 `rg`，也不会因为 shell 差异导致行为漂移。 + +3. **更强的安全与审计** + 调用参数就是“搜索什么、在哪搜、最多返回多少”，天然比任意命令更容易审计和限流。 + +4. **更好的 token 效率** + `grep` 返回的是命中摘要而不是整段文件，模型只对少数候选路径再调用 `read_file`。 + +5. **对 `tool_search` 友好** + 当 DeerFlow 持续扩展工具集时，`grep` / `glob` 会成为非常高频的基础工具，值得保留为 built-in，而不是让模型总是退回通用 bash。 + +## Proposal + +增加两个 built-in sandbox tools： + +- `glob` +- `grep` + +推荐继续放在： + +- `backend/packages/harness/deerflow/sandbox/tools.py` + +并在 `config.example.yaml` 中默认加入 `file:read` 组。 + +### 1. `glob` 工具 + +用途：按路径模式查找文件或目录。 + +建议 schema： + +```python +@tool("glob", parse_docstring=True) +def glob_tool( + runtime: ToolRuntime[ContextT, ThreadState], + description: str, + pattern: str, + path: str, + include_dirs: bool = False, + max_results: int = 200, +) -> str: + ... +``` + +参数语义： + +- `description`: 与现有工具保持一致 +- `pattern`: glob 模式，例如 `**/*.py`、`src/**/test_*.ts` +- `path`: 搜索根目录，必须是绝对路径 +- `include_dirs`: 是否返回目录 +- `max_results`: 最大返回条数，防止一次性打爆上下文 + +建议返回格式： + +```text +Found 3 paths under /mnt/user-data/workspace +1. /mnt/user-data/workspace/backend/app.py +2. /mnt/user-data/workspace/backend/tests/test_app.py +3. /mnt/user-data/workspace/scripts/build.py +``` + +如果后续想更适合前端消费，也可以改成 JSON 字符串；但第一版为了兼容现有工具风格，返回可读文本即可。 + +### 2. `grep` 工具 + +用途：按内容模式搜索文件，返回命中位置摘要。 + +建议 schema： + +```python +@tool("grep", parse_docstring=True) +def grep_tool( + runtime: ToolRuntime[ContextT, ThreadState], + description: str, + pattern: str, + path: str, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, +) -> str: + ... +``` + +参数语义： + +- `pattern`: 搜索词或正则 +- `path`: 搜索根目录，必须是绝对路径 +- `glob`: 可选路径过滤，例如 `**/*.py` +- `literal`: 为 `True` 时按普通字符串匹配，不解释为正则 +- `case_sensitive`: 是否大小写敏感 +- `max_results`: 最大返回命中数，不是文件数 + +建议返回格式： + +```text +Found 4 matches under /mnt/user-data/workspace +/mnt/user-data/workspace/backend/config.py:12: TOOL_GROUPS = [...] +/mnt/user-data/workspace/backend/config.py:48: def load_tool_config(...): +/mnt/user-data/workspace/backend/tools.py:91: "tool_groups" +/mnt/user-data/workspace/backend/tests/test_config.py:22: assert "tool_groups" in data +``` + +第一版建议只返回： + +- 文件路径 +- 行号 +- 命中行摘要 + +不返回上下文块，避免结果过大。模型如果需要上下文，再调用 `read_file(path, start_line, end_line)`。 + +## Design Principles + +### A. 不做 shell wrapper + +不建议把 `grep` 实现为： + +```python +subprocess.run("grep ...") +``` + +也不建议在容器里直接拼 `find` / `rg` 命令。 + +原因： + +- 会引入 shell quoting 和注入面 +- 会依赖不同 sandbox 内镜像是否安装同一套命令 +- Windows / macOS / Linux 行为不一致 +- 很难稳定控制输出条数与格式 + +正确方向是： + +- `glob` 使用 Python 标准库路径遍历 +- `grep` 使用 Python 逐文件扫描 +- 输出由 DeerFlow 自己格式化 + +如果未来为了性能考虑要优先调用 `rg`，也应该封装在 provider 内部，并保证外部语义不变，而不是把 CLI 暴露给模型。 + +### B. 继续沿用 DeerFlow 的路径权限模型 + +这两个工具必须复用当前 `ls` / `read_file` 的路径校验逻辑： + +- 本地模式走 `validate_local_tool_path(..., read_only=True)` +- 支持 `/mnt/skills/...` +- 支持 `/mnt/acp-workspace/...` +- 支持 thread workspace / uploads / outputs 的虚拟路径解析 +- 明确拒绝越权路径与 path traversal + +也就是说，它们属于 **file:read**，不是 `bash` 的替代越权入口。 + +### C. 结果必须硬限制 + +没有硬限制的 `glob` / `grep` 很容易炸上下文。 + +建议第一版至少限制： + +- `glob.max_results` 默认 200，最大 1000 +- `grep.max_results` 默认 100，最大 500 +- 单行摘要最大长度，例如 200 字符 +- 二进制文件跳过 +- 超大文件跳过，例如单文件大于 1 MB 或按配置控制 + +此外，命中数超过阈值时应返回： + +- 已展示的条数 +- 被截断的事实 +- 建议用户缩小搜索范围 + +例如： + +```text +Found more than 100 matches, showing first 100. Narrow the path or add a glob filter. +``` + +### D. 工具语义要彼此互补 + +推荐模型工作流应该是： + +1. `glob` 找候选文件 +2. `grep` 找候选位置 +3. `read_file` 读局部上下文 +4. `str_replace` / `write_file` 执行修改 + +这样工具边界清晰，也更利于 prompt 中教模型形成稳定习惯。 + +## Implementation Approach + +## Option A: 直接在 `sandbox/tools.py` 实现第一版 + +这是我推荐的起步方案。 + +做法： + +- 在 `sandbox/tools.py` 新增 `glob_tool` 与 `grep_tool` +- 在 local sandbox 场景直接使用 Python 文件系统 API +- 在非 local sandbox 场景，优先也通过 DeerFlow 自己控制的路径访问层实现 + +优点： + +- 改动小 +- 能尽快验证 agent 效果 +- 不需要先改 `Sandbox` 抽象 + +缺点： + +- `tools.py` 会继续变胖 +- 如果未来想在 provider 侧做性能优化，需要再抽象一次 + +## Option B: 先扩展 `Sandbox` 抽象 + +例如新增： + +```python +class Sandbox(ABC): + def glob(self, path: str, pattern: str, include_dirs: bool = False, max_results: int = 200) -> list[str]: + ... + + def grep( + self, + path: str, + pattern: str, + *, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, + ) -> list[GrepMatch]: + ... +``` + +优点： + +- 抽象更干净 +- 容器 / 远程 sandbox 可以各自优化 + +缺点： + +- 首次引入成本更高 +- 需要同步改所有 sandbox provider + +结论： + +**第一版建议走 Option A，等工具价值验证后再下沉到 `Sandbox` 抽象层。** + +## Detailed Behavior + +### `glob` 行为 + +- 输入根目录不存在：返回清晰错误 +- 根路径不是目录：返回清晰错误 +- 模式非法：返回清晰错误 +- 结果为空：返回 `No files matched` +- 默认忽略项应尽量与当前 `list_dir` 对齐，例如： + - `.git` + - `node_modules` + - `__pycache__` + - `.venv` + - 构建产物目录 + +这里建议抽一个共享 ignore 集，避免 `ls` 与 `glob` 结果风格不一致。 + +### `grep` 行为 + +- 默认只扫描文本文件 +- 检测到二进制文件直接跳过 +- 对超大文件直接跳过或只扫前 N KB +- regex 编译失败时返回参数错误 +- 输出中的路径继续使用虚拟路径，而不是暴露宿主真实路径 +- 建议默认按文件路径、行号排序，保持稳定输出 + +## Prompting Guidance + +如果引入这两个工具，建议同步更新系统提示中的文件操作建议： + +- 查找文件名模式时优先用 `glob` +- 查找代码符号、配置项、文案时优先用 `grep` +- 只有在工具不足以完成目标时才退回 `bash` + +否则模型仍会习惯性先调用 `bash`。 + +## Risks + +### 1. 与 `bash` 能力重叠 + +这是事实，但不是问题。 + +`ls` 和 `read_file` 也都能被 `bash` 替代，但我们仍然保留它们，因为结构化工具更适合 agent。 + +### 2. 性能问题 + +在大仓库上，纯 Python `grep` 可能比 `rg` 慢。 + +缓解方式： + +- 第一版先加结果上限和文件大小上限 +- 路径上强制要求 root path +- 提供 `glob` 过滤缩小扫描范围 +- 后续如有必要，在 provider 内部做 `rg` 优化，但保持同一 schema + +### 3. 忽略规则不一致 + +如果 `ls` 能看到的路径，`glob` 却看不到，模型会困惑。 + +缓解方式： + +- 统一 ignore 规则 +- 在文档里明确“默认跳过常见依赖和构建目录” + +### 4. 正则搜索过于复杂 + +如果第一版就支持大量 grep 方言，边界会很乱。 + +缓解方式： + +- 第一版只支持 Python `re` +- 并提供 `literal=True` 的简单模式 + +## Alternatives Considered + +### A. 不增加工具，完全依赖 `bash` + +不推荐。 + +这会让 DeerFlow 在代码探索体验上持续落后，也削弱无 bash 或受限 bash 场景下的能力。 + +### B. 只加 `glob`，不加 `grep` + +不推荐。 + +只解决“找文件”，没有解决“找位置”。模型最终还是会退回 `bash grep`。 + +### C. 只加 `grep`，不加 `glob` + +也不推荐。 + +`grep` 缺少路径模式过滤时，扫描范围经常太大；`glob` 是它的天然前置工具。 + +### D. 直接接入 MCP filesystem server 的搜索能力 + +短期不推荐作为主路径。 + +MCP 可以是补充，但 `glob` / `grep` 作为 DeerFlow 的基础 coding tool，最好仍然是 built-in，这样才能在默认安装中稳定可用。 + +## Acceptance Criteria + +- `config.example.yaml` 中可默认启用 `glob` 与 `grep` +- 两个工具归属 `file:read` 组 +- 本地 sandbox 下严格遵守现有路径权限 +- 输出不泄露宿主机真实路径 +- 大结果集会被截断并明确提示 +- 模型可以通过 `glob -> grep -> read_file -> str_replace` 完成典型改码流 +- 在禁用 host bash 的本地模式下，仓库探索能力明显提升 + +## Rollout Plan + +1. 在 `sandbox/tools.py` 中实现 `glob_tool` 与 `grep_tool` +2. 抽取与 `list_dir` 一致的 ignore 规则，避免行为漂移 +3. 在 `config.example.yaml` 默认加入工具配置 +4. 为本地路径校验、虚拟路径映射、结果截断、二进制跳过补测试 +5. 更新 README / backend docs / prompt guidance +6. 收集实际 agent 调用数据，再决定是否下沉到 `Sandbox` 抽象 + +## Suggested Config + +```yaml +tools: + - name: glob + group: file:read + use: deerflow.sandbox.tools:glob_tool + + - name: grep + group: file:read + use: deerflow.sandbox.tools:grep_tool +``` + +## Final Recommendation + +结论是：**可以加，而且应该加。** + +但我会明确卡三个边界： + +1. `grep` / `glob` 必须是 built-in 的只读结构化工具 +2. 第一版不要做 shell wrapper，不要把 CLI 方言直接暴露给模型 +3. 先在 `sandbox/tools.py` 验证价值，再考虑是否下沉到 `Sandbox` provider 抽象 + +如果按这个方向做，它会明显提升 DeerFlow 在 coding / repo exploration 场景下的可用性，而且风险可控。 diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py index 7f8784492..b6041f7ed 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox.py @@ -7,6 +7,7 @@ import uuid from agent_sandbox import Sandbox as AioSandboxClient from deerflow.sandbox.sandbox import Sandbox +from deerflow.sandbox.search import GrepMatch, path_matches, should_ignore_path, truncate_line logger = logging.getLogger(__name__) @@ -135,6 +136,86 @@ class AioSandbox(Sandbox): logger.error(f"Failed to write file in sandbox: {e}") raise + def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]: + if not include_dirs: + result = self._client.file.find_files(path=path, glob=pattern) + files = result.data.files if result.data and result.data.files else [] + filtered = [file_path for file_path in files if not should_ignore_path(file_path)] + truncated = len(filtered) > max_results + return filtered[:max_results], truncated + + result = self._client.file.list_path(path=path, recursive=True, show_hidden=False) + entries = result.data.files if result.data and result.data.files else [] + matches: list[str] = [] + root_path = path.rstrip("/") or "/" + root_prefix = root_path if root_path == "/" else f"{root_path}/" + for entry in entries: + if entry.path != root_path and not entry.path.startswith(root_prefix): + continue + if should_ignore_path(entry.path): + continue + rel_path = entry.path[len(root_path) :].lstrip("/") + if path_matches(pattern, rel_path): + matches.append(entry.path) + if len(matches) >= max_results: + return matches, True + return matches, False + + def grep( + self, + path: str, + pattern: str, + *, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, + ) -> tuple[list[GrepMatch], bool]: + import re as _re + + regex_source = _re.escape(pattern) if literal else pattern + # Validate the pattern locally so an invalid regex raises re.error + # (caught by grep_tool's except re.error handler) rather than a + # generic remote API error. + _re.compile(regex_source, 0 if case_sensitive else _re.IGNORECASE) + regex = regex_source if case_sensitive else f"(?i){regex_source}" + + if glob is not None: + find_result = self._client.file.find_files(path=path, glob=glob) + candidate_paths = find_result.data.files if find_result.data and find_result.data.files else [] + else: + list_result = self._client.file.list_path(path=path, recursive=True, show_hidden=False) + entries = list_result.data.files if list_result.data and list_result.data.files else [] + candidate_paths = [entry.path for entry in entries if not entry.is_directory] + + matches: list[GrepMatch] = [] + truncated = False + + for file_path in candidate_paths: + if should_ignore_path(file_path): + continue + + search_result = self._client.file.search_in_file(file=file_path, regex=regex) + data = search_result.data + if data is None: + continue + + line_numbers = data.line_numbers or [] + matched_lines = data.matches or [] + for line_number, line in zip(line_numbers, matched_lines): + matches.append( + GrepMatch( + path=file_path, + line_number=line_number if isinstance(line_number, int) else 0, + line=truncate_line(line), + ) + ) + if len(matches) >= max_results: + truncated = True + return matches, truncated + + return matches, truncated + def update_file(self, path: str, content: bytes) -> None: """Update a file with binary content in the sandbox. diff --git a/backend/packages/harness/deerflow/sandbox/local/list_dir.py b/backend/packages/harness/deerflow/sandbox/local/list_dir.py index 4c41cbf8a..b1031d340 100644 --- a/backend/packages/harness/deerflow/sandbox/local/list_dir.py +++ b/backend/packages/harness/deerflow/sandbox/local/list_dir.py @@ -1,72 +1,6 @@ -import fnmatch from pathlib import Path -IGNORE_PATTERNS = [ - # Version Control - ".git", - ".svn", - ".hg", - ".bzr", - # Dependencies - "node_modules", - "__pycache__", - ".venv", - "venv", - ".env", - "env", - ".tox", - ".nox", - ".eggs", - "*.egg-info", - "site-packages", - # Build outputs - "dist", - "build", - ".next", - ".nuxt", - ".output", - ".turbo", - "target", - "out", - # IDE & Editor - ".idea", - ".vscode", - "*.swp", - "*.swo", - "*~", - ".project", - ".classpath", - ".settings", - # OS generated - ".DS_Store", - "Thumbs.db", - "desktop.ini", - "*.lnk", - # Logs & temp files - "*.log", - "*.tmp", - "*.temp", - "*.bak", - "*.cache", - ".cache", - "logs", - # Coverage & test artifacts - ".coverage", - "coverage", - ".nyc_output", - "htmlcov", - ".pytest_cache", - ".mypy_cache", - ".ruff_cache", -] - - -def _should_ignore(name: str) -> bool: - """Check if a file/directory name matches any ignore pattern.""" - for pattern in IGNORE_PATTERNS: - if fnmatch.fnmatch(name, pattern): - return True - return False +from deerflow.sandbox.search import should_ignore_name def list_dir(path: str, max_depth: int = 2) -> list[str]: @@ -95,7 +29,7 @@ def list_dir(path: str, max_depth: int = 2) -> list[str]: try: for item in current_path.iterdir(): - if _should_ignore(item.name): + if should_ignore_name(item.name): continue post_fix = "/" if item.is_dir() else "" diff --git a/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py b/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py index bf5cd4017..adcdc37bb 100644 --- a/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py +++ b/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py @@ -6,6 +6,7 @@ from pathlib import Path from deerflow.sandbox.local.list_dir import list_dir from deerflow.sandbox.sandbox import Sandbox +from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches class LocalSandbox(Sandbox): @@ -259,6 +260,39 @@ class LocalSandbox(Sandbox): # Re-raise with the original path for clearer error messages, hiding internal resolved paths raise type(e)(e.errno, e.strerror, path) from None + def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]: + resolved_path = Path(self._resolve_path(path)) + matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results) + return [self._reverse_resolve_path(match) for match in matches], truncated + + def grep( + self, + path: str, + pattern: str, + *, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, + ) -> tuple[list[GrepMatch], bool]: + resolved_path = Path(self._resolve_path(path)) + matches, truncated = find_grep_matches( + resolved_path, + pattern, + glob_pattern=glob, + literal=literal, + case_sensitive=case_sensitive, + max_results=max_results, + ) + return [ + GrepMatch( + path=self._reverse_resolve_path(match.path), + line_number=match.line_number, + line=match.line, + ) + for match in matches + ], truncated + def update_file(self, path: str, content: bytes) -> None: resolved_path = self._resolve_path(path) try: diff --git a/backend/packages/harness/deerflow/sandbox/sandbox.py b/backend/packages/harness/deerflow/sandbox/sandbox.py index 57cab4be6..dc567b503 100644 --- a/backend/packages/harness/deerflow/sandbox/sandbox.py +++ b/backend/packages/harness/deerflow/sandbox/sandbox.py @@ -1,5 +1,7 @@ from abc import ABC, abstractmethod +from deerflow.sandbox.search import GrepMatch + class Sandbox(ABC): """Abstract base class for sandbox environments""" @@ -61,6 +63,25 @@ class Sandbox(ABC): """ pass + @abstractmethod + def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]: + """Find paths that match a glob pattern under a root directory.""" + pass + + @abstractmethod + def grep( + self, + path: str, + pattern: str, + *, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, + ) -> tuple[list[GrepMatch], bool]: + """Search for matches inside text files under a directory.""" + pass + @abstractmethod def update_file(self, path: str, content: bytes) -> None: """Update a file with binary content. diff --git a/backend/packages/harness/deerflow/sandbox/search.py b/backend/packages/harness/deerflow/sandbox/search.py new file mode 100644 index 000000000..a85938870 --- /dev/null +++ b/backend/packages/harness/deerflow/sandbox/search.py @@ -0,0 +1,210 @@ +import fnmatch +import os +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +IGNORE_PATTERNS = [ + ".git", + ".svn", + ".hg", + ".bzr", + "node_modules", + "__pycache__", + ".venv", + "venv", + ".env", + "env", + ".tox", + ".nox", + ".eggs", + "*.egg-info", + "site-packages", + "dist", + "build", + ".next", + ".nuxt", + ".output", + ".turbo", + "target", + "out", + ".idea", + ".vscode", + "*.swp", + "*.swo", + "*~", + ".project", + ".classpath", + ".settings", + ".DS_Store", + "Thumbs.db", + "desktop.ini", + "*.lnk", + "*.log", + "*.tmp", + "*.temp", + "*.bak", + "*.cache", + ".cache", + "logs", + ".coverage", + "coverage", + ".nyc_output", + "htmlcov", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", +] + +DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000 +DEFAULT_LINE_SUMMARY_LENGTH = 200 + + +@dataclass(frozen=True) +class GrepMatch: + path: str + line_number: int + line: str + + +def should_ignore_name(name: str) -> bool: + for pattern in IGNORE_PATTERNS: + if fnmatch.fnmatch(name, pattern): + return True + return False + + +def should_ignore_path(path: str) -> bool: + return any(should_ignore_name(segment) for segment in path.replace("\\", "/").split("/") if segment) + + +def path_matches(pattern: str, rel_path: str) -> bool: + path = PurePosixPath(rel_path) + if path.match(pattern): + return True + if pattern.startswith("**/"): + return path.match(pattern[3:]) + return False + + +def truncate_line(line: str, max_chars: int = DEFAULT_LINE_SUMMARY_LENGTH) -> str: + line = line.rstrip("\n\r") + if len(line) <= max_chars: + return line + return line[: max_chars - 3] + "..." + + +def is_binary_file(path: Path, sample_size: int = 8192) -> bool: + try: + with path.open("rb") as handle: + return b"\0" in handle.read(sample_size) + except OSError: + return True + + +def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]: + matches: list[str] = [] + truncated = False + root = root.resolve() + + if not root.exists(): + raise FileNotFoundError(root) + if not root.is_dir(): + raise NotADirectoryError(root) + + for current_root, dirs, files in os.walk(root): + dirs[:] = [name for name in dirs if not should_ignore_name(name)] + # root is already resolved; os.walk builds current_root by joining under root, + # so relative_to() works without an extra stat()/resolve() per directory. + rel_dir = Path(current_root).relative_to(root) + + if include_dirs: + for name in dirs: + rel_path = (rel_dir / name).as_posix() + if path_matches(pattern, rel_path): + matches.append(str(Path(current_root) / name)) + if len(matches) >= max_results: + truncated = True + return matches, truncated + + for name in files: + if should_ignore_name(name): + continue + rel_path = (rel_dir / name).as_posix() + if path_matches(pattern, rel_path): + matches.append(str(Path(current_root) / name)) + if len(matches) >= max_results: + truncated = True + return matches, truncated + + return matches, truncated + + +def find_grep_matches( + root: Path, + pattern: str, + *, + glob_pattern: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, + max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES, + line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH, +) -> tuple[list[GrepMatch], bool]: + matches: list[GrepMatch] = [] + truncated = False + root = root.resolve() + + if not root.exists(): + raise FileNotFoundError(root) + if not root.is_dir(): + raise NotADirectoryError(root) + + regex_source = re.escape(pattern) if literal else pattern + flags = 0 if case_sensitive else re.IGNORECASE + regex = re.compile(regex_source, flags) + + # Skip lines longer than this to prevent ReDoS on minified / no-newline files. + _max_line_chars = line_summary_length * 10 + + for current_root, dirs, files in os.walk(root): + dirs[:] = [name for name in dirs if not should_ignore_name(name)] + rel_dir = Path(current_root).relative_to(root) + + for name in files: + if should_ignore_name(name): + continue + + candidate_path = Path(current_root) / name + rel_path = (rel_dir / name).as_posix() + + if glob_pattern is not None and not path_matches(glob_pattern, rel_path): + continue + + try: + if candidate_path.is_symlink(): + continue + file_path = candidate_path.resolve() + if not file_path.is_relative_to(root): + continue + if file_path.stat().st_size > max_file_size or is_binary_file(file_path): + continue + with file_path.open(encoding="utf-8", errors="replace") as handle: + for line_number, line in enumerate(handle, start=1): + if len(line) > _max_line_chars: + continue + if regex.search(line): + matches.append( + GrepMatch( + path=str(file_path), + line_number=line_number, + line=truncate_line(line, line_summary_length), + ) + ) + if len(matches) >= max_results: + truncated = True + return matches, truncated + except OSError: + continue + + return matches, truncated diff --git a/backend/packages/harness/deerflow/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py index 40d176a26..55591254e 100644 --- a/backend/packages/harness/deerflow/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -7,6 +7,7 @@ from langchain.tools import ToolRuntime, tool from langgraph.typing import ContextT from deerflow.agents.thread_state import ThreadDataState, ThreadState +from deerflow.config import get_app_config from deerflow.config.paths import VIRTUAL_PATH_PREFIX from deerflow.sandbox.exceptions import ( SandboxError, @@ -16,6 +17,7 @@ from deerflow.sandbox.exceptions import ( from deerflow.sandbox.file_operation_lock import get_file_operation_lock from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.sandbox_provider import get_sandbox_provider +from deerflow.sandbox.search import GrepMatch from deerflow.sandbox.security import LOCAL_HOST_BASH_DISABLED_MESSAGE, is_host_bash_allowed _ABSOLUTE_PATH_PATTERN = re.compile(r"(?()]+)") @@ -31,6 +33,10 @@ _LOCAL_BASH_SYSTEM_PATH_PREFIXES = ( _DEFAULT_SKILLS_CONTAINER_PATH = "/mnt/skills" _ACP_WORKSPACE_VIRTUAL_PATH = "/mnt/acp-workspace" +_DEFAULT_GLOB_MAX_RESULTS = 200 +_MAX_GLOB_MAX_RESULTS = 1000 +_DEFAULT_GREP_MAX_RESULTS = 100 +_MAX_GREP_MAX_RESULTS = 500 def _get_skills_container_path() -> str: @@ -245,6 +251,69 @@ def _get_mcp_allowed_paths() -> list[str]: return allowed_paths +def _get_tool_config_int(name: str, key: str, default: int) -> int: + try: + tool_config = get_app_config().get_tool_config(name) + if tool_config is not None and key in tool_config.model_extra: + value = tool_config.model_extra.get(key) + if isinstance(value, int): + return value + except Exception: + pass + return default + + +def _clamp_max_results(value: int, *, default: int, upper_bound: int) -> int: + if value <= 0: + return default + return min(value, upper_bound) + + +def _resolve_max_results(name: str, requested: int, *, default: int, upper_bound: int) -> int: + requested_max_results = _clamp_max_results(requested, default=default, upper_bound=upper_bound) + configured_max_results = _clamp_max_results( + _get_tool_config_int(name, "max_results", default), + default=default, + upper_bound=upper_bound, + ) + return min(requested_max_results, configured_max_results) + + +def _resolve_local_read_path(path: str, thread_data: ThreadDataState) -> str: + validate_local_tool_path(path, thread_data, read_only=True) + if _is_skills_path(path): + return _resolve_skills_path(path) + if _is_acp_workspace_path(path): + return _resolve_acp_workspace_path(path, _extract_thread_id_from_thread_data(thread_data)) + return _resolve_and_validate_user_data_path(path, thread_data) + + +def _format_glob_results(root_path: str, matches: list[str], truncated: bool) -> str: + if not matches: + return f"No files matched under {root_path}" + + lines = [f"Found {len(matches)} paths under {root_path}"] + if truncated: + lines[0] += f" (showing first {len(matches)})" + lines.extend(f"{index}. {path}" for index, path in enumerate(matches, start=1)) + if truncated: + lines.append("Results truncated. Narrow the path or pattern to see fewer matches.") + return "\n".join(lines) + + +def _format_grep_results(root_path: str, matches: list[GrepMatch], truncated: bool) -> str: + if not matches: + return f"No matches found under {root_path}" + + lines = [f"Found {len(matches)} matches under {root_path}"] + if truncated: + lines[0] += f" (showing first {len(matches)})" + lines.extend(f"{match.path}:{match.line_number}: {match.line}" for match in matches) + if truncated: + lines.append("Results truncated. Narrow the path or add a glob filter.") + return "\n".join(lines) + + def _path_variants(path: str) -> set[str]: return {path, path.replace("\\", "/"), path.replace("/", "\\")} @@ -901,6 +970,126 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: return f"Error: Unexpected error listing directory: {_sanitize_error(e, runtime)}" +@tool("glob", parse_docstring=True) +def glob_tool( + runtime: ToolRuntime[ContextT, ThreadState], + description: str, + pattern: str, + path: str, + include_dirs: bool = False, + max_results: int = _DEFAULT_GLOB_MAX_RESULTS, +) -> str: + """Find files or directories that match a glob pattern under a root directory. + + Args: + description: Explain why you are searching for these paths in short words. ALWAYS PROVIDE THIS PARAMETER FIRST. + pattern: The glob pattern to match relative to the root path, for example `**/*.py`. + path: The **absolute** root directory to search under. + include_dirs: Whether matching directories should also be returned. Default is False. + max_results: Maximum number of paths to return. Default is 200. + """ + try: + sandbox = ensure_sandbox_initialized(runtime) + ensure_thread_directories_exist(runtime) + requested_path = path + effective_max_results = _resolve_max_results( + "glob", + max_results, + default=_DEFAULT_GLOB_MAX_RESULTS, + upper_bound=_MAX_GLOB_MAX_RESULTS, + ) + thread_data = None + if is_local_sandbox(runtime): + thread_data = get_thread_data(runtime) + if thread_data is None: + raise SandboxRuntimeError("Thread data not available for local sandbox") + path = _resolve_local_read_path(path, thread_data) + matches, truncated = sandbox.glob(path, pattern, include_dirs=include_dirs, max_results=effective_max_results) + if thread_data is not None: + matches = [mask_local_paths_in_output(match, thread_data) for match in matches] + return _format_glob_results(requested_path, matches, truncated) + except SandboxError as e: + return f"Error: {e}" + except FileNotFoundError: + return f"Error: Directory not found: {requested_path}" + except NotADirectoryError: + return f"Error: Path is not a directory: {requested_path}" + except PermissionError: + return f"Error: Permission denied: {requested_path}" + except Exception as e: + return f"Error: Unexpected error searching paths: {_sanitize_error(e, runtime)}" + + +@tool("grep", parse_docstring=True) +def grep_tool( + runtime: ToolRuntime[ContextT, ThreadState], + description: str, + pattern: str, + path: str, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = _DEFAULT_GREP_MAX_RESULTS, +) -> str: + """Search for matching lines inside text files under a root directory. + + Args: + description: Explain why you are searching file contents in short words. ALWAYS PROVIDE THIS PARAMETER FIRST. + pattern: The string or regex pattern to search for. + path: The **absolute** root directory to search under. + glob: Optional glob filter for candidate files, for example `**/*.py`. + literal: Whether to treat `pattern` as a plain string. Default is False. + case_sensitive: Whether matching is case-sensitive. Default is False. + max_results: Maximum number of matching lines to return. Default is 100. + """ + try: + sandbox = ensure_sandbox_initialized(runtime) + ensure_thread_directories_exist(runtime) + requested_path = path + effective_max_results = _resolve_max_results( + "grep", + max_results, + default=_DEFAULT_GREP_MAX_RESULTS, + upper_bound=_MAX_GREP_MAX_RESULTS, + ) + thread_data = None + if is_local_sandbox(runtime): + thread_data = get_thread_data(runtime) + if thread_data is None: + raise SandboxRuntimeError("Thread data not available for local sandbox") + path = _resolve_local_read_path(path, thread_data) + matches, truncated = sandbox.grep( + path, + pattern, + glob=glob, + literal=literal, + case_sensitive=case_sensitive, + max_results=effective_max_results, + ) + if thread_data is not None: + matches = [ + GrepMatch( + path=mask_local_paths_in_output(match.path, thread_data), + line_number=match.line_number, + line=match.line, + ) + for match in matches + ] + return _format_grep_results(requested_path, matches, truncated) + except SandboxError as e: + return f"Error: {e}" + except FileNotFoundError: + return f"Error: Directory not found: {requested_path}" + except NotADirectoryError: + return f"Error: Path is not a directory: {requested_path}" + except re.error as e: + return f"Error: Invalid regex pattern: {e}" + except PermissionError: + return f"Error: Permission denied: {requested_path}" + except Exception as e: + return f"Error: Unexpected error searching file contents: {_sanitize_error(e, runtime)}" + + @tool("read_file", parse_docstring=True) def read_file_tool( runtime: ToolRuntime[ContextT, ThreadState], diff --git a/backend/tests/test_sandbox_search_tools.py b/backend/tests/test_sandbox_search_tools.py new file mode 100644 index 000000000..6b6c686c4 --- /dev/null +++ b/backend/tests/test_sandbox_search_tools.py @@ -0,0 +1,393 @@ +from types import SimpleNamespace +from unittest.mock import patch + +from deerflow.community.aio_sandbox.aio_sandbox import AioSandbox +from deerflow.sandbox.local.local_sandbox import LocalSandbox +from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches +from deerflow.sandbox.tools import glob_tool, grep_tool + + +def _make_runtime(tmp_path): + workspace = tmp_path / "workspace" + uploads = tmp_path / "uploads" + outputs = tmp_path / "outputs" + workspace.mkdir() + uploads.mkdir() + outputs.mkdir() + return SimpleNamespace( + state={ + "sandbox": {"sandbox_id": "local"}, + "thread_data": { + "workspace_path": str(workspace), + "uploads_path": str(uploads), + "outputs_path": str(outputs), + }, + }, + context={"thread_id": "thread-1"}, + ) + + +def test_glob_tool_returns_virtual_paths_and_ignores_common_dirs(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "app.py").write_text("print('hi')\n", encoding="utf-8") + (workspace / "pkg").mkdir() + (workspace / "pkg" / "util.py").write_text("print('util')\n", encoding="utf-8") + (workspace / "node_modules").mkdir() + (workspace / "node_modules" / "skip.py").write_text("ignored\n", encoding="utf-8") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + result = glob_tool.func( + runtime=runtime, + description="find python files", + pattern="**/*.py", + path="/mnt/user-data/workspace", + ) + + assert "/mnt/user-data/workspace/app.py" in result + assert "/mnt/user-data/workspace/pkg/util.py" in result + assert "node_modules" not in result + assert str(workspace) not in result + + +def test_glob_tool_supports_skills_virtual_paths(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + skills_dir = tmp_path / "skills" + (skills_dir / "public" / "demo").mkdir(parents=True) + (skills_dir / "public" / "demo" / "SKILL.md").write_text("# Demo\n", encoding="utf-8") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + with ( + patch("deerflow.sandbox.tools._get_skills_container_path", return_value="/mnt/skills"), + patch("deerflow.sandbox.tools._get_skills_host_path", return_value=str(skills_dir)), + ): + result = glob_tool.func( + runtime=runtime, + description="find skills", + pattern="**/SKILL.md", + path="/mnt/skills", + ) + + assert "/mnt/skills/public/demo/SKILL.md" in result + assert str(skills_dir) not in result + + +def test_grep_tool_filters_by_glob_and_skips_binary_files(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "main.py").write_text("TODO = 'ship it'\nprint(TODO)\n", encoding="utf-8") + (workspace / "notes.txt").write_text("TODO in txt should be filtered\n", encoding="utf-8") + (workspace / "image.bin").write_bytes(b"\0binary TODO") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + result = grep_tool.func( + runtime=runtime, + description="find todo references", + pattern="TODO", + path="/mnt/user-data/workspace", + glob="**/*.py", + ) + + assert "/mnt/user-data/workspace/main.py:1: TODO = 'ship it'" in result + assert "notes.txt" not in result + assert "image.bin" not in result + assert str(workspace) not in result + + +def test_grep_tool_truncates_results(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "main.py").write_text("TODO one\nTODO two\nTODO three\n", encoding="utf-8") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + # Prevent config.yaml tool config from overriding the caller-supplied max_results=2. + monkeypatch.setattr("deerflow.sandbox.tools.get_app_config", lambda: SimpleNamespace(get_tool_config=lambda name: None)) + + result = grep_tool.func( + runtime=runtime, + description="limit matches", + pattern="TODO", + path="/mnt/user-data/workspace", + max_results=2, + ) + + assert "Found 2 matches under /mnt/user-data/workspace (showing first 2)" in result + assert "TODO one" in result + assert "TODO two" in result + assert "TODO three" not in result + assert "Results truncated." in result + + +def test_glob_tool_include_dirs_filters_nested_ignored_paths(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "src").mkdir() + (workspace / "src" / "main.py").write_text("x\n", encoding="utf-8") + (workspace / "node_modules").mkdir() + (workspace / "node_modules" / "lib").mkdir() + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + result = glob_tool.func( + runtime=runtime, + description="find dirs", + pattern="**", + path="/mnt/user-data/workspace", + include_dirs=True, + ) + + assert "src" in result + assert "node_modules" not in result + + +def test_grep_tool_literal_mode(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "file.py").write_text("price = (a+b)\nresult = a+b\n", encoding="utf-8") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + # literal=True should treat (a+b) as a plain string, not a regex group + result = grep_tool.func( + runtime=runtime, + description="literal search", + pattern="(a+b)", + path="/mnt/user-data/workspace", + literal=True, + ) + + assert "price = (a+b)" in result + assert "result = a+b" not in result + + +def test_grep_tool_case_sensitive(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "file.py").write_text("TODO: fix\ntodo: also fix\n", encoding="utf-8") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + result = grep_tool.func( + runtime=runtime, + description="case sensitive search", + pattern="TODO", + path="/mnt/user-data/workspace", + case_sensitive=True, + ) + + assert "TODO: fix" in result + assert "todo: also fix" not in result + + +def test_grep_tool_invalid_regex_returns_error(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + + result = grep_tool.func( + runtime=runtime, + description="bad pattern", + pattern="[invalid", + path="/mnt/user-data/workspace", + ) + + assert "Invalid regex pattern" in result + + +def test_aio_sandbox_glob_include_dirs_filters_nested_ignored(monkeypatch) -> None: + with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"): + sandbox = AioSandbox(id="test-sandbox", base_url="http://localhost:8080") + monkeypatch.setattr( + sandbox._client.file, + "list_path", + lambda **kwargs: SimpleNamespace( + data=SimpleNamespace( + files=[ + SimpleNamespace(name="src", path="/mnt/workspace/src"), + SimpleNamespace(name="node_modules", path="/mnt/workspace/node_modules"), + # child of node_modules — should be filtered via should_ignore_path + SimpleNamespace(name="lib", path="/mnt/workspace/node_modules/lib"), + ] + ) + ), + ) + + matches, truncated = sandbox.glob("/mnt/workspace", "**", include_dirs=True) + + assert "/mnt/workspace/src" in matches + assert "/mnt/workspace/node_modules" not in matches + assert "/mnt/workspace/node_modules/lib" not in matches + assert truncated is False + + +def test_aio_sandbox_grep_invalid_regex_raises() -> None: + with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"): + sandbox = AioSandbox(id="test-sandbox", base_url="http://localhost:8080") + + import re + + try: + sandbox.grep("/mnt/workspace", "[invalid") + assert False, "Expected re.error" + except re.error: + pass + + +def test_aio_sandbox_glob_parses_json(monkeypatch) -> None: + with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"): + sandbox = AioSandbox(id="test-sandbox", base_url="http://localhost:8080") + monkeypatch.setattr( + sandbox._client.file, + "find_files", + lambda **kwargs: SimpleNamespace(data=SimpleNamespace(files=["/mnt/user-data/workspace/app.py", "/mnt/user-data/workspace/node_modules/skip.py"])), + ) + + matches, truncated = sandbox.glob("/mnt/user-data/workspace", "**/*.py") + + assert matches == ["/mnt/user-data/workspace/app.py"] + assert truncated is False + + +def test_aio_sandbox_grep_parses_json(monkeypatch) -> None: + with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"): + sandbox = AioSandbox(id="test-sandbox", base_url="http://localhost:8080") + monkeypatch.setattr( + sandbox._client.file, + "list_path", + lambda **kwargs: SimpleNamespace( + data=SimpleNamespace( + files=[ + SimpleNamespace( + name="app.py", + path="/mnt/user-data/workspace/app.py", + is_directory=False, + ) + ] + ) + ), + ) + monkeypatch.setattr( + sandbox._client.file, + "search_in_file", + lambda **kwargs: SimpleNamespace(data=SimpleNamespace(line_numbers=[7], matches=["TODO = True"])), + ) + + matches, truncated = sandbox.grep("/mnt/user-data/workspace", "TODO") + + assert matches == [GrepMatch(path="/mnt/user-data/workspace/app.py", line_number=7, line="TODO = True")] + assert truncated is False + + +def test_find_glob_matches_raises_not_a_directory(tmp_path) -> None: + file_path = tmp_path / "file.txt" + file_path.write_text("x\n", encoding="utf-8") + + try: + find_glob_matches(file_path, "**/*.py") + assert False, "Expected NotADirectoryError" + except NotADirectoryError: + pass + + +def test_find_grep_matches_raises_not_a_directory(tmp_path) -> None: + file_path = tmp_path / "file.txt" + file_path.write_text("TODO\n", encoding="utf-8") + + try: + find_grep_matches(file_path, "TODO") + assert False, "Expected NotADirectoryError" + except NotADirectoryError: + pass + + +def test_find_grep_matches_skips_symlink_outside_root(tmp_path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("TODO outside\n", encoding="utf-8") + (workspace / "outside-link.txt").symlink_to(outside) + + matches, truncated = find_grep_matches(workspace, "TODO") + + assert matches == [] + assert truncated is False + + +def test_glob_tool_honors_smaller_requested_max_results(tmp_path, monkeypatch) -> None: + runtime = _make_runtime(tmp_path) + workspace = tmp_path / "workspace" + (workspace / "a.py").write_text("print('a')\n", encoding="utf-8") + (workspace / "b.py").write_text("print('b')\n", encoding="utf-8") + (workspace / "c.py").write_text("print('c')\n", encoding="utf-8") + + monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: LocalSandbox(id="local")) + monkeypatch.setattr( + "deerflow.sandbox.tools.get_app_config", + lambda: SimpleNamespace(get_tool_config=lambda name: SimpleNamespace(model_extra={"max_results": 50})), + ) + + result = glob_tool.func( + runtime=runtime, + description="limit glob matches", + pattern="**/*.py", + path="/mnt/user-data/workspace", + max_results=2, + ) + + assert "Found 2 paths under /mnt/user-data/workspace (showing first 2)" in result + assert "Results truncated." in result + + +def test_aio_sandbox_glob_include_dirs_enforces_root_boundary(monkeypatch) -> None: + with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"): + sandbox = AioSandbox(id="test-sandbox", base_url="http://localhost:8080") + monkeypatch.setattr( + sandbox._client.file, + "list_path", + lambda **kwargs: SimpleNamespace( + data=SimpleNamespace( + files=[ + SimpleNamespace(name="src", path="/mnt/workspace/src"), + SimpleNamespace(name="src2", path="/mnt/workspace2/src2"), + ] + ) + ), + ) + + matches, truncated = sandbox.glob("/mnt/workspace", "**", include_dirs=True) + + assert matches == ["/mnt/workspace/src"] + assert truncated is False + + +def test_aio_sandbox_grep_skips_mismatched_line_number_payloads(monkeypatch) -> None: + with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"): + sandbox = AioSandbox(id="test-sandbox", base_url="http://localhost:8080") + monkeypatch.setattr( + sandbox._client.file, + "list_path", + lambda **kwargs: SimpleNamespace( + data=SimpleNamespace( + files=[ + SimpleNamespace( + name="app.py", + path="/mnt/user-data/workspace/app.py", + is_directory=False, + ) + ] + ) + ), + ) + monkeypatch.setattr( + sandbox._client.file, + "search_in_file", + lambda **kwargs: SimpleNamespace(data=SimpleNamespace(line_numbers=[7], matches=["TODO = True", "extra"])), + ) + + matches, truncated = sandbox.grep("/mnt/user-data/workspace", "TODO") + + assert matches == [GrepMatch(path="/mnt/user-data/workspace/app.py", line_number=7, line="TODO = True")] + assert truncated is False diff --git a/config.example.yaml b/config.example.yaml index 813da9749..dc3db1a3c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -325,6 +325,16 @@ tools: group: file:read use: deerflow.sandbox.tools:read_file_tool + - name: glob + group: file:read + use: deerflow.sandbox.tools:glob_tool + max_results: 200 + + - name: grep + group: file:read + use: deerflow.sandbox.tools:grep_tool + max_results: 100 + - name: write_file group: file:write use: deerflow.sandbox.tools:write_file_tool From 1694c616ef3e48be10862f5ce66ece1bd224dfaf Mon Sep 17 00:00:00 2001 From: finallylly Date: Fri, 3 Apr 2026 19:46:22 +0800 Subject: [PATCH 29/47] feat(sandbox): add read-only support for local sandbox path mappings (#1808) --- .../deerflow/sandbox/local/local_sandbox.py | 78 +++- .../sandbox/local/local_sandbox_provider.py | 69 +++- .../harness/deerflow/sandbox/tools.py | 88 +++- .../test_local_sandbox_provider_mounts.py | 388 ++++++++++++++++++ backend/tests/test_sandbox_tools_security.py | 172 ++++++++ config.example.yaml | 6 + 6 files changed, 768 insertions(+), 33 deletions(-) create mode 100644 backend/tests/test_local_sandbox_provider_mounts.py diff --git a/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py b/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py index adcdc37bb..83129724b 100644 --- a/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py +++ b/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py @@ -1,7 +1,9 @@ +import errno import ntpath import os import shutil import subprocess +from dataclasses import dataclass from pathlib import Path from deerflow.sandbox.local.list_dir import list_dir @@ -9,6 +11,15 @@ from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches +@dataclass(frozen=True) +class PathMapping: + """A path mapping from a container path to a local path with optional read-only flag.""" + + container_path: str + local_path: str + read_only: bool = False + + class LocalSandbox(Sandbox): @staticmethod def _shell_name(shell: str) -> str: @@ -40,17 +51,42 @@ class LocalSandbox(Sandbox): return None - def __init__(self, id: str, path_mappings: dict[str, str] | None = None): + def __init__(self, id: str, path_mappings: list[PathMapping] | None = None): """ Initialize local sandbox with optional path mappings. Args: id: Sandbox identifier - path_mappings: Dictionary mapping container paths to local paths - Example: {"/mnt/skills": "/absolute/path/to/skills"} + path_mappings: List of path mappings with optional read-only flag. + Skills directory is read-only by default. """ super().__init__(id) - self.path_mappings = path_mappings or {} + self.path_mappings = path_mappings or [] + + def _is_read_only_path(self, resolved_path: str) -> bool: + """Check if a resolved path is under a read-only mount. + + When multiple mappings match (nested mounts), prefer the most specific + mapping (i.e. the one whose local_path is the longest prefix of the + resolved path), similar to how ``_resolve_path`` handles container paths. + """ + resolved = str(Path(resolved_path).resolve()) + + best_mapping: PathMapping | None = None + best_prefix_len = -1 + + for mapping in self.path_mappings: + local_resolved = str(Path(mapping.local_path).resolve()) + if resolved == local_resolved or resolved.startswith(local_resolved + os.sep): + prefix_len = len(local_resolved) + if prefix_len > best_prefix_len: + best_prefix_len = prefix_len + best_mapping = mapping + + if best_mapping is None: + return False + + return best_mapping.read_only def _resolve_path(self, path: str) -> str: """ @@ -65,7 +101,9 @@ class LocalSandbox(Sandbox): path_str = str(path) # Try each mapping (longest prefix first for more specific matches) - for container_path, local_path in sorted(self.path_mappings.items(), key=lambda x: len(x[0]), reverse=True): + for mapping in sorted(self.path_mappings, key=lambda m: len(m.container_path), reverse=True): + container_path = mapping.container_path + local_path = mapping.local_path if path_str == container_path or path_str.startswith(container_path + "/"): # Replace the container path prefix with local path relative = path_str[len(container_path) :].lstrip("/") @@ -85,15 +123,16 @@ class LocalSandbox(Sandbox): Returns: Container path if mapping exists, otherwise original path """ - path_str = str(Path(path).resolve()) + normalized_path = path.replace("\\", "/") + path_str = str(Path(normalized_path).resolve()) # Try each mapping (longest local path first for more specific matches) - for container_path, local_path in sorted(self.path_mappings.items(), key=lambda x: len(x[1]), reverse=True): - local_path_resolved = str(Path(local_path).resolve()) - if path_str.startswith(local_path_resolved): + for mapping in sorted(self.path_mappings, key=lambda m: len(m.local_path), reverse=True): + local_path_resolved = str(Path(mapping.local_path).resolve()) + if path_str == local_path_resolved or path_str.startswith(local_path_resolved + "/"): # Replace the local path prefix with container path relative = path_str[len(local_path_resolved) :].lstrip("/") - resolved = f"{container_path}/{relative}" if relative else container_path + resolved = f"{mapping.container_path}/{relative}" if relative else mapping.container_path return resolved # No mapping found, return original path @@ -112,7 +151,7 @@ class LocalSandbox(Sandbox): import re # Sort mappings by local path length (longest first) for correct prefix matching - sorted_mappings = sorted(self.path_mappings.items(), key=lambda x: len(x[1]), reverse=True) + sorted_mappings = sorted(self.path_mappings, key=lambda m: len(m.local_path), reverse=True) if not sorted_mappings: return output @@ -120,12 +159,11 @@ class LocalSandbox(Sandbox): # Create pattern that matches absolute paths # Match paths like /Users/... or other absolute paths result = output - for container_path, local_path in sorted_mappings: - local_path_resolved = str(Path(local_path).resolve()) + for mapping in sorted_mappings: # Escape the local path for use in regex - escaped_local = re.escape(local_path_resolved) - # Match the local path followed by optional path components - pattern = re.compile(escaped_local + r"(?:/[^\s\"';&|<>()]*)?") + escaped_local = re.escape(str(Path(mapping.local_path).resolve())) + # Match the local path followed by optional path components with either separator + pattern = re.compile(escaped_local + r"(?:[/\\][^\s\"';&|<>()]*)?") def replace_match(match: re.Match) -> str: matched_path = match.group(0) @@ -148,7 +186,7 @@ class LocalSandbox(Sandbox): import re # Sort mappings by length (longest first) for correct prefix matching - sorted_mappings = sorted(self.path_mappings.items(), key=lambda x: len(x[0]), reverse=True) + sorted_mappings = sorted(self.path_mappings, key=lambda m: len(m.container_path), reverse=True) # Build regex pattern to match all container paths # Match container path followed by optional path components @@ -158,7 +196,7 @@ class LocalSandbox(Sandbox): # Create pattern that matches any of the container paths. # The lookahead (?=/|$|...) ensures we only match at a path-segment boundary, # preventing /mnt/skills from matching inside /mnt/skills-extra. - patterns = [re.escape(container_path) + r"(?=/|$|[\s\"';&|<>()])(?:/[^\s\"';&|<>()]*)?" for container_path, _ in sorted_mappings] + patterns = [re.escape(m.container_path) + r"(?=/|$|[\s\"';&|<>()])(?:/[^\s\"';&|<>()]*)?" for m in sorted_mappings] pattern = re.compile("|".join(f"({p})" for p in patterns)) def replace_match(match: re.Match) -> str: @@ -249,6 +287,8 @@ class LocalSandbox(Sandbox): def write_file(self, path: str, content: str, append: bool = False) -> None: resolved_path = self._resolve_path(path) + if self._is_read_only_path(resolved_path): + raise OSError(errno.EROFS, "Read-only file system", path) try: dir_path = os.path.dirname(resolved_path) if dir_path: @@ -295,6 +335,8 @@ class LocalSandbox(Sandbox): def update_file(self, path: str, content: bytes) -> None: resolved_path = self._resolve_path(path) + if self._is_read_only_path(resolved_path): + raise OSError(errno.EROFS, "Read-only file system", path) try: dir_path = os.path.dirname(resolved_path) if dir_path: diff --git a/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py b/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py index 625f80f02..ec4693036 100644 --- a/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py +++ b/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py @@ -1,6 +1,7 @@ import logging +from pathlib import Path -from deerflow.sandbox.local.local_sandbox import LocalSandbox +from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.sandbox_provider import SandboxProvider @@ -14,16 +15,17 @@ class LocalSandboxProvider(SandboxProvider): """Initialize the local sandbox provider with path mappings.""" self._path_mappings = self._setup_path_mappings() - def _setup_path_mappings(self) -> dict[str, str]: + def _setup_path_mappings(self) -> list[PathMapping]: """ Setup path mappings for local sandbox. - Maps container paths to actual local paths, including skills directory. + Maps container paths to actual local paths, including skills directory + and any custom mounts configured in config.yaml. Returns: - Dictionary of path mappings + List of path mappings """ - mappings = {} + mappings: list[PathMapping] = [] # Map skills container path to local skills directory try: @@ -35,10 +37,63 @@ class LocalSandboxProvider(SandboxProvider): # Only add mapping if skills directory exists if skills_path.exists(): - mappings[container_path] = str(skills_path) + mappings.append( + PathMapping( + container_path=container_path, + local_path=str(skills_path), + read_only=True, # Skills directory is always read-only + ) + ) + + # Map custom mounts from sandbox config + _RESERVED_CONTAINER_PREFIXES = [container_path, "/mnt/acp-workspace", "/mnt/user-data"] + sandbox_config = config.sandbox + if sandbox_config and sandbox_config.mounts: + for mount in sandbox_config.mounts: + host_path = Path(mount.host_path) + container_path = mount.container_path.rstrip("/") or "/" + + if not host_path.is_absolute(): + logger.warning( + "Mount host_path must be absolute, skipping: %s -> %s", + mount.host_path, + mount.container_path, + ) + continue + + if not container_path.startswith("/"): + logger.warning( + "Mount container_path must be absolute, skipping: %s -> %s", + mount.host_path, + mount.container_path, + ) + continue + + # Reject mounts that conflict with reserved container paths + if any(container_path == p or container_path.startswith(p + "/") for p in _RESERVED_CONTAINER_PREFIXES): + logger.warning( + "Mount container_path conflicts with reserved prefix, skipping: %s", + mount.container_path, + ) + continue + # Ensure the host path exists before adding mapping + if host_path.exists(): + mappings.append( + PathMapping( + container_path=container_path, + local_path=str(host_path.resolve()), + read_only=mount.read_only, + ) + ) + else: + logger.warning( + "Mount host_path does not exist, skipping: %s -> %s", + mount.host_path, + mount.container_path, + ) except Exception as e: # Log but don't fail if config loading fails - logger.warning("Could not setup skills path mapping: %s", e, exc_info=True) + logger.warning("Could not setup path mappings: %s", e, exc_info=True) return mappings diff --git a/backend/packages/harness/deerflow/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py index 55591254e..acd661db0 100644 --- a/backend/packages/harness/deerflow/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -119,6 +119,54 @@ def _is_acp_workspace_path(path: str) -> bool: return path == _ACP_WORKSPACE_VIRTUAL_PATH or path.startswith(f"{_ACP_WORKSPACE_VIRTUAL_PATH}/") +def _get_custom_mounts(): + """Get custom volume mounts from sandbox config. + + Result is cached after the first successful config load. If config loading + fails an empty list is returned *without* caching so that a later call can + pick up the real value once the config is available. + """ + cached = getattr(_get_custom_mounts, "_cached", None) + if cached is not None: + return cached + try: + from pathlib import Path + + from deerflow.config import get_app_config + + config = get_app_config() + mounts = [] + if config.sandbox and config.sandbox.mounts: + # Only include mounts whose host_path exists, consistent with + # LocalSandboxProvider._setup_path_mappings() which also filters + # by host_path.exists(). + mounts = [m for m in config.sandbox.mounts if Path(m.host_path).exists()] + _get_custom_mounts._cached = mounts # type: ignore[attr-defined] + return mounts + except Exception: + # If config loading fails, return an empty list without caching so that + # a later call can retry once the config is available. + return [] + + +def _is_custom_mount_path(path: str) -> bool: + """Check if path is under a custom mount container_path.""" + for mount in _get_custom_mounts(): + if path == mount.container_path or path.startswith(f"{mount.container_path}/"): + return True + return False + + +def _get_custom_mount_for_path(path: str): + """Get the mount config matching this path (longest prefix first).""" + best = None + for mount in _get_custom_mounts(): + if path == mount.container_path or path.startswith(f"{mount.container_path}/"): + if best is None or len(mount.container_path) > len(best.container_path): + best = mount + return best + + def _extract_thread_id_from_thread_data(thread_data: "ThreadDataState | None") -> str | None: """Extract thread_id from thread_data by inspecting workspace_path. @@ -448,6 +496,8 @@ def mask_local_paths_in_output(output: str, thread_data: ThreadDataState | None) result = pattern.sub(replace_acp, result) + # Custom mount host paths are masked by LocalSandbox._reverse_resolve_paths_in_output() + # Mask user-data host paths if thread_data is None: return result @@ -496,6 +546,7 @@ def validate_local_tool_path(path: str, thread_data: ThreadDataState | None, *, - ``/mnt/user-data/*`` — always allowed (read + write) - ``/mnt/skills/*`` — allowed only when *read_only* is True - ``/mnt/acp-workspace/*`` — allowed only when *read_only* is True + - Custom mount paths (from config.yaml) — respects per-mount ``read_only`` flag Args: path: The virtual path to validate. @@ -527,7 +578,14 @@ def validate_local_tool_path(path: str, thread_data: ThreadDataState | None, *, if path.startswith(f"{VIRTUAL_PATH_PREFIX}/"): return - raise PermissionError(f"Only paths under {VIRTUAL_PATH_PREFIX}/, {_get_skills_container_path()}/, or {_ACP_WORKSPACE_VIRTUAL_PATH}/ are allowed") + # Custom mount paths — respect read_only config + if _is_custom_mount_path(path): + mount = _get_custom_mount_for_path(path) + if mount and mount.read_only and not read_only: + raise PermissionError(f"Write access to read-only mount is not allowed: {path}") + return + + raise PermissionError(f"Only paths under {VIRTUAL_PATH_PREFIX}/, {_get_skills_container_path()}/, {_ACP_WORKSPACE_VIRTUAL_PATH}/, or configured mount paths are allowed") def _validate_resolved_user_data_path(resolved: Path, thread_data: ThreadDataState) -> None: @@ -577,9 +635,10 @@ def validate_local_bash_command_paths(command: str, thread_data: ThreadDataState boundary and must not be treated as isolation from the host filesystem. In local mode, commands must use virtual paths under /mnt/user-data for - user data access. Skills paths under /mnt/skills and ACP workspace paths - under /mnt/acp-workspace are allowed (path-traversal checks only; write - prevention for bash commands is not enforced here). + user data access. Skills paths under /mnt/skills, ACP workspace paths + under /mnt/acp-workspace, and custom mount container paths (configured in + config.yaml) are allowed (path-traversal checks only; write prevention + for bash commands is not enforced here). A small allowlist of common system path prefixes is kept for executable and device references (e.g. /bin/sh, /dev/null). """ @@ -614,6 +673,11 @@ def validate_local_bash_command_paths(command: str, thread_data: ThreadDataState _reject_path_traversal(absolute_path) continue + # Allow custom mount container paths + if _is_custom_mount_path(absolute_path): + _reject_path_traversal(absolute_path) + continue + if any(absolute_path == prefix.rstrip("/") or absolute_path.startswith(prefix) for prefix in _LOCAL_BASH_SYSTEM_PATH_PREFIXES): continue @@ -658,6 +722,8 @@ def replace_virtual_paths_in_command(command: str, thread_data: ThreadDataState result = acp_pattern.sub(replace_acp_match, result) + # Custom mount paths are resolved by LocalSandbox._resolve_paths_in_command() + # Replace user-data paths if VIRTUAL_PATH_PREFIX in result and thread_data is not None: pattern = re.compile(rf"{re.escape(VIRTUAL_PATH_PREFIX)}(/[^\s\"';&|<>()]*)?") @@ -954,8 +1020,9 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: path = _resolve_skills_path(path) elif _is_acp_workspace_path(path): path = _resolve_acp_workspace_path(path, _extract_thread_id_from_thread_data(thread_data)) - else: + elif not _is_custom_mount_path(path): path = _resolve_and_validate_user_data_path(path, thread_data) + # Custom mount paths are resolved by LocalSandbox._resolve_path() children = sandbox.list_dir(path) if not children: return "(empty)" @@ -1117,8 +1184,9 @@ def read_file_tool( path = _resolve_skills_path(path) elif _is_acp_workspace_path(path): path = _resolve_acp_workspace_path(path, _extract_thread_id_from_thread_data(thread_data)) - else: + elif not _is_custom_mount_path(path): path = _resolve_and_validate_user_data_path(path, thread_data) + # Custom mount paths are resolved by LocalSandbox._resolve_path() content = sandbox.read_file(path) if not content: return "(empty)" @@ -1166,7 +1234,9 @@ def write_file_tool( if is_local_sandbox(runtime): thread_data = get_thread_data(runtime) validate_local_tool_path(path, thread_data) - path = _resolve_and_validate_user_data_path(path, thread_data) + if not _is_custom_mount_path(path): + path = _resolve_and_validate_user_data_path(path, thread_data) + # Custom mount paths are resolved by LocalSandbox._resolve_path() with get_file_operation_lock(sandbox, path): sandbox.write_file(path, content, append) return "OK" @@ -1208,7 +1278,9 @@ def str_replace_tool( if is_local_sandbox(runtime): thread_data = get_thread_data(runtime) validate_local_tool_path(path, thread_data) - path = _resolve_and_validate_user_data_path(path, thread_data) + if not _is_custom_mount_path(path): + path = _resolve_and_validate_user_data_path(path, thread_data) + # Custom mount paths are resolved by LocalSandbox._resolve_path() with get_file_operation_lock(sandbox, path): content = sandbox.read_file(path) if not content: diff --git a/backend/tests/test_local_sandbox_provider_mounts.py b/backend/tests/test_local_sandbox_provider_mounts.py new file mode 100644 index 000000000..0eb6d4654 --- /dev/null +++ b/backend/tests/test_local_sandbox_provider_mounts.py @@ -0,0 +1,388 @@ +import errno +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping +from deerflow.sandbox.local.local_sandbox_provider import LocalSandboxProvider + + +class TestPathMapping: + def test_path_mapping_dataclass(self): + mapping = PathMapping(container_path="/mnt/skills", local_path="/home/user/skills", read_only=True) + assert mapping.container_path == "/mnt/skills" + assert mapping.local_path == "/home/user/skills" + assert mapping.read_only is True + + def test_path_mapping_defaults_to_false(self): + mapping = PathMapping(container_path="/mnt/data", local_path="/home/user/data") + assert mapping.read_only is False + + +class TestLocalSandboxPathResolution: + def test_resolve_path_exact_match(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills"), + ], + ) + resolved = sandbox._resolve_path("/mnt/skills") + assert resolved == "/home/user/skills" + + def test_resolve_path_nested_path(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills"), + ], + ) + resolved = sandbox._resolve_path("/mnt/skills/agent/prompt.py") + assert resolved == "/home/user/skills/agent/prompt.py" + + def test_resolve_path_no_mapping(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills"), + ], + ) + resolved = sandbox._resolve_path("/mnt/other/file.txt") + assert resolved == "/mnt/other/file.txt" + + def test_resolve_path_longest_prefix_first(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills"), + PathMapping(container_path="/mnt", local_path="/var/mnt"), + ], + ) + resolved = sandbox._resolve_path("/mnt/skills/file.py") + # Should match /mnt/skills first (longer prefix) + assert resolved == "/home/user/skills/file.py" + + def test_reverse_resolve_path_exact_match(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path=str(skills_dir)), + ], + ) + resolved = sandbox._reverse_resolve_path(str(skills_dir)) + assert resolved == "/mnt/skills" + + def test_reverse_resolve_path_nested(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + file_path = skills_dir / "agent" / "prompt.py" + file_path.parent.mkdir() + file_path.write_text("test") + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path=str(skills_dir)), + ], + ) + resolved = sandbox._reverse_resolve_path(str(file_path)) + assert resolved == "/mnt/skills/agent/prompt.py" + + +class TestReadOnlyPath: + def test_is_read_only_true(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills", read_only=True), + ], + ) + assert sandbox._is_read_only_path("/home/user/skills/file.py") is True + + def test_is_read_only_false_for_writable(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/data", local_path="/home/user/data", read_only=False), + ], + ) + assert sandbox._is_read_only_path("/home/user/data/file.txt") is False + + def test_is_read_only_false_for_unmapped_path(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills", read_only=True), + ], + ) + # Path not under any mapping + assert sandbox._is_read_only_path("/tmp/other/file.txt") is False + + def test_is_read_only_true_for_exact_match(self): + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path="/home/user/skills", read_only=True), + ], + ) + assert sandbox._is_read_only_path("/home/user/skills") is True + + def test_write_file_blocked_on_read_only(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path=str(skills_dir), read_only=True), + ], + ) + # Skills dir is read-only, write should be blocked + with pytest.raises(OSError) as exc_info: + sandbox.write_file("/mnt/skills/new_file.py", "content") + assert exc_info.value.errno == errno.EROFS + + def test_write_file_allowed_on_writable_mount(self, tmp_path): + data_dir = tmp_path / "data" + data_dir.mkdir() + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/data", local_path=str(data_dir), read_only=False), + ], + ) + sandbox.write_file("/mnt/data/file.txt", "content") + assert (data_dir / "file.txt").read_text() == "content" + + def test_update_file_blocked_on_read_only(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + existing_file = skills_dir / "existing.py" + existing_file.write_bytes(b"original") + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path=str(skills_dir), read_only=True), + ], + ) + with pytest.raises(OSError) as exc_info: + sandbox.update_file("/mnt/skills/existing.py", b"updated") + assert exc_info.value.errno == errno.EROFS + + +class TestMultipleMounts: + def test_multiple_read_write_mounts(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + external_dir = tmp_path / "external" + external_dir.mkdir() + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/skills", local_path=str(skills_dir), read_only=True), + PathMapping(container_path="/mnt/data", local_path=str(data_dir), read_only=False), + PathMapping(container_path="/mnt/external", local_path=str(external_dir), read_only=True), + ], + ) + + # Skills is read-only + with pytest.raises(OSError): + sandbox.write_file("/mnt/skills/file.py", "content") + + # Data is writable + sandbox.write_file("/mnt/data/file.txt", "data content") + assert (data_dir / "file.txt").read_text() == "data content" + + # External is read-only + with pytest.raises(OSError): + sandbox.write_file("/mnt/external/file.txt", "content") + + def test_nested_mounts_writable_under_readonly(self, tmp_path): + """A writable mount nested under a read-only mount should allow writes.""" + ro_dir = tmp_path / "ro" + ro_dir.mkdir() + rw_dir = ro_dir / "writable" + rw_dir.mkdir() + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/repo", local_path=str(ro_dir), read_only=True), + PathMapping(container_path="/mnt/repo/writable", local_path=str(rw_dir), read_only=False), + ], + ) + + # Parent mount is read-only + with pytest.raises(OSError): + sandbox.write_file("/mnt/repo/file.txt", "content") + + # Nested writable mount should allow writes + sandbox.write_file("/mnt/repo/writable/file.txt", "content") + assert (rw_dir / "file.txt").read_text() == "content" + + def test_execute_command_path_replacement(self, tmp_path, monkeypatch): + data_dir = tmp_path / "data" + data_dir.mkdir() + test_file = data_dir / "test.txt" + test_file.write_text("hello") + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/data", local_path=str(data_dir)), + ], + ) + + # Mock subprocess to capture the resolved command + captured = {} + original_run = __import__("subprocess").run + + def mock_run(*args, **kwargs): + if len(args) > 0: + captured["command"] = args[0] + return original_run(*args, **kwargs) + + monkeypatch.setattr("deerflow.sandbox.local.local_sandbox.subprocess.run", mock_run) + monkeypatch.setattr("deerflow.sandbox.local.local_sandbox.LocalSandbox._get_shell", lambda self: "/bin/sh") + + sandbox.execute_command("cat /mnt/data/test.txt") + # Verify the command received the resolved local path + assert str(data_dir) in captured.get("command", "") + + def test_reverse_resolve_path_does_not_match_partial_prefix(self, tmp_path): + foo_dir = tmp_path / "foo" + foo_dir.mkdir() + foobar_dir = tmp_path / "foobar" + foobar_dir.mkdir() + target = foobar_dir / "file.txt" + target.write_text("test") + + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/foo", local_path=str(foo_dir)), + ], + ) + + resolved = sandbox._reverse_resolve_path(str(target)) + assert resolved == str(target.resolve()) + + def test_reverse_resolve_paths_in_output_supports_backslash_separator(self, tmp_path): + mount_dir = tmp_path / "mount" + mount_dir.mkdir() + sandbox = LocalSandbox( + "test", + [ + PathMapping(container_path="/mnt/data", local_path=str(mount_dir)), + ], + ) + + output = f"Copied: {mount_dir}\\file.txt" + masked = sandbox._reverse_resolve_paths_in_output(output) + + assert "/mnt/data/file.txt" in masked + assert str(mount_dir) not in masked + + +class TestLocalSandboxProviderMounts: + def test_setup_path_mappings_uses_configured_skills_container_path_as_reserved_prefix(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + custom_dir = tmp_path / "custom" + custom_dir.mkdir() + + from deerflow.config.sandbox_config import SandboxConfig, VolumeMountConfig + + sandbox_config = SandboxConfig( + use="deerflow.sandbox.local:LocalSandboxProvider", + mounts=[ + VolumeMountConfig(host_path=str(custom_dir), container_path="/custom-skills/nested", read_only=False), + ], + ) + config = SimpleNamespace( + skills=SimpleNamespace(container_path="/custom-skills", get_skills_path=lambda: skills_dir), + sandbox=sandbox_config, + ) + + with patch("deerflow.config.get_app_config", return_value=config): + provider = LocalSandboxProvider() + + assert [m.container_path for m in provider._path_mappings] == ["/custom-skills"] + + def test_setup_path_mappings_skips_relative_host_path(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + + from deerflow.config.sandbox_config import SandboxConfig, VolumeMountConfig + + sandbox_config = SandboxConfig( + use="deerflow.sandbox.local:LocalSandboxProvider", + mounts=[ + VolumeMountConfig(host_path="relative/path", container_path="/mnt/data", read_only=False), + ], + ) + config = SimpleNamespace( + skills=SimpleNamespace(container_path="/mnt/skills", get_skills_path=lambda: skills_dir), + sandbox=sandbox_config, + ) + + with patch("deerflow.config.get_app_config", return_value=config): + provider = LocalSandboxProvider() + + assert [m.container_path for m in provider._path_mappings] == ["/mnt/skills"] + + def test_setup_path_mappings_skips_non_absolute_container_path(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + custom_dir = tmp_path / "custom" + custom_dir.mkdir() + + from deerflow.config.sandbox_config import SandboxConfig, VolumeMountConfig + + sandbox_config = SandboxConfig( + use="deerflow.sandbox.local:LocalSandboxProvider", + mounts=[ + VolumeMountConfig(host_path=str(custom_dir), container_path="mnt/data", read_only=False), + ], + ) + config = SimpleNamespace( + skills=SimpleNamespace(container_path="/mnt/skills", get_skills_path=lambda: skills_dir), + sandbox=sandbox_config, + ) + + with patch("deerflow.config.get_app_config", return_value=config): + provider = LocalSandboxProvider() + + assert [m.container_path for m in provider._path_mappings] == ["/mnt/skills"] + + def test_setup_path_mappings_normalizes_container_path_trailing_slash(self, tmp_path): + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + custom_dir = tmp_path / "custom" + custom_dir.mkdir() + + from deerflow.config.sandbox_config import SandboxConfig, VolumeMountConfig + + sandbox_config = SandboxConfig( + use="deerflow.sandbox.local:LocalSandboxProvider", + mounts=[ + VolumeMountConfig(host_path=str(custom_dir), container_path="/mnt/data/", read_only=False), + ], + ) + config = SimpleNamespace( + skills=SimpleNamespace(container_path="/mnt/skills", get_skills_path=lambda: skills_dir), + sandbox=sandbox_config, + ) + + with patch("deerflow.config.get_app_config", return_value=config): + provider = LocalSandboxProvider() + + assert [m.container_path for m in provider._path_mappings] == ["/mnt/skills", "/mnt/data"] diff --git a/backend/tests/test_sandbox_tools_security.py b/backend/tests/test_sandbox_tools_security.py index 02d1b27ce..01aedf6be 100644 --- a/backend/tests/test_sandbox_tools_security.py +++ b/backend/tests/test_sandbox_tools_security.py @@ -8,7 +8,10 @@ import pytest from deerflow.sandbox.tools import ( VIRTUAL_PATH_PREFIX, _apply_cwd_prefix, + _get_custom_mount_for_path, + _get_custom_mounts, _is_acp_workspace_path, + _is_custom_mount_path, _is_skills_path, _reject_path_traversal, _resolve_acp_workspace_path, @@ -96,6 +99,25 @@ def test_validate_local_tool_path_rejects_non_virtual_path() -> None: validate_local_tool_path("/Users/someone/config.yaml", _THREAD_DATA) +def test_validate_local_tool_path_rejects_non_virtual_path_mentions_configured_mounts() -> None: + with pytest.raises(PermissionError, match="configured mount paths"): + validate_local_tool_path("/Users/someone/config.yaml", _THREAD_DATA) + + +def test_validate_local_tool_path_prioritizes_user_data_before_custom_mounts() -> None: + from deerflow.config.sandbox_config import VolumeMountConfig + + mounts = [ + VolumeMountConfig(host_path="/tmp/host-user-data", container_path=VIRTUAL_PATH_PREFIX, read_only=False), + ] + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=mounts): + validate_local_tool_path(f"{VIRTUAL_PATH_PREFIX}/workspace/file.txt", _THREAD_DATA, read_only=True) + + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=mounts): + with pytest.raises(PermissionError, match="path traversal"): + validate_local_tool_path(f"{VIRTUAL_PATH_PREFIX}/workspace/../../etc/passwd", _THREAD_DATA, read_only=True) + + def test_validate_local_tool_path_rejects_bare_virtual_root() -> None: """The bare /mnt/user-data root without trailing slash is not a valid sub-path.""" with pytest.raises(PermissionError, match="Only paths under"): @@ -567,6 +589,156 @@ def test_validate_local_bash_command_paths_allows_mcp_filesystem_paths() -> None validate_local_bash_command_paths("ls /mnt/d/workspace", _THREAD_DATA) +# ---------- Custom mount path tests ---------- + + +def _mock_custom_mounts(): + """Create mock VolumeMountConfig objects for testing.""" + from deerflow.config.sandbox_config import VolumeMountConfig + + return [ + VolumeMountConfig(host_path="/home/user/code-read", container_path="/mnt/code-read", read_only=True), + VolumeMountConfig(host_path="/home/user/data", container_path="/mnt/data", read_only=False), + ] + + +def test_is_custom_mount_path_recognises_configured_mounts() -> None: + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + assert _is_custom_mount_path("/mnt/code-read") is True + assert _is_custom_mount_path("/mnt/code-read/src/main.py") is True + assert _is_custom_mount_path("/mnt/data") is True + assert _is_custom_mount_path("/mnt/data/file.txt") is True + assert _is_custom_mount_path("/mnt/code-read-extra/foo") is False + assert _is_custom_mount_path("/mnt/other") is False + + +def test_get_custom_mount_for_path_returns_longest_prefix() -> None: + from deerflow.config.sandbox_config import VolumeMountConfig + + mounts = [ + VolumeMountConfig(host_path="/var/mnt", container_path="/mnt", read_only=False), + VolumeMountConfig(host_path="/home/user/code", container_path="/mnt/code", read_only=True), + ] + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=mounts): + mount = _get_custom_mount_for_path("/mnt/code/file.py") + assert mount is not None + assert mount.container_path == "/mnt/code" + + +def test_validate_local_tool_path_allows_custom_mount_read() -> None: + """read_file / ls should be able to access custom mount paths.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + validate_local_tool_path("/mnt/code-read/src/main.py", _THREAD_DATA, read_only=True) + validate_local_tool_path("/mnt/data/file.txt", _THREAD_DATA, read_only=True) + + +def test_validate_local_tool_path_blocks_read_only_mount_write() -> None: + """write_file / str_replace must NOT write to read-only custom mounts.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + with pytest.raises(PermissionError, match="Write access to read-only mount is not allowed"): + validate_local_tool_path("/mnt/code-read/src/main.py", _THREAD_DATA, read_only=False) + + +def test_validate_local_tool_path_allows_writable_mount_write() -> None: + """write_file / str_replace should succeed on writable custom mounts.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + validate_local_tool_path("/mnt/data/file.txt", _THREAD_DATA, read_only=False) + + +def test_validate_local_tool_path_blocks_traversal_in_custom_mount() -> None: + """Path traversal via .. in custom mount paths must be rejected.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + with pytest.raises(PermissionError, match="path traversal"): + validate_local_tool_path("/mnt/code-read/../../etc/passwd", _THREAD_DATA, read_only=True) + + +def test_validate_local_bash_command_paths_allows_custom_mount() -> None: + """bash commands referencing custom mount paths should be allowed.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + validate_local_bash_command_paths("cat /mnt/code-read/src/main.py", _THREAD_DATA) + validate_local_bash_command_paths("ls /mnt/data", _THREAD_DATA) + + +def test_validate_local_bash_command_paths_blocks_traversal_in_custom_mount() -> None: + """Bash commands with traversal in custom mount paths should be blocked.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + with pytest.raises(PermissionError, match="path traversal"): + validate_local_bash_command_paths("cat /mnt/code-read/../../etc/passwd", _THREAD_DATA) + + +def test_validate_local_bash_command_paths_still_blocks_non_mount_paths() -> None: + """Paths not matching any custom mount should still be blocked.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + with pytest.raises(PermissionError, match="Unsafe absolute paths"): + validate_local_bash_command_paths("cat /etc/shadow", _THREAD_DATA) + + +def test_get_custom_mounts_caching(monkeypatch, tmp_path) -> None: + """_get_custom_mounts should cache after first successful load.""" + # Clear any existing cache + if hasattr(_get_custom_mounts, "_cached"): + monkeypatch.delattr(_get_custom_mounts, "_cached") + + # Use real directories so host_path.exists() filtering passes + dir_a = tmp_path / "code-read" + dir_a.mkdir() + dir_b = tmp_path / "data" + dir_b.mkdir() + + from deerflow.config.sandbox_config import SandboxConfig, VolumeMountConfig + + mounts = [ + VolumeMountConfig(host_path=str(dir_a), container_path="/mnt/code-read", read_only=True), + VolumeMountConfig(host_path=str(dir_b), container_path="/mnt/data", read_only=False), + ] + mock_sandbox = SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider", mounts=mounts) + mock_config = SimpleNamespace(sandbox=mock_sandbox) + + with patch("deerflow.config.get_app_config", return_value=mock_config): + result = _get_custom_mounts() + assert len(result) == 2 + + # After caching, should return cached value even without mock + assert hasattr(_get_custom_mounts, "_cached") + assert len(_get_custom_mounts()) == 2 + + # Cleanup + monkeypatch.delattr(_get_custom_mounts, "_cached") + + +def test_get_custom_mounts_filters_nonexistent_host_path(monkeypatch, tmp_path) -> None: + """_get_custom_mounts should only return mounts whose host_path exists.""" + if hasattr(_get_custom_mounts, "_cached"): + monkeypatch.delattr(_get_custom_mounts, "_cached") + + from deerflow.config.sandbox_config import SandboxConfig, VolumeMountConfig + + existing_dir = tmp_path / "existing" + existing_dir.mkdir() + + mounts = [ + VolumeMountConfig(host_path=str(existing_dir), container_path="/mnt/existing", read_only=True), + VolumeMountConfig(host_path="/nonexistent/path/12345", container_path="/mnt/ghost", read_only=False), + ] + mock_sandbox = SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider", mounts=mounts) + mock_config = SimpleNamespace(sandbox=mock_sandbox) + + with patch("deerflow.config.get_app_config", return_value=mock_config): + result = _get_custom_mounts() + assert len(result) == 1 + assert result[0].container_path == "/mnt/existing" + + # Cleanup + monkeypatch.delattr(_get_custom_mounts, "_cached") + + +def test_get_custom_mount_for_path_boundary_no_false_prefix_match() -> None: + """_get_custom_mount_for_path must not match /mnt/code-read-extra for /mnt/code-read.""" + with patch("deerflow.sandbox.tools._get_custom_mounts", return_value=_mock_custom_mounts()): + mount = _get_custom_mount_for_path("/mnt/code-read-extra/foo") + assert mount is None + + def test_str_replace_parallel_updates_should_preserve_both_edits(monkeypatch) -> None: class SharedSandbox: def __init__(self) -> None: diff --git a/config.example.yaml b/config.example.yaml index dc3db1a3c..3eb0b9e9e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -375,6 +375,12 @@ sandbox: # not a secure isolation boundary for shell access. Enable only for fully # trusted, single-user local workflows. allow_host_bash: false + # Optional: Mount additional host directories into the sandbox. + # Each mount maps a host path to a virtual container path accessible by the agent. + # mounts: + # - host_path: /home/user/my-project # Absolute path on the host machine + # container_path: /mnt/my-project # Virtual path inside the sandbox + # read_only: true # Whether the mount is read-only (default: false) # Tool output truncation limits (characters). # bash uses middle-truncation (head + tail) since errors can appear anywhere in the output. From 3d4f9a88feaff07b14fcaba69c3c727390eee993 Mon Sep 17 00:00:00 2001 From: Admire <64821731+LittleChenLiya@users.noreply.github.com> Date: Fri, 3 Apr 2026 19:54:42 +0800 Subject: [PATCH 30/47] Add explicit save action for agent creation (#1798) * Add explicit save action for agent creation * Hide internal save prompts and retry agent reads --------- Co-authored-by: Willem Jiang --- .../src/app/workspace/agents/new/page.tsx | 197 ++++++++++++++---- frontend/src/core/i18n/locales/en-US.ts | 11 + frontend/src/core/i18n/locales/types.ts | 7 + frontend/src/core/i18n/locales/zh-CN.ts | 11 + frontend/src/core/messages/utils.ts | 8 + frontend/src/core/threads/hooks.ts | 37 +++- 6 files changed, 219 insertions(+), 52 deletions(-) diff --git a/frontend/src/app/workspace/agents/new/page.tsx b/frontend/src/app/workspace/agents/new/page.tsx index 7b04b4486..33f6de213 100644 --- a/frontend/src/app/workspace/agents/new/page.tsx +++ b/frontend/src/app/workspace/agents/new/page.tsx @@ -1,8 +1,16 @@ "use client"; -import { ArrowLeftIcon, BotIcon, CheckCircleIcon } from "lucide-react"; +import { + ArrowLeftIcon, + BotIcon, + CheckCircleIcon, + InfoIcon, + MoreHorizontalIcon, + SaveIcon, +} from "lucide-react"; import { useRouter } from "next/navigation"; -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { toast } from "sonner"; import { PromptInput, @@ -10,17 +18,20 @@ import { PromptInputSubmit, PromptInputTextarea, } from "@/components/ai-elements/prompt-input"; +import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; import { ArtifactsProvider } from "@/components/workspace/artifacts"; import { MessageList } from "@/components/workspace/messages"; import { ThreadContext } from "@/components/workspace/messages/context"; import type { Agent } from "@/core/agents"; -import { - AgentNameCheckError, - checkAgentName, - getAgent, -} from "@/core/agents/api"; +import { checkAgentName, getAgent } from "@/core/agents/api"; import { useI18n } from "@/core/i18n/hooks"; import { useThreadStream } from "@/core/threads/hooks"; import { uuid } from "@/core/utils/uuid"; @@ -28,23 +39,46 @@ import { isIMEComposing } from "@/lib/ime"; import { cn } from "@/lib/utils"; type Step = "name" | "chat"; +type SetupAgentStatus = "idle" | "requested" | "completed"; const NAME_RE = /^[A-Za-z0-9-]+$/; +const SAVE_HINT_STORAGE_KEY = "deerflow.agent-create.save-hint-seen"; +const AGENT_READ_RETRY_DELAYS_MS = [200, 500, 1_000, 2_000]; + +function wait(ms: number) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +async function getAgentWithRetry(agentName: string) { + for (const delay of [0, ...AGENT_READ_RETRY_DELAYS_MS]) { + if (delay > 0) { + await wait(delay); + } + + try { + return await getAgent(agentName); + } catch { + // Retry until the write settles or the attempts are exhausted. + } + } + + return null; +} export default function NewAgentPage() { const { t } = useI18n(); const router = useRouter(); - // ── Step 1: name form ────────────────────────────────────────────────────── const [step, setStep] = useState("name"); const [nameInput, setNameInput] = useState(""); const [nameError, setNameError] = useState(""); const [isCheckingName, setIsCheckingName] = useState(false); const [agentName, setAgentName] = useState(""); const [agent, setAgent] = useState(null); - // ── Step 2: chat ─────────────────────────────────────────────────────────── + const [showSaveHint, setShowSaveHint] = useState(false); + const [setupAgentStatus, setSetupAgentStatus] = + useState("idle"); - // Stable thread ID — all turns belong to the same thread const threadId = useMemo(() => uuid(), []); const [thread, sendMessage] = useThreadStream({ @@ -53,17 +87,35 @@ export default function NewAgentPage() { mode: "flash", is_bootstrap: true, }, + onFinish() { + if (!agent && setupAgentStatus === "requested") { + setSetupAgentStatus("idle"); + } + }, onToolEnd({ name }) { if (name !== "setup_agent" || !agentName) return; - getAgent(agentName) - .then((fetched) => setAgent(fetched)) - .catch(() => { - // agent write may not be flushed yet — ignore silently - }); + setSetupAgentStatus("completed"); + void getAgentWithRetry(agentName).then((fetched) => { + if (fetched) { + setAgent(fetched); + return; + } + + toast.error(t.agents.agentCreatedPendingRefresh); + }); }, }); - // ── Handlers ─────────────────────────────────────────────────────────────── + useEffect(() => { + if (typeof window === "undefined" || step !== "chat") { + return; + } + if (window.localStorage.getItem(SAVE_HINT_STORAGE_KEY) === "1") { + return; + } + setShowSaveHint(true); + window.localStorage.setItem(SAVE_HINT_STORAGE_KEY, "1"); + }, [step]); const handleConfirmName = useCallback(async () => { const trimmed = nameInput.trim(); @@ -72,6 +124,7 @@ export default function NewAgentPage() { setNameError(t.agents.nameStepInvalidError); return; } + setNameError(""); setIsCheckingName(true); try { @@ -90,6 +143,7 @@ export default function NewAgentPage() { } finally { setIsCheckingName(false); } + setAgentName(trimmed); setStep("chat"); await sendMessage(threadId, { @@ -99,12 +153,12 @@ export default function NewAgentPage() { }, [ nameInput, sendMessage, - threadId, - t.agents.nameStepBootstrapMessage, - t.agents.nameStepInvalidError, t.agents.nameStepAlreadyExistsError, t.agents.nameStepNetworkError, + t.agents.nameStepBootstrapMessage, t.agents.nameStepCheckError, + t.agents.nameStepInvalidError, + threadId, ]); const handleNameKeyDown = (e: React.KeyboardEvent) => { @@ -124,26 +178,82 @@ export default function NewAgentPage() { { agent_name: agentName }, ); }, - [thread.isLoading, sendMessage, threadId, agentName], + [agentName, sendMessage, thread.isLoading, threadId], ); - // ── Shared header ────────────────────────────────────────────────────────── + const handleSaveAgent = useCallback(async () => { + if ( + !agentName || + agent || + thread.isLoading || + setupAgentStatus !== "idle" + ) { + return; + } + + setSetupAgentStatus("requested"); + setShowSaveHint(false); + try { + await sendMessage( + threadId, + { text: t.agents.saveCommandMessage, files: [] }, + { agent_name: agentName }, + { additionalKwargs: { hide_from_ui: true } }, + ); + toast.success(t.agents.saveRequested); + } catch (error) { + setSetupAgentStatus("idle"); + toast.error(error instanceof Error ? error.message : String(error)); + } + }, [ + agent, + agentName, + sendMessage, + setupAgentStatus, + t.agents.saveCommandMessage, + t.agents.saveRequested, + thread.isLoading, + threadId, + ]); const header = ( -
- -

{t.agents.createPageTitle}

+
+
+ +

{t.agents.createPageTitle}

+
+ + {step === "chat" ? ( + + + + + + void handleSaveAgent()} + disabled={ + !!agent || thread.isLoading || setupAgentStatus !== "idle" + } + > + + {setupAgentStatus === "requested" + ? t.agents.saving + : t.agents.save} + + + + ) : null}
); - // ── Step 1: name form ────────────────────────────────────────────────────── - if (step === "name") { return (
@@ -176,9 +286,9 @@ export default function NewAgentPage() { onKeyDown={handleNameKeyDown} className={cn(nameError && "border-destructive")} /> - {nameError && ( + {nameError ? (

{nameError}

- )} + ) : null}