mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
fix(artifacts): preserve balanced ASCII delimiters in references (#6967)
This commit is contained in:
parent
73f6749068
commit
218fff3c76
@ -887,7 +887,7 @@ already received by the browser, without an additional secret-redaction layer.
|
||||
|
||||
In plan mode, malformed TODO statuses return normal tool-validation errors without aborting token attribution, so the agent can correct the call.
|
||||
|
||||
Tool-produced paths and URLs can be retained as short artifact handles across context compaction (`tool_artifacts` in `config.yaml`). Handles distinguish separate tool-result occurrences, even when a provider reuses call IDs. Standard MCP image URLs are captured alongside file references, including results from stdio and HTTP servers. Detected file URLs preserve their query strings and fragments. When PII redaction is enabled, model-visible artifact labels follow that policy; internal references stay intact for tool argument resolution. The configured registry limit retains the newest artifacts, while checkpointed processing identities prevent evicted results from being recaptured after restart. Resolution runs before authorization and write-safety checks; unknown or expired handles return an error without executing the tool. Small unknown structured results may be retained as complete JSON up to 4096 UTF-8 bytes; empty or oversized payloads are skipped. Handles are agent-local: task arguments resolve parent handles to concrete references, and delegated reports must return concrete references rather than child-local handles. A truncated model projection reports how many handles are omitted.
|
||||
Tool-produced paths and URLs can be retained as short artifact handles across context compaction (`tool_artifacts` in `config.yaml`). Handles distinguish separate tool-result occurrences, even when a provider reuses call IDs. Standard MCP image URLs are captured alongside file references, including results from stdio and HTTP servers. Detected file URLs preserve their query strings and fragments. Free-text artifact references retain balanced ASCII and CJK brackets in literal paths and URLs while trimming surrounding prose punctuation. When PII redaction is enabled, model-visible artifact labels follow that policy; internal references stay intact for tool argument resolution. The configured registry limit retains the newest artifacts, while checkpointed processing identities prevent evicted results from being recaptured after restart. Resolution runs before authorization and write-safety checks; unknown or expired handles return an error without executing the tool. Small unknown structured results may be retained as complete JSON up to 4096 UTF-8 bytes; empty or oversized payloads are skipped. Handles are agent-local: task arguments resolve parent handles to concrete references, and delegated reports must return concrete references rather than child-local handles. A truncated model projection reports how many handles are omitted.
|
||||
|
||||
DeerFlow supports configurable MCP servers and skills to extend its capabilities.
|
||||
When durable MCP background tasks are enabled, agents can use `list_background_tasks(status="failed")` or `status="input_required"` to find failed tasks or tasks awaiting input in the current chat. Other supported statuses are `submitted`, `working`, `completed`, and `cancelled`; omitting the status preserves existing behavior. The database applies the filter before limiting results to the 20 most recent matching tasks. When combined with `active_only=true`, both filters apply: active statuses are `submitted`, `working`, and `input_required`, so terminal statuses return an empty list. `GET /api/threads/{thread_id}/mcp-tasks` supports the same `status` and `active_only` filters before its `limit` (default 50, range 1–100); for example, `?status=failed&limit=20`. Unknown statuses return 422, and omitting the filters preserves the existing response.
|
||||
|
||||
@ -7,9 +7,10 @@ source. Both shapes use the existing referenceability gate and per-result dedupl
|
||||
|
||||
`artifact_registry.py` checks remote file suffixes against the complete parsed
|
||||
URL path without case sensitivity. Free-text URL and sandbox-path extraction
|
||||
strips common trailing ASCII/CJK prose punctuation while preserving CJK
|
||||
closers paired with openers inside the detected reference (including literal
|
||||
glob directory names). Structured references remain literal. See
|
||||
strips common trailing ASCII/CJK prose punctuation while preserving ASCII
|
||||
`()`, `[]`, `{}` and CJK closers paired with openers inside the detected
|
||||
reference (including literal glob directory names). Structured references remain
|
||||
literal. See
|
||||
[tool artifact middleware](../agents/middlewares/TOOL_ARTIFACTS.md).
|
||||
|
||||
Deferred-tool regex discovery uses `regex.VERSION0` with a shared 100 ms
|
||||
|
||||
@ -48,7 +48,7 @@ _STRUCTURED_TASK_KEYS = frozenset({"task_id", "job_id"})
|
||||
# commonly wraps paths in, including CJK punctuation. `\S+` would otherwise
|
||||
# consume them into `real_ref`.
|
||||
_REF_TRAILING_NOISE_CHARS = ".,;:)]}\"'`\u3002\uff0c\uff1b\uff1a\u3001\uff09\u3011\u300b\u201d\u2019"
|
||||
_CJK_CLOSING_DELIMITERS = {")": "(", "】": "【", "》": "《", "”": "“", "’": "‘"}
|
||||
_REF_CLOSING_DELIMITERS = {")": "(", "]": "[", "}": "{", ")": "(", "】": "【", "》": "《", "”": "“", "’": "‘"}
|
||||
|
||||
# Content-block and structured-key refs are trusted only in these shapes.
|
||||
# `data:`/`blob:` URIs can carry arbitrarily large embedded payloads (MCP
|
||||
@ -131,16 +131,16 @@ def _make_entry(
|
||||
|
||||
|
||||
def _strip_ref_trailing_noise(raw: str) -> str:
|
||||
"""Strip prose closers while preserving paired CJK characters inside a ref."""
|
||||
"""Strip prose closers while preserving paired ASCII/CJK delimiters inside a ref."""
|
||||
end = len(raw.rstrip(_REF_TRAILING_NOISE_CHARS))
|
||||
if end == len(raw):
|
||||
return raw
|
||||
|
||||
unmatched_openers = dict.fromkeys(_CJK_CLOSING_DELIMITERS.values(), 0)
|
||||
unmatched_openers = dict.fromkeys(_REF_CLOSING_DELIMITERS.values(), 0)
|
||||
for index, char in enumerate(raw):
|
||||
if char in unmatched_openers:
|
||||
unmatched_openers[char] += 1
|
||||
elif (opener := _CJK_CLOSING_DELIMITERS.get(char)) and unmatched_openers[opener]:
|
||||
elif (opener := _REF_CLOSING_DELIMITERS.get(char)) and unmatched_openers[opener]:
|
||||
unmatched_openers[opener] -= 1
|
||||
# An opener outside the detected token is prose; an opener inside
|
||||
# it makes this closer part of a literal directory name or URL.
|
||||
|
||||
49
backend/tests/test_artifact_ascii_delimiters.py
Normal file
49
backend/tests/test_artifact_ascii_delimiters.py
Normal file
@ -0,0 +1,49 @@
|
||||
"""Literal ASCII delimiters must survive artifact capture and resolution."""
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
|
||||
from deerflow.agents.middlewares.artifact_resolution_middleware import ArtifactResolutionMiddleware
|
||||
from deerflow.sandbox.tools import _format_glob_results
|
||||
from deerflow.tools.artifact_registry import _detect_refs_in_text, extract_artifacts_from_result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["archive(old)", "archive[old]", "archive{old}", "archive(old[1])", "archive)old(done)"])
|
||||
@pytest.mark.parametrize("content_block", [False, True])
|
||||
def test_glob_directory_handle_resolves_to_existing_directory(tmp_path, name, content_block):
|
||||
directory = tmp_path / name
|
||||
directory.mkdir()
|
||||
root = "/mnt/user-data/workspace"
|
||||
reference = f"{root}/{name}"
|
||||
text = _format_glob_results(root, [reference], truncated=False)
|
||||
message = ToolMessage(content=[{"type": "text", "text": text}] if content_block else text, tool_call_id="glob-1", name="glob")
|
||||
entries = extract_artifacts_from_result(message, thread_id="ascii-delimiters")
|
||||
request = ToolCallRequest(
|
||||
tool_call={"name": "ls", "args": {"path": entries[-1]["handle"]}, "id": "ls-1", "type": "tool_call"},
|
||||
tool=None,
|
||||
state={"tool_artifacts": entries},
|
||||
runtime=None,
|
||||
)
|
||||
|
||||
def handler(resolved):
|
||||
path = resolved.tool_call["args"]["path"]
|
||||
assert path == reference
|
||||
assert (tmp_path / path.removeprefix(f"{root}/")).is_dir()
|
||||
return ToolMessage(content="directory exists", tool_call_id="ls-1")
|
||||
|
||||
assert ArtifactResolutionMiddleware().wrap_tool_call(request, handler).content == "directory exists"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ref", ["/mnt/user-data/workspace/archive(old)", "/mnt/user-data/workspace/archive[old]", "/mnt/user-data/workspace/archive{old}", "https://files.example/report.pdf?label=(old)", "https://files.example/report.pdf#section[1]"]
|
||||
)
|
||||
@pytest.mark.parametrize("suffix", ["", ".", ")", "]", "。】"])
|
||||
def test_balanced_ascii_closers_survive_prose_trimming(ref, suffix):
|
||||
assert [entry["ref"] for entry in _detect_refs_in_text(f"Reference: ({ref}{suffix}")] == [ref]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", [")", "]", "}", ").", "]。"])
|
||||
def test_unpaired_ascii_prose_closers_still_trimmed(suffix):
|
||||
ref = "/mnt/user-data/outputs/report.pdf"
|
||||
assert [entry["ref"] for entry in _detect_refs_in_text(f"Reference: {ref}{suffix}")] == [ref]
|
||||
@ -134,6 +134,8 @@ def test_handle_projected_into_model_context():
|
||||
("https://files.example/report.pdf?token=abc", "]"),
|
||||
pytest.param("https://files.example/report.pdf?token=part.csvX#page=2", "\u3002\u201d\uff09", id="cjk-punctuation"),
|
||||
pytest.param("https://files.example/report.PdF?token=part.csvX#page=2", "]", id="mixed-case-extension"),
|
||||
pytest.param("https://files.example/report.pdf?label=(old)", "]", id="ascii-query-parentheses"),
|
||||
pytest.param("https://files.example/report.pdf#section[1]", "]", id="ascii-fragment-brackets"),
|
||||
],
|
||||
)
|
||||
def test_remote_url_survives_capture_checkpoint_and_resolved_download(url, closing_punctuation):
|
||||
@ -177,10 +179,10 @@ def test_remote_url_survives_capture_checkpoint_and_resolved_download(url, closi
|
||||
assert persisted["real_ref"] == url
|
||||
|
||||
|
||||
def test_glob_directory_survives_capture_checkpoint_and_resolved_listing(tmp_path):
|
||||
@pytest.mark.parametrize("directory_name", ["项目(归档)", "archive(old)", "archive[old]", "archive{old}", "archive(old[1])"])
|
||||
def test_glob_directory_survives_capture_checkpoint_and_resolved_listing(tmp_path, directory_name):
|
||||
from deerflow.sandbox.tools import _format_glob_results
|
||||
|
||||
directory_name = "项目(归档)"
|
||||
real_ref = f"/mnt/user-data/workspace/{directory_name}"
|
||||
directory = tmp_path / directory_name
|
||||
directory.mkdir()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user