fix(knowledge): scope ragflow link cleanup to its native citation namespace (#6886)

* fix(knowledge): scope ragflow link cleanup to its native citation namespace

budget_source_artifact cleaned a delegated task summary with two regexes that
matched any #knowledge-... destination. When a task result carried both the
ragflow-native contract (#knowledge-<id>-<n>) and the unified contract from the
retrieval refactor (#knowledge-source-<id>), those regexes also flattened the
unified citation to plain text. budget_source_artifact only rebuilds the native
knowledge_sources, so the now-linkless unified evidence then failed the
prune_sources substring check and retrieval_sources was emptied. The native
citations in the same result survived, so a mixed result lost attribution for
its unified sources while keeping it for ragflow ones, even with budget headroom
to spare.

Anchor both regexes to the native namespace with a (?!source-) lookahead so they
leave #knowledge-source-<id> for prune_sources to reconcile downstream. Native
ids are 32-hex and never begin with "source-", so native link cleanup is
unchanged.

Add a regression that drives the real middleware entry point on a mixed-contract
task result and asserts the unified citation stays clickable and its
retrieval_sources record survives alongside the native one.

Signed-off-by: CorgiBoyG <111257566+CorgiBoyG@users.noreply.github.com>

* fix(knowledge): remove dangling citation fragments from task summaries

---------

Signed-off-by: CorgiBoyG <111257566+CorgiBoyG@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

README.md

@@ -1377,6 +1377,8 @@ numbers use the selected interface language. New providers use the shared
 envelope; existing RAGFlow citation histories remain supported.
 Expand a completed subtask card to inspect its report's forwarded knowledge
 sources, including when reopening a saved conversation.
+Shortened mixed-provider task reports remove dangling citation fragments
+without flattening retained shared-source links.
 Existing web citations and the text-only Python `knowledge_search(query)` APIs
 remain supported. No knowledge management service or page is required.
 See [the knowledge contract guide](backend/docs/KNOWLEDGE.md).

backend/packages/harness/deerflow/community/ragflow/AGENTS.md

@@ -26,6 +26,9 @@ together, including delegated results and model-request history. Never shorten
 an excerpt under an existing ID. Drop entries that cannot fit, with an omission
 notice, while preserving unrelated artifact fields. This honors per-tool and
 fallback limits without exempting citation-bearing results from the budget.
+Synopsis cleanup preserves complete shared citation links (including escaped
+labels and sanitized anchors) while removing orphaned/truncated destinations,
+both before and after the synopsis head/tail split.
 
 ## Document validation
 

backend/packages/harness/deerflow/community/ragflow/sources.py

@@ -6,6 +6,16 @@ from collections.abc import Mapping
 from typing import Any
 
 
+def _clean_citation_fragments(summary: str) -> str:
+    """Keep complete shared links; remove orphaned or truncated destinations."""
+    return re.sub(
+        r"(\[(?:\\.|[^\[\]\\\n])*\]\(#(?:user-content-)?knowledge-source-[a-f0-9]{32}\))"
+        r"|#(?:user-content-)?knowledge-[\w-]*",
+        lambda match: match[1] or "",
+        summary,
+    )
+
+
 def budget_source_artifact(content: str, artifact: object, max_chars: int, *, summary: str = "") -> tuple[str, dict[str, Any] | None] | None:
     """Keep whole evidence records and their links together within a tool budget.
 
@@ -38,9 +48,13 @@ def budget_source_artifact(content: str, artifact: object, max_chars: int, *, su
 
     notice = "Knowledge sources omitted to fit output budget; request smaller excerpts."
     # Remove old destinations before shortening the report, including any link
-    # fragments left by the generic synopsis/truncation transform.
-    summary = re.sub(r"\[([^\]\n]*)\]\(#(?:user-content-)?knowledge-[^)\s]*\)", r"\1", summary)
-    summary = re.sub(r"#(?:user-content-)?knowledge-[\w-]*", "", summary)
+    # fragments left by the generic synopsis/truncation transform. Scope the
+    # match to the native ``#knowledge-<id>`` namespace: the unified
+    # ``#knowledge-source-<id>`` contract is a different namespace that
+    # ``prune_sources`` reconciles downstream, so flattening it here would
+    # strip a legitimate citation this budgeter never rebuilds.
+    summary = re.sub(r"\[([^\]\n]*)\]\(#(?:user-content-)?knowledge-(?!source-)[^)\s]*\)", r"\1", summary)
+    summary = _clean_citation_fragments(summary)
     summary_limit = min(1000, max_chars // 4) if max_chars >= 160 else 0
     if not summary_limit:
         summary = ""
@@ -49,7 +63,8 @@ def budget_source_artifact(content: str, artifact: object, max_chars: int, *, su
         # for the full externalized task report.
         head = (summary_limit - 3) // 2
         summary = summary[:head] + "\n…\n" + summary[-(summary_limit - head - 3) :]
-    summary = summary.rstrip()
+    # The head/tail split can cut a previously complete shared link.
+    summary = _clean_citation_fragments(summary).rstrip()
     entries = [summary] if summary else []
     used = len(summary)
     retained = []

backend/tests/test_knowledge_citation_budget.py

@@ -9,8 +9,10 @@ from langgraph.types import Command
 
 from deerflow.agents.middlewares.tool_output_budget_middleware import ToolOutputBudgetMiddleware, _patch_model_messages, _patch_result
 from deerflow.community.ragflow.formatting import format_retrieval_sources
-from deerflow.community.ragflow.sources import budget_source_artifact
+from deerflow.community.ragflow.sources import budget_source_artifact, cited_source_artifact
 from deerflow.config.tool_output_config import ToolOutputConfig
+from deerflow.knowledge.contract import KnowledgeSource
+from deerflow.knowledge.sources import forward_sources
 
 
 def message(name="knowledge_search"):
@@ -125,3 +127,101 @@ def test_delegated_budget_retains_the_report_reference_and_complete_evidence():
     assert content.startswith("Task completed.")
     assert "/mnt/user-data/outputs/.tool-results/report.txt" in content
     assert_paired(original, original.model_copy(update={"content": content, "artifact": artifact}), 7000)
+
+
+def mixed_contract_task_message(unified):
+    """A task result carrying BOTH citation contracts at once.
+
+    RAGFlow-native evidence (``#knowledge-<32hex>-<n>``, dict envelope) is
+    harvested parent-side by ``cited_source_artifact``; the unified contract
+    (``#knowledge-source-<32hex>``, list envelope) is forwarded through
+    ``forward_sources`` the way ``task_tool`` merges an accumulated child's
+    ``knowledge_sources``. Both links sit near the head so a head/tail
+    truncation keeps them in the budgeted window.
+    """
+    rag_id = "a" * 32 + "-1"
+    report = f"Task completed. Synopsis.\n\n[citation:{unified.title}](#knowledge-source-{unified.id})\n{unified.excerpt}\n\n[citation:1](#knowledge-{rag_id}) Engineering / Manual.pdf\nragflow native excerpt text one\n\n" + "filler " * 3000
+    source_messages = [
+        {
+            "type": "tool",
+            "name": "knowledge_search",
+            "content": f"[citation:1](#knowledge-{rag_id}) Engineering / Manual.pdf\nragflow native excerpt text one",
+            "artifact": {"knowledge_sources": {"version": 1, "sources": [{"id": rag_id, "provider": "ragflow", "text": "ragflow native excerpt text one", "dataset_name": "Engineering", "document_name": "Manual.pdf"}]}},
+        }
+    ]
+    native_artifact = cited_source_artifact(source_messages, report, include_omissions=True)
+    content, unified_artifact = forward_sources(report, [unified.model_dump(mode="json", exclude_none=True)])
+    artifact = {**native_artifact, "retrieval_sources": unified_artifact["knowledge_sources"], "knowledge_sources_version": 1}
+    return ToolMessage(content=content, artifact=artifact, name="task", tool_call_id="call-1"), rag_id
+
+
+@pytest.mark.parametrize("limit", [7000, 20000])
+def test_mixed_contract_budget_keeps_unified_citations_alongside_native(limit):
+    """Budgeting a result that carries both contracts must not drop one of them.
+
+    The RAGFlow summary-cleaning step must stay scoped to the native link
+    shape: it may not flatten the unified ``#knowledge-source-<id>`` citation
+    or discard ``retrieval_sources`` as a side effect of retaining the native
+    sources. The content budgeted here is far smaller than ``limit``, so any
+    loss is attribution loss, not conservative trimming under pressure.
+    """
+    unified = KnowledgeSource(provider="lightrag", title="UnifiedDoc", excerpt="unified evidence excerpt alpha that is clearly load bearing content")
+    original, rag_id = mixed_contract_task_message(unified)
+    config = ToolOutputConfig(externalize_min_chars=0, fallback_max_chars=limit)
+    result = _patch_result(original, config, None)
+    content = result.content if isinstance(result.content, str) else result.content[0]["text"]
+    # The native citation keeps its clickable link and record (the existing behavior).
+    assert f"](#knowledge-{rag_id})" in content
+    # The unified citation must remain clickable, not flattened to plain text.
+    assert f"](#knowledge-source-{unified.id})" in content
+    assert f"citation:{unified.title}" not in content.replace(f"[citation:{unified.title}](#knowledge-source-{unified.id})", "")
+    # Its structured record must survive paired with the visible link.
+    retrieval_sources = (result.artifact or {}).get("retrieval_sources") or []
+    assert any(source.get("id") == unified.id for source in retrieval_sources)
+
+
+@pytest.mark.parametrize("prefix", ["", "user-content-"])
+@pytest.mark.parametrize(
+    "fragment",
+    ["](#{}knowledge-source-abc12", "#{}knowledge-source-" + "b" * 32, "[citation:Broken](#{}knowledge-source-" + "b" * 32, "[citation:Broken](#{}knowledge-source-" + "b" * 33 + ")"],
+    ids=["partial-id", "orphan", "unclosed-link", "overlong-id"],
+)
+def test_mixed_contract_budget_removes_dangling_unified_fragments(prefix, fragment):
+    unified = KnowledgeSource(provider="lightrag", title="UnifiedDoc", excerpt="unified evidence excerpt alpha that is clearly load bearing content")
+    original, rag_id = mixed_contract_task_message(unified)
+    fragment = fragment.format(prefix)
+    original = original.model_copy(update={"content": f"Task completed. {fragment}\n{original.content}"})
+
+    result = _patch_result(original, ToolOutputConfig(externalize_min_chars=0, fallback_max_chars=7000), None)
+
+    assert fragment not in result.content
+    assert f"](#knowledge-source-{unified.id})" in result.content
+    assert f"](#knowledge-{rag_id})" in result.content
+    assert [source["id"] for source in result.artifact["retrieval_sources"]] == [unified.id]
+    assert len(result.content) <= 7000
+
+
+@pytest.mark.parametrize("prefix", ["", "user-content-"])
+def test_native_budget_preserves_complete_unified_links_with_escaped_labels(prefix):
+    original = message("task")
+    shared_link = rf"[citation:Guide \[draft\]](#{prefix}knowledge-source-{'b' * 32})"
+    content, _ = budget_source_artifact(original.content, original.artifact, 7000, summary=f"Task completed. {shared_link}")
+
+    assert shared_link in content
+
+
+@pytest.mark.parametrize("prefix", ["", "user-content-"])
+@pytest.mark.parametrize("side", ["head", "tail"])
+def test_native_summary_truncation_removes_new_unified_fragments(prefix, side):
+    original = message("task")
+    destination = f"#{prefix}knowledge-source-{'b' * 32}"
+    link = f"[citation:Guide]({destination})"
+    # A 2,000-character output gives this synopsis a 500-character head/tail
+    # budget. Split the head inside the ID or start the tail at its '#'.
+    summary = "x" * 200 + link + "y" * 1000 if side == "head" else "x" * 1000 + link + "y" * (249 - len(destination) - 1)
+
+    content, _ = budget_source_artifact(original.content, original.artifact, 2000, summary=summary)
+
+    assert "#knowledge-source-" not in content
+    assert "#user-content-knowledge-source-" not in content
+    assert len(content) <= 2000