diff --git a/README.md b/README.md index 2f1ee19496..df07b3f02f 100644 --- a/README.md +++ b/README.md @@ -1377,6 +1377,8 @@ numbers use the selected interface language. New providers use the shared envelope; existing RAGFlow citation histories remain supported. Expand a completed subtask card to inspect its report's forwarded knowledge sources, including when reopening a saved conversation. +Shortened mixed-provider task reports remove dangling citation fragments +without flattening retained shared-source links. Existing web citations and the text-only Python `knowledge_search(query)` APIs remain supported. No knowledge management service or page is required. See [the knowledge contract guide](backend/docs/KNOWLEDGE.md). diff --git a/backend/packages/harness/deerflow/community/ragflow/AGENTS.md b/backend/packages/harness/deerflow/community/ragflow/AGENTS.md index d6e2c089a6..e1f1d0bc7d 100644 --- a/backend/packages/harness/deerflow/community/ragflow/AGENTS.md +++ b/backend/packages/harness/deerflow/community/ragflow/AGENTS.md @@ -26,6 +26,9 @@ together, including delegated results and model-request history. Never shorten an excerpt under an existing ID. Drop entries that cannot fit, with an omission notice, while preserving unrelated artifact fields. This honors per-tool and fallback limits without exempting citation-bearing results from the budget. +Synopsis cleanup preserves complete shared citation links (including escaped +labels and sanitized anchors) while removing orphaned/truncated destinations, +both before and after the synopsis head/tail split. ## Document validation diff --git a/backend/packages/harness/deerflow/community/ragflow/sources.py b/backend/packages/harness/deerflow/community/ragflow/sources.py index 192fa53808..23ed1a68bc 100644 --- a/backend/packages/harness/deerflow/community/ragflow/sources.py +++ b/backend/packages/harness/deerflow/community/ragflow/sources.py @@ -6,6 +6,16 @@ from collections.abc import Mapping from typing import Any +def _clean_citation_fragments(summary: str) -> str: + """Keep complete shared links; remove orphaned or truncated destinations.""" + return re.sub( + r"(\[(?:\\.|[^\[\]\\\n])*\]\(#(?:user-content-)?knowledge-source-[a-f0-9]{32}\))" + r"|#(?:user-content-)?knowledge-[\w-]*", + lambda match: match[1] or "", + summary, + ) + + def budget_source_artifact(content: str, artifact: object, max_chars: int, *, summary: str = "") -> tuple[str, dict[str, Any] | None] | None: """Keep whole evidence records and their links together within a tool budget. @@ -38,9 +48,13 @@ def budget_source_artifact(content: str, artifact: object, max_chars: int, *, su notice = "Knowledge sources omitted to fit output budget; request smaller excerpts." # Remove old destinations before shortening the report, including any link - # fragments left by the generic synopsis/truncation transform. - summary = re.sub(r"\[([^\]\n]*)\]\(#(?:user-content-)?knowledge-[^)\s]*\)", r"\1", summary) - summary = re.sub(r"#(?:user-content-)?knowledge-[\w-]*", "", summary) + # fragments left by the generic synopsis/truncation transform. Scope the + # match to the native ``#knowledge-`` namespace: the unified + # ``#knowledge-source-`` contract is a different namespace that + # ``prune_sources`` reconciles downstream, so flattening it here would + # strip a legitimate citation this budgeter never rebuilds. + summary = re.sub(r"\[([^\]\n]*)\]\(#(?:user-content-)?knowledge-(?!source-)[^)\s]*\)", r"\1", summary) + summary = _clean_citation_fragments(summary) summary_limit = min(1000, max_chars // 4) if max_chars >= 160 else 0 if not summary_limit: summary = "" @@ -49,7 +63,8 @@ def budget_source_artifact(content: str, artifact: object, max_chars: int, *, su # for the full externalized task report. head = (summary_limit - 3) // 2 summary = summary[:head] + "\n…\n" + summary[-(summary_limit - head - 3) :] - summary = summary.rstrip() + # The head/tail split can cut a previously complete shared link. + summary = _clean_citation_fragments(summary).rstrip() entries = [summary] if summary else [] used = len(summary) retained = [] diff --git a/backend/tests/test_knowledge_citation_budget.py b/backend/tests/test_knowledge_citation_budget.py index 685e145e05..136370226b 100644 --- a/backend/tests/test_knowledge_citation_budget.py +++ b/backend/tests/test_knowledge_citation_budget.py @@ -9,8 +9,10 @@ from langgraph.types import Command from deerflow.agents.middlewares.tool_output_budget_middleware import ToolOutputBudgetMiddleware, _patch_model_messages, _patch_result from deerflow.community.ragflow.formatting import format_retrieval_sources -from deerflow.community.ragflow.sources import budget_source_artifact +from deerflow.community.ragflow.sources import budget_source_artifact, cited_source_artifact from deerflow.config.tool_output_config import ToolOutputConfig +from deerflow.knowledge.contract import KnowledgeSource +from deerflow.knowledge.sources import forward_sources def message(name="knowledge_search"): @@ -125,3 +127,101 @@ def test_delegated_budget_retains_the_report_reference_and_complete_evidence(): assert content.startswith("Task completed.") assert "/mnt/user-data/outputs/.tool-results/report.txt" in content assert_paired(original, original.model_copy(update={"content": content, "artifact": artifact}), 7000) + + +def mixed_contract_task_message(unified): + """A task result carrying BOTH citation contracts at once. + + RAGFlow-native evidence (``#knowledge-<32hex>-``, dict envelope) is + harvested parent-side by ``cited_source_artifact``; the unified contract + (``#knowledge-source-<32hex>``, list envelope) is forwarded through + ``forward_sources`` the way ``task_tool`` merges an accumulated child's + ``knowledge_sources``. Both links sit near the head so a head/tail + truncation keeps them in the budgeted window. + """ + rag_id = "a" * 32 + "-1" + report = f"Task completed. Synopsis.\n\n[citation:{unified.title}](#knowledge-source-{unified.id})\n{unified.excerpt}\n\n[citation:1](#knowledge-{rag_id}) Engineering / Manual.pdf\nragflow native excerpt text one\n\n" + "filler " * 3000 + source_messages = [ + { + "type": "tool", + "name": "knowledge_search", + "content": f"[citation:1](#knowledge-{rag_id}) Engineering / Manual.pdf\nragflow native excerpt text one", + "artifact": {"knowledge_sources": {"version": 1, "sources": [{"id": rag_id, "provider": "ragflow", "text": "ragflow native excerpt text one", "dataset_name": "Engineering", "document_name": "Manual.pdf"}]}}, + } + ] + native_artifact = cited_source_artifact(source_messages, report, include_omissions=True) + content, unified_artifact = forward_sources(report, [unified.model_dump(mode="json", exclude_none=True)]) + artifact = {**native_artifact, "retrieval_sources": unified_artifact["knowledge_sources"], "knowledge_sources_version": 1} + return ToolMessage(content=content, artifact=artifact, name="task", tool_call_id="call-1"), rag_id + + +@pytest.mark.parametrize("limit", [7000, 20000]) +def test_mixed_contract_budget_keeps_unified_citations_alongside_native(limit): + """Budgeting a result that carries both contracts must not drop one of them. + + The RAGFlow summary-cleaning step must stay scoped to the native link + shape: it may not flatten the unified ``#knowledge-source-`` citation + or discard ``retrieval_sources`` as a side effect of retaining the native + sources. The content budgeted here is far smaller than ``limit``, so any + loss is attribution loss, not conservative trimming under pressure. + """ + unified = KnowledgeSource(provider="lightrag", title="UnifiedDoc", excerpt="unified evidence excerpt alpha that is clearly load bearing content") + original, rag_id = mixed_contract_task_message(unified) + config = ToolOutputConfig(externalize_min_chars=0, fallback_max_chars=limit) + result = _patch_result(original, config, None) + content = result.content if isinstance(result.content, str) else result.content[0]["text"] + # The native citation keeps its clickable link and record (the existing behavior). + assert f"](#knowledge-{rag_id})" in content + # The unified citation must remain clickable, not flattened to plain text. + assert f"](#knowledge-source-{unified.id})" in content + assert f"citation:{unified.title}" not in content.replace(f"[citation:{unified.title}](#knowledge-source-{unified.id})", "") + # Its structured record must survive paired with the visible link. + retrieval_sources = (result.artifact or {}).get("retrieval_sources") or [] + assert any(source.get("id") == unified.id for source in retrieval_sources) + + +@pytest.mark.parametrize("prefix", ["", "user-content-"]) +@pytest.mark.parametrize( + "fragment", + ["](#{}knowledge-source-abc12", "#{}knowledge-source-" + "b" * 32, "[citation:Broken](#{}knowledge-source-" + "b" * 32, "[citation:Broken](#{}knowledge-source-" + "b" * 33 + ")"], + ids=["partial-id", "orphan", "unclosed-link", "overlong-id"], +) +def test_mixed_contract_budget_removes_dangling_unified_fragments(prefix, fragment): + unified = KnowledgeSource(provider="lightrag", title="UnifiedDoc", excerpt="unified evidence excerpt alpha that is clearly load bearing content") + original, rag_id = mixed_contract_task_message(unified) + fragment = fragment.format(prefix) + original = original.model_copy(update={"content": f"Task completed. {fragment}\n{original.content}"}) + + result = _patch_result(original, ToolOutputConfig(externalize_min_chars=0, fallback_max_chars=7000), None) + + assert fragment not in result.content + assert f"](#knowledge-source-{unified.id})" in result.content + assert f"](#knowledge-{rag_id})" in result.content + assert [source["id"] for source in result.artifact["retrieval_sources"]] == [unified.id] + assert len(result.content) <= 7000 + + +@pytest.mark.parametrize("prefix", ["", "user-content-"]) +def test_native_budget_preserves_complete_unified_links_with_escaped_labels(prefix): + original = message("task") + shared_link = rf"[citation:Guide \[draft\]](#{prefix}knowledge-source-{'b' * 32})" + content, _ = budget_source_artifact(original.content, original.artifact, 7000, summary=f"Task completed. {shared_link}") + + assert shared_link in content + + +@pytest.mark.parametrize("prefix", ["", "user-content-"]) +@pytest.mark.parametrize("side", ["head", "tail"]) +def test_native_summary_truncation_removes_new_unified_fragments(prefix, side): + original = message("task") + destination = f"#{prefix}knowledge-source-{'b' * 32}" + link = f"[citation:Guide]({destination})" + # A 2,000-character output gives this synopsis a 500-character head/tail + # budget. Split the head inside the ID or start the tail at its '#'. + summary = "x" * 200 + link + "y" * 1000 if side == "head" else "x" * 1000 + link + "y" * (249 - len(destination) - 1) + + content, _ = budget_source_artifact(original.content, original.artifact, 2000, summary=summary) + + assert "#knowledge-source-" not in content + assert "#user-content-knowledge-source-" not in content + assert len(content) <= 2000