fix(knowledge): retain later evidence that fits the output budget (#7942)

README.md

@@ -1823,6 +1823,9 @@ and when a subagent performed the retrieval. Missing page numbers and public
 URLs remain absent; sources without a URL are readable as local evidence.
 Inline citations remain readable after Markdown sanitization, and supplied page
 numbers use the selected interface language.
+Shared knowledge results skip entries that exceed the remaining output budget
+and retain later fitting entries in provider order. Retained citations stay
+paired with their visible evidence.
 Copying a source reference produces a Markdown link that escapes brackets in the
 visible title and wraps a URL containing parentheses in angle brackets, so the
 copied reference stays a valid link. New providers use the shared

backend/docs/KNOWLEDGE.md

@@ -200,9 +200,13 @@ Inline shared references accept both the raw `#knowledge-source-ID` and the
 sanitized `#user-content-knowledge-source-ID`; the sanitizer's DOM-clobbering
 protection stays enabled. Missing or malformed references render inert labels.
 
-`format_result` budgets chunk text and source evidence together: a cited hit
-that cannot fit is omitted as a whole. Results display at most 100 hits, matching
-the bounded artifact readers. Only displayed excerpts enter artifacts.
+`format_result` budgets chunk text and source evidence together: a hit
+that cannot fit is omitted as a whole, and later fitting hits remain eligible.
+It examines only the first 100 input hits in provider order, including skipped
+hits. Per-chunk truncation precedes packing; the remaining total budget does
+not shorten a retained excerpt or change its source ID. Only displayed excerpts
+enter artifacts. Omission notices use leftover space rather than displacing
+complete evidence that fits.
 The global tool-output budget prunes evidence removed by truncation or
 externalization and replaces complete orphan citations with their labels.
 Subagents accumulate at most 100 validated sources across values frames before

backend/packages/harness/deerflow/AGENTS.md

@@ -62,6 +62,10 @@ the SDK sentinel can hide timeouts. See `tests/test_image_search_errors.py`.
 
 Screenshot and session rules: [browser guidance](community/browser_automation/AGENTS.md).
 
+### Knowledge budgets
+
+`knowledge.sources.format_result` skips oversized hits within the first 100.
+
 ### Embedded Client (`packages/harness/deerflow/client.py`)
 
 `DeerFlowClient` embeds the harness without HTTP/FastAPI, sharing Gateway modules, config, data directories, and response schemas.

backend/packages/harness/deerflow/knowledge/sources.py

@@ -91,10 +91,10 @@ def format_result(result: RetrievalResult, *, max_chars_per_chunk: int = 800, ma
         remaining = budget - used_chars - len(separator)
         entry = header + content
         if len(entry) > remaining:
-            # Drop the entire hit when it cannot fit with real evidence. This
-            # keeps the stable source identity independent of total budget.
+            # Skip this hit without changing its excerpt or source identity;
+            # a later, shorter entry may still fit the remaining budget.
             truncated = True
-            break
+            continue
         entries.append(separator + entry)
         used_chars += len(separator) + len(entry)
         if source is not None:

backend/tests/test_knowledge_contract.py

@@ -65,7 +65,7 @@ def test_sources_require_real_evidence_and_optional_locators() -> None:
 
 def test_budget_never_leaves_citations_without_visible_evidence() -> None:
     result = RetrievalResult(hits=[RetrievalHit(content="x" * 500, source=KnowledgeSource(provider="demo", title="Guide", excerpt="x" * 500))])
-    for budget in (1, 20, 80, 200, 800):
+    for budget in (0, 1, 20, 80, 200, 800):
         content, artifact = format_result(result, max_total_chars=budget, max_chars_per_chunk=50)
         assert len(content) <= budget
         for source in artifact["knowledge_sources"]:
@@ -76,6 +76,74 @@ def test_budget_never_leaves_citations_without_visible_evidence() -> None:
             assert artifact["knowledge_sources"]
 
 
+@pytest.mark.parametrize("oversized_index", [0, 1])
+@pytest.mark.parametrize("attributed", [False, True])
+def test_budget_skips_oversized_hits_and_retains_later_evidence(oversized_index, attributed):
+    sources = [KnowledgeSource(provider="demo", title=title, excerpt=excerpt) for title, excerpt in [("First", "First evidence"), ("Second", "Second evidence")]]
+    large = KnowledgeSource(provider="demo", title="Large", excerpt="oversized evidence " * 100)
+    hits = [RetrievalHit(content=source.excerpt, source=source if attributed else None) for source in sources]
+    hits.insert(oversized_index, RetrievalHit(content=large.excerpt, source=large if attributed else None))
+    result = RetrievalResult(hits=hits)
+    snapshot = result.model_dump()
+
+    text, artifact = format_result(result, max_total_chars=400)
+
+    assert len(text) <= 400
+    assert text.index("First evidence") < text.index("Second evidence")
+    assert "oversized evidence" not in text
+    assert text.startswith("[")
+    assert text.endswith("… (response truncated)")
+    assert sources_from_messages([{"type": "tool", "content": text, "artifact": artifact}]) == (sources if attributed else [])
+    assert len(artifact["knowledge_sources"]) == (2 if attributed else 0)
+    assert result.model_dump() == snapshot
+
+
+def test_default_budget_retains_short_source_after_oversized_hit():
+    sources = [KnowledgeSource(provider="demo", title=f"Guide {i}", excerpt=f"Evidence {i}: " + "x" * 788) for i in range(1, 9)]
+    large = KnowledgeSource(provider="demo", title="L" * 500, excerpt="z" * 800)
+    short = KnowledgeSource(provider="demo", title="Short policy", excerpt="Annual leave is 15 days.")
+    result = RetrievalResult(hits=[RetrievalHit(content=source.excerpt, source=source) for source in [*sources, large, short]])
+
+    text, artifact = format_result(result)
+
+    assert len(text) <= 8000
+    assert short.excerpt in text
+    assert text.endswith("… (response truncated)")
+    assert sources_from_messages([{"type": "tool", "content": text, "artifact": artifact}]) == [*sources, short]
+    assert len(artifact["knowledge_sources"]) == 9
+
+
+@pytest.mark.parametrize("oversized_first", [False, True])
+def test_exact_budget_retains_complete_source(oversized_first):
+    source = KnowledgeSource(provider="demo", title="Guide", excerpt="evidence")
+    hits = [RetrievalHit(content=source.excerpt, source=source)]
+    if oversized_first:
+        hits.insert(0, RetrievalHit(content="x" * 800))
+    index = 2 if oversized_first else 1
+    expected = f"[{index}] [citation:Guide](#knowledge-source-{source.id})\nevidence"
+
+    text, artifact = format_result(RetrievalResult(hits=hits), max_total_chars=len(expected))
+
+    assert text == expected
+    assert sources_from_messages([{"type": "tool", "content": text, "artifact": artifact}]) == [source]
+    assert len(artifact["knowledge_sources"]) == 1
+
+
+@pytest.mark.parametrize("short_position", [100, 101])
+def test_skipping_oversized_hits_preserves_first_hundred_limit(short_position):
+    source = KnowledgeSource(provider="demo", title="Short", excerpt="Fitting evidence")
+    hits = [RetrievalHit(content="x" * 800)] * (short_position - 1)
+    hits.append(RetrievalHit(content=source.excerpt, source=source))
+
+    text, artifact = format_result(RetrievalResult(hits=hits), max_total_chars=200)
+
+    assert len(text) <= 200
+    assert (source.excerpt in text) == (short_position == 100)
+    assert text.endswith("… (response truncated)")
+    assert sources_from_messages([{"type": "tool", "content": text, "artifact": artifact}]) == ([source] if short_position == 100 else [])
+    assert len(artifact["knowledge_sources"]) == (1 if short_position == 100 else 0)
+
+
 def test_forward_only_valid_tool_evidence_and_deduplicate_replay() -> None:
     source = KnowledgeSource(provider="demo", title="Guide", excerpt="evidence")
     content, artifact = format_result(RetrievalResult(hits=[RetrievalHit(content="evidence", source=source)]))