diff --git a/README.md b/README.md index 15c6cee522..b4d8198502 100644 --- a/README.md +++ b/README.md @@ -1728,6 +1728,8 @@ The Web UI preserves persisted message order when merging history with live upda Compaction keeps the current user request and summarizes older assistant/tool activity. When rescuing that request leaves an assistant/tool-only summary window, input trimming favors its most recent content. For mixed histories whose user-message anchor falls outside the trimming budget, compaction retains the existing final-message fallback. `summarization.trim_tokens_to_summarize` (4000 by default) controls trimming of the raw summary input; escaping and prompt formatting add overhead beyond that budget. Setting this option to `null` disables input trimming for the summary model; choose that only when the model can accept the full history being compacted. +If the same rendered summary prompt and model candidates recur, DeerFlow reuses the existing summary when the first candidate previously returned it unchanged. An unchanged fallback response is not cached, so reuse cannot prevent the primary model from being retried after an invocation failure. Cache failures fall back to normal generation. See [summarization telemetry](backend/docs/summarization.md#reuse-and-telemetry) for the summary result and reuse counters. + Use `/compact` in the Web UI composer to summarize older context for the current thread. DeerFlow keeps the full chat visible, but future model calls use the compacted summary plus recent messages. The command is ignored when there is not enough history to compact, and it is blocked while the thread has a run in flight, including when that run is owned by another Gateway worker. If a multi-worker reservation loses its lease, DeerFlow cancels the checkpoint writer before the replacing run proceeds and returns a retryable conflict after cleanup. Thread-title edits are serialized through the same state-write boundary and show a conflict without closing the rename dialog when a run is active. The chat header also shows a context-window gauge when the selected model has a positive `context_window` configured. It estimates the latest materialized checkpoint's message tokens and keeps the previous same-thread percentage visible while data refetches, independently of the cumulative token-usage setting. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 2c1070aead..4ce0008179 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -369,7 +369,7 @@ Automatic conversation summarization when approaching token limits: manual state updates). Its short-lived `checkpoint_write` thread operation shares the durable active-thread uniqueness constraint with run admission, preventing either worker-local or cross-worker checkpoint-write races. -- Each summarizer call emits a `middleware:summarize` journal event (`MIDDLEWARE_SUMMARIZE_TAG`; `action="summary_result"`, `changes.noop` true when the output is byte-identical to the prior `summary_text`) and bumps `summary_noop_count` / `summary_call_count` on the middleware. This is observation only; the summarizer call is never skipped. Per-call LLM telemetry lives on `RunJournal` (see the runtime guide). +- Cache only first-candidate no-ops; never suppress primary retries with fallback results. See [reuse and telemetry](docs/summarization.md#reuse-and-telemetry) for cache and counter contracts. See [docs/summarization.md](docs/summarization.md) for details. diff --git a/backend/docs/summarization.md b/backend/docs/summarization.md index 39a50140f4..e328277622 100644 --- a/backend/docs/summarization.md +++ b/backend/docs/summarization.md @@ -14,6 +14,36 @@ The summarization feature uses LangChain's `SummarizationMiddleware` to monitor 4. Maintains AI/Tool message pairs together for context continuity 5. Stores the summary in `ThreadState.summary_text` and projects it ephemerally through durable context data +## Reuse and telemetry + +The middleware keeps a bounded cache of exact rendered summary prompts and ordered +model candidate names whose first candidate returned the existing `summary_text` +unchanged. A repeated input reuses that summary through the normal compaction path +only if the current summary still matches the cached output. Responses from fallback +candidates are never cached, so reuse cannot suppress retries after a primary +invocation failure. Changed inputs, generation failures, and cache errors follow normal +generation. This optimization assumes a successful first candidate would return the +same result for identical input; stochastic model outputs can vary. + +Each successful summary result emits a `middleware:summarize` journal event with +`action="summary_result"`. Its `changes` include: + +- `noop`: the result is byte-identical to the previous summary. +- `llm_call_skipped`: this result came from the no-op cache. +- `call_count`: completed summary results, including cache reuse and canned results. +- `noop_count`: completed results identical to the previous summary. +- `skip_count`: completed results reused from the no-op cache. + +The middleware's corresponding `summary_call_count`, `summary_noop_count`, and +`summary_skip_count` counters start at zero and are updated together under a lock. +Event counter snapshots include their own result and remain consistent during +concurrent compactions. Counters are cumulative across runs sharing that middleware +instance; concurrent journal events may arrive in a different order. + +`call_count` is not an LLM request count: a compaction can try multiple candidates +or produce a canned result without invoking a model. Use per-call LLM journal events +for model usage and cost, and `skip_count / call_count` for the cache reuse rate. + ## Todo reminders Compaction filters `HumanMessage(name="todo_reminder")` snapshots after the diff --git a/backend/packages/harness/deerflow/agents/middlewares/summarization_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/summarization_middleware.py index a89bdfdc88..3c0f20be0b 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/summarization_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/summarization_middleware.py @@ -2,8 +2,11 @@ from __future__ import annotations +import hashlib import html import logging +import threading +from collections import OrderedDict from collections.abc import Mapping from dataclasses import dataclass from typing import Any, Literal, Protocol, override, runtime_checkable @@ -34,6 +37,13 @@ _SUMMARY_TRIGGER_MESSAGE_NAME = "summary" _COMPACTION_TRANSFORM_KIND = "summarization" _COMPACTION_TRANSFORM_VERSION = "1" _UNSET = object() +_NOOP_CACHE_MAX_ENTRIES = 128 + + +class _ReusedSummary(str): + """Marks a summary reused from an identical, previously no-op prompt (no LLM call).""" + + # Valid non-generated summaries for the empty / too-long-to-summarize edges; these # short-circuit model invocation (and must not be treated as generation failures). _CANNED_SUMMARIES = frozenset( @@ -166,6 +176,11 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): self._extensions = extensions self.summary_call_count = 0 self.summary_noop_count = 0 + self.summary_skip_count = 0 + self._summary_counters_lock = threading.Lock() + # P1: exact prompt fingerprints whose summarizer output equalled the existing summary. + self._noop_prompt_cache: OrderedDict[str, str] = OrderedDict() + self._noop_prompt_cache_lock = threading.Lock() # Nostream generation models are either prebuilt by the factory or built # lazily by name. An eagerly seeded ``None`` pins the run-model fallback for # this middleware instance; unlike the legacy path, it does not allow one @@ -315,9 +330,16 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): # and any failure at any stage falls through to the next candidate. When all # candidates fail the caller leaves compaction state unchanged. names = self._generation_candidate_names() + noop_key = self._noop_cache_key(prompt, names) + reused = self._reuse_noop_summary(noop_key, previous_summary) + if reused is not None: + return reused for index, name in enumerate(names): text = self._invoke_summary(self._model_for(name), prompt, last=index == len(names) - 1) if text is not None: + # A fallback no-op must not suppress retrying a failed earlier candidate. + if index == 0: + self._remember_noop_summary(noop_key, text, previous_summary) return text return None @@ -333,6 +355,10 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): if prompt is None or prompt in _CANNED_SUMMARIES: return prompt names = self._generation_candidate_names() + noop_key = self._noop_cache_key(prompt, names) + reused = self._reuse_noop_summary(noop_key, previous_summary) + if reused is not None: + return reused for index, name in enumerate(names): text = await self._ainvoke_summary( self._model_for(name), @@ -342,9 +368,55 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): task_store=task_store, ) if text is not None: + if index == 0: + self._remember_noop_summary(noop_key, text, previous_summary) return text return None + @staticmethod + def _noop_cache_key(prompt: str, candidate_names: list[str | None]) -> str | None: + """Fingerprint of the exact summarizer input: the rendered prompt plus the candidate models.""" + try: + payload = "|".join([*(repr(name) for name in candidate_names), prompt]) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + except Exception: + logger.warning("Summarization no-op cache key failed", exc_info=True) + return None + + def _reuse_noop_summary(self, key: str | None, previous_summary: str | None) -> str | None: + """Return the existing summary instead of calling the LLM for a proven no-op input. + + Only an input whose first candidate produced output byte-identical to + ``previous_summary`` is cached, and reuse additionally requires today's + ``previous_summary`` to equal that cached output, so the reused value is exactly + the existing ``summary_text``. Any failure falls through to the normal LLM call. + """ + if key is None or not previous_summary: + return None + try: + with self._noop_prompt_cache_lock: + cached = self._noop_prompt_cache.get(key) + if cached is None or cached != previous_summary: + return None + self._noop_prompt_cache.move_to_end(key) + logger.info("Summarization skipped: identical input previously produced an unchanged summary") + return _ReusedSummary(cached) + except Exception: + logger.warning("Summarization no-op cache lookup failed", exc_info=True) + return None + + def _remember_noop_summary(self, key: str | None, summary: str, previous_summary: str | None) -> None: + if key is None or not previous_summary or summary != previous_summary: + return + try: + with self._noop_prompt_cache_lock: + self._noop_prompt_cache[key] = summary + self._noop_prompt_cache.move_to_end(key) + while len(self._noop_prompt_cache) > _NOOP_CACHE_MAX_ENTRIES: + self._noop_prompt_cache.popitem(last=False) + except Exception: + logger.warning("Summarization no-op cache store failed", exc_info=True) + def _invoke_summary(self, model: Any | None, prompt: str, *, last: bool = False) -> str | None: """Invoke ``model`` for a summary; ``None`` on error or a blank response. @@ -661,17 +733,23 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): ) notify_context_compacted(event, extensions=self._extensions) - def _record_summary_telemetry(self, runtime: Runtime, previous_summary: str | None, summary: str, summarized_count: int) -> None: - """P0 measurement only: count summarizer calls whose output equals the prior ``summary_text``. + def _record_summary_telemetry(self, runtime: Runtime, previous_summary: str | None, summary: str, summarized_count: int, *, llm_call_skipped: bool = False) -> None: + """Count successful summary results, unchanged results, and no-op cache reuse. - Never skips or alters the summarizer call; fails soft. + ``call_count`` includes reuse and canned results, not individual LLM requests. + Counters and event snapshots update atomically; telemetry failures stay soft. """ try: noop = previous_summary is not None and summary.encode("utf-8") == previous_summary.encode("utf-8") - self.summary_call_count = self.summary_call_count + 1 + with self._summary_counters_lock: + self.summary_call_count += 1 + self.summary_noop_count += int(noop) + self.summary_skip_count += int(llm_call_skipped) + call_count = self.summary_call_count + noop_count = self.summary_noop_count + skip_count = self.summary_skip_count if noop: - self.summary_noop_count = self.summary_noop_count + 1 - logger.info("Summarization no-op: summarizer output identical to existing summary_text (%d chars)", len(summary)) + logger.info("Summarization no-op: result identical to existing summary_text (%d chars)", len(summary)) context = getattr(runtime, "context", None) journal = context.get("__run_journal") if isinstance(context, dict) else None if journal is None: @@ -683,11 +761,13 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): action="summary_result", changes={ "noop": noop, + "llm_call_skipped": llm_call_skipped, "summary_chars": len(summary), "previous_summary_chars": len(previous_summary) if previous_summary is not None else None, "summarized_message_count": summarized_count, - "noop_count": self.summary_noop_count, - "call_count": self.summary_call_count, + "noop_count": noop_count, + "call_count": call_count, + "skip_count": skip_count, }, ) except Exception: @@ -719,7 +799,9 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): if raise_on_failure: raise SummaryGenerationError("summary generation failed") return None - self._record_summary_telemetry(runtime, previous_summary, summary, len(messages_to_summarize)) + llm_call_skipped = isinstance(summary, _ReusedSummary) + summary = str(summary) + self._record_summary_telemetry(runtime, previous_summary, summary, len(messages_to_summarize), llm_call_skipped=llm_call_skipped) # Fire hooks only once a replacement summary exists — flushing pre-compaction # messages into durable memory for a summary that never materializes would # duplicate that work on the next attempt. Messages are still removed after @@ -769,7 +851,9 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware): if raise_on_failure: raise SummaryGenerationError("summary generation failed") return None - self._record_summary_telemetry(runtime, previous_summary, summary, len(messages_to_summarize)) + llm_call_skipped = isinstance(summary, _ReusedSummary) + summary = str(summary) + self._record_summary_telemetry(runtime, previous_summary, summary, len(messages_to_summarize), llm_call_skipped=llm_call_skipped) # Fire hooks only once a replacement summary exists (see compact_state). self._fire_hooks(messages_to_summarize, preserved_messages, runtime) self._record_compaction( diff --git a/backend/tests/test_summarization_noop_skip.py b/backend/tests/test_summarization_noop_skip.py new file mode 100644 index 0000000000..880c62fa9a --- /dev/null +++ b/backend/tests/test_summarization_noop_skip.py @@ -0,0 +1,241 @@ +"""P1: skip the summarizer LLM call when the exact input already produced an unchanged summary.""" + +from __future__ import annotations + +import copy +import threading +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from langchain_core.messages import AIMessage, HumanMessage + +from deerflow.agents.middlewares.summarization_middleware import DeerFlowSummarizationMiddleware +from deerflow.runtime.events.store.memory import MemoryRunEventStore +from deerflow.runtime.journal import RunJournal + + +def _model(output: str = "S"): + model = MagicMock() + model.invoke.return_value = SimpleNamespace(text=output) + model.ainvoke = AsyncMock(return_value=SimpleNamespace(text=output)) + model.with_config.return_value = model + return model + + +def _middleware(output: str = "S", **kwargs) -> DeerFlowSummarizationMiddleware: + return DeerFlowSummarizationMiddleware(model=_model(output), trigger=("messages", 4), keep=("messages", 2), token_counter=len, **kwargs) + + +def _state(summary: str | None = "S", first: str = "u1") -> dict: + messages = [ + HumanMessage(content=first, id="m1"), + AIMessage(content="a1", id="m2"), + HumanMessage(content="u2", id="m3"), + AIMessage(content="a2", id="m4"), + ] + return {"messages": messages, "summary_text": summary} + + +def _runtime(journal=None) -> SimpleNamespace: + context = {"thread_id": "t1"} + if journal is not None: + context["__run_journal"] = journal + return SimpleNamespace(context=context) + + +def _calls(mw) -> int: + return mw.model.invoke.call_count + mw.model.ainvoke.await_count + + +def _ids(result) -> tuple: + return (result.summary_text, [m.id for m in result.messages_to_summarize], [m.id for m in result.preserved_messages]) + + +def test_summary_counters_start_at_zero(): + mw = _middleware() + assert (mw.summary_call_count, mw.summary_noop_count, mw.summary_skip_count) == (0, 0, 0) + + +async def _compact(mw, state, runtime, asynchronous: bool): + if asynchronous: + return await mw.acompact_state(state, runtime, force=True) + return mw.compact_state(state, runtime, force=True) + + +@pytest.mark.anyio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_unchanged_window_skips_llm_and_preserves_state(asynchronous): + mw = _middleware("S") + state = _state("S") + snapshot = copy.deepcopy(state) + + first = await _compact(mw, state, _runtime(), asynchronous) + assert _calls(mw) == 1 # first time the input is seen: LLM is called, proves no-op + second = await _compact(mw, state, _runtime(), asynchronous) + + assert _calls(mw) == 1 # skipped + assert second.summary_text == "S" and type(second.summary_text) is str + assert _ids(second) == _ids(first) # same compaction outcome as the LLM path + assert state["summary_text"] == snapshot["summary_text"] + assert [m.id for m in state["messages"]] == [m.id for m in snapshot["messages"]] # input untouched + assert mw.summary_skip_count == 1 + + +@pytest.mark.anyio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_changed_window_still_calls_llm(asynchronous): + mw = _middleware("S") + await _compact(mw, _state("S"), _runtime(), asynchronous) + assert _calls(mw) == 1 + + await _compact(mw, _state("S", first="different window"), _runtime(), asynchronous) + assert _calls(mw) == 2 # new messages -> new prompt + + await _compact(mw, _state("older summary"), _runtime(), asynchronous) + assert _calls(mw) == 3 # new previous summary -> new prompt + + +@pytest.mark.anyio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_non_noop_result_is_never_cached(asynchronous): + mw = _middleware("brand new summary") + await _compact(mw, _state("S"), _runtime(), asynchronous) + await _compact(mw, _state("S"), _runtime(), asynchronous) + assert _calls(mw) == 2 + assert mw.summary_skip_count == 0 + + +@pytest.mark.anyio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("failure", ["exception", "blank"]) +async def test_fallback_noop_does_not_hide_primary_recovery(asynchronous, failure): + fallback = _model("S") + mw = _middleware(configured_model_name="summary-model", run_model_name="run-model", prebuilt_models={"run-model": fallback}) + first_response = RuntimeError("primary temporarily unavailable") if failure == "exception" else SimpleNamespace(text=" \n ") + mw.model.invoke.side_effect = [first_response, SimpleNamespace(text="recovered primary summary")] + mw.model.ainvoke.side_effect = [first_response, SimpleNamespace(text="recovered primary summary")] + state = _state("S") + snapshot = copy.deepcopy(state) + + first = await _compact(mw, state, _runtime(), asynchronous) + second = await _compact(mw, state, _runtime(), asynchronous) + + assert first.summary_text == "S" + assert second.summary_text == "recovered primary summary" + assert _calls(mw) == 2 + assert fallback.invoke.call_count + fallback.ainvoke.await_count == 1 + assert _ids(first)[1:] == _ids(second)[1:] + assert state == snapshot + assert mw.summary_skip_count == 0 + + +@pytest.mark.anyio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_primary_noop_with_fallback_configured_still_skips(asynchronous): + fallback = _model("fallback summary") + mw = _middleware(configured_model_name="summary-model", run_model_name="run-model", prebuilt_models={"run-model": fallback}) + + first = await _compact(mw, _state("S"), _runtime(), asynchronous) + second = await _compact(mw, _state("S"), _runtime(), asynchronous) + + assert _ids(first) == _ids(second) + assert _calls(mw) == 1 + fallback.invoke.assert_not_called() + fallback.ainvoke.assert_not_awaited() + assert mw.summary_skip_count == 1 + + +@pytest.mark.anyio +async def test_sync_and_async_equivalent(): + results = [] + for asynchronous in (False, True): + mw = _middleware("S") + state = _state("S") + await _compact(mw, state, _runtime(), asynchronous) + results.append((_ids(await _compact(mw, state, _runtime(), asynchronous)), _calls(mw))) + assert results[0] == results[1] + + +@pytest.mark.anyio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_skip_is_visible_in_p0_telemetry(asynchronous): + store = MemoryRunEventStore() + journal = RunJournal("r1", "t1", store, flush_threshold=100) + mw = _middleware("S") + await _compact(mw, _state("S"), _runtime(journal), asynchronous) + await _compact(mw, _state("S"), _runtime(journal), asynchronous) + await journal.flush() + + events = [e for e in await store.list_events("t1", "r1") if e["event_type"] == "middleware:summarize"] + changes = [e["content"]["changes"] for e in events] + assert [c["llm_call_skipped"] for c in changes] == [False, True] + assert [c["noop"] for c in changes] == [True, True] # P0 fields intact + assert [c["call_count"] for c in changes] == [1, 2] + assert [c["skip_count"] for c in changes] == [0, 1] + assert [c["noop_count"] for c in changes] == [1, 2] + assert mw.summary_noop_count == 2 and mw.summary_call_count == 2 + assert mw.summary_skip_count == 1 + + +def test_concurrent_compactions_keep_consistent_counter_snapshots(): + first_waiting = threading.Event() + release_first = threading.Event() + journal = MagicMock() + + class PausedContext(dict): + def get(self, key, default=None): + if key == "__run_journal": + first_waiting.set() + assert release_first.wait(timeout=5) + return super().get(key, default) + + mw = _middleware("S") + runtime = SimpleNamespace(context=PausedContext(thread_id="t1", __run_journal=journal)) + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(mw.compact_state, _state("S"), runtime, force=True) + try: + assert first_waiting.wait(timeout=5) + second = executor.submit(mw.compact_state, _state("S"), _runtime(journal), force=True) + assert second.result(timeout=5) is not None + finally: + release_first.set() + assert first.result(timeout=5) is not None + + changes = [call.kwargs["changes"] for call in journal.record_middleware.call_args_list] + assert sorted((c["call_count"], c["skip_count"], c["noop_count"]) for c in changes) == [(1, 0, 1), (2, 1, 2)] + assert (mw.summary_call_count, mw.summary_skip_count, mw.summary_noop_count) == (2, 1, 2) + assert _calls(mw) == 1 + + +def test_cache_failure_falls_back_to_llm(monkeypatch): + mw = _middleware("S") + mw.compact_state(_state("S"), _runtime(), force=True) + + cache_lock = mw._noop_prompt_cache_lock + broken_lock = MagicMock() + broken_lock.__enter__.side_effect = RuntimeError("cache broken") + monkeypatch.setattr(mw, "_noop_prompt_cache_lock", broken_lock) + result = mw.compact_state(_state("S"), _runtime(), force=True) + assert result is not None and result.summary_text == "S" + assert _calls(mw) == 2 # lookup failed -> normal LLM path + assert broken_lock.__enter__.call_count == 2 # lookup and store both exercise the injected exception + broken_lock.__exit__.assert_not_called() + + monkeypatch.setattr(mw, "_noop_prompt_cache_lock", cache_lock) + broken_hash = MagicMock(side_effect=RuntimeError("cache key broken")) + monkeypatch.setattr("deerflow.agents.middlewares.summarization_middleware.hashlib", SimpleNamespace(sha256=broken_hash)) + assert mw.compact_state(_state("S"), _runtime(), force=True).summary_text == "S" + assert _calls(mw) == 3 + broken_hash.assert_called_once() + + +def test_telemetry_failure_does_not_change_skip_behavior(): + journal = MagicMock() + journal.record_middleware.side_effect = RuntimeError("journal down") + mw = _middleware("S") + first = mw.compact_state(_state("S"), _runtime(journal), force=True) + second = mw.compact_state(_state("S"), _runtime(journal), force=True) + assert _calls(mw) == 1 + assert _ids(second) == _ids(first)