diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 8bb4f56fe..207dfd7d8 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -559,7 +559,8 @@ The cached value is reused for both the blocking (`runs.wait`) and streaming (`_ 2. Queue debounces (30s default), batches updates, deduplicates per-thread 3. Background thread invokes LLM to extract context updates and facts, using the stored `user_id` (not the contextvar, which is unavailable on timer threads) 4. Applies updates atomically (temp file + rename) with cache invalidation, skipping duplicate fact content before append -5. Next interaction injects top 15 facts + context into `` tags in system prompt +5. **Staleness pass** (same LLM invocation as step 3, no extra API call): when `staleness_review_enabled` is `true` and at least `staleness_min_candidates` aged facts exist, `_select_stale_candidates` selects facts older than `staleness_age_days` that are not in `staleness_protected_categories` (default: `correction`), surfaces them in the prompt, and the LLM judges each as KEEP or REMOVE. `_apply_updates` enforces the guardrail unconditionally at apply time: it intersects the LLM-returned removal set with `_select_stale_candidates` output before applying the per-cycle cap (`staleness_max_removals_per_cycle`), so protected and non-aged facts can never be deleted regardless of model behavior or the feature flag setting. +6. Next interaction injects top 15 facts + context into `` tags in system prompt **Token counting** (`packages/harness/deerflow/agents/memory/prompt.py`): - `_count_tokens` budgets the injection. In default `tiktoken` mode, the encoding is loaded lazily and cached. @@ -577,6 +578,11 @@ Focused regression coverage for the updater lives in `backend/tests/test_memory_ - `max_facts` / `fact_confidence_threshold` - Fact storage limits (100 / 0.7) - `max_injection_tokens` - Token limit for prompt injection (2000) - `token_counting` - Token counting strategy for the injection budget: `tiktoken` (default, accurate but may download BPE data from a public endpoint on first use — can block for a long time in network-restricted environments, see issues #3402/#3429) or `char` (network-free CJK-aware char estimate, never touches tiktoken) +- `staleness_review_enabled` - Enable proactive staleness pruning of aged facts (default: `true`; only triggers when aged candidates exist) +- `staleness_age_days` - Age in days before a fact becomes a staleness candidate (default: 180; range: 1–3650) +- `staleness_min_candidates` - Minimum aged candidates required to trigger a review cycle (default: 3; range: 1–50) +- `staleness_max_removals_per_cycle` - Maximum facts removed in a single cycle; lowest-confidence entries are kept when the LLM requests more (default: 5; range: 1–20) +- `staleness_protected_categories` - Fact categories that are never pruned by staleness review (default: `["correction"]`) ### Reflection System (`packages/harness/deerflow/reflection/`) @@ -694,7 +700,7 @@ Returns `{}` when Langfuse is not in the enabled providers — LangSmith-only de - `title` - Auto-title generation (enabled, max_words, max_chars, model_name; null model_name uses fast local fallback, explicit model_name uses the prompt_template LLM path) - `summarization` - Context summarization (enabled, trigger conditions, keep policy) - `subagents.enabled` - Master switch for subagent delegation -- `memory` - Memory system (enabled, storage_path, debounce_seconds, model_name, max_facts, fact_confidence_threshold, injection_enabled, max_injection_tokens) +- `memory` - Memory system (enabled, storage_path, debounce_seconds, model_name, max_facts, fact_confidence_threshold, injection_enabled, max_injection_tokens, staleness_review_enabled, staleness_age_days, staleness_min_candidates, staleness_max_removals_per_cycle, staleness_protected_categories) **`extensions_config.json`**: - `mcpServers` - Map of server name → config (enabled, type, command, args, env, url, headers, oauth, description) diff --git a/backend/app/gateway/routers/memory.py b/backend/app/gateway/routers/memory.py index 6d1c356c8..75f41934b 100644 --- a/backend/app/gateway/routers/memory.py +++ b/backend/app/gateway/routers/memory.py @@ -131,6 +131,11 @@ class MemoryConfigResponse(BaseModel): ..., description="Token ceiling for guaranteed-category facts (displaces regular lines in the common case; additive only when guaranteed alone overflows max_injection_tokens)", ) + staleness_review_enabled: bool = Field(..., description="Whether staleness review is enabled for aged facts") + staleness_age_days: int = Field(..., description="Facts older than this many days are candidates for staleness review") + staleness_min_candidates: int = Field(..., description="Minimum stale facts required to trigger a review cycle") + staleness_max_removals_per_cycle: int = Field(..., description="Maximum number of facts staleness review can remove per cycle") + staleness_protected_categories: list[str] = Field(..., description="Fact categories exempt from staleness review") class MemoryStatusResponse(BaseModel): @@ -360,6 +365,11 @@ async def get_memory_config_endpoint() -> MemoryConfigResponse: token_counting=config.token_counting, guaranteed_categories=config.guaranteed_categories, guaranteed_token_budget=config.guaranteed_token_budget, + staleness_review_enabled=config.staleness_review_enabled, + staleness_age_days=config.staleness_age_days, + staleness_min_candidates=config.staleness_min_candidates, + staleness_max_removals_per_cycle=config.staleness_max_removals_per_cycle, + staleness_protected_categories=config.staleness_protected_categories, ) @@ -391,6 +401,11 @@ async def get_memory_status(http_request: Request) -> MemoryStatusResponse: token_counting=config.token_counting, guaranteed_categories=config.guaranteed_categories, guaranteed_token_budget=config.guaranteed_token_budget, + staleness_review_enabled=config.staleness_review_enabled, + staleness_age_days=config.staleness_age_days, + staleness_min_candidates=config.staleness_min_candidates, + staleness_max_removals_per_cycle=config.staleness_max_removals_per_cycle, + staleness_protected_categories=config.staleness_protected_categories, ), data=MemoryResponse(**memory_data), ) diff --git a/backend/packages/harness/deerflow/agents/memory/prompt.py b/backend/packages/harness/deerflow/agents/memory/prompt.py index ac5620236..9fc8ae2bc 100644 --- a/backend/packages/harness/deerflow/agents/memory/prompt.py +++ b/backend/packages/harness/deerflow/agents/memory/prompt.py @@ -115,7 +115,8 @@ Output Format (JSON): "newFacts": [ {{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }} ], - "factsToRemove": ["fact_id_1", "fact_id_2"] + "factsToRemove": ["fact_id_1", "fact_id_2"], + "staleFactsToRemove": [{{ "id": "fact_id", "reason": "brief explanation" }}] }} Important Rules: @@ -135,9 +136,40 @@ Important Rules: session-specific and ephemeral — they will not be accessible in future sessions. Recording upload events causes confusion in subsequent conversations. +{staleness_review_section} + Return ONLY valid JSON, no explanation or markdown.""" +# Prompt section injected into MEMORY_UPDATE_PROMPT when staleness review triggers. +# Surfaces aged facts explicitly so the LLM can semantically judge each one, +# rather than relying on passive contradiction from the current conversation. +STALENESS_REVIEW_PROMPT = """## Staleness Review + +The following facts were created more than {age_days} days ago and may no longer +accurately reflect the user's current situation. Review each one against the full +conversation context and your understanding of the user. + + +{stale_facts} + + +For each fact, decide KEEP or REMOVE: +- KEEP: Still likely valid — even if not mentioned in this conversation. + Stable attributes (native language, core expertise, personality traits) often + remain true indefinitely. +- REMOVE: Outdated, contradicted by recent context, or no longer relevant. + Examples: tech-stack migrations, job changes, relocated offices, abandoned projects. + +Add REMOVE decisions to "staleFactsToRemove" in your output JSON. +Each entry must be {{"id": "fact_id", "reason": "brief explanation"}}. +The reason should cite what signal in the conversation (or absence thereof) +supports the removal. + +Be conservative — when in doubt, KEEP. Removing a valid fact is worse than +keeping a slightly stale one, because the next review cycle will re-evaluate it.""" + + # Prompt template for extracting facts from a single message FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message. diff --git a/backend/packages/harness/deerflow/agents/memory/updater.py b/backend/packages/harness/deerflow/agents/memory/updater.py index 1391c7190..c53cc3c1c 100644 --- a/backend/packages/harness/deerflow/agents/memory/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/updater.py @@ -11,10 +11,12 @@ import os import re import uuid from contextlib import nullcontext +from datetime import UTC, datetime, timedelta from typing import Any from deerflow.agents.memory.prompt import ( MEMORY_UPDATE_PROMPT, + STALENESS_REVIEW_PROMPT, format_conversation_for_update, ) from deerflow.agents.memory.storage import ( @@ -301,11 +303,30 @@ def _normalize_memory_update_data(update_data: dict[str, Any]) -> dict[str, Any] 0, ) + # ── Normalize staleness review removals ── + stale_removals_raw = update_data.get("staleFactsToRemove") + normalized_stale_removals: list[dict[str, str]] = [] + if isinstance(stale_removals_raw, list): + for entry in stale_removals_raw: + if not isinstance(entry, dict): + continue + fact_id = entry.get("id") + if not isinstance(fact_id, str) or not fact_id: + continue + reason = entry.get("reason", "") + normalized_stale_removals.append( + { + "id": fact_id, + "reason": reason if isinstance(reason, str) else "", + } + ) + return { "user": user if isinstance(user, dict) else {}, "history": history if isinstance(history, dict) else {}, "newFacts": normalized_new_facts, "factsToRemove": normalized_facts_to_remove, + "staleFactsToRemove": normalized_stale_removals, } @@ -376,6 +397,73 @@ def _fact_content_key(content: Any) -> str | None: return stripped.casefold() +# ── Staleness review helpers ────────────────────────────────────────────── + + +def _parse_fact_datetime(raw: str) -> datetime | None: + """Parse an ISO-8601 datetime string from a fact's createdAt field. + + Returns ``None`` on any parse failure so callers can safely skip malformed facts. + """ + if not raw: + return None + try: + result = datetime.fromisoformat(raw) + # Naive datetimes (no tzinfo) would cause TypeError when compared + # with the timezone-aware cutoff. Assume UTC for safety. + if result.tzinfo is None: + result = result.replace(tzinfo=UTC) + return result + except (ValueError, TypeError): + return None + + +def _select_stale_candidates( + current_memory: dict[str, Any], + config: Any, +) -> list[dict[str, Any]]: + """Return facts that are older than ``staleness_age_days`` and not protected. + + Protected categories (default: ``correction``) are excluded because they + represent explicit user feedback that should not be auto-pruned by age. + """ + cutoff = datetime.now(UTC) - timedelta(days=config.staleness_age_days) + protected = frozenset(config.staleness_protected_categories) + candidates: list[dict[str, Any]] = [] + for fact in current_memory.get("facts", []): + if not isinstance(fact, dict): + continue + category = fact.get("category", "") + if isinstance(category, str) and category in protected: + continue + created_at = _parse_fact_datetime(fact.get("createdAt", "")) + if created_at is not None and created_at < cutoff: + candidates.append(fact) + return candidates + + +def _build_staleness_section( + stale_candidates: list[dict[str, Any]], + age_days: int, +) -> str: + """Format the staleness review prompt section from candidate facts.""" + if not stale_candidates: + return "" + lines: list[str] = [] + for fact in stale_candidates: + fid = fact.get("id", "?") + cat = str(fact.get("category", "context")).strip() or "context" + conf = fact.get("confidence", 0.0) + created_raw = fact.get("createdAt", "") + created_short = created_raw[:10] if isinstance(created_raw, str) and len(created_raw) >= 10 else created_raw + content = str(fact.get("content", "")) + lines.append(f'- [{fid} | {cat} | {conf:.2f} | {created_short}] "{content}"') + return STALENESS_REVIEW_PROMPT.format( + stale_facts="\n".join(lines), + age_days=age_days, + ) + + class MemoryUpdater: """Updates memory using LLM based on conversation context.""" @@ -443,10 +531,22 @@ class MemoryUpdater: correction_detected=correction_detected, reinforcement_detected=reinforcement_detected, ) + + # ── Build staleness review section ── + staleness_section = "" + if config.staleness_review_enabled: + stale_candidates = _select_stale_candidates(current_memory, config) + if len(stale_candidates) >= config.staleness_min_candidates: + staleness_section = _build_staleness_section( + stale_candidates, + config.staleness_age_days, + ) + prompt = MEMORY_UPDATE_PROMPT.format( current_memory=json.dumps(current_memory, indent=2, ensure_ascii=False), conversation=conversation_text, correction_hint=correction_hint, + staleness_review_section=staleness_section, ) return current_memory, prompt @@ -664,11 +764,49 @@ class MemoryUpdater: "updatedAt": now, } - # Remove facts + # Remove facts (contradiction-based) facts_to_remove = set(update_data.get("factsToRemove", [])) if facts_to_remove: current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove] + # ── Staleness review removals ── + stale_removals = update_data.get("staleFactsToRemove", []) + if isinstance(stale_removals, list) and stale_removals: + stale_ids_to_remove = {entry["id"] for entry in stale_removals if isinstance(entry, dict) and "id" in entry} + + # Deterministic guardrail: intersect with actual staleness + # candidates so an LLM slip that emits a protected-category or + # non-aged fact id is silently rejected. Runs unconditionally + # so the apply-layer protection is independent of model behavior + # AND of the staleness_review_enabled flag. + candidate_ids = {f["id"] for f in _select_stale_candidates(current_memory, config)} + stale_ids_to_remove &= candidate_ids + + if not stale_ids_to_remove: + # After intersection with candidate set, nothing to remove. + stale_removals = [] + else: + # Safety cap: limit max staleness removals per cycle. + # When the LLM returns more than the cap, keep only the + # lowest-confidence entries up to the limit so the most + # questionable facts are removed first. + max_stale = config.staleness_max_removals_per_cycle + if len(stale_ids_to_remove) > max_stale: + stale_facts = [f for f in current_memory.get("facts", []) if f.get("id") in stale_ids_to_remove] + stale_facts.sort(key=lambda f: f.get("confidence", 0)) + stale_ids_to_remove = {f["id"] for f in stale_facts[:max_stale]} + + current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in stale_ids_to_remove] + + # Log removals for observability + for entry in stale_removals: + if isinstance(entry, dict) and entry.get("id") in stale_ids_to_remove: + logger.info( + "Staleness review removed fact %s: %s", + entry["id"], + entry.get("reason", "no reason provided"), + ) + # Add new facts existing_fact_keys = {fact_key for fact_key in (_fact_content_key(fact.get("content")) for fact in current_memory.get("facts", [])) if fact_key is not None} new_facts = update_data.get("newFacts", []) diff --git a/backend/packages/harness/deerflow/client.py b/backend/packages/harness/deerflow/client.py index 70d704173..7fd13a67e 100644 --- a/backend/packages/harness/deerflow/client.py +++ b/backend/packages/harness/deerflow/client.py @@ -1319,6 +1319,11 @@ class DeerFlowClient: "token_counting": config.token_counting, "guaranteed_categories": config.guaranteed_categories, "guaranteed_token_budget": config.guaranteed_token_budget, + "staleness_review_enabled": config.staleness_review_enabled, + "staleness_age_days": config.staleness_age_days, + "staleness_min_candidates": config.staleness_min_candidates, + "staleness_max_removals_per_cycle": config.staleness_max_removals_per_cycle, + "staleness_protected_categories": config.staleness_protected_categories, } def get_memory_status(self) -> dict: diff --git a/backend/packages/harness/deerflow/config/memory_config.py b/backend/packages/harness/deerflow/config/memory_config.py index f14e777e9..867f1e6e1 100644 --- a/backend/packages/harness/deerflow/config/memory_config.py +++ b/backend/packages/harness/deerflow/config/memory_config.py @@ -98,6 +98,40 @@ class MemoryConfig(BaseModel): "safety-truncation ceiling is raised accordingly." ), ) + # ── Staleness review ──────────────────────────────────────────────── + staleness_review_enabled: bool = Field( + default=True, + description=( + "Enable staleness review for aged facts. When enabled, facts older " + "than ``staleness_age_days`` are surfaced in the memory-update prompt " + "so the LLM can semantically judge whether each is still valid or " + "should be removed. This solves the 'silent staleness' problem where " + "outdated facts persist because no future conversation explicitly " + "contradicts them." + ), + ) + staleness_age_days: int = Field( + default=90, + ge=30, + le=365, + description=("Facts older than this many days become candidates for staleness review. 90 days (~one quarter) balances between catching genuine changes (job switches, tech-stack migrations) and avoiding noise on stable facts."), + ) + staleness_min_candidates: int = Field( + default=3, + ge=1, + le=50, + description=("Minimum number of stale facts required to trigger a review cycle. Below this threshold the prompt overhead is not justified."), + ) + staleness_max_removals_per_cycle: int = Field( + default=10, + ge=1, + le=50, + description=("Maximum number of facts the staleness review can remove in a single update cycle. Prevents the LLM from over-pruning when reviewing a large backlog of aged facts."), + ) + staleness_protected_categories: list[str] = Field( + default_factory=lambda: ["correction"], + description=("Fact categories exempt from staleness review. Correction facts represent explicit user feedback and should not be auto-pruned based on age alone."), + ) # Global configuration instance diff --git a/backend/tests/test_client.py b/backend/tests/test_client.py index 0be07e78b..823baaaab 100644 --- a/backend/tests/test_client.py +++ b/backend/tests/test_client.py @@ -2617,6 +2617,11 @@ class TestGatewayConformance: mem_cfg.injection_enabled = True mem_cfg.max_injection_tokens = 2000 mem_cfg.token_counting = "tiktoken" + mem_cfg.staleness_review_enabled = True + mem_cfg.staleness_age_days = 90 + mem_cfg.staleness_min_candidates = 3 + mem_cfg.staleness_max_removals_per_cycle = 10 + mem_cfg.staleness_protected_categories = ["correction"] with patch("deerflow.config.memory_config.get_memory_config", return_value=mem_cfg): result = client.get_memory_config() @@ -2636,6 +2641,11 @@ class TestGatewayConformance: mem_cfg.injection_enabled = True mem_cfg.max_injection_tokens = 2000 mem_cfg.token_counting = "tiktoken" + mem_cfg.staleness_review_enabled = True + mem_cfg.staleness_age_days = 90 + mem_cfg.staleness_min_candidates = 3 + mem_cfg.staleness_max_removals_per_cycle = 10 + mem_cfg.staleness_protected_categories = ["correction"] memory_data = { "version": "1.0", diff --git a/backend/tests/test_memory_staleness_review.py b/backend/tests/test_memory_staleness_review.py new file mode 100644 index 000000000..6ae032c0b --- /dev/null +++ b/backend/tests/test_memory_staleness_review.py @@ -0,0 +1,651 @@ +"""Tests for the staleness review feature in the memory updater. + +Covers: +- Candidate selection (age threshold, protected categories) +- Trigger conditions (min candidates, enabled flag) +- Prompt section formatting +- Staleness removal in _apply_updates (safety cap, observability) +- Normalization of staleFactsToRemove from LLM responses +- Integration with _prepare_update_prompt +""" + +from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock, patch + +from deerflow.agents.memory.updater import ( + MemoryUpdater, + _build_staleness_section, + _normalize_memory_update_data, + _parse_fact_datetime, + _select_stale_candidates, +) +from deerflow.config.memory_config import MemoryConfig + +# ── Helpers ──────────────────────────────────────────────────────────────── + + +def _memory_config(**overrides: object) -> MemoryConfig: + config = MemoryConfig() + for key, value in overrides.items(): + setattr(config, key, value) + return config + + +def _make_fact( + fact_id: str, + content: str = "test content", + category: str = "knowledge", + confidence: float = 0.9, + days_ago: int = 100, +) -> dict: + created = (datetime.now(UTC) - timedelta(days=days_ago)).isoformat().replace("+00:00", "Z") + return { + "id": fact_id, + "content": content, + "category": category, + "confidence": confidence, + "createdAt": created, + "source": "thread-test", + } + + +def _make_memory(facts: list[dict] | None = None) -> dict: + return { + "version": "1.0", + "lastUpdated": "", + "user": { + "workContext": {"summary": "", "updatedAt": ""}, + "personalContext": {"summary": "", "updatedAt": ""}, + "topOfMind": {"summary": "", "updatedAt": ""}, + }, + "history": { + "recentMonths": {"summary": "", "updatedAt": ""}, + "earlierContext": {"summary": "", "updatedAt": ""}, + "longTermBackground": {"summary": "", "updatedAt": ""}, + }, + "facts": facts or [], + } + + +# ── _parse_fact_datetime ────────────────────────────────────────────────── + + +class TestParseFactDatetime: + def test_z_suffix(self): + result = _parse_fact_datetime("2025-06-01T12:00:00Z") + assert result is not None + assert result.year == 2025 + assert result.month == 6 + + def test_offset_format(self): + result = _parse_fact_datetime("2025-06-01T12:00:00+00:00") + assert result is not None + assert result.year == 2025 + + def test_empty_string(self): + assert _parse_fact_datetime("") is None + + def test_invalid_format(self): + assert _parse_fact_datetime("not-a-date") is None + + def test_naive_datetime_gets_utc(self): + """Naive datetime (no tzinfo) should be treated as UTC, not cause TypeError.""" + result = _parse_fact_datetime("2025-06-01T12:00:00") + assert result is not None + assert result.tzinfo is not None + assert result.utcoffset().total_seconds() == 0 + + +# ── _select_stale_candidates ────────────────────────────────────────────── + + +class TestSelectStaleCandidates: + def test_old_facts_selected(self): + memory = _make_memory( + [ + _make_fact("fact_old", days_ago=100), + _make_fact("fact_new", days_ago=10), + ] + ) + config = _memory_config(staleness_age_days=90) + candidates = _select_stale_candidates(memory, config) + assert len(candidates) == 1 + assert candidates[0]["id"] == "fact_old" + + def test_protected_category_excluded(self): + memory = _make_memory( + [ + _make_fact("fact_correction", category="correction", days_ago=200), + _make_fact("fact_knowledge", category="knowledge", days_ago=200), + ] + ) + config = _memory_config(staleness_age_days=90, staleness_protected_categories=["correction"]) + candidates = _select_stale_candidates(memory, config) + assert len(candidates) == 1 + assert candidates[0]["id"] == "fact_knowledge" + + def test_custom_protected_categories(self): + memory = _make_memory( + [ + _make_fact("fact_goal", category="goal", days_ago=200), + ] + ) + config = _memory_config(staleness_age_days=90, staleness_protected_categories=["goal"]) + candidates = _select_stale_candidates(memory, config) + assert len(candidates) == 0 + + def test_no_facts(self): + memory = _make_memory([]) + config = _memory_config(staleness_age_days=90) + assert _select_stale_candidates(memory, config) == [] + + def test_all_recent(self): + memory = _make_memory( + [ + _make_fact("fact_a", days_ago=10), + _make_fact("fact_b", days_ago=30), + ] + ) + config = _memory_config(staleness_age_days=90) + assert _select_stale_candidates(memory, config) == [] + + +# ── Trigger conditions via _select_stale_candidates + config ───────────── + + +class TestStalenessTriggerConditions: + """The old _should_run_staleness_review was removed; trigger logic is now + inlined in _prepare_update_prompt. We verify the gating conditions here + through _select_stale_candidates + config flags directly.""" + + def test_disabled_means_no_section(self): + memory = _make_memory([_make_fact(f"f{i}", days_ago=100) for i in range(5)]) + config = _memory_config(staleness_review_enabled=False, staleness_age_days=90, staleness_min_candidates=3) + candidates = _select_stale_candidates(memory, config) + # Even though candidates exist, the caller checks enabled flag first + assert config.staleness_review_enabled is False + assert len(candidates) >= config.staleness_min_candidates + + def test_below_min_candidates(self): + memory = _make_memory([_make_fact("fact_only", days_ago=100)]) + config = _memory_config(staleness_review_enabled=True, staleness_age_days=90, staleness_min_candidates=3) + candidates = _select_stale_candidates(memory, config) + assert len(candidates) < config.staleness_min_candidates + + def test_at_min_candidates(self): + memory = _make_memory([_make_fact(f"fact_{i}", days_ago=100) for i in range(3)]) + config = _memory_config(staleness_review_enabled=True, staleness_age_days=90, staleness_min_candidates=3) + candidates = _select_stale_candidates(memory, config) + assert len(candidates) >= config.staleness_min_candidates + + def test_above_min_candidates(self): + memory = _make_memory([_make_fact(f"fact_{i}", days_ago=100) for i in range(10)]) + config = _memory_config(staleness_review_enabled=True, staleness_age_days=90, staleness_min_candidates=3) + candidates = _select_stale_candidates(memory, config) + assert len(candidates) >= config.staleness_min_candidates + + +# ── _build_staleness_section ────────────────────────────────────────────── + + +class TestBuildStalenessSection: + def test_empty_candidates(self): + assert _build_staleness_section([], 90) == "" + + def test_includes_fact_details(self): + candidates = [ + _make_fact("fact_vue", "User uses Vue.js", "knowledge", 0.95, days_ago=120), + ] + section = _build_staleness_section(candidates, 90) + assert "fact_vue" in section + assert "User uses Vue.js" in section + assert "0.95" in section + assert "90 days" in section + + def test_multiple_facts(self): + candidates = [ + _make_fact("fact_a", "Fact A", "knowledge", 0.9, days_ago=100), + _make_fact("fact_b", "Fact B", "preference", 0.8, days_ago=150), + ] + section = _build_staleness_section(candidates, 90) + assert "fact_a" in section + assert "fact_b" in section + assert "" in section + + +# ── _apply_updates with staleness removals ───────────────────────────────── + + +class TestApplyUpdatesStaleness: + def test_stale_facts_removed(self): + updater = MemoryUpdater() + current_memory = _make_memory( + [ + _make_fact("fact_keep", "User knows Python", days_ago=100), + _make_fact("fact_stale", "User uses Vue.js", days_ago=120), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [ + {"id": "fact_stale", "reason": "User switched to React"}, + ], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100, staleness_max_removals_per_cycle=10), + ): + result = updater._apply_updates(current_memory, update_data) + + assert len(result["facts"]) == 1 + assert result["facts"][0]["id"] == "fact_keep" + + def test_safety_cap_limits_removals(self): + updater = MemoryUpdater() + # 5 stale facts, but cap is 2 → only 2 lowest-confidence should be removed + current_memory = _make_memory( + [ + _make_fact("fact_high", confidence=0.95, days_ago=100), + _make_fact("fact_mid", confidence=0.80, days_ago=100), + _make_fact("fact_low1", confidence=0.70, days_ago=100), + _make_fact("fact_low2", confidence=0.65, days_ago=100), + _make_fact("fact_low3", confidence=0.60, days_ago=100), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [ + {"id": "fact_high", "reason": "outdated"}, + {"id": "fact_mid", "reason": "outdated"}, + {"id": "fact_low1", "reason": "outdated"}, + {"id": "fact_low2", "reason": "outdated"}, + {"id": "fact_low3", "reason": "outdated"}, + ], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100, staleness_max_removals_per_cycle=2), + ): + result = updater._apply_updates(current_memory, update_data) + + # 5 - 2 = 3 facts remain; the 2 lowest-confidence removed + assert len(result["facts"]) == 3 + remaining_ids = {f["id"] for f in result["facts"]} + assert "fact_high" in remaining_ids + assert "fact_mid" in remaining_ids + assert "fact_low1" in remaining_ids + + def test_empty_stale_removals_no_effect(self): + updater = MemoryUpdater() + current_memory = _make_memory( + [ + _make_fact("fact_a", days_ago=100), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100), + ): + result = updater._apply_updates(current_memory, update_data) + + assert len(result["facts"]) == 1 + + def test_missing_stale_removals_key_no_effect(self): + """When LLM doesn't return staleFactsToRemove, existing behavior is preserved.""" + updater = MemoryUpdater() + current_memory = _make_memory([_make_fact("fact_a")]) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + # no staleFactsToRemove key + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100), + ): + result = updater._apply_updates(current_memory, update_data) + + assert len(result["facts"]) == 1 + + def test_contradiction_and_staleness_removals_combined(self): + """Both factsToRemove and staleFactsToRemove work together.""" + updater = MemoryUpdater() + current_memory = _make_memory( + [ + _make_fact("fact_keep", days_ago=10), + _make_fact("fact_contradicted", days_ago=10), + _make_fact("fact_stale", days_ago=200), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": ["fact_contradicted"], + "staleFactsToRemove": [{"id": "fact_stale", "reason": "old"}], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config(max_facts=100, staleness_max_removals_per_cycle=10), + ): + result = updater._apply_updates(current_memory, update_data) + + assert len(result["facts"]) == 1 + assert result["facts"][0]["id"] == "fact_keep" + + def test_protected_category_fact_refused_at_apply(self): + """Regression: LLM hallucinating a correction-category fact id in + staleFactsToRemove must be silently rejected at the apply layer, + even though it appears in the serialized prompt JSON.""" + updater = MemoryUpdater() + current_memory = _make_memory( + [ + _make_fact("fact_stale", category="knowledge", days_ago=200), + _make_fact("fact_correction", category="correction", days_ago=200), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [ + {"id": "fact_stale", "reason": "outdated"}, + {"id": "fact_correction", "reason": "LLM slip"}, + ], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config( + max_facts=100, + staleness_review_enabled=True, + staleness_age_days=90, + staleness_min_candidates=1, + staleness_max_removals_per_cycle=10, + staleness_protected_categories=["correction"], + ), + ): + result = updater._apply_updates(current_memory, update_data) + + # fact_stale removed, fact_correction kept (protected) + assert len(result["facts"]) == 1 + assert result["facts"][0]["id"] == "fact_correction" + + def test_non_aged_fact_refused_at_apply(self): + """Regression: LLM returning a fresh (non-aged) fact id in + staleFactsToRemove must be silently rejected.""" + updater = MemoryUpdater() + current_memory = _make_memory( + [ + _make_fact("fact_stale", days_ago=200), + _make_fact("fact_fresh", days_ago=10), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [ + {"id": "fact_stale", "reason": "outdated"}, + {"id": "fact_fresh", "reason": "LLM hallucination"}, + ], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config( + max_facts=100, + staleness_review_enabled=True, + staleness_age_days=90, + staleness_min_candidates=1, + staleness_max_removals_per_cycle=10, + staleness_protected_categories=["correction"], + ), + ): + result = updater._apply_updates(current_memory, update_data) + + # fact_stale removed, fact_fresh kept (not in candidate set) + assert len(result["facts"]) == 1 + assert result["facts"][0]["id"] == "fact_fresh" + + def test_guardrail_runs_when_staleness_review_disabled(self): + """Regression: guardrail must reject invalid ids even when + staleness_review_enabled=False, so the protection is independent + of the feature flag and model behavior.""" + updater = MemoryUpdater() + current_memory = _make_memory( + [ + _make_fact("fact_stale", days_ago=200), + _make_fact("fact_fresh", days_ago=5), + ] + ) + update_data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [ + {"id": "fact_stale", "reason": "LLM hallucination"}, + {"id": "fact_fresh", "reason": "LLM hallucination"}, + ], + } + + with patch( + "deerflow.agents.memory.updater.get_memory_config", + return_value=_memory_config( + max_facts=100, + staleness_review_enabled=False, + staleness_age_days=90, + staleness_min_candidates=3, + staleness_max_removals_per_cycle=10, + staleness_protected_categories=["correction"], + ), + ): + result = updater._apply_updates(current_memory, update_data) + + # Guardrail runs regardless of feature flag: + # fact_stale is a valid candidate (200 days old) → removed + # fact_fresh is not a candidate (5 days old) → kept + assert len(result["facts"]) == 1 + assert result["facts"][0]["id"] == "fact_fresh" + + +# ── _normalize_memory_update_data with staleFactsToRemove ───────────────── + + +class TestNormalizeStaleFactsToRemove: + def test_valid_entries(self): + data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [ + {"id": "fact_a", "reason": "User moved offices"}, + {"id": "fact_b", "reason": "Tech stack changed"}, + ], + } + result = _normalize_memory_update_data(data) + assert len(result["staleFactsToRemove"]) == 2 + assert result["staleFactsToRemove"][0]["id"] == "fact_a" + assert result["staleFactsToRemove"][1]["reason"] == "Tech stack changed" + + def test_missing_key(self): + data = {"user": {}, "history": {}, "newFacts": [], "factsToRemove": []} + result = _normalize_memory_update_data(data) + assert result["staleFactsToRemove"] == [] + + def test_non_list_ignored(self): + data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": "not a list", + } + result = _normalize_memory_update_data(data) + assert result["staleFactsToRemove"] == [] + + def test_non_dict_entries_skipped(self): + data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": ["just a string", 42, {"id": "fact_ok", "reason": "valid"}], + } + result = _normalize_memory_update_data(data) + assert len(result["staleFactsToRemove"]) == 1 + assert result["staleFactsToRemove"][0]["id"] == "fact_ok" + + def test_empty_id_skipped(self): + data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [{"id": "", "reason": "no id"}], + } + result = _normalize_memory_update_data(data) + assert result["staleFactsToRemove"] == [] + + def test_non_string_reason_defaulted(self): + data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [{"id": "fact_a", "reason": 123}], + } + result = _normalize_memory_update_data(data) + assert result["staleFactsToRemove"][0]["reason"] == "" + + def test_missing_reason_defaulted(self): + data = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "staleFactsToRemove": [{"id": "fact_a"}], + } + result = _normalize_memory_update_data(data) + assert result["staleFactsToRemove"][0]["reason"] == "" + + +# ── Integration: _prepare_update_prompt ──────────────────────────────────── + + +class TestPrepareUpdatePromptStaleness: + def test_staleness_section_included_when_triggered(self): + updater = MemoryUpdater() + old_facts = [_make_fact(f"fact_{i}", days_ago=100) for i in range(5)] + memory = _make_memory(old_facts) + + msg = MagicMock() + msg.type = "human" + msg.content = "Hello, I'm using React now" + + config = _memory_config( + enabled=True, + staleness_review_enabled=True, + staleness_age_days=90, + staleness_min_candidates=3, + ) + + with ( + patch("deerflow.agents.memory.updater.get_memory_config", return_value=config), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=memory), + ): + result = updater._prepare_update_prompt( + messages=[msg], + agent_name=None, + correction_detected=False, + reinforcement_detected=False, + ) + + assert result is not None + _, prompt = result + assert "Staleness Review" in prompt + assert "" in prompt + + def test_staleness_section_omitted_when_not_triggered(self): + updater = MemoryUpdater() + memory = _make_memory([]) # no facts at all + + msg = MagicMock() + msg.type = "human" + msg.content = "Hello" + + config = _memory_config( + enabled=True, + staleness_review_enabled=True, + staleness_age_days=90, + staleness_min_candidates=3, + ) + + with ( + patch("deerflow.agents.memory.updater.get_memory_config", return_value=config), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=memory), + ): + result = updater._prepare_update_prompt( + messages=[msg], + agent_name=None, + correction_detected=False, + reinforcement_detected=False, + ) + + assert result is not None + _, prompt = result + assert "Staleness Review" not in prompt + assert "" not in prompt + + def test_staleness_section_omitted_when_disabled(self): + updater = MemoryUpdater() + old_facts = [_make_fact(f"fact_{i}", days_ago=200) for i in range(10)] + memory = _make_memory(old_facts) + + msg = MagicMock() + msg.type = "human" + msg.content = "Hello" + + config = _memory_config( + enabled=True, + staleness_review_enabled=False, + ) + + with ( + patch("deerflow.agents.memory.updater.get_memory_config", return_value=config), + patch("deerflow.agents.memory.updater.get_memory_data", return_value=memory), + ): + result = updater._prepare_update_prompt( + messages=[msg], + agent_name=None, + correction_detected=False, + reinforcement_detected=False, + ) + + assert result is not None + _, prompt = result + assert "Staleness Review" not in prompt diff --git a/config.example.yaml b/config.example.yaml index b7d3c04ec..7cb90a0aa 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1342,6 +1342,26 @@ memory: guaranteed_categories: - correction guaranteed_token_budget: 500 + # Staleness review: periodically prune aged facts that may no longer reflect + # the user's current situation. When triggered, the LLM reviews facts older + # than ``staleness_age_days`` during the normal memory-update call (same LLM + # invocation — no extra API call) and decides KEEP or REMOVE for each. + # staleness_review_enabled - master switch (default: true) + # staleness_age_days - facts older than this are candidates (default: 90) + # staleness_min_candidates - minimum stale facts required to trigger a review + # cycle; avoids wasteful LLM calls when there are + # very few candidates (default: 3) + # staleness_max_removals_per_cycle - safety cap on removals per cycle; when + # exceeded, the lowest-confidence entries + # are kept (default: 10) + # staleness_protected_categories - fact categories exempt from review + # (default: ["correction"]) + staleness_review_enabled: true + staleness_age_days: 90 + staleness_min_candidates: 3 + staleness_max_removals_per_cycle: 10 + staleness_protected_categories: + - correction # ============================================================================ # Custom Agent Management API