{
  "version": "continuity-v1",
  "created_before_eval": true,
  "deerflow_commit": "3f0b6ecc811190481897f1ed02c2ba0c1f69799e",
  "llm_model": "qwen3.8-flash-next",
  "embedding_model": "Qwen3-Embedding-0.6B",
  "temperature": 0,
  "reasoning_effort": "none",
  "seed": 20260912,
  "public_dataset": "xiaowu0162/longmemeval-cleaned",
  "dataset_revision": "98d7416c24c778c2fee6e6f3006e7a073259d48f",
  "dataset_file": "longmemeval_s_cleaned.json",
  "dataset_sha256": "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
  "public_test_per_stratum": 6,
  "public_dev_per_stratum": 1,
  "strata": ["single-session-user", "single-session-assistant", "single-session-preference", "temporal-reasoning", "knowledge-update", "multi-session", "abstention"],
  "selection": "Within each stratum sort SHA256(20260912:question_id); first item is dev, next six test. No answer- or score-based selection.",
  "budget_tokenizer": "cl100k_base (fixed proxy tokenizer; actual model usage separately recorded)",
  "archive_chunk_tokens": 384,
  "archive_chunk_overlap": 64,
  "history_batch_tokens": 12000,
  "recent_tail_tokens": 1024,
  "summary_max_output_tokens": 1024,
  "notes_max_output_tokens": 1536,
  "notes_context_tokens": 1536,
  "retrieval_context_tokens": 4096,
  "answer_max_output_tokens": 512,
  "reader_context_limit_tokens": 8192,
  "arms": {"A": "rolling summary + recent tail", "B": "A + task/history notes with source IDs", "C": "B + SQLite FTS5 history retrieval", "D": "B + FTS5/dense reciprocal-rank fusion"},
  "public_reading": "One reader call per arm with direct final-question retrieval; no question/answer/evidence labels in summarization or notes. Gold metadata only in scorer. Identical shared summary/notes artifacts reused across arms.",
  "retrieval": "C and D use same raw chunks, source scope and token budget. RRF k=60, equal rank weights, candidate pool 80 per retriever; pure dense retrieval is diagnostic only. Evidence session recall and token cost reported.",
  "qa_grading": "Official LongMemEval evaluator prompt, same authorized model, blind to arm, keyed by reference + prediction; inspect discordant/ambiguous cases; disclose same-model judging.",
  "statistics": "Paired case bootstrap 10000 resamples seed 20260912; exact McNemar; exploratory pilot, not a population-wide guarantee.",
  "task_continuation": "24 independently authored tasks, six families x four variants, three forced compactions; same objective, simulated environment state and step cap for all arms. Live native tool calls write a real JSON artifact, deterministic verifier evaluates required values/forbidden actions. Controlled environment, not full DeerFlow production acceptance.",
  "limitations": ["Replay prototype outside production runtime; no claim of deployment or integrated DeerFlow score", "Fixed small public subset; no leaderboard claim", "A/B/C differ in consumed context; equal hard cap and all costs reported, B-A is not a token-matched pure representation effect", "D-C isolates the retrieval change", "Generation, embedding, note writing, readback and retries count toward reported overhead", "No private task histories or real user memories are sent"],
  "retry_policy": "Up to three attempts for network/429/5xx only; malformed generations logged, not silently repaired using gold answers.",
  "failure_policy": "Keep all selected rows, including failed or incomplete calls; report operational coverage separately from conditional quality. Protocol changes require a new named version and rerun affected rows."
}