mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-24 05:26:20 +00:00
6d5d7bb1d5
* feat(subagents): add opt-in parent context snapshots * test(subagents): package synthetic snapshot evaluation * fix(subagents): preserve output text and defer snapshot capture * docs(subagents): keep snapshot guidance within chain budget * fix(subagents): omit unpaired tool calls from snapshots * fix(subagents): safely omit unserializable snapshot media
93 lines
4.3 KiB
JSON
93 lines
4.3 KiB
JSON
{
|
|
"synthetic": true,
|
|
"date": "2026-09-12",
|
|
"origin": "Metadata export from original local harness; not generated by the portable runner",
|
|
"runtime_equivalent_revision": "3fc5bf09e5664d3460c1d8b10e7d95d9aebc5853",
|
|
"runtime_note": "Live evaluation ran before this commit; measured production source was unchanged when committed",
|
|
"model": "qwen3.8-flash-next",
|
|
"config": {
|
|
"protocol": "context-snapshot-synthetic-v1",
|
|
"temperature": 0.2,
|
|
"reasoning_effort": "none",
|
|
"max_tokens": 4096,
|
|
"context_window": 262144,
|
|
"http_timeout_seconds": 120,
|
|
"worker_timeout_seconds": 180,
|
|
"max_graph_steps": 50,
|
|
"max_retries": 0,
|
|
"seed": 1703,
|
|
"order_seed": 20260913,
|
|
"concurrency": 2
|
|
},
|
|
"historical_repetition": 3,
|
|
"historical_seed": 1703,
|
|
"canonical_composition": {
|
|
"original_control_rows": 14,
|
|
"corrected_invoice_rows": 2,
|
|
"original_parent_calls_reused": 2
|
|
},
|
|
"phases": {
|
|
"capability-control": {
|
|
"protocol_sha256": "66108c541bdbb307affff2ee0c653da42e804a90c8d7da95d783719caf9d6576",
|
|
"source_hashes": {
|
|
"evaluate.py": "40367eafd88b56164c33bd4bd48d151dc192e7d090c352fe411052560891a7eb",
|
|
"capability_control.py": "76492676353c6f1e5e61dfd1b536765574f5d00f9b9e84810450f4bfcb57b3cb",
|
|
"cases.py": "c2be56cf7a1e63265907d30b8cef4db878b7b352ac21a74dbfac71ba63ab02fb"
|
|
},
|
|
"order": [
|
|
"pagination__3__isolated_handoff",
|
|
"release_plan__3__isolated_handoff",
|
|
"email_cleanup__3__snapshot",
|
|
"rate_limits__3__snapshot",
|
|
"fresh_evidence__3__snapshot",
|
|
"invoice_total__3__isolated_handoff",
|
|
"rate_limits__3__isolated_handoff",
|
|
"fresh_evidence__3__isolated_handoff",
|
|
"storage_policy__3__isolated_handoff",
|
|
"email_cleanup__3__isolated_handoff",
|
|
"pagination__3__snapshot",
|
|
"self_contained__3__snapshot",
|
|
"storage_policy__3__snapshot",
|
|
"self_contained__3__isolated_handoff",
|
|
"release_plan__3__snapshot",
|
|
"invoice_total__3__snapshot"
|
|
],
|
|
"changes": [
|
|
"Explicit worker capabilities in both lead prompts",
|
|
"Accept valid parent prompt rephrasing, measure deviations separately",
|
|
"Forward any unexpected acceptance_criteria into the executor instead of ignoring them"
|
|
],
|
|
"unchanged": [
|
|
"Case briefs/history/summary/reference data",
|
|
"Hidden graders",
|
|
"Worker system/tools/factory",
|
|
"Model parameters except predeclared new repetition seed",
|
|
"Mode validation and one lead-call accounting",
|
|
"No retries of semantic errors, mode/schema failures, timeouts or turn caps"
|
|
]
|
|
},
|
|
"capability-grader-corrected": {
|
|
"protocol_sha256": "42bf2f59414e5a9bd4d73a9d40de4e8a5b593b56736e4d925fe1f3163539b931",
|
|
"source_hashes": {
|
|
"evaluate.py": "40367eafd88b56164c33bd4bd48d151dc192e7d090c352fe411052560891a7eb",
|
|
"evaluate_fixed_grader.py": "55c1f956c03a765af1e2e98c359dc08108bcaf850d3d8fad106e1f6b43be5ab5",
|
|
"corrected_invoice_control.py": "44aa8e275c7997fd4d99d0103c189cac0a90f3655e310074fb2f7a73d48cfc7b",
|
|
"cases.py": "c2be56cf7a1e63265907d30b8cef4db878b7b352ac21a74dbfac71ba63ab02fb"
|
|
},
|
|
"changes": [
|
|
"Expose the standard pure arithmetic builtin divmod to the evaluator"
|
|
],
|
|
"unchanged": [
|
|
"Original 8-case corpus and all predeclared hidden checks",
|
|
"Lead calls and their task arguments",
|
|
"Worker prompts, tools and budget",
|
|
"Model and sampling settings",
|
|
"Actual context snapshot implementation"
|
|
],
|
|
"reason": "The evaluator's safe builtin allowlist omitted Python divmod. That produced false NameError public failures for valid standalone Python, changing worker behavior. Correct the grader and rerun BOTH arms of the affected invoice case, not only the failing arm. Keep all original runs and report this correction explicitly.",
|
|
"lead_accounting": "Reuse each arm's exact already-generated task arguments and original lead HTTP record, tokens and elapsed time. Generate only fresh workers, under the same model settings and prompts. Canonical comparison uses each original lead once and the corrected workers; the discarded workers remain recorded as experimental overhead.",
|
|
"seed": 1703
|
|
}
|
|
}
|
|
}
|