mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-08-13 16:28:38 +00:00
42baed8c8c
* feat(checkpoint): dual-mode checkpoint storage with LangGraph DeltaChannel
Add a restart-required database.checkpoint_channel_mode ("full" default,
"delta") that stores the messages channel via LangGraph 1.2 DeltaChannel,
cutting checkpoint storage from O(n^2) to O(n) for append-only history.
Existing full checkpoints seed delta state transparently; no data migration.
- config: mode schema + freeze-on-first-use with
CheckpointModeReconfigurationError; mode marker persisted in checkpoint
metadata; unsafe delta->full downgrade rejected fail-closed with
CheckpointModeMismatchError (run-level error, failed state read)
- state: delta message state schema; CheckpointStateAccessor centralizes
materialized reads for all consumers (threads API, branches,
regeneration, compaction, state updates, memory, goal workers)
- runtime: raw writers (run durations, interrupted title, thread goal)
parent their checkpoints to the checkpoint they derive from, preserving
delta ancestry; rollback forks the pre-run lineage through a state
mutation graph with Overwrite restores; InMemorySaver delta-history
override delegates to the base walk (fixes dropped first write after
migration, also present upstream)
- tests: conformance suite over {memory, sqlite, postgres} covering
migration replay, stable message IDs, storage shape and writer
preservation; conftest fixture isolates the frozen mode between tests;
stale config fakes refreshed
- ci: backend unit tests gain a postgres service
* fix(checkpoint): close materialization gaps in goal flow, guard public factory
- Route goal-continuation message reads through CheckpointStateAccessor:
raw channel_values reads see the delta sentinel in delta mode, which
disabled goal continuation (stand_down=no_durable_end_of_turn) after
durable assistant turns. Raw tuples remain for tuple-only metadata
(checkpoint id, pending_writes).
- Reject checkpoint_channel_mode='delta' + checkpointer in
create_deerflow_agent at construction: factory-built persisted graphs
bypass mode-marker injection and the fail-closed gate, reproducing
silent mixed-mode state loss. Delta without persistence stays allowed.
- Import the postgres saver lazily (pytest.importorskip in the fixture)
so the documented default install collects the suite; add a CI job
running pytest --collect-only on uv sync --group dev without extras.
- Fix test_checkpointer fallback test to patch get_app_config at its
use site (provider module), making it deterministic when a local
config.yaml selects a persistent backend.
* fix(gateway): preserve extension-owned channels in state mutations, bump config version
- build_state_mutation_graph / build_checkpoint_state_mutation_accessor
accept an explicit state_schema; branch and POST /state now compile the
mutation graph from the thread's effective schema
(graph_state_schema on the assistant graph). The base-ThreadState
fallback silently discarded channels contributed by custom
AgentMiddleware.state_schema on branch (data loss) and returned a
false-success 200 on POST /state.
- POST /state validates values keys against the mutation graph's
channels and rejects unknown fields with 422 instead of ignoring
them; reducer detection covers extension channels
(BinaryOperatorAggregate or DeltaChannel) so Overwrite replace
semantics work for middleware reducers in both modes.
- Endpoint regression: custom AgentMiddleware.state_schema value
survives branch, updates through POST /state, and an unknown field
receives 422.
- config_version 26 -> 27 for the new database.checkpoint_channel_mode
(example, Helm chart values + README, support-bundle fixture), so
existing installs get the outdated-config warning and
make config-upgrade merges the field; covered by a test driving the
real example file and the real config-upgrade script.
* fix(gateway): resolve assistant schema via one boundary, copy branch reducer values with Overwrite
GET /threads/{id}/state now resolves the thread's assistant_id through a
single reusable boundary (thread metadata -> assistant_id -> effective
graph), so channels contributed by a custom AgentMiddleware.state_schema
are materialized instead of dropped by the default lead schema. POST
/state uses the same boundary instead of resolving the schema ad hoc.
Branch writes wrap every copied reducer channel in Overwrite (derived
from the effective mutation graph: BinaryOperatorAggregate + DeltaChannel),
not just messages, so already-aggregated values are never re-merged.
Regression tests use a real AgentMiddleware.state_schema with a
non-identity reducer in both full and delta modes: GET /state returns the
extension value, POST /state replaces it, branch preserves it
byte-for-byte; the unknown-field 422 is a separate assertion.
* refactor(checkpoint): collapse read-path round-trips and ship dual-mode parity tests
Address review round 4 on PR #4292:
- Push ahistory/history limit through Pregel into checkpointer.alist
(SQL LIMIT) instead of materializing all rows and breaking in Python
- Fold the read-side mode-compat gate onto the returned snapshot's
metadata; only writes keep the pre-write tuple fetch (fail-closed)
- Cache factory-built accessor graphs per (assistant_id, mode) with
factory-identity revalidation; state reads no longer build a lead
agent per request
- get_thread: one snapshot fetch + one raw pending_writes fetch on the
resolved checkpoint (post-checkpoint __error__ writes never surface
in snapshot.tasks; verified empirically)
- DeerFlowClient.get_thread: single checkpointer.list walk collects
pending_writes per checkpoint instead of N get_tuple calls
- InMemorySaver delta-history patch: stand-down when the upstream
override disappears, try/except guard, validated-version warning,
guard tests
- make_lead_agent mode precedence: first freeze is owned by app_config
(client-supplied configurable key ignored); once frozen, injected
key/app_config must match or fail closed
- Rollback: lock in non-message channel restoration via fork
inheritance with a dedicated reducer-channel test
- Add tests/test_threads_checkpoint_mode.py and
tests/test_gateway_checkpoint_mode.py referenced by AGENTS.md and
the PR validation section: lifecycle parity (memory + sqlite),
per-step blob-count storage guard, gateway endpoint parity
Counted-saver tests pin checkpoint round-trips for aget/ahistory so
these regressions cannot silently return.
* fix(checkpoint): precise mode-mismatch HTTP mapping, gate E2E, and accessor resilience
- threads router: map CheckpointModeMismatchError to 409 (with cause and
thread id) and CheckpointModeReconfigurationError to 503 across all state
endpoints instead of swallowing both into a generic 500
- gate coverage: seed a real delta checkpoint into AsyncSqliteSaver and
assert aget/aupdate/ahistory fail closed; assert 409 at the HTTP boundary
through the real route stack
- rollback: compile the restore mutation graph with the thread's effective
state schema per the build_state_mutation_graph contract
- inheritance contract locks: rollback and manual compaction preserve
middleware-contributed channels via checkpoint fork cloning
- services: revalidate the accessor-graph cache against app_config identity
so config.yaml hot-reloads never serve a stale compiled graph
- services: degrade full-mode state reads to raw checkpointer reads when the
agent factory is unavailable (delta gate still applies; delta mode has no
fallback)
- deps: override websockets==16.0 (langgraph-sdk 0.4.2's <16 pin silently
downgraded 16.0 -> 15.0.1; pin is not grounded in any API incompatibility)
and bump the langchain lower bound to what the lockfile actually resolves
* fix(checkpoint): include anchor checkpoint in degraded history walk + cover get_thread
- _RawCheckpointReadAccessor.ahistory: alist(before=...) is exclusive while
pregel's get_state_history treats config.checkpoint_id as the inclusive
start; fetch the anchor explicitly so both read paths paginate identically
- extend the degraded-path gateway test: GET /thread returns raw values, and
POST /history with before starts at the anchor checkpoint
* fix(gateway): preserve degraded checkpoint timestamps
* fix(gateway): harden degraded checkpoint access
* fix(gateway): resolve assistants for checkpoint reads
---------
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
84 lines
2.8 KiB
TOML
84 lines
2.8 KiB
TOML
[project]
|
|
name = "deerflow-harness"
|
|
version = "2.1.0"
|
|
description = "DeerFlow agent harness framework"
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
"agent-client-protocol>=0.4.0",
|
|
"agent-sandbox>=0.0.19",
|
|
"croniter>=6.0.0",
|
|
"dotenv>=0.9.9",
|
|
"exa-py>=1.0.0",
|
|
"httpx>=0.28.0",
|
|
"kubernetes>=30.0.0",
|
|
# Lower bound reflects what the lockfile resolves and tests run against
|
|
# (langgraph 1.2.9 pulls langchain >=1.3 transitively).
|
|
"langchain>=1.3",
|
|
"langchain-anthropic>=1.4.1",
|
|
"langchain-deepseek>=1.0.1",
|
|
"langchain-mcp-adapters>=0.2.2",
|
|
"langchain-openai>=1.2.1",
|
|
"langfuse>=3.4.1",
|
|
"langgraph>=1.2.9,<1.3",
|
|
"langgraph-api>=0.8.1",
|
|
"langgraph-cli>=0.4.24",
|
|
"langgraph-runtime-inmem>=0.28.0",
|
|
"markdownify>=1.2.2",
|
|
"markitdown[all,xlsx]>=0.0.1a2",
|
|
"pydantic>=2.12.5",
|
|
"pyyaml>=6.0.3",
|
|
"readabilipy>=0.3.0",
|
|
"tavily-python>=0.7.17",
|
|
"firecrawl-py>=1.15.0",
|
|
"tiktoken>=0.8.0",
|
|
"ddgs>=9.10.0",
|
|
"duckdb>=1.4.4",
|
|
"langchain-google-genai>=4.2.1",
|
|
"langgraph-checkpoint-sqlite>=3.1.0,<3.2",
|
|
"langgraph-sdk>=0.1.51",
|
|
"sqlalchemy[asyncio]>=2.0,<3.0",
|
|
"aiosqlite>=0.19",
|
|
"alembic>=1.13",
|
|
"cryptography>=48.0.1",
|
|
"e2b-code-interpreter>=2.8.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
deerflow = "deerflow.tui.cli:main"
|
|
|
|
[project.optional-dependencies]
|
|
# Terminal workbench (TUI). Kept optional so the core harness install stays lean;
|
|
# the `deerflow` console script degrades to headless help when textual is absent.
|
|
tui = ["textual>=0.80"]
|
|
# GroundRoute needs no extra packages (httpx is already a core dependency). This
|
|
# empty extra exists so the documented `uv add 'deerflow-harness[groundroute]'`
|
|
# install command resolves cleanly without an "unknown extra" warning.
|
|
groundroute = []
|
|
ollama = ["langchain-ollama>=0.3.0"]
|
|
postgres = [
|
|
"asyncpg>=0.29",
|
|
"langgraph-checkpoint-postgres>=3.1.0,<3.2",
|
|
"psycopg[binary]>=3.3.3",
|
|
"psycopg-pool>=3.3.0",
|
|
]
|
|
# Cross-process SSE stream bridge (stream_bridge.type: redis). Optional so
|
|
# single-process / memory-bridge installs do not pull redis. The Docker image
|
|
# always installs this extra because Docker defaults to the redis bridge.
|
|
redis = ["redis>=5.0.0"]
|
|
pymupdf = ["pymupdf4llm>=0.0.17"]
|
|
boxlite = ["boxlite>=0.9.7"]
|
|
# Agent observability (Monocle). Optional so a default install stays free of the
|
|
# OpenTelemetry stack; only pulled in when MONOCLE_TRACING is used.
|
|
monocle = ["monocle_apptrace>=0.8.8"]
|
|
# Agentic browser control (browser_navigate/click/type/... tool group). Optional
|
|
# so the core harness install stays lean; import is lazy inside the private
|
|
# Playwright loop. After install, run `playwright install chromium` once.
|
|
browser = ["playwright>=1.40"]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["deerflow"]
|