mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-08-15 01:08:53 +00:00
3949340610
* fix: harden Postgres async engine with pool_recycle and command_timeout to stop stale-connection 504s * fix(persistence): make postgres command timeout a configurable database setting Default the app-ORM command timeout to 30s (below nginx's 60s proxy deadline), expose it as database.command_timeout with null to disable, and add regression coverage. Addresses P1 review feedback. Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> * fix(persistence): decouple DB command timeout from nonexistent proxy deadline The command timeout bounds stalled ORM queries independently; drop the incorrect coupling to a 60s nginx deadline (actual proxy timeout is 600s). * feat(config): make pool_recycle configurable Expose pool_recycle alongside command_timeout and pool_size in config, example config, and the helm chart, keeping the 300s default, per review. --------- Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
116 lines
4.4 KiB
Python
116 lines
4.4 KiB
Python
"""Unified database backend configuration.
|
|
|
|
Controls BOTH the LangGraph checkpointer and the DeerFlow application
|
|
persistence layer (runs, threads metadata, users, etc.). The user
|
|
configures one backend; the system handles physical separation details.
|
|
|
|
SQLite mode: checkpointer and app share a single .db file
|
|
({sqlite_dir}/deerflow.db) with WAL journal mode enabled on every
|
|
connection. WAL allows concurrent readers and a single writer without
|
|
blocking, making a unified file safe for both workloads. Writers
|
|
that contend for the lock wait via the default 5-second sqlite3
|
|
busy timeout rather than failing immediately.
|
|
|
|
Postgres mode: both use the same database URL but maintain independent
|
|
connection pools with different lifecycles.
|
|
|
|
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
|
|
No database is initialized.
|
|
|
|
Sensitive values (postgres_url) should use $VAR syntax in config.yaml
|
|
to reference environment variables from .env:
|
|
|
|
database:
|
|
backend: postgres
|
|
postgres_url: $DATABASE_URL
|
|
|
|
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
|
before this config is instantiated -- DatabaseConfig itself does not
|
|
need to do any environment variable processing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class DatabaseConfig(BaseModel):
|
|
backend: Literal["memory", "sqlite", "postgres"] = Field(
|
|
default="memory",
|
|
description=("Storage backend for both checkpointer and application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment."),
|
|
)
|
|
sqlite_dir: str = Field(
|
|
default=".deer-flow/data",
|
|
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
|
|
)
|
|
postgres_url: str = Field(
|
|
default="",
|
|
description=(
|
|
"PostgreSQL connection URL, shared by checkpointer and app. "
|
|
"Use $DATABASE_URL in config.yaml to reference .env. "
|
|
"Example: postgresql://user:pass@host:5432/deerflow "
|
|
"(the +asyncpg driver suffix is added automatically where needed)."
|
|
),
|
|
)
|
|
echo_sql: bool = Field(
|
|
default=False,
|
|
description="Echo all SQL statements to log (debug only).",
|
|
)
|
|
pool_size: int = Field(
|
|
default=5,
|
|
description="Connection pool size for the app ORM engine (postgres only).",
|
|
)
|
|
pool_recycle: int = Field(
|
|
default=300,
|
|
gt=0,
|
|
description="Seconds before app ORM PostgreSQL connections are recycled.",
|
|
)
|
|
command_timeout: float | None = Field(
|
|
default=30,
|
|
gt=0,
|
|
description="Timeout in seconds for app ORM PostgreSQL commands. Set to null to disable the command timeout.",
|
|
)
|
|
|
|
# -- Derived helpers (not user-configured) --
|
|
|
|
@property
|
|
def _resolved_sqlite_dir(self) -> str:
|
|
"""Resolve sqlite_dir to an absolute path (relative to CWD)."""
|
|
from pathlib import Path
|
|
|
|
return str(Path(self.sqlite_dir).resolve())
|
|
|
|
@property
|
|
def sqlite_path(self) -> str:
|
|
"""Unified SQLite file path shared by checkpointer and app."""
|
|
return os.path.join(self._resolved_sqlite_dir, "deerflow.db")
|
|
|
|
# Backward-compatible aliases
|
|
@property
|
|
def checkpointer_sqlite_path(self) -> str:
|
|
"""SQLite file path for the LangGraph checkpointer (alias for sqlite_path)."""
|
|
return self.sqlite_path
|
|
|
|
@property
|
|
def app_sqlite_path(self) -> str:
|
|
"""SQLite file path for application ORM data (alias for sqlite_path)."""
|
|
return self.sqlite_path
|
|
|
|
@property
|
|
def app_sqlalchemy_url(self) -> str:
|
|
"""SQLAlchemy async URL for the application ORM engine."""
|
|
if self.backend == "sqlite":
|
|
return f"sqlite+aiosqlite:///{self.sqlite_path}"
|
|
if self.backend == "postgres":
|
|
url = self.postgres_url
|
|
if url.startswith("postgresql://"):
|
|
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
|
elif url.startswith("postgres://"):
|
|
# libpq's short alias: accepted by the psycopg checkpointer, but not a SQLAlchemy dialect.
|
|
url = url.replace("postgres://", "postgresql+asyncpg://", 1)
|
|
return url
|
|
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|