mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-11 06:28:58 +00:00
perf(config): index AppConfig.get_*_config by name (O(n) -> O(1)) (#3688)
get_model_config / get_tool_config / get_tool_group_config did a next(...) linear scan of self.models / self.tools / self.tool_groups on every call. These sit on hot paths: get_tool_config runs 2-3x per community-tool invocation (web_search etc.) and get_model_config several times per agent build, across 40+ call sites. Build name -> config dicts once in a mode="after" validator (stored as PrivateAttr), so each getter is an O(1) dict lookup. A config reload constructs a fresh AppConfig, which rebuilds the indexes; setdefault keeps first-match-wins on duplicate names, matching the prior next(...) semantics. Co-authored-by: ly-wang19 <ly-wang19@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f956682f31
commit
b66e3253a0
@ -8,7 +8,7 @@ from typing import Any, Self
|
|||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
|
||||||
|
|
||||||
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
||||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||||
@ -163,6 +163,14 @@ class AppConfig(BaseModel):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Name -> config lookup tables, (re)built after validation by
|
||||||
|
# ``_build_name_indexes``. They make ``get_model_config`` / ``get_tool_config``
|
||||||
|
# / ``get_tool_group_config`` O(1) instead of an O(n) ``next(...)`` scan per
|
||||||
|
# call. Private attrs are excluded from serialization.
|
||||||
|
_models_by_name: dict[str, ModelConfig] = PrivateAttr(default_factory=dict)
|
||||||
|
_tools_by_name: dict[str, ToolConfig] = PrivateAttr(default_factory=dict)
|
||||||
|
_tool_groups_by_name: dict[str, ToolGroupConfig] = PrivateAttr(default_factory=dict)
|
||||||
|
|
||||||
@model_validator(mode="before")
|
@model_validator(mode="before")
|
||||||
@classmethod
|
@classmethod
|
||||||
def _drop_null_config_sections(cls, data: Any) -> Any:
|
def _drop_null_config_sections(cls, data: Any) -> Any:
|
||||||
@ -375,6 +383,31 @@ class AppConfig(BaseModel):
|
|||||||
return [cls.resolve_env_variables(item) for item in config]
|
return [cls.resolve_env_variables(item) for item in config]
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def _build_name_indexes(self) -> "AppConfig":
|
||||||
|
"""Build name -> config lookup tables for O(1) ``get_*_config``.
|
||||||
|
|
||||||
|
``get_tool_config`` runs 2-3x per community-tool invocation (e.g.
|
||||||
|
web_search) and ``get_model_config`` several times per agent build, so
|
||||||
|
the previous O(n) ``next(...)`` scans sat on hot paths. Rebuilt here so a
|
||||||
|
config reload (which constructs a fresh ``AppConfig``) refreshes them.
|
||||||
|
``setdefault`` keeps the first entry on duplicate names, preserving the
|
||||||
|
prior ``next(...)`` first-match semantics.
|
||||||
|
"""
|
||||||
|
models_by_name: dict[str, ModelConfig] = {}
|
||||||
|
for model in self.models:
|
||||||
|
models_by_name.setdefault(model.name, model)
|
||||||
|
tools_by_name: dict[str, ToolConfig] = {}
|
||||||
|
for tool in self.tools:
|
||||||
|
tools_by_name.setdefault(tool.name, tool)
|
||||||
|
tool_groups_by_name: dict[str, ToolGroupConfig] = {}
|
||||||
|
for group in self.tool_groups:
|
||||||
|
tool_groups_by_name.setdefault(group.name, group)
|
||||||
|
self._models_by_name = models_by_name
|
||||||
|
self._tools_by_name = tools_by_name
|
||||||
|
self._tool_groups_by_name = tool_groups_by_name
|
||||||
|
return self
|
||||||
|
|
||||||
def get_model_config(self, name: str) -> ModelConfig | None:
|
def get_model_config(self, name: str) -> ModelConfig | None:
|
||||||
"""Get the model config by name.
|
"""Get the model config by name.
|
||||||
|
|
||||||
@ -384,7 +417,7 @@ class AppConfig(BaseModel):
|
|||||||
Returns:
|
Returns:
|
||||||
The model config if found, otherwise None.
|
The model config if found, otherwise None.
|
||||||
"""
|
"""
|
||||||
return next((model for model in self.models if model.name == name), None)
|
return self._models_by_name.get(name)
|
||||||
|
|
||||||
def get_tool_config(self, name: str) -> ToolConfig | None:
|
def get_tool_config(self, name: str) -> ToolConfig | None:
|
||||||
"""Get the tool config by name.
|
"""Get the tool config by name.
|
||||||
@ -395,7 +428,7 @@ class AppConfig(BaseModel):
|
|||||||
Returns:
|
Returns:
|
||||||
The tool config if found, otherwise None.
|
The tool config if found, otherwise None.
|
||||||
"""
|
"""
|
||||||
return next((tool for tool in self.tools if tool.name == name), None)
|
return self._tools_by_name.get(name)
|
||||||
|
|
||||||
def get_tool_group_config(self, name: str) -> ToolGroupConfig | None:
|
def get_tool_group_config(self, name: str) -> ToolGroupConfig | None:
|
||||||
"""Get the tool group config by name.
|
"""Get the tool group config by name.
|
||||||
@ -406,7 +439,7 @@ class AppConfig(BaseModel):
|
|||||||
Returns:
|
Returns:
|
||||||
The tool group config if found, otherwise None.
|
The tool group config if found, otherwise None.
|
||||||
"""
|
"""
|
||||||
return next((group for group in self.tool_groups if group.name == name), None)
|
return self._tool_groups_by_name.get(name)
|
||||||
|
|
||||||
|
|
||||||
# Compatibility singleton layer for code paths that have not yet been
|
# Compatibility singleton layer for code paths that have not yet been
|
||||||
|
|||||||
67
backend/tests/test_app_config_name_indexes.py
Normal file
67
backend/tests/test_app_config_name_indexes.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
"""Tests for AppConfig's name-indexed get_*_config lookups.
|
||||||
|
|
||||||
|
``get_model_config`` / ``get_tool_config`` / ``get_tool_group_config`` are
|
||||||
|
served from name -> config dicts built once after validation, instead of an
|
||||||
|
O(n) ``next(...)`` scan per call. These tests lock the indexed lookups to the
|
||||||
|
exact semantics of the linear scan they replaced (match, miss -> None,
|
||||||
|
first-match-wins on duplicate names) and confirm a config reload rebuilds them.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from deerflow.config.app_config import AppConfig
|
||||||
|
|
||||||
|
|
||||||
|
def _build(model_names=(), tool_names=(), group_names=()):
|
||||||
|
return AppConfig.model_validate(
|
||||||
|
{
|
||||||
|
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||||
|
"models": [{"name": n, "use": "pkg:Cls", "model": n} for n in model_names],
|
||||||
|
"tools": [{"name": n, "group": "default", "use": "pkg:fn"} for n in tool_names],
|
||||||
|
"tool_groups": [{"name": n} for n in group_names],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_config_returns_matching_entry():
|
||||||
|
cfg = _build(model_names=["m1", "m2"], tool_names=["t1", "t2"], group_names=["g1"])
|
||||||
|
assert cfg.get_model_config("m2").name == "m2"
|
||||||
|
assert cfg.get_tool_config("t1").name == "t1"
|
||||||
|
assert cfg.get_tool_group_config("g1").name == "g1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_config_returns_none_for_missing():
|
||||||
|
cfg = _build(model_names=["m1"], tool_names=["t1"], group_names=["g1"])
|
||||||
|
assert cfg.get_model_config("nope") is None
|
||||||
|
assert cfg.get_tool_config("nope") is None
|
||||||
|
assert cfg.get_tool_group_config("nope") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_config_first_match_wins_on_duplicate_names():
|
||||||
|
# Two models share a name; the index must return the FIRST, matching the
|
||||||
|
# previous next(...) scan.
|
||||||
|
cfg = AppConfig.model_validate(
|
||||||
|
{
|
||||||
|
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||||
|
"models": [
|
||||||
|
{"name": "dup", "use": "pkg:A", "model": "first"},
|
||||||
|
{"name": "dup", "use": "pkg:B", "model": "second"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert cfg.get_model_config("dup").model == "first"
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_matches_linear_scan_reference():
|
||||||
|
cfg = _build(model_names=["a", "b", "c"], tool_names=["x", "y"], group_names=["g"])
|
||||||
|
for n in ["a", "b", "c", "missing"]:
|
||||||
|
assert cfg.get_model_config(n) == next((m for m in cfg.models if m.name == n), None)
|
||||||
|
for n in ["x", "y", "missing"]:
|
||||||
|
assert cfg.get_tool_config(n) == next((t for t in cfg.tools if t.name == n), None)
|
||||||
|
for n in ["g", "missing"]:
|
||||||
|
assert cfg.get_tool_group_config(n) == next((grp for grp in cfg.tool_groups if grp.name == n), None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_config_lookups_return_none():
|
||||||
|
cfg = _build()
|
||||||
|
assert cfg.get_model_config("anything") is None
|
||||||
|
assert cfg.get_tool_config("anything") is None
|
||||||
|
assert cfg.get_tool_group_config("anything") is None
|
||||||
Loading…
x
Reference in New Issue
Block a user