mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-13 07:28:44 +00:00
fix(scripts): detect the ollama extra from configured models (#5318)
* fix(scripts): detect the ollama extra from configured models make dev synced without --extra ollama, uninstalling langchain-ollama from a working setup. Same failure #2754 hit with postgres; ollama predates the detector added in #2767 and never got a rule. * fix(deps): declare the ollama extra on backend, pin model use: matching Review follow-ups. `ollama` was the only extra in the detector's map that the root `backend` project did not declare, so any consumer syncing without `--all-packages` failed outright: $ cd backend && uv sync --locked --extra ollama error: Extra `ollama` is not defined in the project's `optional-dependencies` table `serve.sh` and `docker/dev-entrypoint.sh` both pass `--all-packages` and were unaffected, but `backend/Dockerfile` does not, and `config.example.yaml` documents `UV_EXTRAS` as an image build-arg — so `UV_EXTRAS=ollama docker compose build` would have hard-failed on a value this branch makes first-class. Declaring `ollama = ["deerflow-harness[ollama]"]` alongside the other delegating extras closes that, and `uv.lock` is regenerated to match. The `use:` match also accepted any nesting depth inside `models:`, so a `use` in a sub-mapping was read as the model's provider: models: - name: doubao use: deerflow.models.patched_deepseek:PatchedChatDeepSeek when_thinking_enabled: use: langchain_ollama:ChatOllama That yielded `--extra ollama` despite the model's own provider pointing elsewhere, and `when_thinking_enabled` appears fifteen times in config.example.yaml, so the shape is common rather than contrived. Pin matching to the list item's own key indent, mirroring how `section_value()` pins `child_indent` and documents deeper nesting as ignored on purpose. Covered by a regression test that fails on the looser parser. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> * chore(deps): drop unrelated standard-aifc marker churn from uv.lock Review follow-up. Regenerating the lock with a newer local uv (0.11.19) also rewrote the `standard-aifc` entry, adding `python_full_version >= '3.13'` markers to its `audioop-lts` and `standard-chunk` dependencies. Unrelated to this change, so restore upstream's lines and keep the lock diff to the `ollama` extra. `uv lock --check` passes with these lines under both the CI- and Dockerfile-pinned uv 0.11.1 and uv 0.11.19. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> * fix(scripts): detect ollama in unindented model lists and after nested sequences Review follow-ups on the models parser. `yaml.safe_dump` — used by the setup wizard (`make setup`) and scripts/config-upgrade.sh — writes list items unindented: models: - name: qwen3-local use: langchain_ollama:ChatOllama The parser treated any column-0 line as the end of the `models:` section, so the first model ended it and nothing was detected. That is the layout new users get from the recommended setup path, so `make dev` still synced without `--extra ollama`. Model entries are now recognised before the section-end test, the same ordering `tools_include_name()` already uses for the unindented tools list (#4367). Separately, every sequence item reset the key indent, including items inside a model option. With `stop:` / `- END` before `use:`, the key indent jumped to the nested item's and the model's own `use` was skipped, so detection depended on key order within the model. The first sequence item under `models:` now fixes the model-list indent; only items at that indent start a model and set where its keys sit. Nested sequence items and deeper mappings are skipped without moving it. Regression tests cover the real setup-wizard output via `build_minimal_config()`, a hand-written unindented list, and `use:` after a nested `stop:` list — all three fail on the previous parser — plus a guard that a `- use:` nested inside a model option is still ignored. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Parthiban Sivakumar
GitHub
parent
e92c2c62f7
commit
a2011996d8
@ -47,6 +47,7 @@ redis = ["deerflow-harness[redis]"]
|
||||
discord = ["discord.py>=2.7.0"]
|
||||
buzz = ["coincurve>=20.0.0"]
|
||||
monocle = ["deerflow-harness[monocle]"]
|
||||
ollama = ["deerflow-harness[ollama]"]
|
||||
browser = ["deerflow-harness[browser]"]
|
||||
memory-zh = ["deerflow-harness[memory-zh]"]
|
||||
|
||||
|
||||
@ -208,6 +208,102 @@ def test_detect_from_config_memory_stream_bridge_returns_no_extras(tmp_path):
|
||||
assert detect.detect_from_config(cfg) == []
|
||||
|
||||
|
||||
def test_detect_from_config_ollama_via_model_use(tmp_path):
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
"models:\n - name: qwen3-local\n use: langchain_ollama:ChatOllama\n model: qwen3:32b\n base_url: http://localhost:11434\n",
|
||||
)
|
||||
assert detect.detect_from_config(cfg) == ["ollama"]
|
||||
|
||||
|
||||
def test_detect_from_config_ollama_when_use_is_the_first_key(tmp_path):
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("models:\n - use: langchain_ollama:ChatOllama\n name: qwen3-local\n")
|
||||
assert detect.detect_from_config(cfg) == ["ollama"]
|
||||
|
||||
|
||||
def test_detect_from_config_ignores_commented_ollama_block(tmp_path):
|
||||
"""config.example.yaml ships the Ollama models fully commented out."""
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
"models:\n # - name: qwen3-local\n # use: langchain_ollama:ChatOllama\n # base_url: http://localhost:11434\n",
|
||||
)
|
||||
assert detect.detect_from_config(cfg) == []
|
||||
|
||||
|
||||
def test_detect_from_config_ignores_use_in_nested_model_mapping(tmp_path):
|
||||
"""A `use` inside a sub-mapping is not the model's own provider.
|
||||
|
||||
`when_thinking_enabled` / `when_thinking_disabled` blocks are common in
|
||||
config.example.yaml, so matching `use:` at any depth would misread them.
|
||||
"""
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
"models:\n - name: doubao\n use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n when_thinking_enabled:\n use: langchain_ollama:ChatOllama\n",
|
||||
)
|
||||
assert detect.detect_from_config(cfg) == []
|
||||
|
||||
|
||||
def test_detect_from_config_ollama_in_setup_wizard_output(tmp_path):
|
||||
"""`make setup` writes config.yaml with yaml.safe_dump, which leaves list items unindented."""
|
||||
from wizard.writer import build_minimal_config
|
||||
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
build_minimal_config(
|
||||
provider_use="langchain_ollama:ChatOllama",
|
||||
model_name="qwen3:32b",
|
||||
display_name="Qwen3 32B (Ollama)",
|
||||
api_key_field="api_key",
|
||||
env_var=None,
|
||||
base_url="http://localhost:11434",
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Pin the layout under test, so a writer change fails here rather than silently passing.
|
||||
assert "\nmodels:\n- " in cfg.read_text(encoding="utf-8")
|
||||
assert "ollama" in detect.detect_from_config(cfg)
|
||||
|
||||
|
||||
def test_detect_from_config_ollama_via_indentless_models_list(tmp_path):
|
||||
"""Same shape as the setup wizard, and as scripts/config-upgrade.sh emits."""
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("models:\n- name: qwen3-local\n use: langchain_ollama:ChatOllama\n model: qwen3:32b\n")
|
||||
assert detect.detect_from_config(cfg) == ["ollama"]
|
||||
|
||||
|
||||
def test_detect_from_config_ollama_when_use_follows_a_nested_list(tmp_path):
|
||||
"""A sequence inside a model option must not move the key indent; key order is irrelevant."""
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
"models:\n - name: local\n model: qwen3:32b\n stop:\n - END\n use: langchain_ollama:ChatOllama\n",
|
||||
)
|
||||
assert detect.detect_from_config(cfg) == ["ollama"]
|
||||
|
||||
|
||||
def test_detect_from_config_ignores_use_in_nested_indentless_sequence(tmp_path):
|
||||
"""A `- use:` item inside a model option is not that model's provider."""
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
"models:\n- name: x\n use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n fallbacks:\n - use: langchain_ollama:ChatOllama\n",
|
||||
)
|
||||
assert detect.detect_from_config(cfg) == []
|
||||
|
||||
|
||||
def test_detect_from_config_non_ollama_model_returns_no_extras(tmp_path):
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("models:\n - name: gpt\n use: langchain_openai:ChatOpenAI\n model: gpt-4o\n")
|
||||
assert detect.detect_from_config(cfg) == []
|
||||
|
||||
|
||||
def test_detect_from_config_combines_ollama_and_postgres(tmp_path):
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text(
|
||||
"database:\n backend: postgres\nmodels:\n - name: qwen3-local\n use: langchain_ollama:ChatOllama\n",
|
||||
)
|
||||
assert detect.detect_from_config(cfg) == ["ollama", "postgres"]
|
||||
|
||||
|
||||
def test_detect_from_config_combines_postgres_and_redis(tmp_path):
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("database:\n backend: postgres\nstream_bridge:\n type: redis\n")
|
||||
|
||||
6
backend/uv.lock
generated
6
backend/uv.lock
generated
@ -843,6 +843,9 @@ memory-zh = [
|
||||
monocle = [
|
||||
{ name = "deerflow-harness", extra = ["monocle"] },
|
||||
]
|
||||
ollama = [
|
||||
{ name = "deerflow-harness", extra = ["ollama"] },
|
||||
]
|
||||
postgres = [
|
||||
{ name = "deerflow-harness", extra = ["postgres"] },
|
||||
]
|
||||
@ -874,6 +877,7 @@ requires-dist = [
|
||||
{ name = "deerflow-harness", extras = ["browser"], marker = "extra == 'browser'", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["memory-zh"], marker = "extra == 'memory-zh'", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["monocle"], marker = "extra == 'monocle'", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["ollama"], marker = "extra == 'ollama'", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["postgres"], marker = "extra == 'postgres'", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["redis"], marker = "extra == 'redis'", editable = "packages/harness" },
|
||||
{ name = "dingtalk-stream", specifier = ">=0.24.3" },
|
||||
@ -894,7 +898,7 @@ requires-dist = [
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" },
|
||||
{ name = "wecom-aibot-python-sdk", specifier = ">=0.1.6" },
|
||||
]
|
||||
provides-extras = ["postgres", "redis", "discord", "buzz", "monocle", "browser", "memory-zh"]
|
||||
provides-extras = ["postgres", "redis", "discord", "buzz", "monocle", "ollama", "browser", "memory-zh"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
|
||||
@ -14,6 +14,7 @@ Order of resolution:
|
||||
- tools[].name == browser_navigate -> browser
|
||||
- sandbox.ownership.type == redis -> redis
|
||||
- channels.buzz.enabled == true -> buzz
|
||||
- models[].use == langchain_ollama:* -> ollama
|
||||
3. Runtime environment toggles that enable optional backends:
|
||||
- DEER_FLOW_STREAM_BRIDGE_REDIS_URL -> redis
|
||||
- DEER_FLOW_SANDBOX_OWNERSHIP_REDIS_URL -> redis
|
||||
@ -79,6 +80,21 @@ _SECTION_RE = re.compile(r"^([A-Za-z_][\w-]*)\s*:\s*$")
|
||||
_INDENTED_SECTION_RE = re.compile(r"^\s+([A-Za-z_][\w-]*)\s*:\s*$")
|
||||
_KEY_RE = re.compile(r"^\s+([A-Za-z_][\w-]*)\s*:\s*(\S.*?)\s*$")
|
||||
_LIST_ITEM_NAME_RE = re.compile(r"^\s*-\s+name\s*:\s*(\S.*?)\s*$")
|
||||
# `use:` on a models list item, whether it is the first key (`- use: X`) or a
|
||||
# later one (` use: X`). Leading whitespace is optional because
|
||||
# `yaml.safe_dump` (the setup wizard, config-upgrade.sh) writes list items
|
||||
# unindented. The caller pins matching to the model's own key indent, so a
|
||||
# `use` nested in a sub-mapping (e.g. `when_thinking_enabled`) is not mistaken
|
||||
# for the model's provider.
|
||||
_MODEL_USE_RE = re.compile(r"^\s*(?:-\s+)?use\s*:\s*(\S.*?)\s*$")
|
||||
# A sequence item: group 1 is the dash's indent, group 2 the dash plus the
|
||||
# spaces before the item's first key, so their combined length is where that
|
||||
# item's keys sit.
|
||||
_LIST_ITEM_RE = re.compile(r"^(\s*)(-\s+)\S")
|
||||
|
||||
# Provider module (the part before `:` in `models[].use`) -> uv extra that
|
||||
# ships it. Mirrors `[project.optional-dependencies]` in the harness package.
|
||||
_PROVIDER_EXTRAS = {"langchain_ollama": "ollama"}
|
||||
|
||||
|
||||
def _strip_comment(line: str) -> str:
|
||||
@ -251,6 +267,60 @@ def tools_include_name(lines: list[str], tool_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def models_use_providers(lines: list[str]) -> set[str]:
|
||||
"""Return provider modules referenced by `models[].use`.
|
||||
|
||||
Only each model's own `use` counts. The first sequence item under `models:`
|
||||
fixes the indent of the model list; later items at that indent start a new
|
||||
model and set where its keys sit. That handles both the indented layout in
|
||||
config.example.yaml and the unindented one `yaml.safe_dump` emits. Deeper
|
||||
content — a sub-mapping such as `when_thinking_enabled`, or a sequence
|
||||
inside a model option such as `stop:` — is skipped and never moves the key
|
||||
indent, so key order within a model does not change the result.
|
||||
|
||||
Commented-out example blocks are dropped by ``_strip_comment`` before
|
||||
matching, which keeps the fully-commented `models:` section shipped in
|
||||
config.example.yaml from enabling an extra.
|
||||
"""
|
||||
inside = False
|
||||
item_indent: int | None = None
|
||||
key_indent: int | None = None
|
||||
providers: set[str] = set()
|
||||
for raw in lines:
|
||||
line = _strip_comment(raw)
|
||||
if not line.strip():
|
||||
continue
|
||||
sect_match = _SECTION_RE.match(line)
|
||||
if sect_match:
|
||||
inside = sect_match.group(1) == "models"
|
||||
item_indent = key_indent = None
|
||||
continue
|
||||
if not inside:
|
||||
continue
|
||||
stripped = line.lstrip()
|
||||
indent = len(line) - len(stripped)
|
||||
item_match = _LIST_ITEM_RE.match(line)
|
||||
if item_match and (item_indent is None or len(item_match.group(1)) == item_indent):
|
||||
# A model entry. Checked before the section-end test below because
|
||||
# `yaml.safe_dump` puts these at column 0.
|
||||
item_indent = len(item_match.group(1))
|
||||
key_indent = item_indent + len(item_match.group(2))
|
||||
elif indent == 0 or (item_indent is not None and indent <= item_indent):
|
||||
# A new top-level key, or a dedent past the model list.
|
||||
inside = False
|
||||
item_indent = key_indent = None
|
||||
continue
|
||||
elif item_match or indent != key_indent:
|
||||
# A sequence item inside a model option, or content nested deeper
|
||||
# than the model's own keys. Neither is the model's provider.
|
||||
continue
|
||||
use_match = _MODEL_USE_RE.match(line)
|
||||
if use_match:
|
||||
target = _unquote(use_match.group(1).strip())
|
||||
providers.add(target.split(":", 1)[0].split(".", 1)[0])
|
||||
return providers
|
||||
|
||||
|
||||
def detect_from_config(path: Path) -> list[str]:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
@ -272,6 +342,10 @@ def detect_from_config(path: Path) -> list[str]:
|
||||
extras.add("buzz")
|
||||
if tools_include_name(lines, "browser_navigate"):
|
||||
extras.add("browser")
|
||||
for provider in models_use_providers(lines):
|
||||
extra = _PROVIDER_EXTRAS.get(provider)
|
||||
if extra is not None:
|
||||
extras.add(extra)
|
||||
return sorted(extras)
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user