From a2011996d842b9374fe2786a75460bb67eecb7e2 Mon Sep 17 00:00:00 2001 From: Parthiban Sivakumar <228711334+parthiban-sivakumar@users.noreply.github.com> Date: Sat, 12 Sep 2026 06:40:47 +0530 Subject: [PATCH] fix(scripts): detect the ollama extra from configured models (#5318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(scripts): detect the ollama extra from configured models make dev synced without --extra ollama, uninstalling langchain-ollama from a working setup. Same failure #2754 hit with postgres; ollama predates the detector added in #2767 and never got a rule. * fix(deps): declare the ollama extra on backend, pin model use: matching Review follow-ups. `ollama` was the only extra in the detector's map that the root `backend` project did not declare, so any consumer syncing without `--all-packages` failed outright: $ cd backend && uv sync --locked --extra ollama error: Extra `ollama` is not defined in the project's `optional-dependencies` table `serve.sh` and `docker/dev-entrypoint.sh` both pass `--all-packages` and were unaffected, but `backend/Dockerfile` does not, and `config.example.yaml` documents `UV_EXTRAS` as an image build-arg — so `UV_EXTRAS=ollama docker compose build` would have hard-failed on a value this branch makes first-class. Declaring `ollama = ["deerflow-harness[ollama]"]` alongside the other delegating extras closes that, and `uv.lock` is regenerated to match. The `use:` match also accepted any nesting depth inside `models:`, so a `use` in a sub-mapping was read as the model's provider: models: - name: doubao use: deerflow.models.patched_deepseek:PatchedChatDeepSeek when_thinking_enabled: use: langchain_ollama:ChatOllama That yielded `--extra ollama` despite the model's own provider pointing elsewhere, and `when_thinking_enabled` appears fifteen times in config.example.yaml, so the shape is common rather than contrived. Pin matching to the list item's own key indent, mirroring how `section_value()` pins `child_indent` and documents deeper nesting as ignored on purpose. Covered by a regression test that fails on the looser parser. Co-Authored-By: Claude Opus 5 * chore(deps): drop unrelated standard-aifc marker churn from uv.lock Review follow-up. Regenerating the lock with a newer local uv (0.11.19) also rewrote the `standard-aifc` entry, adding `python_full_version >= '3.13'` markers to its `audioop-lts` and `standard-chunk` dependencies. Unrelated to this change, so restore upstream's lines and keep the lock diff to the `ollama` extra. `uv lock --check` passes with these lines under both the CI- and Dockerfile-pinned uv 0.11.1 and uv 0.11.19. Co-Authored-By: Claude Opus 5 * fix(scripts): detect ollama in unindented model lists and after nested sequences Review follow-ups on the models parser. `yaml.safe_dump` — used by the setup wizard (`make setup`) and scripts/config-upgrade.sh — writes list items unindented: models: - name: qwen3-local use: langchain_ollama:ChatOllama The parser treated any column-0 line as the end of the `models:` section, so the first model ended it and nothing was detected. That is the layout new users get from the recommended setup path, so `make dev` still synced without `--extra ollama`. Model entries are now recognised before the section-end test, the same ordering `tools_include_name()` already uses for the unindented tools list (#4367). Separately, every sequence item reset the key indent, including items inside a model option. With `stop:` / `- END` before `use:`, the key indent jumped to the nested item's and the model's own `use` was skipped, so detection depended on key order within the model. The first sequence item under `models:` now fixes the model-list indent; only items at that indent start a model and set where its keys sit. Nested sequence items and deeper mappings are skipped without moving it. Regression tests cover the real setup-wizard output via `build_minimal_config()`, a hand-written unindented list, and `use:` after a nested `stop:` list — all three fail on the previous parser — plus a guard that a `- use:` nested inside a model option is still ignored. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- backend/pyproject.toml | 1 + backend/tests/test_detect_uv_extras.py | 96 ++++++++++++++++++++++++++ backend/uv.lock | 6 +- scripts/detect_uv_extras.py | 74 ++++++++++++++++++++ 4 files changed, 176 insertions(+), 1 deletion(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1a8149a6b..291179bbc 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -47,6 +47,7 @@ redis = ["deerflow-harness[redis]"] discord = ["discord.py>=2.7.0"] buzz = ["coincurve>=20.0.0"] monocle = ["deerflow-harness[monocle]"] +ollama = ["deerflow-harness[ollama]"] browser = ["deerflow-harness[browser]"] memory-zh = ["deerflow-harness[memory-zh]"] diff --git a/backend/tests/test_detect_uv_extras.py b/backend/tests/test_detect_uv_extras.py index eb32f260b..668192e92 100644 --- a/backend/tests/test_detect_uv_extras.py +++ b/backend/tests/test_detect_uv_extras.py @@ -208,6 +208,102 @@ def test_detect_from_config_memory_stream_bridge_returns_no_extras(tmp_path): assert detect.detect_from_config(cfg) == [] +def test_detect_from_config_ollama_via_model_use(tmp_path): + cfg = tmp_path / "config.yaml" + cfg.write_text( + "models:\n - name: qwen3-local\n use: langchain_ollama:ChatOllama\n model: qwen3:32b\n base_url: http://localhost:11434\n", + ) + assert detect.detect_from_config(cfg) == ["ollama"] + + +def test_detect_from_config_ollama_when_use_is_the_first_key(tmp_path): + cfg = tmp_path / "config.yaml" + cfg.write_text("models:\n - use: langchain_ollama:ChatOllama\n name: qwen3-local\n") + assert detect.detect_from_config(cfg) == ["ollama"] + + +def test_detect_from_config_ignores_commented_ollama_block(tmp_path): + """config.example.yaml ships the Ollama models fully commented out.""" + cfg = tmp_path / "config.yaml" + cfg.write_text( + "models:\n # - name: qwen3-local\n # use: langchain_ollama:ChatOllama\n # base_url: http://localhost:11434\n", + ) + assert detect.detect_from_config(cfg) == [] + + +def test_detect_from_config_ignores_use_in_nested_model_mapping(tmp_path): + """A `use` inside a sub-mapping is not the model's own provider. + + `when_thinking_enabled` / `when_thinking_disabled` blocks are common in + config.example.yaml, so matching `use:` at any depth would misread them. + """ + cfg = tmp_path / "config.yaml" + cfg.write_text( + "models:\n - name: doubao\n use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n when_thinking_enabled:\n use: langchain_ollama:ChatOllama\n", + ) + assert detect.detect_from_config(cfg) == [] + + +def test_detect_from_config_ollama_in_setup_wizard_output(tmp_path): + """`make setup` writes config.yaml with yaml.safe_dump, which leaves list items unindented.""" + from wizard.writer import build_minimal_config + + cfg = tmp_path / "config.yaml" + cfg.write_text( + build_minimal_config( + provider_use="langchain_ollama:ChatOllama", + model_name="qwen3:32b", + display_name="Qwen3 32B (Ollama)", + api_key_field="api_key", + env_var=None, + base_url="http://localhost:11434", + ), + encoding="utf-8", + ) + # Pin the layout under test, so a writer change fails here rather than silently passing. + assert "\nmodels:\n- " in cfg.read_text(encoding="utf-8") + assert "ollama" in detect.detect_from_config(cfg) + + +def test_detect_from_config_ollama_via_indentless_models_list(tmp_path): + """Same shape as the setup wizard, and as scripts/config-upgrade.sh emits.""" + cfg = tmp_path / "config.yaml" + cfg.write_text("models:\n- name: qwen3-local\n use: langchain_ollama:ChatOllama\n model: qwen3:32b\n") + assert detect.detect_from_config(cfg) == ["ollama"] + + +def test_detect_from_config_ollama_when_use_follows_a_nested_list(tmp_path): + """A sequence inside a model option must not move the key indent; key order is irrelevant.""" + cfg = tmp_path / "config.yaml" + cfg.write_text( + "models:\n - name: local\n model: qwen3:32b\n stop:\n - END\n use: langchain_ollama:ChatOllama\n", + ) + assert detect.detect_from_config(cfg) == ["ollama"] + + +def test_detect_from_config_ignores_use_in_nested_indentless_sequence(tmp_path): + """A `- use:` item inside a model option is not that model's provider.""" + cfg = tmp_path / "config.yaml" + cfg.write_text( + "models:\n- name: x\n use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n fallbacks:\n - use: langchain_ollama:ChatOllama\n", + ) + assert detect.detect_from_config(cfg) == [] + + +def test_detect_from_config_non_ollama_model_returns_no_extras(tmp_path): + cfg = tmp_path / "config.yaml" + cfg.write_text("models:\n - name: gpt\n use: langchain_openai:ChatOpenAI\n model: gpt-4o\n") + assert detect.detect_from_config(cfg) == [] + + +def test_detect_from_config_combines_ollama_and_postgres(tmp_path): + cfg = tmp_path / "config.yaml" + cfg.write_text( + "database:\n backend: postgres\nmodels:\n - name: qwen3-local\n use: langchain_ollama:ChatOllama\n", + ) + assert detect.detect_from_config(cfg) == ["ollama", "postgres"] + + def test_detect_from_config_combines_postgres_and_redis(tmp_path): cfg = tmp_path / "config.yaml" cfg.write_text("database:\n backend: postgres\nstream_bridge:\n type: redis\n") diff --git a/backend/uv.lock b/backend/uv.lock index 44ad076f6..e3179ec07 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -843,6 +843,9 @@ memory-zh = [ monocle = [ { name = "deerflow-harness", extra = ["monocle"] }, ] +ollama = [ + { name = "deerflow-harness", extra = ["ollama"] }, +] postgres = [ { name = "deerflow-harness", extra = ["postgres"] }, ] @@ -874,6 +877,7 @@ requires-dist = [ { name = "deerflow-harness", extras = ["browser"], marker = "extra == 'browser'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["memory-zh"], marker = "extra == 'memory-zh'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["monocle"], marker = "extra == 'monocle'", editable = "packages/harness" }, + { name = "deerflow-harness", extras = ["ollama"], marker = "extra == 'ollama'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["postgres"], marker = "extra == 'postgres'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["redis"], marker = "extra == 'redis'", editable = "packages/harness" }, { name = "dingtalk-stream", specifier = ">=0.24.3" }, @@ -894,7 +898,7 @@ requires-dist = [ { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, { name = "wecom-aibot-python-sdk", specifier = ">=0.1.6" }, ] -provides-extras = ["postgres", "redis", "discord", "buzz", "monocle", "browser", "memory-zh"] +provides-extras = ["postgres", "redis", "discord", "buzz", "monocle", "ollama", "browser", "memory-zh"] [package.metadata.requires-dev] dev = [ diff --git a/scripts/detect_uv_extras.py b/scripts/detect_uv_extras.py index 9625e2d9f..7f71b9f14 100755 --- a/scripts/detect_uv_extras.py +++ b/scripts/detect_uv_extras.py @@ -14,6 +14,7 @@ Order of resolution: - tools[].name == browser_navigate -> browser - sandbox.ownership.type == redis -> redis - channels.buzz.enabled == true -> buzz + - models[].use == langchain_ollama:* -> ollama 3. Runtime environment toggles that enable optional backends: - DEER_FLOW_STREAM_BRIDGE_REDIS_URL -> redis - DEER_FLOW_SANDBOX_OWNERSHIP_REDIS_URL -> redis @@ -79,6 +80,21 @@ _SECTION_RE = re.compile(r"^([A-Za-z_][\w-]*)\s*:\s*$") _INDENTED_SECTION_RE = re.compile(r"^\s+([A-Za-z_][\w-]*)\s*:\s*$") _KEY_RE = re.compile(r"^\s+([A-Za-z_][\w-]*)\s*:\s*(\S.*?)\s*$") _LIST_ITEM_NAME_RE = re.compile(r"^\s*-\s+name\s*:\s*(\S.*?)\s*$") +# `use:` on a models list item, whether it is the first key (`- use: X`) or a +# later one (` use: X`). Leading whitespace is optional because +# `yaml.safe_dump` (the setup wizard, config-upgrade.sh) writes list items +# unindented. The caller pins matching to the model's own key indent, so a +# `use` nested in a sub-mapping (e.g. `when_thinking_enabled`) is not mistaken +# for the model's provider. +_MODEL_USE_RE = re.compile(r"^\s*(?:-\s+)?use\s*:\s*(\S.*?)\s*$") +# A sequence item: group 1 is the dash's indent, group 2 the dash plus the +# spaces before the item's first key, so their combined length is where that +# item's keys sit. +_LIST_ITEM_RE = re.compile(r"^(\s*)(-\s+)\S") + +# Provider module (the part before `:` in `models[].use`) -> uv extra that +# ships it. Mirrors `[project.optional-dependencies]` in the harness package. +_PROVIDER_EXTRAS = {"langchain_ollama": "ollama"} def _strip_comment(line: str) -> str: @@ -251,6 +267,60 @@ def tools_include_name(lines: list[str], tool_name: str) -> bool: return False +def models_use_providers(lines: list[str]) -> set[str]: + """Return provider modules referenced by `models[].use`. + + Only each model's own `use` counts. The first sequence item under `models:` + fixes the indent of the model list; later items at that indent start a new + model and set where its keys sit. That handles both the indented layout in + config.example.yaml and the unindented one `yaml.safe_dump` emits. Deeper + content — a sub-mapping such as `when_thinking_enabled`, or a sequence + inside a model option such as `stop:` — is skipped and never moves the key + indent, so key order within a model does not change the result. + + Commented-out example blocks are dropped by ``_strip_comment`` before + matching, which keeps the fully-commented `models:` section shipped in + config.example.yaml from enabling an extra. + """ + inside = False + item_indent: int | None = None + key_indent: int | None = None + providers: set[str] = set() + for raw in lines: + line = _strip_comment(raw) + if not line.strip(): + continue + sect_match = _SECTION_RE.match(line) + if sect_match: + inside = sect_match.group(1) == "models" + item_indent = key_indent = None + continue + if not inside: + continue + stripped = line.lstrip() + indent = len(line) - len(stripped) + item_match = _LIST_ITEM_RE.match(line) + if item_match and (item_indent is None or len(item_match.group(1)) == item_indent): + # A model entry. Checked before the section-end test below because + # `yaml.safe_dump` puts these at column 0. + item_indent = len(item_match.group(1)) + key_indent = item_indent + len(item_match.group(2)) + elif indent == 0 or (item_indent is not None and indent <= item_indent): + # A new top-level key, or a dedent past the model list. + inside = False + item_indent = key_indent = None + continue + elif item_match or indent != key_indent: + # A sequence item inside a model option, or content nested deeper + # than the model's own keys. Neither is the model's provider. + continue + use_match = _MODEL_USE_RE.match(line) + if use_match: + target = _unquote(use_match.group(1).strip()) + providers.add(target.split(":", 1)[0].split(".", 1)[0]) + return providers + + def detect_from_config(path: Path) -> list[str]: try: text = path.read_text(encoding="utf-8", errors="replace") @@ -272,6 +342,10 @@ def detect_from_config(path: Path) -> list[str]: extras.add("buzz") if tools_include_name(lines, "browser_navigate"): extras.add("browser") + for provider in models_use_providers(lines): + extra = _PROVIDER_EXTRAS.get(provider) + if extra is not None: + extras.add(extra) return sorted(extras)