mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
fix(storage): resolve single-module colon backend imports (#7017)
* fix(storage): resolve single-module colon backend imports * docs(storage): describe both backend import separators --------- Co-authored-by: lakers-abing <236384465+lakers-abing@users.noreply.github.com>
This commit is contained in:
parent
badfd7d3f1
commit
f23a411325
@ -464,6 +464,11 @@ frontend retain their own environment loading. Values they already export win.
|
||||
It does not select `ENV` profiles or isolate databases, storage or tenants.
|
||||
See [backend dotenv selection](backend/docs/CONFIGURATION.md#backend-dotenv-selection).
|
||||
|
||||
Custom blob storage backends accept `blob_storage.backend: module.Class` or
|
||||
`module:Class`, including a single `.py` module importable by the Gateway.
|
||||
The selected class must implement `BlobStore`; backend options go under
|
||||
`blob_storage.backend_config`. See [blob storage](backend/docs/blob-storage.md).
|
||||
|
||||
### Running the Application
|
||||
|
||||
#### Deployment Sizing
|
||||
|
||||
@ -43,7 +43,7 @@ This backend already delivers multi-instance resolution when `root` points at a
|
||||
```yaml
|
||||
blob_storage:
|
||||
enabled: true # default false
|
||||
backend: local_fs # folder name under storage/backends/, or a dotted import path
|
||||
backend: local_fs # folder name under storage/backends/, or module.Class / module:Class
|
||||
backend_config:
|
||||
root: /mnt/shared/deerflow-blobs # default: {runtime_home}/blobs, absolute
|
||||
```
|
||||
|
||||
@ -33,7 +33,8 @@ class BlobStorageConfig(BaseModel):
|
||||
description=(
|
||||
"Blob store backend selector. Either a registered backend name "
|
||||
"(matching a `storage/backends/<name>/` folder that exposes "
|
||||
"`STORE_CLASS`, e.g. `local_fs`) or a dotted import path to a "
|
||||
"`STORE_CLASS`, e.g. `local_fs`) or an import path (`module.Class` "
|
||||
"or `module:Class`, including single-file modules) to a "
|
||||
"`BlobStore` subclass. The factory resolves this at "
|
||||
"`get_blob_store()` time and raises `ValueError` on failure "
|
||||
"(fail-fast: blobs are persistent state, so an unresolved backend "
|
||||
|
||||
@ -61,8 +61,8 @@ backend name == `blob_storage.backend` value:
|
||||
3. Keep the portability rule: a backend talks to the host through the method
|
||||
arguments and its `backend_config` dict only. The single `deerflow` import a
|
||||
backend folder needs is the contract line importing `BlobStore`, so a
|
||||
third-party backend can live outside this tree and be selected with a dotted
|
||||
path (`pkg.mod.Class` or `pkg.mod:Class`).
|
||||
third-party backend can live outside this tree and be selected with an import
|
||||
path (`module.Class` or `module:Class`, including single-file modules).
|
||||
4. Reject an unusable config by raising; a store that silently starts on a
|
||||
different root or backend strands previously written content.
|
||||
5. Add the backend's semantics tests to `backend/tests/test_blob_store.py`; the
|
||||
|
||||
@ -3,7 +3,8 @@
|
||||
Mirrors ``deerflow/agents/memory/manager.py`` (the pattern maintainers pointed
|
||||
at for issue #4189 item 2): backends live in ``storage/backends/<name>/``, each
|
||||
package's ``__init__`` exposes ``STORE_CLASS``, and ``blob_storage.backend``
|
||||
names one of them -- or a dotted import path for a backend that lives elsewhere.
|
||||
names one of them -- or an import path (``module.Class`` or ``module:Class``)
|
||||
for a backend that lives elsewhere, including a single-file module.
|
||||
|
||||
Two accessors, so call sites cannot drift:
|
||||
|
||||
@ -62,18 +63,18 @@ def _resolve_store_class(backend: str) -> type[BlobStore]:
|
||||
|
||||
* a registered backend name -- a folder under ``storage/backends/`` whose
|
||||
``__init__`` exposes ``STORE_CLASS``;
|
||||
* a dotted import path -- ``pkg.mod.Class`` for a backend that lives
|
||||
outside this package (an extension or a deployment-local backend).
|
||||
* an import path -- ``module.Class`` or ``module:Class`` -- for a backend
|
||||
outside this package, including a single-file deployment-local module.
|
||||
"""
|
||||
global _backends_cache
|
||||
|
||||
if "." not in backend:
|
||||
if "." not in backend and ":" not in backend:
|
||||
if _backends_cache is None:
|
||||
_backends_cache = _scan_backends()
|
||||
cls = _backends_cache.get(backend)
|
||||
if cls is None:
|
||||
available = ", ".join(sorted(_backends_cache)) or "(none)"
|
||||
raise ValueError(f"Unknown blob store backend {backend!r}. Registered backends: {available}. Alternatively pass a dotted import path to a BlobStore subclass.")
|
||||
raise ValueError(f"Unknown blob store backend {backend!r}. Registered backends: {available}. Alternatively pass an import path (module.Class or module:Class) to a BlobStore subclass.")
|
||||
return cls
|
||||
|
||||
module_path, _, class_name = backend.partition(":") if ":" in backend else backend.rpartition(".")
|
||||
|
||||
@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
@ -449,6 +450,31 @@ def test_dotted_import_path_backend(tmp_path: Path):
|
||||
assert isinstance(get_blob_store(), LocalFsBlobStore)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("separator", [".", ":"])
|
||||
def test_single_module_import_path_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, separator: str):
|
||||
module_name = "deployment_blob_backend"
|
||||
(tmp_path / f"{module_name}.py").write_text(
|
||||
"from deerflow.storage.backends.local_fs import LocalFsBlobStore\n\nclass DeploymentBlobStore(LocalFsBlobStore):\n pass\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
bsc.set_blob_storage_config(
|
||||
bsc.BlobStorageConfig(
|
||||
enabled=True,
|
||||
backend=f"{module_name}{separator}DeploymentBlobStore",
|
||||
backend_config={"root": str(tmp_path / "blobs")},
|
||||
)
|
||||
)
|
||||
try:
|
||||
store = get_blob_store()
|
||||
assert type(store).__module__ == module_name
|
||||
ref = store.put_bytes(b"custom backend content", kind="tool-output")
|
||||
assert store.get_bytes(ref) == b"custom backend content"
|
||||
assert get_blob_store_if_enabled() is store
|
||||
finally:
|
||||
sys.modules.pop(module_name, None)
|
||||
|
||||
|
||||
def test_relative_root_is_resolved_against_runtime_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
import deerflow.storage.manager as mgr
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user