mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-13 07:28:44 +00:00
feat(tools): filter list_uploaded_files by name and extension (#5341)
* feat(tools): filter list_uploaded_files by name and extension Add optional query and extensions so historical upload discovery can find older matching files instead of dropping them behind the default 20-item mtime cap. Fixes #5339 * fix(tools): strip glob stars from list_uploaded_files extensions Model-supplied tokens like *.pdf were prefixed to .*.pdf and never matched Path.suffix. Also run ruff format so the backend format gate passes.
This commit is contained in:
parent
e20b36b3c7
commit
9f17bbeec7
@ -137,7 +137,8 @@ To work with these files:
|
||||
```
|
||||
|
||||
以前轮次上传的文件不会在每次请求中重复注入。Agent 可按需调用
|
||||
`list_uploaded_files` 查询历史上传;如果已知文件名,也可直接使用
|
||||
`list_uploaded_files` 查询历史上传(可选 `query` 按文件名子串过滤、
|
||||
`extensions` 按类型过滤;过滤发生在默认 20 条上限之前)。如果已知文件名,也可直接使用
|
||||
`read_file` 或 `grep` 访问 `/mnt/user-data/uploads/` 下的文件。
|
||||
|
||||
### 使用上传的文件
|
||||
@ -248,7 +249,7 @@ backend/.deer-flow/threads/
|
||||
2. **Uploads Middleware** (`packages/harness/deerflow/agents/middlewares/uploads_middleware.py`)
|
||||
- 读取当前消息的 `additional_kwargs.files`
|
||||
- 在 Agent 请求前生成并注入 `<current_uploads>` 文件上下文
|
||||
- 历史上传由 `list_uploaded_files` 按需查询,不会每轮自动注入
|
||||
- 历史上传由 `list_uploaded_files` 按需查询(可按文件名/扩展名过滤后再截断),不会每轮自动注入
|
||||
|
||||
3. **Nginx 配置** (`nginx.conf`)
|
||||
- 路由上传请求到 Gateway API
|
||||
|
||||
@ -63,11 +63,57 @@ def _resolve_user_id(runtime: Runtime) -> str:
|
||||
return resolve_runtime_user_id(runtime) or get_effective_user_id()
|
||||
|
||||
|
||||
def _normalize_query(query: str | None) -> str | None:
|
||||
"""Return a stripped query, or None when filtering should be skipped."""
|
||||
if not isinstance(query, str):
|
||||
return None
|
||||
stripped = query.strip()
|
||||
return stripped or None
|
||||
|
||||
|
||||
def _normalize_extensions(extensions: list[str] | None) -> frozenset[str] | None:
|
||||
"""Normalize extension tokens to lowercase dotted suffixes.
|
||||
|
||||
Non-strings, blanks, and a non-list input are dropped. A leading ``*`` is
|
||||
stripped so model-supplied glob tokens like ``*.pdf`` still match
|
||||
``Path.suffix``. An empty result means "no extension filter", matching
|
||||
the unfiltered historical behavior.
|
||||
"""
|
||||
if not isinstance(extensions, list):
|
||||
return None
|
||||
normalized: set[str] = set()
|
||||
for item in extensions:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
token = item.strip().lower().lstrip("*")
|
||||
if not token:
|
||||
continue
|
||||
if not token.startswith("."):
|
||||
token = f".{token}"
|
||||
normalized.add(token)
|
||||
return frozenset(normalized) or None
|
||||
|
||||
|
||||
def _matches_filters(
|
||||
filename: str,
|
||||
suffix: str,
|
||||
query: str | None,
|
||||
extensions: frozenset[str] | None,
|
||||
) -> bool:
|
||||
if extensions is not None and suffix.lower() not in extensions:
|
||||
return False
|
||||
if query is not None and query.casefold() not in filename.casefold():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _list_uploaded_files_impl(
|
||||
include_outline: bool | list[str] = False,
|
||||
max_results: int = _DEFAULT_MAX_RESULTS,
|
||||
runtime: Runtime | None = None,
|
||||
*,
|
||||
query: str | None = None,
|
||||
extensions: list[str] | None = None,
|
||||
_paths: Any | None = None,
|
||||
) -> dict:
|
||||
"""Core implementation — testable without the @tool wrapper."""
|
||||
@ -140,6 +186,17 @@ def _list_uploaded_files_impl(
|
||||
if not candidates:
|
||||
return {"files": [], "message": "No historical uploaded files in this thread."}
|
||||
|
||||
query_filter = _normalize_query(query)
|
||||
extension_filter = _normalize_extensions(extensions)
|
||||
if query_filter is not None or extension_filter is not None:
|
||||
candidates = [item for item in candidates if _matches_filters(item[1].name, item[1].suffix, query_filter, extension_filter)]
|
||||
if not candidates:
|
||||
return {
|
||||
"files": [],
|
||||
"total_count": 0,
|
||||
"message": "No uploaded files matched the given filters.",
|
||||
}
|
||||
|
||||
# Sort by mtime descending (most recent first)
|
||||
candidates.sort(key=lambda item: item[0], reverse=True)
|
||||
|
||||
@ -199,6 +256,14 @@ def list_uploaded_files(
|
||||
int,
|
||||
"Maximum number of files to return (default 20, max 100).",
|
||||
] = _DEFAULT_MAX_RESULTS,
|
||||
query: Annotated[
|
||||
str | None,
|
||||
"Optional case-insensitive substring to match against the filename only (not the virtual path). Omit or leave blank to skip name filtering.",
|
||||
] = None,
|
||||
extensions: Annotated[
|
||||
list[str] | None,
|
||||
'Optional file extensions to keep, e.g. ["pdf", ".PNG"]. With or without a leading dot; matching is case-insensitive. Combined with query using AND. Omit to skip type filtering.',
|
||||
] = None,
|
||||
) -> dict:
|
||||
"""Discover historical uploaded files available in this thread.
|
||||
|
||||
@ -213,9 +278,14 @@ def list_uploaded_files(
|
||||
Skip this tool when:
|
||||
- The user names a specific file — use read_file or grep directly with the path
|
||||
- The file was uploaded in the current run — it's already in <current_uploads>
|
||||
|
||||
Optional filters (`query`, `extensions`) run before the max_results cap, so
|
||||
older matching files are not displaced by newer unrelated uploads.
|
||||
"""
|
||||
return _list_uploaded_files_impl(
|
||||
include_outline=include_outline,
|
||||
max_results=max_results,
|
||||
runtime=runtime,
|
||||
query=query,
|
||||
extensions=extensions,
|
||||
)
|
||||
|
||||
@ -842,6 +842,126 @@ def test_all_string_fields_in_result_are_neutralized(tmp_path):
|
||||
assert result["total_count"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# query / extensions filters
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestListUploadedFilesFilters:
|
||||
"""Filters must run before truncation so 'those PDFs' remain reachable."""
|
||||
|
||||
def test_unfiltered_call_is_unchanged(self, tmp_path):
|
||||
uploads_dir = _uploads_dir(tmp_path)
|
||||
for i in range(25):
|
||||
p = uploads_dir / f"file_{i:02}.txt"
|
||||
p.write_text(f"content {i}", encoding="utf-8")
|
||||
os.utime(p, (i, i))
|
||||
|
||||
result = _list_uploaded_files_impl(max_results=10, runtime=_runtime(), _paths=_paths(tmp_path))
|
||||
|
||||
assert len(result["files"]) == 10
|
||||
assert result["total_count"] == 25
|
||||
assert result["truncated"] is True
|
||||
|
||||
def test_extensions_filter_before_truncation(self, tmp_path):
|
||||
uploads_dir = _uploads_dir(tmp_path)
|
||||
for i in range(25):
|
||||
p = uploads_dir / f"shot_{i:02}.png"
|
||||
p.write_bytes(b"png")
|
||||
os.utime(p, (100 + i, 100 + i))
|
||||
for i, name in enumerate(("old.pdf", "notes.PDF")):
|
||||
p = uploads_dir / name
|
||||
p.write_bytes(b"%PDF")
|
||||
os.utime(p, (i, i))
|
||||
|
||||
result = _list_uploaded_files_impl(
|
||||
max_results=10,
|
||||
runtime=_runtime(),
|
||||
extensions=["pdf", ".PNG"],
|
||||
_paths=_paths(tmp_path),
|
||||
)
|
||||
|
||||
pdf_only = _list_uploaded_files_impl(
|
||||
max_results=10,
|
||||
runtime=_runtime(),
|
||||
extensions=["pdf"],
|
||||
_paths=_paths(tmp_path),
|
||||
)
|
||||
|
||||
assert {f["filename"] for f in pdf_only["files"]} == {"old.pdf", "notes.PDF"}
|
||||
assert pdf_only["total_count"] == 2
|
||||
assert "truncated" not in pdf_only
|
||||
assert len(result["files"]) == 10
|
||||
assert result["total_count"] == 27
|
||||
assert result["truncated"] is True
|
||||
|
||||
def test_glob_star_extension_token_matches_suffix(self, tmp_path):
|
||||
# Models often emit glob-style tokens like "*.pdf". If we only prefix a
|
||||
# dot, that becomes ".*.pdf" and never matches Path.suffix, so the PDFs
|
||||
# the user asked for disappear behind "no files matched".
|
||||
uploads_dir = _uploads_dir(tmp_path)
|
||||
(uploads_dir / "old.pdf").write_bytes(b"%PDF")
|
||||
(uploads_dir / "notes.PDF").write_bytes(b"%PDF")
|
||||
(uploads_dir / "shot.png").write_bytes(b"png")
|
||||
|
||||
result = _list_uploaded_files_impl(
|
||||
runtime=_runtime(),
|
||||
extensions=["*.pdf", "*.PNG"],
|
||||
_paths=_paths(tmp_path),
|
||||
)
|
||||
|
||||
assert {f["filename"] for f in result["files"]} == {"old.pdf", "notes.PDF", "shot.png"}
|
||||
assert result["total_count"] == 3
|
||||
|
||||
def test_query_matches_filename_not_path(self, tmp_path):
|
||||
uploads_dir = _uploads_dir(tmp_path)
|
||||
(uploads_dir / "quarterly-report.pdf").write_bytes(b"%PDF")
|
||||
(uploads_dir / "notes.txt").write_text("report in body", encoding="utf-8")
|
||||
|
||||
result = _list_uploaded_files_impl(runtime=_runtime(), query="REPORT", _paths=_paths(tmp_path))
|
||||
uploads_query = _list_uploaded_files_impl(runtime=_runtime(), query="uploads", _paths=_paths(tmp_path))
|
||||
|
||||
assert [f["filename"] for f in result["files"]] == ["quarterly-report.pdf"]
|
||||
assert uploads_query["files"] == []
|
||||
assert uploads_query["total_count"] == 0
|
||||
assert uploads_query["message"] == "No uploaded files matched the given filters."
|
||||
assert "truncated" not in uploads_query
|
||||
|
||||
def test_query_and_extensions_are_and(self, tmp_path):
|
||||
uploads_dir = _uploads_dir(tmp_path)
|
||||
(uploads_dir / "report.pdf").write_bytes(b"%PDF")
|
||||
(uploads_dir / "report.txt").write_text("txt", encoding="utf-8")
|
||||
(uploads_dir / "other.pdf").write_bytes(b"%PDF")
|
||||
|
||||
result = _list_uploaded_files_impl(
|
||||
runtime=_runtime(),
|
||||
query="report",
|
||||
extensions=[".pdf"],
|
||||
_paths=_paths(tmp_path),
|
||||
)
|
||||
|
||||
assert [f["filename"] for f in result["files"]] == ["report.pdf"]
|
||||
assert result["total_count"] == 1
|
||||
|
||||
def test_blank_and_invalid_filters_are_noop(self, tmp_path):
|
||||
uploads_dir = _uploads_dir(tmp_path)
|
||||
(uploads_dir / "a.txt").write_text("a", encoding="utf-8")
|
||||
|
||||
result = _list_uploaded_files_impl(
|
||||
runtime=_runtime(),
|
||||
query=" ",
|
||||
extensions=["", " ", 1, None], # type: ignore[list-item]
|
||||
_paths=_paths(tmp_path),
|
||||
)
|
||||
|
||||
assert [f["filename"] for f in result["files"]] == ["a.txt"]
|
||||
assert result["message"] == "Found 1 historical file(s)."
|
||||
|
||||
def test_empty_directory_keeps_unfiltered_message(self, tmp_path):
|
||||
_uploads_dir(tmp_path)
|
||||
result = _list_uploaded_files_impl(runtime=_runtime(), query="pdf", extensions=[".pdf"], _paths=_paths(tmp_path))
|
||||
assert result["files"] == []
|
||||
assert "No historical uploaded files" in result["message"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# @tool schema — regression for #4375
|
||||
# ---------------------------------------------------------------------------
|
||||
@ -858,7 +978,7 @@ class TestToolSchema:
|
||||
def test_runtime_excluded_from_model_facing_args(self):
|
||||
from deerflow.tools.builtins.list_uploaded_files_tool import list_uploaded_files
|
||||
|
||||
assert set(list_uploaded_files.args) == {"include_outline", "max_results"}
|
||||
assert set(list_uploaded_files.args) == {"include_outline", "max_results", "query", "extensions"}
|
||||
assert "runtime" not in list_uploaded_files.args
|
||||
|
||||
def test_openai_schema_generation_succeeds(self):
|
||||
@ -869,4 +989,4 @@ class TestToolSchema:
|
||||
# This raised PydanticInvalidForJsonSchema before the fix.
|
||||
oai = convert_to_openai_tool(list_uploaded_files)
|
||||
params = oai["function"]["parameters"]["properties"]
|
||||
assert set(params) == {"include_outline", "max_results"}
|
||||
assert set(params) == {"include_outline", "max_results", "query", "extensions"}
|
||||
|
||||
@ -250,7 +250,7 @@ def test_list_uploaded_files_model_schema_excludes_injected_runtime() -> None:
|
||||
"""The model-facing schema must not expose ToolRuntime internals."""
|
||||
parameters = convert_to_openai_tool(list_uploaded_files)["function"]["parameters"]
|
||||
|
||||
assert set(parameters["properties"]) == {"include_outline", "max_results"}
|
||||
assert set(parameters["properties"]) == {"include_outline", "max_results", "query", "extensions"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_obj", [case[0] for case in _TOOL_CASES], ids=[case[0].name for case in _TOOL_CASES])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user