feat(tools): filter list_uploaded_files by name and extension (#5341)

* feat(tools): filter list_uploaded_files by name and extension

Add optional query and extensions so historical upload discovery can
find older matching files instead of dropping them behind the default
20-item mtime cap.

Fixes #5339

* fix(tools): strip glob stars from list_uploaded_files extensions

Model-supplied tokens like *.pdf were prefixed to .*.pdf and never
matched Path.suffix. Also run ruff format so the backend format gate
passes.
This commit is contained in:
wutongyuonce 2026-09-12 07:25:19 +08:00 committed by GitHub
parent e20b36b3c7
commit 9f17bbeec7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 196 additions and 5 deletions

View File

@ -137,7 +137,8 @@ To work with these files:
```
以前轮次上传的文件不会在每次请求中重复注入。Agent 可按需调用
`list_uploaded_files` 查询历史上传;如果已知文件名,也可直接使用
`list_uploaded_files` 查询历史上传(可选 `query` 按文件名子串过滤、
`extensions` 按类型过滤;过滤发生在默认 20 条上限之前)。如果已知文件名,也可直接使用
`read_file``grep` 访问 `/mnt/user-data/uploads/` 下的文件。
### 使用上传的文件
@ -248,7 +249,7 @@ backend/.deer-flow/threads/
2. **Uploads Middleware** (`packages/harness/deerflow/agents/middlewares/uploads_middleware.py`)
- 读取当前消息的 `additional_kwargs.files`
- 在 Agent 请求前生成并注入 `<current_uploads>` 文件上下文
- 历史上传由 `list_uploaded_files` 按需查询,不会每轮自动注入
- 历史上传由 `list_uploaded_files` 按需查询(可按文件名/扩展名过滤后再截断),不会每轮自动注入
3. **Nginx 配置** (`nginx.conf`)
- 路由上传请求到 Gateway API

View File

@ -63,11 +63,57 @@ def _resolve_user_id(runtime: Runtime) -> str:
return resolve_runtime_user_id(runtime) or get_effective_user_id()
def _normalize_query(query: str | None) -> str | None:
"""Return a stripped query, or None when filtering should be skipped."""
if not isinstance(query, str):
return None
stripped = query.strip()
return stripped or None
def _normalize_extensions(extensions: list[str] | None) -> frozenset[str] | None:
"""Normalize extension tokens to lowercase dotted suffixes.
Non-strings, blanks, and a non-list input are dropped. A leading ``*`` is
stripped so model-supplied glob tokens like ``*.pdf`` still match
``Path.suffix``. An empty result means "no extension filter", matching
the unfiltered historical behavior.
"""
if not isinstance(extensions, list):
return None
normalized: set[str] = set()
for item in extensions:
if not isinstance(item, str):
continue
token = item.strip().lower().lstrip("*")
if not token:
continue
if not token.startswith("."):
token = f".{token}"
normalized.add(token)
return frozenset(normalized) or None
def _matches_filters(
filename: str,
suffix: str,
query: str | None,
extensions: frozenset[str] | None,
) -> bool:
if extensions is not None and suffix.lower() not in extensions:
return False
if query is not None and query.casefold() not in filename.casefold():
return False
return True
def _list_uploaded_files_impl(
include_outline: bool | list[str] = False,
max_results: int = _DEFAULT_MAX_RESULTS,
runtime: Runtime | None = None,
*,
query: str | None = None,
extensions: list[str] | None = None,
_paths: Any | None = None,
) -> dict:
"""Core implementation — testable without the @tool wrapper."""
@ -140,6 +186,17 @@ def _list_uploaded_files_impl(
if not candidates:
return {"files": [], "message": "No historical uploaded files in this thread."}
query_filter = _normalize_query(query)
extension_filter = _normalize_extensions(extensions)
if query_filter is not None or extension_filter is not None:
candidates = [item for item in candidates if _matches_filters(item[1].name, item[1].suffix, query_filter, extension_filter)]
if not candidates:
return {
"files": [],
"total_count": 0,
"message": "No uploaded files matched the given filters.",
}
# Sort by mtime descending (most recent first)
candidates.sort(key=lambda item: item[0], reverse=True)
@ -199,6 +256,14 @@ def list_uploaded_files(
int,
"Maximum number of files to return (default 20, max 100).",
] = _DEFAULT_MAX_RESULTS,
query: Annotated[
str | None,
"Optional case-insensitive substring to match against the filename only (not the virtual path). Omit or leave blank to skip name filtering.",
] = None,
extensions: Annotated[
list[str] | None,
'Optional file extensions to keep, e.g. ["pdf", ".PNG"]. With or without a leading dot; matching is case-insensitive. Combined with query using AND. Omit to skip type filtering.',
] = None,
) -> dict:
"""Discover historical uploaded files available in this thread.
@ -213,9 +278,14 @@ def list_uploaded_files(
Skip this tool when:
- The user names a specific file use read_file or grep directly with the path
- The file was uploaded in the current run it's already in <current_uploads>
Optional filters (`query`, `extensions`) run before the max_results cap, so
older matching files are not displaced by newer unrelated uploads.
"""
return _list_uploaded_files_impl(
include_outline=include_outline,
max_results=max_results,
runtime=runtime,
query=query,
extensions=extensions,
)

View File

@ -842,6 +842,126 @@ def test_all_string_fields_in_result_are_neutralized(tmp_path):
assert result["total_count"] == 1
# ---------------------------------------------------------------------------
# query / extensions filters
# ---------------------------------------------------------------------------
class TestListUploadedFilesFilters:
"""Filters must run before truncation so 'those PDFs' remain reachable."""
def test_unfiltered_call_is_unchanged(self, tmp_path):
uploads_dir = _uploads_dir(tmp_path)
for i in range(25):
p = uploads_dir / f"file_{i:02}.txt"
p.write_text(f"content {i}", encoding="utf-8")
os.utime(p, (i, i))
result = _list_uploaded_files_impl(max_results=10, runtime=_runtime(), _paths=_paths(tmp_path))
assert len(result["files"]) == 10
assert result["total_count"] == 25
assert result["truncated"] is True
def test_extensions_filter_before_truncation(self, tmp_path):
uploads_dir = _uploads_dir(tmp_path)
for i in range(25):
p = uploads_dir / f"shot_{i:02}.png"
p.write_bytes(b"png")
os.utime(p, (100 + i, 100 + i))
for i, name in enumerate(("old.pdf", "notes.PDF")):
p = uploads_dir / name
p.write_bytes(b"%PDF")
os.utime(p, (i, i))
result = _list_uploaded_files_impl(
max_results=10,
runtime=_runtime(),
extensions=["pdf", ".PNG"],
_paths=_paths(tmp_path),
)
pdf_only = _list_uploaded_files_impl(
max_results=10,
runtime=_runtime(),
extensions=["pdf"],
_paths=_paths(tmp_path),
)
assert {f["filename"] for f in pdf_only["files"]} == {"old.pdf", "notes.PDF"}
assert pdf_only["total_count"] == 2
assert "truncated" not in pdf_only
assert len(result["files"]) == 10
assert result["total_count"] == 27
assert result["truncated"] is True
def test_glob_star_extension_token_matches_suffix(self, tmp_path):
# Models often emit glob-style tokens like "*.pdf". If we only prefix a
# dot, that becomes ".*.pdf" and never matches Path.suffix, so the PDFs
# the user asked for disappear behind "no files matched".
uploads_dir = _uploads_dir(tmp_path)
(uploads_dir / "old.pdf").write_bytes(b"%PDF")
(uploads_dir / "notes.PDF").write_bytes(b"%PDF")
(uploads_dir / "shot.png").write_bytes(b"png")
result = _list_uploaded_files_impl(
runtime=_runtime(),
extensions=["*.pdf", "*.PNG"],
_paths=_paths(tmp_path),
)
assert {f["filename"] for f in result["files"]} == {"old.pdf", "notes.PDF", "shot.png"}
assert result["total_count"] == 3
def test_query_matches_filename_not_path(self, tmp_path):
uploads_dir = _uploads_dir(tmp_path)
(uploads_dir / "quarterly-report.pdf").write_bytes(b"%PDF")
(uploads_dir / "notes.txt").write_text("report in body", encoding="utf-8")
result = _list_uploaded_files_impl(runtime=_runtime(), query="REPORT", _paths=_paths(tmp_path))
uploads_query = _list_uploaded_files_impl(runtime=_runtime(), query="uploads", _paths=_paths(tmp_path))
assert [f["filename"] for f in result["files"]] == ["quarterly-report.pdf"]
assert uploads_query["files"] == []
assert uploads_query["total_count"] == 0
assert uploads_query["message"] == "No uploaded files matched the given filters."
assert "truncated" not in uploads_query
def test_query_and_extensions_are_and(self, tmp_path):
uploads_dir = _uploads_dir(tmp_path)
(uploads_dir / "report.pdf").write_bytes(b"%PDF")
(uploads_dir / "report.txt").write_text("txt", encoding="utf-8")
(uploads_dir / "other.pdf").write_bytes(b"%PDF")
result = _list_uploaded_files_impl(
runtime=_runtime(),
query="report",
extensions=[".pdf"],
_paths=_paths(tmp_path),
)
assert [f["filename"] for f in result["files"]] == ["report.pdf"]
assert result["total_count"] == 1
def test_blank_and_invalid_filters_are_noop(self, tmp_path):
uploads_dir = _uploads_dir(tmp_path)
(uploads_dir / "a.txt").write_text("a", encoding="utf-8")
result = _list_uploaded_files_impl(
runtime=_runtime(),
query=" ",
extensions=["", " ", 1, None], # type: ignore[list-item]
_paths=_paths(tmp_path),
)
assert [f["filename"] for f in result["files"]] == ["a.txt"]
assert result["message"] == "Found 1 historical file(s)."
def test_empty_directory_keeps_unfiltered_message(self, tmp_path):
_uploads_dir(tmp_path)
result = _list_uploaded_files_impl(runtime=_runtime(), query="pdf", extensions=[".pdf"], _paths=_paths(tmp_path))
assert result["files"] == []
assert "No historical uploaded files" in result["message"]
# ---------------------------------------------------------------------------
# @tool schema — regression for #4375
# ---------------------------------------------------------------------------
@ -858,7 +978,7 @@ class TestToolSchema:
def test_runtime_excluded_from_model_facing_args(self):
from deerflow.tools.builtins.list_uploaded_files_tool import list_uploaded_files
assert set(list_uploaded_files.args) == {"include_outline", "max_results"}
assert set(list_uploaded_files.args) == {"include_outline", "max_results", "query", "extensions"}
assert "runtime" not in list_uploaded_files.args
def test_openai_schema_generation_succeeds(self):
@ -869,4 +989,4 @@ class TestToolSchema:
# This raised PydanticInvalidForJsonSchema before the fix.
oai = convert_to_openai_tool(list_uploaded_files)
params = oai["function"]["parameters"]["properties"]
assert set(params) == {"include_outline", "max_results"}
assert set(params) == {"include_outline", "max_results", "query", "extensions"}

View File

@ -250,7 +250,7 @@ def test_list_uploaded_files_model_schema_excludes_injected_runtime() -> None:
"""The model-facing schema must not expose ToolRuntime internals."""
parameters = convert_to_openai_tool(list_uploaded_files)["function"]["parameters"]
assert set(parameters["properties"]) == {"include_outline", "max_results"}
assert set(parameters["properties"]) == {"include_outline", "max_results", "query", "extensions"}
@pytest.mark.parametrize("tool_obj", [case[0] for case in _TOOL_CASES], ids=[case[0].name for case in _TOOL_CASES])