From 9f17bbeec74eb3c82a4016c0f7477bc2c9dc1066 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Sat, 12 Sep 2026 07:25:19 +0800 Subject: [PATCH] feat(tools): filter list_uploaded_files by name and extension (#5341) * feat(tools): filter list_uploaded_files by name and extension Add optional query and extensions so historical upload discovery can find older matching files instead of dropping them behind the default 20-item mtime cap. Fixes #5339 * fix(tools): strip glob stars from list_uploaded_files extensions Model-supplied tokens like *.pdf were prefixed to .*.pdf and never matched Path.suffix. Also run ruff format so the backend format gate passes. --- backend/docs/FILE_UPLOAD.md | 5 +- .../builtins/list_uploaded_files_tool.py | 70 ++++++++++ .../tests/test_list_uploaded_files_tool.py | 124 +++++++++++++++++- ...st_tool_args_schema_no_pydantic_warning.py | 2 +- 4 files changed, 196 insertions(+), 5 deletions(-) diff --git a/backend/docs/FILE_UPLOAD.md b/backend/docs/FILE_UPLOAD.md index 778b57210..dd79ca2f8 100644 --- a/backend/docs/FILE_UPLOAD.md +++ b/backend/docs/FILE_UPLOAD.md @@ -137,7 +137,8 @@ To work with these files: ``` 以前轮次上传的文件不会在每次请求中重复注入。Agent 可按需调用 -`list_uploaded_files` 查询历史上传；如果已知文件名，也可直接使用 +`list_uploaded_files` 查询历史上传（可选 `query` 按文件名子串过滤、 +`extensions` 按类型过滤；过滤发生在默认 20 条上限之前）。如果已知文件名，也可直接使用 `read_file` 或 `grep` 访问 `/mnt/user-data/uploads/` 下的文件。 ### 使用上传的文件 @@ -248,7 +249,7 @@ backend/.deer-flow/threads/ 2. **Uploads Middleware** (`packages/harness/deerflow/agents/middlewares/uploads_middleware.py`) - 读取当前消息的 `additional_kwargs.files` - 在 Agent 请求前生成并注入 `` 文件上下文 - - 历史上传由 `list_uploaded_files` 按需查询，不会每轮自动注入 + - 历史上传由 `list_uploaded_files` 按需查询（可按文件名/扩展名过滤后再截断），不会每轮自动注入 3. **Nginx 配置** (`nginx.conf`) - 路由上传请求到 Gateway API diff --git a/backend/packages/harness/deerflow/tools/builtins/list_uploaded_files_tool.py b/backend/packages/harness/deerflow/tools/builtins/list_uploaded_files_tool.py index c6212fc4d..a3ff64b1c 100644 --- a/backend/packages/harness/deerflow/tools/builtins/list_uploaded_files_tool.py +++ b/backend/packages/harness/deerflow/tools/builtins/list_uploaded_files_tool.py @@ -63,11 +63,57 @@ def _resolve_user_id(runtime: Runtime) -> str: return resolve_runtime_user_id(runtime) or get_effective_user_id() +def _normalize_query(query: str | None) -> str | None: + """Return a stripped query, or None when filtering should be skipped.""" + if not isinstance(query, str): + return None + stripped = query.strip() + return stripped or None + + +def _normalize_extensions(extensions: list[str] | None) -> frozenset[str] | None: + """Normalize extension tokens to lowercase dotted suffixes. + + Non-strings, blanks, and a non-list input are dropped. A leading ``*`` is + stripped so model-supplied glob tokens like ``*.pdf`` still match + ``Path.suffix``. An empty result means "no extension filter", matching + the unfiltered historical behavior. + """ + if not isinstance(extensions, list): + return None + normalized: set[str] = set() + for item in extensions: + if not isinstance(item, str): + continue + token = item.strip().lower().lstrip("*") + if not token: + continue + if not token.startswith("."): + token = f".{token}" + normalized.add(token) + return frozenset(normalized) or None + + +def _matches_filters( + filename: str, + suffix: str, + query: str | None, + extensions: frozenset[str] | None, +) -> bool: + if extensions is not None and suffix.lower() not in extensions: + return False + if query is not None and query.casefold() not in filename.casefold(): + return False + return True + + def _list_uploaded_files_impl( include_outline: bool | list[str] = False, max_results: int = _DEFAULT_MAX_RESULTS, runtime: Runtime | None = None, *, + query: str | None = None, + extensions: list[str] | None = None, _paths: Any | None = None, ) -> dict: """Core implementation — testable without the @tool wrapper.""" @@ -140,6 +186,17 @@ def _list_uploaded_files_impl( if not candidates: return {"files": [], "message": "No historical uploaded files in this thread."} + query_filter = _normalize_query(query) + extension_filter = _normalize_extensions(extensions) + if query_filter is not None or extension_filter is not None: + candidates = [item for item in candidates if _matches_filters(item[1].name, item[1].suffix, query_filter, extension_filter)] + if not candidates: + return { + "files": [], + "total_count": 0, + "message": "No uploaded files matched the given filters.", + } + # Sort by mtime descending (most recent first) candidates.sort(key=lambda item: item[0], reverse=True) @@ -199,6 +256,14 @@ def list_uploaded_files( int, "Maximum number of files to return (default 20, max 100).", ] = _DEFAULT_MAX_RESULTS, + query: Annotated[ + str | None, + "Optional case-insensitive substring to match against the filename only (not the virtual path). Omit or leave blank to skip name filtering.", + ] = None, + extensions: Annotated[ + list[str] | None, + 'Optional file extensions to keep, e.g. ["pdf", ".PNG"]. With or without a leading dot; matching is case-insensitive. Combined with query using AND. Omit to skip type filtering.', + ] = None, ) -> dict: """Discover historical uploaded files available in this thread. @@ -213,9 +278,14 @@ def list_uploaded_files( Skip this tool when: - The user names a specific file — use read_file or grep directly with the path - The file was uploaded in the current run — it's already in + + Optional filters (`query`, `extensions`) run before the max_results cap, so + older matching files are not displaced by newer unrelated uploads. """ return _list_uploaded_files_impl( include_outline=include_outline, max_results=max_results, runtime=runtime, + query=query, + extensions=extensions, ) diff --git a/backend/tests/test_list_uploaded_files_tool.py b/backend/tests/test_list_uploaded_files_tool.py index 4987e98b2..04fdb5f06 100644 --- a/backend/tests/test_list_uploaded_files_tool.py +++ b/backend/tests/test_list_uploaded_files_tool.py @@ -842,6 +842,126 @@ def test_all_string_fields_in_result_are_neutralized(tmp_path): assert result["total_count"] == 1 +# --------------------------------------------------------------------------- +# query / extensions filters +# --------------------------------------------------------------------------- +class TestListUploadedFilesFilters: + """Filters must run before truncation so 'those PDFs' remain reachable.""" + + def test_unfiltered_call_is_unchanged(self, tmp_path): + uploads_dir = _uploads_dir(tmp_path) + for i in range(25): + p = uploads_dir / f"file_{i:02}.txt" + p.write_text(f"content {i}", encoding="utf-8") + os.utime(p, (i, i)) + + result = _list_uploaded_files_impl(max_results=10, runtime=_runtime(), _paths=_paths(tmp_path)) + + assert len(result["files"]) == 10 + assert result["total_count"] == 25 + assert result["truncated"] is True + + def test_extensions_filter_before_truncation(self, tmp_path): + uploads_dir = _uploads_dir(tmp_path) + for i in range(25): + p = uploads_dir / f"shot_{i:02}.png" + p.write_bytes(b"png") + os.utime(p, (100 + i, 100 + i)) + for i, name in enumerate(("old.pdf", "notes.PDF")): + p = uploads_dir / name + p.write_bytes(b"%PDF") + os.utime(p, (i, i)) + + result = _list_uploaded_files_impl( + max_results=10, + runtime=_runtime(), + extensions=["pdf", ".PNG"], + _paths=_paths(tmp_path), + ) + + pdf_only = _list_uploaded_files_impl( + max_results=10, + runtime=_runtime(), + extensions=["pdf"], + _paths=_paths(tmp_path), + ) + + assert {f["filename"] for f in pdf_only["files"]} == {"old.pdf", "notes.PDF"} + assert pdf_only["total_count"] == 2 + assert "truncated" not in pdf_only + assert len(result["files"]) == 10 + assert result["total_count"] == 27 + assert result["truncated"] is True + + def test_glob_star_extension_token_matches_suffix(self, tmp_path): + # Models often emit glob-style tokens like "*.pdf". If we only prefix a + # dot, that becomes ".*.pdf" and never matches Path.suffix, so the PDFs + # the user asked for disappear behind "no files matched". + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "old.pdf").write_bytes(b"%PDF") + (uploads_dir / "notes.PDF").write_bytes(b"%PDF") + (uploads_dir / "shot.png").write_bytes(b"png") + + result = _list_uploaded_files_impl( + runtime=_runtime(), + extensions=["*.pdf", "*.PNG"], + _paths=_paths(tmp_path), + ) + + assert {f["filename"] for f in result["files"]} == {"old.pdf", "notes.PDF", "shot.png"} + assert result["total_count"] == 3 + + def test_query_matches_filename_not_path(self, tmp_path): + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "quarterly-report.pdf").write_bytes(b"%PDF") + (uploads_dir / "notes.txt").write_text("report in body", encoding="utf-8") + + result = _list_uploaded_files_impl(runtime=_runtime(), query="REPORT", _paths=_paths(tmp_path)) + uploads_query = _list_uploaded_files_impl(runtime=_runtime(), query="uploads", _paths=_paths(tmp_path)) + + assert [f["filename"] for f in result["files"]] == ["quarterly-report.pdf"] + assert uploads_query["files"] == [] + assert uploads_query["total_count"] == 0 + assert uploads_query["message"] == "No uploaded files matched the given filters." + assert "truncated" not in uploads_query + + def test_query_and_extensions_are_and(self, tmp_path): + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "report.pdf").write_bytes(b"%PDF") + (uploads_dir / "report.txt").write_text("txt", encoding="utf-8") + (uploads_dir / "other.pdf").write_bytes(b"%PDF") + + result = _list_uploaded_files_impl( + runtime=_runtime(), + query="report", + extensions=[".pdf"], + _paths=_paths(tmp_path), + ) + + assert [f["filename"] for f in result["files"]] == ["report.pdf"] + assert result["total_count"] == 1 + + def test_blank_and_invalid_filters_are_noop(self, tmp_path): + uploads_dir = _uploads_dir(tmp_path) + (uploads_dir / "a.txt").write_text("a", encoding="utf-8") + + result = _list_uploaded_files_impl( + runtime=_runtime(), + query=" ", + extensions=["", " ", 1, None], # type: ignore[list-item] + _paths=_paths(tmp_path), + ) + + assert [f["filename"] for f in result["files"]] == ["a.txt"] + assert result["message"] == "Found 1 historical file(s)." + + def test_empty_directory_keeps_unfiltered_message(self, tmp_path): + _uploads_dir(tmp_path) + result = _list_uploaded_files_impl(runtime=_runtime(), query="pdf", extensions=[".pdf"], _paths=_paths(tmp_path)) + assert result["files"] == [] + assert "No historical uploaded files" in result["message"] + + # --------------------------------------------------------------------------- # @tool schema — regression for #4375 # --------------------------------------------------------------------------- @@ -858,7 +978,7 @@ class TestToolSchema: def test_runtime_excluded_from_model_facing_args(self): from deerflow.tools.builtins.list_uploaded_files_tool import list_uploaded_files - assert set(list_uploaded_files.args) == {"include_outline", "max_results"} + assert set(list_uploaded_files.args) == {"include_outline", "max_results", "query", "extensions"} assert "runtime" not in list_uploaded_files.args def test_openai_schema_generation_succeeds(self): @@ -869,4 +989,4 @@ class TestToolSchema: # This raised PydanticInvalidForJsonSchema before the fix. oai = convert_to_openai_tool(list_uploaded_files) params = oai["function"]["parameters"]["properties"] - assert set(params) == {"include_outline", "max_results"} + assert set(params) == {"include_outline", "max_results", "query", "extensions"} diff --git a/backend/tests/test_tool_args_schema_no_pydantic_warning.py b/backend/tests/test_tool_args_schema_no_pydantic_warning.py index ff07b21a7..bfa485c8f 100644 --- a/backend/tests/test_tool_args_schema_no_pydantic_warning.py +++ b/backend/tests/test_tool_args_schema_no_pydantic_warning.py @@ -250,7 +250,7 @@ def test_list_uploaded_files_model_schema_excludes_injected_runtime() -> None: """The model-facing schema must not expose ToolRuntime internals.""" parameters = convert_to_openai_tool(list_uploaded_files)["function"]["parameters"] - assert set(parameters["properties"]) == {"include_outline", "max_results"} + assert set(parameters["properties"]) == {"include_outline", "max_results", "query", "extensions"} @pytest.mark.parametrize("tool_obj", [case[0] for case in _TOOL_CASES], ids=[case[0].name for case in _TOOL_CASES])