From 5951c89b5b8863ecf6aa54103cf83e210cd587ba Mon Sep 17 00:00:00 2001 From: Zeren Wang <53075619+Vanzeren@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:00:26 +0200 Subject: [PATCH] feat(projects): project workspaces with scoped chats and thread membership (#5265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(projects): project workspaces with scoped chats and thread membership Backend: - projects table model and migration; fail-closed ProjectRepository with ownership checks, CRUD/archive/restore/delete router, and atomic thread move between projects - threads_meta.project_id column exposed as reserved deerflow_project_id metadata; project-aware thread create/search with pagination bounds and membership echoed in create responses - first-run admission assigns the project only at genuine first run, seeded at write time and dropped when invalid; serialized against project deletion and thread assignment - branch creation inherits the source thread's project membership (an archived/deleted project degrades the branch to unassigned instead of failing the request) Frontend: - projects data layer, thread move API, and sidebar projects section with flat/grouped modes, archived-project threads, and stable virtual-list offsets - project detail page with project-scoped new chat (/workspace/chats/new?project=) and paginated thread list - move-to-project thread menu, new-project dialog, archived-project gates - project-scoped new chats pre-create the thread with membership before the first submit or /goal set, so runs never proceed outside the project - goal-set preparation is fenced against conversation switches: a stale continuation is dropped instead of saving the goal or launching the abandoned submission on the newly opened conversation - project thread lists join thread lifecycle invalidations (stop, pin) so an open project page never keeps stale titles, recency, or pagination * fix(chats): keep archive undo toast when the sidebar row unmounts The archive success toast was fired from per-mutate callbacks passed to mutation.mutate. React Query drops those handlers when the observer component unmounts before the mutation settles; archiving the open chat removes its sidebar row mid-flight, so the undo toast never appeared and the e2e archive-undo test timed out waiting for it. Move the success/error handlers to the mutation level (useArchiveThread options, same pattern as useMoveThreadToProject) where callbacks are delivered even after the originating row unmounts. * fix(projects): pin project thread listing contract and exclude archived chats GET /api/projects/{id}/threads returned the thread store row verbatim (list[dict], no response_model): user_id/assistant_id leaked, any future ThreadMetaRow column would auto-leak, and the OpenAPI schema was empty. Return a narrow ProjectThreadResponse (the exact fields ProjectThread declares) with the same metadata secret redaction the surrounding thread endpoints get from _MetadataRedactingResponse. The listing also ran search() without the archived filter, so a retired chat rendered as a normal row on the project page while the sidebar hid it. Search archived=False to mirror the sidebar's archived:false lists; restore stays on the global Archived tab. Both regressions pinned by new router tests: wire-shape allowlist and archived-member exclusion. * docs(migrations): record the 0019/0020 chain against the bootstrap reservation The tree now chains 0018 -> 0019_projects -> 0020_threads_meta_project_id, so migrations/AGENTS.md was stale twice over: the revision index stopped at 0018 and the rolling-forward section still claimed the tree 'deliberately remains at 0018'. Document the new head and record the intentional numeric-prefix reuse of 0019: 0019_projects is in-chain while 0019_thread_incarnations stays the reserved, allowlisted out-of-tree rollout id. The owning rollout revision must re-parent onto this tree's head when it merges so alembic never sees two heads off 0018; bootstrap.py now cross-references that note next to _FORWARD_COMPATIBLE_REVISION. * fix(chats): invalidate project thread lists on archive/restore useArchiveThread refreshed the infinite sidebar cache, threads/search and the per-thread metadata cache but not the project-scoped list ([...PROJECTS_QUERY_KEY, 'threads', id]) this PR adds — the one thread mutation not wired to that key, after usePinThread, useRenameThread, useDeleteThread, useMoveThreadToProject and invalidateStoppedThreadCaches. An archive from a sidebar row while a project page is open therefore left the archived chat rendered as a normal row until remount (and undo left it missing). Invalidate the prefix in the mutation-level success handler. Regression test asserts the project-list prefix is invalidated on success. * fix(projects): fetch project discovery only in grouped sidebar mode RecentChatList mounted two useProjects queries per sidebar render, but knownProjectIds is consumed only by the grouped-mode exclusion filter; in the default flat mode every page load paid two GET /api/projects?status= round trips for data nothing read. Gate both queries on grouped mode — GroupedProjectList fetches the same keys when the toggle is on and TanStack dedupes the observers. Also set retry: false on useProject: a deleted or foreign project 404s deterministically, and the page renders a dedicated not-found state for it, so the default 1s/2s/4s retry backoff kept deep links in 'loading' for ~7s before that state appeared. Matches useThreadMetadata / useThreadTokenUsage. * fix(threads): fail closed on project-scoped create in memory mode MemoryThreadMetaStore.create accepted project_id and silently ignored it, making memory mode the one membership path that fails open: POST /api/threads with a project id returned 200 and the run started unassigned, violating the invariant that a run never proceeds outside the selected project (the SQL store raises ProjectNotAssignableError inside the insert transaction for the same request). Raise ProjectNotAssignableError whenever project_id is present so the router's existing 404 mapping applies, the frontend keeps the composer text for a retry, and memory mode behaves exactly like SQL mode. set_project already reports rejection; create now matches it. Store-level test (raises, nothing persisted, project filter stays empty, unscoped creates still work) plus a router-level test asserting the 404 and that no row is left behind. * fix(projects): window the project page thread list ProjectThreadsSection rendered every loaded page as a plain Link row, so a long-lived project accumulated unbounded DOM on the page's scroll surface: each load-more appended another 100 rows and every formatTimeAgo tick re-rendered the whole list. Reuse VirtualThreadList (now generic over any row shape with a thread_id), pointing its scroll parent at this page's ScrollArea viewport via the shared [data-slot="scroll-area-viewport"] selector used by /workspace/chats; under the 60-row threshold it falls back to the plain render, so small projects are unchanged. * fix(projects): restore row dividers and pin them with a render test The row class template literal concatenated transition-colors directly with the conditional border-b token, so non-final rows rendered the invalid class 'transition-colorsborder-b' and lost both the divider and the transition. Compose the row classes with cn() and a boolean guard instead. The section moved out of page.tsx into a testable component so the row markup finally has coverage: a DOM test asserts every row except the final data row carries border-b (index-based, not last: — correct under virtualization where the last mounted row is not the last data row), and the untitled fallback plus load-more button render for a partial page. * fix(projects): validate forward schemas and fence membership reads --- README.md | 16 + backend/app/gateway/AGENTS.md | 10 +- backend/app/gateway/app.py | 3 + backend/app/gateway/auth/pat.py | 13 + backend/app/gateway/authz.py | 7 + backend/app/gateway/deps.py | 4 + backend/app/gateway/routers/projects.py | 167 ++++ backend/app/gateway/routers/threads.py | 100 ++- backend/app/gateway/services.py | 18 +- .../harness/deerflow/persistence/bootstrap.py | 46 +- .../deerflow/persistence/migrations/AGENTS.md | 73 +- .../migrations/versions/0019_projects.py | 45 + .../versions/0020_threads_meta_project_id.py | 37 + .../deerflow/persistence/models/__init__.py | 2 + .../deerflow/persistence/projects/__init__.py | 8 + .../deerflow/persistence/projects/model.py | 36 + .../deerflow/persistence/projects/sql.py | 177 ++++ .../persistence/thread_meta/__init__.py | 5 +- .../deerflow/persistence/thread_meta/base.py | 38 +- .../persistence/thread_meta/memory.py | 22 +- .../deerflow/persistence/thread_meta/model.py | 1 + .../deerflow/persistence/thread_meta/sql.py | 108 ++- .../test_authorization_route_permissions.py | 18 + .../test_memory_thread_meta_isolation.py | 25 + ...est_migration_0004_run_ownership_dedupe.py | 2 +- ...ration_0007_scheduled_run_active_dedupe.py | 2 +- ...t_migration_0015_scheduled_task_enqueue.py | 2 +- .../test_migration_0019_0020_projects.py | 68 ++ backend/tests/test_pat_auth.py | 40 + backend/tests/test_persistence_bootstrap.py | 2 +- .../test_persistence_bootstrap_concurrency.py | 2 +- .../test_persistence_bootstrap_regression.py | 4 +- ...est_persistence_forward_revision_compat.py | 91 +- backend/tests/test_projects_repo.py | 84 ++ backend/tests/test_projects_router.py | 355 ++++++++ backend/tests/test_thread_meta_repo.py | 198 +++++ backend/tests/test_threads_router.py | 353 +++++++- docs/database-forward-revision-recovery.md | 70 ++ frontend/src/AGENTS.md | 9 +- .../src/app/workspace/projects/[id]/page.tsx | 278 ++++++ .../components/workspace/chats/chat-page.tsx | 115 ++- .../workspace/chats/use-thread-chat.ts | 36 + .../src/components/workspace/input-box.tsx | 41 + .../workspace/move-to-project-menu.tsx | 216 +++++ .../components/workspace/projects-section.tsx | 355 ++++++++ .../projects/project-threads-section.tsx | 89 ++ .../components/workspace/recent-chat-list.tsx | 805 ++++++++++-------- .../workspace/thread-list-virtualizer.tsx | 39 +- .../workspace/use-thread-archive-action.ts | 41 +- .../workspace/workspace-container.tsx | 9 +- .../workspace/workspace-sidebar.tsx | 8 +- frontend/src/core/i18n/locales/en-US.ts | 33 + frontend/src/core/i18n/locales/types.ts | 32 + frontend/src/core/i18n/locales/zh-CN.ts | 31 + frontend/src/core/projects/api.ts | 188 ++++ frontend/src/core/projects/hooks.ts | 168 ++++ frontend/src/core/projects/index.ts | 3 + frontend/src/core/projects/types.ts | 41 + frontend/src/core/settings/local.ts | 5 + frontend/src/core/settings/store.ts | 22 +- frontend/src/core/threads/api.ts | 48 ++ frontend/src/core/threads/archive.ts | 42 +- frontend/src/core/threads/hooks.ts | 74 +- frontend/src/core/threads/utils.ts | 14 + frontend/tests/e2e/chat.spec.ts | 173 +++- frontend/tests/e2e/utils/mock-api.ts | 76 +- .../project-threads-section.dom.test.tsx | 102 +++ .../unit/core/threads/archive.dom.test.tsx | 24 + .../tests/unit/core/threads/infinite.test.ts | 5 +- .../core/threads/move-thread.dom.test.tsx | 112 +++ 70 files changed, 4977 insertions(+), 509 deletions(-) create mode 100644 backend/app/gateway/routers/projects.py create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/0019_projects.py create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/0020_threads_meta_project_id.py create mode 100644 backend/packages/harness/deerflow/persistence/projects/__init__.py create mode 100644 backend/packages/harness/deerflow/persistence/projects/model.py create mode 100644 backend/packages/harness/deerflow/persistence/projects/sql.py create mode 100644 backend/tests/test_migration_0019_0020_projects.py create mode 100644 backend/tests/test_projects_repo.py create mode 100644 backend/tests/test_projects_router.py create mode 100644 docs/database-forward-revision-recovery.md create mode 100644 frontend/src/app/workspace/projects/[id]/page.tsx create mode 100644 frontend/src/components/workspace/move-to-project-menu.tsx create mode 100644 frontend/src/components/workspace/projects-section.tsx create mode 100644 frontend/src/components/workspace/projects/project-threads-section.tsx create mode 100644 frontend/src/core/projects/api.ts create mode 100644 frontend/src/core/projects/hooks.ts create mode 100644 frontend/src/core/projects/index.ts create mode 100644 frontend/src/core/projects/types.ts create mode 100644 frontend/tests/unit/components/workspace/projects/project-threads-section.dom.test.tsx create mode 100644 frontend/tests/unit/core/threads/move-thread.dom.test.tsx diff --git a/README.md b/README.md index 3562d4db3..51abab6b2 100644 --- a/README.md +++ b/README.md @@ -1485,6 +1485,22 @@ The HTTP Gateway accepts `values`, `messages-tuple`, `updates`, `debug`, `tasks` All dict-returning methods are validated against Gateway Pydantic response models in CI (`TestGatewayConformance`), ensuring the embedded client stays in sync with the HTTP API schemas. See `backend/packages/harness/deerflow/client.py` for full API documentation. +## Project membership + +A conversation joins a project at creation time (when a project is selected) or +later through the move menu. Runs never modify membership: submitting a message +cannot assign or reassign a conversation. Moving a conversation out of a project +keeps it unassigned until it is explicitly moved again. + +Moving a conversation refreshes its header affiliation as well as the project +lists, including when an older metadata request is still in flight. + +Projects require the current database tables and columns. A database stamped +`0019_thread_incarnations` from the older 0018-based rollout is rejected at +startup if the project schema is missing. Follow the +[offline database recovery procedure](docs/database-forward-revision-recovery.md) +before starting this build against that database. + ## Scheduled Tasks DeerFlow now includes a first-class scheduled-task MVP in the workspace. diff --git a/backend/app/gateway/AGENTS.md b/backend/app/gateway/AGENTS.md index 21e1d4151..1eada7a8b 100644 --- a/backend/app/gateway/AGENTS.md +++ b/backend/app/gateway/AGENTS.md @@ -8,7 +8,15 @@ CORS is same-origin by default when requests enter through nginx on port 2026. S Browser auth sessions are owned by `app.gateway.auth.session_cookie`. Login accepts a `remember_me` form flag, but the Gateway never stores passwords. `SessionCookiePolicy` persists the `HttpOnly access_token` cookie only for HTTPS/trusted-forwarded HTTPS, direct-host localhost HTTP, or explicit operator opt-in for insecure persistence; public HTTP sandbox URLs degrade to session cookies. Session-creating handlers stamp the final `max_age` on `request.state`; CSRF cookie creation mirrors it so the double-submit pair expires together, including re-issue after password changes and OIDC callbacks. A small `HttpOnly` preference cookie preserves the remember choice across re-issues. Logout clears all auth cookies and suppresses CSRF re-issue on the logout response. -Personal Access Tokens (`app.gateway.auth.pat`, `Authorization: Bearer dfp_...`) run as their owning user: an invalid Bearer is a hard 401 with no cookie fallback, which keeps `CSRFMiddleware`'s Bearer skip safe (origin checks still run). Scopes narrow within the allowlisted threads/runs routes; every other authenticated route 403s PAT callers (admin included). PAT management and `/change-password` require session auth; only SHA-256 digests are stored (`0017`). +Personal Access Tokens (`app.gateway.auth.pat`, `Authorization: Bearer dfp_...`) run as their owning user: an invalid Bearer is a hard 401 with no cookie fallback, which keeps `CSRFMiddleware`'s Bearer skip safe (origin checks still run). Scopes narrow within the allowlisted threads/runs/projects routes (including `POST /api/threads/{id}/move`); every other authenticated route 403s PAT callers (admin included). PAT management and `/change-password` require session auth; only SHA-256 digests are stored (`0017`). + +Thread→project membership is written by thread creation (`POST /api/threads` with +a validated `project_id`), branch creation (the new row inherits the source +thread's project; an archived/deleted project degrades the branch to unassigned +instead of failing), and explicit moves (`POST /api/threads/{id}/move`); run +admission never modifies membership. The server-reserved `deerflow_project_id` +metadata key is a read-only exposure of the `threads_meta.project_id` column and +is stripped from client writes. Localhost persistence deliberately reads the direct request `Host` and ignores `Forwarded` / `X-Forwarded-Host`. Scheme and auth-origin reconstruction still consume forwarding headers. The bundled nginx sets `X-Forwarded-Proto`, but preserves an upstream HTTPS value and does not overwrite every forwarded header, so the outer trusted proxy must replace or strip client-supplied forwarding headers before traffic reaches DeerFlow. diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 21571e141..c8f12477d 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -32,6 +32,7 @@ from app.gateway.routers import ( mcp_tasks, memory, models, + projects, runs, scheduled_tasks, skills, @@ -825,6 +826,8 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for # Agents API is mounted at /api/agents app.include_router(agents.router) + # Projects API is mounted at /api/projects + app.include_router(projects.router) # Deployment-level subagent catalog and admin management. app.include_router(subagents.router) diff --git a/backend/app/gateway/auth/pat.py b/backend/app/gateway/auth/pat.py index 34c97e9d2..be039579d 100644 --- a/backend/app/gateway/auth/pat.py +++ b/backend/app/gateway/auth/pat.py @@ -33,6 +33,9 @@ PAT_ALLOWED_SCOPES: frozenset[str] = frozenset( "runs:create", "runs:read", "runs:cancel", + "projects:read", + "projects:write", + "projects:delete", } ) @@ -53,6 +56,16 @@ _PAT_ROUTE_RULES: tuple[tuple[frozenset[str], re.Pattern[str]], ...] = ( (frozenset({"GET", "PUT", "DELETE"}), re.compile(r"^/api/threads/[^/]+/goal$")), (frozenset({"GET", "POST"}), re.compile(r"^/api/threads/[^/]+/state$")), (frozenset({"POST"}), re.compile(r"^/api/threads/[^/]+/(compact|history|branches)$")), + (frozenset({"POST"}), re.compile(r"^/api/threads/[^/]+/move$")), + # Projects subtree: same enumerated-no-dead-methods precision as the + # threads/runs rules — only the methods the projects router implements + # are admitted, so a future projects route is default-denied until + # explicitly listed. Scope narrowing (projects:read|write|delete and + # threads:write for move) stays enforced by ``@require_permission``. + (frozenset({"GET", "POST"}), re.compile(r"^/api/projects$")), + (frozenset({"GET", "PATCH", "DELETE"}), re.compile(r"^/api/projects/[^/]+$")), + (frozenset({"POST"}), re.compile(r"^/api/projects/[^/]+/(archive|restore)$")), + (frozenset({"GET"}), re.compile(r"^/api/projects/[^/]+/threads$")), # Runs subtree: enumerated per implemented subroute instead of a # ``runs(/.*)?`` wildcard, so a route added under /runs is default-denied # until explicitly listed — the same no-dead-methods precision the diff --git a/backend/app/gateway/authz.py b/backend/app/gateway/authz.py index a3f7da407..aed9944f8 100644 --- a/backend/app/gateway/authz.py +++ b/backend/app/gateway/authz.py @@ -68,6 +68,10 @@ class Permissions: RUNS_CREATE = "runs:create" RUNS_READ = "runs:read" RUNS_CANCEL = "runs:cancel" + # Projects + PROJECTS_READ = "projects:read" + PROJECTS_WRITE = "projects:write" + PROJECTS_DELETE = "projects:delete" class AuthContext: @@ -149,6 +153,9 @@ _ALL_PERMISSIONS: list[str] = [ Permissions.RUNS_CREATE, Permissions.RUNS_READ, Permissions.RUNS_CANCEL, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, ] diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index c77f1b4b1..ce1aed5b7 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -513,12 +513,14 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen app.state.thread_store = make_thread_store(sf, app.state.store) if sf is not None: from deerflow.persistence.mcp_tasks import McpTaskRepository + from deerflow.persistence.projects import ProjectRepository from deerflow.persistence.scheduled_task_runs import ( ScheduledTaskRunRepository, ) from deerflow.persistence.scheduled_tasks import ScheduledTaskRepository from deerflow.persistence.subagent_batches import SubagentBatchRepository + app.state.project_repo = ProjectRepository(sf) app.state.scheduled_task_repo = ScheduledTaskRepository( sf, run_repository=app.state.run_store, @@ -531,6 +533,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen app.state.subagent_batch_repo = SubagentBatchRepository(sf) else: app.state.mcp_task_repo = None + app.state.project_repo = None app.state.subagent_batch_repo = None app.state.scheduled_task_repo = None app.state.scheduled_task_run_repo = None @@ -648,6 +651,7 @@ get_checkpointer: Callable[[Request], Checkpointer] = _require("checkpointer", " get_run_event_store: Callable[[Request], RunEventStore] = _require("run_event_store", "Run event store") get_feedback_repo: Callable[[Request], FeedbackRepository] = _require("feedback_repo", "Feedback") get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store") +get_project_repo = _require("project_repo", "Projects") def get_store(request: Request): diff --git a/backend/app/gateway/routers/projects.py b/backend/app/gateway/routers/projects.py new file mode 100644 index 000000000..732bcec09 --- /dev/null +++ b/backend/app/gateway/routers/projects.py @@ -0,0 +1,167 @@ +"""CRUD API for projects (Phase 1: organization only — no documents/trash).""" + +import logging +from typing import Any, Literal + +from fastapi import APIRouter, HTTPException, Query, Request +from pydantic import BaseModel, Field, field_validator + +from app.gateway.authz import require_permission +from app.gateway.deps import get_project_repo, get_thread_store +from deerflow.runtime.secret_context import redact_metadata_secrets +from deerflow.utils.time import coerce_iso + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/projects", tags=["projects"]) + +ProjectStatus = Literal["active", "archived"] + + +class ProjectResponse(BaseModel): + id: str + name: str + instructions: str + presentation: dict + status: str + created_at: str + updated_at: str + + +class ProjectCreateRequest(BaseModel): + name: str = Field(..., min_length=1, max_length=128) + instructions: str = "" + presentation: dict = Field(default_factory=dict) + + +class ProjectPatchRequest(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=128) + instructions: str | None = None + presentation: dict | None = None + + +class ProjectListResponse(BaseModel): + projects: list[ProjectResponse] + + +class ProjectThreadResponse(BaseModel): + """A thread row from ``GET /api/projects/{id}/threads``. + + Deliberately narrow — only the fields ``ProjectThread`` declares in + ``frontend/src/core/projects/types.ts``. Store rows carry ownership + columns (``user_id``, ``assistant_id``) and ``ThreadMetaRow`` may grow; + without this model those would leak onto the wire and the route's + OpenAPI schema stays empty. Metadata is redacted here exactly as the + surrounding thread endpoints redact it via ``_MetadataRedactingResponse``. + """ + + thread_id: str + display_name: str | None = None + created_at: str = "" + updated_at: str = "" + metadata: dict[str, Any] = Field(default_factory=dict) + + @field_validator("metadata", mode="before", check_fields=False) + @classmethod + def _redact_metadata_secrets(cls, value: Any) -> Any: + return redact_metadata_secrets(value) + + +def _to_response(row: dict) -> ProjectResponse: + return ProjectResponse( + id=row["id"], + name=row["name"], + instructions=row.get("instructions", ""), + presentation=row.get("presentation") or {}, + status=row["status"], + created_at=row.get("created_at", ""), + updated_at=row.get("updated_at", ""), + ) + + +def _not_found() -> HTTPException: + # Fail closed: foreign projects are indistinguishable from missing ones. + return HTTPException(status_code=404, detail="Project not found") + + +@router.post("", response_model=ProjectResponse, status_code=201) +@require_permission("projects", "write") +async def create_project(body: ProjectCreateRequest, request: Request) -> ProjectResponse: + repo = get_project_repo(request) + return _to_response(await repo.create(name=body.name, instructions=body.instructions, presentation=body.presentation)) + + +@router.get("", response_model=ProjectListResponse) +@require_permission("projects", "read") +async def list_projects(request: Request, status: ProjectStatus | None = None) -> ProjectListResponse: + repo = get_project_repo(request) + return ProjectListResponse(projects=[_to_response(r) for r in await repo.list(status=status)]) + + +@router.get("/{project_id}", response_model=ProjectResponse) +@require_permission("projects", "read") +async def get_project(project_id: str, request: Request) -> ProjectResponse: + row = await get_project_repo(request).get(project_id) + if row is None: + raise _not_found() + return _to_response(row) + + +@router.patch("/{project_id}", response_model=ProjectResponse) +@require_permission("projects", "write") +async def patch_project(project_id: str, body: ProjectPatchRequest, request: Request) -> ProjectResponse: + row = await get_project_repo(request).patch(project_id, name=body.name, instructions=body.instructions, presentation=body.presentation) + if row is None: + raise _not_found() + return _to_response(row) + + +@router.post("/{project_id}/archive", response_model=ProjectResponse) +@require_permission("projects", "write") +async def archive_project(project_id: str, request: Request) -> ProjectResponse: + row = await get_project_repo(request).set_status(project_id, "archived") + if row is None: + raise _not_found() + return _to_response(row) + + +@router.post("/{project_id}/restore", response_model=ProjectResponse) +@require_permission("projects", "write") +async def restore_project(project_id: str, request: Request) -> ProjectResponse: + row = await get_project_repo(request).set_status(project_id, "active") + if row is None: + raise _not_found() + return _to_response(row) + + +@router.delete("/{project_id}", status_code=204) +@require_permission("projects", "delete") +async def delete_project(project_id: str, request: Request) -> None: + if not await get_project_repo(request).delete(project_id): + raise _not_found() + + +@router.get("/{project_id}/threads", response_model=list[ProjectThreadResponse]) +@require_permission("projects", "read") +@require_permission("threads", "read") +async def list_project_threads(project_id: str, request: Request, limit: int = Query(default=100, ge=1, le=1000), offset: int = Query(default=0, ge=0)) -> list[ProjectThreadResponse]: + if await get_project_repo(request).get(project_id) is None: + raise _not_found() + # Active members only, mirroring the sidebar's `archived: false` lists: + # an archived chat leaves the project's pages the same way it leaves the + # sidebar and returns only via the global Archived tab. + rows = await get_thread_store(request).search( + project_id=project_id, + archived=False, + limit=limit, + offset=offset, + ) + return [ + ProjectThreadResponse( + thread_id=r["thread_id"], + display_name=r.get("display_name"), + created_at=coerce_iso(r.get("created_at", "")), + updated_at=coerce_iso(r.get("updated_at", "")), + metadata=r.get("metadata", {}), + ) + for r in rows + ] diff --git a/backend/app/gateway/routers/threads.py b/backend/app/gateway/routers/threads.py index 4fe6e4419..39cf463bd 100644 --- a/backend/app/gateway/routers/threads.py +++ b/backend/app/gateway/routers/threads.py @@ -46,7 +46,7 @@ from app.gateway.utils import sanitize_log_param from deerflow.agents.thread_state import THREAD_STATE_REDUCER_FIELDS from deerflow.config.paths import Paths, get_paths from deerflow.config.summarization_config import ContextSize -from deerflow.persistence.thread_meta import THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY +from deerflow.persistence.thread_meta import PROJECT_FILTER_UNSET, THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, THREAD_PROJECT_METADATA_KEY from deerflow.runtime import ThreadOperationKind, serialize_channel_values_for_api from deerflow.runtime.checkpoint_mode import CheckpointModeMismatchError, CheckpointModeReconfigurationError from deerflow.runtime.checkpoint_state import graph_reducer_channels, graph_state_schema, graph_writable_channels @@ -111,7 +111,7 @@ def _checkpoint_mode_http_error(exc: Exception, thread_id: str) -> HTTPException # owner identity through the API surface. Defense-in-depth — the # row-level invariant is still ``threads_meta.user_id`` populated from # the auth contextvar; this list closes the metadata-blob echo gap. -_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"}) +_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id", THREAD_PROJECT_METADATA_KEY}) _SIDECAR_METADATA_KEY = "deerflow_sidecar" _BRANCH_METADATA_KEY = "deerflow_branch" _BRANCH_TITLE_SEQUENCE_METADATA_KEY = "branch_title_sequence" @@ -444,6 +444,7 @@ class ThreadCreateRequest(BaseModel): thread_id: ThreadId | None = Field(default=None, description="Optional thread ID (auto-generated if omitted)") assistant_id: str | None = Field(default=None, description="Associate thread with an assistant") metadata: dict[str, Any] = Field(default_factory=dict, description="Initial metadata") + project_id: str | None = Field(default=None, description="Assign the new thread to this project (validated server-side)") _strip_reserved = field_validator("metadata")(classmethod(lambda cls, v: _strip_reserved_metadata(v))) @@ -453,6 +454,7 @@ class ThreadSearchRequest(BaseModel): archived: bool | None = Field(default=None, strict=True, description="Archive filter; omitted includes all, false includes legacy unarchived threads") metadata: dict[str, Any] = Field(default_factory=dict, description="Metadata filter (exact match)") + project_id: str | None = Field(default=None, description="Filter by project; explicit null = unassigned threads; omit key for all") limit: int = Field(default=100, ge=1, le=1000, description="Maximum results") offset: int = Field(default=0, ge=0, description="Pagination offset") status: str | None = Field(default=None, description="Filter by thread status") @@ -826,13 +828,20 @@ async def create_thread(body: ThreadCreateRequest, request: Request) -> ThreadRe return _existing_thread_response(thread_id, existing_record) # Write thread_meta so the thread appears in /threads/search immediately + from deerflow.persistence.projects import ProjectNotAssignableError + try: - await thread_store.create( + created_record = await thread_store.create( thread_id, assistant_id=getattr(body, "assistant_id", None), **thread_owner_kwargs, metadata=body.metadata, + project_id=body.project_id, ) + except ProjectNotAssignableError: + # Fail closed: missing, foreign, or archived projects are + # indistinguishable at the API surface. + raise HTTPException(status_code=404, detail="Project not found") from None except IntegrityError: # The idempotency read above and this insert are not atomic: a # concurrent request for the same thread_id can commit in between, so @@ -869,13 +878,11 @@ async def create_thread(body: ThreadCreateRequest, request: Request) -> ThreadRe raise HTTPException(status_code=500, detail="Failed to create thread") logger.info("Thread created: %s", sanitize_log_param(thread_id)) - return ThreadResponse( - thread_id=thread_id, - status="idle", - created_at=now, - updated_at=now, - metadata=body.metadata, - ) + # Respond from the persisted record — the store stamps + # ``metadata.deerflow_project_id`` from the assigned project_id column, so + # echoing ``body.metadata`` here would omit the membership the retry path + # (``_existing_thread_response``) reports. + return _existing_thread_response(thread_id, created_record) @router.post("/{thread_id}/branches", response_model=ThreadBranchResponse) @@ -952,6 +959,15 @@ async def _branch_thread_with_reservation( "branch_parent_message_id": body.message_id, "branch_created_at": now, } + # A branch extends its source conversation, so the new row inherits the + # source thread's project membership — an unassigned branch would surface + # under Recent chats instead of the source thread's project group. The + # store re-validates the project inside the insert (same fail-closed path + # as create/move), and the inherited id may be stale only when the project + # was archived or deleted after the source read. + from deerflow.persistence.projects import ProjectNotAssignableError + + source_project_id = (source_metadata or {}).get(THREAD_PROJECT_METADATA_KEY) if body.title: display_name = body.title @@ -1026,17 +1042,30 @@ async def _branch_thread_with_reservation( logger.exception("Failed to write branch checkpoint for thread %s", sanitize_log_param(new_thread_id)) raise HTTPException(status_code=500, detail="Failed to create branch") from None + async def _write_branch_row(project_id: str | None) -> None: + try: + await thread_store.create( + new_thread_id, + assistant_id=source_record.get("assistant_id"), + display_name=display_name, + metadata=branch_metadata, + project_id=project_id, + **thread_owner_kwargs, + ) + except ProjectNotAssignableError: + raise + except Exception: + logger.exception("Failed to write branch thread_meta for %s", sanitize_log_param(new_thread_id)) + raise HTTPException(status_code=500, detail="Failed to create branch") from None + try: - await thread_store.create( - new_thread_id, - assistant_id=source_record.get("assistant_id"), - display_name=display_name, - metadata=branch_metadata, - **thread_owner_kwargs, - ) - except Exception: - logger.exception("Failed to write branch thread_meta for %s", sanitize_log_param(new_thread_id)) - raise HTTPException(status_code=500, detail="Failed to create branch") from None + await _write_branch_row(source_project_id) + except ProjectNotAssignableError: + # The source project became unassignable (archived, or deleted in a + # race that cleared the source row's membership after our read): keep + # the branch usable as an unassigned thread — the pre-inheritance + # behavior for sources without an active project. + await _write_branch_row(None) # The thread feed (GET /messages, /messages/page) reads the run-event # store, not checkpoints, and a fresh branch has no run_events — so the @@ -1087,11 +1116,15 @@ async def search_threads(body: ThreadSearchRequest, request: Request) -> list[Th from deerflow.persistence.thread_meta import InvalidMetadataFilterError repo = get_thread_store(request) + # Three-state project filter: key absent → no filter; explicit null → + # unassigned threads only; string → members of that project. + project_filter = body.project_id if "project_id" in body.model_fields_set else PROJECT_FILTER_UNSET try: rows = await repo.search( metadata=body.metadata or None, status=body.status, **({"archived": body.archived} if body.archived is not None else {}), + project_id=project_filter, limit=body.limit, offset=body.offset, ) @@ -1147,6 +1180,33 @@ async def patch_thread(thread_id: ThreadId, body: ThreadPatchRequest, request: R ) +class ThreadMoveRequest(BaseModel): + """Request body for moving a thread into/out of a project.""" + + project_id: str | None = Field(..., description="Target project id, or null to unassign") + + +@router.post("/{thread_id}/move", response_model=ThreadResponse) +@require_permission("threads", "write", owner_check=True, require_existing=True) +async def move_thread(thread_id: ThreadId, body: ThreadMoveRequest, request: Request) -> ThreadResponse: + """Move a thread between projects (or out). Organizational only: history, + run state, and per-thread files are untouched (RFC v2 §6).""" + from app.gateway.deps import get_thread_store + + thread_store = get_thread_store(request) + moved = await thread_store.set_project(thread_id, body.project_id) + if not moved: + raise HTTPException(status_code=404, detail="Thread or project not found") + record = await thread_store.get(thread_id) + return ThreadResponse( + thread_id=thread_id, + status=record.get("status", "idle"), + created_at=coerce_iso(record.get("created_at", "")), + updated_at=coerce_iso(record.get("updated_at", "")), + metadata=record.get("metadata", {}), + ) + + @router.get("/{thread_id}", response_model=ThreadResponse) @require_permission("threads", "read", owner_check=True) async def get_thread(thread_id: ThreadId, request: Request) -> ThreadResponse: diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index dbcbddfdd..f93574575 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -201,14 +201,24 @@ async def _ensure_thread_metadata( if existing is None: if require_existing_thread: raise LookupError(f"Thread {record.thread_id} was deleted during run admission") + from deerflow.persistence.thread_meta import THREAD_PROJECT_METADATA_KEY + + run_metadata = record.metadata or {} + metadata = { + key: value + for key, value in run_metadata.items() + # Strip the run-scoped trace id (existing) and the reserved + # membership key: run admission never modifies project membership — + # the column is written only by POST /api/threads and + # /threads/{id}/move — so the key must not persist either. + if key not in (DEERFLOW_TRACE_METADATA_KEY, THREAD_PROJECT_METADATA_KEY) + } await thread_store.create( record.thread_id, assistant_id=record.assistant_id, - # Seeded from the run that created the thread, minus the run-scoped - # trace id: a thread spans many runs and as many trace ids, so - # pinning the first one here would be misleading rather than useful. - metadata={key: value for key, value in (record.metadata or {}).items() if key != DEERFLOW_TRACE_METADATA_KEY}, + metadata=metadata, ) + return async def _terminal_record_stream_missing(bridge: StreamBridge, record: RunRecord) -> bool: diff --git a/backend/packages/harness/deerflow/persistence/bootstrap.py b/backend/packages/harness/deerflow/persistence/bootstrap.py index b1a8861dd..7a8356d0c 100644 --- a/backend/packages/harness/deerflow/persistence/bootstrap.py +++ b/backend/packages/harness/deerflow/persistence/bootstrap.py @@ -18,7 +18,7 @@ Three-branch decision (see ``_decide_state``) | empty (no DeerFlow tables) | ``create_all`` + ``alembic stamp head`` | | legacy (DeerFlow tables, no alembic) | ``create_all`` (baseline tables only, as backfill) + ``stamp 0001_baseline`` + ``upgrade head`` | | versioned (one locally known revision) | ``alembic upgrade head`` | -| reviewed forward-compatible revision 0019 | warn and skip migration | +| reviewed forward revision with local columns | warn and skip migration | | unknown, empty, or multiple revision rows | refuse to start | The legacy branch handles pre-alembic databases that already have at least one @@ -116,7 +116,14 @@ _KNOWN_REVISIONS: frozenset[str] | None = None # server default, table, index, constraint, or data backfill. The owning 0019 # change must cross-pin this revision id and schema shape in tests. Amending # that DDL requires re-auditing old-repository reads and writes before this -# exception remains valid. +# exception remains valid. The exception also requires every current ORM table +# and column: the original 0018 + incarnation columns shape lacks projects and +# is no longer compatible with this build. Both skip paths validate that floor. +# Note: this tree's own chain already carries ``0019_projects`` / +# ``0020_threads_meta_project_id`` off ``0018_oauth_identity_pg_partial``; the +# ``0019_`` numeric prefix is intentionally reused. When the owning rollout +# revision merges it must re-parent onto the current head (see +# ``migrations/AGENTS.md``) so alembic never sees two heads off 0018. _FORWARD_COMPATIBLE_REVISION = "0019_thread_incarnations" # Baseline (stamp target for legacy DBs). Pinned here so the bootstrap layer @@ -317,6 +324,32 @@ async def _read_database_revision(conn: Any) -> str: return revision +def _validate_forward_schema(sync_conn: Any) -> None: + """Require the local repository schema before skipping unknown migrations. + + This is a presence check, not a general schema compatibility proof. The + allowlisted additive DDL still needs its separate read/write audit. Derive + the local floor from ORM metadata so a new mapped column cannot silently + invalidate the existing exception again. + """ + import deerflow.persistence.models # noqa: F401 + from deerflow.persistence.base import Base + + inspector = sa_inspect(sync_conn) + tables = set(inspector.get_table_names()) + missing = [] + for name, table in sorted(Base.metadata.tables.items()): + if name not in tables: + missing.append(name) + continue + columns = {column["name"] for column in inspector.get_columns(name)} + missing.extend(f"{name}.{column.name}" for column in table.columns if column.name not in columns) + if missing: + raise RuntimeError( + f"bootstrap: revision {_FORWARD_COMPATIBLE_REVISION!r} is missing required local schema: {', '.join(missing)}; refusing to start. See docs/database-forward-revision-recovery.md for the audited offline migration path." + ) + + def _reflect_state(sync_conn: Any) -> dict[str, bool]: """Inspect *sync_conn* (sync connection inside ``run_sync``) and return: @@ -597,15 +630,18 @@ async def bootstrap_schema(engine: AsyncEngine, *, backend: str, postgres_schema raise async with engine.connect() as conn: current_revision = await _read_database_revision(conn) - if current_revision != _FORWARD_COMPATIBLE_REVISION: - raise + if current_revision != _FORWARD_COMPATIBLE_REVISION: + raise + await conn.run_sync(_validate_forward_schema) logger.warning( "bootstrap: database advanced concurrently to explicitly forward-compatible revision %s; skipping the stale local upgrade", current_revision, ) elif database_revision == _FORWARD_COMPATIBLE_REVISION: + async with engine.connect() as conn: + await conn.run_sync(_validate_forward_schema) logger.warning( - "bootstrap: database revision %s is newer than local head %s but is explicitly forward-compatible; skipping migration", + "bootstrap: database revision %s is explicitly forward-compatible with local head %s and has its required tables and columns; skipping migration", database_revision, head, ) diff --git a/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md b/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md index ba7e05057..0d3458f0b 100644 --- a/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md +++ b/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md @@ -2,7 +2,7 @@ DeerFlow's application tables (`runs`, `threads_meta`, `feedback`, `users`, `run_events`, plus the four `channel_*` tables) are owned by alembic via a **hybrid bootstrap** strategy. LangGraph's checkpointer tables (`checkpoints`, `checkpoint_blobs`, `checkpoint_writes`, `checkpoint_migrations`) live in the same database but are owned by LangGraph and excluded from alembic's view via `migrations/_env_filters.py::include_object`. -**Convention**: every ORM model change (new column, new table, new index) MUST ship as an alembic revision under `migrations/versions/`. The Gateway runs `alembic upgrade head` automatically on startup; users do not run `alembic` manually in production. +**Convention**: every ORM model change (new column, new table, new index) MUST ship as an alembic revision under `migrations/versions/`. The Gateway runs `alembic upgrade head` automatically on startup; routine production upgrades do not require manual Alembic commands. The audited offline recovery below is an exception for the out-of-tree incarnation revision. **Hybrid bootstrap** (`persistence/bootstrap.py::bootstrap_schema`, invoked from `persistence/engine.py::init_engine`): @@ -11,35 +11,58 @@ DeerFlow's application tables (`runs`, `threads_meta`, `feedback`, `users`, `run | empty (no DeerFlow tables) | `create_all` + `alembic stamp head` | | legacy (DeerFlow tables, no `alembic_version`) | `create_all` (baseline tables only, backfill) + `alembic stamp 0001_baseline` + `upgrade head` | | versioned (one locally known `alembic_version` row) | `alembic upgrade head` | -| forward-compatible (`0019_thread_incarnations`) | warn and skip migration | +| `0019_thread_incarnations` with all current ORM tables/columns | warn and skip migration | +| `0019_thread_incarnations` missing local tables/columns | refuse startup; offline recovery required | | unknown revision, empty version table, or multiple version rows | fail closed and refuse to start | The legacy branch handles pre-alembic databases that already have at least one DeerFlow-owned table. `create_all` runs first because stamping at `0001_baseline` makes alembic skip the baseline's own `create_table` DDL on the subsequent upgrade — so any baseline table introduced into `Base.metadata` after the user's DB was first provisioned (e.g. the `channel_*` tables from PR #1930 for users upgrading across multiple releases) would otherwise never be created, and the first request hitting that table would 500 with `no such table`. The backfill is **restricted to `_BASELINE_TABLE_NAMES`** so it does not also create tables that future revisions introduce — those revisions' own `op.create_table` would otherwise fail with `relation already exists`. A guard test pins `_BASELINE_TABLE_NAMES` against `0001_baseline.upgrade()`'s actual output, so editing 0001 to add or remove a table forces a matching update to the constant. Column-level shape (pre-#3658 vs post-#3658 vs manual-ALTER for `token_usage_by_model`) is answered by each `versions/*.py` revision via the idempotent helpers in `migrations/_helpers.py` (`safe_add_column` / `safe_drop_column`) which no-op when the change is already present and `logger.warning` on shape drift. **Adding a new ORM column / table only requires a new revision file — no edit to `bootstrap.py` is needed** *unless* the new revision adds a new baseline table (rare; only happens when a new model is part of the baseline rather than introduced by its own revision). The empty-DB path keeps using `create_all` because `Base.metadata` is the only authoritative schema source — `create_all` renders both SQLite (JSON, type affinity) and Postgres (JSONB, partial indexes) correctly without anyone having to keep a hand-written baseline in lockstep. `0001_baseline.upgrade()` is therefore almost never executed in practice; it exists as a stamp target + chain root. -**Rolling forward compatibility**: this migration tree deliberately remains at -`0018_oauth_identity_pg_partial`, but an older Gateway may briefly share a -database with the expand-only `0019_thread_incarnations` deployment. Bootstrap -reads `alembic_version` while holding its backend lock and accepts exactly one -row. A locally known revision follows the normal upgrade path. The one unknown -revision `0019_thread_incarnations` is explicitly allowlisted: bootstrap logs a -warning and leaves the newer schema untouched. Any other unknown revision, an -empty version table, or multiple version rows fails closed. Do not broaden the -allowlist without proving that old repositories can read, insert, and update -through the newer schema; nullable additive columns are covered by -`tests/test_persistence_forward_revision_compat.py`. This exception is reviewed -only for the expand-only 0019 shape: nullable VARCHAR(32) +**Rolling forward compatibility**: the local chain head is `0020_threads_meta_project_id` +(`0018_oauth_identity_pg_partial` → `0019_projects` → `0020_threads_meta_project_id`). +Bootstrap reads `alembic_version` while holding its backend lock and accepts +exactly one row. A locally known revision follows the normal upgrade path. The +one unknown revision `0019_thread_incarnations` is conditionally allowlisted: +bootstrap first requires every current ORM table and column, then logs a +warning and leaves the schema untouched. The original rollout shape (0018 plus +the two incarnation columns) is now rejected: it lacks `projects` and +`threads_meta.project_id`. Seeding current head is only a positive compatibility +fixture; tests must also construct the original 0018-based schema and assert +rejection on both the direct startup and SQLite race-recovery paths. The check +uses `conn.run_sync` reflection and derives its local floor from `Base.metadata` +so future ORM additions cannot silently bypass it. This checks presence only; +the additive DDL audit below still owns type/constraint compatibility. Any other +unknown revision, an empty version table, or multiple version rows fails +closed. Do not broaden the allowlist without proving that old repositories can +read, insert, and update through the newer schema; nullable additive columns +are covered by `tests/test_persistence_forward_revision_compat.py`. This +exception is reviewed only for the expand-only 0019 shape: nullable VARCHAR(32) `threads_meta.incarnation` and `mcp_tasks.thread_incarnation` columns with no -server default, table, index, constraint, or data backfill. The 0019 migration -must cross-pin its revision id and schema shape against the bootstrap contract; -amending that DDL requires a fresh old-repository compatibility audit. Because SQLite has no -cross-process bootstrap mutex, an old process may read 0018 immediately before -another process commits 0019. If its now-stale Alembic upgrade fails, bootstrap -re-reads the version and recovers only for the exact allowlisted 0019 while that -revision remains absent from the local migration tree. Do not generalize this -recovery or apply it to a binary that owns 0019; its migration failures must -remain fatal. +server default, table, index, constraint, or data backfill. The owning +`0019_thread_incarnations` migration must cross-pin its revision id and schema +shape against the bootstrap contract; amending that DDL requires a fresh +old-repository compatibility audit. The `0019_` numeric prefix is intentionally +reused: `0019_projects` is this tree's in-chain revision, while +`0019_thread_incarnations` is the reserved, out-of-tree rollout id allowlisted +above — revision ids only need to be unique, not numerically ordered, but the +owning rollout revision must re-parent from `0018_oauth_identity_pg_partial` +onto this tree's head when it merges so `alembic` never sees two heads off +0018. Because SQLite has no cross-process bootstrap mutex, an old process may +read 0018 immediately before another process commits 0019. If its now-stale +Alembic upgrade fails, bootstrap re-reads the version and recovers only for the +exact allowlisted 0019 with all current ORM tables and columns while that +revision remains absent from the local migration tree. Do not generalize this recovery or apply it to a binary that +owns 0019; its migration failures must remain fatal. + +For an existing database with the original 0018-plus-incarnation shape, use the +[audited offline recovery procedure](../../../../../../docs/database-forward-revision-recovery.md). +Bootstrap never re-stamps an unknown revision automatically. After stopping all +writers, backing up, and verifying the exact additive schema, the operator may +purge-stamp the known 0018 parent and apply this tree's 0019/0020 migrations; +the extra nullable columns and their data remain intact. A regression exercises +that procedure from the original schema and verifies repository reads/inserts +and preservation of incarnation data. **Concurrency safety**: Postgres uses `pg_advisory_lock` to serialise concurrent Gateway instances. SQLite uses a per-engine `asyncio.Lock` for same-process startup and is best-effort across processes via SQLite's file-level write lock + `PRAGMA busy_timeout`; multi-instance deployments should use Postgres. Column revisions in `versions/` additionally use idempotent helpers (`_helpers.py::safe_add_column`, `safe_drop_column`) so repeated post-baseline changes and retries are no-ops when the change is already present. @@ -47,7 +70,7 @@ remain fatal. ```bash cd backend && make migrate-rev MSG="add foo column to runs" ``` -This invokes `alembic revision --autogenerate` against the live ORM models. Review the generated file under `migrations/versions/` and switch raw `op.add_column` / `op.drop_column` calls to the idempotent helpers from `_helpers.py` before committing. There is no `make migrate` / `make migrate-stamp` target on purpose — the only execution path is Gateway startup, which keeps operational mistakes off the table. +This invokes `alembic revision --autogenerate` against the live ORM models. Review the generated file under `migrations/versions/` and switch raw `op.add_column` / `op.drop_column` calls to the idempotent helpers from `_helpers.py` before committing. There is no `make migrate` / `make migrate-stamp` target on purpose — routine upgrades execute at Gateway startup; the documented offline recovery is reserved for the audited out-of-tree schema. **Extension-owned tables.** An extension that persists data owns its schema end to end and must not register models against `deerflow.persistence.base.Base` @@ -113,6 +136,8 @@ on installs that never enabled it. The convention is: - `migrations/versions/0016_subagent_batches.py` — creates durable native-subagent batch and item tables, including owner/submission idempotency, item identity, lease/recovery state, and result fields - `migrations/versions/0017_personal_access_tokens.py` — creates the personal access token table for programmatic API access - `migrations/versions/0018_oauth_identity_pg_partial.py` — converts `idx_users_oauth_identity` to a partial index on Postgres (`postgresql_where`), matching what `UserRow.__table_args__` already builds via `create_all`; `0001_baseline` never applied the predicate on Postgres, so every `alembic upgrade head`-provisioned deployment carried a full index until this revision. Postgres-only, idempotent (checks `pg_index.indpred` directly), no-op on SQLite (already partial via `sqlite_where`) and on a DB where the index doesn't exist yet. Originally generated as 0017 and renumbered to 0018 after 0017_personal_access_tokens merged first and kept that slot +- `migrations/versions/0019_projects.py` — creates the `projects` table (id/user_id/name/instructions/presentation/status + timestamps) for the Projects Phase-1 organization feature; chains after `0018_oauth_identity_pg_partial` +- `migrations/versions/0020_threads_meta_project_id.py` — adds nullable `threads_meta.project_id` plus `ix_threads_meta_project_id` (no FK by design: project delete clears membership first, and the reserved `deerflow_project_id` metadata key stays in sync); chains after `0019_projects` and is the current head. The `0019_` numeric prefix is reused by the reserved out-of-tree `0019_thread_incarnations` — see the rolling-forward section above - `persistence/bootstrap.py` — `bootstrap_schema(engine, backend=...)`, the three-branch provisioning decision, locked revision validation, and the narrow 0019 forward-compatibility exception - `extensions/loader.py::load_extensions` — registers each spec's `table_prefix` with `register_extension_table_prefix()` - Tests: `tests/test_persistence_bootstrap.py` (branches), `tests/test_persistence_bootstrap_concurrency.py` (concurrency), `tests/test_persistence_bootstrap_regression.py` (issue #3682), `tests/test_persistence_migrations_env.py` (filter, including extension-owned tables), `tests/test_extension_loader.py::TestTablePrefixRegistration` (spec-to-filter wiring), `tests/blocking_io/test_persistence_bootstrap.py` (asyncio.to_thread anchor), `tests/test_migration_0004_run_ownership_dedupe.py` + `tests/test_migration_0007_scheduled_run_active_dedupe.py` (dedupe-before-unique-index pre-steps) diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0019_projects.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0019_projects.py new file mode 100644 index 000000000..399045604 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0019_projects.py @@ -0,0 +1,45 @@ +"""projects. + +Revision ID: 0019_projects +Revises: 0018_oauth_identity_pg_partial +Create Date: 2026-09-06 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0019_projects" +down_revision: str | Sequence[str] | None = "0018_oauth_identity_pg_partial" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table("projects"): + op.create_table( + "projects", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=False), + sa.Column("name", sa.String(length=128), nullable=False), + sa.Column("instructions", sa.Text(), nullable=False), + sa.Column("presentation", sa.JSON(), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_projects_user_id", "projects", ["user_id"]) + op.create_index("ix_projects_status", "projects", ["status"]) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table("projects"): + op.drop_index("ix_projects_status", table_name="projects") + op.drop_index("ix_projects_user_id", table_name="projects") + op.drop_table("projects") diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0020_threads_meta_project_id.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0020_threads_meta_project_id.py new file mode 100644 index 000000000..29a71677a --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0020_threads_meta_project_id.py @@ -0,0 +1,37 @@ +"""threads_meta.project_id. + +Revision ID: 0020_threads_meta_project_id +Revises: 0019_projects +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0020_threads_meta_project_id" +down_revision: str | Sequence[str] | None = "0019_projects" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + from deerflow.persistence.migrations._helpers import safe_add_column + + safe_add_column("threads_meta", sa.Column("project_id", sa.String(length=64), nullable=True)) + inspector = sa.inspect(op.get_bind()) + existing = {i["name"] for i in inspector.get_indexes("threads_meta")} + if "ix_threads_meta_project_id" not in existing: + op.create_index("ix_threads_meta_project_id", "threads_meta", ["project_id"]) + + +def downgrade() -> None: + from deerflow.persistence.migrations._helpers import safe_drop_column + + inspector = sa.inspect(op.get_bind()) + existing = {i["name"] for i in inspector.get_indexes("threads_meta")} + if "ix_threads_meta_project_id" in existing: + op.drop_index("ix_threads_meta_project_id", table_name="threads_meta") + safe_drop_column("threads_meta", "project_id") diff --git a/backend/packages/harness/deerflow/persistence/models/__init__.py b/backend/packages/harness/deerflow/persistence/models/__init__.py index c6b374ae2..ff0188d81 100644 --- a/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -26,6 +26,7 @@ from deerflow.persistence.managed_subagents.model import ManagedSubagentRow from deerflow.persistence.mcp_tasks.model import McpTaskRow from deerflow.persistence.models.run_event import RunEventRow from deerflow.persistence.personal_access_tokens.model import PersonalAccessTokenRow +from deerflow.persistence.projects.model import ProjectRow from deerflow.persistence.run.model import RunRow from deerflow.persistence.scheduled_task_runs.model import ScheduledTaskRunRow from deerflow.persistence.scheduled_tasks.model import ScheduledTaskRow @@ -44,6 +45,7 @@ __all__ = [ "McpTaskRow", "ManagedSubagentRow", "PersonalAccessTokenRow", + "ProjectRow", "RunEventRow", "RunRow", "ScheduledTaskRow", diff --git a/backend/packages/harness/deerflow/persistence/projects/__init__.py b/backend/packages/harness/deerflow/persistence/projects/__init__.py new file mode 100644 index 000000000..6e9a0982f --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/projects/__init__.py @@ -0,0 +1,8 @@ +"""Project persistence — ORM model and SQL repository.""" + +from __future__ import annotations + +from deerflow.persistence.projects.model import ProjectRow +from deerflow.persistence.projects.sql import ProjectNotAssignableError, ProjectRepository + +__all__ = ["ProjectNotAssignableError", "ProjectRepository", "ProjectRow"] diff --git a/backend/packages/harness/deerflow/persistence/projects/model.py b/backend/packages/harness/deerflow/persistence/projects/model.py new file mode 100644 index 000000000..dda23e58e --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/projects/model.py @@ -0,0 +1,36 @@ +"""ORM model for projects. + +One row per user-owned project. ``id`` (uuid4 hex) is the only external +identity; ``name``/``presentation`` are display attributes — renaming touches +exactly this row and never affects membership (threads reference +``threads_meta.project_id``). ``instructions`` is user-authored project +context; Phase 1 stores and PATCHes it, Phase 2 injects it. There is +deliberately no ``memory_mode``/sharing/agent-config column: no consumer +exists in Phase 1/2 (RFC v2 §3, §4.1). +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import JSON, DateTime, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from deerflow.persistence.base import Base + + +class ProjectRow(Base): + __tablename__ = "projects" + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + user_id: Mapped[str] = mapped_column(String(64), index=True) + name: Mapped[str] = mapped_column(String(128)) + instructions: Mapped[str] = mapped_column(Text, default="") + presentation: Mapped[dict] = mapped_column(JSON, default=dict) + status: Mapped[str] = mapped_column(String(16), default="active", index=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + ) diff --git a/backend/packages/harness/deerflow/persistence/projects/sql.py b/backend/packages/harness/deerflow/persistence/projects/sql.py new file mode 100644 index 000000000..78ef16f97 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/projects/sql.py @@ -0,0 +1,177 @@ +"""SQLAlchemy-backed project repository. + +Ownership discipline mirrors ``ThreadMetaRepository``: every method resolves +the caller via ``resolve_user_id(..., AUTO)`` and filters by ``user_id``; +a foreign project is indistinguishable from a missing one (callers map +``None``/``False`` to 404). ``delete`` runs the Phase-1 membership-clearing +transaction from RFC v2 §5.1 (the ``project_documents`` statement joins this +transaction in Phase 2). +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import UTC, datetime +from typing import Any, Literal + +from sqlalchemy import delete as sa_delete +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from deerflow.persistence.projects.model import ProjectRow +from deerflow.persistence.thread_meta.model import ThreadMetaRow +from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id +from deerflow.utils.time import coerce_iso + +logger = logging.getLogger(__name__) + +ProjectStatus = Literal["active", "archived"] + + +class ProjectNotAssignableError(ValueError): + """Raised when a thread cannot be assigned to a project. + + The project is missing, foreign to the caller, or archived — the + atomicity rules (RFC v2 §5.2) make these indistinguishable inside the + mutating statement, so callers get one signal and map it to 404 (explicit + API) or a dropped key (run admission). + """ + + +class ProjectRepository: + def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + self._sf = session_factory + + @staticmethod + def _row_to_dict(row: ProjectRow) -> dict[str, Any]: + d = row.to_dict() + for key in ("created_at", "updated_at"): + val = d.get(key) + if isinstance(val, datetime): + d[key] = coerce_iso(val) + return d + + async def create( + self, + *, + name: str, + instructions: str = "", + presentation: dict | None = None, + user_id: str | None | _AutoSentinel = AUTO, + ) -> dict: + resolved_user_id = resolve_user_id(user_id, method_name="ProjectRepository.create") + now = datetime.now(UTC) + row = ProjectRow( + id=uuid.uuid4().hex, + user_id=resolved_user_id, + name=name, + instructions=instructions, + presentation=presentation or {}, + status="active", + created_at=now, + updated_at=now, + ) + async with self._sf() as session: + session.add(row) + await session.commit() + await session.refresh(row) + return self._row_to_dict(row) + + async def get(self, project_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None: + resolved_user_id = resolve_user_id(user_id, method_name="ProjectRepository.get") + async with self._sf() as session: + row = await session.get(ProjectRow, project_id) + if row is None: + return None + if resolved_user_id is not None and row.user_id != resolved_user_id: + return None + return self._row_to_dict(row) + + async def list( + self, + *, + status: ProjectStatus | None = None, + user_id: str | None | _AutoSentinel = AUTO, + ) -> list[dict]: + resolved_user_id = resolve_user_id(user_id, method_name="ProjectRepository.list") + stmt = select(ProjectRow).order_by(ProjectRow.created_at.asc(), ProjectRow.id.asc()) + if resolved_user_id is not None: + stmt = stmt.where(ProjectRow.user_id == resolved_user_id) + if status is not None: + stmt = stmt.where(ProjectRow.status == status) + async with self._sf() as session: + result = await session.execute(stmt) + return [self._row_to_dict(r) for r in result.scalars()] + + async def patch( + self, + project_id: str, + *, + name: str | None = None, + instructions: str | None = None, + presentation: dict | None = None, + user_id: str | None | _AutoSentinel = AUTO, + ) -> dict | None: + resolved_user_id = resolve_user_id(user_id, method_name="ProjectRepository.patch") + async with self._sf() as session: + row = await session.get(ProjectRow, project_id) + if row is None or (resolved_user_id is not None and row.user_id != resolved_user_id): + return None + if name is not None: + row.name = name + if instructions is not None: + row.instructions = instructions + if presentation is not None: + row.presentation = presentation + await session.commit() + await session.refresh(row) + return self._row_to_dict(row) + + async def set_status( + self, + project_id: str, + status: ProjectStatus, + *, + user_id: str | None | _AutoSentinel = AUTO, + ) -> dict | None: + """Idempotent status flip; returns the current row or None (missing/foreign).""" + resolved_user_id = resolve_user_id(user_id, method_name="ProjectRepository.set_status") + async with self._sf() as session: + row = await session.get(ProjectRow, project_id) + if row is None or (resolved_user_id is not None and row.user_id != resolved_user_id): + return None + row.status = status + await session.commit() + await session.refresh(row) + return self._row_to_dict(row) + + async def delete(self, project_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> bool: + """Delete a project and clear membership in one transaction (no filesystem work). + + Returns True when the project row was deleted; False when missing/foreign. + """ + resolved_user_id = resolve_user_id(user_id, method_name="ProjectRepository.delete") + async with self._sf() as session: + async with session.begin(): + # Lock the project row before touching it (FOR UPDATE on + # Postgres; the clause renders nothing on SQLite). Membership + # assignment (ThreadMetaRepository.set_project/create) takes + # the same row lock before writing ``project_id``, so an + # assigner either commits first and has its membership cleared + # below, or blocks until this transaction commits and then + # re-reads the row as gone — no dangling ``threads_meta.project_id`` + # (RFC v2 §14.14). Missing/foreign rows keep the False result. + lock_stmt = select(ProjectRow).where(ProjectRow.id == project_id) + if resolved_user_id is not None: + lock_stmt = lock_stmt.where(ProjectRow.user_id == resolved_user_id) + locked = (await session.execute(lock_stmt.with_for_update())).scalar_one_or_none() + if locked is None: + return False + if resolved_user_id is not None: + await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.project_id == project_id, ThreadMetaRow.user_id == resolved_user_id).values(project_id=None, updated_at=ThreadMetaRow.__table__.c.updated_at)) + result = await session.execute(sa_delete(ProjectRow).where(ProjectRow.id == project_id, ProjectRow.user_id == resolved_user_id)) + else: + await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.project_id == project_id).values(project_id=None, updated_at=ThreadMetaRow.__table__.c.updated_at)) + result = await session.execute(sa_delete(ProjectRow).where(ProjectRow.id == project_id)) + return result.rowcount > 0 diff --git a/backend/packages/harness/deerflow/persistence/thread_meta/__init__.py b/backend/packages/harness/deerflow/persistence/thread_meta/__init__.py index 41bc8283e..7908fc10e 100644 --- a/backend/packages/harness/deerflow/persistence/thread_meta/__init__.py +++ b/backend/packages/harness/deerflow/persistence/thread_meta/__init__.py @@ -4,7 +4,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from deerflow.persistence.thread_meta.base import THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, InvalidMetadataFilterError, ThreadMetaStore +from deerflow.persistence.thread_meta.base import PROJECT_FILTER_UNSET, THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, THREAD_PROJECT_METADATA_KEY, InvalidMetadataFilterError, ThreadMetaStore from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore from deerflow.persistence.thread_meta.model import ThreadMetaRow from deerflow.persistence.thread_meta.sql import ThreadMetaRepository @@ -16,8 +16,11 @@ if TYPE_CHECKING: __all__ = [ "InvalidMetadataFilterError", "MemoryThreadMetaStore", + "PROJECT_FILTER_UNSET", "THREAD_PINNED_METADATA_KEY", "THREAD_ARCHIVED_METADATA_KEY", + "THREAD_PINNED_METADATA_KEY", + "THREAD_PROJECT_METADATA_KEY", "ThreadMetaRepository", "ThreadMetaRow", "ThreadMetaStore", diff --git a/backend/packages/harness/deerflow/persistence/thread_meta/base.py b/backend/packages/harness/deerflow/persistence/thread_meta/base.py index 409fc6673..8b5205f7e 100644 --- a/backend/packages/harness/deerflow/persistence/thread_meta/base.py +++ b/backend/packages/harness/deerflow/persistence/thread_meta/base.py @@ -15,7 +15,7 @@ three-state semantics (see :mod:`deerflow.runtime.user_context`): from __future__ import annotations import abc -from typing import Any +from typing import Any, ClassVar, Final from deerflow.runtime.user_context import AUTO, _AutoSentinel @@ -25,6 +25,28 @@ from deerflow.runtime.user_context import AUTO, _AutoSentinel THREAD_PINNED_METADATA_KEY = "deerflow_pinned" THREAD_ARCHIVED_METADATA_KEY = "deerflow_archived" +# Cross-component metadata key. Keep in sync with +# ``frontend/src/core/threads/utils.ts`` and +# ``frontend/tests/e2e/utils/mock-api.ts``. +THREAD_PROJECT_METADATA_KEY = "deerflow_project_id" + + +class _ProjectFilterUnset: + """Sentinel for ``search(project_id=...)``: absent filter vs explicit unassigned.""" + + _instance: ClassVar[_ProjectFilterUnset | None] = None + + def __new__(cls) -> _ProjectFilterUnset: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __repr__(self) -> str: + return "" + + +PROJECT_FILTER_UNSET: Final = _ProjectFilterUnset() + class InvalidMetadataFilterError(ValueError): """Raised when all client-supplied metadata filter keys are rejected.""" @@ -40,8 +62,19 @@ class ThreadMetaStore(abc.ABC): user_id: str | None | _AutoSentinel = AUTO, display_name: str | None = None, metadata: dict | None = None, + project_id: str | None = None, ) -> dict: - pass + """Create a thread row; when ``project_id`` is set, validate the + project inside the insert transaction and raise + ``ProjectNotAssignableError`` on failure (no partial row).""" + + @abc.abstractmethod + async def set_project(self, thread_id: str, project_id: str | None, *, user_id: str | None | _AutoSentinel = AUTO) -> bool: + """Atomically move a thread into/out of a project (RFC v2 §5.2). + + Returns False when the thread is missing/foreign, or the target + project is missing/foreign/archived. Must not touch ``updated_at``. + """ @abc.abstractmethod async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None: @@ -54,6 +87,7 @@ class ThreadMetaStore(abc.ABC): metadata: dict[str, Any] | None = None, status: str | None = None, archived: bool | None = None, + project_id: str | None | _ProjectFilterUnset = PROJECT_FILTER_UNSET, limit: int = 100, offset: int = 0, user_id: str | None | _AutoSentinel = AUTO, diff --git a/backend/packages/harness/deerflow/persistence/thread_meta/memory.py b/backend/packages/harness/deerflow/persistence/thread_meta/memory.py index bf5c7f320..8b4bb9a87 100644 --- a/backend/packages/harness/deerflow/persistence/thread_meta/memory.py +++ b/backend/packages/harness/deerflow/persistence/thread_meta/memory.py @@ -12,7 +12,7 @@ from typing import Any from langgraph.store.base import BaseStore from deerflow.persistence.json_compat import json_value_matches -from deerflow.persistence.thread_meta.base import THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, ThreadMetaStore +from deerflow.persistence.thread_meta.base import PROJECT_FILTER_UNSET, THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, ThreadMetaStore, _ProjectFilterUnset from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id from deerflow.utils.time import coerce_iso, now_iso @@ -48,7 +48,18 @@ class MemoryThreadMetaStore(ThreadMetaStore): user_id: str | None | _AutoSentinel = AUTO, display_name: str | None = None, metadata: dict | None = None, + project_id: str | None = None, ) -> dict: + # Memory mode has no projects backend in Phase 1. Fail closed exactly + # like the SQL store does for a missing/foreign/archived project: a + # create carrying ``project_id`` raises ``ProjectNotAssignableError`` + # (the router maps it to 404) instead of silently persisting an + # unassigned thread that a run would then proceed under. Mirrors + # ``set_project`` below, which already reports rejection. + if project_id is not None: + from deerflow.persistence.projects import ProjectNotAssignableError + + raise ProjectNotAssignableError(project_id) resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.create") now = now_iso() record: dict[str, Any] = { @@ -65,6 +76,11 @@ class MemoryThreadMetaStore(ThreadMetaStore): await self._store.aput(THREADS_NS, thread_id, record) return record + async def set_project(self, thread_id: str, project_id: str | None, *, user_id: str | None | _AutoSentinel = AUTO) -> bool: + # Memory mode has no projects backend in Phase 1: membership moves + # are unsupported and always report rejection. + return False + async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None: return await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.get") @@ -74,6 +90,7 @@ class MemoryThreadMetaStore(ThreadMetaStore): metadata: dict[str, Any] | None = None, status: str | None = None, archived: bool | None = None, + project_id: str | None | _ProjectFilterUnset = PROJECT_FILTER_UNSET, limit: int = 100, offset: int = 0, user_id: str | None | _AutoSentinel = AUTO, @@ -85,6 +102,9 @@ class MemoryThreadMetaStore(ThreadMetaStore): scalable paginated I/O. """ resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.search") + if not isinstance(project_id, _ProjectFilterUnset) and project_id is not None: + # Memory mode has no projects backend: no thread can be a member. + return [] filter_dict: dict[str, Any] = {} if status: filter_dict["status"] = status diff --git a/backend/packages/harness/deerflow/persistence/thread_meta/model.py b/backend/packages/harness/deerflow/persistence/thread_meta/model.py index fe15315e1..939b12f44 100644 --- a/backend/packages/harness/deerflow/persistence/thread_meta/model.py +++ b/backend/packages/harness/deerflow/persistence/thread_meta/model.py @@ -16,6 +16,7 @@ class ThreadMetaRow(Base): thread_id: Mapped[str] = mapped_column(String(64), primary_key=True) assistant_id: Mapped[str | None] = mapped_column(String(128), index=True) user_id: Mapped[str | None] = mapped_column(String(64), index=True) + project_id: Mapped[str | None] = mapped_column(String(64), index=True, nullable=True) display_name: Mapped[str | None] = mapped_column(String(256)) status: Mapped[str] = mapped_column(String(20), default="idle") metadata_json: Mapped[dict] = mapped_column(JSON, default=dict) diff --git a/backend/packages/harness/deerflow/persistence/thread_meta/sql.py b/backend/packages/harness/deerflow/persistence/thread_meta/sql.py index 9fb4c3db3..28ac16d2e 100644 --- a/backend/packages/harness/deerflow/persistence/thread_meta/sql.py +++ b/backend/packages/harness/deerflow/persistence/thread_meta/sql.py @@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from sqlalchemy.orm.attributes import flag_modified from deerflow.persistence.json_compat import json_match -from deerflow.persistence.thread_meta.base import THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, InvalidMetadataFilterError, ThreadMetaStore +from deerflow.persistence.thread_meta.base import PROJECT_FILTER_UNSET, THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, THREAD_PROJECT_METADATA_KEY, InvalidMetadataFilterError, ThreadMetaStore, _ProjectFilterUnset from deerflow.persistence.thread_meta.model import ThreadMetaRow from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id from deerflow.utils.time import coerce_iso @@ -26,7 +26,10 @@ class ThreadMetaRepository(ThreadMetaStore): @staticmethod def _row_to_dict(row: ThreadMetaRow) -> dict[str, Any]: d = row.to_dict() - d["metadata"] = d.pop("metadata_json", None) or {} + d["metadata"] = dict(d.pop("metadata_json", None) or {}) + project_id = d.pop("project_id", None) + if project_id is not None: + d["metadata"][THREAD_PROJECT_METADATA_KEY] = project_id for key in ("created_at", "updated_at"): val = d.get(key) if isinstance(val, datetime): @@ -43,26 +46,103 @@ class ThreadMetaRepository(ThreadMetaStore): user_id: str | None | _AutoSentinel = AUTO, display_name: str | None = None, metadata: dict | None = None, + project_id: str | None = None, ) -> dict: # Auto-resolve user_id from contextvar when AUTO; explicit None # creates an orphan row (used by migration scripts). resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create") now = datetime.now(UTC) - row = ThreadMetaRow( - thread_id=thread_id, - assistant_id=assistant_id, - user_id=resolved_user_id, - display_name=display_name, - metadata_json=metadata or {}, - created_at=now, - updated_at=now, - ) async with self._sf() as session: + if session.get_bind().dialect.name == "sqlite": + await session.execute(text("BEGIN IMMEDIATE")) + if project_id is not None: + from deerflow.persistence.projects import ProjectNotAssignableError + from deerflow.persistence.projects.model import ProjectRow + + # Lock the project row (FOR UPDATE on Postgres; the clause + # renders nothing on SQLite) so a concurrent + # ProjectRepository.delete — which holds the same lock across + # its membership-clear and DELETE — either commits first (this + # read then finds no row) or waits for this transaction. + # RFC v2 §14.14: no dangling ``threads_meta.project_id``. + locked = await session.scalar( + select(ProjectRow.id) + .where( + ProjectRow.id == project_id, + ProjectRow.user_id == resolved_user_id, + ProjectRow.status == "active", + ) + .with_for_update() + ) + if locked is None: + raise ProjectNotAssignableError(project_id) + row = ThreadMetaRow( + thread_id=thread_id, + assistant_id=assistant_id, + user_id=resolved_user_id, + display_name=display_name, + status="idle", + metadata_json=metadata or {}, + project_id=project_id, + created_at=now, + updated_at=now, + ) session.add(row) await session.commit() await session.refresh(row) return self._row_to_dict(row) + async def set_project( + self, + thread_id: str, + project_id: str | None, + *, + user_id: str | None | _AutoSentinel = AUTO, + ) -> bool: + resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.set_project") + from deerflow.persistence.projects.model import ProjectRow + + async with self._sf() as session: + if session.get_bind().dialect.name == "sqlite": + # Read-then-write transaction: take the write lock up front + # (create() precedent) so a concurrent writer cannot + # interleave between the project check and the UPDATE. + await session.execute(text("BEGIN IMMEDIATE")) + if project_id is not None: + # Lock the project row (FOR UPDATE on Postgres; the clause + # renders nothing on SQLite) so a concurrent + # ProjectRepository.delete — which holds the same lock across + # its membership-clear and DELETE — either commits first (this + # read then finds no row) or waits for this transaction. + # RFC v2 §14.14: no dangling ``threads_meta.project_id``. + locked = await session.scalar( + select(ProjectRow.id) + .where( + ProjectRow.id == project_id, + ProjectRow.user_id == resolved_user_id, + ProjectRow.status == "active", + ) + .with_for_update() + ) + if locked is None: + await session.commit() + return False + stmt = ( + update(ThreadMetaRow) + .where(ThreadMetaRow.thread_id == thread_id) + .values( + project_id=project_id, + # Explicit self-assignment: satisfies the column so the + # ``onupdate`` hook does not bump recency (pin precedent, G5). + updated_at=ThreadMetaRow.__table__.c.updated_at, + ) + ) + if resolved_user_id is not None: + stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id) + result = await session.execute(stmt) + await session.commit() + return result.rowcount > 0 + async def get( self, thread_id: str, @@ -115,6 +195,7 @@ class ThreadMetaRepository(ThreadMetaStore): metadata: dict[str, Any] | None = None, status: str | None = None, archived: bool | None = None, + project_id: str | None | _ProjectFilterUnset = PROJECT_FILTER_UNSET, limit: int = 100, offset: int = 0, user_id: str | None | _AutoSentinel = AUTO, @@ -159,6 +240,11 @@ class ThreadMetaRepository(ThreadMetaStore): # identically on SQLite and Postgres, including for the active view. archive_flag = case((json_match(ThreadMetaRow.metadata_json, THREAD_ARCHIVED_METADATA_KEY, True), 1), else_=0) stmt = stmt.where(archive_flag == int(archived)) + if not isinstance(project_id, _ProjectFilterUnset): + if project_id is None: + stmt = stmt.where(ThreadMetaRow.project_id.is_(None)) + else: + stmt = stmt.where(ThreadMetaRow.project_id == project_id) stmt = stmt.limit(limit).offset(offset) async with self._sf() as session: diff --git a/backend/tests/test_authorization_route_permissions.py b/backend/tests/test_authorization_route_permissions.py index 8318fd0db..c562d7206 100644 --- a/backend/tests/test_authorization_route_permissions.py +++ b/backend/tests/test_authorization_route_permissions.py @@ -91,6 +91,9 @@ async def test_route_permissions_disabled_preserves_all_permissions(monkeypatch) Permissions.RUNS_CREATE, Permissions.RUNS_READ, Permissions.RUNS_CANCEL, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, ] cached.assert_not_called() @@ -107,6 +110,9 @@ async def test_route_permissions_use_async_provider_and_trusted_principal(monkey Permissions.THREADS_WRITE, Permissions.RUNS_CREATE, Permissions.RUNS_READ, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, ] assert [(request.resource, request.action, request.target) for request in provider.requests] == [ ("route", "read", Permissions.THREADS_READ), @@ -115,6 +121,9 @@ async def test_route_permissions_use_async_provider_and_trusted_principal(monkey ("route", "create", Permissions.RUNS_CREATE), ("route", "read", Permissions.RUNS_READ), ("route", "cancel", Permissions.RUNS_CANCEL), + ("route", "read", Permissions.PROJECTS_READ), + ("route", "write", Permissions.PROJECTS_WRITE), + ("route", "delete", Permissions.PROJECTS_DELETE), ] principal = provider.requests[0].principal assert principal.user_id == "user-123" @@ -137,6 +146,9 @@ async def test_route_permissions_fail_closed_denies_only_the_failed_permission(m Permissions.THREADS_DELETE, Permissions.RUNS_CREATE, Permissions.RUNS_READ, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, ] @@ -154,6 +166,9 @@ async def test_route_permissions_fail_open_allows_the_failed_permission(monkeypa Permissions.RUNS_CREATE, Permissions.RUNS_READ, Permissions.RUNS_CANCEL, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, ] @@ -171,6 +186,9 @@ async def test_route_permissions_fail_open_allows_the_failed_permission(monkeypa Permissions.RUNS_CREATE, Permissions.RUNS_READ, Permissions.RUNS_CANCEL, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, ], ), ], diff --git a/backend/tests/test_memory_thread_meta_isolation.py b/backend/tests/test_memory_thread_meta_isolation.py index 25a776010..fdc54e6bc 100644 --- a/backend/tests/test_memory_thread_meta_isolation.py +++ b/backend/tests/test_memory_thread_meta_isolation.py @@ -11,6 +11,7 @@ from types import SimpleNamespace import pytest from langgraph.store.memory import InMemoryStore +from deerflow.persistence.projects import ProjectNotAssignableError from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore from deerflow.runtime.user_context import reset_current_user, set_current_user @@ -209,3 +210,27 @@ async def test_explicit_none_bypasses_filter(store): row = await store.get("t-alpha", user_id=None) assert row is not None + + +@pytest.mark.anyio +@pytest.mark.no_auto_user +async def test_create_with_project_id_fails_closed(store): + """Memory mode has no projects backend; a create carrying a project id + must fail closed (ProjectNotAssignableError) instead of silently + persisting an unassigned thread — the router maps the error to 404 and + the frontend keeps the composer for a retry, matching the SQL store's + missing/foreign/archived project behavior.""" + with _as_user(USER_A): + with pytest.raises(ProjectNotAssignableError): + await store.create("t-proj", project_id="p1") + + # Nothing persisted, and the store's project filter fails closed too. + assert await store.search() == [] + assert await store.search(project_id="p1") == [] + + # Unscoped creates still work. + await store.create("t-plain") + assert [r["thread_id"] for r in await store.search()] == ["t-plain"] + + # Membership moves report rejection (never a silent unassign). + assert await store.set_project("t-plain", "p1") is False diff --git a/backend/tests/test_migration_0004_run_ownership_dedupe.py b/backend/tests/test_migration_0004_run_ownership_dedupe.py index 515d269ea..5f2ef6be3 100644 --- a/backend/tests/test_migration_0004_run_ownership_dedupe.py +++ b/backend/tests/test_migration_0004_run_ownership_dedupe.py @@ -157,7 +157,7 @@ async def test_migration_dedupes_duplicate_active_rows_before_unique_index(tmp_p with sqlite3.connect(db_path) as raw: version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() # Bootstrap upgrades through the later revisions after 0004. - assert version_row[0] == "0018_oauth_identity_pg_partial" + assert version_row[0] == "0020_threads_meta_project_id" # Sanity: the invariant the index enforces is now true — at most one # active row per thread. diff --git a/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py b/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py index 331bc4bb0..ebdcd2f42 100644 --- a/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py +++ b/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py @@ -173,7 +173,7 @@ async def test_migration_supersedes_duplicate_active_runs_before_unique_index(tm with sqlite3.connect(db_path) as raw: version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() - assert version_row[0] == "0018_oauth_identity_pg_partial" + assert version_row[0] == "0020_threads_meta_project_id" # Sanity: the invariant the index enforces now holds — at most one # active row per task_id. diff --git a/backend/tests/test_migration_0015_scheduled_task_enqueue.py b/backend/tests/test_migration_0015_scheduled_task_enqueue.py index e018fe8cf..a50cf1a6c 100644 --- a/backend/tests/test_migration_0015_scheduled_task_enqueue.py +++ b/backend/tests/test_migration_0015_scheduled_task_enqueue.py @@ -57,7 +57,7 @@ async def test_migration_interrupts_legacy_queue_and_adds_claim_fields(tmp_path: # Bootstrap always advances to the repository head after exercising # the 0015 migration behavior below. - assert version == "0018_oauth_identity_pg_partial" + assert version == "0020_threads_meta_project_id" assert {"lease_owner", "lease_expires_at", "attempt_count"} <= columns.keys() assert columns["attempt_count"]["nullable"] is False assert overlap_policy == "enqueue" diff --git a/backend/tests/test_migration_0019_0020_projects.py b/backend/tests/test_migration_0019_0020_projects.py new file mode 100644 index 000000000..806025159 --- /dev/null +++ b/backend/tests/test_migration_0019_0020_projects.py @@ -0,0 +1,68 @@ +"""Migration tests for 0019_projects and 0020_threads_meta_project_id.""" + +from __future__ import annotations + +import pytest +import sqlalchemy as sa + +from deerflow.persistence.engine import close_engine, init_engine +from deerflow.persistence.migrations import _helpers # noqa: F401 (ensures helpers importable) + +pytestmark = pytest.mark.asyncio + + +async def _fresh_db(tmp_path): + url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}" + await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)) + return url + + +async def test_0019_creates_projects_table(tmp_path): + await _fresh_db(tmp_path) + try: + from deerflow.persistence.engine import get_engine + + engine = get_engine() + async with engine.connect() as conn: + + def _inspect(sync_conn): + inspector = sa.inspect(sync_conn) + assert inspector.has_table("projects") + cols = {c["name"]: c for c in inspector.get_columns("projects")} + assert set(cols) == { + "id", + "user_id", + "name", + "instructions", + "presentation", + "status", + "created_at", + "updated_at", + } + index_names = {i["name"] for i in inspector.get_indexes("projects")} + assert "ix_projects_user_id" in index_names + assert "ix_projects_status" in index_names + + await conn.run_sync(_inspect) + finally: + await close_engine() + + +async def test_0020_adds_project_id_column(tmp_path): + await _fresh_db(tmp_path) + try: + from deerflow.persistence.engine import get_engine + + engine = get_engine() + async with engine.connect() as conn: + + def _inspect(sync_conn): + inspector = sa.inspect(sync_conn) + cols = {c["name"] for c in inspector.get_columns("threads_meta")} + assert "project_id" in cols + index_names = {i["name"] for i in inspector.get_indexes("threads_meta")} + assert "ix_threads_meta_project_id" in index_names + + await conn.run_sync(_inspect) + finally: + await close_engine() diff --git a/backend/tests/test_pat_auth.py b/backend/tests/test_pat_auth.py index eeaa05c4b..b46488c5b 100644 --- a/backend/tests/test_pat_auth.py +++ b/backend/tests/test_pat_auth.py @@ -561,6 +561,46 @@ def test_pat_runs_policy_admits_exactly_the_mounted_routes(): assert not is_pat_allowed_route(method, path), f"{method} {path} is not implemented and must stay denied" +def test_pat_projects_policy_admits_exactly_the_mounted_routes(): + """Projects subtree (and the thread move endpoint) follow the same + enumerated-no-dead-methods discipline as the runs subtree: every + method/path the projects router actually implements is admitted (derived + from the mounted router, not a hand-maintained list), and deliberately + unimplemented neighbors stay default-denied. A new projects route fails + here until explicitly allowlisted; a removed one leaves a dead rule + visible.""" + from fastapi.routing import APIRoute + + from app.gateway.auth.pat import is_pat_allowed_route + from app.gateway.routers.projects import router + + def concrete(path: str) -> str: + return path.replace("{project_id}", "p1") + + for route in router.routes: + if not isinstance(route, APIRoute): + continue + path = concrete(route.path) + for method in sorted(route.methods - {"HEAD", "OPTIONS"}): + assert is_pat_allowed_route(method, path), f"{method} {path} is implemented but PAT-denied" + + for method, path in [ + ("PUT", "/api/projects"), + ("DELETE", "/api/projects"), + ("PUT", "/api/projects/p1"), + ("GET", "/api/projects/p1/archive"), + ("GET", "/api/projects/p1/restore"), + ("POST", "/api/projects/p1/threads"), + ]: + assert not is_pat_allowed_route(method, path), f"{method} {path} is not implemented and must stay denied" + + # Thread move (POST /api/threads/{id}/move) is admitted for PATs holding + # threads:write; other methods on the same path stay denied. + move_path = "/api/threads/6f1c2f0e-3b7a-4d2e-9c1a-2b5f0e8a1d3c/move" + assert is_pat_allowed_route("POST", move_path) is True + assert is_pat_allowed_route("GET", move_path) is False + + def test_pat_scopes_enforced_on_stateless_run_entry(client): """Follow-up to the review's P1-1: the stateless run entrypoints now carry @require_permission("runs", "create"), so a threads:read-only PAT diff --git a/backend/tests/test_persistence_bootstrap.py b/backend/tests/test_persistence_bootstrap.py index c369dd9a9..ea365d5d6 100644 --- a/backend/tests/test_persistence_bootstrap.py +++ b/backend/tests/test_persistence_bootstrap.py @@ -48,7 +48,7 @@ from deerflow.persistence.migrations._helpers import _normalize_default asyncio_test = pytest.mark.asyncio -HEAD = "0018_oauth_identity_pg_partial" +HEAD = "0020_threads_meta_project_id" BASELINE = "0001_baseline" diff --git a/backend/tests/test_persistence_bootstrap_concurrency.py b/backend/tests/test_persistence_bootstrap_concurrency.py index a1c404b32..0c77aec51 100644 --- a/backend/tests/test_persistence_bootstrap_concurrency.py +++ b/backend/tests/test_persistence_bootstrap_concurrency.py @@ -28,7 +28,7 @@ from deerflow.persistence.bootstrap import bootstrap_schema pytestmark = pytest.mark.asyncio -HEAD = "0018_oauth_identity_pg_partial" +HEAD = "0020_threads_meta_project_id" def _url(tmp_path: Path) -> str: diff --git a/backend/tests/test_persistence_bootstrap_regression.py b/backend/tests/test_persistence_bootstrap_regression.py index 2c495e087..da2a4d1ca 100644 --- a/backend/tests/test_persistence_bootstrap_regression.py +++ b/backend/tests/test_persistence_bootstrap_regression.py @@ -76,7 +76,7 @@ async def test_legacy_database_recovers_token_usage_column(tmp_path: Path) -> No cols = {row[1] for row in raw.execute("PRAGMA table_info(runs)").fetchall()} assert "token_usage_by_model" in cols version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() - assert version_row[0] == "0018_oauth_identity_pg_partial" + assert version_row[0] == "0020_threads_meta_project_id" # And the read path that originally 500'd must now succeed. sf = get_session_factory() @@ -116,6 +116,6 @@ async def test_legacy_database_with_manual_alter_still_bootstraps(tmp_path: Path # No duplicate column -- list, not set, to catch dupes. assert cols.count("token_usage_by_model") == 1 version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() - assert version_row[0] == "0018_oauth_identity_pg_partial" + assert version_row[0] == "0020_threads_meta_project_id" finally: await close_engine() diff --git a/backend/tests/test_persistence_forward_revision_compat.py b/backend/tests/test_persistence_forward_revision_compat.py index f5ca2e159..36208250a 100644 --- a/backend/tests/test_persistence_forward_revision_compat.py +++ b/backend/tests/test_persistence_forward_revision_compat.py @@ -12,6 +12,7 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import pytest import sqlalchemy as sa +from alembic import command as alembic_command from alembic.util.exc import CommandError from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine @@ -28,7 +29,7 @@ from deerflow.persistence.engine import close_engine, get_engine, init_engine_fr from deerflow.persistence.mcp_tasks import McpTaskRepository from deerflow.persistence.thread_meta import ThreadMetaRepository -HEAD = "0018_oauth_identity_pg_partial" +HEAD = "0020_threads_meta_project_id" POSTGRES_URL = os.environ.get("TEST_POSTGRES_URI") @@ -58,6 +59,94 @@ async def _seed_head(engine) -> None: assert await _database_revision(engine) == HEAD +async def _seed_original_forward_schema(engine) -> None: + # The rollout predates projects: seeding today's head masks missing columns. + await asyncio.to_thread(_upgrade, _get_alembic_config(engine), "0018_oauth_identity_pg_partial") + await _add_forward_columns(engine) + await _set_database_revision(engine, _FORWARD_COMPATIBLE_REVISION) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("concurrent", [False, True]) +async def test_original_forward_schema_fails_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, concurrent: bool) -> None: + engine = create_async_engine(_url(tmp_path, "original-forward.db")) + try: + await _seed_original_forward_schema(engine) + if concurrent: + await _set_database_revision(engine, "0018_oauth_identity_pg_partial") + + def concurrent_upgrade(_cfg, _revision): + asyncio.run(_set_database_revision(engine, _FORWARD_COMPATIBLE_REVISION)) + raise CommandError("revision advanced concurrently") + + monkeypatch.setattr(bootstrap_mod, "_upgrade", concurrent_upgrade) + + with pytest.raises(RuntimeError, match="missing.*projects.*threads_meta.project_id"): + await bootstrap_schema(engine, backend="sqlite") + + assert await _database_revision(engine) == _FORWARD_COMPATIBLE_REVISION + async with engine.connect() as conn: + tables = await conn.run_sync(lambda sync: sa.inspect(sync).get_table_names()) + assert "projects" not in tables + finally: + await engine.dispose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("ddl", "missing"), + [ + ("DROP TABLE projects", "projects"), + ("ALTER TABLE projects DROP COLUMN instructions", "projects.instructions"), + ("ALTER TABLE threads_meta DROP COLUMN project_id", "threads_meta.project_id"), + ], +) +async def test_forward_revision_rejects_partial_project_schema(tmp_path: Path, ddl: str, missing: str) -> None: + engine = create_async_engine(_url(tmp_path, "partial-projects.db")) + try: + await _seed_head(engine) + await _add_forward_columns(engine) + async with engine.begin() as conn: + if "DROP COLUMN project_id" in ddl: + await conn.execute(sa.text("DROP INDEX ix_threads_meta_project_id")) + await conn.execute(sa.text(ddl)) + await _set_database_revision(engine, _FORWARD_COMPATIBLE_REVISION) + + with pytest.raises(RuntimeError, match=f"missing required local schema: {missing};"): + await bootstrap_schema(engine, backend="sqlite") + assert await _database_revision(engine) == _FORWARD_COMPATIBLE_REVISION + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_audited_original_forward_schema_can_upgrade_preserving_incarnations(tmp_path: Path) -> None: + engine = create_async_engine(_url(tmp_path, "forward-recovery.db")) + try: + await _seed_original_forward_schema(engine) + async with engine.begin() as conn: + await conn.execute( + sa.text("INSERT INTO threads_meta (thread_id, status, metadata_json, created_at, updated_at, incarnation) VALUES ('existing', 'idle', '{}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, :incarnation)"), + {"incarnation": "a" * 32}, + ) + + # Documented offline operator recovery, only after verifying the exact + # 0018 + two nullable columns shape. Bootstrap never re-stamps an unknown DB. + await asyncio.to_thread(alembic_command.stamp, _get_alembic_config(engine), "0018_oauth_identity_pg_partial", purge=True) + await bootstrap_schema(engine, backend="sqlite") + + assert await _database_revision(engine) == HEAD + repository = ThreadMetaRepository(async_sessionmaker(engine, expire_on_commit=False)) + assert [row["thread_id"] for row in await repository.search(user_id=None)] == ["existing"] + assert (await repository.create("new", user_id=None))["thread_id"] == "new" + async with engine.connect() as conn: + assert (await conn.execute(sa.text("SELECT incarnation FROM threads_meta WHERE thread_id = 'existing'"))).scalar_one() == "a" * 32 + columns = await conn.run_sync(lambda sync: sa.inspect(sync).get_columns("mcp_tasks")) + assert "thread_incarnation" in {column["name"] for column in columns} + finally: + await engine.dispose() + + @pytest.mark.asyncio async def test_known_older_revision_upgrades_normally(tmp_path: Path) -> None: engine = create_async_engine(_url(tmp_path, "known.db")) diff --git a/backend/tests/test_projects_repo.py b/backend/tests/test_projects_repo.py new file mode 100644 index 000000000..705982e5a --- /dev/null +++ b/backend/tests/test_projects_repo.py @@ -0,0 +1,84 @@ +"""Tests for ProjectRepository (SQLAlchemy-backed).""" + +import pytest + +from deerflow.persistence.projects import ProjectRepository + + +@pytest.fixture +async def repo(tmp_path): + from deerflow.persistence.engine import close_engine, get_session_factory, init_engine + + url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}" + await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)) + yield ProjectRepository(get_session_factory()) + await close_engine() + + +class TestProjectRepository: + @pytest.mark.anyio + async def test_create_and_get(self, repo): + row = await repo.create(name="Infra overhaul", user_id="u1") + assert row["status"] == "active" + assert row["instructions"] == "" + fetched = await repo.get(row["id"], user_id="u1") + assert fetched is not None and fetched["name"] == "Infra overhaul" + + @pytest.mark.anyio + async def test_get_is_fail_closed_for_foreign_user(self, repo): + row = await repo.create(name="p", user_id="u1") + assert await repo.get(row["id"], user_id="u2") is None + + @pytest.mark.anyio + async def test_list_filters_status(self, repo): + a = await repo.create(name="a", user_id="u1") + b = await repo.create(name="b", user_id="u1") + await repo.set_status(b["id"], "archived", user_id="u1") + active = await repo.list(status="active", user_id="u1") + assert [r["id"] for r in active] == [a["id"]] + archived = await repo.list(status="archived", user_id="u1") + assert [r["id"] for r in archived] == [b["id"]] + # other users see nothing + assert await repo.list(user_id="u2") == [] + + @pytest.mark.anyio + async def test_patch_rename_and_instructions(self, repo): + row = await repo.create(name="old", user_id="u1") + updated = await repo.patch(row["id"], name="new", instructions="ctx", presentation={"icon": "folder"}, user_id="u1") + assert updated is not None + assert updated["name"] == "new" and updated["instructions"] == "ctx" + assert updated["presentation"] == {"icon": "folder"} + + @pytest.mark.anyio + async def test_patch_foreign_is_none(self, repo): + row = await repo.create(name="p", user_id="u1") + assert await repo.patch(row["id"], name="x", user_id="u2") is None + + @pytest.mark.anyio + async def test_set_status_is_idempotent(self, repo): + row = await repo.create(name="p", user_id="u1") + first = await repo.set_status(row["id"], "archived", user_id="u1") + second = await repo.set_status(row["id"], "archived", user_id="u1") + assert first is not None and second is not None + assert first["status"] == second["status"] == "archived" + + @pytest.mark.anyio + async def test_delete_returns_false_for_missing_or_foreign(self, repo): + row = await repo.create(name="p", user_id="u1") + assert await repo.delete(row["id"], user_id="u2") is False + assert await repo.delete("nope", user_id="u1") is False + assert await repo.delete(row["id"], user_id="u1") is True + assert await repo.get(row["id"], user_id="u1") is None + + @pytest.mark.anyio + async def test_delete_clears_membership_and_keeps_thread(self, repo, tmp_path): + from deerflow.persistence.thread_meta import ThreadMetaRepository + + threads = ThreadMetaRepository(repo._sf) + p = await repo.create(name="P", user_id="u1") + await threads.create("t1", user_id="u1", project_id=p["id"]) + + assert await repo.delete(p["id"], user_id="u1") is True + record = await threads.get("t1", user_id="u1") + assert record is not None # thread row intact + assert "deerflow_project_id" not in record["metadata"] diff --git a/backend/tests/test_projects_router.py b/backend/tests/test_projects_router.py new file mode 100644 index 000000000..6760cd496 --- /dev/null +++ b/backend/tests/test_projects_router.py @@ -0,0 +1,355 @@ +"""Router tests for the projects CRUD API (Phase 1). + +Harness mirrors ``test_channel_connections_router.py`` (real SQLAlchemy repo +on a temp sqlite engine + TestClient) and ``_router_auth_helpers`` (stub auth +middleware), extended to also set the request-scoped user ContextVar that +``ProjectRepository`` / ``ThreadMetaRepository`` resolve ownership from, and +to take the user id from a header so cross-user isolation can be exercised. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import anyio +import pytest +from fastapi import FastAPI, Request, Response +from fastapi.testclient import TestClient +from starlette.middleware.base import BaseHTTPMiddleware + +from app.gateway.authz import AuthContext, Permissions +from app.gateway.routers import projects +from deerflow.persistence.engine import close_engine, get_session_factory, init_engine +from deerflow.persistence.projects import ProjectRepository +from deerflow.persistence.thread_meta import THREAD_ARCHIVED_METADATA_KEY, THREAD_PROJECT_METADATA_KEY, ThreadMetaRepository +from deerflow.runtime.user_context import reset_current_user, set_current_user + +_STUB_PERMISSIONS: list[str] = [ + Permissions.THREADS_READ, + Permissions.THREADS_WRITE, + Permissions.THREADS_DELETE, + Permissions.RUNS_CREATE, + Permissions.RUNS_READ, + Permissions.RUNS_CANCEL, + Permissions.PROJECTS_READ, + Permissions.PROJECTS_WRITE, + Permissions.PROJECTS_DELETE, +] + +_USER_HEADER = "x-test-user" +_PERMISSIONS_HEADER = "x-test-permissions" + + +class _StubAuthMiddleware(BaseHTTPMiddleware): + """Stamp a fake AuthContext and set the user ContextVar per request. + + Mirrors production ``AuthMiddleware`` (``request.state.auth`` + + ``set_current_user``) so ``@require_permission`` and the + ContextVar-resolving repositories behave as in the real gateway. + The user id comes from the ``x-test-user`` header (default ``user-a``) so a + single app can drive multiple identities. The granted permissions come from + the ``x-test-permissions`` header (comma-separated; default the full stub + list) so scope-narrowed callers can be exercised. + """ + + async def dispatch(self, request: Request, call_next: Any) -> Response: + user_id = request.headers.get(_USER_HEADER, "user-a") + user = SimpleNamespace(id=user_id, system_role="user") + request.state.user = user + permissions_header = request.headers.get(_PERMISSIONS_HEADER) + permissions = permissions_header.split(",") if permissions_header else list(_STUB_PERMISSIONS) + request.state.auth = AuthContext(user=user, permissions=permissions) + token = set_current_user(user) + try: + return await call_next(request) + finally: + reset_current_user(token) + + +async def _init_db(tmp_path) -> None: + await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / 'projects.db'}", sqlite_dir=str(tmp_path)) + + +def _build_projects_app(tmp_path, *, project_repo: Any = "default") -> FastAPI: + """Build a stub-authed FastAPI app with real SQL project/thread repos.""" + anyio.run(_init_db, tmp_path) + sf = get_session_factory() + app = FastAPI() + app.add_middleware(_StubAuthMiddleware) + app.state.project_repo = ProjectRepository(sf) if project_repo == "default" else project_repo + app.state.thread_store = ThreadMetaRepository(sf) + app.include_router(projects.router) + return app + + +def _as_user(user_id: str) -> dict[str, str]: + return {_USER_HEADER: user_id} + + +def _seed_thread(app: FastAPI, thread_id: str, *, user_id: str, project_id: str | None = None, metadata: dict | None = None) -> dict: + """Create a thread row directly through the store as ``user_id``.""" + + async def _run() -> dict: + token = set_current_user(SimpleNamespace(id=user_id)) + try: + return await app.state.thread_store.create(thread_id, project_id=project_id, metadata=metadata) + finally: + reset_current_user(token) + + return anyio.run(_run) + + +def _search_threads(app: FastAPI, *, user_id: str, **kwargs: Any) -> list[dict]: + async def _run() -> list[dict]: + token = set_current_user(SimpleNamespace(id=user_id)) + try: + return await app.state.thread_store.search(**kwargs) + finally: + reset_current_user(token) + + return anyio.run(_run) + + +@pytest.fixture(autouse=True) +def _close_engine_after_test(): + yield + anyio.run(close_engine) + + +def test_create_list_get_project(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + created = client.post("/api/projects", json={"name": "Infra overhaul", "instructions": "ctx"}) + assert created.status_code == 201, created.text + project = created.json() + assert project["name"] == "Infra overhaul" + assert project["instructions"] == "ctx" + assert project["status"] == "active" + assert project["presentation"] == {} + + listing = client.get("/api/projects") + assert listing.status_code == 200 + assert [p["id"] for p in listing.json()["projects"]] == [project["id"]] + + fetched = client.get(f"/api/projects/{project['id']}") + assert fetched.status_code == 200 + assert fetched.json()["id"] == project["id"] + + +def test_get_patch_delete_foreign_project_returns_404(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "mine"}).json() + + # Fail closed: a foreign project is indistinguishable from a missing one. + assert client.get(f"/api/projects/{project['id']}", headers=_as_user("user-b")).status_code == 404 + assert client.patch(f"/api/projects/{project['id']}", json={"name": "x"}, headers=_as_user("user-b")).status_code == 404 + assert client.delete(f"/api/projects/{project['id']}", headers=_as_user("user-b")).status_code == 404 + + # Owner is unaffected. + assert client.get(f"/api/projects/{project['id']}").status_code == 200 + assert client.get("/api/projects", headers=_as_user("user-b")).json()["projects"] == [] + + +def test_patch_rename_does_not_touch_membership(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "old"}).json() + _seed_thread(app, "thread-1", user_id="user-a", project_id=project["id"]) + + patched = client.patch(f"/api/projects/{project['id']}", json={"name": "new"}) + assert patched.status_code == 200 + assert patched.json()["name"] == "new" + + members = _search_threads(app, user_id="user-a", project_id=project["id"]) + assert [t["thread_id"] for t in members] == ["thread-1"] + + +def test_archive_restore_idempotent(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + + for _ in range(2): + archived = client.post(f"/api/projects/{pid}/archive") + assert archived.status_code == 200 + assert archived.json()["status"] == "archived" + assert client.get("/api/projects", params={"status": "active"}).json()["projects"] == [] + assert [p["id"] for p in client.get("/api/projects", params={"status": "archived"}).json()["projects"]] == [pid] + + for _ in range(2): + restored = client.post(f"/api/projects/{pid}/restore") + assert restored.status_code == 200 + assert restored.json()["status"] == "active" + assert [p["id"] for p in client.get("/api/projects").json()["projects"]] == [pid] + + +def test_delete_unlinks_threads_and_keeps_them(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + _seed_thread(app, "thread-1", user_id="user-a", project_id=pid) + + deleted = client.delete(f"/api/projects/{pid}") + assert deleted.status_code == 204 + assert client.get(f"/api/projects/{pid}").status_code == 404 + + # Thread row survives with membership cleared. + threads = _search_threads(app, user_id="user-a") + assert [t["thread_id"] for t in threads] == ["thread-1"] + assert THREAD_PROJECT_METADATA_KEY not in threads[0]["metadata"] + assert _search_threads(app, user_id="user-a", project_id=pid) == [] + + +def test_project_threads_lists_members_only(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + p1 = client.post("/api/projects", json={"name": "p1"}).json() + p2 = client.post("/api/projects", json={"name": "p2"}).json() + _seed_thread(app, "thread-p1", user_id="user-a", project_id=p1["id"]) + _seed_thread(app, "thread-p2", user_id="user-a", project_id=p2["id"]) + _seed_thread(app, "thread-loose", user_id="user-a") + + response = client.get(f"/api/projects/{p1['id']}/threads") + assert response.status_code == 200 + assert [t["thread_id"] for t in response.json()] == ["thread-p1"] + + # Unknown project -> 404, not an empty list. + assert client.get("/api/projects/nope/threads").status_code == 404 + + +@pytest.mark.parametrize( + "params", + [ + {"limit": -1}, + {"limit": 0}, + {"limit": 1001}, + {"offset": -1}, + ], +) +def test_project_threads_rejects_out_of_bounds_pagination(tmp_path, params): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + response = client.get(f"/api/projects/{project['id']}/threads", params=params) + assert response.status_code == 422 + + +@pytest.mark.parametrize("params", [{"limit": 1, "offset": 0}, {"limit": 1000}]) +def test_project_threads_accepts_boundary_pagination(tmp_path, params): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + _seed_thread(app, "thread-1", user_id="user-a", project_id=project["id"]) + response = client.get(f"/api/projects/{project['id']}/threads", params=params) + assert response.status_code == 200 + assert [t["thread_id"] for t in response.json()] == ["thread-1"] + + +def test_project_threads_requires_threads_read(tmp_path): + """The listing returns thread records (titles/metadata), so it must require + ``threads:read`` alongside ``projects:read`` — matching + ``/api/threads/search``. A caller holding only ``projects:read`` (e.g. a + scoped PAT) must be denied.""" + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + _seed_thread(app, "thread-1", user_id="user-a", project_id=pid) + + without_threads_read = { + **_as_user("user-a"), + _PERMISSIONS_HEADER: ",".join(p for p in _STUB_PERMISSIONS if p != Permissions.THREADS_READ), + } + denied = client.get(f"/api/projects/{pid}/threads", headers=without_threads_read) + assert denied.status_code == 403 + + allowed = client.get(f"/api/projects/{pid}/threads", headers=_as_user("user-a")) + assert allowed.status_code == 200 + assert [t["thread_id"] for t in allowed.json()] == ["thread-1"] + + +def test_project_threads_redacts_legacy_auth_token(tmp_path): + """Legacy ``auth_token`` metadata must not leak through the project thread + listing — thread endpoints already redact it via + ``_MetadataRedactingResponse``; this listing must match.""" + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + _seed_thread(app, "thread-1", user_id="user-a", project_id=pid, metadata={"auth_token": "sk-legacy-secret", "keep": "x"}) + + response = client.get(f"/api/projects/{pid}/threads") + assert response.status_code == 200 + metadata = response.json()[0]["metadata"] + assert "auth_token" not in metadata + assert metadata["keep"] == "x" + assert metadata[THREAD_PROJECT_METADATA_KEY] == pid + + # Redaction is response-side only; the stored row keeps the legacy key. + stored = _search_threads(app, user_id="user-a", project_id=pid) + assert stored[0]["metadata"]["auth_token"] == "sk-legacy-secret" + + +def test_project_threads_metadata_unchanged_without_legacy_key(tmp_path): + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + _seed_thread(app, "thread-1", user_id="user-a", project_id=pid, metadata={"keep": "x"}) + + response = client.get(f"/api/projects/{pid}/threads") + assert response.status_code == 200 + assert response.json()[0]["metadata"] == {"keep": "x", THREAD_PROJECT_METADATA_KEY: pid} + + +def test_memory_backend_unavailable(tmp_path): + app = _build_projects_app(tmp_path, project_repo=None) + with TestClient(app) as client: + assert client.get("/api/projects").status_code == 503 + assert client.post("/api/projects", json={"name": "p"}).status_code == 503 + + +def test_project_threads_wire_shape_is_narrow(tmp_path): + """The listing must not leak store-row internals: ownership columns + (``user_id``/``assistant_id``) and any future ``ThreadMetaRow`` column + stay off the wire, and the OpenAPI schema is no longer empty. The model + pins exactly the fields ``ProjectThread`` declares.""" + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + _seed_thread(app, "thread-1", user_id="user-a", project_id=pid, metadata={"keep": "x"}) + + response = client.get(f"/api/projects/{pid}/threads") + assert response.status_code == 200 + rows = response.json() + assert len(rows) == 1 + row = rows[0] + assert set(row) == {"thread_id", "display_name", "created_at", "updated_at", "metadata"} + assert row["thread_id"] == "thread-1" + assert row["metadata"] == {"keep": "x", THREAD_PROJECT_METADATA_KEY: pid} + + +def test_project_threads_excludes_archived_members(tmp_path): + """Archived chats leave the project listing the same way they leave the + sidebar (``archived: false`` semantics): no silent normal-row rendering + of a retired chat on the project page. The store keeps the row; restore + flows through the global Archived tab as elsewhere.""" + app = _build_projects_app(tmp_path) + with TestClient(app) as client: + project = client.post("/api/projects", json={"name": "p"}).json() + pid = project["id"] + _seed_thread(app, "active-1", user_id="user-a", project_id=pid) + _seed_thread(app, "archived-1", user_id="user-a", project_id=pid, metadata={THREAD_ARCHIVED_METADATA_KEY: True}) + + response = client.get(f"/api/projects/{pid}/threads") + assert response.status_code == 200 + assert [t["thread_id"] for t in response.json()] == ["active-1"] + + # The archived row still exists in the store (unfiltered search). + stored = _search_threads(app, user_id="user-a", project_id=pid) + assert {t["thread_id"] for t in stored} == {"active-1", "archived-1"} diff --git a/backend/tests/test_thread_meta_repo.py b/backend/tests/test_thread_meta_repo.py index e18b9ccec..e8f64cc94 100644 --- a/backend/tests/test_thread_meta_repo.py +++ b/backend/tests/test_thread_meta_repo.py @@ -468,6 +468,204 @@ class TestThreadMetaRepository: hits = {r["thread_id"] for r in await repo.search(metadata={"id": large})} assert hits == {"t1"} + @pytest.mark.anyio + async def test_membership_exposed_via_reserved_metadata_key(self, repo): + from deerflow.persistence.thread_meta import THREAD_PROJECT_METADATA_KEY + + record = await repo.create("t1", user_id="u1") + assert THREAD_PROJECT_METADATA_KEY not in record["metadata"] + # membership is set in Task 4; the raw column must never leak top-level + assert "project_id" not in record + + @pytest.mark.anyio + async def test_row_to_dict_does_not_leak_reserved_key_into_stored_metadata(self, repo): + """Regression (Task 3 review): ``_row_to_dict`` must copy ``metadata_json`` + before injecting the reserved project key. Without the copy the injected + dict IS the ORM row's ``metadata_json`` object, so reading a member + thread mutates the row in place and a later update in the same session + persists ``deerflow_project_id`` into stored user metadata.""" + from deerflow.persistence.projects import ProjectRepository + from deerflow.persistence.thread_meta.model import ThreadMetaRow + + projects = ProjectRepository(repo._sf) + p = await projects.create(name="P", user_id="u1") + await repo.create("t1", user_id="u1", metadata={"keep": 1}) + # Assign membership directly on the row (store-level assignment is Task 4). + async with repo._sf() as session: + row = await session.get(ThreadMetaRow, "t1") + row.project_id = p["id"] + await session.commit() + + record = await repo.get("t1", user_id="u1") + assert record["metadata"]["deerflow_project_id"] == p["id"] + + # The review's failure mode is same-session: converting a row for read + # must not dirty the ORM row's stored dict, or a later update in that + # session persists the reserved key. ``repo.get`` never exposes its + # session, so replicate its conversion on a row from this session. + async with repo._sf() as session: + row = await session.get(ThreadMetaRow, "t1") + repo._row_to_dict(row) # same conversion repo.get performs + assert "deerflow_project_id" not in row.metadata_json + + await repo.update_metadata("t1", {"new": 2}, user_id="u1") + + async with repo._sf() as session: + row = await session.get(ThreadMetaRow, "t1") + assert "deerflow_project_id" not in row.metadata_json + assert row.metadata_json["keep"] == 1 + assert row.metadata_json["new"] == 2 + + @pytest.mark.anyio + async def test_set_project_moves_and_preserves_updated_at(self, repo): + from deerflow.persistence.projects import ProjectRepository + + projects = ProjectRepository(repo._sf) + p = await projects.create(name="P", user_id="u1") + await repo.create("t1", user_id="u1") + before = (await repo.get("t1", user_id="u1"))["updated_at"] + + assert await repo.set_project("t1", p["id"], user_id="u1") is True + record = await repo.get("t1", user_id="u1") + assert record["metadata"]["deerflow_project_id"] == p["id"] + assert record["updated_at"] == before # G5: move must not bump recency + + # move out + assert await repo.set_project("t1", None, user_id="u1") is True + assert "deerflow_project_id" not in (await repo.get("t1", user_id="u1"))["metadata"] + + @pytest.mark.anyio + async def test_set_project_rejects_foreign_thread_foreign_project_archived(self, repo): + from deerflow.persistence.projects import ProjectRepository + + projects = ProjectRepository(repo._sf) + mine = await projects.create(name="mine", user_id="u1") + await projects.create(name="theirs", user_id="u2") # a foreign-owned project exists + archived = await projects.create(name="arch", user_id="u1") + await projects.set_status(archived["id"], "archived", user_id="u1") + await repo.create("t1", user_id="u1") + await repo.create("t2", user_id="u2") + + assert await repo.set_project("t1", mine["id"], user_id="u2") is False # foreign thread + assert await repo.set_project("t2", mine["id"], user_id="u2") is False # foreign project + assert await repo.set_project("t1", archived["id"], user_id="u1") is False # archived + assert await repo.set_project("t1", "missing", user_id="u1") is False # missing + assert (await repo.get("t1", user_id="u1"))["metadata"].get("deerflow_project_id") is None + + @pytest.mark.anyio + async def test_run_admission_never_seeds_project_membership(self, repo): + """Negative contract: run admission never writes thread→project membership. + + A run admitted with the reserved ``deerflow_project_id`` metadata key + must leave the row's ``project_id`` column NULL, and the key must not + persist into ``metadata_json`` either — membership is written only by + POST /api/threads (create) and /threads/{id}/move. + """ + from app.gateway.services import _ensure_thread_metadata + from deerflow.persistence.projects import ProjectRepository + from deerflow.persistence.thread_meta.model import ThreadMetaRow + from deerflow.runtime.runs.manager import RunRecord + from deerflow.runtime.runs.schemas import DisconnectMode, RunStatus + from deerflow.runtime.runs.worker import RunContext + + projects = ProjectRepository(repo._sf) + project = await projects.create(name="P") + record = RunRecord(run_id="run-1", thread_id="t1", assistant_id="lead-agent", status=RunStatus.pending, on_disconnect=DisconnectMode.cancel, metadata={"deerflow_project_id": project["id"]}) + run_ctx = RunContext(checkpointer=None, thread_store=repo) + await _ensure_thread_metadata(run_ctx, record, owner_user_id=None) + + async with repo._sf() as session: + row = await session.get(ThreadMetaRow, "t1") + assert row is not None and row.project_id is None + assert "deerflow_project_id" not in row.metadata_json + + @pytest.mark.anyio + async def test_create_with_project_assignment_and_rejection(self, repo): + from deerflow.persistence.projects import ProjectNotAssignableError, ProjectRepository + + projects = ProjectRepository(repo._sf) + p = await projects.create(name="P", user_id="u1") + record = await repo.create("t1", user_id="u1", project_id=p["id"]) + assert record["metadata"]["deerflow_project_id"] == p["id"] + + with pytest.raises(ProjectNotAssignableError): + await repo.create("t2", user_id="u1", project_id="missing") + assert await repo.get("t2", user_id="u1") is None # no partial row + + @pytest.mark.anyio + async def test_search_project_filter_three_states(self, repo): + from deerflow.persistence.projects import ProjectRepository + from deerflow.persistence.thread_meta.base import PROJECT_FILTER_UNSET + + projects = ProjectRepository(repo._sf) + p = await projects.create(name="P", user_id="u1") + await repo.create("t1", user_id="u1", project_id=p["id"]) + await repo.create("t2", user_id="u1") + + all_rows = await repo.search(user_id="u1", project_id=PROJECT_FILTER_UNSET) + assert {r["thread_id"] for r in all_rows} == {"t1", "t2"} + unassigned = await repo.search(user_id="u1", project_id=None) + assert [r["thread_id"] for r in unassigned] == ["t2"] + in_project = await repo.search(user_id="u1", project_id=p["id"]) + assert [r["thread_id"] for r in in_project] == ["t1"] + + @pytest.mark.anyio + async def test_concurrent_move_vs_project_delete_never_dangles(self, repo): + """§5.2 race: move-vs-delete resolves to cleared membership or rejection.""" + import asyncio + + from deerflow.persistence.projects import ProjectRepository + + projects = ProjectRepository(repo._sf) + for i in range(10): + p = await projects.create(name=f"P{i}", user_id="u1") + tid = f"trace-{i}" + await repo.create(tid, user_id="u1") + moved, deleted = await asyncio.gather( + repo.set_project(tid, p["id"], user_id="u1"), + projects.delete(p["id"], user_id="u1"), + ) + record = await repo.get(tid, user_id="u1") + membership = record["metadata"].get("deerflow_project_id") + if moved and not deleted: + # delete lost the race before our read: membership may still be + # set only if the project row still exists + assert membership is None or await projects.get(membership, user_id="u1") is not None + else: + assert membership is None + + @pytest.mark.anyio + async def test_concurrent_create_vs_project_delete_never_dangles(self, repo): + """§14.14 race: create-with-assignment vs delete resolves to a + rejected create or cleared membership — never a thread whose + metadata references a deleted project.""" + import asyncio + + from deerflow.persistence.projects import ProjectNotAssignableError, ProjectRepository + + projects = ProjectRepository(repo._sf) + + async def create_in_project(tid: str, project_id: str) -> bool: + try: + await repo.create(tid, user_id="u1", project_id=project_id) + except ProjectNotAssignableError: + return False + return True + + for i in range(10): + p = await projects.create(name=f"P{i}", user_id="u1") + tid = f"trace-{i}" + await asyncio.gather( + create_in_project(tid, p["id"]), + projects.delete(p["id"], user_id="u1"), + ) + record = await repo.get(tid, user_id="u1") + if record is not None: + membership = record["metadata"].get("deerflow_project_id") + if membership is not None: + # A carried key is only valid while the project row exists. + assert await projects.get(membership, user_id="u1") is not None + class TestJsonMatchCompilation: """Verify compiled SQL for both SQLite and PostgreSQL dialects.""" diff --git a/backend/tests/test_threads_router.py b/backend/tests/test_threads_router.py index 31fcce0bd..840f00d73 100644 --- a/backend/tests/test_threads_router.py +++ b/backend/tests/test_threads_router.py @@ -4,6 +4,7 @@ from contextlib import asynccontextmanager from types import SimpleNamespace from unittest.mock import AsyncMock, patch +import anyio import pytest from _router_auth_helpers import make_authed_test_app from fastapi import FastAPI, HTTPException @@ -17,10 +18,19 @@ from langgraph.types import Overwrite from app.gateway import services as gateway_services from app.gateway.routers import thread_runs, threads from deerflow.config.paths import Paths -from deerflow.persistence.thread_meta import THREAD_PINNED_METADATA_KEY, InvalidMetadataFilterError +from deerflow.persistence.engine import close_engine, get_session_factory, init_engine +from deerflow.persistence.projects import ProjectRepository +from deerflow.persistence.thread_meta import ( + PROJECT_FILTER_UNSET, + THREAD_PINNED_METADATA_KEY, + THREAD_PROJECT_METADATA_KEY, + InvalidMetadataFilterError, + ThreadMetaRepository, +) from deerflow.persistence.thread_meta.memory import THREADS_NS, MemoryThreadMetaStore from deerflow.runtime import ConflictError, ThreadOperationKind from deerflow.runtime.checkpoint_state import CheckpointStateAccessor +from deerflow.runtime.user_context import reset_current_user, set_current_user _ISO_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}") @@ -47,11 +57,11 @@ class _PermissiveThreadMetaStore(MemoryThreadMetaStore): return not require_existing return True - async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None): # type: ignore[override] - return await super().create(thread_id, assistant_id=assistant_id, user_id=None, display_name=display_name, metadata=metadata) + async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None, project_id=None): # type: ignore[override] + return await super().create(thread_id, assistant_id=assistant_id, user_id=None, display_name=display_name, metadata=metadata, project_id=project_id) - async def search(self, *, metadata=None, status=None, limit=100, offset=0, user_id=None, archived=None): # type: ignore[override] - return await super().search(metadata=metadata, status=status, limit=limit, offset=offset, user_id=None, archived=archived) + async def search(self, *, metadata=None, status=None, limit=100, offset=0, user_id=None, archived=None, project_id=PROJECT_FILTER_UNSET): # type: ignore[override] + return await super().search(metadata=metadata, status=status, limit=limit, offset=offset, user_id=None, archived=archived, project_id=project_id) class _ThreadTestRunManager: @@ -692,7 +702,7 @@ def test_create_thread_returns_existing_when_insert_loses_race() -> None: super().__init__(backing) self._raised = False - async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None): # type: ignore[override] + async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None, project_id=None): # type: ignore[override] if not self._raised: self._raised = True await super().create( @@ -701,6 +711,7 @@ def test_create_thread_returns_existing_when_insert_loses_race() -> None: user_id=user_id, display_name=display_name, metadata=metadata, + project_id=project_id, ) raise IntegrityError( "INSERT INTO threads_meta", @@ -713,6 +724,7 @@ def test_create_thread_returns_existing_when_insert_loses_race() -> None: user_id=user_id, display_name=display_name, metadata=metadata, + project_id=project_id, ) app.state.thread_store = _RacingThreadMetaStore(store) @@ -752,10 +764,10 @@ def test_insert_race_recovery_claims_unscoped_row_for_trusted_owner() -> None: """Our insert loses to a competing create that already wrote an unscoped row, exactly the interleaving the recovery path exists for.""" - async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None): # type: ignore[override] + async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None, project_id=None): # type: ignore[override] # The competing request commits its (owner-less) row here, then our # insert loses the primary-key race. - await super().create(thread_id, user_id=None, metadata=metadata) + await super().create(thread_id, user_id=None, metadata=metadata, project_id=project_id) raise IntegrityError( "INSERT INTO threads_meta", {}, @@ -3934,3 +3946,328 @@ def test_archive_patch_cannot_modify_another_users_thread(): response = client.patch("/api/threads/private", json={"metadata": {"deerflow_archived": True}}) assert response.status_code == 404 assert asyncio.run(store.aget(THREADS_NS, "private")).value["metadata"] == {} + + +# --------------------------------------------------------------------------- +# Project membership surface (Phase 1): create/search/move + reserved key +# --------------------------------------------------------------------------- +# +# The memory harness above cannot exercise project membership (the memory +# store ignores it by design), so these tests build a stub-authed app on real +# SQL repos — same harness shape as ``test_projects_router.py``. + +from test_projects_router import _StubAuthMiddleware # noqa: E402 + + +async def _init_threads_db(tmp_path) -> None: + await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / 'threads.db'}", sqlite_dir=str(tmp_path)) + + +def _build_project_threads_app(tmp_path) -> FastAPI: + """Stub-authed app with real SQL thread/project repos.""" + anyio.run(_init_threads_db, tmp_path) + session_factory = get_session_factory() + app = FastAPI() + app.add_middleware(_StubAuthMiddleware) + app.state.thread_store = ThreadMetaRepository(session_factory) + app.state.project_repo = ProjectRepository(session_factory) + app.state.checkpointer = InMemorySaver() + app.state.run_manager = _ThreadTestRunManager() + app.include_router(threads.router) + return app + + +def _create_project(app: FastAPI, *, user_id: str = "user-a", name: str = "Project") -> dict: + async def _run() -> dict: + token = set_current_user(SimpleNamespace(id=user_id)) + try: + return await app.state.project_repo.create(name=name) + finally: + reset_current_user(token) + + return anyio.run(_run) + + +def _archive_project(app: FastAPI, project_id: str, *, user_id: str = "user-a") -> None: + async def _run() -> None: + token = set_current_user(SimpleNamespace(id=user_id)) + try: + await app.state.project_repo.set_status(project_id, "archived") + finally: + reset_current_user(token) + + anyio.run(_run) + + +@pytest.fixture(autouse=True) +def _close_sql_engine_after_test(): + yield + anyio.run(close_engine) + + +def test_create_thread_with_project_assigns(tmp_path): + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + with TestClient(app) as client: + created = client.post("/api/threads", json={"project_id": project["id"]}) + assert created.status_code == 200, created.text + thread_id = created.json()["thread_id"] + + fetched = client.get(f"/api/threads/{thread_id}") + assert fetched.json()["metadata"][THREAD_PROJECT_METADATA_KEY] == project["id"] + + hits = client.post("/api/threads/search", json={"project_id": project["id"]}).json() + assert [h["thread_id"] for h in hits] == [thread_id] + + +def test_create_thread_response_includes_persisted_project_membership(tmp_path): + """The create response must echo the persisted record, not body.metadata. + + The store stamps ``metadata.deerflow_project_id`` from the assigned + ``project_id`` column; a response built from ``body.metadata`` omits it + and disagrees with the idempotent-retry response for the same thread. + """ + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + with TestClient(app) as client: + created = client.post("/api/threads", json={"project_id": project["id"]}) + assert created.status_code == 200, created.text + assert created.json()["metadata"][THREAD_PROJECT_METADATA_KEY] == project["id"] + + retry = client.post("/api/threads", json={"thread_id": created.json()["thread_id"], "project_id": project["id"]}) + assert retry.status_code == 200, retry.text + assert retry.json() == created.json() + + +def test_create_thread_response_without_project_has_no_membership_key(tmp_path): + """Regression guard: no project_id → the key must not appear in the response.""" + app = _build_project_threads_app(tmp_path) + with TestClient(app) as client: + created = client.post("/api/threads", json={"metadata": {"keep": "v"}}) + assert created.status_code == 200, created.text + assert created.json()["metadata"] == {"keep": "v"} + + +def test_create_thread_with_missing_or_foreign_project_404(tmp_path): + app = _build_project_threads_app(tmp_path) + foreign = _create_project(app, user_id="user-b", name="Foreign") + with TestClient(app) as client: + missing = client.post("/api/threads", json={"thread_id": "thread-missing-proj", "project_id": "no-such-project"}) + assert missing.status_code == 404, missing.text + assert missing.json()["detail"] == "Project not found" + + foreign_resp = client.post("/api/threads", json={"thread_id": "thread-foreign-proj", "project_id": foreign["id"]}) + assert foreign_resp.status_code == 404, foreign_resp.text + + +def test_create_thread_with_project_in_memory_mode_404(): + """Memory mode has no projects backend: a project-scoped create must fail + closed with the same 404 the SQL store produces for a missing project — + not silently persist an unassigned thread whose run would then proceed + outside the selected project (``ensureProjectThread`` keeps the composer + text for a retry on this failure).""" + app, _, _ = _build_thread_app() + with TestClient(app) as client: + created = client.post("/api/threads", json={"thread_id": "thread-mem-proj", "project_id": "p1"}) + assert created.status_code == 404, created.text + assert created.json()["detail"] == "Project not found" + + # The store's project filter fails closed too; no row was persisted. + hits = client.post("/api/threads/search", json={"project_id": "p1"}).json() + assert hits == [] + + # Unscoped creates still work in memory mode. + plain = client.post("/api/threads", json={"thread_id": "thread-mem-plain"}) + assert plain.status_code == 200, plain.text + + +def test_create_and_patch_strip_deerflow_project_id_metadata_key(tmp_path): + app = _build_project_threads_app(tmp_path) + with TestClient(app) as client: + created = client.post("/api/threads", json={"metadata": {THREAD_PROJECT_METADATA_KEY: "forged", "keep": "v"}}) + assert created.status_code == 200, created.text + thread_id = created.json()["thread_id"] + assert created.json()["metadata"] == {"keep": "v"} + + fetched = client.get(f"/api/threads/{thread_id}") + assert fetched.json()["metadata"] == {"keep": "v"} + + patched = client.patch(f"/api/threads/{thread_id}", json={"metadata": {THREAD_PROJECT_METADATA_KEY: "forged-2"}}) + assert patched.status_code == 200, patched.text + assert THREAD_PROJECT_METADATA_KEY not in patched.json()["metadata"] + + +def test_search_threads_project_filter_absent_null_value(tmp_path): + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + with TestClient(app) as client: + in_project = client.post("/api/threads", json={"project_id": project["id"]}).json()["thread_id"] + unassigned = client.post("/api/threads", json={}).json()["thread_id"] + + all_hits = {t["thread_id"] for t in client.post("/api/threads/search", json={}).json()} + assert all_hits == {in_project, unassigned} + + only_project = {t["thread_id"] for t in client.post("/api/threads/search", json={"project_id": project["id"]}).json()} + assert only_project == {in_project} + + only_unassigned = {t["thread_id"] for t in client.post("/api/threads/search", json={"project_id": None}).json()} + assert only_unassigned == {unassigned} + + +def test_move_thread_to_project_and_out(tmp_path): + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + with TestClient(app) as client: + thread_id = client.post("/api/threads", json={}).json()["thread_id"] + + moved = client.post(f"/api/threads/{thread_id}/move", json={"project_id": project["id"]}) + assert moved.status_code == 200, moved.text + assert moved.json()["metadata"][THREAD_PROJECT_METADATA_KEY] == project["id"] + + out = client.post(f"/api/threads/{thread_id}/move", json={"project_id": None}) + assert out.status_code == 200, out.text + assert THREAD_PROJECT_METADATA_KEY not in out.json()["metadata"] + + # The key is required-but-nullable: omitting it is a 422. + missing_key = client.post(f"/api/threads/{thread_id}/move", json={}) + assert missing_key.status_code == 422, missing_key.text + + +def test_move_thread_to_archived_or_foreign_project_404(tmp_path): + app = _build_project_threads_app(tmp_path) + archived = _create_project(app, name="Archived") + foreign = _create_project(app, user_id="user-b", name="Foreign") + _archive_project(app, archived["id"]) + with TestClient(app) as client: + thread_id = client.post("/api/threads", json={}).json()["thread_id"] + + to_archived = client.post(f"/api/threads/{thread_id}/move", json={"project_id": archived["id"]}) + assert to_archived.status_code == 404, to_archived.text + + to_foreign = client.post(f"/api/threads/{thread_id}/move", json={"project_id": foreign["id"]}) + assert to_foreign.status_code == 404, to_foreign.text + + +def test_move_thread_does_not_bump_updated_at(tmp_path): + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + with TestClient(app) as client: + thread_id = client.post("/api/threads", json={}).json()["thread_id"] + before = client.get(f"/api/threads/{thread_id}").json()["updated_at"] + + moved = client.post(f"/api/threads/{thread_id}/move", json={"project_id": project["id"]}) + assert moved.status_code == 200, moved.text + assert moved.json()["updated_at"] == before + + after = client.get(f"/api/threads/{thread_id}").json()["updated_at"] + assert after == before + + +def _seed_branchable_thread(app: FastAPI, thread_id: str) -> None: + """Write a three-turn conversation so a middle AI turn can be branched.""" + human_1 = HumanMessage(id="human-1", content="First question") + ai_1 = AIMessage(id="ai-1", content="First answer") + human_2 = HumanMessage(id="human-2", content="Second question") + + async def _seed(parent_config: dict) -> None: + after_human_1 = await _write_checkpoint( + app.state.checkpointer, + thread_id, + str(uuid6()), + [human_1], + step=1, + parent_config=parent_config, + ) + after_ai_1 = await _write_checkpoint( + app.state.checkpointer, + thread_id, + str(uuid6()), + [human_1, ai_1], + step=2, + parent_config=after_human_1, + ) + await _write_checkpoint( + app.state.checkpointer, + thread_id, + str(uuid6()), + [human_1, ai_1, human_2], + step=3, + parent_config=after_ai_1, + ) + + initial = asyncio.run(app.state.checkpointer.aget_tuple({"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}})) + assert initial is not None + asyncio.run(_seed(initial.config)) + + +def test_branch_inherits_source_project_membership(tmp_path): + """A branch of a project thread stays in the source thread's project. + + Branching writes a new thread_meta row; without inheritance it is + unassigned and the sidebar surfaces it under Recent chats instead of the + source thread's project group. + """ + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + source_thread_id = "source-project-branch" + + with TestClient(app) as client: + created = client.post( + "/api/threads", + json={"thread_id": source_thread_id, "project_id": project["id"]}, + ) + assert created.status_code == 200, created.text + _seed_branchable_thread(app, source_thread_id) + + branch = client.post( + f"/api/threads/{source_thread_id}/branches", + json={"message_id": "ai-1", "message_ids": ["ai-1"]}, + ) + assert branch.status_code == 200, branch.text + branch_id = branch.json()["thread_id"] + assert branch.json()["parent_thread_id"] == source_thread_id + + fetched = client.get(f"/api/threads/{branch_id}") + assert fetched.status_code == 200, fetched.text + assert fetched.json()["metadata"][THREAD_PROJECT_METADATA_KEY] == project["id"] + + hits = client.post("/api/threads/search", json={"project_id": project["id"]}).json() + assert {h["thread_id"] for h in hits} == {source_thread_id, branch_id} + + unassigned = client.post("/api/threads/search", json={"project_id": None}).json() + assert [h["thread_id"] for h in unassigned] == [] + + +def test_branch_from_archived_project_thread_degrades_to_unassigned(tmp_path): + """Branching stays available when the source project was archived meanwhile. + + The branch inherits through the same validated create path as assignment; + an archived project is no longer assignable, so the branch row is created + unassigned (pre-inheritance behavior) instead of failing the request. + """ + app = _build_project_threads_app(tmp_path) + project = _create_project(app) + source_thread_id = "source-archived-branch" + + with TestClient(app) as client: + created = client.post( + "/api/threads", + json={"thread_id": source_thread_id, "project_id": project["id"]}, + ) + assert created.status_code == 200, created.text + _seed_branchable_thread(app, source_thread_id) + _archive_project(app, project["id"]) + + branch = client.post( + f"/api/threads/{source_thread_id}/branches", + json={"message_id": "ai-1", "message_ids": ["ai-1"]}, + ) + assert branch.status_code == 200, branch.text + branch_id = branch.json()["thread_id"] + + fetched = client.get(f"/api/threads/{branch_id}") + assert fetched.status_code == 200, fetched.text + assert THREAD_PROJECT_METADATA_KEY not in fetched.json()["metadata"] + + unassigned = client.post("/api/threads/search", json={"project_id": None}).json() + assert {h["thread_id"] for h in unassigned} == {branch_id} diff --git a/docs/database-forward-revision-recovery.md b/docs/database-forward-revision-recovery.md new file mode 100644 index 000000000..fce9ab22e --- /dev/null +++ b/docs/database-forward-revision-recovery.md @@ -0,0 +1,70 @@ +# Recovering the original thread-incarnation database revision + +The Projects build requires `projects` and `threads_meta.project_id`. An older +deployment may have stamped `0019_thread_incarnations` on a database containing +only `0018_oauth_identity_pg_partial` plus two nullable `VARCHAR(32)` columns: +`threads_meta.incarnation` and `mcp_tasks.thread_incarnation`. That shape cannot +serve this build's repositories. Startup now rejects it without changing the +schema or revision, and reports the missing tables/columns. + +Normal databases on this tree's known migration chain upgrade automatically. +The procedure below is only for the exact original incarnation rollout shape. +An incarnation-stamped database that already has all current ORM tables and +columns can still use the audited compatibility exception without re-stamping. + +## Offline migration + +1. Stop every Gateway, scheduler, and other process writing to the database. + Take a restorable database backup and rehearse these steps on a copy. +2. Verify there is exactly one `alembic_version` row, containing + `0019_thread_incarnations`. Inspect the owning deployment's migration and + actual database schema: it must be the local 0018 schema plus only the two + nullable columns above, with no added defaults, constraints, tables, indexes, + or data backfills. Neither `projects` nor `threads_meta.project_id` may already + exist for this recovery path. If the shape differs, use a migration reviewed + for that deployment; do not use the commands below. +3. From this checkout's `backend/`, set `DEERFLOW_RECOVERY_DATABASE_URL` to the + target async SQLAlchemy URL (`sqlite+aiosqlite:////absolute/path/database.db` + or `postgresql+asyncpg://…`). For Postgres, also set + `DEERFLOW_RECOVERY_POSTGRES_SCHEMA` to the configured application schema, if + one is used. Keep credentials out of shell history. +4. Rebase the version marker to the verified common parent and run the normal + migrations. `purge=True` is necessary because this tree does not contain the + out-of-tree revision; it replaces the version row, not application data. + + ```bash + uv run python - <<'PY' + import asyncio + import os + + from alembic import command + from sqlalchemy.ext.asyncio import create_async_engine + + from deerflow.persistence.bootstrap import _get_alembic_config + + engine = create_async_engine(os.environ["DEERFLOW_RECOVERY_DATABASE_URL"]) + cfg = _get_alembic_config( + engine, + postgres_schema=os.environ.get("DEERFLOW_RECOVERY_POSTGRES_SCHEMA", ""), + ) + command.stamp(cfg, "0018_oauth_identity_pg_partial", purge=True) + command.upgrade(cfg, "head") + asyncio.run(engine.dispose()) + PY + ``` + + This applies `0019_projects` and `0020_threads_meta_project_id`, preserving + the two incarnation columns and their existing values. Do not stamp directly + to head: that would skip the DDL and reproduce the missing-column failure. +5. Confirm the version is `0020_threads_meta_project_id`, the project table and + membership column/index exist, and existing incarnation values are retained. + Start this build, verify existing conversations load and a new conversation + can be created, then resume service. Do not restart older binaries that + cannot read this tree's head revision. + +Bootstrap never performs this re-stamp itself. The regression in +`backend/tests/test_persistence_forward_revision_compat.py` constructs the +original schema, verifies startup rejection, and exercises the recovery while +checking thread reads/inserts and preservation of incarnation data. The future +incarnation migration must chain from the current local head and handle these +already-present nullable columns idempotently. diff --git a/frontend/src/AGENTS.md b/frontend/src/AGENTS.md index a13bb4292..875218dff 100644 --- a/frontend/src/AGENTS.md +++ b/frontend/src/AGENTS.md @@ -63,6 +63,13 @@ as removable "missing" entries instead of silently widening the allowlist. 6. Components subscribe to thread state and render updates +Project moves in `core/threads/hooks.ts` cancel all per-thread metadata query +variants after the write succeeds, merge only `deerflow_project_id`, then +invalidate/refetch that metadata prefix. This fences delayed pre-move reads and +restarts initial reads that have no cached snapshot. Search and project-thread +lists are invalidated on settlement. Keep the delayed-read regression in +`tests/unit/core/threads/move-thread.dom.test.tsx` for moves and removal. + The chat header's context-window control is intentionally persistent: while `context_usage` is unavailable, `ContextUsageBadge` renders a gauge placeholder rather than unmounting; once data arrives, the same position shows the percentage. `useThreadTokenUsage` retains placeholder data only when the response `thread_id` still matches the active route, so same-thread refetches do not flicker and cross-thread navigation never displays the previous chat's usage. Settings skill uploads reject archives larger than 100 MiB before starting the @@ -79,7 +86,7 @@ Composer drafts are tab-scoped browser state. `core/threads/composer-draft.ts` s Auth UI note: the login page's "keep me signed in" option submits only `remember_me` to the Gateway and may persist only the email address through `core/auth/remember-login.ts`. Passwords and tokens must never be stored in frontend storage; the `HttpOnly access_token` and readable `csrf_token` cookies remain Gateway-owned. -`/goal` and `/compact` are built-in composer commands, not skill activations. `src/components/workspace/input-box.tsx` intercepts `/goal`, `/goal clear`, and `/goal ` before normal chat submission, calling Gateway `GET/PUT/DELETE /api/threads/{thread_id}/goal`. Setting `/goal ` also submits the condition text as the next user task so the agent starts running immediately; status and clear do not start a run. Goal and compact requests are tied to the current `threadId` with an `AbortController`, so switching threads or unmounting the composer aborts in-flight requests and stale responses cannot update the new thread's composer state. The chat pages render `GoalStatus` above the composer from `AgentThreadState.goal`, with local optimistic state until an incremental goal update or final state reload arrives. `/compact` calls `POST /api/threads/{thread_id}/compact` to summarize older active context while leaving the full visible chat history intact; it is skipped on new/empty threads and blocked server-side while a run is in flight. Thread rename uses the same serialized state-write route; the rename dialog stays open and surfaces the server error when an active run returns 409. +`/goal` and `/compact` are built-in composer commands, not skill activations. `src/components/workspace/input-box.tsx` intercepts `/goal`, `/goal clear`, and `/goal ` before normal chat submission, calling Gateway `GET/PUT/DELETE /api/threads/{thread_id}/goal`. Setting `/goal ` also submits the condition text as the next user task so the agent starts running immediately; status and clear do not start a run. On a project-scoped new chat (`/workspace/chats/new?project=…`), the chat page's project pre-create runs before the goal PUT via the composer's `onPrepareThread` callback: the goal endpoint materializes a missing thread row itself, and an unassigned row would make the later idempotent thread create return it without assigning the project. Goal and compact requests are tied to the current `threadId` with an `AbortController`, so switching threads or unmounting the composer aborts in-flight requests and stale responses cannot update the new thread's composer state. The chat pages render `GoalStatus` above the composer from `AgentThreadState.goal`, with local optimistic state until an incremental goal update or final state reload arrives. `/compact` calls `POST /api/threads/{thread_id}/compact` to summarize older active context while leaving the full visible chat history intact; it is skipped on new/empty threads and blocked server-side while a run is in flight. Thread rename uses the same serialized state-write route; the rename dialog stays open and surfaces the server error when an active run returns 409. The `/` skill list stays reachable after a skill is selected: typing `/` in the editable text beside the chip reopens it, and picking an entry swaps the chip rather than adding a second one, because the wire format carries exactly one leading `/skill`. That list offers skills only while a chip is selected — a builtin command owns the whole composer line, so `/goal` behind a selected skill would submit as chat text instead of running the command. The trigger itself is unchanged: a slash only opens the list at the start of the input (`getLeadingSlashSkillQuery`), pinned by `tests/e2e/chat.spec.ts`. diff --git a/frontend/src/app/workspace/projects/[id]/page.tsx b/frontend/src/app/workspace/projects/[id]/page.tsx new file mode 100644 index 000000000..c6fd6c618 --- /dev/null +++ b/frontend/src/app/workspace/projects/[id]/page.tsx @@ -0,0 +1,278 @@ +"use client"; + +import { + Archive, + Folder, + MessageSquarePlus, + RotateCcw, + Trash2, +} from "lucide-react"; +import Link from "next/link"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + Empty, + EmptyHeader, + EmptyMedia, + EmptyTitle, +} from "@/components/ui/empty"; +import { Input } from "@/components/ui/input"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { ProjectThreadsSection } from "@/components/workspace/projects/project-threads-section"; +import { + WorkspaceBody, + WorkspaceContainer, + WorkspaceHeader, +} from "@/components/workspace/workspace-container"; +import { useI18n } from "@/core/i18n/hooks"; +import { + useArchiveProject, + useDeleteProject, + useInfiniteProjectThreads, + usePatchProject, + useProject, + useRestoreProject, + type Project, +} from "@/core/projects"; +import { isIMEComposing } from "@/lib/ime"; + +function newProjectChatPath(projectId: string): string { + return `/workspace/chats/new?project=${encodeURIComponent(projectId)}`; +} + +export default function ProjectPage() { + const { t } = useI18n(); + const { id: projectId } = useParams<{ id: string }>(); + const projectQuery = useProject(projectId); + const project = projectQuery.data; + // Skip the threads request entirely when the project 404s — the endpoint + // itself 404s for a missing project, so firing it would be pure noise. + const threadsQuery = useInfiniteProjectThreads(projectId, { + enabled: project != null, + }); + + useEffect(() => { + document.title = project?.name + ? `${project.name} - ${t.pages.appName}` + : `${t.projects.title} - ${t.pages.appName}`; + }, [project?.name, t.projects.title, t.pages.appName]); + + return ( + + + + +
+ {projectQuery.isError ? ( + + ) : project == null ? ( +
+ {t.common.loading} +
+ ) : ( + <> + + + {/* Keying on updated_at re-syncs the rename draft whenever the + project changes underneath (e.g. rename round-trip). */} + + + )} +
+ + + + ); +} + +function ProjectNotFoundState() { + const { t } = useI18n(); + return ( + + + + + + {t.projects.notFound} + + + ); +} + +function ProjectHeader({ project }: { project: Project }) { + const { t } = useI18n(); + return ( +
+

+ {project.name} +

+ {project.status === "archived" && ( + {t.projects.archived} + )} + {project.status !== "archived" && ( + + )} +
+ ); +} + +function ProjectSettingsSection({ project }: { project: Project }) { + const { t } = useI18n(); + const router = useRouter(); + const patchProject = usePatchProject(); + const archiveProject = useArchiveProject(); + const restoreProject = useRestoreProject(); + const deleteProject = useDeleteProject(); + + const [name, setName] = useState(project.name); + const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false); + + const trimmedName = name.trim(); + const canSave = + trimmedName.length > 0 && + trimmedName !== project.name && + !patchProject.isPending; + const isArchived = project.status === "archived"; + + const handleRename = () => { + if (!canSave) { + return; + } + patchProject.mutate( + { projectId: project.id, input: { name: trimmedName } }, + { + onError: (error) => { + toast.error( + error instanceof Error && error.message + ? error.message + : t.common.renameFailed, + ); + }, + }, + ); + }; + + const handleArchiveToggle = () => { + const mutation = isArchived ? restoreProject : archiveProject; + const fallback = isArchived + ? t.projects.restoreFailed + : t.projects.archiveFailed; + mutation.mutate(project.id, { + onError: (error) => { + toast.error( + error instanceof Error && error.message ? error.message : fallback, + ); + }, + }); + }; + + const handleDelete = () => { + deleteProject.mutate(project.id, { + onSuccess: () => { + router.push("/workspace/chats"); + }, + onError: (error) => { + toast.error( + error instanceof Error && error.message + ? error.message + : t.projects.deleteFailed, + ); + }, + }); + }; + + return ( +
+

+ {t.projects.settings} +

+
+ +
+ setName(e.target.value)} + placeholder={t.projects.namePlaceholder} + onKeyDown={(e) => { + if (e.key === "Enter" && !isIMEComposing(e)) { + e.preventDefault(); + handleRename(); + } + }} + /> + +
+
+
+ + +
+ + + + {t.projects.deleteProject} + + {t.projects.deleteProjectConfirm} + + + + + + + + +
+ ); +} diff --git a/frontend/src/components/workspace/chats/chat-page.tsx b/frontend/src/components/workspace/chats/chat-page.tsx index 28a9a344e..2b2a16a86 100644 --- a/frontend/src/components/workspace/chats/chat-page.tsx +++ b/frontend/src/components/workspace/chats/chat-page.tsx @@ -1,6 +1,9 @@ "use client"; -import { useRouter } from "next/navigation"; +import { useQueryClient } from "@tanstack/react-query"; +import { Folder } from "lucide-react"; +import Link from "next/link"; +import { useRouter, useSearchParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; @@ -44,9 +47,12 @@ import { import { isHiddenFromUIMessage } from "@/core/messages/utils"; import { useModels } from "@/core/models/hooks"; import { useNotification } from "@/core/notification/hooks"; +import { useProject } from "@/core/projects"; import { useLocalSettings, useThreadSettings } from "@/core/settings"; +import { createThread } from "@/core/threads/api"; import { useBranchThread, + INFINITE_THREADS_QUERY_KEY_PREFIX, useThreadMetadata, useThreadStream, useThreadTokenUsage, @@ -55,7 +61,7 @@ import { selectContextUsage, threadTokenUsageToTokenUsage, } from "@/core/threads/token-usage"; -import { textOfMessage } from "@/core/threads/utils"; +import { projectIdOfThread, textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; import { cn } from "@/lib/utils"; @@ -66,14 +72,25 @@ import { useThreadChat } from "./use-thread-chat"; export default function ChatPage() { const { t } = useI18n(); const router = useRouter(); + const searchParams = useSearchParams(); const { threadId, setThreadId, isNewThread, setIsNewThread, isMock } = useThreadChat(); + // Project-scoped new chat: `/workspace/chats/new?project={id}` assigns the + // thread to the project on the FIRST submit — primarily via an explicit + // `POST /api/threads` pre-create with `project_id`, with the run-request + // metadata seed as the fallback channel (the SDK's own threads.create + // strips the reserved key, so the seed alone races the sidebar). Only + // meaningful while the thread is still lazy (`isNewThread`); once + // materialized the URL is replaced with the thread route and the param is + // gone. Invalid ids are dropped by backend admission. + const projectParam = isNewThread ? searchParams.get("project") : null; // `isNewThread` tracks whether the backend has the thread yet — gates the // SDK's history fetch (see issue #2746). `isWelcomeMode` is the visual // welcome layout (centered input, hero, quick actions); we flip it to false // the moment the user submits so the UI animates immediately, even though // `isNewThread` stays true until the backend actually creates the thread. const [isWelcomeMode, setIsWelcomeMode] = useState(isNewThread); + const queryClient = useQueryClient(); const [settings, setSettings] = useThreadSettings(threadId); const [localSettings, setLocalSettings] = useLocalSettings(); const { enabled: browserControlEnabled } = useBrowserControlEnabled(); @@ -178,15 +195,70 @@ export default function ChatPage() { threadMetadata.isLoading, ]); + // Born assigned: pre-create the thread row with its project so the sidebar + // lists it under the project immediately. Idempotent server-side on + // `thread_id`, so retrying the same first message (same `threadId` while + // `isNewThread`) reuses the existing row instead of double-creating. This + // is the sole membership channel — run requests never carry the project + // key. + // + // Also runs before InputBox issues a `/goal ` PUT (via + // `onPrepareThread`): the goal endpoint materializes a missing thread row + // itself, and an unassigned row would make this later idempotent create a + // membership no-op. + const ensureProjectThread = useCallback(async () => { + if (!projectParam) { + return; + } + try { + await createThread(threadId, projectParam); + void queryClient.invalidateQueries({ + queryKey: INFINITE_THREADS_QUERY_KEY_PREFIX, + }); + } catch (error) { + // Any failure (e.g. the project was deleted/archived between page load + // and submit): do NOT submit unassigned. Reject so PromptInput keeps + // the composer's text for a retry; the send in-flight guard is never + // engaged on this path, so retrying works immediately. + toast.error(t.projects.projectUnavailable); + throw error; + } + }, [threadId, projectParam, queryClient, t]); + + // Submission fence. The cleanup runs when `threadId` changes (conversation + // switch on a persisted page — sidebar navigation keeps this component + // mounted), when the new-chat project scope changes (the sidebar "New + // chat" link can drop `?project=` without a pathname change), or on + // unmount. handleSubmit awaits the project pre-create before sending, and + // a navigation during that await must not let the stale continuation + // start a run for the abandoned conversation: its onStart would rewrite + // the newly selected conversation's URL and its completion would clear + // the new composer. + const submissionEpochRef = useRef(0); + useEffect(() => { + return () => { + submissionEpochRef.current += 1; + }; + }, [threadId, projectParam]); + const handleSubmit = useCallback( - (message: PromptInputMessage, options?: InputBoxSubmitOptions) => { + async (message: PromptInputMessage, options?: InputBoxSubmitOptions) => { + const submissionEpoch = submissionEpochRef.current; + await ensureProjectThread(); + // Conversation switched (or the page unmounted) while the project + // pre-create was pending: drop the submission. Reject silently — the + // user has already moved on, so a toast would land on the new + // conversation — and PromptInput keeps the current composer text. + if (submissionEpochRef.current !== submissionEpoch) { + throw new Error("thread-submission-stale"); + } const sendPromise = sendMessage(threadId, message, undefined, options); if (message.files.length > 0) { return sendPromise; } void sendPromise; }, - [sendMessage, threadId], + [sendMessage, threadId, ensureProjectThread], ); const handleSubmitHumanInput = useCallback( async (request: HumanInputRequest, response: HumanInputResponse) => { @@ -270,6 +342,14 @@ export default function ChatPage() { [thread.messages], ); + // Project affiliation chip: shown once the materialized thread's metadata + // carries `deerflow_project_id` (written by the create/move endpoints and + // exposed here read-only). + const affiliatedProjectId = + !isNewThread && !isMock && threadMetadata.data + ? projectIdOfThread(threadMetadata.data) + : null; + return ( {!isMock && } -
+
)} + {affiliatedProjectId && ( + + )}
{!isNewThread && @@ -463,6 +546,7 @@ export default function ChatPage() { setSettings("context", context) } onGoalChange={setLocalGoal} + onPrepareThread={ensureProjectThread} onSubmit={handleSubmit} onStop={handleStop} /> @@ -489,3 +573,24 @@ export default function ChatPage() { ); } + +/** + * Small chip in the chat header linking to the thread's project. Hidden + * while the project lookup is pending or when it fails (e.g. the project + * was deleted) — an unresolvable affiliation degrades silently. + */ +function ProjectAffiliationBadge({ projectId }: { projectId: string }) { + const { data: project } = useProject(projectId); + if (!project) { + return null; + } + return ( + + + {project.name} + + ); +} diff --git a/frontend/src/components/workspace/chats/use-thread-chat.ts b/frontend/src/components/workspace/chats/use-thread-chat.ts index 3a324d5b9..c65fa001e 100644 --- a/frontend/src/components/workspace/chats/use-thread-chat.ts +++ b/frontend/src/components/workspace/chats/use-thread-chat.ts @@ -25,6 +25,10 @@ export function resetThreadChatAfterDelete(detail: ThreadChatResetDetail) { ); } +// Sentinel distinguishing "no identity minted yet" from a minted identity +// whose scope is a project-less new chat (project === null). +const NEW_CHAT_SCOPE_UNSET = Symbol("deerflow.newChatScopeUnset"); + export function useThreadChat() { const { thread_id: threadIdFromPath } = useParams<{ thread_id: string }>(); const pathname = usePathname(); @@ -79,6 +83,38 @@ export function useThreadChat() { setThreadIdState(threadIdFromPath); }, [pathname, threadIdFromPath]); + // A new-chat identity is minted for the entry scope (the new chat's + // `project` query). The sidebar "New chat" link can leave + // `/new?project=…` for plain `/new` without a pathname change, so the + // pathname sync above keeps the old identity — which may already be + // pre-created under the previous project, and whose in-flight submission + // would otherwise survive the navigation. Re-mint whenever the project + // scope changes while still on a new-chat path. + const newChatScopeRef = useRef(NEW_CHAT_SCOPE_UNSET); + useEffect(() => { + const project = isNewPath ? searchParams.get("project") : null; + if (!isNewPath) { + newChatScopeRef.current = NEW_CHAT_SCOPE_UNSET; + return; + } + if (newChatScopeRef.current === project) { + return; + } + const previousScope = newChatScopeRef.current; + newChatScopeRef.current = project; + if (previousScope === NEW_CHAT_SCOPE_UNSET) { + return; + } + // Scope changed (project added, removed, or swapped): the old identity + // belongs to the previous scope, so allocate a fresh one. The threadId + // flip also runs every fence keyed on `threadId` (composer goal aborts, + // chat-page submission epoch). + const nextThreadId = uuid(); + newThreadIdRef.current = nextThreadId; + setIsNewThreadState(true); + setThreadIdState(nextThreadId); + }, [isNewPath, pathname, searchParams]); + useEffect(() => { const handleReset = (event: Event) => { const detail = (event as CustomEvent).detail; diff --git a/frontend/src/components/workspace/input-box.tsx b/frontend/src/components/workspace/input-box.tsx index 235e7bba8..4500bc2ed 100644 --- a/frontend/src/components/workspace/input-box.tsx +++ b/frontend/src/components/workspace/input-box.tsx @@ -297,6 +297,7 @@ export function InputBox({ onContextChange, onFollowupsVisibilityChange, onGoalChange, + onPrepareThread, onSubmit, onStop, ...props @@ -340,6 +341,16 @@ export function InputBox({ ) => void; onFollowupsVisibilityChange?: (visible: boolean) => void; onGoalChange?: (goal: GoalState | null) => void; + /** + * Prepare a not-yet-materialized thread before a builtin command creates + * it server-side. The `/goal ` PUT endpoint materializes a + * missing thread row itself, so a project-scoped new chat uses this to + * assign membership first — the later idempotent thread create would + * otherwise return that unassigned row without the project. Only runs for + * goal-set: status/clear never create a thread server-side. Rejecting + * aborts the command and keeps the composer's text for a retry. + */ + onPrepareThread?: () => void | Promise; onSubmit?: ( message: PromptInputMessage, options?: InputBoxSubmitOptions, @@ -1193,6 +1204,35 @@ export function InputBox({ // clearing it (PromptInput only preserves input on a rejected submit). return Promise.reject(new Error("goal-too-long")); } + if (submitAction.command.kind === "set") { + // A goal-set PUT creates the thread server-side when missing, so a + // project-scoped new chat must assign membership first. The prepare + // callback toasts its own failure; reject so the composer keeps the + // text for a retry instead of issuing an unassigned goal. + // + // Fence against conversation switches while preparation runs: the + // goal PUT registers its AbortController only when it starts, so + // the thread-change/unmount cleanup cannot cancel an in-flight + // prepare. Capture the goal-request epoch before the await — the + // cleanup bumps it via abortGoalRequest — and drop the stale + // continuation before it can clear the new conversation's composer + // or launch the abandoned submission. + const requestEpoch = goalRequestStateRef.current.sequence; + try { + await onPrepareThread?.(); + } catch (error) { + return Promise.reject( + error instanceof Error + ? error + : new Error("thread preparation failed"), + ); + } + if (goalRequestStateRef.current.sequence !== requestEpoch) { + // Reject (not resolve) so PromptInput keeps the current + // conversation's composer text untouched. + return Promise.reject(new Error("goal-preparation-stale")); + } + } promptHistoryIndexRef.current = null; promptHistoryDraftRef.current = ""; setFollowups([]); @@ -1229,6 +1269,7 @@ export function InputBox({ handleCompactCommand, handleGoalCommand, handleStopStreaming, + onPrepareThread, selectedSlashSkill, status, submitThreadMessage, diff --git a/frontend/src/components/workspace/move-to-project-menu.tsx b/frontend/src/components/workspace/move-to-project-menu.tsx new file mode 100644 index 000000000..402d2e874 --- /dev/null +++ b/frontend/src/components/workspace/move-to-project-menu.tsx @@ -0,0 +1,216 @@ +"use client"; + +import { Check, FolderInput, FolderMinus, Plus } from "lucide-react"; +import { useCallback, useState } from "react"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuSub, + DropdownMenuSubContent, + DropdownMenuSubTrigger, +} from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; +import { useI18n } from "@/core/i18n/hooks"; +import { useCreateProject, useProjects } from "@/core/projects"; +import { useMoveThreadToProject } from "@/core/threads/hooks"; +import type { AgentThread } from "@/core/threads/types"; +import { projectIdOfThread } from "@/core/threads/utils"; +import { isIMEComposing } from "@/lib/ime"; + +/** + * "Move to project" submenu for a thread's dropdown menu: active projects + * (check mark on the current one), "New project…" (opens the create dialog), + * and "Remove from project" when the thread is assigned. Archived projects + * never appear because only the "active" list is queried. + * + * Renders ONLY menu content. DropdownMenuContent unmounts its children when + * the menu closes, so the create dialog lives outside the menu: the + * "New project…" item delegates to `onNewProject`, and the caller mounts + * `NewProjectDialog` as a sibling of the DropdownMenu (the same pattern the + * rename dialog uses in `ThreadSidebarItem`). + */ +export function MoveToProjectMenu({ + thread, + onNewProject, + onMoveProject, +}: { + thread: AgentThread; + onNewProject: () => void; + onMoveProject: (projectId: string | null) => void; +}) { + const { t } = useI18n(); + const { data: projects } = useProjects("active"); + + const currentProjectId = projectIdOfThread(thread); + + // Presentational only: the move mutation lives on the persistent + // `ThreadSidebarItem` (this submenu unmounts when the dropdown closes, so + // a mutation owned here would lose its error handling on failure). + const handleMove = useCallback( + (projectId: string | null) => { + if (projectId === currentProjectId) { + return; + } + onMoveProject(projectId); + }, + [currentProjectId, onMoveProject], + ); + + return ( + + + + {t.projects.moveToProject} + + + {projects?.map((project) => ( + handleMove(project.id)} + > + {project.id === currentProjectId ? ( + + ) : ( + + ))} + {projects && projects.length > 0 && } + + + {t.projects.newProject}… + + {currentProjectId !== null && ( + handleMove(null)}> + + {t.projects.removeFromProject} + + )} + + + {t.projects.moveToProjectHint} + + + + ); +} + +/** + * "New project" dialog for the move-to-project flow. Must be mounted + * OUTSIDE the thread's DropdownMenuContent (as a sibling of the + * DropdownMenu) so it survives the menu closing. On successful create, the + * thread is moved into the new project. + */ +export function NewProjectDialog({ + thread, + open, + onOpenChange, +}: { + thread: AgentThread; + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const { t } = useI18n(); + const { mutate: createProject, isPending: isCreating } = useCreateProject(); + const { mutate: moveThreadToProject } = useMoveThreadToProject(); + + const [createName, setCreateName] = useState(""); + + const handleOpenChange = useCallback( + (nextOpen: boolean) => { + if (!nextOpen) { + setCreateName(""); + } + onOpenChange(nextOpen); + }, + [onOpenChange], + ); + + const handleCreateSubmit = useCallback(() => { + const name = createName.trim(); + if (!name || isCreating) { + return; + } + createProject( + { name }, + { + onSuccess: (project) => { + handleOpenChange(false); + moveThreadToProject( + { threadId: thread.thread_id, projectId: project.id }, + { + onError: (error) => { + toast.error( + error instanceof Error && error.message + ? error.message + : t.projects.moveFailed, + ); + }, + }, + ); + }, + onError: (error) => { + toast.error( + error instanceof Error && error.message + ? error.message + : t.projects.createFailed, + ); + }, + }, + ); + }, [ + createProject, + createName, + isCreating, + handleOpenChange, + moveThreadToProject, + t.projects.createFailed, + t.projects.moveFailed, + thread.thread_id, + ]); + + return ( + + + + {t.projects.newProject} + +
+ setCreateName(e.target.value)} + placeholder={t.projects.namePlaceholder} + onKeyDown={(e) => { + if (e.key === "Enter" && !isIMEComposing(e)) { + e.preventDefault(); + handleCreateSubmit(); + } + }} + /> +
+ + + + + + + ); +} diff --git a/frontend/src/components/workspace/projects-section.tsx b/frontend/src/components/workspace/projects-section.tsx new file mode 100644 index 000000000..281ca003c --- /dev/null +++ b/frontend/src/components/workspace/projects-section.tsx @@ -0,0 +1,355 @@ +"use client"; + +import { + Archive, + ChevronRight, + Folder, + FolderTree, + List, + Plus, +} from "lucide-react"; +import Link from "next/link"; +import { useParams, usePathname } from "next/navigation"; +import { useCallback, useMemo, useState } from "react"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { + SidebarGroup, + SidebarGroupContent, + SidebarGroupLabel, + SidebarMenu, + SidebarMenuAction, + SidebarMenuButton, + SidebarMenuItem, +} from "@/components/ui/sidebar"; +import { useI18n } from "@/core/i18n/hooks"; +import { useCreateProject, useProjects, type Project } from "@/core/projects"; +import { useLocalSettings } from "@/core/settings"; +import { isStaticWebsiteOnly } from "@/core/static-mode"; +import { useInfiniteThreads } from "@/core/threads/hooks"; +import { flattenThreadBranches } from "@/core/threads/thread-branch-tree"; +import { buildThreadListModel } from "@/core/threads/thread-list-model"; +import type { AgentThread } from "@/core/threads/types"; +import { pathOfThread, projectIdOfThread } from "@/core/threads/utils"; +import { env } from "@/env"; +import { isIMEComposing } from "@/lib/ime"; + +import { ThreadSidebarItem } from "./recent-chat-list"; + +function projectPath(projectId: string): string { + return `/workspace/projects/${projectId}`; +} + +function ProjectThreadGroup({ + project, + threads, + recentThreadId, +}: { + project: Project; + threads: readonly AgentThread[]; + /** Global most-recent thread id — mirrors flat mode's `threads[0]?.thread_id`. */ + recentThreadId: string | undefined; +}) { + const pathname = usePathname(); + const [open, setOpen] = useState(true); + const href = projectPath(project.id); + const branchEntries = useMemo( + () => flattenThreadBranches([...threads]), + [threads], + ); + return ( + + + + + + {project.name} + + + + + + + + + + + {branchEntries.map((entry) => ( + + ))} + + + + ); +} + +function ArchivedProjectsGroup({ + projects, + threadsByProject, + recentThreadId, +}: { + projects: readonly Project[]; + threadsByProject: ReadonlyMap; + recentThreadId: string | undefined; +}) { + const { t } = useI18n(); + const [open, setOpen] = useState(false); + return ( + + + + + + {t.projects.archived} + + + + + + + {projects.map((project) => ( + + ))} + + + + ); +} + +/** + * Grouped mode: active projects as collapsible headers with their recent + * threads (grouped client-side from the already-fetched infinite thread + * pages), plus a collapsed "Archived" section at the bottom. + */ +function GroupedProjectList() { + const { data: activeProjects } = useProjects("active"); + const { data: archivedProjects } = useProjects("archived"); + const { data: infiniteThreads } = useInfiniteThreads({ + archived: + env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" ? undefined : false, + }); + const { thread_id: threadIdFromPath } = useParams<{ + thread_id: string; + agent_name?: string; + }>(); + const threadListModel = useMemo( + () => buildThreadListModel(infiniteThreads?.pages ?? []), + [infiniteThreads?.pages], + ); + // Same value flat mode passes to `ThreadSidebarItem` — the global most-recent + // thread, NOT the capped displayedThreads or any group-local first entry. + const globalRecentThreadId = threadListModel.threads[0]?.thread_id; + // Mirror `RecentChatList`'s active-thread exception: the path-active thread + // is appended even when it falls beyond the unpinned display cap, so it + // must join the partition input too — otherwise an active assigned chat + // renders in neither the flat list nor its project group. + const partitionableThreads = useMemo(() => { + if ( + !threadIdFromPath || + threadListModel.displayedThreads.some( + (thread) => thread.thread_id === threadIdFromPath, + ) + ) { + return threadListModel.displayedThreads; + } + const activeThread = threadListModel.byId.get(threadIdFromPath); + return activeThread + ? [...threadListModel.displayedThreads, activeThread] + : threadListModel.displayedThreads; + }, [threadIdFromPath, threadListModel]); + const threadsByProject = useMemo(() => { + const grouped = new Map(); + for (const thread of partitionableThreads) { + const projectId = projectIdOfThread(thread); + if (projectId === null) { + continue; + } + const projectThreads = grouped.get(projectId); + if (projectThreads) { + projectThreads.push(thread); + } else { + grouped.set(projectId, [thread]); + } + } + return grouped; + }, [partitionableThreads]); + + if ( + (!activeProjects || activeProjects.length === 0) && + (!archivedProjects || archivedProjects.length === 0) + ) { + return null; + } + return ( + + {activeProjects?.map((project) => ( + + ))} + {archivedProjects && archivedProjects.length > 0 && ( + + )} + + ); +} + +export function ProjectsSection() { + const { t } = useI18n(); + const [settings, setSettings] = useLocalSettings(); + const groupByProject = settings.projectsDisplayMode === "grouped"; + const { mutate: createProject, isPending: isCreating } = useCreateProject(); + + const [createDialogOpen, setCreateDialogOpen] = useState(false); + const [createName, setCreateName] = useState(""); + + const handleCreateSubmit = useCallback(() => { + const name = createName.trim(); + if (!name || isCreating) { + return; + } + createProject( + { name }, + { + onSuccess: () => { + setCreateDialogOpen(false); + setCreateName(""); + }, + onError: (error) => { + toast.error( + error instanceof Error && error.message + ? error.message + : t.projects.createFailed, + ); + }, + }, + ); + }, [createProject, createName, isCreating, t.projects.createFailed]); + + // Static-demo mode has no Gateway and no projects: hide the whole section + // (New project button, grouped toggle, and groups are all dead actions). + if (isStaticWebsiteOnly()) { + return null; + } + + return ( + + + {t.projects.title} + + + + + + {groupByProject && ( + + + + )} + + {/* New project dialog */} + + + + {t.projects.newProject} + +
+ setCreateName(e.target.value)} + placeholder={t.projects.namePlaceholder} + onKeyDown={(e) => { + if (e.key === "Enter" && !isIMEComposing(e)) { + e.preventDefault(); + handleCreateSubmit(); + } + }} + /> +
+ + + + + + + + ); +} diff --git a/frontend/src/components/workspace/projects/project-threads-section.tsx b/frontend/src/components/workspace/projects/project-threads-section.tsx new file mode 100644 index 000000000..f66f25179 --- /dev/null +++ b/frontend/src/components/workspace/projects/project-threads-section.tsx @@ -0,0 +1,89 @@ +"use client"; + +import Link from "next/link"; + +import { Button } from "@/components/ui/button"; +import { VirtualThreadList } from "@/components/workspace/thread-list-virtualizer"; +import { useI18n } from "@/core/i18n/hooks"; +import { type ProjectThreadsQueryResult } from "@/core/projects"; +import { pathOfThread } from "@/core/threads/utils"; +import { formatTimeAgo } from "@/core/utils/datetime"; +import { cn } from "@/lib/utils"; + +export function ProjectThreadsSection({ + query, +}: { + query: ProjectThreadsQueryResult; +}) { + const { t } = useI18n(); + const threads = query.data?.pages.flatMap((page) => page) ?? []; + return ( +
+

+ {t.projects.threads} +

+
+ {query.isError ? ( +
+ {t.projects.threadsLoadFailed} +
+ ) : threads.length === 0 && !query.isLoading ? ( +
+ {t.projects.empty} +
+ ) : ( + // The page scrolls inside its own ScrollArea; the list windows rows + // against that viewport so paging through a long-lived project never + // grows unbounded DOM (same windowing the sidebar and + // /workspace/chats use). + ( + +
+
+ {thread.display_name?.trim() + ? thread.display_name + : t.projects.untitled} +
+ {thread.updated_at && ( +
+ {formatTimeAgo(thread.updated_at)} +
+ )} +
+ + )} + /> + )} +
+ {query.hasNextPage && ( + + )} +
+ ); +} diff --git a/frontend/src/components/workspace/recent-chat-list.tsx b/frontend/src/components/workspace/recent-chat-list.tsx index b1f0e5b35..15af7cf3e 100644 --- a/frontend/src/components/workspace/recent-chat-list.tsx +++ b/frontend/src/components/workspace/recent-chat-list.tsx @@ -49,32 +49,55 @@ import { resetThreadChatAfterDelete } from "@/components/workspace/chats/use-thr import { getAPIClient } from "@/core/api"; import { writeTextToClipboard } from "@/core/clipboard"; import { useI18n } from "@/core/i18n/hooks"; +import { useProjects } from "@/core/projects"; +import { useLocalSettings } from "@/core/settings"; +import { isStaticWebsiteOnly } from "@/core/static-mode"; import { exportThread, type ThreadExportFormat } from "@/core/threads/export"; import { useDeleteThread, useInfiniteThreads, + useMoveThreadToProject, usePinThread, useRenameThread, } from "@/core/threads/hooks"; -import { flattenThreadBranches } from "@/core/threads/thread-branch-tree"; +import { + flattenThreadBranches, + type ThreadBranchEntry, +} from "@/core/threads/thread-branch-tree"; import { buildThreadListModel } from "@/core/threads/thread-list-model"; import type { AgentThread, AgentThreadState } from "@/core/threads/types"; import { channelSourceOfThread, isThreadPinned, pathOfThread, + projectIdOfThread, titleOfThread, } from "@/core/threads/utils"; import { env } from "@/env"; import { isIMEComposing } from "@/lib/ime"; +import { MoveToProjectMenu, NewProjectDialog } from "./move-to-project-menu"; import { ThreadChannelIcon } from "./thread-channel-source"; import { VirtualThreadList } from "./thread-list-virtualizer"; import { useThreadArchiveAction } from "./use-thread-archive-action"; -export function RecentChatList() { +/** + * A single thread row in the sidebar: link + hover action menu (pin, rename, + * share, export, delete) + rename dialog. Shared by the flat recent-chat list + * and the grouped-by-project rendering in `ProjectsSection`. + */ +export function ThreadSidebarItem({ + thread, + isActive, + branchEntry, + recentThreadId, +}: { + thread: AgentThread; + isActive: boolean; + branchEntry?: ThreadBranchEntry | undefined; + recentThreadId?: string | undefined; +}) { const { t } = useI18n(); - const archiveAction = useThreadArchiveAction(); const router = useRouter(); const pathname = usePathname(); const { thread_id: threadIdFromPath, agent_name: agentNameFromPath } = @@ -82,131 +105,80 @@ export function RecentChatList() { thread_id: string; agent_name?: string; }>(); - const { - data: infiniteThreads, - fetchNextPage, - hasNextPage, - isFetchingNextPage, - } = useInfiniteThreads({ - archived: - env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" ? undefined : false, - }); - const threadListModel = useMemo( - () => buildThreadListModel(infiniteThreads?.pages ?? []), - [infiniteThreads?.pages], - ); - const { threads } = threadListModel; - const displayedThreads = useMemo(() => { - if ( - !threadIdFromPath || - threadListModel.displayedThreads.some( - (thread) => thread.thread_id === threadIdFromPath, - ) - ) { - return threadListModel.displayedThreads; - } - const activeThread = threadListModel.byId.get(threadIdFromPath); - return activeThread - ? [...threadListModel.displayedThreads, activeThread] - : threadListModel.displayedThreads; - }, [threadIdFromPath, threadListModel]); - const branchList = useMemo(() => { - const entries = flattenThreadBranches(displayedThreads); - return { - entriesById: new Map( - entries.map((entry) => [entry.thread.thread_id, entry]), - ), - threads: entries.map((entry) => entry.thread), - }; - }, [displayedThreads]); - - const sentinelRef = useRef(null); - useEffect(() => { - const element = sentinelRef.current; - if (!element || !hasNextPage || !threadListModel.canLoadMore) { - return; - } - const observer = new IntersectionObserver( - ([entry]) => { - if (entry?.isIntersecting && hasNextPage && !isFetchingNextPage) { - void fetchNextPage(); - } - }, - { rootMargin: "120px 0px 120px 0px" }, - ); - observer.observe(element); - return () => observer.disconnect(); - }, [ - fetchNextPage, - hasNextPage, - isFetchingNextPage, - threadListModel.canLoadMore, - ]); - const { mutate: deleteThread } = useDeleteThread(); const { mutate: renameThread } = useRenameThread(); const { mutate: updatePinnedThread } = usePinThread(); + // The move mutation is owned here (not inside `MoveToProjectMenu`) because + // selecting a project closes the dropdown and unmounts the menu — a + // per-mutate `onError` registered there would be dropped before a failed + // request settles, failing silently. This row persists, so a hook-level + // `onError` always fires. Same ownership-hoisting precedent as the + // `NewProjectDialog` below. + const { mutate: moveThreadToProject } = useMoveThreadToProject({ + onError: (error) => { + toast.error( + error instanceof Error && error.message + ? error.message + : t.projects.moveFailed, + ); + }, + }); + + const handleMoveProject = useCallback( + (projectId: string | null) => { + moveThreadToProject({ threadId: thread.thread_id, projectId }); + }, + [moveThreadToProject, thread.thread_id], + ); + const archiveAction = useThreadArchiveAction(); - // Rename dialog state const [renameDialogOpen, setRenameDialogOpen] = useState(false); - const [renameThreadId, setRenameThreadId] = useState(null); const [renameValue, setRenameValue] = useState(""); + const [newProjectDialogOpen, setNewProjectDialogOpen] = useState(false); - const handleDelete = useCallback( - (thread: AgentThread) => { - const currentPathname = - typeof window === "undefined" ? pathname : window.location.pathname; - const threadPath = pathOfThread(thread); - const nextThreadPath = pathOfThread("new", { - agent_name: agentNameFromPath, - }); - const isNewThreadPath = currentPathname === nextThreadPath; - const isCurrentThread = - thread.thread_id === threadIdFromPath || - threadPath === currentPathname || - (isNewThreadPath && threads[0]?.thread_id === thread.thread_id); + const handleDelete = useCallback(() => { + const currentPathname = + typeof window === "undefined" ? pathname : window.location.pathname; + const threadPath = pathOfThread(thread); + const nextThreadPath = pathOfThread("new", { + agent_name: agentNameFromPath, + }); + const isNewThreadPath = currentPathname === nextThreadPath; + const isCurrentThread = + thread.thread_id === threadIdFromPath || + threadPath === currentPathname || + (isNewThreadPath && recentThreadId === thread.thread_id); - deleteThread({ - threadId: thread.thread_id, - onRemoteDeleted: isCurrentThread - ? () => { - resetThreadChatAfterDelete({ - deletedThreadId: thread.thread_id, - nextPath: nextThreadPath, - force: true, - }); - void router.replace(nextThreadPath); - } - : undefined, - }); - }, - [ - agentNameFromPath, - deleteThread, - pathname, - router, - threadIdFromPath, - threads, - ], - ); - - const handleRenameClick = useCallback( - (threadId: string, currentTitle: string) => { - setRenameThreadId(threadId); - setRenameValue(currentTitle); - setRenameDialogOpen(true); - }, - [], - ); + deleteThread({ + threadId: thread.thread_id, + onRemoteDeleted: isCurrentThread + ? () => { + resetThreadChatAfterDelete({ + deletedThreadId: thread.thread_id, + nextPath: nextThreadPath, + force: true, + }); + void router.replace(nextThreadPath); + } + : undefined, + }); + }, [ + agentNameFromPath, + deleteThread, + pathname, + recentThreadId, + router, + thread, + threadIdFromPath, + ]); const handleRenameSubmit = useCallback(() => { - if (renameThreadId && renameValue.trim()) { + if (renameValue.trim()) { renameThread( - { threadId: renameThreadId, title: renameValue.trim() }, + { threadId: thread.thread_id, title: renameValue.trim() }, { onSuccess: () => { setRenameDialogOpen(false); - setRenameThreadId(null); setRenameValue(""); }, onError: (error) => { @@ -219,54 +191,48 @@ export function RecentChatList() { }, ); } - }, [renameThread, renameThreadId, renameValue, t.common.renameFailed]); + }, [renameThread, thread.thread_id, renameValue, t.common.renameFailed]); - const handleTogglePin = useCallback( - (thread: AgentThread) => { - updatePinnedThread( - { - threadId: thread.thread_id, - pinned: !isThreadPinned(thread), + const handleTogglePin = useCallback(() => { + updatePinnedThread( + { + threadId: thread.thread_id, + pinned: !isThreadPinned(thread), + }, + { + onError: (err) => { + toast.error( + err instanceof Error ? err.message : t.chats.pinChatFailed, + ); }, - { - onError: (err) => { - toast.error( - err instanceof Error ? err.message : t.chats.pinChatFailed, - ); - }, - }, - ); - }, - [t.chats.pinChatFailed, updatePinnedThread], - ); + }, + ); + }, [t.chats.pinChatFailed, thread, updatePinnedThread]); - const handleShare = useCallback( - async (thread: AgentThread) => { - // Always use Vercel URL for sharing so others can access - const VERCEL_URL = "https://deer-flow-v2.vercel.app"; - const isLocalhost = - window.location.hostname === "localhost" || - window.location.hostname === "127.0.0.1"; - // On localhost: use Vercel URL; On production: use current origin - const baseUrl = isLocalhost ? VERCEL_URL : window.location.origin; - const shareUrl = `${baseUrl}${pathOfThread(thread)}`; - try { - const didCopy = await writeTextToClipboard(shareUrl); - if (!didCopy) { - toast.error(t.clipboard.failedToCopyToClipboard); - return; - } - - toast.success(t.clipboard.linkCopied); - } catch { + const handleShare = useCallback(async () => { + // Always use Vercel URL for sharing so others can access + const VERCEL_URL = "https://deer-flow-v2.vercel.app"; + const isLocalhost = + window.location.hostname === "localhost" || + window.location.hostname === "127.0.0.1"; + // On localhost: use Vercel URL; On production: use current origin + const baseUrl = isLocalhost ? VERCEL_URL : window.location.origin; + const shareUrl = `${baseUrl}${pathOfThread(thread)}`; + try { + const didCopy = await writeTextToClipboard(shareUrl); + if (!didCopy) { toast.error(t.clipboard.failedToCopyToClipboard); + return; } - }, - [t], - ); + + toast.success(t.clipboard.linkCopied); + } catch { + toast.error(t.clipboard.failedToCopyToClipboard); + } + }, [t, thread]); const handleExport = useCallback( - async (thread: AgentThread, format: ThreadExportFormat) => { + async (format: ThreadExportFormat) => { try { const apiClient = getAPIClient(); const state = await apiClient.threads.getState( @@ -283,217 +249,137 @@ export function RecentChatList() { toast.error(t.common.exportFailed); } }, - [t], + [t, thread], ); - if (threads.length === 0) { - return null; - } + const channelSource = channelSourceOfThread(thread); + const pinned = isThreadPinned(thread); + const parentTitle = branchEntry?.parentThread + ? titleOfThread(branchEntry.parentThread) + : null; + const title = titleOfThread(thread); + const branchLabel = parentTitle + ? t.chats.branchLabel(title, parentTitle) + : undefined; + return ( - <> - - - {env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" - ? t.sidebar.recentChats - : t.sidebar.demoChats} - - - - {/* Keep pagination at the old list boundary when this switches to virtual rows. */} -
+ + 0 ? branchEntry.depth : undefined + } + data-branch-parent-id={branchEntry?.parentThread?.thread_id} + href={pathOfThread(thread)} + title={branchLabel} + > + {branchEntry && branchEntry.depth > 0 && ( +