Merge remote-tracking branch 'origin/develop' into iop-vc-preview

Conflicts resolved as in git-cascade candidate ab64912569, with the
design review migrations 0809 to 0814 kept after develop's 0158 and
0159. Fixes for develop's changes that merged without conflict:

- files_branch.clj reads main FOR UPDATE: develop's get-file* refuses
  :lock-for-share? on a data read.
- sidebar.cljs passes the scroll-store prop in review mode: develop
  removed the local scroll-store* atom.
- The snapshot rename guard tests pass :file-id, now required by
  update-file-snapshot, and the branch import test expects develop's
  :not-supported refusal of every in-place import.

AI-assisted-by: claude-opus-5-5
Signed-off-by: Álvaro Tejero Cantero <alvorithm@teje.ro>
This commit is contained in:
Álvaro Tejero Cantero 2026-11-17 22:30:52 +01:00
commit b7df44cad0
No known key found for this signature in database
564 changed files with 51553 additions and 6642 deletions

View File

@ -31,14 +31,17 @@ memory; it does not repeat it.
layout, entry rules, the `## Breaking change note` comment contract, the
advisory (`:lock:`) contract, pre-flight checks, anomaly types, and pitfalls.
2. `mem:scripts/changelog` — `boom`, `security`, `check-merged`, `cross-ref`,
`report`.
`dev-only-bugs`, `report`.
3. `mem:scripts/gh` — `issues`, `issue`, `comments`, `prs`, `advisories`.
## Commands
```bash
python3 scripts/gh.py issues "<VERSION>" --exclude "release blocker,no changelog"
python3 scripts/gh.py issues "<VERSION>" --exclude "no changelog"
python3 scripts/gh.py issues "<VERSION>" --label "release blocker" --exclude "no changelog" # closed non-bug residue to review by hand
python3 scripts/gh.py advisories --patched-version "<VERSION>"
python3 scripts/changelog.py dev-only-bugs "<VERSION>" # list bugs that never reached production
python3 scripts/changelog.py dev-only-bugs "<VERSION>" --apply # label the sure ones `no changelog`
python3 scripts/changelog.py check-merged <PR numbers...>
python3 scripts/changelog.py boom "<VERSION>" # print the :boom: section
python3 scripts/changelog.py boom "<VERSION>" --write # replace it in CHANGES.md
@ -54,4 +57,5 @@ python3 scripts/changelog.py report "<VERSION>"
- Only closed, non-excluded milestone issues are included; the issue is the changelog unit and the PR is inline.
- `:boom:` and `:rocket:` are label-generated and `:lock:` is advisory-generated: never hand-edit them; edit the label, the `## Breaking change note` comment or the advisory and re-run the command.
- Verify PR merge status before writing; only merged PRs are referenced.
- Bugs that never reached production stay out: `dev-only-bugs --apply` labels the sure ones; candidates are the user's call, never label them yourself.
- Fill `patched_versions` in the advisories before running `security`; unpublished advisories are valid.

View File

@ -9,7 +9,8 @@
app.common.data.macros/get-in clojure.core/get-in
app.common.data.macros/with-open clojure.core/with-open
app.common.data.macros/select-keys clojure.core/select-keys
app.common.logging/with-context clojure.core/do}
app.common.logging/with-context clojure.core/do
benches.render-wasm.errors/attempt clojure.core/do}
:hooks
{:analyze-call
@ -24,7 +25,11 @@
shadow.lazy/loadable hooks.export/rumext-lazycomponent
app.util.i18n/tr hooks.i18n/tr-dynamic
app.common.i18n/tr hooks.i18n/tr-dynamic
}}
benches.render-wasm.declarations/defscene hooks.benches/defscene
benches.render-wasm.declarations/defcase hooks.benches/defcase
benches.render-wasm.runtime.commands/defcommand hooks.benches/defcommand
benches.render-wasm.runtime.protocol/resolve-or-reject hooks.benches/resolve-or-reject
benches.render-wasm.runtime.protocol/resolve-or-reject-with hooks.benches/resolve-or-reject-with}}
:output
{:exclude-files
@ -36,7 +41,7 @@
"^(?:backend|frontend|exporter|common)/deps.edn"
"^(?:backend|frontend|exporter|common)/scripts/"
"^(?:backend|frontend|exporter|common)/dev/"
"^(?:backend|frontend|exporter|common)/test/"]
"^(?:backend|frontend|exporter|common)/test/(?!performance_tests/)"]
:linter-name true}
@ -101,6 +106,4 @@
:unused-binding
{:exclude-destructured-as true
:exclude-destructured-keys-in-fn-args false
}
}}
:exclude-destructured-keys-in-fn-args false}}}

View File

@ -0,0 +1,77 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns hooks.benches
(:require [clj-kondo.hooks-api :as api]))
(defn defscene
"Analyzes scene options and its compiled builder body."
[{:keys [node]}]
(let [[_ _ opts argv & body] (:children node)]
(if (and argv (api/vector-node? argv))
{:node (api/list-node [(api/token-node (quote do)) opts
(api/list-node (into [(api/token-node (quote fn)) argv] body))])}
{:node node})))
(defn defcase
"Analyzes case options and its compiled operation body."
[{:keys [node]}]
(let [[_ _ opts argv & body] (:children node)]
(if (and (seq body) argv (api/vector-node? argv))
{:node (api/list-node [(api/token-node (quote do)) opts
(api/list-node (into [(api/token-node (quote fn)) argv] body))])}
{:node node})))
(defn defcommand
"Analyzes command options and both generated functions."
[{:keys [node]}]
(let [[_ command & forms] (:children node)
[doc forms] (if (string? (api/sexpr (first forms)))
[(first forms) (next forms)]
[nil forms])
[options forms] (if (api/map-node? (first forms))
[(first forms) (next forms)]
[nil forms])
[argv & body] forms
[conditions body] (if (and (api/map-node? (first body)) (next body))
[(first body) (next body)]
[nil body])
handlers (:async (some-> options api/sexpr))
core (api/list-node
(into [(api/token-node 'fn)
(api/vector-node (mapv api/token-node handlers))]
body))
starred (api/token-node (symbol (str (api/sexpr command) "*")))
measured (get (some-> options api/sexpr) :measured-variant true)
definition (api/list-node
(into [(api/token-node 'defn) command]
(concat (when doc [doc]) [argv]
(when conditions [conditions]) [core])))]
{:node (api/list-node
(into [(api/token-node 'do)]
(concat (when options [(api/coerce (dissoc (api/sexpr options) :async))])
[definition]
(when measured
[(api/list-node [(api/token-node 'def)
starred command])]))))}))
(defn resolve-or-reject
"Analyzes the body of a Promise-returning form."
[{:keys [node]}]
(let [[_ & body] (:children node)]
{:node (api/list-node (into [(api/token-node (quote do))] body))}))
(defn resolve-or-reject-with
"Analyzes the cause handler and body. Binds `cause` in the handler only."
[{:keys [node]}]
(let [[_ cause on-error & body] (:children node)]
{:node (api/list-node
(into [(api/token-node (quote do))
(api/list-node
[(api/token-node (quote let))
(api/vector-node [cause (api/token-node nil)])
on-error])]
body))}))

View File

@ -1,3 +1,9 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns hooks.export
(:require [clj-kondo.hooks-api :as api]))

View File

@ -10,5 +10,7 @@
:extra-indents {rumext.v2/fnc [[:inner 0]]
cljs.test/async [[:inner 0]]
promesa.exec/thread [[:inner 0]]
specify! [[:inner 0] [:inner 1]]}
specify! [[:inner 0] [:inner 1]]
benches.render-wasm.runtime.commands/defcommand [[:inner 0]]
benches.render-wasm.runtime.protocol/resolve-or-reject-with [[:inner 0]]}
}

View File

@ -48,10 +48,10 @@ jobs:
# No dependency cache here: a deploy must not restore a cache that
# other workflows can write to.
- name: Setup pnpm + Node.js
uses: pnpm/setup@fa36a5ef7dcdefc3a7cb25655c122e00cf6e636e # v3.1.2
uses: pnpm/setup@731dfb6463bbd11c11fd0ee11fe00ed0e36d48a1 # v3.2.0
with:
working-directory: plugins
runtime: node@26.13.0
runtime: node@26.13.1
install: false
- name: Install deps

View File

@ -46,10 +46,10 @@ jobs:
# No dependency cache here: a deploy must not restore a cache that
# other workflows can write to.
- name: Setup pnpm + Node.js
uses: pnpm/setup@fa36a5ef7dcdefc3a7cb25655c122e00cf6e636e # v3.1.2
uses: pnpm/setup@731dfb6463bbd11c11fd0ee11fe00ed0e36d48a1 # v3.2.0
with:
working-directory: plugins
runtime: node@26.13.0
runtime: node@26.13.1
install: false
- name: Install deps

2
.gitignore vendored
View File

@ -121,7 +121,7 @@ opencode.json
/.opencode/reports
/.opencode/prompts
/.ci-logs
/tmp/
/tmp
/.codex/
/scripts/__pycache__/
/tools/__pycache__

2
.nvmrc
View File

@ -1 +1 @@
v24.22.2
v24.22.3

View File

@ -11,8 +11,10 @@
- Account lockout (flag `:account-lockout`, `app.auth.login-lockout`) is Redis-backed and keyed per profile id. Password and LDAP flows check/increment on the profile derived from the typed email and clear on the profile that actually logs in.
- Lockout and RPC rate limits share the `app.http.errors/handle-error :rate-limit` HTTP path: status 429, body `{:type :rate-limit :code ... :hint ... :ttl ...}`, and any supplied `::http/headers` preserved. Account lockout raises `:code :account-locked` and, when it carries a non-nil `:ttl` (seconds), the handler adds `retry-after`. The RPC limiter (`app.rpc.rlimit`) raises `:code :request-blocked` and sets `retry-after` itself in `::http/headers` (seconds until the longest rejecting limit resets), alongside `x-rate-limit-remaining`/`x-rate-limit-reset`. CORS (`app.http.middleware/with-cors-headers`) exposes `content-type`, `retry-after`, and both `x-rate-limit-*` headers. External flags: `enable-account-lockout`, `enable-rpc-rlimit`.
- Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI.
- Organization SSO (Nitrate): the RPC `check-nitrate-sso` takes the route's **query string proper** (`:query-string`, e.g. `screen=workspace&team-id=…`), never a URL and never with the `?` separator: the client strips it (`rt/token->query-string`) and the server puts it back. The server builds the destination with `app.auth.oidc/build-dest-url` (`public-uri` + `?` + the params, parsed and rendered back), so the redirect can never leave the public URI; a value whose parsed keys are not param names is not a route: a URL parses into a single key holding the whole URL, and a key carrying `:`, `/`, `?` or `#` is rejected with a `:warn` and the user lands on the public URI (the RPC never fails because of it). Do not reason from `u/map->query-string`: it keeps a valueless key as `key=`, so rendering is not what filters a URL out — the key shape is. A leading `?` is still trimmed server-side, so a client bundled before the trim keeps its destination. Never require `screen`: the root of the app carries its params instead of a screen (`?template=…`), and `check-nitrate-sso` is called from that branch too (`dtm/resolve-login-team-id`, which today reads only `:authorized` and never follows the redirect, so a wrong destination there is a silent `:warn` and not a visible break); the callback re-checks the destination before redirecting, which covers state tokens minted before that. The state token also carries `:profile-id`: only that session is granted the `props :sso` entry (`session-mismatch` otherwise, and the redirect still happens).
- Organization SSO needs `app.nitrate/organization-access?` whenever the RPC receives an `:organization-id` (that id always comes from the client): the profile must be a member of the organization or its owner; otherwise the RPC answers `{:authorized true :reason :no-organization-access}`, the same short-circuit shape as `:no-team-access`.
- Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation.
- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile.
- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. SSO registration (OIDC and compatible providers) follows the OIDC callback gate: it requires `:registration` or `:oidc-registration`, or a team invitation token issued for the registering email (an invitation substitutes the flags only when `member-email` matches the OIDC/`prepared-register` email via `clean-email`, as in the password flow, which raises `:email-does-not-match-invitation` otherwise). The OIDC callback verifies the invitation JWT and the email before redirecting to register, and `register-profile` revalidates the invitation under a row lock when no SSO flag is set; it never requires password login. Only OIDC-typed backends (`oidc-provider-types` in `app.auth.oidc`) take the SSO registration gate; any other `:backend` in a prepared-register token is rejected as `:invalid-token`.
- Access tokens (`app.http.access-token`): personal tokens have JWT issuer `access-token` and DB type other than `mcp`; MCP tokens have issuer `urn:penpot:mcp-token` (`actoken/mcp-token-issuer`) and DB type `mcp`. Authentication requires issuer and type to agree, and the claim `uid` to match the row's profile; other issuers and undecryptable tokens are treated as anonymous.
- Management RPC (`/api/management/methods/:name`) is shared-key only. The middleware authenticates `<id> <secret>`; the RPC wrapper `wrap-shared-key-authorization` authorizes the key id against the method's `::rpc/allowed-shared-keys` set. Absent metadata accepts any configured key; all current methods declare one (`exporter`/`media-processor` for `upload-tempfile`, `nexus` for file analysis, `exporter` for jobs, `admin-console` for Nitrate). A valid key outside the set answers 403 `:shared-key-not-allowed`; denied calls do not reach the audit wrapper.
- Personal tokens need flag `:access-tokens`. MCP tokens need `:mcp` or `:access-tokens`, and only on routes whose `actoken/authz` gets `::actoken/allow-mcp true` (the RPC routes); elsewhere (e.g. assets) they are anonymous.
@ -24,6 +26,7 @@
- `app.rpc.permissions` provides predicate/check factories. Failed permission checks intentionally raise `:not-found` / `:object-not-found`, not an authorization-specific error, to avoid leaking object existence.
- Team role flags are normalized as owner > admin > editor > viewer. Owner/admin imply edit; any membership row implies read.
- File/project role rows only count with a live team membership: the file/project permission queries require a matching `team_profile_rel` row, and leaving or removing a member deletes their team-scoped `file`/`project`/`team-project-profile-rel` rows (GHSA-v9r9-h77c-55m2).
- The organization-owner fallback (`get-organization-owner-permissions` via Nitrate over HTTP) grants read-only access for a non-member organization owner. It resolves the owning team through `resolve-team-id`, which refuses soft-deleted file/project/team: the file and project lookups filter `deleted_at`, and a final `db/get*` check covers the team row itself — `nil` means no access. `db/get*` only filters soft-deleted rows when `deleted_at` is in the projection. The re-checks of open websocket subscriptions rely on this (`mem:backend/websocket-subscriptions`).
- File/project/comment checks are implemented in the owning command namespaces, often via helpers imported from `files`, `teams`, or `projects`; do not bypass those helpers with direct DB lookups unless preserving their not-found semantics.
- Comment permission includes both logged-in state and the file/team comment policy. Shared viewer paths may pass `share-id`; preserve that path when changing comment queries. RPCs that accept `share-id` enforce the link's page scope before mutating (`share-link-pages` + `check-page-scope`): a thread or page outside the link raises `:not-found`, and that check runs before the ownership checks. `get-profiles-for-file-comments` filters its query by the same pages. `mark-all-threads-as-read` does not take `share-id`.
@ -32,6 +35,7 @@
- Team/project commands mix DB changes, email, message bus notifications, media/storage cleanup, feature flags, quotas, and audit metadata. Keep mutations transactional when the existing command does so.
- Invitation flows validate muted/bounced emails before sending and use tokenized invitation state. Accepting an invitation is tied to the invited member identity, not just possession of a token.
- The whole invitation family is team-admin only: creating, updating and deleting invitations, minting the invitation token (`get-team-invitation-token`) and listing the pending ones (`get-team-invitations`, which carries the invited email and the offered role). The token of an invitation that grants the `owner` role is restricted to the team owner, because an admin cannot create nor promote an owner invitation (`cant-promote-to-owner`). This family is the exception to the rule above: it answers `:validation` / `:insufficient-permissions` instead of `:not-found`, so the five commands speak one language.
- With the `:admin-console` flag, the organization `send-invitations` policy (role value plus Nitrate `restrictions-met`, through `cto/allowed?`) gates every invitation command that hands out or manages access: create (and resend, which reuses it), `update-team-invitation-role`, `delete-team-invitation` and `get-team-invitation-token`. Only `get-team-invitations` stays on team admin alone, so a blocked admin still sees the list. The shared helper `check-send-invitations-policy` in `teams_invitations.clj` does a strict Nitrate lookup (fails closed; 404 means no organization) and runs outside a DB transaction, which is why the role and delete commands carry no `::db/transaction`.
- Logical deletion is used for many product objects; prefer existing logical-deletion helpers over hard deletes unless the command already performs permanent cleanup.
- Bounced/spam-complaint emails can mute/block a profile for login/registration and email sending. Devenv Mailpit is the normal local path for registration/email-flow testing.

View File

@ -9,6 +9,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m
- Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment`
- Session lifetime config, token `:exp`, and idle/absolute GC: `mem:backend/session-expiration`.
- Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains`
- WebSocket subscriptions, the connection registry (local per backend, unlike the bus), and how a lost permission gets revoked: `mem:backend/websocket-subscriptions`.
- Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:audit-log` (guía completa, también para perfiles junior; el stub `mem:backend/audit-log` solo redirige aquí)
- Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`.
@ -67,7 +68,8 @@ Every background job is a job-def: a plain `(defn execute-X [cfg params] ...)` i
- Job resources (`app.jobs.storage`) are the objects a job owns through `job.resource_id`: they live in the `job-resource` bucket, are written with the profile of the job as owner (only that profile can read them back from `/assets/by-id`) and are never deduplicated. They carry no expiry of their own, because `expired-at` is stored as `deleted_at` and an object with it is unreadable from birth: the retention of the artifact is the retention of its job row. The writer marks the object as touched right away, so an artifact that no row references (a crash between the upload and the completion) is reclaimed by `storage-gc-touched`, and a row that does reference it freezes it until the jobs GC deletes the row and touches the object.
- The binfile jobs (`:export-binfile` and `:import-binfile`) run the core of `bf.v1`/`bf.v3` through `app.binfile.jobs`, which turns the `:progress` taps of the core into `progress` events of the job with `jobs/heartbeat` (the `modified_at` touch at most once per second, progress events at up to ~200ms). Each event is a self-contained milestone: a `:stage` keyword of one vocabulary shared by the two formats (the same work has a different name in v1 and v3) and an optional `:counters` map, with the counter of the activity named after its own stage and the outer scopes in course as context, which the tap carries itself (the reader puts `::outer-counters` in every tap inside them — today only `:files`, extendable to `:teams`/`:projects` for multi-scope exports — so the adapter remembers nothing between taps and a tap that happens outside any scope has no context); the ids and names the taps carry never reach the event. Every tap is reported, so a tap with no units to count still keeps the job alive, and a milestone the throttle drops loses nothing because the next one carries the same state. The adapter checks that the job is still active before the run, and the beat raises an `:interrupt` right there on the run thread when the job is no longer active, so the work stops at its next event point inside the transaction of its handler, which rolls back. A run whose last tap predates the interruption is discarded by the check once it returns. The adapter passes the job id from the handler context. Their job-def components carry pool, storage, metrics and msgbus, because a job with a `profile_id` cannot store an event without a msgbus on its cfg. Both readers answer with the same shape (`:file-ids` and, in version 3, the resolution of the libraries), because the version 1 reader returns a bare set of file ids.
- Both binfile handlers revalidate their permission at run time, not only when the job is created, because it can be revoked while the job waits: the export checks the read permission of every file before reading anything and stores the package in `job-resource` as the resource of the job (answering with the completion envelope), and the import checks the edition permission of the destination and runs the core inside a transaction, which is what makes a cancelled import roll back instead of leaving half of a file. The import releases its package (marks it as deleted and drops the reference from the job row in the same transaction) and deletes its temporary copy once the job is terminal, whatever the outcome. Both handlers check their own result against a schema before returning it (the import job-def declares that same schema as its result schema, which is what types the resolution when the row is read).
- The v3 import only materializes a library relation when both ids belong to the package and differ; a relation that points outside the package or a self link is ignored with a warning. Migration `0158-del-forbidden-file-library-rels` deletes the legacy cross-team and self rows from `file_library_rel` and their `file_library_sync` rows, sync rows first.
- The v3 and v1 imports only materialize a library relation when both ids belong to the package and the link would neither be a self link nor close a dependency cycle (the same `bfc/library-link-status` check that `link-file-to-library` uses to raise); relations are processed in sorted `(file_id, library_file_id)` order, which `get-files-rels` also returns, so the first edge wins deterministically, and every edge it ignores is logged with a warning. The v2 import stays out of that guard on purpose: it is only reachable from the developer REPL (`app.srepl.binfile`) and keeps inserting the relations as they come. Migration `0158-del-forbidden-file-library-rels` deletes the legacy cross-team and self rows from `file_library_rel` and their `file_library_sync` rows, sync rows first.
- A version 1 package lists its files in the metadata and the importer creates one file per entry. The v1 export (gone from the product; only tests and the developer REPL use it) unioned the requested files with their libraries without deduplicating, so a package exported from a file inside a library cycle lists that file twice. The importer reads the pair of objects of every entry to keep the stream in sync, imports the first occurrence only, warns about the repeats, and answers with one id per imported file.
- The domain cfg of a binfile request lives in `app.binfile.common` (`export-cfg`, `import-cfg`), so the legacy RPC and the jobs freeze the same ids, resolve the same team features and apply the same anti-zip-bomb limits. `resolve-export-type` is the translation the legacy RPC needs for its deprecated boolean flags; the creation command asks the caller for the type instead of guessing a default.
- Two retention knobs, easy to confuse: `:jobs-retention` sweeps terminal internal rows (the ones without a profile), while `:jobs-user-ttl` is the default expiry a user-facing job gets when it is created; the jobs GC deletes the row once `expires_at` passes and touches its resource. The heavy work of a user job runs on its own `:binfile` queue (`:worker-binfile-parallelism`, default 1) so it does not block the default runner.
- The quote of a family of jobs (`::export-jobs-per-profile`, `::import-jobs-per-profile`) counts the jobs of the profile that are still on their way (`new`/`scheduled`/`running`/`retry`) whose name the registry gives for that family, so a new job of the family is covered without touching the quote.

View File

@ -19,6 +19,12 @@
- Renewal fires when `modified-at` is older than 6h (`default-renewal-max-age`, not configurable). It `UPDATE`s the same row (`modified-at` only) and issues a new token that keeps the original `:exp`.
- `read-session` does not check age. Idle expiration is enforced on every request by `wrap-authz` (sessions past the idle window authenticate like missing ones and are never renewed); the GC only deletes the rows afterwards.
## Revocation paths
- Profile deletion → `session/invalidate-all` (all sessions of the profile die).
- Authenticated password change → `session/invalidate-others` (every session except the caller's dies).
- Password recovery (`recover-profile`, GHSA-3j2p-qr45-q365) → `session/invalidate-all` inside the same transaction as the password `UPDATE`; there is no current session to preserve, and a captured session must not survive recovery. Sibling of GHSA-mj9f-5cwq-7p3q (logout deletes the current session row).
## GC (`::tasks/gc`, cron `session-gc`, daily)
- Deletes from `http_session_v2` where `modified_at < now - :auth-token-cookie-max-age` **or** `created_at < now - :auth-token-cookie-max-age-absolute`.

View File

@ -93,7 +93,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `
| `organization` | Organization logos uploaded by the Nitrate management API. | Yes | Public | No reference scan. A touched object is deleted. |
| `tempfile` | Export files and temporary font downloads. | No | Authentication required | No reference scan. A touched object uses a two-hour deletion delay. |
| `upload-session` | Chunked-upload chunks. References: `upload_session_chunk.object_id` and `upload_session_chunk.session_id` (both NO ACTION DEFERRABLE: restrict semantics, procedural deletion). | No | Authentication required | Reference scan on `upload_session_chunk.object_id`; a referenced chunk freezes, an orphan is deleted after the delay. Deferred rows keep their mappings until give-up. |
| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference metadata has `storage-ref-id`, `file-id`, and the `file_data` row ID. | Yes | Authentication required | Reference scan. |
| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference: the `file_data.storage_object_id` column (FK `NO ACTION DEFERRABLE INITIALLY IMMEDIATE`); the object metadata keeps `file-id` and the `file_data` row ID. | Yes | Authentication required | Reference scan. |
| `file-data-fragment` | Compatibility value for file-data fragments. The current backend has no dedicated producer for this bucket. | No current write semantics | Public | No touched-object collector case. |
| `file-change` | Compatibility value for file changes. Current snapshots store data in `file_data`, not this bucket. | No current write semantics | Authentication required | No touched-object collector case. |
| `job-resource` | Storage objects owned by job rows (`job.resource_id`), written by `app.jobs.storage`. | No | Authentication required, owner-scoped | jobs-GC touch → reference scan. |
@ -118,18 +118,40 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `
## File Data
- `file-data-backend` accepts `legacy-db`, `db`, or `storage`.
- `legacy-db` stores main data in `file.data` and snapshots in `file_change.data`.
- `file-data-backend` accepts `legacy-db`, `db`, or `storage`; `legacy-db` is read-only and the default is `db`. The configuration reads `legacy-db` as `db` at startup, with a warning (`normalize-file-data-backend`), so a deployment that kept the variable saves instead of failing on every write.
- `legacy-db` reads main data from `file.data` and snapshots from `file_change.data`; it no longer writes them (`fdata/upsert!` rejects it).
- `db` stores encoded data in `file_data.data`.
- `storage` stores encoded data in storage subsystem with `file-data` bucket and keeps `data` nil in `file_data` table.
- The `file_data.metadata.storage-ref-id` value points to the storage object.
- `fdata/upsert!` touches a storage object from incoming metadata before it stores the new row.
- `storage` stores encoded data in storage subsystem with `file-data` bucket, keeps `data` nil in `file_data` table, and points at the object with `file_data.storage_object_id`.
- `file_data.storage_object_id` is a `NO ACTION DEFERRABLE INITIALLY IMMEDIATE` foreign key to `storage_object(id)`: the database refuses to physically delete a referenced object. The `IMMEDIATE` is load-bearing: the RI violation fires inside the `storage-gc-deleted` chunk transaction, so the chunk rolls back, is logged at error level with a generic hint plus the `constraint` and `table` the database reports (any `NO ACTION` foreign key on `storage_object` can block the delete, not only `file_data`'s) and is retried on the next run; with `INITIALLY DEFERRED` the check would fire at commit, outside the catch, and the sweep would never move past the blocked chunk.
- The column is the only source of truth. The release is a coordinated restart (no rolling deploy) when the instance uses the storage backend (`PENPOT_FILE_DATA_BACKEND=storage`) or still has rows carrying `metadata.storage-ref-id`; with the default `db` backend and no such rows the previous and the new code read the same `file_data.data` column, so a rolling deploy is safe. `metadata.storage-ref-id` is no longer written or read, and the `file-data` metadata schema does not declare it. The migration backfills the rows it can resolve (valid uuid with an existing `storage_object`) and removes the key from those rows; a corrupt or orphan pointer keeps the key, which nothing reads. A large instance runs `operations/2.20.0-file-data-storage-object-id.md`: column, index and FK online, then the backfill (only if a row still carries `metadata.storage-ref-id`) with every instance stopped, verifies, and marks the migration as applied (`INSERT INTO migrations`). The backfill must never run while the previous code serves when the storage backend is in use (or rows with the pointer remain), because that code writes and scans the metadata pointer. A rollback to the previous code cannot read the rows written by this release when the storage backend is in use.
- `fdata/upsert!` touches the `storage_object_id` it receives in the params (the pointer the persisted row had) before replacing it, so the replaced blob is reclaimed. Callers pass the column from the file map (main/restore), `nil` for a new row (fragments, snapshots, new files) or the row itself (offload); there is no query in the upsert path, which is why every caller reads the file under the row lock first (the migration-on-read path included, see `mem:backend/subtleties`). It returns `{:written <bool> :storage-object-id <uuid or nil>}`, the pointer of the object the write created (nil for `db`/`legacy-db`), and `binfile.common/update-file!` propagates it so a migrating read returns the map with the pointer of the row it persisted instead of the pre-write one. The write also compares the expected pointer with `IS NOT DISTINCT FROM` (the caller's value, not the new one) and raises `:stale-file-data-pointer` when the row changed under it, so a stale map fails instead of leaking the replaced blob; the check lives in `upsert-in-database` because the `legacy-db` branches do not use the upsert SQL.
- `binfile.common/update-file!` takes the file map read from the database plus the fields to change (`[cfg file & params]`), merges them and builds the `file_data` params from the merged map, so the pointer travels in the read map and a caller cannot lose it by building a partial map; the params cannot override `:storage-object-id`. `insert-file!` always drops it (an INSERT never replaces a persisted pointer; `duplicate-file` hands the source map there). `bfc/get-file` keeps the pointer for the write paths, and every response goes through `bfc/file->response`, which strips it and any residual `storage-ref-id` left in the metadata of the rows the migration does not backfill: each `file-data` object is authentication-only and any logged-in profile could otherwise fetch it from `/assets/by-id`.
- The in-place binfile import was removed: `import-files!` raises `:not-supported` when the cfg carries `::bfc/file-id`, so no caller can overwrite a storage-backed row without carrying its previous pointer.
- File snapshots use `file_data` for snapshot data and `file_change` for snapshot metadata.
- Tiered offload (`:tiered-file-data-storage` flag + `db` backend): `file-gc` submits an `:offload-file-data` job (priority 10, delay 1000, same params) only when it actually processed the file; the handler takes the `file` row with `FOR UPDATE SKIP LOCKED` and skips (`{:skipped 1}`, no error) when the row is locked or gone, then moves every live `file_data` row of that file with `backend = 'db'` to `backend = 'storage'` via `fdata/upsert!` (one `put-object!` in bucket `file-data` per row, `data` becomes nil) inside a single all-or-nothing transaction per file. The skip only serializes because every `file_data` writer holds at least `FOR SHARE` on the `file` row. A storage failure aborts only that file and bubbles for a runner retry; orphans go to `storage-gc-touched`. A second run is a no-op because the query only matches `backend = 'db'`.
- Tiered offload (`:tiered-file-data-storage` flag + `db` backend): `file-gc` submits an `:offload-file-data` job (priority 10, delay 1000, same params) only when it actually processed the file; the handler takes the `file` row with `FOR UPDATE SKIP LOCKED` and skips (`{:skipped 1}`, no error) when the row is locked or gone, then moves every live `file_data` row of that file with `backend = 'db'` to `backend = 'storage'` via `fdata/upsert!` (one `put-object!` in bucket `file-data` per row, `data` becomes nil and `storage_object_id` points at the new object) inside a single all-or-nothing transaction per file. The skip only serializes because every `file_data` writer locks the `file` row first (the update RPC and the snapshot paths with `FOR UPDATE` through `lock-for-update?`, the migration-on-read path with `FOR UPDATE`). A storage failure aborts only that file and bubbles for a runner retry; orphans go to `storage-gc-touched`. A second run is a no-op because the query only matches `backend = 'db'`.
- The offload normalizes each `SELECT fd.*` row before `upsert!`: the row builder omits NULL columns, jsonb metadata arrives as a `PGobject`, and a JSON `null` decodes to nil and must disappear instead of travelling as `:metadata nil`, which the update schema rejects.
- Offload pacing: none by deliberate decision. Sleeps only make sense when batching multiple transactions; the per-file transaction is short by construction and the scheduler run takes minutes per production metrics, so neither loop sleeps. `jobs/heartbeat` (pool-based liveness, ~1 write/s) stays on both paths.
- Offload metrics (bounded labels only, never ids or error text): `penpot_file_data_offload_rows_total{type}` and `penpot_file_data_offload_bytes_total{type}` (`type`: `main`/`snapshot`/`fragment`, successful runs only) plus `penpot_file_data_offload_timing_ms{result}` (`result`: `ok`/`skip`/`error`). Helpers live in `app.jobs.metrics`; the job-def requires `::mtx/metrics`.
- Writes always resolve the backend through `fdata/default-backend` (`cf/get :file-data-backend`), so post-offload edits land back in hot (`db`) storage without extra code; reads stay transparent through `fdata/resolve-file-data` + `decode-file-data` + `realize`.
- Deleting an offloaded row touches its storage object first (`objects-gc/delete-file-data-chunk` via `fdata/process-metadata`), so `storage-gc-touched` reclaims the blob with the usual delay; restoring a file unmarks the referenced objects.
- Deleting an offloaded row touches its storage object first (`objects-gc/delete-file-data-chunk` via `fdata/touch-storage-object` with the row's `storage_object_id`), so `storage-gc-touched` reclaims the blob with the usual delay; restoring a file unmarks the referenced objects.
- `storage-gc-touched` resolves the `file-data` reference with `SELECT EXISTS (SELECT 1 FROM file_data WHERE storage_object_id = ?)`, an indexed lookup that no longer parses the object metadata.
- A row with `backend='storage'` and no readable object makes `resolve-file-data` raise `:missing-storage-object` with the file and row ids instead of failing later in the storage backend.
- The `file-id` and row id kept in the storage object metadata are informational for diagnostics; nothing parses them to resolve the reference (the reference is the column).
## File Change
- `file_change` is partitioned by `HASH (file_id)` in 16 partitions, with the same bounds as `file_data`, so a join between them is resolved partition by partition (`enable_partitionwise_join` is set on every pooled connection).
- The automatic migration (`0158-partition-file-change`) refuses to run when `file_change` is above 10 GiB total size or 2M rows (`pg_total_relation_size` + `pg_class.reltuples`, catalog reads). It analyzes the table first, because `reltuples` is -1 without statistics. It raises with the path of `operations/2.20.0-file-change-partition.md`, because the copy happens inside the migration transaction and a big instance would restart in a loop. Below the guard it raises `statement_timeout` to 30 minutes for the copy and the index builds, and analyzes the new partitions at the end, after the copy and the indexes, so the planner does not start with empty statistics.
- Its primary key is `(file_id, id)`: a partitioned table needs the partition key in the primary key, so every access filters by `file_id`. From 2.20 the four snapshot commands (`update-file-snapshot`, `delete-file-snapshot`, `lock-file-snapshot`, `unlock-file-snapshot`) require the `file-id` param, because they use it to prune the partition; without it they answer `:params-validation`.
- It has a `kind` column: `1` is a snapshot (it has a label and its data lives in `file_data`; the read-only `legacy-db` backend keeps old payloads in `data`) and `2` is a revision (it has no label and it holds the changes). The values are in `app.features.file-changes`.
- `kind` is a deliberate reserve, not a leftover: today no query filters by it, the queries still tell a snapshot from a revision by its label (`label IS NOT NULL`), and `kind` takes over when they are rewritten. No constraint ties `kind` and `label` together on purpose: the day `kind` is the source of truth, that constraint would have to go. It stays nullable on purpose: the previous version inserts without it, and a rollback to it has to keep writing; the `NOT NULL` comes in a later migration.
- Every write of the application to a row of `file_change` moves `updated_at` to the moment of the write (`lock-by!`, `unlock!`, the snapshot rename and delete, `delete-object`, the restore of a file). The actions of the foreign keys (`ON DELETE SET NULL` on `locked_by` and `profile_id`) change the row without touching it; during the partition both tables carry the same keys, so both receive them. The runbook of the partition uses it as the watermark of its delta passes: a write of the application that leaves it behind is lost at the cutover.
- `file_change_legacy` is a frozen copy of the previous layout of the partition: it keeps the rows and the indexes of the old table. A new instance (empty table) does not keep it. On an upgraded instance the table stays until the operator drops it by hand days later (`DROP TABLE file_change_legacy;`, see `operations/2.20.0-file-change-partition.md`); nothing drops it automatically.
- A revision carries only the delta of a save (the changes). The features of the file are not repeated on it: nobody read them from there.
- A snapshot carries the features and the migrations of the file.
- It keeps the `data` column for reading only: migration `0157-migrate-legacy-file-data` moved the existing blobs to `file_data`, and `legacy-db` remains as a read-only backend for the payloads still in `data`. `fdata/upsert!` rejects `legacy-db` and the `file-data-backend` default is `db`. It has no `data_ref_id` or `data_backend` columns: nothing read them since the file storage refactor.
- It has no index for the system snapshots. The old one filtered them by `data IS NOT NULL`, and with the payload living in `file_data` the column no longer tells snapshots apart. The new one could only filter them by `deleted_at IS NULL`, but the system snapshots always carry a deletion date in the future, so it would never match a row. The query that would use it filters that date with an "or", which no partial index can serve.
- The `objects-gc` purge groups each chunk by `file_id` and locks it with `file_id = ?` and `id = ANY(?::uuid[])`: a tuple `IN ((file_id, id), ...)` gives the planner no constant partition key, so it scans all 16 partitions in every chunk. Each group is one `FOR UPDATE SKIP LOCKED` query inside the chunk transaction.
## Metrics

View File

@ -45,7 +45,9 @@
- `job` is deletion-protected at the SQL level (`deletion_protection__tgr`): a stray `DELETE FROM job` raises instead of dropping the row and its `job_event` cascade without touching the referenced `storage_object`. The only intended deleters, the jobs GC and the submit dedupe, disable the guard with `SET LOCAL rules.deletion_protection TO off` inside their transaction; tests use `th/db-force-delete`.
- A missing job-def raises (`:no-job-definition`) instead of completing. Throwing with `ex-data :type ::retry` still controls retry behavior; `:strategy ::noop` retries without incrementing retry count.
- Cron entries claim their `scheduled_task` row with `FOR UPDATE SKIP LOCKED`, disable statement/idle-in-transaction timeouts locally, submit one `job` row per entry when no active instance exists (no-overlap), and reschedule themselves in `finally` unless interrupted. Worker, dispatcher, and cron components do not start when the DB pool is read-only.
- Integrant params schemas name injected components as bare `::ns/key` entries (pool, metrics, msgbus, job defs), and a bare entry is a malli registry lookup, not `any?`: the owning namespace must `sm/register!` it at load, or the backend crashes at boot with `:malli.core/invalid-schema` instead of asserting. Unit tests never build those keys, so `backend-tests.worker-test` compiles the three worker schemas to catch it.
- Integrant params schemas name injected components as bare `::ns/key` entries (pool, metrics, msgbus, job defs), and a bare entry is a malli registry lookup, not `any?`: the owning namespace must `sm/register!` it at load, or the backend crashes at boot with `:malli.core/invalid-schema` instead of asserting. Unit tests never build those keys, so `backend-tests.worker-test` compiles the three worker schemas and `backend-tests.http-websocket-test` compiles the revocation-watcher one to catch it. The assertion only runs with `PENPOT_FLAGS=enable-backend-asserts`, so without it a broken schema reaches production and only crashes at boot.
- Whether a bare entry works has nothing to do with its position or with how many of them there are: it is a registry lookup, so it works only if that exact keyword was `sm/register!`'d. `[:map ::db/pool ::metrics/metrics]` is fine, `[:map :app.aaa/unregistered :app.ccc/unregistered]` throws. When a component is **not** registered as a malli type, spell it as an explicit pair instead of registering a type just to satisfy the lookup — `[:app.nitrate/client {:optional true} [:maybe :map]]` is how `auth.oidc/schema:routes-params` and `loggers.audit.archive-task/schema:audit-log-archive-params` declare it. `{:optional true}` matters because `init-key` yields `nil` when the owning feature flag (`:admin-console`) is off, and a required key would then fail the assert on a perfectly normal boot.
- `sm/valid?` catches every `Throwable` and returns `false`, so an uncompilable schema and a genuinely invalid value both surface as a bare `Assert failed:` with no cause. When an `assert-key` fires with no message, compile the schema by hand to see the real exception.
## Config and HTTP/session middleware
@ -92,21 +94,21 @@
## File data persistence
- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data.
- `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob.
- File data backends for writing are `db` and `storage`; `legacy-db` is a read-only alias for old payloads. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores nil data and points at the object with the `storage_object_id` column (FK `NO ACTION DEFERRABLE`).
- `fdata/upsert!` touches the storage object the persisted row pointed at before replacing it; the pointer travels in the params (`file->file-data-params` from the read file map, `nil` for a new row, the row for the offload) and the params of `update-file!` cannot override it. `binfile.common/update-file!` merges the read file map with the fields to change, so the pointer survives; `insert-file!` drops it because an INSERT never replaces a persisted object. The column is the only source of truth and the metadata no longer carries `storage-ref-id` (the migration removes it from the backfilled rows and the write path never stores it); the release is a coordinated restart when the storage backend is in use (the previous code cannot read the new rows), while the default `db` backend reads the same data column and tolerates a rolling deploy. Responses built from a file map go through `bfc/file->response`, which strips the pointer and a residual `storage-ref-id` from the metadata.
- Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. A pointer keeps its id while nothing changes; content (`assoc`/`without`) and metadata (`with-meta`) changes produce a fresh id, so a fragment row is never rewritten after it is written. Re-assoc with an identical value returns the same map in array and hash maps alike, so a save keeps the fragment id of every page it does not change.
- `enable-objects-map` skips unloaded pointer pages: their fragment was already wrapped when it was written, and loading it only to wrap it again costs one page read per untouched page on every save. A page with a plain fragment is wrapped when it is first loaded or edited; a data migration that enables `fdata/objects-map` must rewrite the data itself.
- Snapshots share fragment rows with the live file (they are stored with pointers, not materialized). `file-gc` marks every fragment referenced neither by the current main file nor by an unpurged snapshot; the file is read unrealized and unchanged pages keep their ids via reattach, so a pass over an unchanged file writes no new fragments.
- A save attaches its automatic snapshot before applying pending file migrations: the snapshot keeps the pre-migration pointers, data and migration set, and restoring it leaves the next read to migrate the file. This also holds for files without the feature. The preview (`get-file-snapshot`) applies the pending migrations in memory with the live file's libraries and never persists them: the stored version keeps its data, the payload carries the snapshot's `:migrations` (an ordered set) and `fdata/objects-map` is realized for clients that do not declare it. The historical feature set is deliberately not checked against the enabled one: a version may predate a feature change. The window is narrow: the `get-file` RPC reads with `migrate?` on and its ETag hashes `available-migrations`, so a post-deploy read cannot be a 304 from the pre-deploy state and migrates before any save in the normal flow; the pre-migration snapshot only appears when the save is the first request to touch the file after the deploy (an open tab that saves without re-reading, or a non-UI writer). `create-file-snapshot` and the restore backup read with `migrate?` on, so those versions are stored already migrated.
- `:migrations` is always a `d/ordered-set`: `file-migrations/resolve-applied-migrations` builds one, `fmg/migrate` re-wraps the union after applying, and `file-snapshots/decode-snapshot` converts the `file_change` pgarray (as a plain vector, `set/difference` and `contains?` would compare indexes instead of names). A snapshot with NULL `:migrations` and an old `:version` predates the `file_change.migrations` column (SQL migration 0140): the preview regenerates the set from the stored version and applies that full chain, while `restore!` uses a fixed 67 baseline, so preview and restore can disagree for very old snapshots.
- The migration read path (`binfile.common/migrate-file`) captures `fdata/original-pointers` before realizing, reattaches unchanged positions after applying the migrations, and persists the reattached file; it returns the realized file because reads expose plain data. A migration that adds `fdata/objects-map` skips the reattach so the pages get wrapped.
- The migration read path (`binfile.common/get-file*`) reads the `file` row once (`sql:get-file`, with its data columns; with the storage backend the blob lives in the object, not in the column) plus an explicit `resolve-applied-migrations`, because a row read does not resolve them. A read with no pending migration is completed by `attach-file-data` on that map: one row read and one migrations resolution. Only when a migration is pending is the row re-read under `FOR UPDATE` on the `file` row, where `need-migration?` is re-checked on the fresh map and the migrated file persisted (`binfile.common/migrate-file`); that re-read resolves the migrations again, and `migrate-file` once more after persisting, to return the applied set. The migration path holds two raw copies of the blob (the first read and the locked re-read) but a single decoded one: the alternative, a row-only probe before the read, avoided the first raw copy at the cost of an extra row read on every open, and was removed. `get-file` runs the read inside `db/tx-run!` (joining the caller's transaction when the cfg carries a connection) so the persist commits with the read; a read-only read (`:read-only?`) takes no lock and writes nothing, so it runs on a plain connection (`db/run!`) instead of opening a transaction of its own, and with a cfg that already carries a connection it still joins the caller's. It accepts a cfg map only and throws an `IllegalArgumentException` otherwise (`assert` is elided without `:backend-asserts`); a data read rejects `:lock-for-share?` (only row-only reads accept it), because the migration branch re-reads with `FOR UPDATE`, and a lock option needs a read-write transaction from the caller because PostgreSQL rejects row locks in a read-only transaction. A row-only read (`load-data?` false) resolves the applied migrations by default too (`:load-migrations?`), because the row can be completed later and the caller may need them; a caller that does not want them passes `:load-migrations? false`. Persisting is an optimization of the read (the read already returns migrated data): a row that changed between the first read and the locked read is not overwritten, and a pending migration discovered on the unlocked read is applied in memory only (the next read persists it under the lock). This was the only writer of the file row outside the lock protocol (save, library absorption, restore, `file-gc` and offload already take it); without it, a save that committed in between was overwritten by the stale map (lost content, `revn` moving back so the next save fails with `:revn-conflict`, and the object the save left as a frozen orphan). `migrate-file` captures `fdata/original-pointers` before realizing, reattaches unchanged positions after applying the migrations, and persists the reattached file; it returns that reattached map when it persists (the pages the migration does not change keep their pointers and fragment ids) and the realized map when it does not, so a read is plain only when the caller asks with `:realize?`, and a migration that adds `fdata/objects-map` skips the reattach so the pages get wrapped.
- `fdata/load-pointer` reads rows that a previous `file-gc` pass marked on purpose (explicit `::db/remove-deleted false`): the fragment is only gone once `objects-gc` purges it. The `get-file-fragment` RPC keeps rejecting marked rows.
- Bind the fragment loader through `fdata/with-pointers` (reads), `fdata/with-tracked-pointers` (writes with a loader) or `fdata/with-tracked` (writes with nothing to load yet, e.g. file creation), and `fdata/with-untracked` (validation passes that must track nothing) instead of hand-rolled `binding` forms. Two direct bindings remain on purpose: the fine-grained tracked toggles of `app.rpc.commands.files-update` (validation/repair turn tracking off, restore the existing atom and read it before repairing), and `binfile.common/encode-file` (a fresh tracked set inheriting the current one). Converge them only if atom-scoped macros (`with-tracking`, `with-inherited-tracking`) are added; today the extra indirection buys nothing.
- Snapshot creation takes `FOR SHARE` on the file row: both `create-file-snapshot` and the backup read of `restore-file-snapshot` hold it from the read to the insert, so they serialize with a `file-gc` pass (`FOR UPDATE ... SKIP LOCKED`) instead of pinning fragments the pass is about to mark. `get-file-snapshot` realizes the snapshot pointers with the live file id (fragment rows are keyed by file, not by snapshot).
- Snapshot creation takes `FOR UPDATE` on the file row: both `create-file-snapshot` and the backup read of `restore-file-snapshot` hold it from the read to the insert, so they serialize with a `file-gc` pass (`FOR UPDATE ... SKIP LOCKED`) and with a concurrent save, and never pin fragments the pass is about to mark. The lock is exclusive and not `FOR SHARE` because the migration-on-read path may re-read the same row with `FOR UPDATE`: a shared lock that asks for the exclusive one deadlocks two concurrent readers (one creating a snapshot, the other saving, both seeing a pending migration). `get-file-snapshot` realizes the snapshot pointers with the live file id (fragment rows are keyed by file, not by snapshot).
- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders.
- A `file_data` row is one of three classes, told apart by `type`: `main` is the file's current state and carries `id = file_id`; `snapshot` is a point-in-time copy whose `id` is the one its `file_change` row shares; `fragment` is a piece of the pointer-map, referenced from the file's data.
- Their lifecycles differ: `main` and `snapshot` live with the file (dated when it is deleted, undated when it is restored), while a `fragment` carries no date and is collected by `file_gc` once the file stops referencing it. Neither is a session buffer, so do not read a `fragment` as an ephemeral revision.
- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only written by the `legacy-db` backend, plus the rows written before the 2025-07 file storage refactor.
- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only read by the `legacy-db` backend (it is no longer written), plus the rows written before the 2025-07 file storage refactor.
- Tell a snapshot apart by `label IS NOT NULL` (with `created_by` / `deleted_at` as needed), never by `file_change.data IS NOT NULL`: the label holds with every file data backend, and the payload column stopped meaning "this is a snapshot" when the payload moved out. Queries that identify versions must use the label. `0157-migrate-legacy-file-data` moved the labelled leftovers to `file_data` and deleted the unlabelled payload rows, so from 2.19 on the label is the only marker there is.
- The version quota counts of `app.rpc.quotes` (`sql:get-snapshots-per-file`, `sql:get-snapshots-per-team`) are served by the partial index `file_change__user_snapshots__idx`, whose predicate mirrors their where clause; changing one without the other drops the index only scan and the count walks the whole table again.
- Revisions live only in `file_change` and have no `file_data` counterpart. They are born already scheduled for deletion one hour later (`app.rpc.commands.files-update`) and the snapshot paths cannot reach one, because they resolve their row through a `label IS NOT NULL` filter.

View File

@ -3,12 +3,15 @@
JVM `clojure.test` (kaocha runner) under `backend/test/backend_tests/`.
- READ `mem:testing` FIRST — it defines the execution discipline (no piping, tee to file, preferred commands) that applies to all JVM test runs.
- Test DB/Redis come from the `PENPOT_TEST_*` env vars already set in the shell (`PENPOT_TEST_DATABASE_URI`, `PENPOT_TEST_REDIS_URI`, ...); per instance they point at `penpot_test` / `penpot_test_wsN` and valkey DB `6+N` (`mem:devenv/core`). Do not override them with values copied from CI logs or other sources unless the user asks (e.g. "use a temporary database"); the CI env (`redis://redis/1`, `postgresql://postgres/penpot_test`) only works inside the CI docker network.
- The test database may have migrations applied from newer versions (it is shared dev infra). Reinitializing it — `dropdb`/`createdb` of the `PENPOT_TEST_*` database only — is explicitly permitted when that blocks a test run; never drop the main `penpot` database or another instance's test database.
- All CLI commands must be executed from the `backend/` subdirectory.
- Tests are invoked directly via `clojure -M:dev:test` (kaocha) — there is no pnpm wrapper. Kaocha auto-discovers test namespaces, so no runner registration is needed.
- **Preferred way to run them: `scripts/ci <module> --test`**, which runs lint, tests and format per module and writes a log per task under `.ci-logs/`. It exports `PENPOT_FLAGS` with `enable-backend-asserts` appended, so use it rather than calling `clojure -M:dev:test` by hand.
- **Why the flag matters:** `clojure.core/assert` is elided at compile time when `*assert*` is false, and `app.config` only sets that var from the flags when it loads. The test fixture redefines the flags after that, so they never reach it: without the flag the suite runs with every assertion compiled out, and a test that depends on an assert only means something with it set. `scripts/ci` and `tests-backend.yml` export the flag (GitHub Actions sets it too). When running the suite directly — including a `--focus` run — the command is `PENPOT_FLAGS="enable-backend-asserts" clojure -M:dev:test`.
- Coverage: if code is added or modified in `src/`, corresponding tests in `test/backend_tests/` must be added or updated.
- Isolated run: `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace, or `clojure -M:dev:test --focus backend-tests.my-ns-test/my-test-var` for a specific test var.
- Isolated run: `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace, or `clojure -M:dev:test --focus backend-tests.my-ns-test/my-test-var` for a specific test var. Keep the `PENPOT_FLAGS="enable-backend-asserts"` prefix on these too; a focused run without it hides assert-dependent failures exactly like a full one.
- A **whole-suite** crash reported as `Execution error ... Assert failed: ...` with no `FAIL in` line is not a test failure: an `ig/assert-key` ran while the test system was being built, which happens outside `clojure.test`, so kaocha reports it as a load/build error and the summary has no counts. Fix the offending schema (see `mem:backend/subtleties`) rather than hunting for a test.
- Multiple focus symbols are supported by REPEATING the flag (kaocha accumulates them, last-wins does not apply): `clojure -M:dev:test --focus backend-tests.ns-one --focus backend-tests.ns-two` — preferred over a shell loop of isolated runs.
- Regression run: `clojure -M:dev:test` to ensure no regressions in related functional areas.
- If you need to filter output, tee to a temp file first: `clojure -M:dev:test 2>&1 | tee /tmp/penpot-test-output.txt`.

View File

@ -0,0 +1,44 @@
# Backend WebSocket Subscriptions
## Registry
- `app.http.websocket/state` holds `{:connections {conn-id → wsp} :by-profile {profile-id → #{conn-id}}}` in **one** atom, so a single `swap!` keeps both consistent; `register-connection`/`unregister-connection` maintain them together and drop a profile from the index once its last connection leaves.
- Each connection's own `::ws/state` atom holds its subscriptions (`::file-subscription`, `::team-subscription`). `state` is shadowed by `::ws/state` in every `handle-message` destructuring, so only `on-connect`, `on-disconnect` and the `repl-*` helpers touch the registry — reading the registry from a handler is a bug.
- `::profile-id` and `::session-id` on a connection live in the `app.http.websocket` namespace, while `::ws/id`, `::ws/state`, `::ws/output-ch`, `::ws/session-id` live in `app.util.websocket`. Mixing them up yields silently `nil` values (a `nil` `session-id` makes the channel's `:xf` drop **every** message).
- The registry is local to one backend instance. The message bus is shared (`mem:prod-infra/core`). Anything that must reach connections owned by another instance has to go through the bus — closing a connection from an RPC handler only ever works in a single-backend deployment.
## Revocation model
A subscription is authorized once, at `:subscribe-file`/`:subscribe-team`, and that decision used to be trusted for the whole connection life (GHSA-m53j-2766-6jqw, medium). Two independent mechanisms now bound it:
- **Announcement (primary).** A mutation that can change access publishes on `app.rpc.notifications/internal-revocation-topic` (`notify-permissions-changed` for one profile, `notify-team-permissions-changed` for one team id — the watcher resolves the members itself, so a large team is one event, not N). The `::revocation-watcher` Integrant component in `app.http.websocket` subscribes and calls `revalidate-profile-subscriptions`, which looks the profile up in `:by-profile` and re-checks every subscription with `file-readable?`/`team-readable?`: existence (the file gate is the same `sql:get-minimal-file` as the `:subscribe-file` gate, made non-throwing via `{:db/throw-if-not-exists false}`; the team gate is `db/get*` with `::sql/columns [:id :deleted-at]` — `deleted_at` must be in the projection or the soft-delete filter is a no-op) AND the `has-read-permissions?` predicates — and `sql:file-permissions` additionally filters `deleted_at` on file, project and team, as do `sql:get-team-id-for-file`/`-project` plus a final team-row check in `resolve-team-id`, so the organization-owner fallback grants nothing through soft-deleted file/project/team. The existence gate closes that fallback (expelled org owner, deleted resource). The event is only the trigger; the fresh check is the decision, which is what keeps a downgrade to `viewer` (still readable) and a non-member organization owner on a live resource from being mistaken for a revocation. Both publishers defer through `db/after-commit`, because the watcher checks on another connection and would otherwise see the pre-commit state. `:subscribe-team` deliberately has no existence gate of its own; only the re-checks add one, matching the file side (user decision, PR #12189 review).
- **Interval (safety net).** `start-relay` runs the relay as a `go-loop` that waits on the subscription channel and a one-shot tick channel together via `sp/alts!` with `:priority true` on the tick: when both hold something, the tick wins, so a steady stream of messages can never push the re-check away. The cadence is `subscription-revalidation-interval` (default `default-revalidation-interval`, 5 min, a code constant in `app.http.websocket` and **not** in the `config.clj` default map, same rule as session lifetimes). The deadline is fixed per cadence: a single tick channel covers a whole cadence and message turns reuse it (minting a fresh timeout per turn would both churn timers and starve under load exactly like an in-loop timer does); a new one is minted only after a tick fires. It covers what no announcement can: Nitrate transferring organization ownership (an HTTP call to another service) and rows deleted straight over SQL. Closing the channel ends the loop either way, because a take on a closed channel is always ready with nil — every way a subscription can end closes it, so a finished relay never keeps polling. Cost is three or four queries per subscription per interval (existence row + predicates), plus one Nitrate call per team per interval when the `:admin-console` flag is on (cached 30 s); one `timeout-chan` per subscription per cadence on promesa's default scheduler (one or two threads shared with every other scheduled task) — do not shorten the interval without sizing that. A tick abandoned at teardown (the subscription closed while it was pending) still fires its scheduled close harmlessly. Two places fail open, deliberately: `still-authorized?` (a transient DB error logs and keeps the subscription; `InterruptedException` is rethrown) and the `catch Throwable` in `watch-revocations`.
Which mutations announce: `delete-team-member`, `leave-team`, `update-team-member-role` (member only), `delete-team`, `delete-file`, `delete-project`, `move-files`, `move-project` (one team event each). Anything not listed here is covered only by the interval. `delete-share-link` deliberately does not: a share-link visitor never passes the websocket subscribe check, because `check-read-permissions!` is called without a `share-id` and that arity only considers team membership.
`close-file-subscription`/`close-team-subscription` are idempotent: no-op on a mismatched id, and they `dissoc` the subscription so a repeat call and `:pointer-update` do not publish to a dead topic. Closing the channel is what ends the relay (`take!` on a closed channel returns `nil`).
## Topic and layering
- `internal-revocation-topic` is a fixed uuid that is never a real resource id, so it cannot collide with a file, team or profile topic — and deliberately not the `uuid/zero` system topic, which is `sp/pipe`d straight into the client output and anything published there reaches every client. No client can subscribe to the internal topic: `:subscribe-file`/`:subscribe-team` only accept ids that already passed a permission check, and `:broadcast`/`:pointer-update` only republish to ids the server already holds.
- The topic and the publish helpers live in `app.rpc.notifications`, not in `app.http.websocket`: the command namespaces cannot require the watcher, because `app.http.websocket` already requires `app.rpc.commands.files`/`teams`. The watcher keeps the registry, which is the part it owns.
## Time and channel traps in this namespace
- The relay waits with `sp/alts!` over `[tick-ch channel]` with `:priority true`, following the `msgbus.clj` pattern (`start-io-loop` destructures the same `[val port]` shape and compares ports with `identical?`). A take on a closed channel is always ready with nil, which is what ends the loop — and what makes closing the channel from any teardown path prompt even while parked in `alts!`. `sp/timeout-chan` is one-shot: it closes once after the delay, and one abandoned at teardown still fires its scheduled close harmlessly.
- `ct/diff a b` returns **b − a** (`Duration/between`), so "time until a future instant" is `(inst-ms (ct/diff (ct/now) future))`.
- `ct/duration` unit keywords are singular: `{:millis 150}`, `{:minutes 5}`, never `:milliseconds`/`:days`.
- `still-authorized?` takes the check itself: `(boolean (check-access))`. Calling `(boolean pred)` on the closure is always true and silently disables the interval — this exact bug shipped once and no test caught it until the resubscribe-counting test existed. Naming rule here: no `?` for the action that queries and can fail (`check-access`), `?` only for the safe yes/no answer (`still-authorized?`).
## Tests
`backend/test/backend_tests/http_websocket_test.clj` drives the real Redis-backed msgbus, so the revocation tests are end-to-end: register the connection in the real registry, run the real command, wait for the subscription to drop, then assert no content arrives. A connection built without `register-connection` is invisible to the watcher, so the watcher-based tests must go through `open-registered-connection`. Interval tests shorten the config via `with-redefs` on `cf/get` rather than waiting minutes. `with-clean-registry` runs as an `:each` fixture, so no test can leak a live relay loop into the next one. `captured-events` forwards the keyword args through to the real `mbus/pub!` — rebuilding them positionally publishes to the topic `:message` with a `nil` payload and silently untests every other notice. Interval tests take access away with a direct row delete, never with a command: a command announces, so the test would pass even with the interval removed. The same rule covers the two counting/fan-out tests: `revocation-closes-subscriptions-that-are-no-longer-authorized` takes access away with `delete-team-membership!` (raw row delete, publishes nothing) because a command announcement lets the live watcher close both subscriptions before the test's own `revalidate-profile-subscriptions` call, and its `(= 2 …)` count then reads 0; `team-event-closes-every-member-subscription` soft-deletes the team row (`sql:team-members` joins `team_profile_rel` and `profile` only, so the watcher still resolves every member) and asserts nothing closed before one synthetic `notify-team-permissions-changed`, leaving the command path to `deleting-a-team-announces-once` and `deleting-a-team-closes-open-subscriptions`. The resubscribe-counting test is the one that proves finished relays stop polling. The soft-delete + org-owner re-check tests pair a direct row soft-delete with `with-redefs` on `nitrate/organization-owner-of-team?`: before the existence gate, the org-owner fallback kept the subscription open on a deleted resource. The soft-deleted-project/team-closes-org-owner-file-subscription tests pin the root fix in `app.rpc.permissions` (`resolve-team-id` refuses dead file/project/team for every caller of the fallback). `watcher-consumes-revocation-events-from-the-bus` takes access away with a direct row delete and drives the watcher with a synthetic bus event — never a command and never a second `notify-*` — so the "still subscribed" assertion cannot race an after-commit hook and a broken command announcement cannot mask a broken watcher. `moving-a-project-closes-source-team-subscriptions` pins the move-project asymmetry: the file subscription drops, the source-team subscription survives (the member is still a member).
`revocation-watcher-schema-resolves-every-ref` covers the `::revocation-watcher` params schema, which no other test compiles. It existed broken for a while: the three injected keys were written as bare keywords, which is a valid malli *property* position and not a key, so the schema raised `:malli.core/invalid-schema` — invisible unless `PENPOT_FLAGS=enable-backend-asserts` was set, and a crash of the whole system build rather than one test (see `mem:backend/testing` on the flag).
## See also
- `mem:backend/auth-permissions-product-domains` for the permission predicates and the not-found semantics that the re-checks reuse.
- `mem:prod-infra/core` for why the bus crosses backends and the registry does not.
- `mem:frontend/routing-app-shell-subtleties` for the client side: the frontend already reacts to `team-membership-change` by navigating away, which is why revocation needs no new client message.
- `.agents/plans/2026-10-07-websocket-subscription-revocation.md` for the original plan, and the sibling `.agents/plans/2026-10-05-leave-team-file-access-cleanup.md` whose hardened permission queries this re-verification depends on (without live team membership required in `sql:file-permissions`, the re-check would keep reporting access for a member who already left).

View File

@ -8,6 +8,7 @@ How to shape a function in this codebase. Each rule came from a review round; do
- **Functions that obtain data start with `get-` or `fetch-`.** A reader must see at a glance that the function reads (`get-file`, `get-snapshot`, `get-file-json`); a bare noun hides it (`file-json` says nothing about reading). New helpers that fetch by id follow this, never a `current-` prefix: the id parameter already says which one it is.
- **`?` marks a predicate, not a data prop.** A var or function that answers a question takes `?` (`export-error?`, `(defn editable? …)`); a map key that names a property does not (`:export-error`, never `:export-error?`), even when its value is a boolean. Existing `?` keys are legacy to remove gradually; do not add a new one.
- **Do not shape production code for tests.** No return values added just to assert them; assert on the observable effect. When preconditions move, move the tests with them.
- **All code-facing text MUST be in English** — identifiers, comments, docstrings, test descriptions. No exceptions; see the hard rule in `AGENTS.md`.
Reference implementation: the http metrics samplers in `app.http` (`sample-worker-metrics` / `sample-connector-metrics` are guard-free; `sample-http-metrics` guards at the boundary).

View File

@ -11,4 +11,5 @@ Behaviors confirmed against the language/stdlib — do not re-derive from assump
- Inside an `^:async` fn, a nested call that needs a temp binding (e.g. calling a local fn `(g (f x))`, whose arg gets a `G__` binding) compiles to `(await (async function(){…})())`: `await` unwraps a returned promise, so `(settled (upload (make-ctx)))` blocks until the upload promise settles instead of passing the promise on (a test that pushes the answer later hangs, idle). Bind each step in its own `let` binding.
- `!` suffix allowed cases (only exceptions to the no-new-`!` rule in `mem:clojure/design-rules`): mutating public `app.db` API (`insert!`, `update!`, `delete!`, `run!`, `tx-run!`, `exec!`, ...) and fns whose contract is to work over transients (stdlib `conj!`, `assoc!`, `dissoc!`, `disj!`, `persistent!` and custom helpers over them); any further case must be registered here with justification before use.
- Penpot extends `clojure.core/Inst` in `app.common.time` ( JVM-only section), so `inst-ms` also accepts `Duration` (returns `.toMillis`, NOT epoch millis), `FileTime` and `OffsetDateTime`. It only works where that namespace is loaded; on a bare `Duration`, prefer explicit `.toMillis`, which reads as what it is (an amount, not an instant).
- The threading macros insert the threaded value at DIFFERENT positions: `->` and `some->` put it FIRST (`(-> x (f a))` → `(f x a)`), `->>` and `some->>` put it LAST (`(->> x (f a))` → `(f a x)`). `some->`/`some->>` also short-circuit on nil. With unary calls both nest the same way (`(-> x (f) (g))` and `(->> x (f) (g))` are both `(g (f x))`): what changes between them is the position inside each call, not the order of the steps. A pipeline that calls a fn whose first argument is a config/context (`app.features.fdata/resolve-file-data` is `(cfg, file)`) must use `->>`/`some->>`; with `->`/`some->` the arguments are swapped and the call silently returns the wrong value (it cost a broken `get-snapshot` in `app.features.file-snapshots`).
- `cuerdas.core/index-of` returns `nil` (not `-1`) when the substring is absent, like `clojure.string/index-of`. `(pos? nil)` **throws on the JVM** but evaluates to `false` in CLJS (JS coercion), so `(if (pos? (str/index-of s "\n")) …)` works in the browser and crashes the backend. Guard with `some?` before `pos?` (this was `app.common.exceptions/first-line`).

View File

@ -0,0 +1,7 @@
# Clojure Logging Conventions
Structured logging through `app.common.logging` (`l/trc`/`l/dbg`/`l/info`/`l/wrn`/`l/error`/`l/fatal`): `:hint` carries the human-readable message, every other datum goes in its own keyword field (`:id`, `:container`, `:removed`, ...).
- `:hint` values start lowercase: `"fix dangling children"`, not `"Fix dangling children"`. Uppercase-first hints exist as legacy (~57 occurrences, the densest in `backend/src`; also `l/trc` in migration `legacy-28` and `log/error` in `app.common.files.repair`) — do not copy them, and do not mass-rewrite them unless the surrounding code needs the change.
- A name that requires uppercase (an env var like `PENPOT_MEDIA_PROCESSING_SERVICE_URI`, an acronym like `MCP` or `LDAP`) stays uppercase inside an otherwise lowercase sentence.
- Repair work done by a file migration logs at `l/trc`, only when it changes something, and with the ids it touched: the house style of `migrations.cljc` (`legacy-28` precedes; the file has no `warn`/`info` lines). The shipped `log4j2.xml` pins the `app` logger to `info` (static file, `monitorInterval="60"`), so those trace lines appear only after the operator raises the level; the durable record of a repaired file is the migration id in its `:migrations`.

View File

@ -52,3 +52,8 @@ its own drift. Comparing `:selrect.width` from shape A to
`:selrect.width` from shape B is comparing two values that
"semantically" should be equal but were computed via different
operation chains — exact equality will often be false.
## Library sync comparison
- `update-attrs` (`app.common.logic.libraries`) compares main vs copy attrs exactly first, then with tolerance for: `drifting-geometry-attrs` (`:x :y :width :height :selrect :points`, via `gsh/close-attrs?`); path and bool `:content` (`segment/content-close?`; bool shapes keep their path in `:content`); and text `:position-data` (`position-data-close?` over `mth/close-map?`: every numeric field of every entry within `mth/close?`, other fields such as fills, fonts and text exact). All other attrs (`:opacity`, radii, `:transform`, `:rotation`, text `:content`) stay exact: tolerance there hides real changes.
- `close-attrs?` pairs lists with `d/zip`, which truncates to the shorter list and treats nil vs vector as equal for `:points`/`:position-data`. Do not use it for variable-length data. That is why `:position-data` has its own helper, and why `attr-equal?` checks the `:points` count before calling it.

View File

@ -33,4 +33,5 @@
- `migrate-file` synthesizes legacy migration ids from old numeric versions when `:migrations` is absent, migrates legacy features, and records feature flags created through `cfeat/*new*`.
- When a file had no previous `:migrations`, `migrate-file` marks all migrations as migrated in metadata so callers persist the complete migration set, not only transformations that changed data.
- Closed schemas (`schema:stroke-attrs` and others) reject a key that was dropped from the declaration, so removing a declared attr needs a migration in the same change, for data already written with it.
- Rebase collision on a migration number: upstream keeps the id. Renumber the branch migration to the next free one, keep it last in `available-migrations`, and rename everything that spells the number — its test file and namespace (`files_migrations_00NN_test.cljc`), the test names, and comments in production code or memories that name it.
- Rebase collision on a migration number: upstream keeps the id. Renumber the branch migration to the next free one, keep it last in `available-migrations`, and rename everything that spells the number — its test file and namespace (`files_migrations_00NN_test.cljc`), the test names, and comments in production code or memories that name it.
- Withdrawing a migration id: remove both the `defmethod` and the id from `available-migrations`; the replacement migration subsumes it and files that recorded the old id re-migrate through the replacement (persisted `:migrations` keep the old id via union — do not rewrite them, including playwright fixture JSONs). Idempotent/pure-cleanup migrations only; never withdraw one whose transform must not run again unless the replacement is safe for those files. Precedents: "0021-fix-shape-svg-attrs" -> "0035-fix-shape-svg-attrs-and-defs"; "legacy-36" -> "0036-clear-nil-keys-in-objects".

View File

@ -6,6 +6,7 @@ You are working on the GitHub project `penpot/penpot`, a monorepo.
- A section's top-level memory is `<section>/core`. When a section is relevant, read the core memory before focused memories.
- Edits/stale refs/duplication cleanup: `mem:memory-maintenance`.
- Cross-cutting testing principles, TDD workflow, and anti-patterns: `mem:testing`.
- Structured logging conventions (`:hint` casing and fields, log levels): `mem:clojure/logging` — read before adding or changing any log call.
- Verified Clojure language behaviors that contradict common assumptions (e.g. `int?` covers `Long`; `integer?` is the general predicate): `mem:clojure/idioms` — read before assuming stdlib predicate semantics.
- Read `mem:clojure/design-rules` (function shape, boundary checks, naming) before writing or modifying Clojure/ClojureScript code — not only when designing a solution or an API.
- Promises: which call style to use where (`fmap`/`mcat`/`merr` vs `then`/`catch`) and how they bridge to rx: `mem:clojure/promesa` — read before chaining promises or mixing them with observables.

View File

@ -15,6 +15,7 @@ Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Par
- For ws1+, `instance-env-overrides` computes the per-instance overrides (container/volume names, host ports offset `10000·N`, `PENPOT_PUBLIC_URI`, `PENPOT_REDIS_URI`, `PENPOT_TEST_DATABASE_URI`, `PENPOT_TEST_REDIS_URI`) and `instance-compose` injects them as env vars at compose time — never written to disk, recomputed each call so they can't drift. ws0 uses `defaults.env` as-is.
- Per-instance test databases are created by `ensure-instance-test-database` (called from `start-instance`): `CREATE DATABASE` guarded by a `pg_database` existence check, so it is idempotent. Needed because `postgresql_init.sql` only runs on first volume creation. The backend applies migrations itself on first use, so an empty database is enough.
- `backend/scripts/_env`: backend-internal only — secret keys, `PENPOT_FLAGS` (with `enable-backend-worker` gated on `PENPOT_BACKEND_WORKER`), `JAVA_OPTS`, `setup_s3_bucket()`. Never duplicates `defaults.env`.
- Frontend admin-console is off by default in devenv: `docker-compose.main.yml` sets `PENPOT_DEVENV=true`, which makes `app.config` (`goog-define devenv-mode`) prepend `disable-admin-console` ahead of the user `config.js` flags; flags parse in order and user flags win, so `enable-admin-console` in `frontend/resources/public/js/config.js` re-enables it. Outside devenv (prod builds) the default is unset and the admin-console stays on, matching `flags/default`.
- Compose files use pure `${VAR}` substitution; missing var = compose fails.
## Invariants

View File

@ -12,7 +12,7 @@ Frontend: CLJS SPA; React/Rumext; Potok; RxJS; okulary refs; SCSS modules; share
- `app.render_wasm.*`: frontend bridge to Rust/WASM renderer.
- `app.util.*`: DOM, HTTP, i18n, keyboard, codegen, and general frontend utilities.
- `frontend/packages/*` and `frontend/text-editor`: JS/TS workspace packages consumed by the app.
- Nitrate subscription/organization UI and flows live under `app.main.data.nitrate` and `app.main.ui.nitrate*`; backend/API behavior is covered by backend memories, and shared permission rules are in `common/src/app/common/types/nitrate_permissions.cljc`.
- Nitrate subscription/organization UI and flows live under `app.main.data.nitrate` and `app.main.ui.nitrate*`; backend/API behavior is covered by backend memories, and shared organization permission rules (`allowed?`, `can-send-invitations?`) are in `common/src/app/common/types/organization.cljc`.
- `app.config` external bridges (`external-feature-flag`, `external-session-id`, `external-context-info`, `external-notify-register-success`, `initialize-external-context-info`) delegate to `globalThis` hooks injected by the SaaS host. Never delete `external-feature-flag`, even with zero call sites: it is the seam for future A/B tests.

View File

@ -5,9 +5,11 @@
- Dashboard initialization fetches projects and fonts for the team, then listens to websocket messages only for global topic `uuid/zero` or the current profile id.
- Project fetch replaces each project map completely instead of merging, so fields such as `deleted-at` can disappear cleanly.
- Dashboard file/project mutations are often optimistic local updates with fire-and-forget RPC watchers. Bulk permanent delete/restore paths use SSE progress and progress notifications.
- The import wizard runs the package as a job of the backend: it reads the manifest of the file in the main thread (`app.main.data.imports`, no web worker) and follows the job, so its header paints the milestone of the job (the stage and its counter) instead of the messages the old SSE stream sent; every file of a container reports the same milestone, so `jp/current-milestone-text` writes it once and the rows never repeat it. Before the job exists, the header paints a client-side `:upload` milestone built from the chunked upload (`uploads/upload-blob-chunked` `:on-progress`, chunks done/total), which never comes from the backend; the chunks go one at a time and it writes them as a percentage of the file, not as a counter. Once created and still waiting for a worker, no row says anything: the header says the import waits for a slot while no job of it has started. The bar of the header measures the whole import, never a file: indeterminate while checking, waiting for a slot or rebuilding, because the client cannot know a global index and total of operations. The upload is the exception — the chunked upload knows its chunks, so the bar is determinate while the containers are sent, weighted by their size (`imp/upload-pct` over `:container-size`, the only percentage the client really knows). The export modal follows the same one-job-per-file model, but every file is queued up front (even the ones whose job is created only after the previous one is over) and its queue ends only with a milestone: `:start`/`:retry` do not end it there. Neither modal paints a generic "…ing file…" line on the rows any more: the rows of the import wizard carry only what is not progress — the name, the size, `Ready` once their container finished uploading, and the error text —, and the button that drops a file only shows while the wizard still offers its import step (the confirmation screen). Above the rows, the import wizard paints a stepper (Check/Upload/Import) plus one phase line: `imp/import-step` derives the step from the dialog status and the entries (upload holds while any file is pending, either `:import-ready` or carrying its own `:upload` milestone), so Check covers analyze, Upload covers the chunked client-side upload, and Import covers queue plus job. A file of a version 3 package shows what it occupies inside its container, computed while analyzing (`imp/file-sizes`): its own `files/<id>…` entries plus the storage objects its media and thumbnails reference, whose metadata is the only link to `objects/<id>…`. The container's own size stays in `:container-size` (what the dropped file weighs) and is what the upload bar weighs, never what a row shows. A file the manifest lists but the container holds no entry of has no size at all, and its row then shows no subtitle: what the package weighs never stands in for it. The subtitle is the size alone, with no extension. While analyzing with no failures nothing renders below the phase header — no rows, no spinner (`imp/awaiting-analysis?`); the header text is the only "Checking…" message. Rows appear once the analysis resolves (one per file inside the package), and a failed analysis takes the error-summary path instead. The ready step and its Continue wait for the end of the analysis (`imp/ready-to-import?`, fed with the completion of `imp/analyze`): a ready file is not enough, because the files still to arrive would never be imported. While rebuilding, the phase title counts the files and `imp/imported-count` counts them done with the job's own `:files` counter, so the number moves while a container rebuilds its files one by one. The `:error` of an import entry is user-facing text (the backend hint, or a translated label), never an i18n key: the wizard renders it as-is, it does not translate it again. The link-later step takes the file id from the `:id` of its resolution entry, never from the key of the resolution map: the map comes from the job result, so what a key holds depends on the result schema of the job-def that wrote it, while the wizard indexes its selection by the id of the entry. The legacy SSE import stays in the backend for other clients.
- The v3 format carries no size per file: the size of one is the sum of its own entries plus the `objects/` its media and thumbnail metadata reference, the only link between a file and that shared storage.
- Those metadata entries are read one at a time on purpose: a container holds thousands and every read opens its own decompression stream, which makes Chromium fail the analysis — zip.js then masks the cause behind a `Cannot close a ERRORED writable stream` — while Firefox tolerates it. Do not turn this back into a parallel read. The read is best-effort: a reference that cannot be read leaves that file without size, never the whole package.
- The `:error` of an import entry is user-facing text, never an i18n key. The legacy SSE import stays in the backend for other clients.
- File creation/duplication strips file `:data` before putting file summaries into dashboard state.
- The pending invitations are a team-admin only list (backend `:is-admin` already includes the owner). The dashboard hides the invitations entry from the header tabs and the team-options dropdown for non-admins, `invitations-section-content` keys the section on `:is-admin`, and `fetch-invitations` answers `insufficient-permissions` by emitting `invitations-fetched` with `[]`, so a demoted admin does not keep the cached emails and the header does not paint the invite button from a stale list. The organization `send-invitations` policy only gates inviting and resending, not the list.
- The pending invitations are a team-admin only list (backend `:is-admin` already includes the owner). The dashboard hides the invitations entry from the header tabs and the team-options dropdown for non-admins, `invitations-section-content` keys the section on `:is-admin`, and `fetch-invitations` answers `insufficient-permissions` by emitting `invitations-fetched` with `[]`, so a demoted admin does not keep the cached emails and the header does not paint the invite button from a stale list. The organization `send-invitations` policy (`can-invite?`) does not gate the list, but it gates every action on it: inviting (header and empty-state buttons render disabled with the `dashboard.invite-profile-disabled` tooltip), the role selector (read-only label), row selection (so resend and delete never show) and copy link (`invitation-link-copyable?`). The backend enforces the same rule.
## Viewer

View File

@ -15,6 +15,9 @@
## Runtime initialization and permissions
- The frontend initializes `@penpot/plugins-runtime` only after `features/initialize` and only when feature `plugins/runtime` is active. It also installs the runtime `isPluginError` predicate into frontend error handling.
- `app.plugins/init-plugins-runtime` returns whether `ɵloadPlugin` was installed. It calls `preg/signal-runtime-ready` even on failure: `workspace.cljs` waits on that promise before `mcp/init`, so skipping it hangs MCP initialization forever.
- When SES bootstrap fails, `app.plugins` emits one `plugins-runtime-unavailable` telemetry event (with the SES error message), and `app.main.data.plugins/load-plugin!` shows the lazy `plugins.runtime-unavailable` toast when the user opens a plugin. `start-plugin!` (MCP path) stays silent by design.
- Runtime availability is detected by the existing `(fn? (unchecked-get global "ɵloadPlugin"))` check; do not add a second mechanism.
- Manifest parsing expands write permissions to read permissions (`content:write` => `content:read`, etc.). Permission checks also allow the all-zero plugin id and the hard-coded MCP plugin id.
- Manifest URL origin differs by manifest version: v1 clears the path; v2 joins `.` to the plugin URL. Existing plugin ids are reused by matching manifest name and host.
- The MCP plugin id is defined in `app.plugins.register` to avoid a circular dependency with workspace MCP code.

View File

@ -7,6 +7,8 @@
- `set-token!` and `replace-token!` are deliberately asymmetric. `set-token!` is a total no-op when the token already is the current one (no URL write, no report), so navigating to the screen you are on cannot grow the history stack. `replace-token!` always writes and always reports, because it is how the app corrects the URL in place (legacy `#/…` translation, cleaned share links) and that correction must reach the router even when the token did not change. This matches `Html5History`, which did the same; do not "fix" the asymmetry without deciding what navigates to the current screen.
- `app.util.dom/replace-history-state!` is the deliberate bypass used by `app.main.ui.dashboard`/`app.main.ui.static` to rewrite the URL **without** routing. Keep using it there instead of `replace-token!`, which would re-enter the router.
- Route params live entirely in the query map under the reserved `screen` key; duplicate query params can become vectors, so use `rt/get-query-param` when a scalar is required.
- The organization SSO flow sends the **query string** (`rt/get-current-query-string`), not the URL and not the token: the backend builds the destination URL from `public-uri` + `?` + those params, so the client never chooses the origin of the redirect. The history token (`rt/get-current-token`, `location.search`) keeps its leading `?` because every other consumer concatenates it over the base path (`str cf/public-uri token`, `dom/append-query-param`, `dom/remove-query-param`); `rt/token->query-string` is the only bridge between the two shapes, so do not trim at a call site by hand; `rt/token->url` is the single client builder of an absolute destination from either shape (used by the SSO retry and error flow), so do not assemble `public-uri + "?" + params` at call sites. The gate is not the only caller: `dtm/resolve-login-team-id` runs from the root branch of `on-navigate` too, and there the URL carries its params instead of a screen (`?template=…`) — that call only reads `:authorized`, so it never follows the destination, which is exactly why a wrong destination there goes unnoticed. Do not add a rule that requires `screen`. See `mem:backend/auth-permissions-product-domains`.
- The router events expose their payload through `@event` (`IDeref`): `rt/navigate` returns `{:id :params :options}` and `rt/nav-raw` returns its `:href`/`:uri`. `rt/nav-raw` is the way out of the router (raw URL, outside the token scheme) and nothing in the app listens to `::rt/nav-raw`; the destination travels in the effect, so `@event` is the only way to tell where the user is being sent (tests included).
- Legacy `#/…` hash URLs translate client-side to the query format (one-version compat; see `legacy-routes` in `app.main.ui.routes`, TODO(next-version) to delete).
- Unknown/empty routes trigger an extra `get-profile`/`get-teams` check before redirecting. This avoids invitation and root-route race conditions.
- The root app has two error layers, and they reset differently. `:exception` in the store renders `static/exception-page*` directly and is cleared by `rt/navigated`; a React render error is caught by `app.main.ui.error-boundary` (`react-error-boundary`), whose error lives in internal React state that only `reset-error-boundary` clears. Because every exit from the error page (the logo, the no-permission dialogs, the SSO dialog, the feedback link) only *navigates*, the boundary is given `:resetKeys [(eb/reset-key route)]` — the screen name — so a navigation counts as a fresh attempt at rendering. Three traps in that `resetKeys` value, all learned the hard way: it must be a real JS array (`#js [...]`), never a ClojureScript vector, because `react-error-boundary` reads `.length` and `.some` and a `PersistentVector` has neither, so `componentDidUpdate` throws `some is not a function`; it must hold the screen name as a *string*, because the library compares entries with `Object.is` and ClojureScript does not intern keywords, so two equal keywords are two objects and a keyword key would read as new on every route rebuilt, resetting the boundary on a param change inside the same screen; and it must be the screen rather than the params, so a param change does not replay the screen that just crashed. Navigating to the *same* screen therefore does not reset, and a reload is the only way out of that case.
@ -22,6 +24,9 @@
- `flash-persistence` only delegates to another handler for `:authentication`, `:not-found` and `:code :vern-conflict`. Everything else it reports itself, and the report type is derived from the cause: `#{:validation :server-error :internal}` are application defects and report as `unhandled-exception` so they reach the internal reports and the alerts; the rest stay `handled-exception`. `:cause-type` wins over `:type` when classifying, because `persistence-failed` wraps the original cause under `:type :persistence`. Environment failures are deliberately kept handled: they land here after the retry budget is spent and the user's connection is not our bug. `:assertion` is also kept handled, matching its own handler.
- The report governor is global and stateful across tests: two reports with the same fingerprint inside `report-window-ms` collapse. Call `errors/reset-report-governor!` between assertions that expect an emission.
- Error messages are bounded at the source: `app.common.exceptions/get-hint` clamps to `max-message-length` (1000 chars) and the CLJS `format-throwable` clamps the message line of the stack trace and honors `:trace-length`. A message can embed a whole payload (a serialized file, for example) and reach megabytes; the bound keeps the audit `:hint` prop and the `:report` string small. The `Context: Hint:` header in `generate-report` prints `get-hint` (not the raw `:hint` from `ex-data`) for the same reason.
- A team scoped navigation carries a team whenever any of the four tiers exists, and `app.main.data.common/resolve-team-id` decides it for every team scoped `go-to-*` helper of that namespace: the UUID the caller passes, `:current-team-id`, the team of the route already on screen (`?team-id`), then the profile `:default-team-id`. `go-to-feedback` and `go-to-viewer` are not team scoped and do not go through it, and `:team-id :default` keeps meaning the profile default, resolved before the rest. The route goes before the profile default because an emitter that runs from a team screen knows which team the user is looking at, and the personal team would open their file under another team. Two `rt/nav` emitters outside that namespace still decide their own id and can still emit nil, and they are the residual: `app.main.ui.nitrate.entry/nitrate-entry*` (the profile default) and the root branch of `app.main.ui.routes/on-query-navigate` (`dtm/resolve-login-team-id`, which answers nil when the profile has no default team). When none of the tiers exists the helper still fires the navigation with a nil id, so the not-found page appears instead of the click doing nothing.
- `app.main.ui/team-container*` raises `:not-found :invalid-team-id` for a dashboard or workspace route without a valid `team-id`; the raise is what keeps a broken URL from painting a blank shell, so it stays after the hook fix of #12728. `exception-section*` skips `errors/submit-report` for `:not-found`, so this failure never reaches PostHog and cannot be measured from the error dashboard. The exit from that page is the `error-container*` header (logo plus `not-found.no-permission.go-dashboard`), which emits `go-to-dashboard-recent` with the profile default and falls back to `rt/nav-root`; every not-found and authentication page already carries it, so there is no button to add there.
- A click cannot reach a team screen before its team is resolved: `team-container*` paints its children only when the route id matches the store team, and `refs/team` derives from `:current-team-id`, so with that key empty the dashboard and the workspace paint nothing clickable. The nil-id emissions come from events that run on their own (login, invitation acceptance, membership change, the sidebar logo, the workspace shortcuts) and from typed or bookmarked URLs.
## Store and websocket

View File

@ -17,7 +17,8 @@ Frontend testing is async-first: everything essentially asynchronous is modeled
- `mock/with-mocks` (callback style, legacy compat): installs with `set!` so mocks survive async boundaries; bodies run deferred past the current tick via `asap`, so only done-chained (`t/async`) contexts are allowed; `done'` restores and completes exactly once (twice only warns; never calling it stalls the run and leaks the mocks). Prefer `mock/with-mocks*` for new tests.
- `mock/with-mocks*` (direction): body forms wrapped in a generated `^:async` fn, evaluates to a promise — `await` it, `await` nested scopes too, no `done` in test code. Rejections and non-promise returns report as `:error` via `run-mocked`.
- `mock/stub` wraps fns for arities 0-6 (the `:esm` test build dispatches multi-arity vars as `cljs$core$IFn$_invoke$arity$N`); when the mocked var is variadic-defined (or called with more than 6 args), the call compiles to variadic dispatch, so use a plain variadic `fn` — the stub does not forward variadic. A mock must not call the mocked var again (self-delegation inside a multi-arity function recurses).
- Helpers in `frontend-tests.helpers.async`: `->promise` (single-value observable → promise; beicon has no `to-promise`), `await-response` (subscribe→push→await, atomic), `settle`, `wait-for` (immediate check + bounded poll, fails instead of hanging), `observe` (stream → termination promise; asserts provided, timeout rejects).
- Helpers in `frontend-tests.helpers.async`: `->promise` (single-value observable → promise; beicon has no `to-promise`), `await-response` (subscribe→push→await, atomic), `settle`, `wait-for`, `wait-for-store`, `observe` (stream → termination promise; asserts provided, timeout rejects).
- `wait-for` waits on a **signal**, not on turns: `:signal` (an observable) re-checks the predicate on every emission and resolves on the event that changes the state; `:timeout-ms` (default 5000) only bounds failure then. Without a signal it polls against a real-time deadline — 50 turns were ~54 ms idle, and turns are not a unit of time. `wait-for-store` passes `ptk/input-stream` as the signal (the store notifies after applying the update, so the predicate sees the new state); the persistence fixture exposes `requests$` for `(= N (count @requests))`. A wait that resolves on an event consumes no turn. The waits with nothing to listen to stay on the poll: test atoms filled outside the store (`@cycles`, `@reports`, `@done`), `@failures`, and predicates over JS objects or closures.
- Fixtures return promises (`with-watchdog`, `with-persistence`); `await` them from `^:async` tests. `main_errors` and `fonts` are fully migrated (no legacy `with-mocks` left).
- Valid `^:async` placements: `mem:clojure/idioms`.
@ -37,9 +38,14 @@ To assert over emitted event sequences, observe termination: subscribe through `
- `st/emit!` doubles collect heterogeneous events: audit `DataEvent`s deref, toast reify-objects do not — discriminate with `ptk/type` (total, never throws), never blind `deref`.
- Subscription order decides delivery order: never resolve settlement from a pre-subscribed branch racing the pipeline.
- Auto-answering mocks lose deadline expressiveness (can't time answers), need teardown timer-cancellation, and post-teardown deliveries hit real implementations — explicit pushes + async delivery + `wait-for` won on every axis.
- A wait that observes an effect the test asserts takes a signal. A wait that means "the side effect advanced before the next emit" has no event to listen to: keep the poll (or an explicit `settle`) and say so, because a signal would resolve inside the same `emit!` and the next step would overtake the pipeline (`persistence-rollback-test` drove its four emits this way).
- With `rp/cmd!` mocked, a store-state wait can resolve before the flow that changes it ends, and the mock scope then closes in flight: the next call hits the real HTTP layer (malformed URL from a nil `cf/public-uri`) and the error is reported on a later test. `unlink-file-from-library` dissocs the library before its async chain, so the only proof the chain ran inside the mock is its second RPC call; wait for that, through the mock.
- Teardown belongs to the terminal continuation, never to `finally`-around-triggers (it would dispose in-flight flows).
- A body that awaits must be `(^:async fn …)` even if the rest is sync; sync sequences are atomic vs the event loop.
- `app.main.errors` prints every handled cause with `ex/print-throwable` before notifying, and the app logger does not gate it. A test that runs a real handler end to end stubs that var (`main-errors-test/quiet-throwable`): without it, the 10 000-error governor test alone wrote 190 k of the 192 k lines of a run log (13,8 MB), and GitHub truncates a step log past ~4 MB, cutting the summary and any failing assertion. Same for a production `console.error` a test provokes on purpose: `code-gen-markup-svg-wasm-test` silences `(.-error js/console)` in a `try/finally` while two WASM panics go through `get-svg-markup` (2 blocks, 70 lines, and the exception dumps its own `data`/`arr`).
- The `:esm` test build cannot load the CommonJS `sax` module behind `tubax`, so SVG parsing fails in tests (`dwm/valid-svg-string?` answers false, `dwm/svg->clj` errors): mock both with the parsed data.
- A hook-order violation in a mounted component reaches neither the root's `onUncaughtError`/`onCaughtError`/`onRecoverableError` nor the markup: React logs an internal error to `console.error` and recovers, so the callbacks stay empty and the tree repaints. Assert on a captured `console.error` too. It only fires when a render *adds* hooks (nil → uuid); a render that calls none at all slips through React's bookkeeping, so the regression test needs the "hooks arrive" direction. `frontend-tests.ui.team-container-test` does both, with a jsdom document.
- `beicon.v2.core` has no `never`. A stream that must **not emit** is `(rx/empty)`: it completes at once, which is enough for `merge`, `filter` and `take` where only the emission matters. A stream that must **stay open** is a `(rx/subject)` nobody pushes to; `rx/empty` would complete the flow behind it. An undeclared `rx/never` compiles to an undefined var and the test dies as a rejected body, which `cljs.test` reports as a test with zero assertions instead of a failure.
From `frontend/`:
- Full unit test run (always builds, suppressed output): `pnpm run test:quiet`.
@ -55,7 +61,7 @@ New frontend test namespaces must be required/listed in `frontend_tests/runner.c
## Playwright integration tests
Do not add, modify, or run Playwright integration tests under `frontend/playwright` unless explicitly asked. When explicitly asked, use `pnpm run test:e2e` or `pnpm run test:e2e --grep "pattern"` from `frontend/`; ensure dependencies are installed through `./scripts/setup` if the environment is not prepared.
Do not add, modify, or run Playwright integration tests under `frontend/playwright` unless explicitly asked. When explicitly asked, run the whole suite with a worker count sized to the machine: `pnpm run test:e2e --workers=N` from `frontend/`, about one worker per two hardware threads, because each keeps two or three busy and holds about 2 GB of browser memory. The config's default of one worker leaves it slow; four take a 16-thread host from 48 minutes to 35 and fit a 64 GB machine beside a watching devenv, while a 32 GB machine takes two, or four with a slim devenv. `--grep "pattern"` narrows the run. Ensure dependencies are installed through `./scripts/setup` if the environment is not prepared.
Integration tests fake backend behavior by intercepting network/websocket traffic, so every RPC or websocket the page needs must be mocked. Use existing Page Object Models:
- `BasePage.mockRPC` intercepts RPC calls and already prefixes `/api/rpc/command/`; pass command names such as `get-profile`, not full URLs.

View File

@ -18,6 +18,7 @@
## Sandbox and global cleanup
- `src/index.ts` calls `bootstrapSES()` (in `src/lib/bootstrap.ts`) at module load. If `repairIntrinsics` throws (e.g. SES 2.2+ rejects a `Temporal` wrapped by a browser extension), the error is caught, `getPluginsRuntimeError()` exposes it, and `initPluginsRuntime` installs no `ɵ*` globals: the frontend app still boots, plugins and MCP are unavailable in that environment. On failure, `Function.prototype.toString` has already been replaced but no intrinsics are hardened and no compartment/sandbox is created — never run plugins on that path.
- The runtime uses SES compartments. Public API return values are passed through `ses.safeReturn` before crossing back to plugin code.
- Plugin `fetch` is sanitized: credentials are omitted and Authorization is blanked. The exposed response only includes ok/status/statusText/url/text/json.
- Timer callbacks are wrapped to mark plugin-originated errors, and timeout/interval IDs are tracked so plugin close can clear them.

View File

@ -43,7 +43,13 @@ Production runs the deployed toolbox, not the devenv one:
- **Start from the ids, not from an inventory.** The ids come from the error or the logs. Unbounded queries over tables with tens of millions of rows (`storage_object`, `file_media_object`) do not finish in a reasonable time; bounded to an id list they are index lookups.
- **One query per reference column.** Two columns in one `IN (col_a, col_b)` lose the index and scan the whole table; split them.
- The read-only replica cuts long queries: see `mem:prod-infra/grafana`.
- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table (file saves write to `file_change` on every save) build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block); the shipped migration uses `IF NOT EXISTS`, finds it and does nothing. A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_indexes` and `pg_index.indisvalid`).
- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block). A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_index` and `pg_index.indisvalid`).
- A partitioned parent cannot take `CREATE INDEX CONCURRENTLY` (`cannot create index on partitioned table ... concurrently`). Build one index per partition with `CONCURRENTLY`, then `CREATE INDEX ... ON ONLY` on the parent and `ALTER INDEX ... ATTACH PARTITION` for each child; the parent index only becomes valid when every partition is attached.
- Write the migration as plain, idempotent SQL: `ADD COLUMN IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, and `DROP CONSTRAINT IF EXISTS` followed by `ADD CONSTRAINT`. Do not encode shape checks as `pg_catalog` introspection: they are easy to get wrong (a missing `NOT condeferred` accepts an `INITIALLY DEFERRED` foreign key) and they duplicate what the runbook verifies by hand. `IF NOT EXISTS` matches the name, not the definition, so the out-of-band procedure verifies the index shape and completeness.
- The runbook is the contract for a large instance: it builds and verifies the out-of-band work and then marks the migration step as applied (`INSERT INTO migrations (module, step) ...`), so the body does not run at startup. Verify the column, the pending rows, the index (valid, no predicate or expression, every partition attached) and the constraint shape before inserting the row; the marker is the only thing that skips the migration.
- A reference that a GC task resolves must be `DEFERRABLE INITIALLY IMMEDIATE`, never `INITIALLY DEFERRED`: the immediate check fires inside the transaction that catches the violation and retries the chunk; the deferred one fires at commit, outside the catch. A validated foreign key belongs out of band: `ADD FOREIGN KEY` runs as an `ALTER TABLE` and it scans the whole referencing table; `NOT VALID` on a partitioned table only exists since PostgreSQL 18.
- A backfill that removes a key an old reader uses (for example `metadata.storage-ref-id`, which the previous storage GC scans) must run with the old code stopped. Otherwise the old reader loses the reference and can delete a live object, and the rows it writes in the meantime miss the backfill.
- `ADD COLUMN` takes a brief `ACCESS EXCLUSIVE` even when `IF NOT EXISTS` skips it, and `ADD FOREIGN KEY` takes `ACCESS EXCLUSIVE` on the referencing table and `SHARE ROW EXCLUSIVE` on the referenced one. A migration accepts those locks; a deployment that cannot take them runs the runbook and marks the step as applied.
## See also

View File

@ -8,7 +8,8 @@
- `boom <MILESTONE> [--changes CHANGES.md --write]` — generate `### :boom: Breaking changes & Deprecations` from the closed, non-excluded milestone issues labelled `breaking change`, sorted by issue number. Each issue becomes a `#### <issue title>` block: content = body of the issue's last `## Breaking change note` comment (whole markdown, headings demoted to `#####` outside code fences; fallback = heading plus refs with a stderr warning), refs = issue link plus merged closing PRs on their own line, with `(by @author)` for `community contribution`. `--write` replaces only that subsection of the version (inserts it after the version header when missing, keeps the heading when there are no entries; idempotent). Print-only by default. Uses `gh.py comments`.
- `security <MILESTONE> [--changes CHANGES.md --write]` — generate `### :lock: Security fixes` from the advisories whose `patched_versions` include the version (via `gh.py advisories --patched-version`), sorted by GHSA id. Entry: `- Fix <description> (<advisory URL>) (PR: [#N](...))`; description = `Fix` + advisory summary, first word decapitalized unless it starts with an acronym; PRs = merged milestone PRs labelled `pr pt` whose body contains the GHSA id, comma-separated, and no PR is legal. Warns on stderr when a `pr pt` PR references an advisory with no matching `patched_versions`. `--write` replaces only that subsection, placed after `:rocket:` (or `:boom:`, or the version header when neither exists); empty entries remove the subsection; idempotent. Print-only by default.
- `cross-ref <MILESTONE> [--changes CHANGES.md]` — fetch milestone PRs with `--state all` once, report merged PRs missing from the version section plus CLOSED (unmerged) warnings. PR links inside `:lock:` entries count as referenced; advisory URLs never appear here by design.
- `report <MILESTONE> [--changes CHANGES.md --output CHANGES-ISSUES.md]` — overwrite the anomaly report: A/B milestone mismatches, E `:boom:` entry without the label, F `:rocket:` sync, G `:boom:` sync, plus the C/D `:rocket:` gaps (warnings). `:boom:`/`:rocket:` rule violations are fixed by regenerating or relabelling, not by editing entries in place.
- `dev-only-bugs <MILESTONE> [--apply]` — list closed milestone bugs that never reached production, without `no changelog` (bugs carrying `release blocker` are analyzed too). Sure: parent issue is a `Feature`/`Enhancement` of the same milestone; `--apply` adds `no changelog` to these only. Candidates (never labelled automatically): parent is an epic of the milestone, `regression` label, or the "Affected versions" / "Environment" body field names only hourly, develop, staging, PRE or the milestone version itself (`only_unreleased_envs`; any production hint or other version keeps the bug in — versions compare as full semver with missing patch = 0, so for milestone 2.19.2 the released 2.19.0 keeps the bug in). Reads `body` and `parent` from one batched `gh.py issue` lookup.
- `report <MILESTONE> [--changes CHANGES.md --output CHANGES-ISSUES.md]` — overwrite the anomaly report: A/B milestone mismatches, E `:boom:` entry without the label, F `:rocket:` sync, G `:boom:` sync, plus the C/D `:rocket:` gaps and the `dev-only-bugs` lists (warnings). `:boom:`/`:rocket:` rule violations are fixed by regenerating or relabelling, not by editing entries in place.
## Conventions
@ -18,4 +19,4 @@
- PR/issue reference parsing covers standard `[#N](.../pull/N)` / `[#N](.../issues/N)` links plus legacy `PR:[N]` / `[Github #N]` forms.
- Shared parsing helpers (`extract_version_section`, `extract_subsection`, `collect_changelog_prs/issues`) live at module top so future subcommands reuse them instead of duplicating regexes.
- Report anomaly types: A/B milestone mismatches, E `:boom:` entry whose issue lacks the `breaking change` label (counted, entries preserved — human labels the issue or moves the entry), G `breaking change` issue missing from `:boom:` (counted, same contract as F); C/D are `:rocket:` gaps (warnings, never counted).
- `issue_excluded` (closed, non-Task, non-Rejected, no `release blocker`/`no changelog` label) is module-level and shared by `report` and `boom` so the exclusions cannot drift.
- `issue_excluded` (closed, non-Task, non-Rejected, no `no changelog` label) is module-level and shared by `report` and `boom` so the exclusions cannot drift.

View File

@ -40,7 +40,7 @@ python3 scripts/gh.py issues "2.16.0" --label "bug"
python3 scripts/gh.py issues "2.16.0" --label "bug,regression"
# Exclude by label
python3 scripts/gh.py issues "2.16.0" --exclude "release blocker,no changelog"
python3 scripts/gh.py issues "2.16.0" --exclude "no changelog"
# Show only issues NOT yet in CHANGES.md
python3 scripts/gh.py issues "2.16.0" --compare CHANGES.md
@ -81,7 +81,7 @@ python3 scripts/gh.py prs --milestone "2.16.0" --state all
Fetch issue details by number in batches of 50 (same input styles as `prs`: numbers, `--file`, `--stdin`). No filters. A dead number fails its whole batch (unlike PRs, issues error instead of resolving to null), so the batch falls back to one-by-one lookups and reports misses as `{"number", "error": "not_found"}`. Any other failure aborts.
**Output**: JSON array to stdout; progress to stderr.
**Output**: JSON array to stdout; progress to stderr. Each issue carries the shared issue fields plus `body` and `parent` (`number`, `issue_type`, `milestone`, or null); `changelog.py dev-only-bugs` reads both.
### `comments`

View File

@ -73,6 +73,8 @@ a test that reproduces it:
- **One Assertion Per Concept** — each test verifies one behavior; split
compound assertions
- **Descriptive Test Names** — names read like specifications
- **English Only** — test names, `testing` strings, comments, and every
code-facing text MUST be in English; see the hard rule in `AGENTS.md`.
## Prefer Real Implementations Over Mocks

View File

@ -56,6 +56,14 @@ that breaks them is wrong, even if the code is right.
`:bug:` bug fix · `:sparkles:` enhancement · `:tada:` new feature · `:recycle:` refactor · `:lipstick:` cosmetic · `:ambulance:` critical fix · `:books:` docs · `:construction:` WIP · `:boom:` breaking · `:wrench:` config · `:zap:` perf · `:whale:` docker · `:paperclip:` other · `:arrow_up:` dep upgrade · `:arrow_down:` dep downgrade · `:fire:` removal · `:globe_with_meridians:` translations · `:rocket:` epic/highlight
## Atomic Commits
A PR squash-merges into one commit by default, so the split inside the branch
does not reach `main` and nothing depends on it. When the PR asks for a
regular merge (`mem:workflow/creating-prs` → Merge Strategy), every commit
must stand alone: it builds, passes the linter, and needs no later commit on
the same branch to make sense.
## Referencing Issues
Use `Closes #NNNN` (not `Fixes #NNNN`) in commit and PR body text for readable context. The explicit issue-to-PR link is assigned with `python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER>` as described in `mem:workflow/creating-prs`; do not rely on the keyword to create that link.

View File

@ -32,6 +32,19 @@ PR titles follow commit title conventions:
See `mem:workflow/creating-commits` for emoji codes. Squash merge uses the PR title as the final commit subject, so title format matters.
## Merge Strategy
Policy: `CONTRIBUTING.md` → "Merging", under "Pull Requests".
- Squash merge is the default, so the PR's own commits never reach `main`
and the PR title becomes the landed subject.
- A regular merge is possible on request only. The request is the **first
line** of the PR body, above the `**Note:**` line:
`> **Merge:** do not squash; each commit is self-contained.`
- Ask for it only when every commit on the branch is self-contained (builds
and passes lint on its own). Default to squash unless the user asks for
the opposite.
## Description Body
Include concise sections covering:
@ -60,6 +73,8 @@ PR descriptions follow this structure:
```
The "Note:" line is required at the top. Adjust if this is a manual (non-AI) PR.
When the PR requests a regular merge instead of a squash merge, its
`**Merge:**` line goes above the `**Note:**` line (see Merge Strategy).
## Writing Principles

View File

@ -24,11 +24,12 @@ Editing `CHANGES.md` happens only on explicit request or as part of the release
| Is it a breaking change? | `breaking change` label |
| Is it a highlight? | `release highlight` label |
- **Issue Type decides the section**: `Bug` → `:bug:`, `Feature`/`Enhancement` → `:sparkles:`, `Task` is excluded (internal chores). Labels `bug`/`enhancement` and title emoji prefixes are often wrong or missing; do not use them.
- **Issue Type decides the section**: `Feature` → `:tada:`, `Enhancement` → `:sparkles:`, `Bug` → `:bug:`, `Task` is excluded (internal chores). Labels `bug`/`enhancement` and title emoji prefixes are often wrong or missing; do not use them.
- **Security fixes come from advisories**: an advisory whose `patched_versions` include the version becomes a `:lock:` entry (section 7). `triage`, `draft` and `published` are all valid states; the advisory may still be unpublished when the changelog is written. A security fix may also carry a regular bug issue; both entries are legal and independent (the issue lives in `:bug:` as any other bug).
- **Two labels override the Issue Type**: `breaking change` → an entry only under `:boom:`, never in its regular section; `release highlight` → an extra entry under `:rocket:` on top of its regular entry.
- **Only closed issues**. An open issue in the milestone is not generated, even if someone added a provisional entry by hand before the merge.
- **Excluded issues**: `no changelog`, `release blocker`, `Task` issue type, `Rejected` project status.
- **Excluded issues**: `no changelog`, `Task` issue type, `Rejected` project status. `release blocker` is a workflow label (the issue blocks the release), not a changelog exclusion: bugs carrying it are analyzed by `dev-only-bugs` (section 8, step 3), and closed non-bug issues carrying it are reviewed by hand (section 8, step 3).
- **Bugs that never reached production** stay out: a bug found and fixed while building a feature of the same, unreleased version never affected users. The team rarely labels them, so the flow finds them with `dev-only-bugs` and labels them `no changelog` (section 8, step 3).
- **Excluded PRs**: a PR with `release blocker` or `no issue required` does not bring an entry of its own.
- **An issue with no milestone** belongs to another (probably private) project: it is neither changelog candidate nor anomaly.
@ -39,7 +40,7 @@ The newest version goes at the top, right after the `# CHANGELOG` header. Inside
```markdown
# CHANGELOG
## 2.19.0 (Unreleased)
## 2.20.0 (Unreleased)
### :boom: Breaking changes & Deprecations
@ -57,21 +58,27 @@ The newest version goes at the top, right after the `# CHANGELOG` header. Inside
- Fix <user-facing description of the vulnerability> (<advisory URL>) (PR: [#PR](https://github.com/penpot/penpot/pull/PR))
### :bug: Bugs fixed
- Fix <user-facing description of the bug> [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR))
### :sparkles: New features & Enhancements
### :tada: Features
- Add <user-facing description of the feature> [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR))
### :sparkles: Enhancements
- Improve <user-facing description of the existing behaviour> [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR))
### :bug: Bugs
- Fix <user-facing description of the bug> [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR))
```
- `:boom:` first, then `:rocket:`, `:lock:`, `:bug:`, `:sparkles:`. Other headings exist in old versions (`:heart:` community contributions); do not add them to new versions.
- `:boom:` first, then `:rocket:`, `:lock:`, `:tada:`, `:sparkles:`, `:bug:`. Features come before enhancements so they stand out to readers deciding whether to update.
- **This layout applies from 2.20.0.** Versions up to 2.19.x use the old one: `:bug: Bugs fixed` before `:sparkles: New features & Enhancements`, which mixes features and enhancements. Keep the old layout in those sections, including new 2.19.x patches, so a minor line reads the same. The in-app update dialog only reads `:rocket:`, so the change does not affect it.
- Other headings exist in old versions (`:heart:` community contributions); do not add them to new versions.
- Include a section only when it has entries. Historical versions may have an empty `:boom:` or `:rocket:` heading; `boom --write` keeps the `:boom:` heading even when there are no entries. `:lock:` is omitted (and removed on `--write`) when the version has no security advisories.
- One blank line between the last entry of a section and the next heading.
- No manual line wrapping: one line per bullet or paragraph, however long.
## 4. Regular entries (`:bug:`, `:sparkles:`)
## 4. Regular entries (`:tada:`, `:sparkles:`, `:bug:`)
The entry shape is:
@ -152,16 +159,20 @@ Generated from the advisories that declare the version in `patched_versions`; ne
## 8. Flow
1. **Confirm the version** (a semver string such as `2.19.0`) with the user when it is not obvious.
2. **Fetch the milestone issues**: `python3 scripts/gh.py issues "<VERSION>" --exclude "release blocker,no changelog"` (closed by default; add `--state all` only to inspect open issues). `gh.py` already drops `Task` and `Rejected`; pass `--include-tasks` / `--include-rejected` only to inspect them. To find what is missing from an existing section, add `--compare CHANGES.md`.
3. **Fetch PR details when needed**: `python3 scripts/gh.py prs <numbers...>` (also `--milestone`). Use it for the PR author, labels and merge state.
4. **Categorize** by Issue Type and the two labels (section 2), and **build the descriptions** (section 4).
5. **Verify PR merge status** before writing: `python3 scripts/changelog.py check-merged <PR numbers...>`. Only merged PRs go in the changelog; if a closing PR was closed unmerged, look for the PR that superseded it.
6. **Reconcile the existing text** (section 9).
7. **Generate `:boom:`**: `python3 scripts/changelog.py boom "<VERSION>"` prints the section; add `--write` to replace it in `CHANGES.md`. The write replaces only that subsection, inserts it after the version header when missing, keeps the heading when there are no entries, and is idempotent.
8. **Generate `:rocket:`** from the `release highlight` label (section 6).
9. **Generate `:lock:`**: `python3 scripts/changelog.py security "<VERSION>"` prints the section; add `--write` to replace it in `CHANGES.md`. The contract lives in section 7.
10. **Cross-check PRs**: `python3 scripts/changelog.py cross-ref "<VERSION>"`. It lists merged milestone PRs missing from the section and warns about closed (unmerged) PRs. PRs linked from `:lock:` count as referenced; an advisory with no PR is invisible here by design.
11. **Generate the anomaly report**: `python3 scripts/changelog.py report "<VERSION>"`. It overwrites `CHANGES-ISSUES.md` with the current state (section 10).
2. **Fetch the milestone issues**: `python3 scripts/gh.py issues "<VERSION>" --exclude "no changelog"` (closed by default; add `--state all` only to inspect open issues). `gh.py` already drops `Task` and `Rejected`; pass `--include-tasks` / `--include-rejected` only to inspect them. To find what is missing from an existing section, add `--compare CHANGES.md`.
3. **Keep out bugs that never reached production**: `python3 scripts/changelog.py dev-only-bugs "<VERSION>"` lists them; add `--apply` to label the sure ones `no changelog` on GitHub, then re-fetch step 2. The label puts the decision on the issue, so it survives a regeneration. Bugs carrying `release blocker` are analyzed like any other bug.
- **Sure**: a bug whose parent issue is a `Feature` or `Enhancement` of the same milestone. `--apply` labels these.
- **Candidates**: a bug whose parent is an epic of the same milestone, that carries the `regression` label, or whose "Affected versions" / "Environment" field names only hourly, develop, staging, PRE or the milestone version itself. `regression` also marks regressions users hit in production, and the field says where the bug was seen, not since when it exists, so never label these yourself: show them to the user, who labels the ones they confirm. The anomaly report lists them too (section 10).
- **Closed non-bug issues carrying `release blocker`** have no automatic check: `python3 scripts/gh.py issues "<VERSION>" --label "release blocker" --exclude "no changelog"` lists them (closed, non-Task). Show them to the user: the internal ones get the `no changelog` label, the rest keep their entry.
4. **Fetch PR details when needed**: `python3 scripts/gh.py prs <numbers...>` (also `--milestone`). Use it for the PR author, labels and merge state.
5. **Categorize** by Issue Type and the two labels (section 2), and **build the descriptions** (section 4).
6. **Verify PR merge status** before writing: `python3 scripts/changelog.py check-merged <PR numbers...>`. Only merged PRs go in the changelog; if a closing PR was closed unmerged, look for the PR that superseded it.
7. **Reconcile the existing text** (section 9).
8. **Generate `:boom:`**: `python3 scripts/changelog.py boom "<VERSION>"` prints the section; add `--write` to replace it in `CHANGES.md`. The write replaces only that subsection, inserts it after the version header when missing, keeps the heading when there are no entries, and is idempotent.
9. **Generate `:rocket:`** from the `release highlight` label (section 6).
10. **Generate `:lock:`**: `python3 scripts/changelog.py security "<VERSION>"` prints the section; add `--write` to replace it in `CHANGES.md`. The contract lives in section 7.
11. **Cross-check PRs**: `python3 scripts/changelog.py cross-ref "<VERSION>"`. It lists merged milestone PRs missing from the section and warns about closed (unmerged) PRs. PRs linked from `:lock:` count as referenced; an advisory with no PR is invisible here by design.
12. **Generate the anomaly report**: `python3 scripts/changelog.py report "<VERSION>"`. It overwrites `CHANGES-ISSUES.md` with the current state (section 10).
## 9. Pre-flight: reconcile before writing
@ -169,10 +180,10 @@ Re-fetch the milestone (do not trust cached data) and fix these in `CHANGES.md`
1. **Duplicate across versions** → remove it from the current section; the earlier version wins for backports. The same holds after merging a patch branch into `staging`: the fix keeps its released patch entry and leaves the newer, unreleased section (the merge conflict in `CHANGES.md` is resolved by keeping both sections, newest first, minus the duplicate).
2. **Stale milestone assignment** (the issue moved out of this milestone) → remove the entry.
3. **Newly applied exclusion labels** (`no changelog`, `release blocker`) → remove the entry.
3. **Newly applied exclusion labels** (`no changelog`) → remove the entry. (`release blocker` is not an exclusion label; it never removes an entry by itself.)
4. **Issue no longer closed, deleted, or now `Rejected`** → remove the entry.
5. **Unmerged or moved PR reference** → fix the reference or remove the entry. A PR merged in a different milestone is an anomaly (section 10); do not remove it silently.
6. **Issue Type changed to `Task`** → remove the entry.
6. **Issue Type changed** → to `Task`: remove the entry; to another type: move the entry to that type's section.
7. **Breaking change out of sync** (a `breaking change` issue outside `:boom:`, or a `:boom:` entry without the label) → regenerate the section; a labelled issue has no regular entry.
8. **Highlight out of sync** (`:rocket:` entry without the label, or a labelled issue missing) → remove it or add it.
9. **Missing valid issues** (closed, non-excluded, referenced nowhere) → add them to their section.
@ -189,6 +200,8 @@ Re-fetch the milestone (do not trust cached data) and fix these in `CHANGES.md`
- **C — released X.Y.0 version without a `:rocket:` section** (gap; patches never carry highlights).
- **D — `:rocket:` entry without issue AND PR references** (gap).
**Bugs that may never have reached production** are warnings too (section 8, step 3): sub-issues of a milestone feature still unlabelled (run `dev-only-bugs --apply`) and candidates for the user to confirm. They never count toward the total.
Anything else is a rule violation, not a report item: fix it in the pre-flight and re-run.
## 11. Pitfalls
@ -203,6 +216,6 @@ Anything else is a rule violation, not a report item: fix it in the pre-flight a
## 12. Where the code is
- `scripts/gh.py`: `issues`, `issue`, `comments`, `prs`, `link-issue`, `advisories` (with `--patched-version`). Details: `mem:scripts/gh`.
- `scripts/changelog.py`: `check-merged`, `boom`, `security`, `cross-ref`, `report`. Details: `mem:scripts/changelog`.
- `scripts/changelog.py`: `check-merged`, `boom`, `security`, `cross-ref`, `dev-only-bugs`, `report`. Details: `mem:scripts/changelog`.
- Tests: `scripts/test_gh.py`, `scripts/test_changelog.py`.
- The note contract for issue authors: `mem:workflow/creating-issues`, section **Breaking changes**.

View File

@ -28,6 +28,12 @@
- Before `gh pr create` / `gh pr edit` → `mem:workflow/creating-prs` (title format, body structure, AI note)
Don't infer format from the title of a previous commit/issue/PR — the memory
is the source of truth.
- **Everything in code MUST be in English.** Identifiers (functions,
variables, types, namespaces), code comments, docstrings, test
descriptions, log hints — always English, regardless of the language the
author works in. Tests are not an exception. The only place another
language may appear in the codebase is product copy rendered through the
i18n system.
## CRITICAL: Read module memories BEFORE writing any code

View File

@ -1,11 +1,116 @@
# CHANGELOG
## 2.21.0
### :boom: Breaking changes & Deprecations
### :lock: Security fixes
- Fix real-time collaboration WebSocket never re-validates file access after `:subscribe-file`, letting revoked users keep receiving live edit content (https://github.com/penpot/penpot/security/advisories/GHSA-m53j-2766-6jqw) (PR: [#12189](https://github.com/penpot/penpot/pull/12189))
### :tada: Features
- Auto-link libraries during import based on slugified name [#9263](https://github.com/penpot/penpot/issues/9263) (PR: [#9958](https://github.com/penpot/penpot/pull/9958))
- Implement basic bullet lists [#12630](https://github.com/penpot/penpot/issues/12630) (PR: [#12720](https://github.com/penpot/penpot/pull/12720))
### :sparkles: Enhancements
- Improve performance of payload decoding in the workspace (by @lukel97) [#12916](https://github.com/penpot/penpot/issues/12916) (PR: [#12704](https://github.com/penpot/penpot/pull/12704))
- Keep the search when changing library in the Swap component menu (by @desty) [#12335](https://github.com/penpot/penpot/issues/12335) (PR: [#12873](https://github.com/penpot/penpot/pull/12873))
### :bug: Bugs
- Fix workspace stuttering when switching pages in a large file (by @lukel97) [#12917](https://github.com/penpot/penpot/issues/12917) (PR: [#12811](https://github.com/penpot/penpot/pull/12811))
- Fix workspace freezes when undoing or redoing a move of a board with many components (by @lukel97) [#12913](https://github.com/penpot/penpot/issues/12913) (PR: [#12870](https://github.com/penpot/penpot/pull/12870))
- Fix dashboard freezes for about a second after duplicating a large file (by @lukel97) [#12914](https://github.com/penpot/penpot/issues/12914) (PR: [#12911](https://github.com/penpot/penpot/pull/12911))
- Fix not being able to collapse a token set group after reordering it [#12625](https://github.com/penpot/penpot/issues/12625) (PR: [#12671](https://github.com/penpot/penpot/pull/12671))
- Fix slow variant switch with the WebGL renderer [#12968](https://github.com/penpot/penpot/issues/12968) (PR: [#12985](https://github.com/penpot/penpot/pull/12985))
- Fix login page GitHub logo showing with low contrast in the light theme [#12938](https://github.com/penpot/penpot/issues/12938) (PR: [#12943](https://github.com/penpot/penpot/pull/12943))
- Fix leak and black screen on wasm-critical errors [#12977](https://github.com/penpot/penpot/issues/12977) (PR: [#12978](https://github.com/penpot/penpot/pull/12978))
## 2.20.0
### :bug: Bugs fixed
### :boom: Breaking changes & Deprecations
#### Add table partitioning for file-change
The file history table (`file_change`) is now partitioned by file, with the same 16 partitions as the file payload table (`file_data`). Queries that filter by file, and the join between the two tables, resolve against a single partition, so they stay fast as the table grows. The rewrite also removes the dead columns and stores the table in its final layout; the space the old table held comes back when it is dropped. A save no longer writes the file features on every revision, and the team snapshot quota is a single lookup.
The four snapshot RPC methods (`update-file-snapshot`, `delete-file-snapshot`, `lock-file-snapshot` and `unlock-file-snapshot`) now require `file-id`. The frontend already sends it; API clients that call them must add it.
On small and medium instances the table is rewritten automatically on boot; there is nothing to do. On large instances (hundreds of GB in `file_change`) it does not fit in the startup transaction: run it by hand **before upgrading to 2.20.0** following the [large-instance runbook](https://github.com/penpot/penpot/blob/staging/operations/2.20.0-file-change-partition.md). The copy runs while the instance is online, followed by a cutover of a few minutes with the backend and the workers stopped. The old table is kept for rollback; check the disk headroom before starting, and drop the old table days later to get the space back.
[#12416](https://github.com/penpot/penpot/issues/12416) (PR: [#12417](https://github.com/penpot/penpot/pull/12417))
#### Protect the file data storage objects with a foreign key in the backend
This release moves the file data storage pointer out of `file_data.metadata->>'storage-ref-id'` and into a new `file_data.storage_object_id` column with a foreign key to `storage_object(id)`. The metadata key is gone: nothing writes or reads it.
What it means for self-hosted instances:
- The release is a coordinated restart. The new code reads the column; the migration drops `storage-ref-id` from the rows it backfills. Rolling back to the previous binary is not supported after this release runs.
- A fresh database or a small instance needs nothing special: the startup migration does the whole job.
- A large instance should not let the startup migration run. Build the column, the index and the foreign key online beforehand, stop the instances, run the backfill only if a row still carries `metadata.storage-ref-id`, verify the end state, and mark the migration as applied. The full procedure is documented in [`operations/2.20.0-file-data-storage-object-id.md`](https://github.com/penpot/penpot/blob/main/operations/2.20.0-file-data-storage-object-id.md).
- If the storage backend was never enabled, there is nothing to backfill and the stopped window is just the restart.
If you upgrade blindly: on fresh databases and small instances the startup migration completes the change. On a large instance, the foreign key validation scan can block writes for a long time; follow the runbook instead.
[#12768](https://github.com/penpot/penpot/issues/12768) (PR: [#12769](https://github.com/penpot/penpot/pull/12769))
### :rocket: Epics and highlights
- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759)
- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761)
### :tada: Features
- Add a feedback channel to the MCP server [#12490](https://github.com/penpot/penpot/issues/12490) (PR: [#12551](https://github.com/penpot/penpot/pull/12551))
- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759)
- Implement full text format paste within Penpot [#12047](https://github.com/penpot/penpot/issues/12047) (PR: [#12137](https://github.com/penpot/penpot/pull/12137))
- Add team, project and file management to the Plugin API for global plugins [#12170](https://github.com/penpot/penpot/issues/12170) (PR: [#12169](https://github.com/penpot/penpot/pull/12169))
- Serve Penpot under a subpath [#3194](https://github.com/penpot/penpot/issues/3194) (PR: [#12358](https://github.com/penpot/penpot/pull/12358))
- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761)
- Add OAuth support for MCP client connections [#12034](https://github.com/penpot/penpot/issues/12034) (PR: [#12480](https://github.com/penpot/penpot/pull/12480))
### :sparkles: Enhancements
- Auto-scroll sortable lists when dragging an item near their top or bottom edge (by @ShreyashAgare26) [#10478](https://github.com/penpot/penpot/issues/10478) (PR: [#11595](https://github.com/penpot/penpot/pull/11595))
- Make WebGL the default renderer for new accounts [#11855](https://github.com/penpot/penpot/issues/11855) (PR: [#12880](https://github.com/penpot/penpot/pull/12880))
- Prevent the hosted MCP server from running execute_code twice when the plugin is open in two tabs [#12008](https://github.com/penpot/penpot/issues/12008) (PR: [#11927](https://github.com/penpot/penpot/pull/11927))
- Allow detaching typography tokens [#11887](https://github.com/penpot/penpot/issues/11887) (PR: [#11969](https://github.com/penpot/penpot/pull/11969))
- Complete the Russian translations [#11955](https://github.com/penpot/penpot/issues/11955) (PR: [#11956](https://github.com/penpot/penpot/pull/11956))
- Run .penpot file import and export as durable background jobs [#12194](https://github.com/penpot/penpot/issues/12194) (PR: [#12056](https://github.com/penpot/penpot/pull/12056), [#12192](https://github.com/penpot/penpot/pull/12192))
- Open files and projects in a new tab with Ctrl/Cmd+click in the dashboard [#12228](https://github.com/penpot/penpot/issues/12228) (PR: [#12230](https://github.com/penpot/penpot/pull/12230))
- Add an RPC method to read file-level plugin data [#12258](https://github.com/penpot/penpot/issues/12258) (PR: [#10171](https://github.com/penpot/penpot/pull/10171))
- Add a get-file-tokens RPC method to read a file's design tokens [#12260](https://github.com/penpot/penpot/issues/12260) (PR: [#12172](https://github.com/penpot/penpot/pull/12172))
- Show the progress and the size of each file in the import and export dialogs [#12340](https://github.com/penpot/penpot/issues/12340) (PR: [#12192](https://github.com/penpot/penpot/pull/12192))
- Focus the search field when opening the color tokens tab of the color picker [#12344](https://github.com/penpot/penpot/issues/12344) (PR: [#12345](https://github.com/penpot/penpot/pull/12345))
- Harden the file data offload job with a row lock, per-file transactions, throttle and metrics [#12398](https://github.com/penpot/penpot/issues/12398) (PR: [#12403](https://github.com/penpot/penpot/pull/12403))
- Render email templates inside the mail job in the backend [#12445](https://github.com/penpot/penpot/issues/12445) (PR: [#12448](https://github.com/penpot/penpot/pull/12448))
- Run local WASM exports for shapes and boards without freezing the UI [#12512](https://github.com/penpot/penpot/issues/12512) (PR: [#12515](https://github.com/penpot/penpot/pull/12515))
- Use WASM to generate SVG markup in the code inspector when render-wasm is active [#12567](https://github.com/penpot/penpot/issues/12567) (PR: [#12608](https://github.com/penpot/penpot/pull/12608))
- Close the workspace search with the Escape key (by @filipsajdak) [#9540](https://github.com/penpot/penpot/issues/9540) (PR: [#12374](https://github.com/penpot/penpot/pull/12374))
- Restrict management RPC methods to their declared shared keys [#12730](https://github.com/penpot/penpot/issues/12730) (PR: [#12736](https://github.com/penpot/penpot/pull/12736))
- Reduce memory used by duplicate shape ids in the workspace [#12791](https://github.com/penpot/penpot/issues/12791) (PR: [#12792](https://github.com/penpot/penpot/pull/12792))
- Reduce memory consumption on shape decoding [#12789](https://github.com/penpot/penpot/issues/12789) (PR: [#12790](https://github.com/penpot/penpot/pull/12790))
- Notify other users when a snapshot is restored [#12425](https://github.com/penpot/penpot/issues/12425) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Normalize the configured tenant to lowercase [#12842](https://github.com/penpot/penpot/issues/12842) (PR: [#12843](https://github.com/penpot/penpot/pull/12843))
- Speed up saves on large files by checking only the pages a save changes [#12602](https://github.com/penpot/penpot/issues/12602) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Retry saves after short server failures and keep the workspace open when a save fails [#12601](https://github.com/penpot/penpot/issues/12601) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Switch the workspace to read-only mode when the file can't be saved [#12507](https://github.com/penpot/penpot/issues/12507) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Validate upload content against real file bytes in media uploads [#12796](https://github.com/penpot/penpot/issues/12796) (PR: [#12801](https://github.com/penpot/penpot/pull/12801))
- Add the file and MCP session to audit events sent from plugins and the MCP [#12902](https://github.com/penpot/penpot/issues/12902) (PR: [#12972](https://github.com/penpot/penpot/pull/12972))
- Add skip-if-active option to job submission in the backend jobs substrate [#12265](https://github.com/penpot/penpot/issues/12265) (PR: [#12266](https://github.com/penpot/penpot/pull/12266))
- Add HTML paste from clipboard in text editor v3 [#10465](https://github.com/penpot/penpot/issues/10465)
- Skip loading the thumbnail rasterizer in WebGL renderer sessions to cut workspace memory [#13029](https://github.com/penpot/penpot/issues/13029) (PR: [#12835](https://github.com/penpot/penpot/pull/12835))
- Speed up uploading page shapes to WASM [#13030](https://github.com/penpot/penpot/issues/13030) (PR: [#12854](https://github.com/penpot/penpot/pull/12854))
### :bug: Bugs
- Fix font upload failing with a regex error [#10436](https://github.com/penpot/penpot/issues/10436) (PR: [#11951](https://github.com/penpot/penpot/pull/11951))
- Fix dashboard file thumbnails loading slow for large multi-page files [#12053](https://github.com/penpot/penpot/issues/12053) (PR: [#12054](https://github.com/penpot/penpot/pull/12054))
- Fix Plugin API resolving a font family token to its separate words when the family name has several words [#11748](https://github.com/penpot/penpot/issues/11748) (PR: [#11750](https://github.com/penpot/penpot/pull/11750))
- Fix file import recreating circular library references when the exported libraries use each other [#12052](https://github.com/penpot/penpot/issues/12052) (PR: [#12062](https://github.com/penpot/penpot/pull/12062))
- Fix asset redirect responses advertising a cache lifetime of about 1000 days instead of 24 hours (by @ners) [#12761](https://github.com/penpot/penpot/issues/12761) (PR: [#12758](https://github.com/penpot/penpot/pull/12758))
- Fix an internal error on typography tokens due to circular dependencies on other token types [#9580](https://github.com/penpot/penpot/issues/9580)
- Fix the MCP server rejecting request bodies over 100kb (express.json default) with PayloadTooLargeError (by @GitNimay) [#12080](https://github.com/penpot/penpot/issues/12080) (PR: [#12450](https://github.com/penpot/penpot/pull/12450))
@ -22,40 +127,24 @@
- Fix token not showing when selecting two text layers (by @ShreyashAgare26) [#11924](https://github.com/penpot/penpot/issues/11924) (PR: [#11944](https://github.com/penpot/penpot/pull/11944))
- Fix inner shadow fill not working in the SVG render while it works in WebGL [#11967](https://github.com/penpot/penpot/issues/11967) (PR: [#12141](https://github.com/penpot/penpot/pull/12141))
- Fix @penpot/plugin-styles loading Work Sans from a third-party CDN (by @ump45nose) [#12023](https://github.com/penpot/penpot/issues/12023) (PR: [#12087](https://github.com/penpot/penpot/pull/12087))
- Fix schema error on swatch component for image fills [#12135](https://github.com/penpot/penpot/issues/12135) (PR: [#12136](https://github.com/penpot/penpot/pull/12136))
- Fix worker storing more shape data than needed [#12740](https://github.com/penpot/penpot/issues/12740) (PR: [#12840](https://github.com/penpot/penpot/pull/12840))
- Fix worker not freeing pages [#12738](https://github.com/penpot/penpot/issues/12738) (PR: [#12751](https://github.com/penpot/penpot/pull/12751))
- Fix editing the L value in HSL mode resetting H and S to 0 (by @makesomethingshit) [#12615](https://github.com/penpot/penpot/issues/12615) (PR: [#12629](https://github.com/penpot/penpot/pull/12629))
- Restrict dashboard plugins to the MCP plugin and localhost plugins [#12948](https://github.com/penpot/penpot/issues/12948) (PR: [#12949](https://github.com/penpot/penpot/pull/12949))
- Fix workspace memory growing on every page switch with the WebGL renderer [#13028](https://github.com/penpot/penpot/issues/13028) (PR: [#12877](https://github.com/penpot/penpot/pull/12877))
- Fix workspace showing React error #310 when the team container renders without a team id (by @filipsajdak) [#12728](https://github.com/penpot/penpot/issues/12728) (PR: [#12920](https://github.com/penpot/penpot/pull/12920))
### :sparkles: New features & Enhancements
## 2.19.2
- Add a feedback channel to the MCP server [#12490](https://github.com/penpot/penpot/issues/12490) (PR: [#12551](https://github.com/penpot/penpot/pull/12551))
- Auto-scroll sortable lists when dragging an item near their top or bottom edge (by @ShreyashAgare26) [#10478](https://github.com/penpot/penpot/issues/10478) (PR: [#11595](https://github.com/penpot/penpot/pull/11595))
- Make WebGL the default renderer for new accounts [#11855](https://github.com/penpot/penpot/issues/11855) (PR: [#12880](https://github.com/penpot/penpot/pull/12880))
- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759)
- Complete the Russian translations [#11955](https://github.com/penpot/penpot/issues/11955) (PR: [#11956](https://github.com/penpot/penpot/pull/11956))
- Implement full text format paste within Penpot [#12047](https://github.com/penpot/penpot/issues/12047) (PR: [#12137](https://github.com/penpot/penpot/pull/12137))
- Prevent the hosted MCP server from running execute_code twice when the plugin is open in two tabs [#12008](https://github.com/penpot/penpot/issues/12008) (PR: [#11927](https://github.com/penpot/penpot/pull/11927))
- Allow detaching typography tokens [#11887](https://github.com/penpot/penpot/issues/11887) (PR: [#11969](https://github.com/penpot/penpot/pull/11969))
- Add team, project and file management to the Plugin API for global plugins [#12170](https://github.com/penpot/penpot/issues/12170) (PR: [#12169](https://github.com/penpot/penpot/pull/12169))
- Run .penpot file import and export as durable background jobs [#12194](https://github.com/penpot/penpot/issues/12194) (PR: [#12056](https://github.com/penpot/penpot/pull/12056), [#12192](https://github.com/penpot/penpot/pull/12192))
- Open files and projects in a new tab with Ctrl/Cmd+click in the dashboard [#12228](https://github.com/penpot/penpot/issues/12228) (PR: [#12230](https://github.com/penpot/penpot/pull/12230))
- Add an RPC method to read file-level plugin data [#12258](https://github.com/penpot/penpot/issues/12258) (PR: [#10171](https://github.com/penpot/penpot/pull/10171))
- Add a get-file-tokens RPC method to read a file's design tokens [#12260](https://github.com/penpot/penpot/issues/12260) (PR: [#12172](https://github.com/penpot/penpot/pull/12172))
- Show the progress and the size of each file in the import and export dialogs [#12340](https://github.com/penpot/penpot/issues/12340) (PR: [#12192](https://github.com/penpot/penpot/pull/12192))
- Focus the search field when opening the color tokens tab of the color picker [#12344](https://github.com/penpot/penpot/issues/12344) (PR: [#12345](https://github.com/penpot/penpot/pull/12345))
- Harden the file data offload job with a row lock, per-file transactions, throttle and metrics [#12398](https://github.com/penpot/penpot/issues/12398) (PR: [#12403](https://github.com/penpot/penpot/pull/12403))
- Render email templates inside the mail job in the backend [#12445](https://github.com/penpot/penpot/issues/12445) (PR: [#12448](https://github.com/penpot/penpot/pull/12448))
- Serve Penpot under a subpath [#3194](https://github.com/penpot/penpot/issues/3194) (PR: [#12358](https://github.com/penpot/penpot/pull/12358))
- Run local WASM exports for shapes and boards without freezing the UI [#12512](https://github.com/penpot/penpot/issues/12512) (PR: [#12515](https://github.com/penpot/penpot/pull/12515))
- Use WASM to generate SVG markup in the code inspector when render-wasm is active [#12567](https://github.com/penpot/penpot/issues/12567) (PR: [#12608](https://github.com/penpot/penpot/pull/12608))
- Close the workspace search with the Escape key (by @filipsajdak) [#9540](https://github.com/penpot/penpot/issues/9540) (PR: [#12374](https://github.com/penpot/penpot/pull/12374))
- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761)
- Add OAuth support for MCP client connections [#12034](https://github.com/penpot/penpot/issues/12034) (PR: [#12480](https://github.com/penpot/penpot/pull/12480))
- Restrict management RPC methods to their declared shared keys [#12730](https://github.com/penpot/penpot/issues/12730) (PR: [#12736](https://github.com/penpot/penpot/pull/12736))
- Reduce memory used by duplicate shape ids in the workspace [#12791](https://github.com/penpot/penpot/issues/12791) (PR: [#12792](https://github.com/penpot/penpot/pull/12792))
- Reduce memory consumption on shape decoding [#12789](https://github.com/penpot/penpot/issues/12789) (PR: [#12790](https://github.com/penpot/penpot/pull/12790))
- Notify other users when a snapshot is restored [#12425](https://github.com/penpot/penpot/issues/12425) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Normalize the configured tenant to lowercase [#12842](https://github.com/penpot/penpot/issues/12842) (PR: [#12843](https://github.com/penpot/penpot/pull/12843))
- Speed up saves on large files by checking only the pages a save changes [#12602](https://github.com/penpot/penpot/issues/12602) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Retry saves after short server failures and keep the workspace open when a save fails [#12601](https://github.com/penpot/penpot/issues/12601) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
- Switch the workspace to read-only mode when the file can't be saved [#12507](https://github.com/penpot/penpot/issues/12507) (PR: [#12555](https://github.com/penpot/penpot/pull/12555))
### :bug: Bugs fixed
- Fix referential integrity error when copying component variants with nested components across libraries and teams [#10704](https://github.com/penpot/penpot/issues/10704) (PR: [#12928](https://github.com/penpot/penpot/pull/12928))
- Fix migration 0033 inserting nil shape entries for dangling child references that blocked file opening [#12936](https://github.com/penpot/penpot/issues/12936) (PR: [#12979](https://github.com/penpot/penpot/pull/12979))
- Fix switching a variant and adding a component instance taking over 10 seconds in files with many token-styled shapes [#13019](https://github.com/penpot/penpot/issues/13019) (PR: [#13021](https://github.com/penpot/penpot/pull/13021))
- Fix autosave failing after uploading an SVG as a component in the assets panel, and on other mains named unlike their component [#12773](https://github.com/penpot/penpot/issues/12773) (PR: [#12794](https://github.com/penpot/penpot/pull/12794))
- Fix `enable-oidc-registration` not working when password login is disabled [#12964](https://github.com/penpot/penpot/issues/12964) (PR: [#12974](https://github.com/penpot/penpot/pull/12974))
- Fix self-hosted backend opening 60 database connections at startup and keeping them open while idle (by @UsamaSadiq) [#12622](https://github.com/penpot/penpot/issues/12622) (PR: [#13052](https://github.com/penpot/penpot/pull/13052))
## 2.19.1

View File

@ -20,6 +20,7 @@ Center](https://help.penpot.app/).
- [Title format](#title)
- [Description](#description)
- [Review process](#review-process)
- [Merging](#merging)
- [What we won't accept](#what-we-wont-accept)
- [Good first issues](#good-first-issues)
- [Commit Guidelines](#commit-guidelines)
@ -111,8 +112,8 @@ Branch names are not enforced, but we recommend the following:
- Otherwise, use a short, descriptive name with words separated by hyphens
and no slashes (e.g. `fix-ellipse-icon-typo`, `feat-auto-link-libraries`).
Since PRs are squash-merged, the branch name does not survive into the
commit history — what matters is the [PR title](#title).
Since PRs are squash-merged by default, the branch name does not survive
into the commit history — what matters is the [PR title](#title).
### Format
@ -156,9 +157,27 @@ for more concrete information.
- Address review feedback by **pushing new commits** — do not
force-push during review, as it breaks comment threads.
- PRs require at least **one approval** before merge.
- We use **squash-merge** by default. The PR title becomes the final
- We **squash-merge** by default; see [Merging](#merging) for the reason
and for how to ask for a regular merge. The PR title becomes the final
commit message, so follow the [title format](#title) above.
### Merging
PRs are **squash-merged** by default. One PR lands as one commit, which keeps
the change atomic: the history stays readable, and `git bisect` never stops
on an intermediate commit that fails the tests or the linter.
This is the default, not a hard rule. If you keep your commits atomic and
self-contained — each one builds, passes the linter and makes sense on its
own — you may ask for a regular merge instead. Put the request in the first
line of the PR description, above everything else:
> **Merge:** do not squash; each commit is self-contained.
Maintainers honor the request when the commits hold up, and squash-merge
otherwise. After a squash merge the PR title becomes the commit subject on
the main branch; see [Title format](#title).
### What we won't accept
To save time on both sides, please avoid submitting PRs that:

View File

@ -4,7 +4,7 @@
"license": "MPL-2.0",
"author": "Kaleidos INC Sucursal en España SL",
"private": true,
"packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3",
"packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0",
"repository": {
"type": "git",
"url": "https://github.com/penpot/penpot"

131
backend/pnpm-lock.yaml generated
View File

@ -7,153 +7,162 @@ importers:
configDependencies: {}
packageManagerDependencies:
pnpm:
specifier: 12.18.0
version: 12.18.0
specifier: 12.21.0
version: 12.21.0
packages:
'@pnpm/exe.android-arm64@12.18.0':
resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==}
'@pnpm/exe.android-arm64@12.21.0':
resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==}
cpu: [arm64]
os: [android]
'@pnpm/exe.android-x64@12.18.0':
resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==}
'@pnpm/exe.android-x64@12.21.0':
resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==}
cpu: [x64]
os: [android]
'@pnpm/exe.darwin-arm64@12.18.0':
resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==}
'@pnpm/exe.darwin-arm64@12.21.0':
resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==}
cpu: [arm64]
os: [darwin]
'@pnpm/exe.darwin-x64@12.18.0':
resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==}
'@pnpm/exe.darwin-x64@12.21.0':
resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==}
cpu: [x64]
os: [darwin]
'@pnpm/exe.freebsd-x64@12.18.0':
resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==}
'@pnpm/exe.freebsd-x64@12.21.0':
resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==}
cpu: [x64]
os: [freebsd]
'@pnpm/exe.linux-arm64-musl@12.18.0':
resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==}
'@pnpm/exe.linux-arm64-musl@12.21.0':
resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==}
cpu: [arm64]
os: [linux]
libc: [musl]
'@pnpm/exe.linux-arm64@12.18.0':
resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==}
'@pnpm/exe.linux-arm64@12.21.0':
resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==}
cpu: [arm64]
os: [linux]
libc: [glibc]
'@pnpm/exe.linux-ppc64@12.18.0':
resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==}
'@pnpm/exe.linux-ppc64@12.21.0':
resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==}
cpu: [ppc64]
os: [linux]
libc: [glibc]
'@pnpm/exe.linux-riscv64@12.18.0':
resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==}
'@pnpm/exe.linux-riscv64@12.21.0':
resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==}
cpu: [riscv64]
os: [linux]
libc: [glibc]
'@pnpm/exe.linux-s390x@12.18.0':
resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==}
'@pnpm/exe.linux-s390x@12.21.0':
resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==}
cpu: [s390x]
os: [linux]
libc: [glibc]
'@pnpm/exe.linux-x64-musl@12.18.0':
resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==}
'@pnpm/exe.linux-x64-musl@12.21.0':
resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==}
cpu: [x64]
os: [linux]
libc: [musl]
'@pnpm/exe.linux-x64@12.18.0':
resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==}
'@pnpm/exe.linux-x64@12.21.0':
resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==}
cpu: [x64]
os: [linux]
libc: [glibc]
'@pnpm/exe.win32-arm64@12.18.0':
resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==}
'@pnpm/exe.netbsd-x64@12.21.0':
resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==}
cpu: [x64]
os: [netbsd]
'@pnpm/exe.win32-arm64@12.21.0':
resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==}
cpu: [arm64]
os: [win32]
'@pnpm/exe.win32-x64@12.18.0':
resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==}
'@pnpm/exe.win32-x64@12.21.0':
resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==}
cpu: [x64]
os: [win32]
pnpm@12.18.0:
resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==}
pnpm@12.21.0:
resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==}
engines: {node: '>=18'}
hasBin: true
snapshots:
'@pnpm/exe.android-arm64@12.18.0':
'@pnpm/exe.android-arm64@12.21.0':
optional: true
'@pnpm/exe.android-x64@12.18.0':
'@pnpm/exe.android-x64@12.21.0':
optional: true
'@pnpm/exe.darwin-arm64@12.18.0':
'@pnpm/exe.darwin-arm64@12.21.0':
optional: true
'@pnpm/exe.darwin-x64@12.18.0':
'@pnpm/exe.darwin-x64@12.21.0':
optional: true
'@pnpm/exe.freebsd-x64@12.18.0':
'@pnpm/exe.freebsd-x64@12.21.0':
optional: true
'@pnpm/exe.linux-arm64-musl@12.18.0':
'@pnpm/exe.linux-arm64-musl@12.21.0':
optional: true
'@pnpm/exe.linux-arm64@12.18.0':
'@pnpm/exe.linux-arm64@12.21.0':
optional: true
'@pnpm/exe.linux-ppc64@12.18.0':
'@pnpm/exe.linux-ppc64@12.21.0':
optional: true
'@pnpm/exe.linux-riscv64@12.18.0':
'@pnpm/exe.linux-riscv64@12.21.0':
optional: true
'@pnpm/exe.linux-s390x@12.18.0':
'@pnpm/exe.linux-s390x@12.21.0':
optional: true
'@pnpm/exe.linux-x64-musl@12.18.0':
'@pnpm/exe.linux-x64-musl@12.21.0':
optional: true
'@pnpm/exe.linux-x64@12.18.0':
'@pnpm/exe.linux-x64@12.21.0':
optional: true
'@pnpm/exe.win32-arm64@12.18.0':
'@pnpm/exe.netbsd-x64@12.21.0':
optional: true
'@pnpm/exe.win32-x64@12.18.0':
'@pnpm/exe.win32-arm64@12.21.0':
optional: true
pnpm@12.18.0:
'@pnpm/exe.win32-x64@12.21.0':
optional: true
pnpm@12.21.0:
optionalDependencies:
'@pnpm/exe.android-arm64': 12.18.0
'@pnpm/exe.android-x64': 12.18.0
'@pnpm/exe.darwin-arm64': 12.18.0
'@pnpm/exe.darwin-x64': 12.18.0
'@pnpm/exe.freebsd-x64': 12.18.0
'@pnpm/exe.linux-arm64': 12.18.0
'@pnpm/exe.linux-arm64-musl': 12.18.0
'@pnpm/exe.linux-ppc64': 12.18.0
'@pnpm/exe.linux-riscv64': 12.18.0
'@pnpm/exe.linux-s390x': 12.18.0
'@pnpm/exe.linux-x64': 12.18.0
'@pnpm/exe.linux-x64-musl': 12.18.0
'@pnpm/exe.win32-arm64': 12.18.0
'@pnpm/exe.win32-x64': 12.18.0
'@pnpm/exe.android-arm64': 12.21.0
'@pnpm/exe.android-x64': 12.21.0
'@pnpm/exe.darwin-arm64': 12.21.0
'@pnpm/exe.darwin-x64': 12.21.0
'@pnpm/exe.freebsd-x64': 12.21.0
'@pnpm/exe.linux-arm64': 12.21.0
'@pnpm/exe.linux-arm64-musl': 12.21.0
'@pnpm/exe.linux-ppc64': 12.21.0
'@pnpm/exe.linux-riscv64': 12.21.0
'@pnpm/exe.linux-s390x': 12.21.0
'@pnpm/exe.linux-x64': 12.21.0
'@pnpm/exe.linux-x64-musl': 12.21.0
'@pnpm/exe.netbsd-x64': 12.21.0
'@pnpm/exe.win32-arm64': 12.21.0
'@pnpm/exe.win32-x64': 12.21.0
---
lockfileVersion: '9.0'

View File

@ -690,15 +690,113 @@
(assoc :query (u/map->query-string params)))]
(redirect-response uri))))
(defn- public-uri
"The configured public URI, parsed, with the app base path normalized with
a trailing slash."
[]
(u/ensure-path-slash (u/uri (cf/get :public-uri))))
(defn- public-uri-base
"The public URI as a string, with the app base path normalized with a
trailing slash. Every URL the server builds starts here, so the app base
path is never lost."
[]
(str (public-uri)))
(defn- redirect-with-organization-sso-error
[{:keys [dest-url organization-id organization-name]}]
(-> (str (or dest-url (cf/get :public-uri)))
(-> (str (or dest-url (public-uri-base)))
(u/append-query-param :sso-error true)
(u/append-query-param :organization-id organization-id)
(cond-> organization-name
(u/append-query-param :organization-name organization-name))
(redirect-response)))
(defn- routable-params?
"True when `params` are params of a route: at least one, and no key carries
a scheme, a path or a fragment. `screen=workspace&team-id=…` and the root
params of the application (`template=…`) qualify; `https://evil.example/`
does not."
[params]
(and (seq params)
(every? #(nil? (re-find #"[/:?#]" (name %))) (keys params))))
(defn build-dest-url
"Return the absolute URL for `query-string` under the current public URI.
The query string is the route of the client (`screen=…&params`, and the root
of the application may carry only its params, like `template=…`): it does not
carry the `?` that separates it from the path, which is put back here, so the
destination is always built by the server and the client only picks the route.
A value that still comes with the separator is trimmed, so a client bundled
before that change keeps its destination.
The params are parsed and rendered back, so a value that is not a route never
becomes a destination: it is ignored with a warning and the user lands on the
public URI, which is also where a blank one lands."
[query-string]
(try
(let [base (public-uri-base)
query (str query-string)
route (if (str/starts-with? query "?")
(subs query 1)
query)
params (u/query-string->map route)]
(cond
(str/blank? query)
base
(not (routable-params? params))
(do (l/warn :hint "ignoring a redirect target that is not a route"
:query-string (subs route 0 (min 200 (count route))))
base)
:else
(str base "?" (u/map->query-string params))))
(catch Exception e
;; The destination must never break the SSO flow: a query string the
;; server cannot parse lands on the public URI instead.
(l/warn :hint "could not build the redirect target" :cause e)
(public-uri-base))))
(defn- same-origin?
"True when `uri` has the same origin as `base`: scheme, host (case
insensitive) and port, with the default port made explicit."
[uri base]
(let [default-port (fn [scheme] (if (= "https" (str/lower (str scheme))) 443 80))
port (fn [u] (or (:port u) (default-port (:scheme u))))]
(and (= (str/lower (str (:scheme uri)))
(str/lower (str (:scheme base))))
(= (str/lower (str (:host uri)))
(str/lower (str (:host base))))
(= (port uri) (port base)))))
(defn- under-base-path?
"True when the path of `uri` is under the base path of `base`."
[uri base]
(str/starts-with? (str (:path uri)) (:path base)))
(defn- trusted-dest-url
"Return `dest-url` when it points inside the current public URI, nil
otherwise.
The destination is built by the server (see `build-dest-url`), but state
tokens minted before that was the case may carry any URL, so the callback
checks the destination before redirecting to it. The check compares the
parsed origin, so neither a userinfo prefix
(`https://penpot.example.com@evil.example/`) nor a look-alike host
(`https://penpot.example.com.evil.example/`) can pass, and it requires
the path to be under the base path of the public URI, so the destination
stays inside the application."
[dest-url]
(when (some? dest-url)
(let [dest (try (u/uri (str dest-url)) (catch Exception _ nil))
base (public-uri)]
(when (and (some? dest)
(same-origin? dest base)
(under-base-path? dest base))
(str dest-url)))))
(defn- redirect-to-register
[cfg info provider]
(let [info (assoc info
@ -915,11 +1013,18 @@
(defn build-organization-sso-auth-redirect-uri
"Build the OIDC authorization redirect URI for an organization SSO config.
Raises if the config is incomplete or OIDC discovery fails."
[cfg sso & {:keys [dest-url organization-id provider]}]
Raises if the config is incomplete or OIDC discovery fails.
`query-string` is the route of the client (`screen=…&params`, the query
string proper, without its separator): the destination is built here, so it
can never point outside the public URI.
`profile-id` is the profile that started the flow; it travels in the
state token so the callback only grants the organization SSO entry to that
very session."
[cfg sso & {:keys [query-string organization-id provider profile-id]}]
(let [organization-id (or organization-id (:organization-id sso))
issuer (organization-sso-discovery-uri sso)
dest-url (or dest-url (str (cf/get :public-uri)))]
dest-url (build-dest-url query-string)]
(when-not issuer
(raise-invalid-sso-config
:hint "missing issuer"
@ -929,6 +1034,7 @@
state-token (tokens/generate cfg {:iss "oidc"
:dest-url dest-url
:organization-id organization-id
:profile-id profile-id
:issuer issuer
:exp (ct/in-future "4h")})]
(build-auth-redirect-uri oidc-provider state-token))
@ -1024,25 +1130,39 @@
(defn- organization-sso-callback-handler
"Handle the organization-SSO branch of the OIDC callback: state carries
:dest-url — exchange the authorization code with the OIDC provider to
verify authentication actually occurred, then redirect back to dest-url."
verify authentication actually occurred, then redirect back to dest-url.
The state also carries the :profile-id that started the flow: only that
session is granted the organization SSO entry, so a state prepared by
someone else cannot mark the session of the one who completes it."
[cfg request state code]
(let [dest-url (:dest-url state)]
(let [dest-url (trusted-dest-url (:dest-url state))]
(try
(let [organization-id (:organization-id state)
sso (nitrate/call cfg :get-organization-sso {:organization-id organization-id})
provider (prepare-organization-sso-provider cfg sso)
_info (get-info cfg provider state code)
session (session/get-session request)
exp (ct/in-future {:hours 4})]
(when (and session organization-id)
(let [props (-> (or (:props session) {})
(update :sso assoc organization-id exp))]
(session/update-session (::session/manager cfg) (assoc session :props props))
exp (ct/in-future {:hours 4})
;; A state minted before the profile claim existed never
;; matches, and neither does the session of another profile.
same-profile? (and (some? session)
(some? (:profile-id state))
(= (:profile-id state) (:profile-id session)))]
(if (and (some? session) (not same-profile?))
(submit-organization-sso-auth-event
cfg request (:profile-id session) organization-id "organization-sso-auth-failed"
:failure-reason "session-mismatch")
(do
(when (and session organization-id)
(let [props (-> (or (:props session) {})
(update :sso assoc organization-id exp))]
(session/update-session (::session/manager cfg) (assoc session :props props))
(submit-organization-sso-auth-event
cfg request (:profile-id session) organization-id "organization-sso-token-issued")))
(submit-organization-sso-auth-event
cfg request (:profile-id session) organization-id "organization-sso-token-issued")))
(submit-organization-sso-auth-event
cfg request (:profile-id session) organization-id "organization-sso-auth-succeeded")
(redirect-response dest-url))
cfg request (:profile-id session) organization-id "organization-sso-auth-succeeded")))
(redirect-response (or dest-url (public-uri-base))))
(catch Throwable cause
(let [{:keys [code]} (ex-data cause)]
(binding [l/*context* (errors/request->context request)]
@ -1060,10 +1180,36 @@
(let [organization-id (:organization-id state)
organization-name (:name (nitrate/call cfg :get-organization-summary {:organization-id organization-id}))]
(redirect-with-organization-sso-error
{:dest-url dest-url
{:dest-url (or dest-url (public-uri-base))
:organization-id organization-id
:organization-name organization-name}))))))
(def oidc-provider-types
"Backends authenticating through an OIDC-compatible provider.
Only these backends take the SSO registration gate."
#{"oidc" "google" "github" "gitlab"})
(defn sso-registration-allowed?
"SSO registration is allowed when public registration or explicit
OIDC registration is enabled. Shared by the callback gate and the
`register-profile` token gate — keep one definition."
[]
(or (contains? cf/flags :registration)
(contains? cf/flags :oidc-registration)))
(defn- valid-invitation-for?
"Returns true when `invitation-token` is a verifiable
`:team-invitation` JWT issued for `email`. Never throws: a garbage
token means there is no valid invitation."
[cfg invitation-token email]
(try
(boolean
(when (and (some? invitation-token) (some? email))
(let [invitation (tokens/verify cfg {:token invitation-token :iss :team-invitation})]
(= (profile/clean-email (:member-email invitation))
(profile/clean-email email)))))
(catch Throwable _ false)))
(defn- callback-handler
[cfg {:keys [params] :as request}]
(if-let [error (get params :error)]
@ -1096,8 +1242,8 @@
(redirect-with-error "email-domain-not-allowed")
:else
(if (or (contains? cf/flags :registration)
(contains? cf/flags :oidc-registration))
(if (or (sso-registration-allowed?)
(valid-invitation-for? cfg (:invitation-token info) (:email info)))
(redirect-to-register cfg info provider)
(redirect-with-error "registration-disabled")))

View File

@ -45,7 +45,6 @@
(def ^:dynamic *state* nil)
(def ^:dynamic *options* nil)
(def ^:dynamic *reference-file* nil)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; DEFAULTS
@ -161,7 +160,7 @@
[index obj & attrs]
(reduce (fn [index attr-fn]
(let [old-id (attr-fn obj)
new-id (if (::overwrite *options*) old-id (uuid/next))]
new-id (uuid/next)]
(assoc index old-id new-id)))
index
attrs))
@ -212,6 +211,7 @@
p.team_id,
coalesce(fd.backend, 'legacy-db') AS backend,
fd.metadata AS metadata,
fd.storage_object_id AS storage_object_id,
fd.data AS data
FROM file AS f
LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
@ -221,27 +221,37 @@
(str sql:files-with-data " WHERE f.id = ?"))
(def sql:get-file-without-data
(str "WITH files AS (" sql:files-with-data ")"
"SELECT f.id,
f.project_id,
f.created_at,
f.modified_at,
f.deleted_at,
f.name,
f.is_shared,
f.has_media_trimmed,
f.revn,
f.ignore_sync_until,
f.comment_thread_seqn,
f.features,
f.version,
f.vern,
f.team_id
FROM files AS f
WHERE f.id = ?"))
"SELECT f.id,
f.project_id,
f.created_at,
f.modified_at,
f.deleted_at,
f.name,
f.is_shared,
f.has_media_trimmed,
f.revn,
f.ignore_sync_until,
f.comment_thread_seqn,
f.features,
f.version,
f.vern,
p.team_id
FROM file AS f
INNER JOIN project AS p ON (p.id = f.project_id)
WHERE f.id = ?")
(defn- migrate-file
[{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}]
"Apply the pending migrations to the file.
With `persist-after-migration?` the migrated map is also persisted, which is
only safe from a map read under the `file` row lock (see `get-file*`). The
returned map is then the reattached one: decoded data where the pages the
migration did not change keep their pointers and their fragment ids, plus
the `:storage-object-id` of the row the write left, so a caller that
persists again touches the current object. Without `persist-after-migration?`
the realized map is returned, which is what a read-only context can do.
`get-file*` realizes the result when the caller asks for it."
[cfg {:keys [id] :as file} {:keys [persist-after-migration?]}]
(fdata/with-tracked-pointers cfg id
(let [libs (delay (get-resolved-file-libraries cfg file))
;; Capture the pointer layout before realizing: the migration
@ -255,17 +265,93 @@
objects-map-added? (and (not objects-map-before?)
(contains? (:features realized) "fdata/objects-map"))]
(if (or read-only? (db/read-only? conn))
realized
(let [file (if objects-map-added?
realized
(fdata/reattach-unchanged original realized))]
(if persist-after-migration?
(let [file (if objects-map-added?
realized
(fdata/reattach-unchanged original realized))
written (update-file! cfg file)]
;; When the file is migrated, we break the rule of not
;; performing mutations on get operations and update the file
;; with all migrations applied. The caller still receives the
;; realized file: the read contract is plain data.
(update-file! cfg file)
(fmigr/resolve-applied-migrations cfg realized))))))
;; with all migrations applied. The reattached map is also what
;; the read returns: the caller realizes it only when it asks.
(assoc (fmigr/resolve-applied-migrations cfg file)
:storage-object-id (:storage-object-id written)))
realized))))
(defn- fetch-row
"Read the file row. With `load-data?` the file data columns come along, but
they are not resolved nor decoded (see `attach-file-data`). With
`load-migrations?` the applied migrations are resolved, which is what
`need-migration?` expects. A skip-locked read answers nil instead of raising
when the row is gone."
[{:keys [::db/conn] :as cfg} id
{:keys [include-deleted?
skip-locked?
lock-for-update?
lock-for-share?
throw-if-not-exists?
load-migrations?
load-data?]}]
(let [sql (if load-data?
sql:get-file
sql:get-file-without-data)
sql (cond
lock-for-update?
(str sql " FOR UPDATE of f")
lock-for-share?
(str sql " FOR SHARE of f")
:else
sql)
sql (if (and skip-locked? (or lock-for-share? lock-for-update?))
(str sql " SKIP LOCKED")
sql)
file (db/get-with-sql conn [sql id] {::db/throw-if-not-exists false
::db/remove-deleted (not include-deleted?)})
file (some-> file
(d/update-when :features db/decode-pgarray #{})
(d/update-when :metadata fdata/decode-metadata))
file (if (and file load-migrations?)
(fmigr/resolve-applied-migrations cfg file)
file)]
(or file
(when-not (or skip-locked? (not throw-if-not-exists?))
(ex/raise :type :not-found
:code :object-not-found
:hint "database object not found"
:table :file
:file-id id)))))
(defn- attach-file-data
"Resolve the data of an already read row and decode it when `decode?`. It
completes the map it receives instead of reading the row again. It does not
realize: `get-file*` owns that step, once per path."
[cfg file {:keys [decode?]}]
(let [file (fdata/resolve-file-data cfg file)]
(if decode?
(fdata/decode-file-data cfg file)
file)))
(defn- fetch-file
"Read the row with its data resolved and decoded, under the requested lock."
[cfg id options]
(when-let [file (fetch-row cfg id options)]
(attach-file-data cfg file options)))
(def ^:private
default-get-file-options
{:load-migrations? true
:throw-if-not-exists? true
:persist-after-migration? true
:load-data? true
:migrate? true
:decode? true})
;; --- Branch files: derived data --------------------------------------
;;
@ -561,110 +647,123 @@
:branch-oplog-depth depth
:branch-oplog-batches (count rows)}))))
(defn- attach-branch-file-data
"Complete a branch row read with its data columns: the branch counterpart
of `attach-file-data`. A branch file derives its data from the pinned base
snapshot and the op log (see `branch-file-data`), so data migrations apply
in memory only and the derived state stays current without persisting a
data payload. Realizes when `realize?`, as `get-file*` does on every path."
[cfg file {:keys [migrate? realize? decode? include-base-data?] :as options}]
(let [file (cond->> (branch-file-data cfg file decode? include-base-data?)
decode?
(fdata/decode-file-data cfg))]
(cond-> file
(and migrate? decode? (fmg/need-migration? file))
(as-> $ (migrate-file cfg $ (assoc options :persist-after-migration? false)))
realize?
(->> (fdata/realize cfg)))))
(defn- get-file*
[{:keys [::db/conn] :as cfg} id
{:keys [migrate?
realize?
decode?
skip-locked?
include-deleted?
load-data?
throw-if-not-exists?
lock-for-update?
lock-for-share?
include-base-data?]
:or {lock-for-update? false
lock-for-share? false
load-data? true
migrate? true
decode? true
include-deleted? false
throw-if-not-exists? true
realize? false
include-base-data? false}
:as options}]
[{:keys [::db/conn] :as cfg} id options]
(assert (db/connection? conn) "expected cfg with valid connection")
(when (and (not load-data?)
(or lock-for-share? lock-for-update? skip-locked?))
(throw (IllegalArgumentException. "locking is incompatible when `load-data?` is false")))
(let [sql
(if load-data?
sql:get-file
sql:get-file-without-data)
(let [{:keys [migrate?
realize?
decode?
load-data?
lock-for-share?
read-only?]
:as options}
(merge default-get-file-options options)
sql
read-only?
(or read-only? (db/read-only? conn))]
(when (and realize? (or (not load-data?) (not decode?)))
(throw (IllegalArgumentException. "invalid options, you only can realize when loading data and decoding")))
(when (and lock-for-share? load-data?)
(throw (IllegalArgumentException. "invalid options, `:lock-for-share?` is only allowed on row-only reads")))
(cond
;; The caller only wants the row: that read is the answer.
(not load-data?)
(fetch-row cfg id options)
(and migrate? decode? (not read-only?))
(when-let [file (fetch-row cfg id options)]
(cond
lock-for-update?
(str sql " FOR UPDATE of f")
(:is-branch file)
(attach-branch-file-data cfg file options)
lock-for-share?
(str sql " FOR SHARE of f")
(fmg/need-migration? file)
(when-let [file (fetch-file cfg id (assoc options :lock-for-update? true))]
(cond-> file
(fmg/need-migration? file)
(as-> $ (migrate-file cfg $ options))
realize?
(->> (fdata/realize cfg))))
:else
sql)
(let [file (attach-file-data cfg file options)]
(cond-> file
realize?
(->> (fdata/realize cfg))))))
sql
(if skip-locked?
(str sql " SKIP LOCKED")
sql)
:else
(when-let [file (fetch-row cfg id options)]
(if (:is-branch file)
(attach-branch-file-data cfg file options)
(let [file (attach-file-data cfg file options)]
(cond-> file
(and migrate? decode? (fmg/need-migration? file))
(as-> $ (migrate-file cfg $ (assoc options :persist-after-migration? false)))
file
(db/get-with-sql conn [sql id]
{::db/throw-if-not-exists false
::db/remove-deleted (not include-deleted?)})
file
(-> file
(d/update-when :features db/decode-pgarray #{})
(d/update-when :metadata fdata/decode-metadata))]
(if file
(if load-data?
(let [branch? (:is-branch file)
file
(if branch?
(branch-file-data cfg file decode? include-base-data?)
(->> file
(fmigr/resolve-applied-migrations cfg)
(fdata/resolve-file-data cfg)))
;; a branch file derives its data from the pinned base
;; snapshot and the op log; data migrations apply in
;; memory only (read-only) so the derived state stays
;; current without persisting a data payload
will-migrate?
(and migrate? (fmg/need-migration? file))]
(if decode?
(cond->> (fdata/decode-file-data cfg file)
(and realize? (not will-migrate?))
(fdata/realize cfg)
will-migrate?
(migrate-file cfg (cond-> options
branch? (assoc :read-only? true))))
file))
file)
(when-not (or skip-locked? (not throw-if-not-exists?))
(ex/raise :type :not-found
:code :object-not-found
:hint "database object not found"
:table :file
:file-id id)))))
realize?
(->> (fdata/realize cfg)))))))))
(defn get-file
"Get file, resolve all features and apply migrations.
Runs inside a transaction: it joins the caller's when the cfg already
carries a connection and opens one otherwise, so the migration write
commits with the read. A read-only read (`:read-only?`) takes no lock and
writes nothing, so it runs on a plain connection instead of opening a
transaction of its own; with a cfg that already carries a connection it
still joins the caller's. Accepts a cfg map only: a bare pool or a
connection raises `IllegalArgumentException`.
Useful when you have plan to apply massive or not surgical
operations on file, because it removes the overhead of lazy fetching
and decoding."
and decoding.
Options:
- `:realize?` returns plain data instead of fragment pointers; it
requires `:load-data?` and `:decode?`.
- `:decode?` decodes the stored blob (default true).
- `:load-data?` reads the data columns (default true). A row-only read
(`false`) is the only mode that accepts `:lock-for-share?`.
- `:migrate?` applies the pending migrations on read (default true).
- `:load-migrations?` resolves the applied migrations on a row read
(default true).
- `:read-only?` applies the pending migrations in memory only and skips
the transaction. It does not take a lock.
- `:lock-for-update?`, `:skip-locked?`, `:include-deleted?` and
`:throw-if-not-exists?` are the usual read knobs. A lock needs a
read-write transaction from the caller: PostgreSQL rejects row locks in
a read-only transaction.
- `:persist-after-migration?` is internal: `get-file*` turns it off
for the in-memory migration of a read-only context. Do not pass it."
[cfg file-id & {:as opts}]
(db/run! cfg get-file* file-id opts))
(when-not (map? cfg)
(throw (IllegalArgumentException.
"expected cfg map, not a pool or a connection")))
(if (:read-only? opts)
(db/run! cfg get-file* file-id opts)
(db/tx-run! cfg get-file* file-id opts)))
(defn clean-file-features
[file]
@ -692,19 +791,27 @@
inner join project as p on (p.id = f.project_id)
where fpr.file_id = ?
and fpr.profile_id = ?
and f.deleted_at is null
and p.deleted_at is null
and exists (select 1
from team_profile_rel as tpr
inner join team as t on (t.id = tpr.team_id)
where tpr.team_id = p.team_id
and tpr.profile_id = fpr.profile_id)
and tpr.profile_id = fpr.profile_id
and t.deleted_at is null)
union all
select tpr.is_owner,
tpr.is_admin,
tpr.can_edit
from team_profile_rel as tpr
inner join team as t on (t.id = tpr.team_id)
inner join project as p on (p.team_id = tpr.team_id)
inner join file as f on (p.id = f.project_id)
where f.id = ?
and tpr.profile_id = ?
and t.deleted_at is null
and p.deleted_at is null
and f.deleted_at is null
union all
select ppr.is_owner,
ppr.is_admin,
@ -714,10 +821,14 @@
inner join project as p on (p.id = ppr.project_id)
where f.id = ?
and ppr.profile_id = ?
and f.deleted_at is null
and p.deleted_at is null
and exists (select 1
from team_profile_rel as tpr
inner join team as t on (t.id = tpr.team_id)
where tpr.team_id = p.team_id
and tpr.profile_id = ppr.profile_id)")
and tpr.profile_id = ppr.profile_id
and t.deleted_at is null)")
(defn- get-file-permissions*
[conn profile-id file-id]
@ -796,10 +907,13 @@
ON fls.file_id = flr.file_id
AND fls.library_file_id = flr.library_file_id
WHERE flr.file_id = ANY(?)
AND l.deleted_at IS NULL;")
AND l.deleted_at IS NULL
ORDER BY flr.file_id, flr.library_file_id;")
(defn get-files-rels
"Given a set of file-id's, return all matching relations with the libraries"
"Given a set of file-id's, return all matching relations with the libraries.
Rows come back in stable `(file-id, library-file-id)` order so export
and import resolve cycles deterministically."
[cfg ids]
(assert (set? ids) "expected a set of uuids")
@ -842,6 +956,20 @@
(->> (db/exec! conn [sql:get-libraries ids'])
(into #{} xf-map-id))))))
(defn library-link-status
"Check whether linking `file-id` to `library-id` is safe. Returns
`:ok` when the link can be created, `:self-link` when both ids are
the same, and `:circular` when the link would close a dependency
cycle (i.e. `file-id` is already among the transitive libraries of
`library-id`). Shared by the UI command (`link-file-to-library`,
which raises) and the v3 importer (`import-file-relations`, which
skips and logs), so both stay consistent."
[cfg file-id library-id]
(cond
(= file-id library-id) :self-link
(contains? (get-libraries cfg [library-id]) file-id) :circular
:else :ok))
(defn get-file-object-thumbnails
"Return all file object thumbnails for a given file."
[cfg file-id]
@ -1017,7 +1145,12 @@
(let [library-ids (get-libraries cfg [file-id])
data (reduce (fn [data library-id]
(if-let [library (get-file cfg library-id :include-deleted? true)]
(cond-> (ctf/absorb-assets data (:data library))
;; The library is read without `:realize?`, so its
;; `:components` may be a pointer map. Bind the
;; library's own loader: the absorb walks the
;; components and would raise without one.
(cond-> (fdata/with-pointers cfg library-id
(ctf/absorb-assets data (:data library)))
(= library-id (cfo/get-tokens-source data))
(cfo/absorb-tokens-lib (:data library)))
data))
@ -1102,18 +1235,33 @@
(dissoc :metadata)
(dissoc :migrations)))
(defn file->response
"Strip the internal keys of a file map before it leaves the backend in a
response. The file data storage pointer is bookkeeping for the write path:
a `file-data` object is authentication-only, so a leaked id would let any
logged-in profile fetch the blob from `/assets/by-id`. The metadata key of
the old pointer is stripped too: a row the migration does not backfill keeps
it, and the metadata schema is open, so it would travel in the response."
[file]
(-> file
(dissoc :storage-object-id)
(d/update-when :metadata #(dissoc % :storage-ref-id))))
(defn- file->file-data-params
[{:keys [id] :as file} & {:as opts}]
[{:keys [id] :as file}]
(let [created-at (or (:created-at file) (ct/now))
modified-at (or (:modified-at file) created-at)]
(d/without-nils
{:id id
:type "main"
:file-id id
:data (:data file)
:metadata (:metadata file)
:created-at created-at
:modified-at modified-at})))
(-> (d/without-nils
{:id id
:type "main"
:file-id id
:data (:data file)
:metadata (:metadata file)
:created-at created-at
:modified-at modified-at})
;; The storage object the persisted row pointed at (nil for a new or
;; imported file). `upsert!` touches it before replacing the row.
(assoc :storage-object-id (:storage-object-id file)))))
(defn insert-file!
"Insert a new file into the database table. Expectes a not-encoded file.
@ -1137,34 +1285,63 @@
:cause cause)
(throw cause))))
(->> (file->file-data-params file)
;; An INSERT never replaces a persisted pointer: `duplicate-file` hands
;; the source file map here, and touching the source object would be a
;; write on a live object.
(->> (file->file-data-params (dissoc file :storage-object-id))
(fdata/upsert! cfg))
nil))
(defn update-file!
"Update an existing file on the database. Expects not encoded file."
[{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
"Update an existing file on the database. Expects a not-encoded file map,
as returned by `get-file`, and optionally the fields to change as keyword
args; they are merged into the file map before persisting.
(if (::reset-migrations? opts false)
(fmigr/reset-migrations! conn file)
(fmigr/upsert-migrations! conn file))
The storage object the persisted row pointed at travels in the map
(`:storage-object-id`), and `upsert!` touches it before replacing the row.
That is why the caller must pass the map it read instead of building a
partial one: nothing is queried here. The params only change mutable fields
of the file: the pointer and the identity (`:id`, `:file-id`) always come
from the read map.
(let [file
(encode-file cfg file)
Returns the map `fdata/upsert!` answers with: `:written` and the
`:storage-object-id` of the object this write created, so a caller that
persists again touches the current object instead of the replaced one."
[{:keys [::db/conn] :as cfg} file & {:as params}]
file-params
(file->params (dissoc file :id))
;; A map read with `:load-data? false` has no `:data`; persisting it would
;; wipe the file content. `ex/raise` instead of `assert`: the guard must
;; exist in the deployed binary, where `clojure.core/assert` is compiled
;; out.
(when-not (contains? file :data)
(ex/raise :type :internal
:code :missing-file-data
:hint "expected a file map read with data"
:file-id (:id file)))
file-data-params
(file->file-data-params file)]
(let [reset? (::reset-migrations? params false)
;; The pointer and the identity are bookkeeping of the persisted row,
;; not editable fields: they can only come from the read map. A pointer
;; in the params would skip the touch and leak the replaced blob; an
;; `:id` would retarget the UPDATE.
file (merge file (dissoc params ::reset-migrations?
:storage-object-id
:id
:file-id))]
(db/update! conn :file file-params
{:id id}
{::db/return-keys false})
(if reset?
(fmigr/reset-migrations! conn file)
(fmigr/upsert-migrations! conn file))
(fdata/upsert! cfg file-data-params)
nil))
(let [file (encode-file cfg file)
file-params (file->params (dissoc file :id))]
(db/update! conn :file file-params
{:id (:id file)}
{::db/return-keys false})
(fdata/upsert! cfg (file->file-data-params file)))))
(defn insert-file-row!
"Insert only the `file` row of a new file, without writing the
@ -1223,8 +1400,7 @@
(let [file (-> file
(assoc :created-at timestamp)
(assoc :modified-at timestamp)
(cond-> (not (::overwrite cfg))
(assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5}))))
(assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5})))
(update :features
(fn [features]
(-> (::features cfg #{})
@ -1241,9 +1417,7 @@
(when (ex/exception? result)
(l/error :hint "file schema validation error" :cause result))))
(if (::overwrite cfg)
(update-file! cfg file (assoc opts ::reset-migrations? true))
(insert-file! cfg file opts))))
(insert-file! cfg file opts)))
(def ^:private sql:get-file-libraries
"WITH RECURSIVE libs AS (

View File

@ -482,9 +482,11 @@
(bfm/apply-pending-migrations! cfg)
;; Knowing that the ids of the created files are in index,
;; just lookup them and return them in the same shape as v3.
;; just lookup them and return them in the same shape as v3. The
;; metadata of an old package may repeat a file id, so the answer
;; carries one id per imported file.
(let [files (-> bfc/*state* deref :files)]
{:file-ids (into [] (keep #(get-in @bfc/*state* [:index %])) files)})))))
{:file-ids (into [] (comp (keep #(get-in @bfc/*state* [:index %])) (distinct)) files)})))))
(defmethod read-import :v1
[options]
@ -510,89 +512,114 @@
(defmethod read-section :v1/files
[{:keys [::bfc/input ::bfc/project-id ::bfc/name] :as system}]
(doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))]
(let [file (read-obj! input)
media (read-obj! input)
(let [imported (volatile! #{})]
(doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))]
(let [file (read-obj! input)
media (read-obj! input)
file-id (:id file)]
file-id (:id file)
file-id' (bfc/lookup-index file-id)
(when (not= file-id expected-file-id)
(ex/raise :type :validation
:code :inconsistent-penpot-file
:found-id file-id
:expected-id expected-file-id
:hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
file (bfc/clean-file-features file)
thumbnails (:thumbnails file)]
(if (contains? @imported file-id)
;; Old exports could repeat a file id in the metadata (the file was
;; also one of its own libraries). The entry is read above to keep
;; the stream in sync, but the file is imported only once.
(l/warn :hint "ignoring repeated file entry"
:file-id (str file-id)
::l/sync? true)
(when (not= file-id expected-file-id)
(ex/raise :type :validation
:code :inconsistent-penpot-file
:found-id file-id
:expected-id expected-file-id
:hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
(let [file-id' (bfc/lookup-index file-id)
file (bfc/clean-file-features file)
thumbnails (:thumbnails file)]
(l/dbg :hint "processing file"
:id (str file-id)
:features (:features file)
:version (-> file :data :version)
:media (count media)
:thumbnails (count thumbnails)
::l/sync? true)
(vswap! imported conj file-id)
(when (seq thumbnails)
(let [thumbnails (remap-thumbnails thumbnails file-id')]
(l/dbg :hint "updated index with thumbnails"
:total (count thumbnails)
::l/sync? true)
(vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails)))
(l/dbg :hint "processing file"
:id (str file-id)
:features (:features file)
:version (-> file :data :version)
:media (count media)
:thumbnails (count thumbnails)
::l/sync? true)
(when (seq media)
;; Update index with media
(l/dbg :hint "update index with media" :total (count media) ::l/sync? true)
(vswap! bfc/*state* update :index bfc/update-index (map :id media))
(when (seq thumbnails)
(let [thumbnails (remap-thumbnails thumbnails file-id')]
(l/dbg :hint "updated index with thumbnails"
:total (count thumbnails)
::l/sync? true)
(vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails)))
;; Store file media for later insertion
(l/dbg :hint "update media references" ::l/sync? true)
(vswap! bfc/*state* update :media into (map #(update % :id bfc/lookup-index)) media))
(when (seq media)
;; Update index with media
(l/dbg :hint "update index with media" :total (count media) ::l/sync? true)
(vswap! bfc/*state* update :index bfc/update-index (map :id media))
(let [file (-> file
(assoc :id file-id')
(cond-> (and (= idx 0) (some? name))
(assoc :name name))
(assoc :project-id project-id)
(dissoc :thumbnails))
file (bfc/process-file system file)]
;; Store file media for later insertion
(l/dbg :hint "update media references" ::l/sync? true)
(vswap! bfc/*state* update :media into (map #(update % :id bfc/lookup-index)) media))
;; All features that are enabled and requires explicit migration are
;; added to the state for a posterior migration step.
(doseq [feature (-> (::bfc/features system)
(set/difference cfeat/no-migration-features)
(set/difference (:features file)))]
(vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id']))
(let [file (-> file
(assoc :id file-id')
(cond-> (and (= idx 0) (some? name))
(assoc :name name))
(assoc :project-id project-id)
(dissoc :thumbnails))
file (bfc/process-file system file)]
(l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
(bfc/save-file! system file ::db/return-keys false)
;; All features that are enabled and requires explicit migration
;; are added to the state for a posterior migration step.
(doseq [feature (-> (::bfc/features system)
(set/difference cfeat/no-migration-features)
(set/difference (:features file)))]
(vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id']))
file-id'))))
(l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
(bfc/save-file! system file ::db/return-keys false)
file-id')))))))
(defmethod read-section :v1/rels
[{:keys [::db/conn ::bfc/input ::bfc/timestamp]}]
(let [rels (read-obj! input)
[{:keys [::db/conn ::bfc/input ::bfc/timestamp] :as cfg}]
(let [rels (->> (read-obj! input)
(sort-by (fn [{:keys [file-id library-file-id]}]
[(str file-id) (str library-file-id)])))
ids (into #{} (-> bfc/*state* deref :files))]
;; Insert all file relations
(doseq [{:keys [library-file-id] :as rel} rels]
(let [rel (-> rel
(update :file-id bfc/lookup-index)
(update :library-file-id bfc/lookup-index))]
(if (contains? ids library-file-id)
(do
(l/dbg :hint "create file library link"
:file-id (:file-id rel)
:lib-id (:library-file-id rel)
::l/sync? true)
(let [rel-params (dissoc rel :synced-at)]
(db/insert! conn :file-library-rel rel-params)
(bfc/upsert-file-library-sync! conn {:file-id (:file-id rel-params)
:library-file-id (:library-file-id rel-params)
:synced-at (or (:synced-at rel)
timestamp)})))
(case (bfc/library-link-status cfg (:file-id rel) (:library-file-id rel))
:self-link
(l/warn :hint "ignoring self-referencing library link"
:file-id (:file-id rel)
:lib-id (:library-file-id rel)
::l/sync? true)
:circular
(l/warn :hint "ignoring circular library link"
:file-id (:file-id rel)
:lib-id (:library-file-id rel)
::l/sync? true)
:ok
(do
(l/dbg :hint "create file library link"
:file-id (:file-id rel)
:lib-id (:library-file-id rel)
::l/sync? true)
(let [rel-params (dissoc rel :synced-at)]
(db/insert! conn :file-library-rel rel-params)
(bfc/upsert-file-library-sync! conn {:file-id (:file-id rel-params)
:library-file-id (:library-file-id rel-params)
:synced-at (or (:synced-at rel)
timestamp)}))))
(l/warn :hint "ignoring file library link"
:file-id (:file-id rel)

View File

@ -319,6 +319,11 @@
(db/insert! conn :file-tagged-object-thumbnail thumbnail
{::db/return-keys false})))
;; NOTE: v2 packages are only produced and imported through the
;; developer REPL (`app.srepl.binfile`), never through the product, so
;; this path keeps inserting the relations as they come: unlike v1 and
;; v3 it does not skip self-referencing or circular links (see
;; `bfc/library-link-status`).
(doseq [rel (read-obj cfg :file-rels file-id)]
(let [rel (-> rel
(update :file-id bfc/lookup-index)

View File

@ -988,8 +988,7 @@
:old-id (str (:id item))
::l/sync? true)
(db/insert! conn :file-media-object params
::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
(db/insert! conn :file-media-object params)))
(doseq [[index item] (d/enumerate thumbnails)]
(let [media-id (bfc/lookup-index (:media-id item))
@ -1038,26 +1037,49 @@
file-id')))
(defn- import-file-relations
"Import library relations from the manifest, ignoring relations whose
endpoints are not both part of the package. Of the rest, skip edges that
would close a dependency cycle or link a file to itself (same check as
`link-file-to-library`, see `bfc/library-link-status`). Relations are
processed in sorted manifest order and the first edge wins: an edge
that would close a cycle with previously imported relations is skipped
and logged."
[{:keys [::db/conn ::manifest ::bfc/timestamp] :as cfg}]
(let [relations (:relations manifest)
(let [relations (sort-by (fn [[file-id libr-id]] [(str file-id) (str libr-id)])
(:relations manifest))
file-ids (into #{} (map :id) (:files manifest))]
(doseq [[index [file-id libr-id]] (d/enumerate relations)]
(tap-progress cfg {:section :relations
:current (inc index) :total (count relations)})
:current (inc index)
:total (count relations)})
(if (and (not= file-id libr-id)
(contains? file-ids file-id)
(if (and (contains? file-ids file-id)
(contains? file-ids libr-id))
(let [file-id (bfc/lookup-index file-id)
libr-id (bfc/lookup-index libr-id)]
(l/dbg :hint "create file library link"
:file-id (str file-id)
:lib-id (str libr-id)
::l/sync? true)
(let [rel-params {:file-id file-id
:library-file-id libr-id}]
(db/insert! conn :file-library-rel rel-params)
(bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))
(case (bfc/library-link-status cfg file-id libr-id)
:self-link
(l/warn :hint "ignoring self-referencing library link"
:file-id (str file-id)
:lib-id (str libr-id)
::l/sync? true)
:circular
(l/warn :hint "ignoring circular library link"
:file-id (str file-id)
:lib-id (str libr-id)
::l/sync? true)
:ok
(do
(l/dbg :hint "create file library link"
:file-id (str file-id)
:lib-id (str libr-id)
::l/sync? true)
(let [rel-params {:file-id file-id
:library-file-id libr-id}]
(db/insert! conn :file-library-rel rel-params)
(bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))))
(l/warn :hint "ignoring invalid file library relation"
:file-id (str file-id)
:lib-id (str libr-id))))))
@ -1282,70 +1304,6 @@
{:file-ids file-ids
:resolution resolution})))
(defn- invalidate-thumbnails
[cfg file-id]
(let [storage (sto/resolve cfg ::db/reuse-conn true)
sql-1
(str "update file_tagged_object_thumbnail "
" set deleted_at = now() "
" where file_id=? returning media_id")
sql-2
(str "update file_thumbnail "
" set deleted_at = now() "
" where file_id=? returning media_id")]
(run! #(sto/touch-object! storage %)
(sequence
(keep :media-id)
(concat
(db/exec! cfg [sql-1 file-id])
(db/exec! cfg [sql-2 file-id]))))))
(def ^:private sql:is-branch-file
"SELECT f.is_branch
FROM file AS f
WHERE f.id = ?")
(defn- import-file-and-overwrite*
[{:keys [::manifest ::bfc/file-id] :as cfg}]
(when (not= 1 (count (:files manifest)))
(ex/raise :type :validation
:code :invalid-condition
:hint "unable to perform in-place update with binfile containing more than 1 file"
:manifest manifest))
;; A branch file's data payload is derived from the branch base and
;; the operation log; overwriting it would persist a `file_data` row
;; no read path consults.
(when (:is-branch (db/exec-one! cfg [sql:is-branch-file file-id]))
(ex/raise :type :validation
:code :branch-file-cant-be-overwritten
:hint "branch file data is derived from the branch base and the operation log and can't be overwritten by an import"
:file-id file-id))
(bfc/configure-import-transaction cfg)
(let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
file (first (get manifest :files))
cfg (assoc cfg ::bfc/overwrite true)]
(vswap! bfc/*state* update :index assoc (:id file) file-id)
(binding [bfc/*options* cfg
bfc/*reference-file* ref-file]
(import-storage-objects cfg)
(import-file cfg file {:current 1 :total 1})
(invalidate-thumbnails cfg file-id)
(bfm/apply-pending-migrations! cfg)
{:file-ids [file-id]
:resolution {}})))
(defn- import-files
[{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
@ -1396,9 +1354,7 @@
(tap-progress cfg {:section :manifest})
(binding [bfc/*state* (volatile! {:media [] :index {}})]
(if (::bfc/file-id cfg)
(db/tx-run! cfg import-file-and-overwrite*)
(db/tx-run! cfg import-files*)))))
(db/tx-run! cfg import-files*))))
;; --- PUBLIC API
@ -1467,6 +1423,12 @@
(defn import-files!
[{:keys [::bfc/input] :as cfg}]
(when (::bfc/file-id cfg)
(ex/raise :type :validation
:code :not-supported
:hint "in-place import is not supported"
:file-id (::bfc/file-id cfg)))
(assert
(and (uuid? (::bfc/profile-id cfg))
(uuid? (::bfc/project-id cfg)))

View File

@ -10,6 +10,7 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.flags :as flags]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
@ -168,6 +169,10 @@
[:http-server-io-threads {:optional true} ::sm/int]
[:http-server-max-worker-threads {:optional true} ::sm/int]
;; How often a websocket subscription re-checks that its access still
;; holds. The default lives in `app.http.websocket`, not here.
[:subscription-revalidation-interval {:optional true} ::ct/duration]
;; Explicit CORS allowlist used when the :cors flag is enabled.
;; Configured via PENPOT_ALLOWED_ORIGINS as a comma/whitespace
;; separated list of origins (e.g. "https://plugins.example.com").
@ -431,6 +436,22 @@
(def explain-config
(sm/explainer schema:config))
(defn- normalize-file-data-backend
"The file data moved to `file_data` (migration `0157-migrate-legacy-file-data`), so
`file.data` and `file_change.data` are read-only leftovers and the `legacy-db`
back end is read-only: a write against it fails on every save. Accepting it would
start a healthy instance that cannot save a single file, so it reads as `db`,
which serves the legacy rows too. Warn instead of aborting: the operator may
still be halfway through the migration."
[config]
(if (= "legacy-db" (c/get config :file-data-backend))
(do
(l/warn :hint "file-data-backend=legacy-db is read-only, using db"
:file-data-backend "legacy-db"
:replacement "db")
(assoc config :file-data-backend "db"))
config))
(defn read-config
"Reads the configuration from environment variables and decodes all
known values."
@ -438,7 +459,8 @@
(->> (read-env prefix)
(merge default)
(normalize-tenant)
(decode-config)))
(decode-config)
(normalize-file-data-backend)))
(def version
(v/parse (or (some-> (io/resource "version.txt")

View File

@ -70,6 +70,7 @@
(def defaults
{::name :main
::min-size 0
::max-size 60
::connection-timeout 30000
::validation-timeout 10000
@ -92,8 +93,7 @@
:read-only read-only
:credentials (and (contains? cfg ::username)
(contains? cfg ::password))
:min-size (or (::min-size cfg)
(::max-size cfg))
:min-size (::min-size cfg)
:max-size (::max-size cfg))
(create-pool cfg))))
@ -108,15 +108,16 @@
(def initsql
(str "SET statement_timeout = 300000;\n"
"SET idle_in_transaction_session_timeout = 300000;"))
"SET idle_in_transaction_session_timeout = 300000;\n"
;; lets the planner resolve a join between two partitioned tables
;; pairwise, when both are partitioned with the same bounds
"SET enable_partitionwise_join = on;"))
(defn- create-datasource-config
[{:keys [::uri] :as cfg}]
;; (app.common.pprint/pprint cfg)
(let [config (HikariConfig.)
max-size (::max-size cfg)
min-size (or (::min-size cfg) max-size)]
(let [config (HikariConfig.)]
(doto config
(.setJdbcUrl (str "jdbc:" uri))
(.setPoolName (d/name (::name cfg)))
@ -126,8 +127,8 @@
(.setValidationTimeout (::validation-timeout cfg))
(.setIdleTimeout (::idle-timeout cfg))
(.setMaxLifetime (::max-lifetime cfg))
(.setMinimumIdle min-size)
(.setMaximumPoolSize max-size)
(.setMinimumIdle (::min-size cfg))
(.setMaximumPoolSize (::max-size cfg))
(.setConnectionInitSql initsql)
(.setInitializationFailTimeout -1))

View File

@ -105,11 +105,16 @@
(dissoc file :legacy-data))
(defmethod resolve-file-data "storage"
[cfg {:keys [metadata] :as file}]
[cfg {:keys [storage-object-id] :as file}]
(let [storage (sto/resolve cfg ::db/reuse-conn true)
ref-id (:storage-ref-id metadata)
data (->> (sto/get-object storage ref-id)
(sto/get-object-bytes storage))]
sobject (or (sto/get-object storage storage-object-id)
(ex/raise :type :internal
:code :missing-storage-object
:hint "file data row has no readable storage object"
:file-id (or (:file-id file) (:id file))
:id (:id file)
:storage-object-id storage-object-id))
data (->> (sto/get-object-bytes storage sobject))]
(-> file
(assoc :data data)
(dissoc :legacy-data))))
@ -122,8 +127,8 @@
(def ^:private sql:insert-file-data
"INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at,
type, backend, metadata, data)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
type, backend, metadata, data, storage_object_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
(def ^:private sql:upsert-file-data
(str sql:insert-file-data
@ -132,105 +137,103 @@
deleted_at=?,
backend=?,
metadata=?,
data=?"))
data=?,
storage_object_id=?
WHERE file_data.storage_object_id IS NOT DISTINCT FROM ?"))
(defn- upsert-in-database
[cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata]}]
[cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata storage-object-id]} expected-storage-object-id]
(let [created-at (or created-at (ct/now))
metadata (some-> metadata db/json)
modified-at (or modified-at created-at)]
;; The metadata pointer is dead (the column is the only source of
;; truth). Clean it here, the single point where every branch
;; serializes the metadata, so no write can resurrect it.
metadata (some-> metadata (dissoc :storage-ref-id) (db/json))
modified-at (or modified-at created-at)
(db/exec-one! cfg [sql:upsert-file-data
file-id id
created-at
modified-at
deleted-at
type
backend
metadata
data
modified-at
deleted-at
backend
metadata
data])))
result
(db/exec-one! cfg [sql:upsert-file-data
file-id id
created-at
modified-at
deleted-at
type
backend
metadata
data
storage-object-id
modified-at
deleted-at
backend
metadata
data
storage-object-id
expected-storage-object-id])]
;; A zero count means the row exists with another pointer: the caller
;; read it before someone else replaced it. Every writer holds the `file`
;; row lock, so this is a protocol violation, not a race to retry.
(when-not (pos? (db/get-update-count result))
(ex/raise :type :internal
:code :stale-file-data-pointer
:hint "the file data row changed since it was read"
:file-id file-id
:id id))
result))
(defn- handle-persistence
[cfg {:keys [type backend id file-id data] :as params}]
(cond
(= backend "storage")
;; IMPORTANT: we strongly do not reuse the main connection that can
;; run inside a transaction because the storage upload process can
;; fail in the middle of uploading and leave garbage on the underlying
;; backend, if we participate in the main transaction and it aborts
;; we will lose all registry of the pending to reconcile blobs
;; what the storage subsystem registers in other parallel
;; transaction
(let [storage (sto/resolve cfg)
content (sto/content data)
sobject (sto/put-object! storage
{::sto/content content
::sto/touch true
:bucket "file-data"
:content-type "application/octet-stream"
:file-id file-id
:id id})
metadata (-> (:metadata params)
(assoc :storage-ref-id (:id sobject)))
params (-> params
(assoc :metadata metadata)
(assoc :data nil))]
(upsert-in-database cfg params))
(= backend "db")
(let [metadata (dissoc (:metadata params) :storage-ref-id)
params (assoc params :metadata metadata)]
(upsert-in-database cfg params))
(= backend "legacy-db")
[cfg {:keys [backend id file-id data] :as params}]
(let [expected (:storage-object-id params)]
(cond
(= type "main")
(do
(db/delete! cfg :file-data
{:id id :file-id file-id :type "main"}
{::db/return-keys false})
(db/update! cfg :file
{:data data}
{:id file-id}
{::db/return-keys false}))
(= backend "storage")
;; IMPORTANT: we strongly do not reuse the main connection that can
;; run inside a transaction because the storage upload process can
;; fail in the middle of uploading and leave garbage on the underlying
;; backend, if we participate in the main transaction and it aborts
;; we will lose all registry of the pending to reconcile blobs
;; what the storage subsystem registers in other parallel
;; transaction
(let [storage (sto/resolve cfg)
content (sto/content data)
sobject (sto/put-object! storage
{::sto/content content
::sto/touch true
:bucket "file-data"
:content-type "application/octet-stream"
:file-id file-id
:id id})
params (-> params
(assoc :storage-object-id (:id sobject))
(assoc :data nil))]
{:result (upsert-in-database cfg params expected)
:storage-object-id (:id sobject)})
(= type "snapshot")
(do
(db/delete! cfg :file-data
{:id id :file-id file-id :type "snapshot"}
{::db/return-keys false})
(db/update! cfg :file-change
{:data data}
{:file-id file-id :id id}
{::db/return-keys false}))
(= type "fragment")
(upsert-in-database cfg
(-> (dissoc params :metadata)
(assoc :backend "db")))
(= backend "db")
{:result (upsert-in-database cfg (assoc params :storage-object-id nil) expected)
:storage-object-id nil}
:else
(throw (RuntimeException. "not implemented")))
(throw (IllegalArgumentException.
(str "backend '" backend "' not supported"))))))
:else
(throw (IllegalArgumentException.
(str "backend '" backend "' not supported")))))
(defn process-metadata
[cfg metadata]
(when-let [storage-id (:storage-ref-id metadata)]
(let [storage (sto/resolve cfg ::db/reuse-conn true)]
(sto/touch-object! storage storage-id))))
(defn touch-storage-object
"Mark a storage object as touched so the storage GC can reclaim it once
nothing references it anymore. Used when a file data row replaces or
deletes the object it pointed at. Expects a non-nil object id; callers
guard with `some->`/`some->>`."
[cfg storage-object-id]
(-> (sto/resolve cfg ::db/reuse-conn true)
(sto/touch-object! storage-object-id)))
(defn- default-backend
[backend]
(or backend (cf/get :file-data-backend)))
(let [backend (or backend (cf/get :file-data-backend))]
(when (= "legacy-db" backend)
(ex/raise :type :validation
:code :legacy-backend-read-only
:hint "the legacy-db file data backend is read-only; set the file data backend to db or storage"
:backend backend))
backend))
(defn decode-metadata
[metadata]
@ -243,8 +246,12 @@
[:id ::sm/uuid]
[:type [:enum "main" "snapshot" "fragment"]]
[:file-id ::sm/uuid]
[:backend {:optional true} [:enum "db" "legacy-db" "storage"]]
[:backend {:optional true} [:enum "db" "storage"]]
[:metadata {:optional true} ctf/schema:file-metadata]
;; The storage object the persisted row pointed at before this write; it is
;; the object to touch when the row replaces it. Required so a new caller
;; must decide it explicitly (nil for a new row) instead of leaking it.
[:storage-object-id [:maybe ::sm/uuid]]
[:data {:optional true} bytes?]
[:created-at {:optional true} ::ct/inst]
[:modified-at {:optional true} [:maybe ::ct/inst]]
@ -254,17 +261,28 @@
(sm/check-fn schema:update-params :hint "invalid params received for update"))
(defn upsert!
"Create or update file data"
"Create or update a file data row.
`:storage-object-id` is the storage object the persisted row pointed at
before this write (nil for a new row or a row that had none), not the
object the new content will live in: the write resolves that itself. The
previous object is marked as touched before the row is replaced, so the
storage GC can reclaim the blob once nothing references it.
The function does not query the row: the caller must hand the pointer it
read. Returns a map with `:written` (whether a row was written) and the
`:storage-object-id` of the object this write created, nil for the `db`
and `legacy-db` backends."
[cfg params & {:as opts}]
(let [params (-> (check-update-params params)
(update :backend default-backend))]
(some->> (:metadata params)
(process-metadata cfg))
(some->> (:storage-object-id params)
(touch-storage-object cfg))
(-> (handle-persistence cfg params)
(db/get-update-count)
(pos?))))
(let [{:keys [result storage-object-id]} (handle-persistence cfg params)]
{:written (pos? (db/get-update-count result))
:storage-object-id storage-object-id})))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; POINTER-MAP
@ -278,7 +296,7 @@
;; previous pass marked must still load until `objects-gc` purges it.
(let [fragment (some-> (db/get* cfg :file-data
{:id id :file-id file-id :type "fragment"}
{::sql/columns [:data :backend :id :metadata]
{::sql/columns [:data :backend :file_id :id :metadata :storage_object_id]
::db/remove-deleted false})
(update :metadata decode-metadata))]
@ -305,9 +323,12 @@
(when (pmap/modified? item)
(l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
(let [content (-> item deref blob/encode)]
;; A modified pointer has a fresh id, so the fragment row is always a
;; new insert: there is no previous storage object to touch.
(upsert! cfg {:id id
:file-id file-id
:type "fragment"
:storage-object-id nil
:data content})))))
(defmacro with-pointers

View File

@ -0,0 +1,21 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.features.file-changes
"The kinds of the rows of the `file_change` table.
A snapshot is the state of a file at a point: it has a label and its
data lives in `file_data` (old `legacy-db` payloads live in
`file_change.data`). A revision is the delta of a save: it has
no label and it holds the changes.")
(def snapshot-kind
"The kind of a snapshot row."
1)
(def revision-kind
"The kind of a revision row."
2)

View File

@ -19,6 +19,7 @@
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.fdata :as fdata]
[app.features.file-changes :as fchanges]
[app.jobs :as jobs]
[app.storage :as sto]
[app.util.blob :as blob]
@ -41,7 +42,8 @@
c.data AS legacy_data,
fd.data AS data,
coalesce(fd.backend, 'legacy-db') AS backend,
fd.metadata AS metadata
fd.metadata AS metadata,
fd.storage_object_id AS storage_object_id
FROM file_change AS c
LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id
AND fd.id = c.id
@ -64,68 +66,88 @@
(d/update-when :features db/decode-pgarray #{}))))
(def ^:private sql:get-minimal-file
;; `fd.storage_object_id` is load-bearing: `restore!` hands this map to
;; `update-file!` so the previous main object is touched before the row is
;; replaced. Removing it would leave the old blob unreclaimed with no error.
"SELECT f.id,
f.revn,
f.modified_at,
f.deleted_at,
f.is_branch,
fd.backend AS backend,
fd.metadata AS metadata
fd.metadata AS metadata,
fd.storage_object_id AS storage_object_id
FROM file AS f
LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
WHERE f.id = ?")
(def ^:private sql:snapshot-columns
"c.id,
c.label,
c.revn,
c.created_at,
c.modified_at,
c.deleted_at,
c.profile_id,
c.created_by,
c.locked_by,
c.features,
c.metadata,
c.migrations,
c.version,
c.file_id")
(def ^:private sql:snapshot-visible
"AND CASE WHEN c.created_by = 'user'
THEN c.deleted_at IS NULL
WHEN c.created_by = 'system'
THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz
END")
(def ^:private sql:get-snapshot-without-data
(str "WITH snapshots AS (" sql:snapshots ")"
"SELECT c.id,
c.label,
c.revn,
c.created_at,
c.modified_at,
c.deleted_at,
c.profile_id,
c.created_by,
c.locked_by,
c.features,
c.metadata,
c.migrations,
c.version,
c.file_id
FROM snapshots AS c
WHERE c.id = ?
AND CASE WHEN c.created_by = 'user'
THEN c.deleted_at IS NULL
WHEN c.created_by = 'system'
THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz
END"))
"SELECT " sql:snapshot-columns
" FROM snapshots AS c
WHERE c.file_id = ?
AND c.id = ? " sql:snapshot-visible))
(defn get-minimal-snapshot
[cfg snapshot-id]
"Get the minimal snapshot info (without the data blob). The query filters
by `file_id`, which lets the planner prune the partitions of `file_change`
when it chooses an index scan. Raises `:snapshot-not-found` when there is
no matching snapshot."
[cfg file-id snapshot-id]
(let [now (ct/now)]
(-> (db/get-with-sql cfg [sql:get-snapshot-without-data snapshot-id now]
{::db/remove-deleted false})
(decode-snapshot))))
(or (-> (db/get-with-sql cfg [sql:get-snapshot-without-data file-id snapshot-id now]
{::db/remove-deleted false
::db/throw-if-not-exists false})
(decode-snapshot))
(ex/raise :type :not-found
:code :snapshot-not-found
:hint "unable to find snapshot with the provided id"
:snapshot-id snapshot-id
:file-id file-id))))
(def ^:private sql:get-snapshot
(str sql:snapshots
" AND c.file_id = ?
AND c.id = ?
AND CASE WHEN c.created_by = 'user'
THEN (c.deleted_at IS NULL)
WHEN c.created_by = 'system'
THEN (c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz)
END"))
AND c.id = ? "
sql:snapshot-visible))
(defn get-snapshot
"Get a fully decoded snapshot for read-only preview or restoration.
Returns the snapshot map with decoded :data field."
Returns the snapshot map with decoded :data field, or nil when the file has
no snapshot with that id. The internal storage pointer of the row is not
part of the result."
[cfg file-id snapshot-id]
(let [now (ct/now)]
(->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now]
{::db/remove-deleted false})
(decode-snapshot)
(fdata/resolve-file-data cfg)
(fdata/decode-file-data cfg))))
(some->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now]
{::db/remove-deleted false
::db/throw-if-not-exists false})
(decode-snapshot)
(fdata/resolve-file-data cfg)
(fdata/decode-file-data cfg)
(bfc/file->response))))
(def ^:private sql:get-visible-snapshots
(str "WITH "
@ -268,6 +290,7 @@
(-> snapshot
(update :features into-array)
(update :migrations into-array)
(assoc :kind fchanges/snapshot-kind)
(assoc :updated-at created-at)
(assoc :profile-id profile-id)
(assoc :session-id session-id)
@ -278,6 +301,7 @@
{:id snapshot-id
:file-id (:id file)
:type "snapshot"
:storage-object-id nil
:data (blob/encode (:data file))
:created-at created-at
:deleted-at deleted-at})
@ -314,7 +338,10 @@
(-> (db/update! cfg :file-change
{:label label
:created-by "user"
:updated-at modified-at
;; the partition runbook uses updated_at as the
;; watermark of its delta passes: a rename has to move
;; it, unlike file_data.modified_at above
:updated-at (ct/now)
:deleted-at nil}
{:file-id file-id
:id id}
@ -360,7 +387,7 @@
(when-not snapshot
(ex/raise :type :not-found
:code :snapshot-not-found
:hint "unable to find snapshot with the provided label"
:hint "unable to find snapshot with the provided id"
:snapshot-id snapshot-id
:file-id file-id))
@ -440,7 +467,8 @@
:file-id file-id
:id id}})
(db/update! cfg :file-change
{:deleted-at deleted-at}
{:deleted-at deleted-at
:updated-at (ct/now)}
{:id id :file-id file-id}
{::db/return-keys false})
true)
@ -449,19 +477,25 @@
(str sql:snapshots " AND c.file_id = ?"))
(defn lock-by!
[conn id profile-id]
[conn file-id id profile-id]
(assert (uuid? file-id) "missing file-id")
(assert (uuid? id) "missing id")
(-> (db/update! conn :file-change
{:locked-by profile-id}
{:id id}
{:locked-by profile-id
:updated-at (ct/now)}
{:id id :file-id file-id}
{::db/return-keys false})
(db/get-update-count)
(pos?)))
(defn unlock!
[conn id]
[conn file-id id]
(assert (uuid? file-id) "missing file-id")
(assert (uuid? id) "missing id")
(-> (db/update! conn :file-change
{:locked-by nil}
{:id id}
{:locked-by nil
:updated-at (ct/now)}
{:id id :file-id file-id}
{::db/return-keys false})
(db/get-update-count)
(pos?)))

View File

@ -123,7 +123,8 @@
(defn- get-resolved-file
[cfg file-id]
(bfc/get-file cfg file-id :migrate? false :decode? false))
(-> (bfc/get-file cfg file-id :migrate? false :decode? false)
(bfc/file->response)))
(defn prepare-download
[file filename]
@ -372,7 +373,6 @@
:profile-id profile-id
:project-id project-id)
cfg (assoc cfg
::bfc/overwrite false
::bfc/profile-id profile-id
::bfc/project-id project-id
::bfc/team-id (:id team)

View File

@ -222,7 +222,8 @@
(defn invalidate-all
"Delete all sessions for a given profile. Used when a profile is deleted
to ensure immediate access revocation across all devices."
or a password is recovered, to ensure immediate access revocation across
all devices."
[cfg profile-id]
(let [sql "delete from http_session_v2 where profile_id = ?"]
(-> (db/exec-one! cfg [sql profile-id])

View File

@ -15,13 +15,16 @@
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as sql]
[app.http.session :as session]
[app.metrics :as mtx]
[app.msgbus :as mbus]
[app.nitrate :as nitrate]
[app.rpc.commands.files :as files]
[app.rpc.commands.teams :as teams]
[app.rpc.notifications :as notifications]
[app.util.websocket :as ws]
[integrant.core :as ig]
[promesa.exec.csp :as sp]
@ -37,31 +40,79 @@
;; WEBSOCKET HOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def state (atom {}))
(def state
"Registry of the websocket connections held by this backend instance.
Holds `:connections` (connection id -> connection data) and
`:by-profile` (profile id -> set of connection ids). Both live in the
same atom so a single `swap!` keeps them consistent, and so the
profile index never has to be rebuilt by scanning every connection.
NOTE: the registry is local to one backend instance, unlike the
message bus, which is shared. Code that must reach connections
regardless of which instance owns them has to go through the bus."
(atom {:connections {}
:by-profile {}}))
(defn register-connection
"Adds `wsp` to the registry under `id`, indexing it by its profile."
[id wsp]
(let [profile-id (::profile-id wsp)]
(swap! state
(fn [st]
(-> st
(update :connections assoc id wsp)
(cond-> profile-id
(update-in [:by-profile profile-id]
(fnil conj #{}) id)))))))
(defn unregister-connection
"Removes the connection `id` from the registry and from the profile
index."
[id]
(swap! state
(fn [{:keys [connections] :as st}]
(let [owner (::profile-id (get connections id))
indexed (disj (get-in st [:by-profile owner] #{}) id)]
(cond-> (assoc st :connections (dissoc connections id))
(and owner (seq indexed))
(update :by-profile assoc owner indexed)
(and owner (empty? indexed))
(update :by-profile dissoc owner))))))
(defn get-connection
"Returns the connection data registered under `id`, or nil."
[id]
(get-in @state [:connections id]))
(defn connections-for-profile
"Returns the set of connection ids currently held by `profile-id`."
[profile-id]
(get-in @state [:by-profile profile-id] #{}))
;; REPL HELPERS
(defn repl-get-connections-for-file
[file-id]
(->> (vals @state)
(->> (vals (:connections @state))
(filter #(= file-id (-> % ::ws/state deref ::file-subscription :file-id)))
(map ::ws/id)))
(defn repl-get-connections-for-team
[team-id]
(->> (vals @state)
(->> (vals (:connections @state))
(filter #(= team-id (-> % ::ws/state deref ::team-subscription :team-id)))
(map ::ws/id)))
(defn repl-close-connection
[id]
(when-let [{:keys [::ws/close-ch] :as wsp} (get @state id)]
(when-let [{:keys [::ws/close-ch]} (get-connection id)]
(sp/put! close-ch [8899 "closed from server"])
(sp/close! close-ch)))
(defn repl-get-connection-info
[id]
(when-let [wsp (get @state id)]
(when-let [wsp (get-connection id)]
(let [subs (some-> wsp ::ws/state deref)]
{:id id
:created-at (::created-at wsp)
@ -85,6 +136,160 @@
(pp/pprint)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; SUBSCRIPTIONS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; A subscription is authorized once, when it is opened, and that
;; single decision is then trusted for as long as the connection
;; lives. Two things bound that: a mutation that changes access
;; announces it (see the revocation section below), and the relay
;; re-checks on an interval.
(def default-revalidation-interval
"How long a subscription may rely on the authorization it was granted
with before its relay loop re-checks it.
This is the safety net for what the announcement cannot cover: an
access change made outside a Penpot command. Nitrate transferring the
ownership of an organization is an HTTP call to another service that
fires no RPC here, and rows deleted straight over SQL fire nothing
either. Both are rare, so the interval is generous; the cost is one
permission query per subscription per interval."
(ct/duration {:minutes 5}))
(defn revalidation-interval
[]
(cf/get :subscription-revalidation-interval default-revalidation-interval))
(defn- still-authorized?
"Runs the `check-access` permission check, reporting instead of
throwing when the lookup itself fails.
A transient database error must not close a subscription that is still
legitimate, and the next tick tries again. This is one of two places
that fail open, deliberately: the announcement path (`watch-revocations`
below) also logs and moves on when a re-check throws. An interrupt is
not a lookup failure and is rethrown."
[check-access]
(try
(boolean (check-access))
(catch InterruptedException cause
(throw cause))
(catch Throwable cause
(l/error :hint "cannot re-check a websocket subscription"
:cause cause)
true)))
(defn- file-readable?
"Non-throwing mirror of the `:subscribe-file` gates: the file must
still exist (a soft-deleted row grants nothing) and the profile must
read it. Same `sql:get-minimal-file` as the subscribe gate, with the
existence failure turned into a boolean — the fresh answer is the
decision."
[cfg profile-id file-id]
(and (some? (bfc/get-minimal-file cfg file-id
{::db/throw-if-not-exists false}))
(files/has-read-permissions? cfg profile-id file-id)))
(defn- team-readable?
"Same shape for teams: the team must still exist (a soft-deleted row
grants nothing) and the profile must read it. `:subscribe-team` has
no existence gate of its own; the re-check adds one, matching the
file side. Selects only `:id` and `:deleted-at` — `db/get*` filters
soft-deleted rows through the `deleted_at` column, so it must be
in the projection."
[cfg profile-id team-id]
(and (some? (db/get* cfg :team {:id team-id}
{::sql/columns [:id :deleted-at]}))
(teams/has-read-permissions? cfg profile-id team-id)))
(defn close-file-subscription
"Tears down the file subscription held by the connection `wsp`, if it
is subscribed to `file-id`.
Announces the departure so the remaining participants drop the
presence of this session, closes the relay channel (which in turn
stops the `:subscribe-file` go-loop, because `take!` on a closed
channel returns nil) and removes the subscription from the bus.
Does nothing when the connection is subscribed to a different file,
so it is safe to call for a subscription that is already gone."
[{:keys [::mbus/msgbus]} {:keys [::ws/state ::session-id ::profile-id]} file-id]
(let [subs (::file-subscription @state)]
(when (= (:file-id subs) file-id)
(mbus/pub! msgbus
:topic file-id
:message {:type :leave-file
:file-id file-id
:session-id session-id
:profile-id profile-id})
(let [ch (:channel subs)]
(sp/close! ch)
(mbus/purge! msgbus [ch])
(swap! state dissoc ::file-subscription)))))
(defn close-team-subscription
"Tears down the team subscription held by the connection `wsp`, if it
is subscribed to `team-id`.
Closing the channel is what stops the relay, so no further team or
organization traffic reaches this connection.
Does nothing when the connection is subscribed to a different team,
so it is safe to call for a subscription that is already gone."
[{:keys [::mbus/msgbus]} {:keys [::ws/state]} team-id]
(let [subs (::team-subscription @state)]
(when (= (:team-id subs) team-id)
(let [ch (:channel subs)]
(sp/close! ch)
(mbus/purge! msgbus [ch])
(swap! state dissoc ::team-subscription)))))
(defn- start-relay
"Forwards `channel` into the client output channel of `wsp`, and keeps
a re-check of `check-access` running on a fixed cadence. When access no
longer holds, `close-fn` tears the subscription down, which closes the
channel and ends the loop.
The loop waits on the subscription channel and a one-shot tick channel
together, with priority on the tick: when both hold something, the tick
wins, so a steady stream of messages can never push the re-check away.
A single tick channel covers a whole cadence and message turns reuse
it, so busy files don't churn timers; the cadence holds under load
because nothing moves the tick. Closing the channel ends the loop
either way, because a take on a closed channel is always ready
with nil.
`on-forward` runs for every forwarded message, which is how the file
relay announces presence."
[{:keys [::ws/output-ch] :as wsp} channel check-access close-fn on-forward]
(let [interval-ms (inst-ms (revalidation-interval))]
(sp/go-loop [tick-ch (sp/timeout-chan interval-ms)]
(let [[message port] (sp/alts! [tick-ch channel] :priority true)]
(cond
(identical? port tick-ch)
(do
(when-not (still-authorized? check-access)
(l/info :hint
"closing websocket subscription on re-check"
:profile-id (::profile-id wsp))
(close-fn))
(when-not (sp/closed? channel)
(recur (sp/timeout-chan interval-ms))))
(nil? message)
;; The channel was closed, which means the subscription is gone
;; and the relay ends.
nil
:else
(do
(sp/put! output-ch message)
(when on-forward (on-forward message))
(recur tick-ch)))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; WEBSOCKET HANDLER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@ -151,10 +356,10 @@
nil)))
(defmethod handle-message :subscribe-team
[cfg {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [team-id] :as params}]
[cfg {:keys [::ws/id ::ws/state ::session-id ::profile-id] :as wsp} {:keys [team-id] :as params}]
(l/trace :fn "handle-message" :event "subscribe-team" :team-id team-id :conn-id id)
(teams/check-read-permissions! cfg profile-id team-id)
(let [prev-subs (get @state ::team-subscription)
(let [prev-subs (::team-subscription @state)
organization-id (get-team-organization-id cfg team-id)
;; Resolved server-side so a client only hears its readable team's org
topics (cond-> [team-id]
@ -163,23 +368,29 @@
channel (sp/chan :buf (sp/dropping-buffer 64)
:xf (remove #(= (:session-id %) session-id)))]
(sp/pipe channel output-ch false)
(mbus/sub! (::mbus/msgbus cfg) :topics topics :chan channel)
;; The team topic also carries library change diffs, so a
;; subscription that outlives access to the team leaks content, not
;; just presence.
(let [subs {:team-id team-id
:organization-id organization-id
:channel channel
:topic team-id}]
(swap! state assoc ::team-subscription subs))
(start-relay wsp channel
#(team-readable? cfg profile-id team-id)
#(close-team-subscription cfg wsp team-id)
nil)
(mbus/sub! (::mbus/msgbus cfg) :topics topics :chan channel)
;; Close previous subscription if exists
(when-let [ch (:channel prev-subs)]
(sp/close! ch)
(mbus/purge! (::mbus/msgbus cfg) [ch]))))
(defmethod handle-message :subscribe-file
[cfg {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [file-id] :as params}]
[cfg {:keys [::ws/id ::ws/state ::session-id ::profile-id] :as wsp} {:keys [file-id] :as params}]
(l/trace :fn "handle-message" :event "subscribe-file" :file-id file-id :conn-id id)
(bfc/check-file-exists cfg file-id)
(files/check-read-permissions! cfg profile-id file-id)
@ -190,51 +401,40 @@
(let [subs {:file-id file-id :channel fch :topic file-id}]
(swap! state assoc ::file-subscription subs))
(start-relay wsp fch
#(file-readable? cfg profile-id file-id)
#(close-file-subscription cfg wsp file-id)
(fn [{:keys [type]}]
(when (or (= :join-file type)
(= :leave-file type)
(= :disconnect type))
(mbus/pub! (::mbus/msgbus cfg)
:topic file-id
:message {:type :presence
:file-id file-id
:session-id session-id
:profile-id profile-id}))))
;; Close previous subscription if exists
(when-let [ch (:channel psub)]
(sp/close! ch)
(mbus/purge! (::mbus/msgbus cfg) [ch]))
(sp/go-loop []
(when-let [{:keys [type] :as message} (sp/take! fch)]
(sp/put! output-ch message)
(when (or (= :join-file type)
(= :leave-file type)
(= :disconnect type))
(let [message {:type :presence
:file-id file-id
:session-id session-id
:profile-id profile-id}]
(mbus/pub! (::mbus/msgbus cfg)
:topic file-id
:message message)))
(recur)))
;; Subscribe to file topic
(mbus/sub! (::mbus/msgbus cfg) :topic file-id :chan fch)
;; Notifify the rest of participants of the new connection.
(let [message {:type :join-file
:file-id file-id
:session-id session-id
:profile-id profile-id}]
(mbus/pub! (::mbus/msgbus cfg) :topic file-id :message message))))
(mbus/pub! (::mbus/msgbus cfg)
:topic file-id
:message {:type :join-file
:file-id file-id
:session-id session-id
:profile-id profile-id})))
(defmethod handle-message :unsubscribe-file
[{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::session-id ::profile-id]} {:keys [file-id] :as params}]
[cfg {:keys [::ws/id] :as wsp} {:keys [file-id] :as params}]
(l/trace :fn "handle-message" :event "unsubscribe-file" :file-id file-id :conn-id id)
(let [subs (::file-subscription @state)
message {:type :leave-file
:file-id file-id
:session-id session-id
:profile-id profile-id}]
(when (= (:file-id subs) file-id)
(mbus/pub! msgbus :topic file-id :message message)
(let [ch (:channel subs)]
(sp/close! ch)
(mbus/purge! msgbus [ch])))))
(close-file-subscription cfg wsp file-id))
(defmethod handle-message :keepalive
[_ _ _]
@ -265,6 +465,110 @@
:message message
:conn-id id))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; SUBSCRIPTION REVOCATION
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; A subscription is authorized once, when it is opened, and that single
;; decision is then trusted for as long as the connection lives. The
;; mutation that changes access announces it on
;; `app.rpc.notifications/internal-revocation-topic`, and every backend
;; re-verifies the subscriptions it owns for that profile.
;;
;; The announcement travels over the message bus precisely because this
;; registry is local: the RPC that revokes access runs on whichever
;; instance received the request, which is usually not the one holding
;; the socket.
(defn revalidate-profile-subscriptions
"Re-checks every subscription held by the connections of `profile-id`
and closes the ones that are no longer authorized.
The event is the trigger, the fresh check is the decision.
`file-readable?` and `team-readable?` mirror the gates of
`:subscribe-file` and extend them to teams: resource exists
(a soft-deleted row grants nothing) and the profile reads it.
The organization-owner fallback keeps a downgrade to viewer and a
non-member owner from being mistaken for revocations — but not a
soft-deleted resource.
Returns the number of subscriptions it closed."
[cfg profile-id]
(let [closed (volatile! 0)]
(doseq [id (connections-for-profile profile-id)]
(when-let [wsp (get-connection id)]
(let [subs (some-> wsp ::ws/state deref)
fsub (get subs ::file-subscription)
tsub (get subs ::team-subscription)]
(when (and fsub
(not (file-readable? cfg profile-id
(:file-id fsub))))
(close-file-subscription cfg wsp (:file-id fsub))
(vswap! closed inc))
(when (and tsub
(not (team-readable? cfg profile-id
(:team-id tsub))))
(close-team-subscription cfg wsp (:team-id tsub))
(vswap! closed inc)))))
@closed))
(defn- watch-revocations
"Consumes revocation events and applies them to the connections this
instance owns. Returns the subscription channel, which `ig/halt-key!`
closes to stop the loop."
[{:keys [::mbus/msgbus] :as cfg}]
(let [ch (sp/chan :buf (sp/dropping-buffer 64))]
(mbus/sub! msgbus
:topic notifications/internal-revocation-topic
:chan ch)
(sp/go-loop []
(when-let [{:keys [type profile-id team-id]} (sp/take! ch)]
(try
(cond
(= :profile-permissions-changed type)
(when (pos? (revalidate-profile-subscriptions cfg profile-id))
(l/debug :hint "revoked websocket subscriptions"
:profile-id profile-id))
(= :team-permissions-changed type)
(let [revoked (->> (teams/get-team-members cfg team-id)
(map :id)
(map #(revalidate-profile-subscriptions cfg %))
(reduce + 0))]
(when (pos? revoked)
(l/debug :hint "revoked websocket subscriptions"
:team-id team-id
:count revoked))))
(catch Throwable cause
(l/error :hint "cannot revalidate websocket subscriptions"
:profile-id profile-id
:team-id team-id
:cause cause)))
(recur)))
ch))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; INTEGRANT
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def ^:private schema:revocation-watcher
[:map
::mbus/msgbus
::db/pool
[:app.nitrate/client {:optional true} [:maybe :map]]])
(defmethod ig/assert-key ::revocation-watcher
[_ params]
(assert (sm/valid? schema:revocation-watcher params)))
(defmethod ig/init-key ::revocation-watcher
[_ cfg]
(watch-revocations cfg))
(defmethod ig/halt-key! ::revocation-watcher
[_ ch]
(sp/close! ch))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HTTP HANDLER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@ -273,7 +577,7 @@
[{:keys [::mtx/metrics]} {:keys [::ws/id] :as wsp}]
(let [created-at (ct/now)]
(l/trace :fn "on-connect" :conn-id id)
(swap! state assoc id wsp)
(register-connection id wsp)
(mtx/run! metrics
:id :websocket-active-connections
:inc 1)
@ -281,7 +585,7 @@
(assoc wsp ::ws/on-disconnect
(fn []
(l/trace :fn "on-disconnect" :conn-id id)
(swap! state dissoc id)
(unregister-connection id)
(mtx/run! metrics :id :websocket-active-connections :dec 1)
(mtx/run! metrics
:id :websocket-session-timing

View File

@ -571,6 +571,11 @@
::session/manager (ig/ref ::session/manager)
:app.nitrate/client (ig/ref :app.nitrate/client)}
::http.ws/revocation-watcher
{::db/pool (ig/ref ::db/pool)
::mbus/msgbus (ig/ref ::mbus/msgbus)
:app.nitrate/client (ig/ref :app.nitrate/client)}
:app.http.assets/routes
{::http.assets/path (cf/get :assets-path)
::http.assets/cache-max-age (ct/duration {:hours 24})

View File

@ -241,8 +241,12 @@
foutput (fs/path (str finput ".ttf"))]
(try
(io/write* finput data)
;; CID-keyed fonts (common in CJK) load as several sub-fonts and
;; Generate only writes the active one unless they are merged.
(let [res (exec-font! system ["fontforge" "-lang=ff" "-c"
(str/fmt "Open('%s'); Generate('%s')"
(str/fmt (str "Open('%s'); "
"if ($cidfontname != \"\"); CIDFlatten(); endif; "
"Generate('%s')")
(str finput)
(str foutput))])]
(when (zero? (:exit res))

View File

@ -541,6 +541,12 @@
{:name "0158-del-forbidden-file-library-rels"
:fn (mg/resource "app/migrations/sql/0158-del-forbidden-file-library-rels.sql")}
{:name "0158-partition-file-change"
:fn (mg/resource "app/migrations/sql/0158-partition-file-change.sql")}
{:name "0159-add-file-data-storage-object-id"
:fn (mg/resource "app/migrations/sql/0159-add-file-data-storage-object-id.sql")}
{:name "0800-add-file-branch-table"
:fn (mg/resource "app/migrations/sql/0800-add-file-branch-table.sql")}

View File

@ -0,0 +1,258 @@
-- Partition `file_change` by `file_id`.
--
-- The table grows with every file change and its TOAST holds the blobs of
-- the changes, so it is the biggest relation of the database. Partitioning it
-- by `file_id` lets the queries that filter by file prune to a single
-- partition, and lets the join with `file_data` (already partitioned with the
-- same modulus) be resolved partition by partition.
--
-- The primary key has to include the partition key, so it becomes
-- `(file_id, id)`. Every access by id alone would lose its index, so the
-- access paths of the application filter by `file_id` too.
--
-- The new table is created with the final schema: the columns that nobody
-- used anymore are gone (`data_ref_id` and `data_backend`, dead since the
-- file storage refactor), the `data` column is kept as-is (the `legacy-db`
-- file data backend reads old payloads from it: migration
-- 0157-migrate-legacy-file-data moved the existing blobs), and the new
-- `kind` column tells the snapshots and the revisions apart.
--
-- The `kind` column is a reserve: the queries still tell a snapshot from a
-- revision by its label (`label IS NOT NULL`), and `kind` takes over when
-- they are rewritten. That is why no constraint ties the two together: the
-- day `kind` is the source of truth, the constraint would have to go.
--
-- It stays nullable on purpose: the previous version inserts without it,
-- and a rollback to that version has to keep writing. The `NOT NULL` comes
-- in a later migration, once the rollback window is closed.
--
-- The old table is kept aside as `file_change_legacy`: a frozen copy of the
-- previous layout. It is dropped by hand, days later (or right here, when it
-- is empty: on a new instance there is nothing to keep).
--
-- This migration is not meant to run on a big instance: the copy of the
-- rows of the table happens inside the transaction of the migration, and
-- the pool sets a 5 minute statement timeout. On those instances mark the
-- migration as applied and partition the table by hand, with the procedure
-- of `operations/2.20.0-file-change-partition.md`.
--
-- It assumes the service is stopped (a stop-start upgrade): it renames the
-- table, and a process of the previous version would keep writing to
-- `file_change_legacy` through its server-side prepared statements, which
-- hold the OID of the table and survive the rename. A rolling deployment is
-- the scenario of the runbook, not of this path.
--
-- The GRANTs of the table are not carried over: this creates a new table,
-- so an instance with custom grants on `file_change` has to re-apply them
-- after the upgrade.
-- 0. The guard for the big instances, before anything is touched: a copy
-- that does not fit keeps the backend down in a restart loop, and the
-- operator has to learn it from a clear message instead of from a
-- `statement timeout`. Both numbers are catalog reads, not table scans.
--
-- The row count comes from `pg_class.reltuples`, which is -1 on a
-- relation that was never analyzed (a restore without `--analyze`, for
-- instance). Analyze first so the number is real; on a table that fails
-- the guard this costs seconds per boot attempt, not a table scan.
--
-- The copy, the index builds and the `ANALYZE` of an instance that
-- passes the guard can take longer than the 5 minutes of the pool. The
-- timeout is bounded, so a wrong estimate still fails instead of hanging
-- the boot, and it covers the `ANALYZE`: on a cold, large table it can
-- outlast the pool too, and the guard has to be the one that speaks.
SET LOCAL statement_timeout = '30min';
ANALYZE file_change;
DO $$
BEGIN
IF pg_total_relation_size('file_change'::regclass) > 10::bigint * 1024 * 1024 * 1024
OR (SELECT reltuples FROM pg_class WHERE oid = 'file_change'::regclass) > 2000000
THEN
RAISE EXCEPTION
'file_change is too large for the automatic partition migration: mark 0158-partition-file-change as applied and follow operations/2.20.0-file-change-partition.md';
END IF;
END $$;
-- 1. The legacy blobs of the snapshots were already moved to `file_data`
-- by migration 0157-migrate-legacy-file-data (on the big instances, by
-- hand, with its runbook), so there is nothing to copy here. The `data`
-- column travels with the row in step 4 below: the `legacy-db` backend
-- still reads old payloads from it.
-- 2. The current table becomes the legacy one. The names of its indexes and
-- constraints live in the schema, not in the table, so they are renamed
-- with a `_legacy` suffix to free the canonical names.
ALTER TABLE file_change RENAME TO file_change_legacy;
ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_pkey
TO file_change_pkey_legacy;
ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_file_id_fkey
TO file_change_file_id_fkey_legacy;
ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_locked_by_fkey
TO file_change_locked_by_fkey_legacy;
ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_profile_id_fkey
TO file_change_profile_id_fkey_legacy;
ALTER INDEX file_change__data_ref_id__idx
RENAME TO file_change__data_ref_id__idx_legacy;
ALTER INDEX file_change__deleted_at__idx
RENAME TO file_change__deleted_at__idx_legacy;
ALTER INDEX file_change__file_id__revn__idx
RENAME TO file_change__file_id__revn__idx_legacy;
ALTER INDEX file_change__locked_by__idx
RENAME TO file_change__locked_by__idx_legacy;
ALTER INDEX file_change__profile_id__idx
RENAME TO file_change__profile_id__idx_legacy;
ALTER INDEX file_change__system_snapshots__idx
RENAME TO file_change__system_snapshots__idx_legacy;
-- This one comes from another branch, so an instance may not have it.
ALTER INDEX IF EXISTS file_change__user_snapshots__idx
RENAME TO file_change__user_snapshots__idx_legacy;
-- 3. The new table, partitioned with the same bounds as `file_data`, so both
-- relations can be joined partition by partition.
CREATE TABLE file_change (
file_id uuid NOT NULL,
id uuid NOT NULL DEFAULT gen_random_uuid(),
created_at timestamptz NOT NULL DEFAULT now(),
session_id uuid,
revn bigint NOT NULL DEFAULT 0,
data bytea,
changes bytea,
profile_id uuid,
features text[],
label text,
version integer,
created_by text NOT NULL DEFAULT 'system',
updated_at timestamptz DEFAULT now(),
deleted_at timestamptz,
migrations text[],
locked_by uuid,
-- Nullable on purpose: the previous version inserts without `kind`, and
-- a rollback to it has to keep writing. The `NOT NULL` comes in a later
-- migration, once the rollback window is closed.
kind smallint,
CONSTRAINT file_change_kind_check CHECK (kind IN (1, 2))
) PARTITION BY HASH (file_id);
-- the columns keep the storage of PostgreSQL by default (migration
-- 0156-reset-pg-storage-defaults dropped the explicit EXTERNAL overrides)
COMMENT ON COLUMN file_change.kind IS
'The kind of the row: 1 is a snapshot (it has a label and its data lives in file_data) and 2 is a revision (it has no label and it holds the changes).';
COMMENT ON COLUMN file_change.locked_by IS
'Profile ID of user who has locked this version. Only the creator can lock/unlock their own versions. Locked versions cannot be deleted by others.';
CREATE TABLE file_change_00 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 0);
CREATE TABLE file_change_01 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 1);
CREATE TABLE file_change_02 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 2);
CREATE TABLE file_change_03 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 3);
CREATE TABLE file_change_04 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 4);
CREATE TABLE file_change_05 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 5);
CREATE TABLE file_change_06 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 6);
CREATE TABLE file_change_07 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 7);
CREATE TABLE file_change_08 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 8);
CREATE TABLE file_change_09 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 9);
CREATE TABLE file_change_10 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 10);
CREATE TABLE file_change_11 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 11);
CREATE TABLE file_change_12 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 12);
CREATE TABLE file_change_13 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 13);
CREATE TABLE file_change_14 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 14);
CREATE TABLE file_change_15 PARTITION OF file_change
FOR VALUES WITH (MODULUS 16, REMAINDER 15);
-- 4. The rows move to the partitioned table. The column list is explicit
-- because the legacy table has columns that the new one does not have
-- (`data_ref_id` and `data_backend`).
-- The kind comes from the label: a row with a label is a snapshot, a row
-- without one is a revision.
INSERT INTO file_change (id, file_id, created_at, session_id, revn, data, changes,
profile_id, features, label, version, created_by,
updated_at, deleted_at, migrations, locked_by, kind)
SELECT id, file_id, created_at, session_id, revn, data, changes,
profile_id, features, label, version, created_by,
updated_at, deleted_at, migrations, locked_by,
CASE WHEN label IS NOT NULL THEN 1 ELSE 2 END
FROM file_change_legacy;
-- 5. The primary key includes the partition key now.
ALTER TABLE file_change ADD PRIMARY KEY (file_id, id);
-- 6. The foreign keys.
ALTER TABLE file_change ADD FOREIGN KEY (file_id)
REFERENCES file(id) DEFERRABLE;
ALTER TABLE file_change ADD FOREIGN KEY (locked_by)
REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;
ALTER TABLE file_change ADD FOREIGN KEY (profile_id)
REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;
-- 7. The indexes, with the canonical names. The one of the deleted rows now
-- carries the `file_id`, so the garbage collector can read the rows it has
-- to delete without touching the table.
--
-- There is no index for the system snapshots: the old one filtered them by
-- `data IS NOT NULL`, and with the payload living in `file_data` the
-- column no longer tells snapshots apart. The new one could only filter
-- them by `deleted_at IS NULL`, but the system snapshots always carry a
-- deletion date in the future, so it would never match a row (and
-- the query that would use it filters that date with an "or", which no
-- partial index can serve).
CREATE INDEX file_change__deleted_at__idx
ON file_change (deleted_at, file_id, id) WHERE deleted_at IS NOT NULL;
CREATE INDEX file_change__file_id__revn__idx
ON file_change (file_id, revn);
CREATE INDEX file_change__locked_by__idx
ON file_change (locked_by) WHERE locked_by IS NOT NULL;
CREATE INDEX file_change__profile_id__idx
ON file_change (profile_id) WHERE profile_id IS NOT NULL;
CREATE INDEX file_change__user_snapshots__idx
ON file_change (file_id)
WHERE label IS NOT NULL
AND created_by = 'user'
AND deleted_at IS NULL;
-- 8. Statistics for the new partitions. The `ANALYZE` of step 0 ran against
-- the old table, before the copy: without this one the planner starts
-- with empty statistics on the partitions and picks plans by guesswork
-- until autovacuum gets to them. The backend starts serving right after
-- this migration, so it is the worst moment to leave them unanalyzed.
ANALYZE file_change;
-- 9. The legacy table is kept as a frozen copy of the previous layout. It is
-- dropped by hand, days after the cutover:
-- DROP TABLE file_change_legacy;
--
-- Its foreign key on `file_id` goes away with the copy: it is `NO
-- ACTION`, so while the legacy table has rows, the objects GC cannot
-- delete a file (it deletes the rows of the file first, and the legacy
-- ones are not its business). The legacy table is frozen: nothing writes
-- to it again.
ALTER TABLE file_change_legacy
DROP CONSTRAINT file_change_file_id_fkey_legacy;
-- On a new instance there is nothing to go back to, so the empty table
-- is dropped here.
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM file_change_legacy) THEN
DROP TABLE file_change_legacy;
END IF;
END $$;

View File

@ -0,0 +1,65 @@
-- Turn the file data storage pointer into a column with a foreign key.
--
-- Until now the pointer lived in `file_data.metadata->>'storage-ref-id'`.
-- The storage GC parsed the JSON, and nothing stopped it from deleting the
-- blob of a live row. With the column and the foreign key the database
-- refuses that delete.
--
-- With the storage backend the release is a coordinated restart: the column is
-- the only source of truth, and the previous code cannot read the rows this
-- release writes. With the default `db` backend the new code writes the same
-- `file_data.data` column the previous code reads, so a rolling deploy is
-- safe.
--
-- This migration runs on a fresh database and on small instances. A large
-- instance builds the column and the index out of band, runs this same
-- backfill and foreign key with the instances stopped, and marks this step as
-- applied; the end state is identical. See
-- `operations/2.20.0-file-data-storage-object-id.md`.
--
-- `DEFERRABLE` follows the schema convention (all foreign keys of the project
-- are deferrable, so `th/database-reset` can delete table by table with
-- `SET CONSTRAINTS ALL DEFERRED`). It is `INITIALLY IMMEDIATE`, not
-- `DEFERRED`: the storage GC catches the violation inside the chunk
-- transaction and retries the chunk, and a deferred check would fire at
-- commit, outside that transaction, so the GC would never move past it.
ALTER TABLE file_data
ADD COLUMN IF NOT EXISTS storage_object_id uuid;
-- The CTE is MATERIALIZED so the uuid cast only sees rows that passed the
-- regex. A plain subquery can be inlined, and a corrupt value would reach the
-- cast and abort the migration.
WITH candidates AS MATERIALIZED (
SELECT file_id,
id,
(metadata->>'storage-ref-id')::uuid AS soid
FROM file_data
WHERE backend = 'storage'
AND storage_object_id IS NULL
AND (metadata->>'storage-ref-id') ~*
'^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$'
)
UPDATE file_data AS fd
SET storage_object_id = c.soid,
metadata = fd.metadata - 'storage-ref-id'
FROM candidates AS c
WHERE fd.file_id = c.file_id
AND fd.id = c.id
AND EXISTS (SELECT 1 FROM storage_object AS so WHERE so.id = c.soid);
-- The deletion check of the foreign key looks a row up by this column for
-- every deleted object; without the index it scans all partitions.
CREATE INDEX IF NOT EXISTS file_data__storage_object_id__idx
ON file_data (storage_object_id);
-- Rebuild the constraint so the definition is always the one above. This is
-- also how a constraint created by hand with the wrong shape (CASCADE, not
-- deferrable, or `INITIALLY DEFERRED`) gets repaired.
ALTER TABLE file_data
DROP CONSTRAINT IF EXISTS file_data_storage_object_id_fkey;
ALTER TABLE file_data
ADD CONSTRAINT file_data_storage_object_id_fkey
FOREIGN KEY (storage_object_id) REFERENCES storage_object(id)
ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE;

View File

@ -398,6 +398,20 @@
(generate-nitrate-uri "api/subscriptions/" profile-id)
schema:subscription params))
(def ^:private schema:profile-subscription
[:map {:title "ProfileSubscription"}
[:profile-id ::sm/uuid]
[:type ::sm/text]
[:status ::sm/text]])
(defn- get-subscriptions-api
[cfg {:keys [profile-ids] :as params}]
(let [params (assoc params :request-params {:profile-ids profile-ids})]
(request-to-nitrate cfg :post
(generate-nitrate-uri "api/subscriptions/bulk")
[:vector schema:profile-subscription]
params)))
(def ^:private schema:subscription-warning
[:maybe
[:map {:title "SubscriptionWarning"}
@ -533,6 +547,7 @@
:delete-team (partial delete-team-api cfg)
:remove-team-from-organization (partial remove-team-from-organization-api cfg)
:get-subscription (partial get-subscription-api cfg)
:get-subscriptions (partial get-subscriptions-api cfg)
:get-subscription-warning (partial get-subscription-warning-api cfg)
:connectivity (partial get-connectivity-api cfg)
:get-identity (partial get-identity-api cfg)
@ -699,6 +714,47 @@
team))
teams))))
(def ^:private valid-licence-statuses
"The statuses `app.main.data.nitrate/is-valid-license?` accepts on the
frontend."
#{"active" "past_due" "trialing"})
(defn- get-valid-licence-statuses
"Returns a map of profile id to the status of its valid Nitrate licence.
Returns an empty map when Nitrate does not answer: the licences of team
owners only decide which plan a team shows."
[cfg profile-ids params]
(try
(let [params (assoc (or params {}) :profile-ids (vec profile-ids))
subscriptions (call cfg :get-subscriptions params)]
(into {}
(comp (filter #(contains? valid-licence-statuses (:status %)))
(map (juxt :profile-id :status)))
subscriptions))
(catch Throwable cause
(l/wrn :hint "failed to get team owner licences"
:cause cause)
{})))
(defn add-owner-licence-to-teams
"Sets type \"nitrate\" in the `:subscription` of each team outside an
organization whose owner has a valid Nitrate licence, so every member
sees the plan of the owner, not only the owner. The database only knows
the Stripe subscription of the owner, which teams keep when Nitrate does
not answer. Teams in an organization take their plan from it."
[cfg teams params]
(let [owner-id #(when (nil? (:organization %))
(get-in % [:subscription :owner-id]))
owner-ids (into #{} (keep owner-id) teams)]
(if (empty? owner-ids)
teams
(let [statuses (get-valid-licence-statuses cfg owner-ids params)]
(mapv (fn [team]
(if-let [status (get statuses (owner-id team))]
(update team :subscription assoc :type "nitrate" :status status)
team))
teams)))))
(defn set-team-organization
"Associates a team with an organization in Nitrate.
Requires organization-id and is-default in params.
@ -716,12 +772,19 @@
:organization-id (:organization-id params)}))
team))
(defn- organization-membership
"The membership of `profile-id` in `organization-id`, as Nitrate reports it.
The profile travels in the path of the Nitrate endpoint, so no acting
profile is sent."
[cfg profile-id organization-id]
(call cfg :get-organization-membership {:profile-id profile-id
:organization-id organization-id}))
(defn assert-membership
"Verifies that the user is a member of the organization.
Raises an exception if the organization doesn't exist or the user is not a member."
[cfg profile-id organization-id]
(let [membership (call cfg :get-organization-membership {:profile-id profile-id
:organization-id organization-id})]
(let [membership (organization-membership cfg profile-id organization-id)]
(when-not (:organization-id membership)
(ex/raise :type :validation
:code :organization-does-not-exist))
@ -729,3 +792,18 @@
(when-not (:is-member membership)
(ex/raise :type :validation
:code :user-doesnt-belong-organization))))
(defn organization-access?
"True when `profile-id` may act for `organization-id`: it is a member of
the organization or its owner.
The organization id reaches the backend from the client, so the flows that
take one have to check it: without this check any authenticated profile
could start a flow for an organization it does not belong to."
[cfg profile-id organization-id]
(let [membership (organization-membership cfg profile-id organization-id)]
(if (:is-member membership)
true
(= (:owner-id (call cfg :get-organization-summary {:organization-id organization-id
::rpc/profile-id profile-id}))
profile-id))))

View File

@ -46,6 +46,10 @@
(def schema:token
[::sm/word-string {:max 6000}])
(def ^:private penpot-backend
"Auth backend identifier for the password flow."
"penpot")
(defn- elapsed-verify-threshold?
[profile]
(let [elapsed (ct/diff (:modified-at profile) (ct/now))
@ -201,6 +205,11 @@
(let [profile (some->> (validate-token token)
(update-password conn))]
(when profile
;; Recovery must revoke every existing session: a previously
;; captured cookie otherwise keeps working after the password
;; change (GHSA-3j2p-qr45-q365). Runs inside the same transaction
;; as the password update (::db/transaction true).
(session/invalidate-all cfg (:id profile))
(login-lockout/clear-attempts! cfg (:id profile))
(eml/send-template cfg {::eml/reuse-conn true
::eml/template "password-changed"
@ -226,25 +235,72 @@
;; ---- COMMAND: Prepare Register
(defn- validate-registration-flags!
([cfg invitation]
(validate-registration-flags! cfg invitation false))
([cfg invitation lock-invitation?]
(when-not (contains? cf/flags :login-with-password)
(ex/raise :type :restriction
:code :registration-disabled
:hint "registration disabled"))
(defn- check-registration-flags
"The `:backend` is the one embedded in the verified
`:prepared-register` token: the password flow backend, one of the
`oidc/oidc-provider-types` backends for SSO flows, or any other
value, which is rejected loudly."
[backend invitation]
(cond
(= backend penpot-backend)
(do
(when-not (contains? cf/flags :login-with-password)
(ex/raise :type :restriction
:code :registration-disabled
:hint "registration disabled"))
(when-not (contains? cf/flags :registration)
(if (nil? invitation)
(ex/raise :type :restriction
:code :registration-disabled
:hint "registration disabled")
(let [opts (when lock-invitation? {::db/for-update true})]
(when (nil? (teams-invitations/active-invitation cfg invitation opts))
(ex/raise :type :validation
:code :invalid-token
:hint "no active invitation associated with the token")))))))
(when-not (or (contains? cf/flags :registration)
(some? invitation))
(ex/raise :type :restriction
:code :registration-disabled
:hint "registration disabled")))
(contains? oidc/oidc-provider-types backend)
;; SSO backends share the OIDC callback gate
;; (`oidc/sso-registration-allowed?`): registration is allowed with
;; `:registration` or `:oidc-registration`, without requiring
;; password login. An invitation token is a substitute for the
;; flags, as in the password flow.
(when-not (or (oidc/sso-registration-allowed?)
(some? invitation))
(ex/raise :type :restriction
:code :registration-disabled
:hint "registration disabled"))
:else
(ex/raise :type :validation
:code :invalid-token
:hint "unsupported registration backend")))
(defn- check-registration-invitation
"Verifies that the invitation is still active. No-op unless a live
invitation is required: a backend whose registration flags do not
allow public registration, with an invitation at hand."
[cfg backend invitation lock-invitation?]
(let [flag-allows? (if (= backend penpot-backend)
(contains? cf/flags :registration)
(oidc/sso-registration-allowed?))]
(when (and (some? invitation)
(not flag-allows?))
(let [opts (when lock-invitation? {::db/for-update true})]
(when (nil? (teams-invitations/active-invitation cfg invitation opts))
(ex/raise :type :validation
:code :invalid-token
:hint "no active invitation associated with the token"))))))
(defn- check-registration
"Shared registration gate (register-profile and prepare-register):
validates the flags, the invitation email and the team invitation
row in order."
[cfg backend invitation email lock-invitation?]
(check-registration-flags backend invitation)
(when (and (some? invitation)
(not= (profile/clean-email (:member-email invitation))
(profile/clean-email email)))
(ex/raise :type :restriction
:code :email-does-not-match-invitation
:hint "email should match the invitation"))
(check-registration-invitation cfg backend invitation lock-invitation?))
(defn- validate-register-attempt!
[cfg params]
@ -254,23 +310,7 @@
{:token (:invitation-token params)
:iss :team-invitation}))]
(when (or (not (contains? cf/flags :login-with-password))
(and (not (contains? cf/flags :registration))
(not invitation?)))
(ex/raise :type :restriction
:code :registration-disabled
:hint "registration disabled"))
(when (and invitation?
(not= (profile/clean-email (:email params))
(profile/clean-email (:member-email invitation))))
(ex/raise :type :restriction
:code :email-does-not-match-invitation
:hint "email should match the invitation"))
(when (and invitation?
(not (contains? cf/flags :registration)))
(validate-registration-flags! cfg invitation)))
(check-registration cfg penpot-backend invitation (:email params) false))
(when (and (email.blacklist/enabled? cfg)
(email.blacklist/contains? cfg (:email params)))
@ -342,7 +382,7 @@
:fullname fullname
:password (:password params)
:invitation-token (:invitation-token params)
:backend "penpot"
:backend penpot-backend
:iss :prepared-register
:exp (ct/in-future {:days 7})
:props props}
@ -419,7 +459,7 @@
locale (when (and (string? locale) (not (str/blank? locale)))
locale)
backend (:backend params "penpot")
backend (:backend params penpot-backend)
is-demo (:is-demo params false)
is-muted (:is-muted params false)
is-active (:is-active params false)
@ -505,12 +545,25 @@
:token vtoken
:extra-data ptoken}))))
(def ^:private schema:prepared-register-claims
[:map {:title "PreparedRegisterClaims"}
[:backend ::sm/text]
[:email ::sm/email]])
(def ^:private valid-prepared-register-claims?
(sm/lazy-validator schema:prepared-register-claims))
(defn register-profile
[{:keys [::db/conn] :as cfg} {:keys [token] :as params}]
(let [claims (tokens/verify cfg {:token token :iss :prepared-register})
_ (when-not (valid-prepared-register-claims? claims)
(ex/raise :type :validation
:code :invalid-token
:hint "prepared-register token contains unexpected data"))
backend (:backend claims)
invitation (when-let [token (:invitation-token claims)]
(tokens/verify cfg {:token token :iss :team-invitation}))
_ (validate-registration-flags! cfg invitation true)
_ (check-registration cfg backend invitation (:email claims) true)
params (cond-> claims
(:accept-newsletter-updates params)
(update :props assoc :newsletter-updates true))

View File

@ -195,7 +195,10 @@
(-> file
(assoc :page-name (dm/get-in data [:pages-index page-id :name]))
(assoc :page-id page-id)
(dissoc :data)))))
(dissoc :data)
;; The storage pointer is internal bookkeeping: it never leaves the
;; backend in a response.
(bfc/file->response)))))
;; FIXME: rename
(defn- get-comment-thread

View File

@ -42,8 +42,8 @@
[app.rpc.cond :as-alias cond]
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.rpc.notifications :as ntf]
[app.rpc.permissions :as perms]
[app.util.blob :as blob]
[app.util.events :as events]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
@ -222,7 +222,11 @@
(if (and (contains? (:features file) "fdata/objects-map")
(not (contains? (:features params) "fdata/objects-map")))
(feat.fdata/realize-objects cfg file)
file)))))
file)
;; The storage pointer is internal bookkeeping: it never leaves the
;; backend in a response.
(bfc/file->response file)))))
;; --- COMMAND QUERY: get-file-fragment (by id)
@ -231,7 +235,7 @@
[:id ::sm/uuid]
[:file-id ::sm/uuid]
[:created-at ::ct/inst]
[:content ::sm/any]])
[:data ::sm/any]])
(def schema:get-file-fragment
[:map {:title "get-file-fragment"}
@ -241,8 +245,17 @@
(defn- get-file-fragment
[cfg file-id fragment-id]
(some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"})
(update :data blob/decode)))
;; Resolve through the file data backend: a fragment can live on cold
;; storage, and reading its row directly would hand `nil` to `blob/decode`.
;; Only the public fields leave the backend: the row also carries the
;; internal storage pointer and the backend metadata.
(when-let [row (db/get cfg :file-data {:file-id file-id
:id fragment-id
:type "fragment"})]
(let [fragment (->> row
(feat.fdata/resolve-file-data cfg)
(feat.fdata/decode-file-data cfg))]
(select-keys fragment (sm/keys schema:file-fragment)))))
(defn- check-fragment-scope!
"Checks that the fragment is reachable from the pages authorized by
@ -561,17 +574,23 @@
(let [file (bfc/get-file cfg id :load-data? false)
summary (t/decode-str result)]
(-> (assoc file :library-summary summary)
(dissoc :data)))))
(dissoc :data :migrations)
(bfc/file->response)))))
calculate-from-db
(fn []
(let [file (bfc/get-file cfg id)
result (feat.fdata/with-pointers cfg id
(calculate-library-summary file))]
;; The storage pointer is internal bookkeeping: this summary is an
;; RPC response and must not expose it. The applied migrations go
;; for the same reason: this endpoint is not where a client
;; completes a read, and `schema:file` declares the key optional.
(-> file
(assoc :library-summary result)
(dissoc :legacy-data)
(dissoc :data))))
(dissoc :data :migrations)
(bfc/file->response))))
persist-to-cache
(fn [{:keys [::rds/conn]} data cache-key]
@ -1039,13 +1058,10 @@
:library-id (str (:id ldata))
:file-id (str file-id))
(bfc/update-file! cfg {:id file-id
:features (:features file)
:migrations (:migrations file)
:revn (inc (:revn file))
:data (:data file)
:modified-at (ct/now)
:has-media-trimmed false}))))
(bfc/update-file! cfg file
:revn (inc (:revn file))
:modified-at (ct/now)
:has-media-trimmed false))))
(defn- get-referenced-files
[cfg library-id]
@ -1200,6 +1216,12 @@
:file-id id
:profile-id profile-id})
;; Deleting the file ends read access for whoever had it open, and
;; which profiles those are is only known to the connections
;; themselves, so the whole team is announced and the watcher
;; re-checks each of them.
(ntf/notify-team-permissions-changed cfg (:id team))
(rph/with-meta (rph/wrap)
{::audit/props {:team-id (:id team)
:project-id (:project-id file)
@ -1272,11 +1294,10 @@
(check-edition-permissions! conn profile-id library-id)
(check-library-team-ownership! conn file-id library-id)
(let [transitive-deps (bfc/get-libraries cfg [library-id])]
(when (contains? transitive-deps file-id)
(ex/raise :type :validation
:code :circular-library-reference
:hint "linking this library would create a circular dependency")))
(when (= :circular (bfc/library-link-status cfg file-id library-id))
(ex/raise :type :validation
:code :circular-library-reference
:hint "linking this library would create a circular dependency"))
(link-file-to-library conn params)
(bfc/get-libraries cfg [library-id]))
@ -1434,7 +1455,9 @@
;; for ever. Rows that still carry a legacy payload without a label are
;; garbage that no read path reaches; 0157-migrate-legacy-file-data deletes
;; them, so they must keep their date and go with the collector.
"UPDATE file_change SET deleted_at = null
;; Every write moves `updated_at`: the delta passes copy the rows whose
;; `updated_at` changed, so a restore that left it stale could be missed.
"UPDATE file_change SET deleted_at = null, updated_at = now()
WHERE file_id = ANY(?::uuid[])
AND label IS NOT NULL")
@ -1452,8 +1475,8 @@
(def ^:private sql:restore-storage-objects
;; Collect referenced storage ids first (UUID set), then join by PK.
;; Casting `storage_object.id` to text (or OR-ing that with `id IN`) forces
;; a scan of deleted rows; cast the JSON side instead.
;; `file_data` points at its object with the `storage_object_id` column, so
;; the reference is already a uuid and no JSON cast is needed.
"WITH ids AS (SELECT unnest(?::uuid[]) AS file_id),
refs AS (
SELECT fmo.media_id AS id
@ -1480,11 +1503,11 @@
WHERE fot.deleted_at IS NULL
AND fot.media_id IS NOT NULL
UNION
SELECT (fd.metadata->>'storage-ref-id')::uuid
SELECT fd.storage_object_id
FROM file_data AS fd
JOIN ids ON (ids.file_id = fd.file_id)
WHERE fd.deleted_at IS NULL
AND (fd.metadata->>'storage-ref-id') IS NOT NULL
AND fd.storage_object_id IS NOT NULL
)
UPDATE storage_object AS so
SET deleted_at = null

View File

@ -295,12 +295,12 @@
fragment pointers, and `file` is the same document realized and
checked against the size limits.
The row is read FOR SHARE. File-gc takes it FOR UPDATE and marks every
fragment that neither main nor a committed snapshot names, so a
snapshot that copies main's pointers must commit before the next pass
can read the row."
The row is read FOR UPDATE, since a data read cannot take FOR SHARE.
File-gc takes it FOR UPDATE too and marks every fragment that neither
main nor a committed snapshot names, so a snapshot that copies main's
pointers must commit before the next pass can read the row."
[cfg main-id operation]
(let [stored (bfc/get-file cfg main-id :lock-for-share? true)
(let [stored (bfc/get-file cfg main-id :lock-for-update? true)
file (-> (fdata/realize cfg stored)
(check-file-size-limits! operation))]
[stored file]))
@ -590,10 +590,10 @@
;; GC while the branch is open (it is released when the
;; branch is deleted or its base moves). The snapshot
;; stores main's fragment pointers, not its pages, so
;; main's row is read FOR SHARE: file-gc takes it FOR
;; UPDATE, and a pass that ran between this read and
;; the commit could mark a fragment the base names.
(let [file (bfc/get-file cfg file-id :lock-for-share? true)
;; main's row is read FOR UPDATE: file-gc takes it FOR
;; UPDATE as well, and a pass that ran between this read
;; and the commit could mark a fragment the base names.
(let [file (bfc/get-file cfg file-id :lock-for-update? true)
_ (fdata/with-pointers cfg file-id
(check-file-size-limits! file :create-branch))
base (fsnap/create! cfg file

View File

@ -68,7 +68,10 @@
{:modified-at (ct/now)}
{:id project-id})
(bfc/get-file cfg (:id file))))))
;; The storage pointer is internal bookkeeping: it never leaves the
;; backend in a response.
(-> (bfc/get-file cfg (:id file))
(bfc/file->response))))))
(def ^:private schema:create-file
[:map {:title "create-file" :closed true}

View File

@ -52,6 +52,7 @@
"Retrieve a file bundle with data from a specific snapshot for
read-only preview. Does not modify any database state."
{::doc/added "2.16"
::doc/changes [["2.20" "An unknown snapshot is reported with the :snapshot-not-found error code"]]
::sm/params schema:get-file-snapshot
::sm/result files/schema:file-with-permissions
::db/transaction true}
@ -125,12 +126,15 @@
;; every unpurged snapshot as a liveness root, and readers
;; (get-file-snapshot, restore!, reduce-snapshots) resolve pointers
;; on demand.
;; NOTE: the file is read with FOR SHARE inside this transaction so
;; NOTE: the file is read with FOR UPDATE inside this transaction so
;; snapshot creation serializes with a concurrent :file-gc pass
;; (which takes FOR UPDATE + SKIP LOCKED on the same row). Without
;; the lock, a version pinned mid-pass could reference fragment rows
;; the pass is recreating.
(let [file (bfc/get-file cfg file-id :lock-for-share? true)
;; (which takes FOR UPDATE + SKIP LOCKED on the same row) and with a
;; concurrent save. Without the lock, a version pinned mid-pass could
;; reference fragment rows the pass is recreating. The lock is
;; exclusive, not shared, because the migration-on-read path may
;; re-read the same row with FOR UPDATE; holding a shared lock while
;; asking for the exclusive one deadlocks two concurrent readers.
(let [file (bfc/get-file cfg file-id :lock-for-update? true)
project (db/get-by-id cfg :project (:project-id file))]
(-> cfg
@ -166,12 +170,14 @@
delay (ldel/get-deletion-delay team)
;; Create backup snapshot of the current state (committed immediately
;; independent of the restore outcome). The read takes FOR SHARE so a
;; independent of the restore outcome). The read takes FOR UPDATE so a
;; concurrent :file-gc pass (which holds FOR UPDATE + SKIP LOCKED on the
;; file row) either finishes first -- and we snapshot the new fragments --
;; or skips the file and retries later. Without the lock, the backup
;; could reference fragments the pass is about to mark as deleted,
;; breaking the backup permanently.
;; breaking the backup permanently. The lock is exclusive because the
;; migration-on-read path may re-read the row with FOR UPDATE: a shared
;; lock that upgrades deadlocks two concurrent readers.
;;
;; The backup read also provides the revn the guard below compares
;; against. An edit committed before this read is captured in the
@ -180,7 +186,7 @@
backup-revn
(db/tx-run! cfg
(fn [cfg]
(let [file (bfc/get-file cfg file-id :lock-for-share? true)]
(let [file (bfc/get-file cfg file-id :lock-for-update? true)]
;; `fsnap/restore!` refuses a branch file too. The
;; command refuses before it takes the backup
;; snapshot, which is committed whatever the
@ -201,7 +207,7 @@
(ex/raise :type :conflict
:code :file-modified
:hint "the file was modified during the restore process, please retry")))
(let [snapshot (fsnap/get-minimal-snapshot cfg id)
(let [snapshot (fsnap/get-minimal-snapshot cfg file-id id)
vern (fsnap/restore! cfg file-id id)]
;; Peers name the restored version by its label when a
;; user named it, else by its date
@ -219,15 +225,17 @@
(def ^:private schema:update-file-snapshot
[:map {:title "update-file-snapshot"}
[:file-id ::sm/uuid]
[:id ::sm/uuid]
[:label ::sm/text]])
(sv/defmethod ::update-file-snapshot
{::doc/added "1.20"
::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]]
::sm/params schema:update-file-snapshot
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id id label]}]
(let [snapshot (fsnap/get-minimal-snapshot conn id)]
[{:keys [::db/conn]} {:keys [::rpc/profile-id file-id id label]}]
(let [snapshot (fsnap/get-minimal-snapshot conn file-id id)]
(files/check-edition-permissions! conn profile-id (:file-id snapshot))
(when (not= (:created-by snapshot) "user")
@ -241,14 +249,16 @@
(def ^:private schema:remove-file-snapshot
[:map {:title "remove-file-snapshot"}
[:file-id ::sm/uuid]
[:id ::sm/uuid]])
(sv/defmethod ::delete-file-snapshot
{::doc/added "1.20"
::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]]
::sm/params schema:remove-file-snapshot
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id]}]
(let [snapshot (fsnap/get-minimal-snapshot conn id)]
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id id]}]
(let [snapshot (fsnap/get-minimal-snapshot conn file-id id)]
(files/check-edition-permissions! conn profile-id (:file-id snapshot))
(when (not= (:created-by snapshot) "user")
@ -276,14 +286,16 @@
(def ^:private schema:lock-file-snapshot
[:map {:title "lock-file-snapshot"}
[:file-id ::sm/uuid]
[:id ::sm/uuid]])
(sv/defmethod ::lock-file-snapshot
{::doc/added "1.20"
::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]]
::sm/params schema:lock-file-snapshot
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
(let [snapshot (fsnap/get-minimal-snapshot conn id)]
[{:keys [::db/conn]} {:keys [::rpc/profile-id file-id id]}]
(let [snapshot (fsnap/get-minimal-snapshot conn file-id id)]
(files/check-edition-permissions! conn profile-id (:file-id snapshot))
(when (not= (:created-by snapshot) "user")
@ -311,18 +323,20 @@
:profile-id profile-id
:locked-by (:locked-by snapshot)))
(fsnap/lock-by! conn id profile-id)))
(fsnap/lock-by! conn (:file-id snapshot) id profile-id)))
(def ^:private schema:unlock-file-snapshot
[:map {:title "unlock-file-snapshot"}
[:file-id ::sm/uuid]
[:id ::sm/uuid]])
(sv/defmethod ::unlock-file-snapshot
{::doc/added "1.20"
::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]]
::sm/params schema:unlock-file-snapshot
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
(let [snapshot (fsnap/get-minimal-snapshot conn id)]
[{:keys [::db/conn]} {:keys [::rpc/profile-id file-id id]}]
(let [snapshot (fsnap/get-minimal-snapshot conn file-id id)]
(files/check-edition-permissions! conn profile-id (:file-id snapshot))
(when (not= (:created-by snapshot) "user")
@ -349,4 +363,4 @@
:snapshot-id id
:profile-id profile-id))
(fsnap/unlock! conn id)))
(fsnap/unlock! conn (:file-id snapshot) id)))

View File

@ -23,6 +23,7 @@
[app.config :as cf]
[app.db :as db]
[app.features.fdata :as fdata]
[app.features.file-changes :as fchanges]
[app.features.file-snapshots :as fsnap]
[app.features.logical-deletion :as ldel]
[app.http.errors :as errors]
@ -469,6 +470,7 @@
(when repair
(db/insert! conn :file-change
{:id (uuid/next)
:kind fchanges/revision-kind
:session-id repair-session-id
:profile-id profile-id
:created-at timestamp
@ -477,7 +479,6 @@
:file-id (:id file)
:revn (:revn repair)
:version (:version file)
:features (into-array (:features file))
:changes (blob/encode (:changes repair))}
{::db/return-keys false})
(l/wrn :hint "file auto-repaired"
@ -489,8 +490,13 @@
;; Insert change (xlog) with deleted_at in a future data for
;; make them automatically eleggible for GC once they expires
;;
;; The revision carries only the delta of the save: the features of the
;; file are not repeated here (nobody reads them from a revision, and
;; they are duplicated on every save).
(db/insert! conn :file-change
{:id (uuid/next)
:kind fchanges/revision-kind
:session-id session-id
:profile-id profile-id
:created-at change-created-at
@ -499,7 +505,6 @@
:file-id (:id file)
:revn (:revn file)
:version (:version file)
:features (into-array (:features file))
:changes (blob/encode changes)}
{::db/return-keys false})
@ -572,9 +577,15 @@
:branch-duration-ms duration))})))))
(defn get-file
"Get not-decoded file, only decodes the features set."
"Get not-decoded file, only decodes the features set.
The read takes the `file` row with FOR UPDATE: the save persists the
map it read, so it must hold the row from the read to the write. The
lock is exclusive, not shared, because the migration-on-read path may
re-read the same row with FOR UPDATE, and a shared lock that upgrades
deadlocks two concurrent readers."
[cfg id]
(bfc/get-file cfg id :decode? false :lock-for-share? true))
(bfc/get-file cfg id :decode? false :lock-for-update? true))
(defn persist-file!
"Function responsible of persisting already encoded file. Should be

View File

@ -112,6 +112,7 @@
[:font-family types.font/schema:font-family]
[:font-weight [::sm/one-of {:format "number"} valid-weight]]
[:font-style [::sm/one-of {:format "string"} valid-style]]
[:variant-name {:optional true} [:maybe [::sm/text {:max 250}]]]
[:uploads [:map-of ::sm/text ::sm/uuid]]])
(defn- prepare-font-data-from-uploads
@ -139,7 +140,8 @@
chunked-upload API)."
{::doc/added "1.18"
::doc/changes [["2.16" "Add :uploads param for chunked upload support"]
["2.18" "Remove :data param, use :uploads exclusively"]]
["2.18" "Remove :data param, use :uploads exclusively"]
["2.20" "Add optional :variant-name param"]]
::climit/id [[:process-font/by-profile ::rpc/profile-id]
[:process-font/global]]
::webhooks/event? true

View File

@ -25,6 +25,8 @@
[app.rpc.commands.projects :as proj]
[app.rpc.commands.teams :as teams]
[app.rpc.doc :as-alias doc]
[app.rpc.notifications :as ntf]
[app.rpc.permissions :as perms]
[app.setup :as-alias setup]
[app.setup.templates :as tmpl]
[app.storage.tmp :as tmp]
@ -88,7 +90,9 @@
fmeds)]
(db/insert! conn :file-media-object params ::db/return-keys false))
file)))
;; The storage pointer of the source file is internal bookkeeping: the
;; new file does not inherit it, and it never leaves the backend.
(bfc/file->response file))))
(def ^:private
schema:duplicate-file
@ -272,6 +276,17 @@
;; --- COMMAND: Move file
(defn- notify-team-access-changed
"Announces a possible read-access change for `team-ids`.
Moving content can take it out of the reach of a whole team, and the
only place that knows who was relying on it is the set of open
connections. One event per team: the watcher resolves the members
itself, so a large team cannot overflow the bounded publish buffers."
[cfg team-ids]
(doseq [team-id (into #{} (remove nil?) team-ids)]
(ntf/notify-team-permissions-changed cfg team-id)))
(def sql:get-files
"select id, features, project_id, is_branch from file where id = ANY(?)")
@ -356,6 +371,14 @@
(recur (rest project-ids)
(ct/plus modified-at 10))))
;; Moving files can take them out of the reach of the team they came
;; from. Only the open connections know who was relying on them, so
;; the source teams are announced wholesale and the watcher re-checks
;; each member against the subscriptions they actually hold.
(notify-team-access-changed
cfg
(keep (partial perms/get-team-id-for-project conn) source))
nil))
(def ^:private
@ -425,6 +448,10 @@
;; delete possible broken relations on moved files
(db/exec-one! conn [sql:delete-broken-relations pids])
;; The project, and everything under it, leaves the source team. The
;; destination team only gains access, so it is not announced.
(notify-team-access-changed cfg [(:team-id project)])
nil))
(def ^:private

View File

@ -704,7 +704,7 @@
[:map {:title "CheckNitrateSsoParams"}
[:team-id {:optional true} ::sm/uuid]
[:organization-id {:optional true} ::sm/uuid]
[:url ::sm/uri]]
[:query-string [:string {:max 2048}]]]
[::sm/contains-any #{:team-id :organization-id}]])
(sv/defmethod ::check-nitrate-sso
@ -714,40 +714,57 @@
session already holds a valid entry for the organization, and
{:authorized true :reason :no-team-access} when the gate was skipped because the
user cannot access the team; the reason lets the client tell a usable session
apart from a plain permission failure.
apart from a plain permission failure. When an organization id comes with the
request, the gate is also skipped with
{:authorized true :reason :no-organization-access} when the profile neither
belongs to that organization nor owns it.
Returns {:authorized false :redirect-uri <url>} when SSO is active;
the client must redirect there. The OIDC provider itself handles
re-authentication transparently if the user already has an active SSO session.
A nil :redirect-uri means SSO is required but the provider is not usable."
A nil :redirect-uri means SSO is required but the provider is not usable.
The :query-string is the route of the client as a query string
(`screen=…&params`, without the `?` that separates it from the path; the root
of the application may carry only its params, like `template=…`): the server
builds the destination URL from it, so it can never point outside the public
URI. A value without params of its own is not a route, is ignored with a
warning, and the user lands on the home page."
{::rpc/auth true
::doc/added "2.18"
::sm/params schema:check-nitrate-sso
::nitrate/sso false}
[cfg {:keys [::rpc/profile-id team-id organization-id url] :as params}]
[cfg {:keys [::rpc/profile-id team-id organization-id query-string] :as params}]
(if (contains? cf/flags :admin-console)
(if (and team-id
(not (teams/has-read-permissions? cfg profile-id team-id)))
;; Let the destination RPC enforce its own permissions. Starting SSO before
;; access is established sends unrelated users through the organization's IdP.
{:authorized true :reason :no-team-access}
(let [request (rph/get-request params)
{:keys [authorized sso]} (nitrate/sso-session-authorized? cfg organization-id team-id request)]
(if authorized
{:authorized true :reason :sso-satisfied}
(if (oidc/organization-sso-discovery-uri sso)
(try
(let [redirect-uri (oidc/build-organization-sso-auth-redirect-uri
cfg sso
:dest-url url
:organization-id organization-id)
organization-id (or organization-id (:organization-id sso))]
(oidc/submit-organization-sso-auth-started-event
cfg request profile-id organization-id)
{:authorized false :redirect-uri redirect-uri})
(catch Throwable cause
(oidc/submit-organization-sso-auth-failed-event
cfg request profile-id (or organization-id (:organization-id sso)) cause)
(throw cause)))
{:authorized false
:redirect-uri nil}))))
(if (and organization-id
(not (nitrate/organization-access? cfg profile-id organization-id)))
;; The organization id comes from the client: without this check any
;; authenticated profile could start a flow for an organization it does
;; not belong to.
{:authorized true :reason :no-organization-access}
(let [request (rph/get-request params)
{:keys [authorized sso]} (nitrate/sso-session-authorized? cfg organization-id team-id request)]
(if authorized
{:authorized true :reason :sso-satisfied}
(if (oidc/organization-sso-discovery-uri sso)
(try
(let [redirect-uri (oidc/build-organization-sso-auth-redirect-uri
cfg sso
:query-string query-string
:organization-id organization-id
:profile-id profile-id)
organization-id (or organization-id (:organization-id sso))]
(oidc/submit-organization-sso-auth-started-event
cfg request profile-id organization-id)
{:authorized false :redirect-uri redirect-uri})
(catch Throwable cause
(oidc/submit-organization-sso-auth-failed-event
cfg request profile-id (or organization-id (:organization-id sso)) cause)
(throw cause)))
{:authorized false
:redirect-uri nil})))))
{:authorized true :reason :sso-satisfied}))

View File

@ -70,6 +70,7 @@
[:onboarding-questions {:optional true} [:map-of :keyword :string]]
[:onboarding-questions-answered {:optional true} ::sm/boolean]
[:nitrate-onboarding-viewed {:optional true} ::sm/boolean]
[:promotional-banner-collapsed {:optional true} ::sm/boolean]
[:v2-info-shown {:optional true} ::sm/boolean]
[:release-notes-viewed {:optional true}
[::sm/text {:max 100}]]

View File

@ -22,6 +22,7 @@
[app.rpc.commands.teams :as teams]
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.rpc.notifications :as ntf]
[app.rpc.permissions :as perms]
[app.rpc.quotes :as quotes]
[app.util.services :as sv]))
@ -321,6 +322,12 @@
:profile-id profile-id
:project-id id)
project (delete-project cfg team id)]
;; Deleting the project ends read access for whoever had one of its
;; files open, so the whole team is announced and the watcher
;; re-checks each member against the subscriptions they hold.
(ntf/notify-team-permissions-changed cfg (:id team))
(rph/with-meta (rph/wrap)
{::audit/props {:team-id (:team-id project)
:name (:name project)

View File

@ -31,6 +31,7 @@
[app.rpc.commands.profile :as profile]
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.rpc.notifications :as ntf]
[app.rpc.permissions :as perms]
[app.rpc.quotes :as quotes]
[app.setup :as-alias setup]
@ -161,7 +162,8 @@
WHEN 'professional' THEN 'active'
ELSE COALESCE(p.props->'~:subscription'->>'~:status', 'incomplete')
END,
'~:seats', p.props->'~:subscription'->'~:quantity'
'~:seats', p.props->'~:subscription'->'~:quantity',
'~:owner-id', '~u' || p.id::text
) AS subscription
FROM team_profile_rel AS tp
JOIN team AS t ON (t.id = tp.team_id)
@ -211,8 +213,9 @@
(dm/with-open [conn (db/open pool)]
(let [teams (get-teams conn profile-id)]
(if (contains? cf/flags :admin-console)
(->> (nitrate/add-organization-info-to-teams cfg teams params)
(remove #(get-in % [:organization :expired-license])))
(as-> (nitrate/add-organization-info-to-teams cfg teams params) teams
(remove #(get-in % [:organization :expired-license]) teams)
(nitrate/add-owner-licence-to-teams cfg teams params))
teams))))
(def ^:private sql:get-owned-teams
@ -258,7 +261,8 @@
[cfg {:keys [::rpc/profile-id id file-id] :as params}]
(let [team (get-team cfg :profile-id profile-id :team-id id :file-id file-id)]
(if (contains? cf/flags :admin-console)
(nitrate/add-organization-info-to-team cfg team params)
(let [team (nitrate/add-organization-info-to-team cfg team params)]
(first (nitrate/add-owner-licence-to-teams cfg [team] params)))
team)))
(defn- get-organization-owner-viewer-team
@ -268,13 +272,14 @@
[cfg profile-id default-team-id params]
(when-let [team-id (perms/resolve-team-id cfg params)]
(when (nitrate/organization-owner-of-team? cfg profile-id team-id)
;; resolve-team-id already proved the team row live; the guard
;; that used to live here (when-not is-row-deleted?) was dead.
(when-let [team (db/get* cfg :team {:id team-id})]
(when-not (db/is-row-deleted? team)
(-> team
(decode-row)
(merge perms/viewer-role-flags)
(assoc :is-default (= team-id default-team-id))
(process-permissions)))))))
(-> team
(decode-row)
(merge perms/viewer-role-flags)
(assoc :is-default (= team-id default-team-id))
(process-permissions))))))
(defn get-team
[cfg & {:keys [profile-id team-id project-id file-id] :as params}]
@ -822,9 +827,10 @@
:team-id team-id}))
(defn leave-team
[{:keys [::db/conn ::mbus/msgbus]} {:keys [profile-id id reassign-to]}]
[{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [profile-id id reassign-to]}]
(let [perms (get-permissions conn profile-id id)
members (get-team-members conn id)]
members (get-team-members conn id)
team (db/get conn :team {:id id})]
(cond
;; we can only proceed if there are more members in the team
@ -873,6 +879,16 @@
(delete-team-rels conn profile-id id)
;; The leaver is told so, exactly as a removed member is, so the
;; session stops asking for content it can no longer read.
(mbus/pub! msgbus
:topic profile-id
:message {:type :team-membership-change
:change :removed
:team-id id
:team-name (:name team)})
(ntf/notify-permissions-changed cfg profile-id)
nil))
(def ^:private schema:leave-team
@ -938,6 +954,10 @@
::jobs/params {:object :team
:deleted-at (:deleted-at team)
:id team-id}})
;; The team is soft-deleted, so every subscription its members hold
;; loses its authorization at once. One event: the watcher resolves
;; the members itself.
(ntf/notify-team-permissions-changed cfg team-id)
(rph/with-meta (rph/wrap team)
{::audit/props {:organization-id organization-id
:team-id team-id
@ -970,7 +990,7 @@
;; --- Mutation: Team Update Role
(defn update-team-member-role
[{:keys [::db/conn ::mbus/msgbus]} {:keys [profile-id team-id member-id role] :as params}]
[{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [profile-id team-id member-id role] :as params}]
;; We retrieve all team members instead of query the
;; database for a single member. This is just for
;; convenience, if this becomes a bottleneck or problematic,
@ -1020,6 +1040,12 @@
params
{:team-id team-id
:profile-id member-id})
;; A role change never removes read access on its own, but the open
;; subscriptions are re-checked rather than assumed: the watcher
;; decides from fresh data, which is also what keeps a downgrade to
;; viewer from being mistaken for a revocation.
(ntf/notify-permissions-changed cfg member-id)
nil)))
(def ^:private schema:update-team-member-role
@ -1093,6 +1119,11 @@
:team-id team-id
:team-name (:name team)}))
;; Whatever this member may still read through another path (an
;; organization owner keeps read-only access), the subscriptions they
;; already hold have to be re-checked, not trusted from here on.
(ntf/notify-permissions-changed cfg member-id)
nil))
;; --- Mutation: Update Team Photo

View File

@ -474,6 +474,31 @@
"The number of invitations can be sent in a single rpc request"
25)
(defn- check-send-invitations-policy
"Raises `:insufficient-permissions` when the organization of the team does
not let the profile send invitations. The policy covers the pending
invitations too: changing their role, deleting them and minting their token
manage the same access that sending one grants.
Only applies with the `:admin-console` flag. Then it returns the team with
its organization info, so the caller reuses the same Nitrate answer."
[cfg profile-id team-id perms]
(when (contains? cf/flags :admin-console)
(let [team (db/get-by-id cfg :team team-id)
;; Strict lookup: a failed Nitrate call must not skip the
;; organization policy by looking like a team without one.
team (nitrate/add-organization-info-to-team-strict cfg team {::rpc/profile-id profile-id})
organization (:organization team)]
(when (and organization
(not (cto/allowed? :send-invitations
{:organization-perms organization
:profile-id profile-id
:team-perms perms})))
(ex/raise :type :validation
:code :insufficient-permissions
:hint "Organization policy does not allow you to send invitations"))
team)))
(sv/defmethod ::create-team-invitations
"A rpc call that allows to send single or multiple invitations to join the team.
@ -486,12 +511,10 @@
[cfg {:keys [::rpc/profile-id team-id role emails] :as params}]
(let [perms (teams/check-invitation-permissions cfg profile-id team-id)
profile (db/get-by-id cfg :profile profile-id)
team (db/get-by-id cfg :team team-id)
;; Strict lookup: a failed Nitrate call must not skip the
;; organization policy below by looking like a team without one.
team-with-org (when (contains? cf/flags :admin-console)
(nitrate/add-organization-info-to-team-strict cfg team {::rpc/profile-id profile-id}))
organization (:organization team-with-org)
;; Reuse the organization lookup of the policy check, so every
;; organization rule below sees the same Nitrate answer.
team (or (check-send-invitations-policy cfg profile-id team-id perms)
(db/get-by-id cfg :team team-id))
;; Determine which format is being used
using-emails-format? (and emails role)
;; Handle both parameter formats
@ -503,16 +526,6 @@
(count emails)
(count (:invitations params)))]
(when (and (contains? cf/flags :admin-console)
organization
(not (cto/allowed? :send-invitations
{:organization-perms organization
:profile-id profile-id
:team-perms perms})))
(ex/raise :type :validation
:code :insufficient-permissions
:hint "Organization policy does not allow you to send invitations"))
;; Don't allow promote to owner to admin users.
(when (and (not (:is-owner perms))
(or (= role :owner)
@ -536,10 +549,7 @@
;; Check if the current profile is allowed to send emails
(teams/check-profile-muted cfg profile)
(let [;; Reuse the strict organization lookup made for the policy
;; check above, so every rule sees the same Nitrate answer.
team (or team-with-org team)
;; NOTE: Is important pass RPC method params down to the
(let [;; NOTE: Is important pass RPC method params down to the
;; `create-team-invitations` because it uses the implicit
;; RPC properties from params for fill necessary data on
;; emiting an entry to the audit-log
@ -647,6 +657,9 @@
:code :cant-promote-to-owner
:hint "only the team owner can get an owner invitation token"))
;; Handing out the link invites as much as sending the email does.
(check-send-invitations-policy cfg profile-id team-id perms)
(let [member (profile/get-profile-by-email pool (:email-to invit))
token (create-invitation-token cfg {:team-id (:team-id invit)
;; The inviter is the creator of
@ -678,17 +691,20 @@
(sv/defmethod ::update-team-invitation-role
{::doc/added "1.17"
::doc/module :teams
::sm/params schema:update-team-invitation-role
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email role] :as params}]
(let [perms (teams/check-invitation-permissions conn profile-id team-id)]
::sm/params schema:update-team-invitation-role}
[cfg {:keys [::rpc/profile-id team-id email role] :as params}]
(let [perms (teams/check-invitation-permissions cfg profile-id team-id)]
;; Don't allow promote to owner to admin users.
(when (and (not (:is-owner perms)) (= role :owner))
(ex/raise :type :validation
:code :cant-promote-to-owner))
(db/update! conn :team-invitation
;; Checked outside a transaction: it may call Nitrate over HTTP, and the
;; update below is a single statement.
(check-send-invitations-policy cfg profile-id team-id perms)
(db/update! cfg :team-invitation
{:role (name role) :updated-at (ct/now)}
{:team-id team-id :email-to (profile/clean-email email)})
@ -703,16 +719,19 @@
(sv/defmethod ::delete-team-invitation
{::doc/added "1.17"
::sm/params schema:delete-team-invition
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email] :as params}]
(teams/check-invitation-permissions conn profile-id team-id)
::sm/params schema:delete-team-invition}
[cfg {:keys [::rpc/profile-id team-id email] :as params}]
(let [perms (teams/check-invitation-permissions cfg profile-id team-id)]
(let [invitation (db/delete! conn :team-invitation
{:team-id team-id
:email-to (profile/clean-email email)}
{::db/return-keys true})]
(rph/wrap nil {::audit/props {:invitation-id (:id invitation)}})))
;; Checked outside a transaction: it may call Nitrate over HTTP, and the
;; delete below is a single statement.
(check-send-invitations-policy cfg profile-id team-id perms)
(let [invitation (db/delete! cfg :team-invitation
{:team-id team-id
:email-to (profile/clean-email email)}
{::db/return-keys true})]
(rph/wrap nil {::audit/props {:invitation-id (:id invitation)}}))))
;; --- Mutation: Request Team Invitation
@ -770,6 +789,9 @@
(-> file
(dissoc :data)
(dissoc :deleted-at)
;; The storage pointer is internal bookkeeping: it never leaves the
;; backend in a response.
(bfc/file->response)
(assoc :page-id (-> file :data :pages first)))))
(def ^:private schema:create-team-access-request

View File

@ -135,7 +135,10 @@
(defn- get-view-only-bundle
[{:keys [::db/conn] :as cfg} {:keys [profile-id file-id share-id ::perms] :as params}]
(let [file (bfc/get-file cfg file-id)
(let [;; The storage pointer is internal bookkeeping: this bundle is an RPC
;; response and must not expose it.
file (-> (bfc/get-file cfg file-id)
(bfc/file->response))
;; pull request review sandbox: the viewer shows the pinned
;; review snapshot of the branch, not its live state
@ -175,7 +178,8 @@
libs (->> (bfc/get-file-libraries conn file-id)
(mapv (fn [{:keys [id] :as lib}]
(merge lib (bfc/get-file cfg id)))))
(-> (merge lib (bfc/get-file cfg id))
(bfc/file->response)))))
;; A share link never grants pages of a linked library, so trim
;; each library to the components the allowed pages reference.

View File

@ -6,6 +6,8 @@
(ns app.rpc.notifications
(:require
[app.common.uuid :as uuid]
[app.db :as db]
[app.msgbus :as mbus]))
(defn notify-team-change
@ -49,3 +51,68 @@
:topic organization-id
:message {:type :organization-change-sso
:organization-id organization-id})))
;; --- Subscription revocation
;; A websocket subscription is authorized once, when it is opened, and
;; that single decision is trusted for as long as the connection lives.
;; A mutation that can take access away announces it here, and every
;; backend re-verifies the subscriptions it owns for that profile.
;;
;; The announcement travels over the message bus precisely because the
;; connection registry is local to one backend instance, while the RPC
;; that revokes access runs on whichever instance received the request,
;; which is usually not the one holding the socket.
(def internal-revocation-topic
"Message-bus topic carrying backend-internal revocation events.
A fixed uuid that is never a real resource id, so it cannot collide
with a file, team or profile topic — and deliberately not the
`uuid/zero` system topic, which is piped straight into the client
output channel, so anything published there would reach every
connected client.
No client can subscribe here. `:subscribe-file` and `:subscribe-team`
only accept ids that first pass a permission check, so a client can
never make the bus deliver this topic to it."
(uuid/uuid "ffffffff-ffff-ffff-ffff-ffffffffffff"))
(defn notify-permissions-changed
"Asks every backend to re-verify the websocket subscriptions currently
held by `profile-id`.
Deferred past the commit: the watcher decides from a fresh permission
check on another connection, so announcing before the transaction
commits would let it see the old state and close nothing. Outside a
transaction the announcement goes out immediately.
The watcher always decides from a fresh permission check, so an extra
announcement is safe: it closes nothing whose access still holds."
[cfg profile-id]
(db/after-commit
(fn []
(let [msgbus (::mbus/msgbus cfg)]
(mbus/pub! msgbus
:topic internal-revocation-topic
:message {:type :profile-permissions-changed
:profile-id profile-id})))))
(defn notify-team-permissions-changed
"Asks every backend to re-verify the subscriptions of every member of
`team-id`.
Used when a change can move a resource out of the reach of a whole
team (moving, deleting or soft-deleting files, projects and teams). One
event per team: the watcher resolves the members itself, so a large
team cannot overflow the bounded publish buffers on the way out.
Like `notify-permissions-changed`, deferred past the commit."
[cfg team-id]
(db/after-commit
(fn []
(let [msgbus (::mbus/msgbus cfg)]
(mbus/pub! msgbus
:topic internal-revocation-topic
:message {:type :team-permissions-changed
:team-id team-id})))))

View File

@ -108,13 +108,16 @@
:can-edit false})
(def ^:private sql:get-team-id-for-project
"SELECT team_id FROM project WHERE id = ?")
"SELECT team_id FROM project WHERE id = ?
AND deleted_at IS NULL")
(def ^:private sql:get-team-id-for-file
"SELECT p.team_id
FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
WHERE f.id = ?")
WHERE f.id = ?
AND f.deleted_at IS NULL
AND p.deleted_at IS NULL")
(defn get-team-id-for-project
[cfg project-id]
@ -127,11 +130,18 @@
(:team-id)))
(defn resolve-team-id
"Resolves the team that owns the project/file `params` references, or
`params`' own `:team-id`. A soft-deleted file/project/team resolves to
nil: no caller may rely on a dead team for access decisions. The team
row is checked for all branches — the SQL filters cover file and
project, this final check covers the team itself."
[cfg {:keys [team-id project-id file-id]}]
(cond
(some? team-id) team-id
(some? project-id) (get-team-id-for-project cfg project-id)
(some? file-id) (get-team-id-for-file cfg file-id)))
(let [resolved-team-id (cond (some? team-id) team-id
(some? project-id) (get-team-id-for-project cfg project-id)
(some? file-id) (get-team-id-for-file cfg file-id))]
(when (and resolved-team-id
(some? (db/get* cfg :team {:id resolved-team-id})))
resolved-team-id)))
(defn get-organization-owner-permissions
"When `profile-id` is a non-member owner of the organization that owns

View File

@ -423,6 +423,8 @@
(sm/lazy-validator schema:snapshots-per-file))
(def ^:private sql:get-snapshots-per-file
;; A version is told apart by its label, and the partial index with the same
;; predicate answers the count without touching the table.
"SELECT count(*) AS total
FROM file_change AS fc
WHERE fc.file_id = ?
@ -454,11 +456,16 @@
(sm/lazy-validator schema:snapshots-per-team))
(def ^:private sql:get-snapshots-per-team
;; The files of the team are resolved first, so the count filters by
;; `file_id`: the planner can prune the partitions of `file_change` when it
;; chooses an index scan (with a sequential scan it reads all of them). A
;; version is told apart by its label.
"SELECT count(*) AS total
FROM file_change AS fc
JOIN file AS f ON (f.id = fc.file_id)
JOIN project AS p ON (p.id = f.project_id)
WHERE p.team_id = ?
WHERE fc.file_id = ANY (SELECT f.id
FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
WHERE p.team_id = ?)
AND fc.created_by = 'user'
AND fc.deleted_at IS NULL
AND fc.label IS NOT NULL")

View File

@ -39,6 +39,7 @@
`[.data[\"pages-index\"][] | {id, name}]`, etc."
(:require
[app.binfile.common :as bfc]
[app.common.exceptions :as ex]
[app.common.json :as json]
[app.db :as db]
[app.features.file-snapshots :as fsnap]
@ -183,16 +184,23 @@
(defn get-snapshot-json
"Return a snapshot as JSON (realized). For nREPL.
Takes the same `format` and `output` as `get-file-json`."
Takes the same `format` and `output` as `get-file-json`. It raises when the
file has no snapshot with that id."
[system file-id snapshot-id & {:keys [format output] :or {format :full}}]
(let [snapshot (db/run! system
(fn [cfg]
(fsnap/get-snapshot cfg file-id snapshot-id)))
data (case format
:full snapshot
:data (:data snapshot)
:summary (summarize-data (:data snapshot))
:pages (:pages (summarize-data (:data snapshot))))]
data (if snapshot
(case format
:full snapshot
:data (:data snapshot)
:summary (summarize-data (:data snapshot))
:pages (:pages (summarize-data (:data snapshot))))
(ex/raise :type :not-found
:code :snapshot-not-found
:hint "unable to find snapshot with the provided id"
:snapshot-id snapshot-id
:file-id file-id))]
(write-output (data->json data) output)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

View File

@ -192,5 +192,5 @@
:created-by "system"}))
(let [file' (update file' :revn inc)]
(bfc/update-file! system file' opts)
(bfc/update-file! system file' ::bfc/reset-migrations? (::bfc/reset-migrations? opts))
true))))

View File

@ -118,6 +118,16 @@
[(ct/now) chunk-size]])]
(db/exec! conn (into [sql] params))))
(defn- server-error
"Return the PostgreSQL server error of a foreign key violation, or nil. It
carries the constraint and the referencing table, which is what the operator
needs: any `NO ACTION` foreign key on `storage_object` can block the delete,
not only the `file_data` one."
[cause]
(when (and (instance? org.postgresql.util.PSQLException cause)
(= "23503" (.getSQLState ^java.sql.SQLException cause)))
(.getServerErrorMessage ^org.postgresql.util.PSQLException cause)))
(defn- clean-deleted
"Processes the pending objects chunk by chunk with keyset pagination: each
chunk is one short transaction that reads and locks the next `chunk-size`
@ -127,11 +137,16 @@
marked as deleted.
A chunk that fails aborts its transaction: nothing of the chunk is deleted,
its locks are released, and it is logged and skipped. The cursor advances
to the `(deleted_at, id)` of the last row of each read chunk regardless of
the deletion result, so a failed chunk is not visited again in the same
run. There is no long-lived read transaction: each transaction lives only
as long as one chunk, letting the vacuum proceed between chunks. Rows
its locks are released, and it is logged and skipped. A chunk blocked by a
foreign key (SQLState 23503) keeps its log at error level with a hint and the
constraint and table the database reports, and is retried on the next run;
the reference must be fixed for the chunk to make progress. Any `NO ACTION`
foreign key on `storage_object` can block the delete, not only the one of
`file_data`. The cursor advances to the `(deleted_at, id)` of
the last row of each read chunk regardless of the deletion result, so a
failed chunk is not visited again in the same run. There is no long-lived
read transaction: each transaction lives only as long as one chunk, letting
the vacuum proceed between chunks. Rows
locked by a concurrent run are skipped and deferred to the next run; a
chunk that fails after a partial backend delete self-heals on the next
run (physical deletes are idempotent). Every `throttle-every` deleted
@ -155,9 +170,22 @@
;; afterwards is a JDBC no-op on the already
;; rolled-back connection.
(db/rollback! conn)
(l/err :hint "unexpected error on chunk deletion"
:ids (map :id rows)
:cause cause)
(if-let [serr (server-error cause)]
;; A NO ACTION foreign key refused a physical
;; delete: a live row of another table still
;; references an object marked for deletion.
;; That is a real anomaly (the reference scan
;; should have frozen it), so it stays an
;; error and the chunk is retried on the next
;; run until the reference is fixed.
(l/err :hint "chunk deletion blocked by a live reference; fix the reference and let the next run reclaim the chunk"
:ids (map :id rows)
:constraint (.getConstraint ^org.postgresql.util.ServerErrorMessage serr)
:table (.getTable ^org.postgresql.util.ServerErrorMessage serr)
:cause cause)
(l/err :hint "unexpected error on chunk deletion"
:ids (map :id rows)
:cause cause))
[rows {}])))))]
;; Reported only after the chunk commits: a rolled-back chunk leaves

View File

@ -108,16 +108,13 @@
(def sql:exists-file-data-refs
"SELECT EXISTS (
SELECT 1 FROM file_data
WHERE file_id = ?
AND id = ?
AND metadata->>'storage-ref-id' = ?::text
WHERE storage_object_id = ?
) AS has_refs")
(defn- has-file-data-refs?
[conn sobject]
(let [{:keys [file-id id]} (:metadata sobject)]
(-> (db/exec-one! conn [sql:exists-file-data-refs file-id id (:id sobject)])
(get :has-refs))))
(-> (db/exec-one! conn [sql:exists-file-data-refs (:id sobject)])
(get :has-refs)))
(def ^:private sql:mark-freeze-in-bulk
"UPDATE storage_object

View File

@ -53,7 +53,8 @@
:deleted-at (ct/format-inst deleted-at))
(db/update! conn :file-change
{:deleted-at deleted-at}
{:deleted-at deleted-at
:updated-at (ct/now)}
{:id id :file-id file-id}
{::db/return-keys false})
@ -139,7 +140,8 @@
;; lifetime; the collector would still take it, just much later than it
;; should.
(db/update! conn :file-change
{:deleted-at deleted-at}
{:deleted-at deleted-at
:updated-at (ct/now)}
{:file-id id :deleted-at nil}
{::db/return-keys false})

View File

@ -613,7 +613,7 @@
ORDER BY deleted_at ASC")
(def ^:private sql:lock-file-data
"SELECT file_id, id, type, deleted_at, metadata, backend
"SELECT file_id, id, type, deleted_at, backend, storage_object_id
FROM file_data
WHERE (file_id, id) IN (SELECT file_id, id FROM unnest(?::uuid[], ?::uuid[]) AS u(file_id, id))
AND deleted_at IS NOT NULL
@ -630,11 +630,10 @@
(let [file-ids (db/create-array conn "uuid" (map :file-id rows))
ids (db/create-array conn "uuid" (map :id rows))]
(->> (db/exec! conn [sql:lock-file-data file-ids ids threshold])
(reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}]
(reduce (fn [total {:keys [file-id id type deleted-at backend storage-object-id]}]
(some->> metadata
(fdata/decode-metadata)
(fdata/process-metadata cfg))
(some->> storage-object-id
(fdata/touch-storage-object cfg))
(l/trc :obj "file-data"
:id (str id)
@ -673,41 +672,48 @@
{:fetch-size 5}))))))
(def ^:private sql:get-file-change
"SELECT id FROM file_change
"SELECT id, file_id FROM file_change
WHERE deleted_at IS NOT NULL
AND deleted_at <= ?
ORDER BY deleted_at ASC")
(def ^:private sql:lock-file-change
"SELECT id, file_id, deleted_at FROM file_change
WHERE id = ANY(?::uuid[])
"SELECT file_id, id, deleted_at FROM file_change
WHERE file_id = ?
AND id = ANY(?::uuid[])
AND deleted_at IS NOT NULL
AND deleted_at <= ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-changes-chunk
[cfg ids]
[cfg rows]
(try
(db/tx-run! cfg
(fn [{:keys [::db/conn ::threshold]}]
(db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
(let [ids (db/create-array conn "uuid" ids)]
(->> (db/exec! conn [sql:lock-file-change ids threshold])
(reduce (fn [total {:keys [id file-id deleted-at]}]
(l/trc :obj "file-change"
:id (str id)
:file-id (str file-id)
:deleted-at (ct/format-inst deleted-at))
;; The rows are locked per file, with a constant `file_id`,
;; so the planner prunes to the partition that holds each
;; one instead of scanning all of them for a tuple `IN`.
(->> (group-by :file-id rows)
(mapcat (fn [[file-id file-rows]]
(let [ids (db/create-array conn "uuid" (map :id file-rows))]
(db/exec! conn [sql:lock-file-change file-id ids threshold]))))
(reduce (fn [total {:keys [id file-id deleted-at]}]
(l/trc :obj "file-change"
:id (str id)
:file-id (str file-id)
:deleted-at (ct/format-inst deleted-at))
(let [affected (-> (db/delete! conn :file-change {:id id})
(db/get-update-count))]
(+ total affected)))
0)))))
(let [affected (-> (db/delete! conn :file-change
{:id id :file-id file-id})
(db/get-update-count))]
(+ total affected)))
0))))
(catch Throwable cause
(l/err :obj "file-change"
:hint "error on chunk deletion, skipping it"
:chunk ids
:chunk rows
:cause cause)
0)))
@ -720,7 +726,10 @@
(+ total deleted))))]
(db/tx-run! (assoc cfg ::db/read-only true)
(fn [{:keys [::db/conn]}]
(transduce (comp (map :id) (partition-all chunk-size)) rf 0
(transduce (comp (map (fn [row] {:id (:id row)
:file-id (:file-id row)}))
(partition-all chunk-size))
rf 0
(db/plan conn [sql:get-file-change threshold]
{:fetch-size 5}))))))

View File

@ -28,18 +28,33 @@
(def ^:private sql:lock-file
;; NOTE: SKIP LOCKED only serializes against writers that also lock
;; the file row. Every file_data writer must therefore hold at least
;; FOR SHARE on it (the update RPC via lock-for-share?, file-gc via
;; FOR UPDATE); a writer skipping that lock would reopen the
;; lost-update race this lock closes.
;; the file row. Every file_data writer must therefore lock it too:
;; the update RPC and the snapshot paths via `lock-for-update?`, and
;; file-gc via FOR UPDATE. A writer skipping that lock would reopen
;; the lost-update race this lock closes.
"SELECT id FROM file WHERE id = ? AND deleted_at IS NULL FOR UPDATE SKIP LOCKED")
(defn- decode-row
"Normalize a `file_data` row read with `SELECT fd.*`: NULL columns are
absent (the db layer uses `next.jdbc.optional`) and jsonb metadata arrives
as a `PGobject`, while `fdata/upsert!` expects a decoded metadata map. A
JSON `null` decodes to nil, and the key has to disappear rather than travel
as `:metadata nil`, which the schema rejects."
[row]
(let [metadata (some-> (:metadata row) fdata/decode-metadata)]
(cond-> (dissoc row :metadata)
(some? metadata) (assoc :metadata metadata))))
(defn- offload-file-data
"Offload one hot-storage row to cold storage. Returns the row type
and the migrated bytes for the caller to aggregate."
[cfg {:keys [id file-id type data] :as fdata}]
(jobs/heartbeat cfg)
(fdata/upsert! cfg (assoc fdata :backend "storage"))
;; The pointer key is normalized here because a NULL `storage_object_id`
;; is omitted by the row builder; `decode-row` already normalized metadata.
(fdata/upsert! cfg (assoc fdata
:backend "storage"
:storage-object-id (:storage-object-id fdata)))
(let [size (if (bytes? data) (alength ^bytes data) 0)]
(l/trc :hint "file data row offloaded"
:file-id (str file-id)
@ -61,16 +76,19 @@
[{:keys [::db/conn] :as cfg} file-id]
(if (db/exec-one! conn [sql:lock-file file-id])
(let [{:keys [rows bytes by-type]}
(reduce (fn [stats fdata]
(let [{:keys [type size]} (offload-file-data cfg fdata)]
(-> stats
(update :rows inc)
(update :bytes + size)
(update-in [:by-type type]
(fn [{r :rows b :bytes :or {r 0 b 0}}]
{:rows (inc r) :bytes (+ b size)})))))
{:rows 0 :bytes 0 :by-type {}}
(db/plan conn [sql:get-file-data file-id]))]
(transduce
(map decode-row)
(completing
(fn [stats fdata]
(let [{:keys [type size]} (offload-file-data cfg fdata)]
(-> stats
(update :rows inc)
(update :bytes + size)
(update-in [:by-type type]
(fn [{r :rows b :bytes :or {r 0 b 0}}]
{:rows (inc r) :bytes (+ b size)}))))))
{:rows 0 :bytes 0 :by-type {}}
(db/plan conn [sql:get-file-data file-id]))]
(l/trc :hint "file data offloaded"
:file-id (str file-id)
:offloaded rows

View File

@ -10,6 +10,7 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.time :as ct]
[app.common.uri :as u]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.http.session :as session]
@ -448,6 +449,76 @@
loc (redirect-location result)]
(t/is (.contains loc "error=registration-disabled")))))))
(t/deftest callback-with-invitation-and-no-flags-redirects-to-register
;; Review of #12974: a verifiable invitation issued for the OIDC
;; email acts as a substitute for the SSO registration flags, so the
;; user reaches the register screen instead of the
;; registration-disabled error. The JWT is verified and the email is
;; compared here; the invitation row is revalidated under a row lock
;; in `register-profile`.
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
invitation (tokens/generate cfg {:iss :team-invitation
:exp (ct/in-future {:hours 1})
:member-email "u@e.com"})
state (make-state-token cfg {:invitation-token invitation})
request (default-request cfg :state state)]
(binding [cf/config {:public-uri "http://localhost:3449"}
cf/flags #{}]
(with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"})
app.auth.oidc/get-info (constantly {:email "u@e.com" :fullname "U"
:backend "oidc" :email-verified false
:invitation-token invitation
:props {}})
app.auth.oidc/get-profile (constantly nil)]
(let [result (#'oidc/callback-handler cfg request)
loc (redirect-location result)
claims (tokens/decode cfg (location-token loc))]
(t/is (.contains loc "screen=auth-register-validate"))
(t/is (.contains loc "token="))
(t/is (= :prepared-register (:iss claims))
"the redirect token must be a :prepared-register token")
(t/is (= invitation (:invitation-token claims))
"the invitation token must travel inside the prepared-register claims"))))))
(t/deftest callback-with-invitation-email-mismatch-redirects-to-registration-disabled
;; Review of #12974: an invitation issued for another email must not
;; unlock registration with closed registration.
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
invitation (tokens/generate cfg {:iss :team-invitation
:exp (ct/in-future {:hours 1})
:member-email "alice@example.com"})
state (make-state-token cfg {:invitation-token invitation})
request (default-request cfg :state state)]
(binding [cf/config {:public-uri "http://localhost:3449"}
cf/flags #{}]
(with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"})
app.auth.oidc/get-info (constantly {:email "attacker@example.com" :fullname "A"
:backend "oidc" :email-verified false
:invitation-token invitation
:props {}})
app.auth.oidc/get-profile (constantly nil)]
(let [result (#'oidc/callback-handler cfg request)
loc (redirect-location result)]
(t/is (.contains loc "error=registration-disabled")))))))
(t/deftest callback-with-garbage-invitation-redirects-to-registration-disabled
;; Review of #12974: raw presence is not enough — an unverifiable
;; invitation string must not reach the register screen.
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
state (make-state-token cfg {:invitation-token "inv-123"})
request (default-request cfg :state state)]
(binding [cf/config {:public-uri "http://localhost:3449"}
cf/flags #{}]
(with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"})
app.auth.oidc/get-info (constantly {:email "u@e.com" :fullname "U"
:backend "oidc" :email-verified false
:invitation-token "inv-123"
:props {}})
app.auth.oidc/get-profile (constantly nil)]
(let [result (#'oidc/callback-handler cfg request)
loc (redirect-location result)]
(t/is (.contains loc "error=registration-disabled")))))))
(t/deftest callback-profile-blocked
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
state (make-state-token cfg {})
@ -578,69 +649,73 @@
(t/deftest organization-sso-callback-success-emits-succeeded
;; No session on the request → auth-succeeded only (no token-issued).
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
state (make-state-token cfg {:dest-url "https://penpot.example.com/#/workspace"
state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace"
:organization-id test-organization-id})
request (default-request cfg :state state)
events (atom [])]
(with-redefs [app.nitrate/call (constantly {:active true})
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (constantly {})
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(let [result (#'oidc/callback-handler cfg request)]
(t/is (= "https://penpot.example.com/#/workspace" (redirect-location result)))
(t/is (= ["organization-sso-auth-succeeded"] (mapv :name @events)))
(t/is (= test-organization-id (get-in (first @events) [:props :organization-id])))))))
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(with-redefs [app.nitrate/call (constantly {:active true})
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (constantly {})
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(let [result (#'oidc/callback-handler cfg request)]
(t/is (= "https://penpot.example.com/?screen=workspace" (redirect-location result)))
(t/is (= ["organization-sso-auth-succeeded"] (mapv :name @events)))
(t/is (= test-organization-id (get-in (first @events) [:props :organization-id]))))))))
(t/deftest organization-sso-callback-success-with-session-emits-token-issued
(let [manager (session/inmemory-manager)
cfg (assoc (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
::session/manager manager)
state (make-state-token cfg {:dest-url "https://penpot.example.com/#/workspace"
:organization-id test-organization-id})
state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace"
:organization-id test-organization-id
:profile-id test-profile-id})
request (default-request cfg :state state)
events (atom [])
sess {:id (uuid/next)
:profile-id test-profile-id
:props {}}]
(with-redefs [app.nitrate/call (constantly {:active true})
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (constantly {})
app.http.session/get-session (constantly sess)
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(let [result (#'oidc/callback-handler cfg request)
names (mapv :name @events)
stored (session/read-session manager (:id sess))]
(t/is (= "https://penpot.example.com/#/workspace" (redirect-location result)))
(t/is (= ["organization-sso-token-issued"
"organization-sso-auth-succeeded"]
names))
(t/is (= test-organization-id
(get-in (first @events) [:props :organization-id])))
(t/is (= test-profile-id (:profile-id (first @events))))
(t/is (= #{:organization-id}
(set (keys (:props (first @events))))))
(t/is (contains? (get-in stored [:props :sso]) test-organization-id))))))
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(with-redefs [app.nitrate/call (constantly {:active true})
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (constantly {})
app.http.session/get-session (constantly sess)
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(let [result (#'oidc/callback-handler cfg request)
names (mapv :name @events)
stored (session/read-session manager (:id sess))]
(t/is (= "https://penpot.example.com/?screen=workspace" (redirect-location result)))
(t/is (= ["organization-sso-token-issued"
"organization-sso-auth-succeeded"]
names))
(t/is (= test-organization-id
(get-in (first @events) [:props :organization-id])))
(t/is (= test-profile-id (:profile-id (first @events))))
(t/is (= #{:organization-id}
(set (keys (:props (first @events))))))
(t/is (contains? (get-in stored [:props :sso]) test-organization-id)))))))
(t/deftest organization-sso-callback-error-emits-failed
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
state (make-state-token cfg {:dest-url "https://penpot.example.com/#/workspace"
state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace"
:organization-id test-organization-id})
request (default-request cfg :state state)
events (atom [])]
(with-redefs [app.nitrate/call (fn [_cfg method _params]
(case method
:get-organization-sso {:active true}
:get-organization-summary {:name "Organization"}))
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (fn [& _]
(ex/raise :type :internal
:code :unable-to-retrieve-user-info))
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(#'oidc/callback-handler cfg request)
(t/is (= ["organization-sso-auth-failed"] (mapv :name @events)))
(t/is (= {:organization-id test-organization-id
:failure-reason "user-info-failed"}
(:props (first @events)))))))
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(with-redefs [app.nitrate/call (fn [_cfg method _params]
(case method
:get-organization-sso {:active true}
:get-organization-summary {:name "Organization"}))
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (fn [& _]
(ex/raise :type :internal
:code :unable-to-retrieve-user-info))
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(#'oidc/callback-handler cfg request)
(t/is (= ["organization-sso-auth-failed"] (mapv :name @events)))
(t/is (= {:organization-id test-organization-id
:failure-reason "user-info-failed"}
(:props (first @events))))))))
(t/deftest organization-sso-oauth-error-emits-failed-without-changing-redirect
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
@ -659,6 +734,155 @@
:failure-reason "access-denied"}
(:props (first @events)))))))))
(t/deftest build-dest-url-builds-the-destination-on-the-public-uri
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(t/testing "the route is appended to the public uri, separator included"
(t/is (= "https://penpot.example.com/?screen=workspace&team-id=0e5f"
(oidc/build-dest-url "screen=workspace&team-id=0e5f"))))
(t/testing "the app base path of the public uri is kept"
(binding [cf/config {:public-uri "https://host/penpot"}]
(t/is (= "https://host/penpot/?screen=workspace"
(oidc/build-dest-url "screen=workspace")))))
(t/testing "a blank query string lands on the public uri"
(t/is (= "https://penpot.example.com/" (oidc/build-dest-url "")))
(t/is (= "https://penpot.example.com/" (oidc/build-dest-url nil))))
(t/testing "a value without params of its own is not a route"
(doseq [query-string ["https://evil.example/"
"//evil.example/x"]]
(t/is (= "https://penpot.example.com/"
(oidc/build-dest-url query-string)) query-string)))
(t/testing "the root of the app keeps its params (invitation, template)"
(t/is (= "https://penpot.example.com/?template=xyz"
(oidc/build-dest-url "template=xyz"))))
(t/testing "a value that still carries the separator is trimmed"
(t/is (= "https://penpot.example.com/?screen=workspace"
(oidc/build-dest-url "?screen=workspace"))))
(t/testing "control characters never reach the destination"
(let [dest (oidc/build-dest-url "screen=x\r\nSet-Cookie: a=b")]
(t/is (not (.contains dest "\r")))
(t/is (not (.contains dest "\n")))))
(t/testing "repeated params are kept"
(t/is (= "https://penpot.example.com/?a=1&a=2"
(oidc/build-dest-url "a=1&a=2"))))))
(def ^:private foreign-dest-urls
"Destinations that a naive prefix check would let through: the first one
parses the prefix as userinfo, the second one is a look-alike host."
["https://evil.example/"
"https://penpot.example.com@evil.example/"
"https://penpot.example.com.evil.example/"])
(t/deftest build-organization-sso-auth-redirect-uri-seals-a-server-built-destination
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
provider {:id "oidc"
:type "oidc"
:client-id "client"
:auth-uri "https://idp.example.com/authorize"
:scopes ["openid"]}
sso {:active true
:issuer "https://idp.example.com"
:organization-id test-organization-id}]
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(doseq [[query-string expected]
[["screen=workspace" "https://penpot.example.com/?screen=workspace"]
["template=xyz" "https://penpot.example.com/?template=xyz"]
["https://evil.example/" "https://penpot.example.com/"]
["" "https://penpot.example.com/"]]]
(t/testing (str "the destination sealed for " (pr-str query-string))
(let [uri (oidc/build-organization-sso-auth-redirect-uri
cfg sso
:query-string query-string
:organization-id test-organization-id
:profile-id test-profile-id
:provider provider)
claims (tokens/decode cfg (-> (u/uri uri)
:query
(u/query-string->map)
:state))]
;; The destination is always composed by the server: the route
;; of the client, never a URL of its own.
(t/is (= expected (:dest-url claims)))
(t/is (= test-profile-id (:profile-id claims)))))))))
(t/deftest organization-sso-callback-ignores-a-destination-outside-the-public-uri
(doseq [dest-url foreign-dest-urls]
(t/testing (str "the success path ignores " dest-url)
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
state (make-state-token cfg {:dest-url dest-url
:organization-id test-organization-id})
request (default-request cfg :state state)]
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(with-redefs [app.nitrate/call (constantly {:active true})
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (constantly {})]
(let [result (#'oidc/callback-handler cfg request)]
(t/is (= "https://penpot.example.com/" (redirect-location result))))))))
(t/testing (str "the error path ignores " dest-url " as well")
(let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
state (make-state-token cfg {:dest-url dest-url
:organization-id test-organization-id})
request (default-request cfg :state state)]
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(with-redefs [app.nitrate/call (fn [_cfg method _params]
(case method
:get-organization-sso {:active true}
:get-organization-summary {:name "Organization"}))
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (fn [& _]
(ex/raise :type :internal
:code :unable-to-retrieve-user-info))]
(let [result (#'oidc/callback-handler cfg request)
loc (redirect-location result)]
(t/is (not (.contains loc "evil.example")))
(t/is (.contains loc "https://penpot.example.com/")))))))))
(t/deftest trusted-dest-url-requires-the-base-path
(binding [cf/config {:public-uri "https://host/penpot"}]
(t/testing "a destination under the app base path is trusted"
(t/is (= "https://host/penpot/?screen=workspace"
(#'oidc/trusted-dest-url "https://host/penpot/?screen=workspace"))))
(t/testing "a destination in another path of the same host is not"
(t/is (nil? (#'oidc/trusted-dest-url "https://host/otra/?screen=workspace"))))
(t/testing "a destination in another host is not"
(t/is (nil? (#'oidc/trusted-dest-url "https://otro.example/penpot/?screen=workspace"))))))
(t/deftest organization-sso-callback-does-not-grant-the-sso-entry-to-another-session
(let [manager (session/inmemory-manager)
cfg (assoc (dissoc base-cfg :app.email/blacklist :app.email/whitelist)
::session/manager manager)
;; The state was prepared by another profile: the session that
;; completes the flow must not be granted the organization entry.
state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace"
:organization-id test-organization-id
:profile-id (uuid/random)})
request (default-request cfg :state state)
events (atom [])
sess {:id (uuid/next)
:profile-id test-profile-id
:props {}}]
(binding [cf/config {:public-uri "https://penpot.example.com"}]
(with-redefs [app.nitrate/call (constantly {:active true})
app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"})
app.auth.oidc/get-info (constantly {})
app.http.session/get-session (constantly sess)
app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))]
(let [result (#'oidc/callback-handler cfg request)
stored (session/read-session manager (:id sess))]
(t/is (= "https://penpot.example.com/?screen=workspace" (redirect-location result)))
(t/is (= ["organization-sso-auth-failed"] (mapv :name @events)))
(t/is (= "session-mismatch" (get-in (first @events) [:props :failure-reason])))
(t/is (nil? (get-in stored [:props :sso]))))))))
(t/deftest prepare-organization-sso-provider-does-not-skip-ssrf-check
(t/testing "organization SSO provider must use SSRF protection"
(let [captured-params (atom nil)]
@ -800,7 +1024,7 @@
{:client-id "test-client"
:client-secret "test-secret"
:issuer "https://wrong-idp.example.com"}
:dest-url "https://localhost:3449/#/dashboard"
:query-string "screen=dashboard"
:organization-id #uuid "00000000-0000-0000-0000-000000000001")
(catch Throwable t t))]
(t/is (ex/error? e))

View File

@ -801,6 +801,95 @@
(t/is (cfo/token-applied? token-radius shape [:r1 :r2 :r3 :r4]))
(t/is (cfo/token-applied? token-opacity shape [:opacity]))))))
(t/deftest embed-assets-loads-the-library-components
;; The library is read without `:realize?`, so its `:components` is an
;; unloaded pointer map: `absorb-assets` walks it with `vals`, which calls
;; `load!`, and that raises when no loader is bound. The absorb has to run
;; under the library's own loader. This is a pre-existing failure, not a
;; regression of the migration-on-read work: the migration path returns the
;; pointer already loaded by the reattach comparison.
(let [profile (th/create-profile* 1)
library (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true})
file (th/create-file* 2 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
lib-page (first (get-in library [:data :pages]))
file-page (first (get-in file [:data :pages]))
component-id (uuid/random)
main-id (uuid/random)
copy-id (uuid/random)]
(th/link-file-to-library* {:file-id (:id file) :library-id (:id library)})
;; The library owns one component.
(update-file!
:file-id (:id library)
:profile-id (:id profile)
:changes
[{:type :add-obj
:page-id lib-page
:id main-id
:parent-id uuid/zero
:frame-id uuid/zero
:components-v2 true
:obj (cts/setup-shape
{:id main-id
:name "Board"
:frame-id uuid/zero
:parent-id uuid/zero
:type :frame
:main-instance true
:component-root true
:component-file (:id library)
:component-id component-id})}
{:type :add-component
:path ""
:name "Board"
:main-instance-id main-id
:main-instance-page lib-page
:id component-id
:anotation nil}])
;; The file instances it, so the absorb has a usage to embed.
(update-file!
:file-id (:id file)
:profile-id (:id profile)
:changes
[{:type :add-obj
:page-id file-page
:id copy-id
:parent-id uuid/zero
:frame-id uuid/zero
:components-v2 true
:obj (cts/setup-shape
{:id copy-id
:name "Board"
:frame-id uuid/zero
:parent-id uuid/zero
:type :frame
:main-instance false
:component-root true
:component-file (:id library)
:component-id component-id})}])
(t/testing "a plain library is absorbed"
(let [file-data (-> (bfc/get-file th/*system* (:id file) :realize? true) :data)
result (bfc/embed-assets th/*system* file-data (:id file))]
(t/is (contains? (:components result) component-id))))
(t/testing "a library that needed a migration is absorbed too"
;; The scenario the review reported. It works even without the loader,
;; because `reattach-unchanged` loads the pointers it puts back; it
;; stays pinned so a future reattach that stops loading does not
;; silently break the absorb.
(th/db-delete! :file-migration
{:file-id (:id library)
:name "0031-migrate-stroke-width-token-attr"})
(let [file-data (-> (bfc/get-file th/*system* (:id file) :realize? true) :data)
result (bfc/embed-assets th/*system* file-data (:id file))]
(t/is (contains? (:components result) component-id))))))
(t/deftest export-detach-libraries-preserves-own-tokens-and-status
(let [profile (th/create-profile* 1)
{:keys [library file page-id shape-id token-radius token-opacity]}
@ -1410,26 +1499,23 @@
(t/is (some? file-res))
(t/is (= 1 (count (:done file-res))))))))
(t/deftest link-later-overwrite-import-no-resolution
(t/deftest import-rejects-an-in-place-import
;; The in-place import was removed: it never carried the previous storage
;; pointer, so it could leave the replaced object unreclaimed. A caller that
;; still sets `::bfc/file-id` fails instead of importing as new files.
(let [{:keys [profile file-id team-id]} (import-sample-file)
output (tmp/tempfile :suffix ".zip")]
;; Export file with link-later
(v3/export-files!
(-> th/*system*
(assoc ::bfc/file-ids #{file-id})
(assoc ::bfc/export-type :link-later)
(assoc ::bfc/output (io/output-stream output))))
;; Import with overwrite (file-id set)
(let [result (-> th/*system*
output (tmp/tempfile :suffix ".zip")
cause (try
(-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/team-id team-id)
(assoc ::bfc/file-id file-id)
(assoc ::bfc/input output)
(v3/import-files!))]
;; Overwrite should have empty resolution
(t/is (= {} (:resolution result))))))
(v3/import-files!))
nil
(catch clojure.lang.ExceptionInfo cause cause))]
(t/is (= :not-supported (:code (ex-data cause))))))
;; -----------------------------------------------------------------------------
;; Category 2: Cross-Team Migration
@ -3082,17 +3168,51 @@
(:object-id thumb)))
(t/is (some? (:media-id thumb)))))))
(t/deftest get-file-without-data-rejects-locks
;; Locking is incompatible with `load-data? false`: the guard fires
;; before any query, so no DB row is needed. Regression test for the
;; `lock-for-share?`/`lock-for-share?` typo that left `lock-for-update?`
;; unchecked. NOTE: the guard is an unconditional `when`/`throw` in
;; `get-file*` (not an `assert`), so this test is not flag-dependent:
;; it behaves the same with or without `enable-backend-asserts`.
(doseq [opt [:lock-for-share? :lock-for-update? :skip-locked?]]
(t/deftest get-file-without-data-can-lock
;; Locking is legal without loading the data: the row-only query selects
;; directly from `file`, so `FOR UPDATE`/`FOR SHARE` apply to the row and
;; the lock lives until the enclosing transaction ends. `SKIP LOCKED` only
;; modifies a locking read, so on its own it is ignored. The SQL is pinned
;; because a plain `SELECT` also returns the row: the lock clause is what
;; serializes with the other writers.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})]
(doseq [[opt sql] {:lock-for-share? (str bfc/sql:get-file-without-data " FOR SHARE of f")
:lock-for-update? (str bfc/sql:get-file-without-data " FOR UPDATE of f")
:skip-locked? bfc/sql:get-file-without-data}]
(let [[row sqls] (th/read-recording-sql
#(bfc/get-file th/*system* (:id file) :load-data? false opt true))]
(t/is (= (:id file) (:id row)))
(t/is (nil? (:data row)))
(t/is (= [sql] sqls))))))
(t/deftest get-file-rejects-realize-without-data
;; `:realize?` walks the decoded data; asking for it without loading or
;; decoding the data is a caller bug. The guard fires before any query, so
;; no file row is needed.
(let [id (uuid/random)]
(t/is (thrown? IllegalArgumentException
(bfc/get-file th/*system* (uuid/random)
:load-data? false opt true)))))
(bfc/get-file th/*system* id :realize? true :decode? false)))
(t/is (thrown? IllegalArgumentException
(bfc/get-file th/*system* id :realize? true :load-data? false)))))
(t/deftest get-file-rejects-a-shared-lock-with-data
;; The probe inherits the caller's lock and the migration branch re-reads
;; with FOR UPDATE, so a shared lock on a data read is a lock upgrade. The
;; guard fires before any query, so no row is needed.
(t/is (thrown? IllegalArgumentException
(bfc/get-file th/*system* (uuid/random) :lock-for-share? true))))
(t/deftest get-file-rejects-a-bare-pool-or-connection
;; The read opens a transaction (unless it is read-only) and resolves the
;; cfg, so it takes a cfg map only: a bare pool or connection is a caller
;; bug. The guard is an `IllegalArgumentException`, not an `assert`, which
;; is elided when the backend runs without `:backend-asserts`.
(let [id (uuid/random)]
(t/is (thrown? IllegalArgumentException (bfc/get-file th/*pool* id)))
(with-open [conn (db/open th/*system*)]
(t/is (thrown? IllegalArgumentException (bfc/get-file conn id))))))
;; -----------------------------------------------------------------------------
;; Branch target protection
@ -3101,7 +3221,8 @@
(t/deftest import-overwrite-refuses-branch-file
;; A branch file's data payload is derived from its merge base and op
;; log, so an in-place import would write a `file_data` main row that
;; no read path consults.
;; no read path consults. `import-files!` refuses every in-place import,
;; and this keeps a branch file among the files it refuses.
(with-redefs [cf/flags (conj cf/flags :branching)]
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
@ -3134,7 +3255,7 @@
(t/is (uuid? branch-id))
(t/is (= :validation (:type error)))
(t/is (= :branch-file-cant-be-overwritten (:code error)))
(t/is (= :not-supported (:code error)))
(t/is (empty? (th/db-query :file-data {:file-id branch-id :type "main"})))))))
;; --- Library relations security tests
@ -3197,7 +3318,8 @@
[[(str (:id file)) (str (:id file))]])
[file-id :as result] (import-bundle profile evil)]
(t/is (= 1 (count result)))
(t/is (empty? (th/db-query :file-library-rel {:file-id file-id})))))
(t/is (empty? (th/db-query :file-library-rel {:file-id file-id})))
(t/is (empty? (th/db-query :file-library-sync {:file-id file-id})))))
(t/deftest import-v3-keeps-valid-library-relation
(let [profile (th/create-profile* 1)
@ -3216,6 +3338,251 @@
(t/is (= 1 (count (th/db-query :file-library-rel
{:file-id (:id new-file)})))))))
(t/deftest import-skips-circular-library-relations
;; https://github.com/penpot/penpot/issues/12052: two libraries that use
;; each other (possible before the 2.17.1 guard) export both relations in
;; the manifest; the import must skip the one closing the cycle instead
;; of recreating it. The cycle is built with direct inserts because the
;; UI command `link-file-to-library` already rejects it. Relations are
;; processed in sorted manifest order and the first edge wins.
(let [profile (th/create-profile* 1)
lib-a (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library A"})
lib-b (th/create-file* 2 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library B"})]
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-a)
:library-file-id (:id lib-b)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-b)
:library-file-id (:id lib-a)})
(let [output (tmp/tempfile :suffix ".zip")]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/file-ids #{(:id lib-a) (:id lib-b)})
(assoc ::bfc/export-type :include-libraries)
(assoc ::bfc/output (io/output-stream output))))
(let [result (import-bundle profile output)
rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %})
result)
syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %})
result)]
(t/is (= 2 (count result)))
;; only one of the two cycle edges survives the import
(t/is (= 1 (count rels)))
;; the skipped edge leaves no sync row behind
(t/is (= 1 (count syncs)))
(t/is (= (set (map (juxt :file-id :library-file-id) syncs))
(set (map (juxt :file-id :library-file-id) rels))))
;; no imported file lists itself among its transitive libraries
(doseq [file-id result]
(t/is (not (contains? (bfc/get-libraries th/*system* [file-id])
file-id))))))))
(t/deftest import-skips-transitive-cycle
;; A three-node cycle (A -> B -> C -> A) only breaks when the check uses
;; the transitive closure: a direct-edge-only check would keep all three
;; edges. Built with direct inserts because the UI command rejects it.
(let [profile (th/create-profile* 1)
lib-a (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library A"})
lib-b (th/create-file* 2 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library B"})
lib-c (th/create-file* 3 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library C"})]
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-a)
:library-file-id (:id lib-b)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-b)
:library-file-id (:id lib-c)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-c)
:library-file-id (:id lib-a)})
(let [output (tmp/tempfile :suffix ".zip")]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/file-ids #{(:id lib-a) (:id lib-b) (:id lib-c)})
(assoc ::bfc/export-type :include-libraries)
(assoc ::bfc/output (io/output-stream output))))
(let [result (import-bundle profile output)
rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %})
result)
syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %})
result)]
(t/is (= 3 (count result)))
;; only two of the three cycle edges survive the import
(t/is (= 2 (count rels)))
;; the skipped edge leaves no sync row behind
(t/is (= 2 (count syncs)))
(t/is (= (set (map (juxt :file-id :library-file-id) syncs))
(set (map (juxt :file-id :library-file-id) rels))))
;; no imported file lists itself among its transitive libraries
(doseq [file-id result]
(t/is (not (contains? (bfc/get-libraries th/*system* [file-id])
file-id))))))))
(t/deftest import-keeps-acyclic-library-relations
;; Control test: a chain without cycles must be imported untouched, so
;; the new guard only skips edges that would close a cycle.
(let [profile (th/create-profile* 1)
lib-a (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library A"})
lib-b (th/create-file* 2 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library B"})
lib-c (th/create-file* 3 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library C"})]
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-a)
:library-file-id (:id lib-b)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-b)
:library-file-id (:id lib-c)})
(let [output (tmp/tempfile :suffix ".zip")]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/file-ids #{(:id lib-a) (:id lib-b) (:id lib-c)})
(assoc ::bfc/export-type :include-libraries)
(assoc ::bfc/output (io/output-stream output))))
(let [result (import-bundle profile output)
rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %})
result)
syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %})
result)]
(t/is (= 3 (count result)))
(t/is (= 2 (count rels)))
;; every kept relation has its sync row
(t/is (= 2 (count syncs)))
(t/is (= (set (map (juxt :file-id :library-file-id) syncs))
(set (map (juxt :file-id :library-file-id) rels))))))))
(t/deftest import-v1-skips-circular-library-relations
;; Issue #12052 also names the version 1 importer: importing a package
;; whose libraries use each other must not recreate the cycle. The cycle
;; is built with direct inserts because `link-file-to-library` already
;; rejects it. The exported file only *uses* one of the libraries, so the
;; package carries the cycle the same way a real export does.
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
lib-a (th/create-file* 2 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library A"})
lib-b (th/create-file* 3 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library B"})]
(th/link-file-to-library* {:file-id (:id file)
:library-id (:id lib-a)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-a)
:library-file-id (:id lib-b)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-b)
:library-file-id (:id lib-a)})
(let [output (tmp/tempfile :suffix ".penpot")]
(v1/export-files!
(-> th/*system*
(assoc ::bfc/file-ids #{(:id file)})
(assoc ::bfc/include-libraries true))
(io/output-stream output))
(let [result (:file-ids (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output)
(v1/import-files!)))
rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %})
result)
syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %})
result)]
(t/is (= 3 (count result)))
;; the relation of the file to its library, plus one of the two
;; cycle edges; the edge that would close the cycle is skipped
(t/is (= 2 (count rels)))
;; the skipped edge leaves no sync row behind
(t/is (= 2 (count syncs)))
(t/is (= (set (map (juxt :file-id :library-file-id) syncs))
(set (map (juxt :file-id :library-file-id) rels))))
;; no imported file lists itself among its transitive libraries
(doseq [file-id result]
(t/is (not (contains? (bfc/get-libraries th/*system* [file-id])
file-id))))))))
(t/deftest import-v1-tolerates-repeated-file-entries
;; Old v1 exports could list the same file twice in the package metadata:
;; the exporter unioned the requested files with their libraries without
;; deduplicating, and a file inside a library cycle is one of its own
;; libraries. The import must read every entry and import each file once.
(let [profile (th/create-profile* 1)
lib-a (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library A"})
lib-b (th/create-file* 2 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared true
:name "Library B"})]
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-a)
:library-file-id (:id lib-b)})
(db/insert! th/*system* :file-library-rel
{:file-id (:id lib-b)
:library-file-id (:id lib-a)})
(let [output (tmp/tempfile :suffix ".penpot")]
;; exporting only lib-a repeats its id in the metadata, because it is
;; among its own transitive libraries
(v1/export-files!
(-> th/*system*
(assoc ::bfc/file-ids #{(:id lib-a)})
(assoc ::bfc/include-libraries true))
(io/output-stream output))
(let [result (:file-ids (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output)
(v1/import-files!)))
rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %})
result)
syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %})
result)]
;; two files: the repeated entry is imported once
(t/is (= 2 (count result)))
;; one of the two cycle edges survives the import
(t/is (= 1 (count rels)))
(t/is (= 1 (count syncs)))
(t/is (= (set (map (juxt :file-id :library-file-id) syncs))
(set (map (juxt :file-id :library-file-id) rels))))
;; no imported file lists itself among its transitive libraries
(doseq [file-id result]
(t/is (not (contains? (bfc/get-libraries th/*system* [file-id])
file-id))))))))
(t/deftest export-multi-file-includes-package-relations
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
@ -3314,3 +3681,45 @@
(t/is (empty? (th/db-query :file-library-sync {:library-file-id (:id victim)})))
(t/is (empty? (th/db-query :file-library-rel {:library-file-id (:id file)})))
(t/is (empty? (th/db-query :file-library-sync {:library-file-id (:id file)}))))))
(t/deftest file-permissions-ignore-soft-deleted
;; sql:file-permissions must not grant anything through a file,
;; project or team that is soft-deleted (GHSA-m53j-2766-6jqw: the
;; websocket re-check relies on this query, and the subscribe handler
;; additionally refuses deleted files via check-file-exists).
(let [owner (th/create-profile* 1 {:is-active true})
editor (th/create-profile* 2 {:is-active true})
team (th/create-team* 1 {:profile-id (:id owner)})]
(th/create-team-role* {:team-id (:id team)
:profile-id (:id editor)
:role :editor})
(let [project (th/create-project* 1 {:profile-id (:id editor)
:team-id (:id team)})
file (th/create-file* 1 {:profile-id (:id editor)
:project-id (:id project)})
perms? #(some? (bfc/get-file-permissions th/*system* (:id editor) (:id file)))
undelete (fn [table id]
(db/update! th/*system* table {:deleted-at nil} {:id id}))
soft-delete (fn [table id]
(db/update! th/*system* table {:deleted-at (ct/now)} {:id id}))]
(t/testing "a live member can read the live file"
(t/is (perms?)))
(t/testing "a soft-deleted file grants nothing"
(soft-delete :file (:id file))
(t/is (nil? (bfc/get-file-permissions th/*system* (:id editor) (:id file))))
(undelete :file (:id file))
(t/is (perms?)))
(t/testing "a soft-deleted project grants nothing"
(soft-delete :project (:id project))
(t/is (nil? (bfc/get-file-permissions th/*system* (:id editor) (:id file))))
(undelete :project (:id project))
(t/is (perms?)))
(t/testing "a soft-deleted team grants nothing"
(soft-delete :team (:id team))
(t/is (nil? (bfc/get-file-permissions th/*system* (:id editor) (:id file))))
(undelete :team (:id team))
(t/is (perms?))))))

View File

@ -452,10 +452,9 @@
(t/is (thrown? clojure.lang.ExceptionInfo
(invoke! cfg (fn [_ _] :never) {}))))
;; the count came back down instead of staying up for good. Only the
;; last publication is asserted, not the exact list: the sampler
;; republishes the same count when its first tick lands inside the
;; window.
;; the count came back down instead of staying up for good. The
;; republisher daemon thread can add publications inside the window,
;; so what is asserted is the last value, not the whole sequence.
(t/is (= 0 (waiting-count climit name)))
(t/is (= 0 (waiting-last name)))

View File

@ -6,9 +6,11 @@
(ns backend-tests.config-test
(:require
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.config :as cf]
[clojure.core :as c]
[clojure.test :as t]))
(t/deftest tenant-is-normalized-to-lowercase
@ -86,3 +88,51 @@
(t/testing "and falls back to a week"
(with-redefs [cf/config (dissoc cf/config :jobs-user-ttl)]
(t/is (= (ct/duration {:days 7}) (cf/get-jobs-user-ttl))))))
;; `legacy-db` is read-only: the file data moved to `file_data`, and
;; `file.data` / `file_change.data` are read-only leftovers. An instance
;; that keeps it starts healthy and then fails on every save. It is read
;; as `db` instead, which also serves the legacy rows.
(defn- read-config
"`read-config` with the environment replaced by `env`."
[env & {:keys [default] :or {default {}}}]
(with-redefs [cf/read-env (constantly env)]
(cf/read-config :default default)))
(defn- read-config-with-warning
"Reads the configuration and collects the warnings it emits. Logging is
emitted asynchronously, so the record is awaited (bounded)."
[env]
(let [records (atom [])]
(add-watch l/log-record ::capture
(fn [_ _ _ record] (swap! records conj record)))
(try
(let [config (read-config env)]
[(loop [remaining 100]
(let [warns (filter #(= :warn (::l/level %)) @records)]
(cond
(seq warns) warns
(zero? remaining) warns
:else (do (Thread/sleep 10) (recur (dec remaining))))))
config])
(finally
(remove-watch l/log-record ::capture)))))
(t/deftest read-config-replaces-the-read-only-legacy-db
(let [[warns config] (read-config-with-warning {:file-data-backend "legacy-db"})]
(t/testing "the configuration writes through db"
(t/is (= "db" (c/get config :file-data-backend))))
(t/testing "the operator is warned"
(t/is (contains? (into #{} (map #(c/get (::l/props %) :hint)) warns)
"file-data-backend=legacy-db is read-only, using db")))))
(t/deftest read-config-keeps-the-writable-back-ends
(doseq [backend ["db" "storage"]]
(t/testing backend
(t/is (= backend (c/get (read-config {:file-data-backend backend})
:file-data-backend))))))
(t/deftest read-config-takes-the-default-back-end
(t/is (= "db" (c/get (read-config {} :default cf/default)
:file-data-backend))))

View File

@ -80,3 +80,11 @@
(t/deftest uuid->hash-code-stable-for-zero-uuid
(t/is (= 3659997967308761462 (db/uuid->hash-code uuid/zero))))
(t/deftest pool-enables-the-partitionwise-join
;; The pool sets the GUC on every connection, so the planner can resolve a
;; join between two partitioned tables pairwise, partition by partition.
(let [row (db/exec-one! th/*pool*
["SELECT current_setting('enable_partitionwise_join') AS partitionwise_join"])]
(t/is (= "on" (:partitionwise-join row)))))

View File

@ -0,0 +1,42 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns backend-tests.fdata-resolve-test
"Resolution of file data by backend (`app.features.fdata/resolve-file-data`).
A row without a `file_data` twin carries no backend: the read queries mark
it with `coalesce(fd.backend, 'legacy-db')` and hand its blob over as
`:legacy-data`. The `legacy-db` backend is still supported, so that case
has to resolve to the blob instead of nil."
(:require
[app.features.fdata :as fdata]
[clojure.test :as t]))
(t/deftest legacy-db-resolves-the-legacy-blob
(let [blob (vec (.getBytes "legacy-blob" "UTF-8"))
file {:backend "legacy-db"
:legacy-data (byte-array blob)}
out (fdata/resolve-file-data nil file)]
(t/is (= blob (vec (:data out))))
(t/is (not (contains? out :legacy-data)))))
(t/deftest missing-backend-defaults-to-legacy-db
;; The queries leave `:backend` nil when the `LEFT JOIN` with `file_data`
;; finds no twin; the dispatch default keeps that row on `legacy-db`.
(let [blob (vec (.getBytes "legacy-blob" "UTF-8"))
file {:legacy-data (byte-array blob)}
out (fdata/resolve-file-data nil file)]
(t/is (= blob (vec (:data out))))
(t/is (not (contains? out :legacy-data)))))
(t/deftest db-backend-drops-the-legacy-key
(let [data (vec (.getBytes "db-blob" "UTF-8"))
file {:backend "db"
:data (byte-array data)
:legacy-data (byte-array (vec (.getBytes "stale" "UTF-8")))}
out (fdata/resolve-file-data nil file)]
(t/is (= data (vec (:data out))))
(t/is (not (contains? out :legacy-data)))))

View File

@ -0,0 +1,127 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns backend-tests.fdata-upsert-test
"Covers the write contract of `fdata/upsert!` after the `legacy-db` backend
became read-only."
(:require
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.features.fdata :as fdata]
[backend-tests.helpers :as th]
[clojure.test :as t]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(defn- upsert-cause
[params]
(try
(fdata/upsert! th/*system* params)
nil
(catch Exception cause
cause)))
(t/deftest legacy-db-param-is-rejected
;; `legacy-db` is not a write backend anymore: the schema refuses it before
;; touching the database.
(let [cause (upsert-cause {:id (uuid/next)
:file-id (uuid/next)
:type "snapshot"
:backend "legacy-db"
:storage-object-id nil
:data (byte-array [1 2 3])})]
(t/is (some? cause))
(t/is (= :assertion (:type (ex-data cause))))
(t/is (= :data-validation (:code (ex-data cause))))))
(t/deftest upsert-reports-the-pointer-it-created
;; The write reports the object it created, so a caller that persists again
;; touches the current pointer instead of the replaced one.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
data (byte-array [1 2 3])]
(let [out (db/tx-run! th/*system*
(fn [cfg]
(fdata/upsert! cfg {:id (:id file)
:file-id (:id file)
:type "main"
:backend "storage"
:storage-object-id nil
:data data})))]
(t/testing "a storage write reports the object it created"
(t/is (true? (:written out)))
(t/is (uuid? (:storage-object-id out)))
(t/is (= (:storage-object-id out)
(:storage-object-id (th/db-get :file-data
{:file-id (:id file)
:id (:id file)}))))))
(let [previous (:storage-object-id (th/db-get :file-data
{:file-id (:id file)
:id (:id file)}))
out (db/tx-run! th/*system*
(fn [cfg]
(fdata/upsert! cfg {:id (:id file)
:file-id (:id file)
:type "main"
:backend "db"
:storage-object-id previous
:data data})))]
(t/testing "a db write has no object to report"
(t/is (true? (:written out)))
(t/is (nil? (:storage-object-id out)))
(t/is (nil? (:storage-object-id (th/db-get :file-data
{:file-id (:id file)
:id (:id file)}))))))))
(t/deftest upsert-rejects-a-stale-pointer
;; The write only replaces the row whose pointer the caller read. A stale
;; map must fail instead of touching the wrong object and leaking the blob.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
data (byte-array [1 2 3])
before (th/db-get :file-data {:file-id (:id file) :id (:id file)})
cause (try
(db/tx-run! th/*system*
(fn [cfg]
(fdata/upsert! cfg {:id (:id file)
:file-id (:id file)
:type "main"
:backend "db"
:storage-object-id (uuid/next)
:data data})))
nil
(catch clojure.lang.ExceptionInfo cause cause))]
(t/is (= :stale-file-data-pointer (:code (ex-data cause))))
(let [after (th/db-get :file-data {:file-id (:id file) :id (:id file)})]
(t/testing "the row is untouched"
(t/is (= (:backend before) (:backend after)))
(t/is (= (:storage-object-id before) (:storage-object-id after)))
(t/is (= (vec (:data before)) (vec (:data after))))))))
(t/deftest legacy-db-config-is-rejected-as-read-only
;; The configuration value is kept for reading old data, but a write with
;; it as the default backend fails with a clear message.
(with-redefs [cf/config (assoc cf/config :file-data-backend "legacy-db")]
(let [cause (upsert-cause {:id (uuid/next)
:file-id (uuid/next)
:type "snapshot"
:storage-object-id nil
:data (byte-array [1 2 3])})]
(t/is (some? cause))
(t/is (= :validation (:type (ex-data cause))))
(t/is (= :legacy-backend-read-only (:code (ex-data cause)))))))
(t/deftest file-data-backend-defaults-to-db
(t/is (= "db" (cf/get :file-data-backend))))

View File

@ -10,7 +10,7 @@
A file has two kinds of row. Durable history: the snapshots, which a restore
brings back. And ephemeral revisions: the xlog of an edit, born already
scheduled for deletion one hour later, which nobody needs once the sessions
that were going to read them are gone.
that were going to read them are gone. The `kind` column tells them apart.
The invariant under test is that a deletion date is assigned once and never
moved: restoring a file must not unmark a revision, and deleting a file must
@ -21,6 +21,7 @@
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.db :as db]
[app.features.file-changes :as fchanges]
[app.rpc :as-alias rpc]
[app.rpc.commands.files :as files]
[app.util.blob :as blob]
@ -64,6 +65,7 @@
:version 0
:created-by (or created-by "system")
:label label
:kind (if label fchanges/snapshot-kind fchanges/revision-kind)
:data data
:changes changes}
{::db/return-keys false})
@ -170,6 +172,7 @@
out (th/command! {::th/type :update-file-snapshot
::rpc/profile-id (:id profile)
:file-id file-id
:id revision-id
:label "renamed"})]

View File

@ -0,0 +1,115 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns backend-tests.file-change-schema-test
(:require
[app.db :as db]
[backend-tests.helpers :as th]
[clojure.test :as t]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(defn- query
"Runs a query that returns a single value and returns it."
[sql & params]
(-> (db/exec! th/*pool* (into [sql] params))
first
(vals)
first))
(defn- partition-names
[]
(->> (db/exec! th/*pool* ["SELECT c.relname AS name
FROM pg_class c
JOIN pg_partition_tree('file_change') t ON t.relid = c.oid
WHERE t.isleaf
ORDER BY c.relname"])
(mapv :name)))
(t/deftest file-change-is-partitioned-in-sixteen-named-partitions
(t/is (= (mapv #(format "file_change_%02d" %) (range 16))
(partition-names))))
(t/deftest file-change-primary-key-is-file-id-and-id
(t/is (= "PRIMARY KEY (file_id, id)"
(query "SELECT pg_get_constraintdef(oid) AS def
FROM pg_constraint
WHERE conrelid = 'file_change'::regclass AND contype = 'p'"))))
(t/deftest file-change-has-three-foreign-keys-and-one-check
(t/is (= 3 (query "SELECT count(*) AS total
FROM pg_constraint
WHERE conrelid = 'file_change'::regclass AND contype = 'f'")))
(t/is (= 1 (query "SELECT count(*) AS total
FROM pg_constraint
WHERE conrelid = 'file_change'::regclass AND contype = 'c'")))
(t/is (re-find #"kind"
(query "SELECT pg_get_constraintdef(oid) AS def
FROM pg_constraint
WHERE conrelid = 'file_change'::regclass AND contype = 'c'"))))
(t/deftest file-change-has-the-six-canonical-indexes
(t/is (= ["file_change__deleted_at__idx"
"file_change__file_id__revn__idx"
"file_change__locked_by__idx"
"file_change__profile_id__idx"
"file_change__user_snapshots__idx"
"file_change_pkey"]
(->> (db/exec! th/*pool* ["SELECT indexname AS name
FROM pg_indexes
WHERE tablename = 'file_change'
AND schemaname = current_schema()
ORDER BY indexname"])
(mapv :name)))))
(t/deftest file-change-keeps-the-data-column-and-a-kind-column
;; The `data` column is kept: the `legacy-db` backend reads old payloads
;; from it (migration 0157 moved the existing blobs). The dead
;; `data_ref_id`/`data_backend` columns stay gone: no code read them since
;; the file storage refactor.
(t/is (= []
(->> (db/exec! th/*pool* ["SELECT attname AS name
FROM pg_attribute
WHERE attrelid = 'file_change'::regclass
AND attnum > 0 AND NOT attisdropped
AND attname IN ('data_ref_id', 'data_backend')"])
(mapv :name))))
(t/is (= ["bytea"]
(->> (db/exec! th/*pool* ["SELECT pg_catalog.format_type(atttypid, atttypmod) AS type
FROM pg_attribute
WHERE attrelid = 'file_change'::regclass
AND attname = 'data'"])
(mapv :type))))
;; Nullable on purpose: a rollback to the previous version inserts
;; without `kind` and has to keep writing (migration 0158).
(t/is (false? (query "SELECT attnotnull AS notnull
FROM pg_attribute
WHERE attrelid = 'file_change'::regclass
AND attname = 'kind'"))))
(t/deftest inserting-a-row-without-a-kind-succeeds
;; The column is nullable on purpose: the previous version inserts without
;; it, and a rollback to that version has to keep writing. The backfill of
;; a later NOT NULL migration turns these NULLs into 1/2.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})]
(db/exec! th/*pool* ["INSERT INTO file_change (file_id, id)
VALUES (?, gen_random_uuid())"
(:id file)])
(t/is (nil? (query "SELECT kind AS k
FROM file_change
WHERE file_id = ?" (:id file))))))
(t/deftest inserting-a-row-with-an-unknown-kind-fails
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})]
(t/is (thrown? java.sql.SQLException
(db/exec! th/*pool* ["INSERT INTO file_change (file_id, id, kind)
VALUES (?, gen_random_uuid(), 3)"
(:id file)])))))

View File

@ -0,0 +1,323 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns backend-tests.file-data-test
(:require
[app.binfile.common :as bfc]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.features.fdata :as fdata]
[app.storage :as sto]
[app.util.blob :as blob]
[backend-tests.helpers :as th]
[clojure.test :as t]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
;; The storage pointer of a `file_data` row is a column with a foreign key and
;; it is the only source of truth: the release is a coordinated restart and
;; the metadata no longer carries `storage-ref-id`.
(defn- find-object
[id]
(th/db-get :storage-object {:id id} {::db/remove-deleted false}))
(defn- put-object!
([bucket metadata content]
(sto/put-object! (::sto/storage th/*system*)
(merge {::sto/content content
:content-type "application/octet-stream"
:bucket bucket}
metadata)))
([bucket metadata]
(put-object! bucket metadata (sto/content "sobject"))))
(defn- create-file!
[]
(let [profile (th/create-profile* 1)]
(th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})))
(defn- upsert!
[params]
(db/tx-run! th/*system*
(fn [cfg]
(fdata/upsert! cfg params))))
(defn- resolve!
[row]
(db/tx-run! th/*system*
(fn [cfg]
(fdata/resolve-file-data cfg row))))
(t/deftest upsert-to-storage-writes-the-column
(let [file (create-file!)
data (blob/encode {:pages [] :pages-index {}})]
;; `main` always carries `id = file_id`: update the row the file already
;; has instead of inventing a second one.
(upsert! {:id (:id file)
:file-id (:id file)
:type "main"
:backend "storage"
:storage-object-id nil
:data data
:metadata {:generated-by "test"}})
(let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})
metadata (fdata/decode-metadata (:metadata row))
sobject (:storage-object-id row)]
(t/testing "the row points at the object with the column"
(t/is (= "storage" (:backend row)))
(t/is (nil? (:data row)))
(t/is (uuid? sobject)))
(t/testing "the metadata no longer carries the pointer"
(t/is (nil? (:storage-ref-id metadata)))
(t/is (= "test" (:generated-by metadata))))
(t/testing "the object is stored in the file-data bucket"
(t/is (= "file-data" (:bucket (meta (sto/get-object (::sto/storage th/*system*)
sobject))))))
(t/testing "the row resolves to the stored bytes"
(t/is (= (vec data) (vec (:data (resolve! row)))))))))
(t/deftest upsert-back-to-db-clears-the-column-and-touches-the-object
(let [file (create-file!)]
(upsert! {:id (:id file)
:file-id (:id file)
:type "main"
:backend "storage"
:storage-object-id nil
:data (blob/encode {:pages [] :pages-index {}})})
(let [sobject (:storage-object-id (th/db-get :file-data
{:file-id (:id file) :id (:id file)}))]
;; The caller passes the object the persisted row points at; `upsert!`
;; touches it before replacing the row. The dead metadata pointer must
;; not survive the write either.
(upsert! {:id (:id file)
:file-id (:id file)
:type "main"
:backend "db"
:storage-object-id sobject
:data (blob/encode {:pages [] :pages-index {}})
:metadata {:generated-by "test"
:storage-ref-id (uuid/next)}})
(let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})
metadata (fdata/decode-metadata (:metadata row))]
(t/testing "the hot row no longer references the object"
(t/is (= "db" (:backend row)))
(t/is (nil? (:storage-object-id row)))
(t/is (nil? (:storage-ref-id metadata)))
(t/is (= "test" (:generated-by metadata))))
(t/testing "the replaced object is touched for the storage GC"
(t/is (some? (:touched-at (find-object sobject)))))))))
(t/deftest resolve-file-data-uses-the-column-and-ignores-the-metadata
(let [file (create-file!)
data (blob/encode {:pages [] :pages-index {}})
obj (put-object! "file-data"
{:file-id (:id file) :id (:id file)}
(sto/content data))
;; A different, well-formed pointer in the metadata: it must be
;; ignored now that the column is the only source of truth.
other (put-object! "file-data"
{:file-id (uuid/next) :id (uuid/next)})]
(th/db-update! :file-data
{:backend "storage"
:data nil
:storage-object-id (:id obj)
:metadata (db/json {"storage-ref-id" (str (:id other))})}
{:file-id (:id file) :id (:id file)})
(t/testing "the column wins over the metadata"
(let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})]
(t/is (= (:id obj) (:storage-object-id row)))
(t/is (= (vec data) (vec (:data (resolve! row)))))))))
(t/deftest gc-touched-freezes-an-object-referenced-only-by-the-column
(let [file (create-file!)
;; The object metadata points at other ids: the freeze must come from
;; the column, not from the old metadata lookup.
sobject (put-object! "file-data"
{:file-id (uuid/next) :id (uuid/next)})]
(th/db-update! :storage-object
{:touched-at (ct/in-past {:hours 3})}
{:id (:id sobject)})
(th/db-update! :file-data
{:backend "storage"
:data nil
:storage-object-id (:id sobject)}
{:file-id (:id file) :id (:id file)})
(let [result (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:freeze result)))
(t/is (= 0 (:delete result)))
(t/is (nil? (:touched-at (find-object (:id sobject))))))))
(t/deftest update-file!-ignores-the-pointer-in-the-params
;; `:storage-object-id` is bookkeeping of the persisted row, not an editable
;; field: a param must not be able to drop the pointer and skip the touch.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})]
(th/run-task! :offload-file-data {:file-id (:id file)})
(th/run-task! :storage-gc-touched {:skip-delay true})
(let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file)
:id (:id file)}))]
(t/is (uuid? sobject))
(t/is (nil? (:touched-at (find-object sobject))))
(db/tx-run! th/*system*
(fn [cfg]
(let [file (bfc/get-file cfg (:id file)
:migrate? false
:lock-for-update? true)]
(bfc/update-file! cfg file
:storage-object-id nil
:modified-at (ct/now)))))
(t/testing "the previous object is still touched"
(t/is (some? (:touched-at (find-object sobject))))))))
(t/deftest resolve-file-data-reports-a-missing-storage-object
(let [file (create-file!)]
(th/db-update! :file-data
{:backend "storage"
:data nil
:storage-object-id nil}
{:file-id (:id file) :id (:id file)})
(let [cause (try
(db/tx-run! th/*system*
(fn [cfg]
(->> (th/db-get :file-data {:file-id (:id file)
:id (:id file)})
(fdata/resolve-file-data cfg))))
nil
(catch clojure.lang.ExceptionInfo cause cause))]
(t/is (some? cause))
(t/is (= :missing-storage-object (:code (ex-data cause))))
(t/is (= (:id file) (:file-id (ex-data cause)))))))
(t/deftest load-pointer-reports-the-file-when-the-object-is-gone
(let [file (create-file!)
fragment (first (th/db-query :file-data {:file-id (:id file)
:type "fragment"}))
sobject (put-object! "file-data"
{:file-id (:id file) :id (:id fragment)})]
(t/is (uuid? (:id fragment)))
;; The foreign key accepts the row (the object exists); a soft-deleted
;; object is simply unreadable, which is the case the error reports.
(th/db-update! :storage-object
{:deleted-at (ct/in-past "1h")}
{:id (:id sobject)})
(th/db-update! :file-data
{:backend "storage"
:data nil
:storage-object-id (:id sobject)}
{:file-id (:id file) :id (:id fragment)})
(let [cause (try
(db/tx-run! th/*system*
(fn [cfg]
(fdata/load-pointer cfg (:id file) (:id fragment))))
nil
(catch clojure.lang.ExceptionInfo cause cause))
data (ex-data cause)]
(t/is (some? cause))
(t/is (= :missing-storage-object (:code data)))
(t/is (= (:id file) (:file-id data)))
(t/is (= (:id fragment) (:id data))))))
(t/deftest update-file!-rejects-a-map-without-data
;; A map read with `:load-data? false` has no `:data` and no storage pointer;
;; persisting it would wipe the content and skip the touch. The guard is an
;; explicit raise, not an assert, so it also exists in production.
(let [file (create-file!)
cause (try
(db/tx-run! th/*system*
(fn [cfg]
(bfc/update-file! cfg (bfc/get-file cfg (:id file)
:load-data? false))))
nil
(catch clojure.lang.ExceptionInfo cause cause))]
(t/is (some? cause))
(t/is (= :missing-file-data (:code (ex-data cause))))))
(t/deftest update-file!-ignores-the-identity-in-the-params
;; The params change mutable fields only: `:id` cannot retarget the UPDATE.
(let [file (create-file!)
other (uuid/next)
when (ct/inst "2020-01-01T00:00:00Z")]
(db/tx-run! th/*system*
(fn [cfg]
(let [file (bfc/get-file cfg (:id file)
:migrate? false
:lock-for-update? true)]
(bfc/update-file! cfg file :id other :modified-at when))))
(t/testing "the read file is updated"
(t/is (= when (:modified-at (th/db-get :file {:id (:id file)})))))
(t/testing "the id in the params is ignored"
(t/is (nil? (th/db-get :file {:id other}))))))
(t/deftest update-file!-touches-the-previous-object-on-a-storage-write
;; With `:file-data-backend` = "storage", every ordinary save replaces the
;; object of the row: the previous one must be touched so the storage GC can
;; reclaim it. The storage -> db and db -> storage directions are covered
;; above; this pins the configuration where storage is the default.
(with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})]
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
previous (:storage-object-id (th/db-get :file-data
{:file-id (:id file) :id (:id file)}))]
(t/is (uuid? previous))
;; A new object is born touched; clear the mark so the assertion below
;; proves the save is what touches it.
(th/db-update! :storage-object {:touched-at nil} {:id previous})
(t/is (nil? (:touched-at (find-object previous))))
(db/tx-run! th/*system*
(fn [cfg]
(let [file (bfc/get-file cfg (:id file)
:migrate? false
:lock-for-update? true)]
(bfc/update-file! cfg file :modified-at (ct/now)))))
(let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})
current (:storage-object-id row)]
(t/testing "the row points at a new object"
(t/is (uuid? current))
(t/is (not= previous current)))
(t/testing "the previous object is touched for the storage GC"
(t/is (some? (:touched-at (find-object previous)))))))))

View File

@ -0,0 +1,365 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns backend-tests.file-migration-read-test
"Persisting the migrations applied on read is an optimization of the read:
the read already returns migrated data, so the write must never overwrite a
row written after that read, and it must keep a single decoded copy of the
file in the heap."
(:require
[app.binfile.common :as bfc]
[app.common.features :as cfeat]
[app.common.files.migrations :as fmg]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.features.fdata :as fdata]
[app.features.file-migrations :as fmigr]
[app.rpc :as-alias rpc]
[app.util.pointer-map :as pmap]
[backend-tests.helpers :as th]
[clojure.test :as t]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(def ^:private migration-name
"0031-migrate-stroke-width-token-attr")
(defn- create-file!
[]
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared false})]
{:profile-id (:id profile)
:file-id (:id file)
:page-id (first (:pages (:data (bfc/get-file th/*system* (:id file)))))}))
(defn- force-migration!
"Make the next read see a pending migration, the way the snapshot tests do:
the version matches, so only a missing row triggers the migration."
[file-id]
(th/db-delete! :file-migration
{:file-id file-id
:name migration-name}))
(defn- file-data-row
[file-id]
(th/db-get :file-data {:file-id file-id :id file-id}))
(defn- storage-object
[id]
(th/db-get :storage-object {:id id} {::db/remove-deleted false}))
(defn- save!
"Run a real save request so the test exercises the same write path a client
takes, not a hand-built row."
[profile-id file-id revn changes]
(th/command! {::th/type :update-file
::rpc/profile-id profile-id
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features (cfeat/get-enabled-features cf/flags)
:changes changes}))
(t/deftest migrate-on-read-does-not-overwrite-a-concurrent-save
;; A save commits after the read that saw the pending migration. The read
;; must decide again on the map it takes under the `file` row lock, so it
;; writes nothing: the save survives, `revn` never goes back (a client whose
;; revn is higher than the stored one cannot save again) and the object the
;; save left stays referenced instead of becoming a frozen orphan.
(with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})]
(let [{:keys [profile-id file-id page-id]} (create-file!)
shape-id (uuid/random)
fired? (atom false)
replaced (atom nil)
saved-at (atom nil)
saved-revn (atom nil)]
(force-migration! file-id)
(with-redefs [fmigr/resolve-applied-migrations
(let [orig fmigr/resolve-applied-migrations]
(fn [cfg file]
(let [result (orig cfg file)]
(when (compare-and-set! fired? false true)
;; The save commits once the read has already
;; seen the pending migration. It applies the
;; migration on its own and records it.
(let [out (save! profile-id file-id 0
[(th/add-rect-change page-id shape-id)])]
(t/is (nil? (:error out))))
(let [row (file-data-row file-id)]
(reset! replaced (:storage-object-id row))
(reset! saved-revn (:revn (th/db-get :file {:id file-id})))
(reset! saved-at (:modified-at (th/db-get :file {:id file-id}))))
;; Freeze the object the save left, as the storage
;; GC would if it saw it before a stale write
;; replaced the row again: a write that does not
;; touch it leaks the blob forever.
(th/db-update! :storage-object
{:touched-at nil}
{:id @replaced}))
result)))]
(let [returned (db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))
row (th/db-get :file {:id file-id})
persisted (db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id
:migrate? false
:realize? true)))]
(t/testing "the save survives the read"
(t/is (= @saved-revn (:revn returned)))
(t/is (= @saved-revn (:revn row)))
(t/is (= @saved-at (:modified-at row)))
(t/is (some? (get-in persisted [:data :pages-index page-id :objects shape-id]))))
(t/testing "the read writes nothing and the file ends migrated"
(t/is (not (fmg/need-migration? returned)))
(t/is (= @replaced (:storage-object-id (file-data-row file-id))))
(t/is (seq (th/db-query :file-migration
{:file-id file-id
:name migration-name}))))
(t/testing "the next save of the same client does not conflict"
(let [out (save! profile-id file-id @saved-revn
[(th/add-rect-change page-id (uuid/random))])]
(t/is (nil? (:error out))))))))))
(t/deftest migration-already-applied-by-another-reader-is-not-persisted
;; Two readers see the migration pending; the first to get the lock migrates
;; and persists. The second must re-check on its locked read and write
;; nothing (a second write would create another object for the same data).
(with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})]
(let [{:keys [file-id]} (create-file!)
fired? (atom false)
pointer (atom nil)]
(force-migration! file-id)
(with-redefs [fmg/need-migration?
(let [orig fmg/need-migration?]
(fn [file]
(let [result (orig file)]
(when (and result (compare-and-set! fired? false true))
;; The other reader wins the lock, migrates and
;; persists before this one gets it.
(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))
(reset! pointer (:storage-object-id (file-data-row file-id))))
result)))]
(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))
(t/testing "the reader that lost the race does not write"
(t/is (uuid? @pointer))
(t/is (= @pointer (:storage-object-id (file-data-row file-id)))))
(t/testing "the file ends migrated"
(t/is (seq (th/db-query :file-migration
{:file-id file-id
:name migration-name}))))))))
(t/deftest migration-read-loads-the-file-data-once
;; The probe decides with a row read without data, so the migration path
;; keeps a single decoded copy of the file in the heap (a large file can be
;; hundreds of MB and a second copy can exhaust it).
(let [{:keys [file-id]} (create-file!)
reads (atom 0)]
(force-migration! file-id)
(with-redefs [fdata/resolve-file-data
(let [orig fdata/resolve-file-data]
(fn [cfg file]
(when (= file-id (:id file))
(swap! reads inc))
(orig cfg file)))]
(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))
(t/is (= 1 @reads)))))
(t/deftest realize-runs-once-per-read-path
;; `fdata/realize` is a full walk of the decoded data. The read owns it once
;; per path; the migration path pays it twice on purpose, once inside
;; `migrate-file` (the chain needs plain data) and once on its result.
(let [{:keys [file-id]} (create-file!)
calls (atom 0)
read! (fn [opts]
(reset! calls 0)
(with-redefs [fdata/realize
(let [orig fdata/realize]
(fn [cfg file]
(swap! calls inc)
(orig cfg file)))]
(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id opts)))))]
(t/testing "a plain read realizes once"
(read! {:realize? true})
(t/is (= 1 @calls)))
(t/testing "a read-only read realizes once"
(read! {:realize? true :read-only? true})
(t/is (= 1 @calls)))
(t/testing "the migration path realizes twice"
(force-migration! file-id)
(read! {:realize? true})
(t/is (= 2 @calls)))))
(t/deftest row-only-read-resolves-the-applied-migrations-on-demand
;; A row-only read resolves the applied migrations by default, because the
;; row can be completed later and the caller may need them; the explicit
;; opt-out leaves them out.
(let [{:keys [file-id]} (create-file!)
with-migrations (bfc/get-file th/*system* file-id :load-data? false)
without (bfc/get-file th/*system* file-id
:load-data? false
:load-migrations? false)]
(t/is (seq (:migrations with-migrations)))
(t/is (nil? (:migrations without)))))
(t/deftest migrating-read-returns-the-pointer-it-persisted
;; The map a migrating read returns must carry the pointer of the row it
;; left: a caller that writes with it touches that object instead of the
;; replaced one. The storage GC freezes a referenced object, so a write
;; that misses it leaks the blob forever.
(with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})]
(let [{:keys [file-id]} (create-file!)]
(force-migration! file-id)
(let [file (db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))
left (:storage-object-id (file-data-row file-id))]
(t/testing "the read reports the pointer of the persisted row"
(t/is (uuid? left))
(t/is (= left (:storage-object-id file))))
;; Freeze the object while the row still references it, as the storage
;; GC does; the later write must touch it (see the concurrent-save
;; test for the same technique).
(th/db-update! :storage-object {:touched-at nil} {:id left})
(db/tx-run! th/*system*
(fn [cfg] (bfc/update-file! cfg file)))
(let [row (file-data-row file-id)]
(t/testing "the write replaces the row and touches the object it left"
(t/is (not= left (:storage-object-id row)))
(t/is (some? (:touched-at (storage-object left))))))))))
(t/deftest read-completes-with-a-single-row-read
;; The common read is one row read (with its data columns) and
;; `attach-file-data` completes it.
(let [{:keys [file-id page-id]} (create-file!)
[file sqls] (th/read-recording-sql
#(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id))))]
(t/is (= [bfc/sql:get-file] sqls))
(t/is (some? (get-in file [:data :pages-index page-id])))))
(t/deftest read-resolves-the-applied-migrations-once
;; The common read resolves the applied migrations once. The migration path
;; resolves them again on the locked re-read (it must decide on the fresh
;; row) and once more after persisting (to return the applied set); no path
;; resolves them twice without a reason.
(let [{:keys [file-id]} (create-file!)
calls (atom 0)
read! (fn []
(reset! calls 0)
(with-redefs [fmigr/resolve-applied-migrations
(let [orig fmigr/resolve-applied-migrations]
(fn [cfg file]
(swap! calls inc)
(orig cfg file)))]
(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))))]
(read!)
(t/is (= 1 @calls))
(force-migration! file-id)
(read!)
(t/is (= 3 @calls))))
(t/deftest migration-read-rereads-the-row-under-lock
;; The first read decides; only when a migration is pending the row is read
;; again, and that second read takes the `file` row lock.
(let [{:keys [file-id]} (create-file!)]
(force-migration! file-id)
(let [[_ sqls] (th/read-recording-sql
#(db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id))))]
(t/is (= [bfc/sql:get-file
(str bfc/sql:get-file " FOR UPDATE of f")]
sqls)))))
(t/deftest read-only-migration-never-writes
;; A read-only context migrates in memory: no lock, no row write and no
;; storage write. The pending migration stays pending.
(with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})]
(let [{:keys [file-id page-id]} (create-file!)
pointer (:storage-object-id (file-data-row file-id))]
(force-migration! file-id)
(let [returned (db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id :read-only? true)))]
(t/testing "the read returns migrated, realized data"
(t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-id])))))
(t/testing "nothing is persisted"
(t/is (= pointer (:storage-object-id (file-data-row file-id))))
(t/is (empty? (th/db-query :file-migration
{:file-id file-id
:name migration-name}))))))))
(t/deftest read-only-cfg-migration-never-writes
;; A cfg marked `::db/read-only` migrates in memory too: `db/tx-run!` opens
;; the transaction read-only, `get-file*` sees the connection read-only and
;; takes the no-persist branch. The config flag that fills this key is
;; legacy, but the key is set by hand (objects-gc, the REPL), so the read
;; must keep working under it.
(with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})]
(let [{:keys [file-id page-id]} (create-file!)
pointer (:storage-object-id (file-data-row file-id))]
(force-migration! file-id)
(let [returned (db/tx-run! (assoc th/*system* ::db/read-only true)
(fn [cfg] (bfc/get-file cfg file-id)))]
(t/testing "the read returns migrated data"
(t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-id])))))
(t/testing "nothing is persisted"
(t/is (= pointer (:storage-object-id (file-data-row file-id))))
(t/is (empty? (th/db-query :file-migration
{:file-id file-id
:name migration-name}))))))))
(t/deftest read-only-read-does-not-open-a-transaction
;; A read-only read takes no lock and writes nothing, so `get-file` runs it
;; on a plain connection instead of opening a transaction of its own. The
;; redefinition proves both directions: the same call without the option
;; does open one.
(let [{:keys [file-id]} (create-file!)]
(with-redefs [db/tx-run! (fn [& _]
(throw (ex-info "opened a transaction" {})))]
(t/is (some? (bfc/get-file th/*system* file-id :read-only? true)))
(t/is (thrown? clojure.lang.ExceptionInfo
(bfc/get-file th/*system* file-id))))))

View File

@ -856,7 +856,8 @@
(t/deftest migration-keeps-unchanged-page-fragments
;; Applying a pending migration on read keeps the fragment ids of the
;; pages the migration does not change; a page it changes gets a new
;; id. The caller still receives realized data, not pointers.
;; id. The read returns decoded data with pointers for the pages the
;; migration does not change; `:realize?` is what returns it plain.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
@ -884,7 +885,9 @@
(let [returned (db/tx-run! th/*system*
(fn [cfg] (bfc/get-file cfg file-id)))]
(t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-a]))))
;; The migration changed page-b, so it comes plain; page-a keeps
;; its pointer (and its fragment id) until someone realizes it.
(t/is (pmap/pointer-map? (get-in returned [:data :pages-index page-a])))
(t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-b]))))
(t/is (not (fmg/need-migration? returned)))

View File

@ -373,8 +373,8 @@
(let [tpoint (ct/tpoint)]
(-> system
(assoc ::files.update/timestamp (ct/now))
(db/tx-run! (fn [{:keys [::db/conn] :as system}]
(let [file (files.update/get-file conn file-id)]
(db/tx-run! (fn [system]
(let [file (files.update/get-file system file-id)]
(#'files.update/update-file* system
{:id file-id
:revn revn
@ -848,6 +848,19 @@
[& params]
(apply db/get* *pool* params))
(defn read-recording-sql
"Run `f` and return [result sqls], where `sqls` are the SQL statements of
every `db/get-with-sql` call. Those are exactly the file reads of
`binfile.common/get-file`: the fragment loads go through `db/get*`."
[f]
(let [sqls (atom [])]
(with-redefs [db/get-with-sql
(let [orig db/get-with-sql]
(fn [ds sql & opts]
(swap! sqls conj (first sql))
(apply orig ds sql opts)))]
[(f) @sqls])))
(defn sleep
[ms-or-duration]
(Thread/sleep (inst-ms (ct/duration ms-or-duration))))

View File

@ -10,7 +10,9 @@
resolve the pointer-mapped pages with the file's own loader."
(:require
[app.http.debug :as debug]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.string :as str]
[clojure.test :as t]
[yetti.response :as-alias yres]))
@ -27,3 +29,28 @@
(t/is (= 200 (::yres/status result)))
(t/is (= "NO VALIDATION ERRORS FOUND" (::yres/body result)))))
(t/deftest debug-download-does-not-expose-the-storage-object-id
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared false})
sobject (sto/put-object! (::sto/storage th/*system*)
{::sto/content (sto/content "sobject")
:content-type "application/octet-stream"
:bucket "file-data"
:file-id (:id file)
:id (:id file)})
_ (th/db-update! :file-data {:storage-object-id (:id sobject)}
{:file-id (:id file)
:id (:id file)})
handler (deref #'debug/download-file-data)
result (handler th/*system* {:params {:file-id (str (:id file))
:download true}})
body (String. ^bytes (::yres/body result))]
(t/is (= 200 (::yres/status result)))
(t/is (str/includes? (get-in result [::yres/headers "content-disposition"])
"attachment"))
(t/is (not (str/includes? body "storage-object-id")))
(t/is (not (str/includes? body "storage_object_id")))))

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More