diff --git a/.agents/skills/update-changelog/SKILL.md b/.agents/skills/update-changelog/SKILL.md index c83ab8d1f7..0c5ca79f23 100644 --- a/.agents/skills/update-changelog/SKILL.md +++ b/.agents/skills/update-changelog/SKILL.md @@ -31,14 +31,17 @@ memory; it does not repeat it. layout, entry rules, the `## Breaking change note` comment contract, the advisory (`:lock:`) contract, pre-flight checks, anomaly types, and pitfalls. 2. `mem:scripts/changelog` — `boom`, `security`, `check-merged`, `cross-ref`, - `report`. + `dev-only-bugs`, `report`. 3. `mem:scripts/gh` — `issues`, `issue`, `comments`, `prs`, `advisories`. ## Commands ```bash -python3 scripts/gh.py issues "" --exclude "release blocker,no changelog" +python3 scripts/gh.py issues "" --exclude "no changelog" +python3 scripts/gh.py issues "" --label "release blocker" --exclude "no changelog" # closed non-bug residue to review by hand python3 scripts/gh.py advisories --patched-version "" +python3 scripts/changelog.py dev-only-bugs "" # list bugs that never reached production +python3 scripts/changelog.py dev-only-bugs "" --apply # label the sure ones `no changelog` python3 scripts/changelog.py check-merged python3 scripts/changelog.py boom "" # print the :boom: section python3 scripts/changelog.py boom "" --write # replace it in CHANGES.md @@ -54,4 +57,5 @@ python3 scripts/changelog.py report "" - Only closed, non-excluded milestone issues are included; the issue is the changelog unit and the PR is inline. - `:boom:` and `:rocket:` are label-generated and `:lock:` is advisory-generated: never hand-edit them; edit the label, the `## Breaking change note` comment or the advisory and re-run the command. - Verify PR merge status before writing; only merged PRs are referenced. +- Bugs that never reached production stay out: `dev-only-bugs --apply` labels the sure ones; candidates are the user's call, never label them yourself. - Fill `patched_versions` in the advisories before running `security`; unpublished advisories are valid. diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 0caf92fbc4..2a7f6ea61b 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -9,7 +9,8 @@ app.common.data.macros/get-in clojure.core/get-in app.common.data.macros/with-open clojure.core/with-open app.common.data.macros/select-keys clojure.core/select-keys - app.common.logging/with-context clojure.core/do} + app.common.logging/with-context clojure.core/do + benches.render-wasm.errors/attempt clojure.core/do} :hooks {:analyze-call @@ -24,7 +25,11 @@ shadow.lazy/loadable hooks.export/rumext-lazycomponent app.util.i18n/tr hooks.i18n/tr-dynamic app.common.i18n/tr hooks.i18n/tr-dynamic - }} + benches.render-wasm.declarations/defscene hooks.benches/defscene + benches.render-wasm.declarations/defcase hooks.benches/defcase + benches.render-wasm.runtime.commands/defcommand hooks.benches/defcommand + benches.render-wasm.runtime.protocol/resolve-or-reject hooks.benches/resolve-or-reject + benches.render-wasm.runtime.protocol/resolve-or-reject-with hooks.benches/resolve-or-reject-with}} :output {:exclude-files @@ -36,7 +41,7 @@ "^(?:backend|frontend|exporter|common)/deps.edn" "^(?:backend|frontend|exporter|common)/scripts/" "^(?:backend|frontend|exporter|common)/dev/" - "^(?:backend|frontend|exporter|common)/test/"] + "^(?:backend|frontend|exporter|common)/test/(?!performance_tests/)"] :linter-name true} @@ -101,6 +106,4 @@ :unused-binding {:exclude-destructured-as true - :exclude-destructured-keys-in-fn-args false - } - }} + :exclude-destructured-keys-in-fn-args false}}} diff --git a/.clj-kondo/hooks/benches.clj b/.clj-kondo/hooks/benches.clj new file mode 100644 index 0000000000..5a100be0d8 --- /dev/null +++ b/.clj-kondo/hooks/benches.clj @@ -0,0 +1,77 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns hooks.benches + (:require [clj-kondo.hooks-api :as api])) + +(defn defscene + "Analyzes scene options and its compiled builder body." + [{:keys [node]}] + (let [[_ _ opts argv & body] (:children node)] + (if (and argv (api/vector-node? argv)) + {:node (api/list-node [(api/token-node (quote do)) opts + (api/list-node (into [(api/token-node (quote fn)) argv] body))])} + {:node node}))) + +(defn defcase + "Analyzes case options and its compiled operation body." + [{:keys [node]}] + (let [[_ _ opts argv & body] (:children node)] + (if (and (seq body) argv (api/vector-node? argv)) + {:node (api/list-node [(api/token-node (quote do)) opts + (api/list-node (into [(api/token-node (quote fn)) argv] body))])} + {:node node}))) + +(defn defcommand + "Analyzes command options and both generated functions." + [{:keys [node]}] + (let [[_ command & forms] (:children node) + [doc forms] (if (string? (api/sexpr (first forms))) + [(first forms) (next forms)] + [nil forms]) + [options forms] (if (api/map-node? (first forms)) + [(first forms) (next forms)] + [nil forms]) + [argv & body] forms + [conditions body] (if (and (api/map-node? (first body)) (next body)) + [(first body) (next body)] + [nil body]) + handlers (:async (some-> options api/sexpr)) + core (api/list-node + (into [(api/token-node 'fn) + (api/vector-node (mapv api/token-node handlers))] + body)) + starred (api/token-node (symbol (str (api/sexpr command) "*"))) + measured (get (some-> options api/sexpr) :measured-variant true) + definition (api/list-node + (into [(api/token-node 'defn) command] + (concat (when doc [doc]) [argv] + (when conditions [conditions]) [core])))] + {:node (api/list-node + (into [(api/token-node 'do)] + (concat (when options [(api/coerce (dissoc (api/sexpr options) :async))]) + [definition] + (when measured + [(api/list-node [(api/token-node 'def) + starred command])]))))})) + +(defn resolve-or-reject + "Analyzes the body of a Promise-returning form." + [{:keys [node]}] + (let [[_ & body] (:children node)] + {:node (api/list-node (into [(api/token-node (quote do))] body))})) + +(defn resolve-or-reject-with + "Analyzes the cause handler and body. Binds `cause` in the handler only." + [{:keys [node]}] + (let [[_ cause on-error & body] (:children node)] + {:node (api/list-node + (into [(api/token-node (quote do)) + (api/list-node + [(api/token-node (quote let)) + (api/vector-node [cause (api/token-node nil)]) + on-error])] + body))})) diff --git a/.clj-kondo/hooks/export.clj b/.clj-kondo/hooks/export.clj index 50e617de55..f4c0ebcf0c 100644 --- a/.clj-kondo/hooks/export.clj +++ b/.clj-kondo/hooks/export.clj @@ -1,3 +1,9 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + (ns hooks.export (:require [clj-kondo.hooks-api :as api])) diff --git a/.cljfmt.edn b/.cljfmt.edn index ad56b8c70d..2b988af01d 100644 --- a/.cljfmt.edn +++ b/.cljfmt.edn @@ -10,5 +10,7 @@ :extra-indents {rumext.v2/fnc [[:inner 0]] cljs.test/async [[:inner 0]] promesa.exec/thread [[:inner 0]] - specify! [[:inner 0] [:inner 1]]} + specify! [[:inner 0] [:inner 1]] + benches.render-wasm.runtime.commands/defcommand [[:inner 0]] + benches.render-wasm.runtime.protocol/resolve-or-reject-with [[:inner 0]]} } diff --git a/.github/workflows/plugins-deploy-api-doc.yml b/.github/workflows/plugins-deploy-api-doc.yml index f5301f9844..4d22311892 100644 --- a/.github/workflows/plugins-deploy-api-doc.yml +++ b/.github/workflows/plugins-deploy-api-doc.yml @@ -48,10 +48,10 @@ jobs: # No dependency cache here: a deploy must not restore a cache that # other workflows can write to. - name: Setup pnpm + Node.js - uses: pnpm/setup@fa36a5ef7dcdefc3a7cb25655c122e00cf6e636e # v3.1.2 + uses: pnpm/setup@731dfb6463bbd11c11fd0ee11fe00ed0e36d48a1 # v3.2.0 with: working-directory: plugins - runtime: node@26.13.0 + runtime: node@26.13.1 install: false - name: Install deps diff --git a/.github/workflows/plugins-deploy-styles-doc.yml b/.github/workflows/plugins-deploy-styles-doc.yml index d084affd59..8fd4157a41 100644 --- a/.github/workflows/plugins-deploy-styles-doc.yml +++ b/.github/workflows/plugins-deploy-styles-doc.yml @@ -46,10 +46,10 @@ jobs: # No dependency cache here: a deploy must not restore a cache that # other workflows can write to. - name: Setup pnpm + Node.js - uses: pnpm/setup@fa36a5ef7dcdefc3a7cb25655c122e00cf6e636e # v3.1.2 + uses: pnpm/setup@731dfb6463bbd11c11fd0ee11fe00ed0e36d48a1 # v3.2.0 with: working-directory: plugins - runtime: node@26.13.0 + runtime: node@26.13.1 install: false - name: Install deps diff --git a/.gitignore b/.gitignore index 5e38fb2361..974b11f495 100644 --- a/.gitignore +++ b/.gitignore @@ -121,7 +121,7 @@ opencode.json /.opencode/reports /.opencode/prompts /.ci-logs -/tmp/ +/tmp /.codex/ /scripts/__pycache__/ /tools/__pycache__ diff --git a/.nvmrc b/.nvmrc index 86456ddf02..22fbfc060a 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v24.22.2 +v24.22.3 diff --git a/.serena/memories/backend/auth-permissions-product-domains.md b/.serena/memories/backend/auth-permissions-product-domains.md index 4cd1658906..284f21cde6 100644 --- a/.serena/memories/backend/auth-permissions-product-domains.md +++ b/.serena/memories/backend/auth-permissions-product-domains.md @@ -11,8 +11,10 @@ - Account lockout (flag `:account-lockout`, `app.auth.login-lockout`) is Redis-backed and keyed per profile id. Password and LDAP flows check/increment on the profile derived from the typed email and clear on the profile that actually logs in. - Lockout and RPC rate limits share the `app.http.errors/handle-error :rate-limit` HTTP path: status 429, body `{:type :rate-limit :code ... :hint ... :ttl ...}`, and any supplied `::http/headers` preserved. Account lockout raises `:code :account-locked` and, when it carries a non-nil `:ttl` (seconds), the handler adds `retry-after`. The RPC limiter (`app.rpc.rlimit`) raises `:code :request-blocked` and sets `retry-after` itself in `::http/headers` (seconds until the longest rejecting limit resets), alongside `x-rate-limit-remaining`/`x-rate-limit-reset`. CORS (`app.http.middleware/with-cors-headers`) exposes `content-type`, `retry-after`, and both `x-rate-limit-*` headers. External flags: `enable-account-lockout`, `enable-rpc-rlimit`. - Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI. +- Organization SSO (Nitrate): the RPC `check-nitrate-sso` takes the route's **query string proper** (`:query-string`, e.g. `screen=workspace&team-id=…`), never a URL and never with the `?` separator: the client strips it (`rt/token->query-string`) and the server puts it back. The server builds the destination with `app.auth.oidc/build-dest-url` (`public-uri` + `?` + the params, parsed and rendered back), so the redirect can never leave the public URI; a value whose parsed keys are not param names is not a route: a URL parses into a single key holding the whole URL, and a key carrying `:`, `/`, `?` or `#` is rejected with a `:warn` and the user lands on the public URI (the RPC never fails because of it). Do not reason from `u/map->query-string`: it keeps a valueless key as `key=`, so rendering is not what filters a URL out — the key shape is. A leading `?` is still trimmed server-side, so a client bundled before the trim keeps its destination. Never require `screen`: the root of the app carries its params instead of a screen (`?template=…`), and `check-nitrate-sso` is called from that branch too (`dtm/resolve-login-team-id`, which today reads only `:authorized` and never follows the redirect, so a wrong destination there is a silent `:warn` and not a visible break); the callback re-checks the destination before redirecting, which covers state tokens minted before that. The state token also carries `:profile-id`: only that session is granted the `props :sso` entry (`session-mismatch` otherwise, and the redirect still happens). +- Organization SSO needs `app.nitrate/organization-access?` whenever the RPC receives an `:organization-id` (that id always comes from the client): the profile must be a member of the organization or its owner; otherwise the RPC answers `{:authorized true :reason :no-organization-access}`, the same short-circuit shape as `:no-team-access`. - Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation. -- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. +- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. SSO registration (OIDC and compatible providers) follows the OIDC callback gate: it requires `:registration` or `:oidc-registration`, or a team invitation token issued for the registering email (an invitation substitutes the flags only when `member-email` matches the OIDC/`prepared-register` email via `clean-email`, as in the password flow, which raises `:email-does-not-match-invitation` otherwise). The OIDC callback verifies the invitation JWT and the email before redirecting to register, and `register-profile` revalidates the invitation under a row lock when no SSO flag is set; it never requires password login. Only OIDC-typed backends (`oidc-provider-types` in `app.auth.oidc`) take the SSO registration gate; any other `:backend` in a prepared-register token is rejected as `:invalid-token`. - Access tokens (`app.http.access-token`): personal tokens have JWT issuer `access-token` and DB type other than `mcp`; MCP tokens have issuer `urn:penpot:mcp-token` (`actoken/mcp-token-issuer`) and DB type `mcp`. Authentication requires issuer and type to agree, and the claim `uid` to match the row's profile; other issuers and undecryptable tokens are treated as anonymous. - Management RPC (`/api/management/methods/:name`) is shared-key only. The middleware authenticates ` `; the RPC wrapper `wrap-shared-key-authorization` authorizes the key id against the method's `::rpc/allowed-shared-keys` set. Absent metadata accepts any configured key; all current methods declare one (`exporter`/`media-processor` for `upload-tempfile`, `nexus` for file analysis, `exporter` for jobs, `admin-console` for Nitrate). A valid key outside the set answers 403 `:shared-key-not-allowed`; denied calls do not reach the audit wrapper. - Personal tokens need flag `:access-tokens`. MCP tokens need `:mcp` or `:access-tokens`, and only on routes whose `actoken/authz` gets `::actoken/allow-mcp true` (the RPC routes); elsewhere (e.g. assets) they are anonymous. @@ -24,6 +26,7 @@ - `app.rpc.permissions` provides predicate/check factories. Failed permission checks intentionally raise `:not-found` / `:object-not-found`, not an authorization-specific error, to avoid leaking object existence. - Team role flags are normalized as owner > admin > editor > viewer. Owner/admin imply edit; any membership row implies read. - File/project role rows only count with a live team membership: the file/project permission queries require a matching `team_profile_rel` row, and leaving or removing a member deletes their team-scoped `file`/`project`/`team-project-profile-rel` rows (GHSA-v9r9-h77c-55m2). +- The organization-owner fallback (`get-organization-owner-permissions` via Nitrate over HTTP) grants read-only access for a non-member organization owner. It resolves the owning team through `resolve-team-id`, which refuses soft-deleted file/project/team: the file and project lookups filter `deleted_at`, and a final `db/get*` check covers the team row itself — `nil` means no access. `db/get*` only filters soft-deleted rows when `deleted_at` is in the projection. The re-checks of open websocket subscriptions rely on this (`mem:backend/websocket-subscriptions`). - File/project/comment checks are implemented in the owning command namespaces, often via helpers imported from `files`, `teams`, or `projects`; do not bypass those helpers with direct DB lookups unless preserving their not-found semantics. - Comment permission includes both logged-in state and the file/team comment policy. Shared viewer paths may pass `share-id`; preserve that path when changing comment queries. RPCs that accept `share-id` enforce the link's page scope before mutating (`share-link-pages` + `check-page-scope`): a thread or page outside the link raises `:not-found`, and that check runs before the ownership checks. `get-profiles-for-file-comments` filters its query by the same pages. `mark-all-threads-as-read` does not take `share-id`. @@ -32,6 +35,7 @@ - Team/project commands mix DB changes, email, message bus notifications, media/storage cleanup, feature flags, quotas, and audit metadata. Keep mutations transactional when the existing command does so. - Invitation flows validate muted/bounced emails before sending and use tokenized invitation state. Accepting an invitation is tied to the invited member identity, not just possession of a token. - The whole invitation family is team-admin only: creating, updating and deleting invitations, minting the invitation token (`get-team-invitation-token`) and listing the pending ones (`get-team-invitations`, which carries the invited email and the offered role). The token of an invitation that grants the `owner` role is restricted to the team owner, because an admin cannot create nor promote an owner invitation (`cant-promote-to-owner`). This family is the exception to the rule above: it answers `:validation` / `:insufficient-permissions` instead of `:not-found`, so the five commands speak one language. +- With the `:admin-console` flag, the organization `send-invitations` policy (role value plus Nitrate `restrictions-met`, through `cto/allowed?`) gates every invitation command that hands out or manages access: create (and resend, which reuses it), `update-team-invitation-role`, `delete-team-invitation` and `get-team-invitation-token`. Only `get-team-invitations` stays on team admin alone, so a blocked admin still sees the list. The shared helper `check-send-invitations-policy` in `teams_invitations.clj` does a strict Nitrate lookup (fails closed; 404 means no organization) and runs outside a DB transaction, which is why the role and delete commands carry no `::db/transaction`. - Logical deletion is used for many product objects; prefer existing logical-deletion helpers over hard deletes unless the command already performs permanent cleanup. - Bounced/spam-complaint emails can mute/block a profile for login/registration and email sending. Devenv Mailpit is the normal local path for registration/email-flow testing. diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index 16978ac1ff..1d9cbe0ba6 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -9,6 +9,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m - Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment` - Session lifetime config, token `:exp`, and idle/absolute GC: `mem:backend/session-expiration`. - Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains` +- WebSocket subscriptions, the connection registry (local per backend, unlike the bus), and how a lost permission gets revoked: `mem:backend/websocket-subscriptions`. - Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:audit-log` (guía completa, también para perfiles junior; el stub `mem:backend/audit-log` solo redirige aquí) - Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`. @@ -67,7 +68,8 @@ Every background job is a job-def: a plain `(defn execute-X [cfg params] ...)` i - Job resources (`app.jobs.storage`) are the objects a job owns through `job.resource_id`: they live in the `job-resource` bucket, are written with the profile of the job as owner (only that profile can read them back from `/assets/by-id`) and are never deduplicated. They carry no expiry of their own, because `expired-at` is stored as `deleted_at` and an object with it is unreadable from birth: the retention of the artifact is the retention of its job row. The writer marks the object as touched right away, so an artifact that no row references (a crash between the upload and the completion) is reclaimed by `storage-gc-touched`, and a row that does reference it freezes it until the jobs GC deletes the row and touches the object. - The binfile jobs (`:export-binfile` and `:import-binfile`) run the core of `bf.v1`/`bf.v3` through `app.binfile.jobs`, which turns the `:progress` taps of the core into `progress` events of the job with `jobs/heartbeat` (the `modified_at` touch at most once per second, progress events at up to ~200ms). Each event is a self-contained milestone: a `:stage` keyword of one vocabulary shared by the two formats (the same work has a different name in v1 and v3) and an optional `:counters` map, with the counter of the activity named after its own stage and the outer scopes in course as context, which the tap carries itself (the reader puts `::outer-counters` in every tap inside them — today only `:files`, extendable to `:teams`/`:projects` for multi-scope exports — so the adapter remembers nothing between taps and a tap that happens outside any scope has no context); the ids and names the taps carry never reach the event. Every tap is reported, so a tap with no units to count still keeps the job alive, and a milestone the throttle drops loses nothing because the next one carries the same state. The adapter checks that the job is still active before the run, and the beat raises an `:interrupt` right there on the run thread when the job is no longer active, so the work stops at its next event point inside the transaction of its handler, which rolls back. A run whose last tap predates the interruption is discarded by the check once it returns. The adapter passes the job id from the handler context. Their job-def components carry pool, storage, metrics and msgbus, because a job with a `profile_id` cannot store an event without a msgbus on its cfg. Both readers answer with the same shape (`:file-ids` and, in version 3, the resolution of the libraries), because the version 1 reader returns a bare set of file ids. - Both binfile handlers revalidate their permission at run time, not only when the job is created, because it can be revoked while the job waits: the export checks the read permission of every file before reading anything and stores the package in `job-resource` as the resource of the job (answering with the completion envelope), and the import checks the edition permission of the destination and runs the core inside a transaction, which is what makes a cancelled import roll back instead of leaving half of a file. The import releases its package (marks it as deleted and drops the reference from the job row in the same transaction) and deletes its temporary copy once the job is terminal, whatever the outcome. Both handlers check their own result against a schema before returning it (the import job-def declares that same schema as its result schema, which is what types the resolution when the row is read). -- The v3 import only materializes a library relation when both ids belong to the package and differ; a relation that points outside the package or a self link is ignored with a warning. Migration `0158-del-forbidden-file-library-rels` deletes the legacy cross-team and self rows from `file_library_rel` and their `file_library_sync` rows, sync rows first. +- The v3 and v1 imports only materialize a library relation when both ids belong to the package and the link would neither be a self link nor close a dependency cycle (the same `bfc/library-link-status` check that `link-file-to-library` uses to raise); relations are processed in sorted `(file_id, library_file_id)` order, which `get-files-rels` also returns, so the first edge wins deterministically, and every edge it ignores is logged with a warning. The v2 import stays out of that guard on purpose: it is only reachable from the developer REPL (`app.srepl.binfile`) and keeps inserting the relations as they come. Migration `0158-del-forbidden-file-library-rels` deletes the legacy cross-team and self rows from `file_library_rel` and their `file_library_sync` rows, sync rows first. +- A version 1 package lists its files in the metadata and the importer creates one file per entry. The v1 export (gone from the product; only tests and the developer REPL use it) unioned the requested files with their libraries without deduplicating, so a package exported from a file inside a library cycle lists that file twice. The importer reads the pair of objects of every entry to keep the stream in sync, imports the first occurrence only, warns about the repeats, and answers with one id per imported file. - The domain cfg of a binfile request lives in `app.binfile.common` (`export-cfg`, `import-cfg`), so the legacy RPC and the jobs freeze the same ids, resolve the same team features and apply the same anti-zip-bomb limits. `resolve-export-type` is the translation the legacy RPC needs for its deprecated boolean flags; the creation command asks the caller for the type instead of guessing a default. - Two retention knobs, easy to confuse: `:jobs-retention` sweeps terminal internal rows (the ones without a profile), while `:jobs-user-ttl` is the default expiry a user-facing job gets when it is created; the jobs GC deletes the row once `expires_at` passes and touches its resource. The heavy work of a user job runs on its own `:binfile` queue (`:worker-binfile-parallelism`, default 1) so it does not block the default runner. - The quote of a family of jobs (`::export-jobs-per-profile`, `::import-jobs-per-profile`) counts the jobs of the profile that are still on their way (`new`/`scheduled`/`running`/`retry`) whose name the registry gives for that family, so a new job of the family is covered without touching the quote. diff --git a/.serena/memories/backend/session-expiration.md b/.serena/memories/backend/session-expiration.md index 99c0d6acac..7efbf49089 100644 --- a/.serena/memories/backend/session-expiration.md +++ b/.serena/memories/backend/session-expiration.md @@ -19,6 +19,12 @@ - Renewal fires when `modified-at` is older than 6h (`default-renewal-max-age`, not configurable). It `UPDATE`s the same row (`modified-at` only) and issues a new token that keeps the original `:exp`. - `read-session` does not check age. Idle expiration is enforced on every request by `wrap-authz` (sessions past the idle window authenticate like missing ones and are never renewed); the GC only deletes the rows afterwards. +## Revocation paths + +- Profile deletion → `session/invalidate-all` (all sessions of the profile die). +- Authenticated password change → `session/invalidate-others` (every session except the caller's dies). +- Password recovery (`recover-profile`, GHSA-3j2p-qr45-q365) → `session/invalidate-all` inside the same transaction as the password `UPDATE`; there is no current session to preserve, and a captured session must not survive recovery. Sibling of GHSA-mj9f-5cwq-7p3q (logout deletes the current session row). + ## GC (`::tasks/gc`, cron `session-gc`, daily) - Deletes from `http_session_v2` where `modified_at < now - :auth-token-cookie-max-age` **or** `created_at < now - :auth-token-cookie-max-age-absolute`. diff --git a/.serena/memories/backend/storage.md b/.serena/memories/backend/storage.md index 529fcd1cc8..4a9b98ea31 100644 --- a/.serena/memories/backend/storage.md +++ b/.serena/memories/backend/storage.md @@ -93,7 +93,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` | `organization` | Organization logos uploaded by the Nitrate management API. | Yes | Public | No reference scan. A touched object is deleted. | | `tempfile` | Export files and temporary font downloads. | No | Authentication required | No reference scan. A touched object uses a two-hour deletion delay. | | `upload-session` | Chunked-upload chunks. References: `upload_session_chunk.object_id` and `upload_session_chunk.session_id` (both NO ACTION DEFERRABLE: restrict semantics, procedural deletion). | No | Authentication required | Reference scan on `upload_session_chunk.object_id`; a referenced chunk freezes, an orphan is deleted after the delay. Deferred rows keep their mappings until give-up. | -| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference metadata has `storage-ref-id`, `file-id`, and the `file_data` row ID. | Yes | Authentication required | Reference scan. | +| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference: the `file_data.storage_object_id` column (FK `NO ACTION DEFERRABLE INITIALLY IMMEDIATE`); the object metadata keeps `file-id` and the `file_data` row ID. | Yes | Authentication required | Reference scan. | | `file-data-fragment` | Compatibility value for file-data fragments. The current backend has no dedicated producer for this bucket. | No current write semantics | Public | No touched-object collector case. | | `file-change` | Compatibility value for file changes. Current snapshots store data in `file_data`, not this bucket. | No current write semantics | Authentication required | No touched-object collector case. | | `job-resource` | Storage objects owned by job rows (`job.resource_id`), written by `app.jobs.storage`. | No | Authentication required, owner-scoped | jobs-GC touch → reference scan. | @@ -118,18 +118,40 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` ## File Data -- `file-data-backend` accepts `legacy-db`, `db`, or `storage`. -- `legacy-db` stores main data in `file.data` and snapshots in `file_change.data`. +- `file-data-backend` accepts `legacy-db`, `db`, or `storage`; `legacy-db` is read-only and the default is `db`. The configuration reads `legacy-db` as `db` at startup, with a warning (`normalize-file-data-backend`), so a deployment that kept the variable saves instead of failing on every write. +- `legacy-db` reads main data from `file.data` and snapshots from `file_change.data`; it no longer writes them (`fdata/upsert!` rejects it). - `db` stores encoded data in `file_data.data`. -- `storage` stores encoded data in storage subsystem with `file-data` bucket and keeps `data` nil in `file_data` table. -- The `file_data.metadata.storage-ref-id` value points to the storage object. -- `fdata/upsert!` touches a storage object from incoming metadata before it stores the new row. +- `storage` stores encoded data in storage subsystem with `file-data` bucket, keeps `data` nil in `file_data` table, and points at the object with `file_data.storage_object_id`. +- `file_data.storage_object_id` is a `NO ACTION DEFERRABLE INITIALLY IMMEDIATE` foreign key to `storage_object(id)`: the database refuses to physically delete a referenced object. The `IMMEDIATE` is load-bearing: the RI violation fires inside the `storage-gc-deleted` chunk transaction, so the chunk rolls back, is logged at error level with a generic hint plus the `constraint` and `table` the database reports (any `NO ACTION` foreign key on `storage_object` can block the delete, not only `file_data`'s) and is retried on the next run; with `INITIALLY DEFERRED` the check would fire at commit, outside the catch, and the sweep would never move past the blocked chunk. +- The column is the only source of truth. The release is a coordinated restart (no rolling deploy) when the instance uses the storage backend (`PENPOT_FILE_DATA_BACKEND=storage`) or still has rows carrying `metadata.storage-ref-id`; with the default `db` backend and no such rows the previous and the new code read the same `file_data.data` column, so a rolling deploy is safe. `metadata.storage-ref-id` is no longer written or read, and the `file-data` metadata schema does not declare it. The migration backfills the rows it can resolve (valid uuid with an existing `storage_object`) and removes the key from those rows; a corrupt or orphan pointer keeps the key, which nothing reads. A large instance runs `operations/2.20.0-file-data-storage-object-id.md`: column, index and FK online, then the backfill (only if a row still carries `metadata.storage-ref-id`) with every instance stopped, verifies, and marks the migration as applied (`INSERT INTO migrations`). The backfill must never run while the previous code serves when the storage backend is in use (or rows with the pointer remain), because that code writes and scans the metadata pointer. A rollback to the previous code cannot read the rows written by this release when the storage backend is in use. +- `fdata/upsert!` touches the `storage_object_id` it receives in the params (the pointer the persisted row had) before replacing it, so the replaced blob is reclaimed. Callers pass the column from the file map (main/restore), `nil` for a new row (fragments, snapshots, new files) or the row itself (offload); there is no query in the upsert path, which is why every caller reads the file under the row lock first (the migration-on-read path included, see `mem:backend/subtleties`). It returns `{:written :storage-object-id }`, the pointer of the object the write created (nil for `db`/`legacy-db`), and `binfile.common/update-file!` propagates it so a migrating read returns the map with the pointer of the row it persisted instead of the pre-write one. The write also compares the expected pointer with `IS NOT DISTINCT FROM` (the caller's value, not the new one) and raises `:stale-file-data-pointer` when the row changed under it, so a stale map fails instead of leaking the replaced blob; the check lives in `upsert-in-database` because the `legacy-db` branches do not use the upsert SQL. +- `binfile.common/update-file!` takes the file map read from the database plus the fields to change (`[cfg file & params]`), merges them and builds the `file_data` params from the merged map, so the pointer travels in the read map and a caller cannot lose it by building a partial map; the params cannot override `:storage-object-id`. `insert-file!` always drops it (an INSERT never replaces a persisted pointer; `duplicate-file` hands the source map there). `bfc/get-file` keeps the pointer for the write paths, and every response goes through `bfc/file->response`, which strips it and any residual `storage-ref-id` left in the metadata of the rows the migration does not backfill: each `file-data` object is authentication-only and any logged-in profile could otherwise fetch it from `/assets/by-id`. +- The in-place binfile import was removed: `import-files!` raises `:not-supported` when the cfg carries `::bfc/file-id`, so no caller can overwrite a storage-backed row without carrying its previous pointer. - File snapshots use `file_data` for snapshot data and `file_change` for snapshot metadata. -- Tiered offload (`:tiered-file-data-storage` flag + `db` backend): `file-gc` submits an `:offload-file-data` job (priority 10, delay 1000, same params) only when it actually processed the file; the handler takes the `file` row with `FOR UPDATE SKIP LOCKED` and skips (`{:skipped 1}`, no error) when the row is locked or gone, then moves every live `file_data` row of that file with `backend = 'db'` to `backend = 'storage'` via `fdata/upsert!` (one `put-object!` in bucket `file-data` per row, `data` becomes nil) inside a single all-or-nothing transaction per file. The skip only serializes because every `file_data` writer holds at least `FOR SHARE` on the `file` row. A storage failure aborts only that file and bubbles for a runner retry; orphans go to `storage-gc-touched`. A second run is a no-op because the query only matches `backend = 'db'`. +- Tiered offload (`:tiered-file-data-storage` flag + `db` backend): `file-gc` submits an `:offload-file-data` job (priority 10, delay 1000, same params) only when it actually processed the file; the handler takes the `file` row with `FOR UPDATE SKIP LOCKED` and skips (`{:skipped 1}`, no error) when the row is locked or gone, then moves every live `file_data` row of that file with `backend = 'db'` to `backend = 'storage'` via `fdata/upsert!` (one `put-object!` in bucket `file-data` per row, `data` becomes nil and `storage_object_id` points at the new object) inside a single all-or-nothing transaction per file. The skip only serializes because every `file_data` writer locks the `file` row first (the update RPC and the snapshot paths with `FOR UPDATE` through `lock-for-update?`, the migration-on-read path with `FOR UPDATE`). A storage failure aborts only that file and bubbles for a runner retry; orphans go to `storage-gc-touched`. A second run is a no-op because the query only matches `backend = 'db'`. +- The offload normalizes each `SELECT fd.*` row before `upsert!`: the row builder omits NULL columns, jsonb metadata arrives as a `PGobject`, and a JSON `null` decodes to nil and must disappear instead of travelling as `:metadata nil`, which the update schema rejects. - Offload pacing: none by deliberate decision. Sleeps only make sense when batching multiple transactions; the per-file transaction is short by construction and the scheduler run takes minutes per production metrics, so neither loop sleeps. `jobs/heartbeat` (pool-based liveness, ~1 write/s) stays on both paths. - Offload metrics (bounded labels only, never ids or error text): `penpot_file_data_offload_rows_total{type}` and `penpot_file_data_offload_bytes_total{type}` (`type`: `main`/`snapshot`/`fragment`, successful runs only) plus `penpot_file_data_offload_timing_ms{result}` (`result`: `ok`/`skip`/`error`). Helpers live in `app.jobs.metrics`; the job-def requires `::mtx/metrics`. - Writes always resolve the backend through `fdata/default-backend` (`cf/get :file-data-backend`), so post-offload edits land back in hot (`db`) storage without extra code; reads stay transparent through `fdata/resolve-file-data` + `decode-file-data` + `realize`. -- Deleting an offloaded row touches its storage object first (`objects-gc/delete-file-data-chunk` via `fdata/process-metadata`), so `storage-gc-touched` reclaims the blob with the usual delay; restoring a file unmarks the referenced objects. +- Deleting an offloaded row touches its storage object first (`objects-gc/delete-file-data-chunk` via `fdata/touch-storage-object` with the row's `storage_object_id`), so `storage-gc-touched` reclaims the blob with the usual delay; restoring a file unmarks the referenced objects. +- `storage-gc-touched` resolves the `file-data` reference with `SELECT EXISTS (SELECT 1 FROM file_data WHERE storage_object_id = ?)`, an indexed lookup that no longer parses the object metadata. +- A row with `backend='storage'` and no readable object makes `resolve-file-data` raise `:missing-storage-object` with the file and row ids instead of failing later in the storage backend. +- The `file-id` and row id kept in the storage object metadata are informational for diagnostics; nothing parses them to resolve the reference (the reference is the column). + +## File Change + +- `file_change` is partitioned by `HASH (file_id)` in 16 partitions, with the same bounds as `file_data`, so a join between them is resolved partition by partition (`enable_partitionwise_join` is set on every pooled connection). +- The automatic migration (`0158-partition-file-change`) refuses to run when `file_change` is above 10 GiB total size or 2M rows (`pg_total_relation_size` + `pg_class.reltuples`, catalog reads). It analyzes the table first, because `reltuples` is -1 without statistics. It raises with the path of `operations/2.20.0-file-change-partition.md`, because the copy happens inside the migration transaction and a big instance would restart in a loop. Below the guard it raises `statement_timeout` to 30 minutes for the copy and the index builds, and analyzes the new partitions at the end, after the copy and the indexes, so the planner does not start with empty statistics. +- Its primary key is `(file_id, id)`: a partitioned table needs the partition key in the primary key, so every access filters by `file_id`. From 2.20 the four snapshot commands (`update-file-snapshot`, `delete-file-snapshot`, `lock-file-snapshot`, `unlock-file-snapshot`) require the `file-id` param, because they use it to prune the partition; without it they answer `:params-validation`. +- It has a `kind` column: `1` is a snapshot (it has a label and its data lives in `file_data`; the read-only `legacy-db` backend keeps old payloads in `data`) and `2` is a revision (it has no label and it holds the changes). The values are in `app.features.file-changes`. +- `kind` is a deliberate reserve, not a leftover: today no query filters by it, the queries still tell a snapshot from a revision by its label (`label IS NOT NULL`), and `kind` takes over when they are rewritten. No constraint ties `kind` and `label` together on purpose: the day `kind` is the source of truth, that constraint would have to go. It stays nullable on purpose: the previous version inserts without it, and a rollback to it has to keep writing; the `NOT NULL` comes in a later migration. +- Every write of the application to a row of `file_change` moves `updated_at` to the moment of the write (`lock-by!`, `unlock!`, the snapshot rename and delete, `delete-object`, the restore of a file). The actions of the foreign keys (`ON DELETE SET NULL` on `locked_by` and `profile_id`) change the row without touching it; during the partition both tables carry the same keys, so both receive them. The runbook of the partition uses it as the watermark of its delta passes: a write of the application that leaves it behind is lost at the cutover. +- `file_change_legacy` is a frozen copy of the previous layout of the partition: it keeps the rows and the indexes of the old table. A new instance (empty table) does not keep it. On an upgraded instance the table stays until the operator drops it by hand days later (`DROP TABLE file_change_legacy;`, see `operations/2.20.0-file-change-partition.md`); nothing drops it automatically. +- A revision carries only the delta of a save (the changes). The features of the file are not repeated on it: nobody read them from there. +- A snapshot carries the features and the migrations of the file. +- It keeps the `data` column for reading only: migration `0157-migrate-legacy-file-data` moved the existing blobs to `file_data`, and `legacy-db` remains as a read-only backend for the payloads still in `data`. `fdata/upsert!` rejects `legacy-db` and the `file-data-backend` default is `db`. It has no `data_ref_id` or `data_backend` columns: nothing read them since the file storage refactor. +- It has no index for the system snapshots. The old one filtered them by `data IS NOT NULL`, and with the payload living in `file_data` the column no longer tells snapshots apart. The new one could only filter them by `deleted_at IS NULL`, but the system snapshots always carry a deletion date in the future, so it would never match a row. The query that would use it filters that date with an "or", which no partial index can serve. +- The `objects-gc` purge groups each chunk by `file_id` and locks it with `file_id = ?` and `id = ANY(?::uuid[])`: a tuple `IN ((file_id, id), ...)` gives the planner no constant partition key, so it scans all 16 partitions in every chunk. Each group is one `FOR UPDATE SKIP LOCKED` query inside the chunk transaction. ## Metrics diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md index fa0def492b..d387fb3b34 100644 --- a/.serena/memories/backend/subtleties.md +++ b/.serena/memories/backend/subtleties.md @@ -45,7 +45,9 @@ - `job` is deletion-protected at the SQL level (`deletion_protection__tgr`): a stray `DELETE FROM job` raises instead of dropping the row and its `job_event` cascade without touching the referenced `storage_object`. The only intended deleters, the jobs GC and the submit dedupe, disable the guard with `SET LOCAL rules.deletion_protection TO off` inside their transaction; tests use `th/db-force-delete`. - A missing job-def raises (`:no-job-definition`) instead of completing. Throwing with `ex-data :type ::retry` still controls retry behavior; `:strategy ::noop` retries without incrementing retry count. - Cron entries claim their `scheduled_task` row with `FOR UPDATE SKIP LOCKED`, disable statement/idle-in-transaction timeouts locally, submit one `job` row per entry when no active instance exists (no-overlap), and reschedule themselves in `finally` unless interrupted. Worker, dispatcher, and cron components do not start when the DB pool is read-only. -- Integrant params schemas name injected components as bare `::ns/key` entries (pool, metrics, msgbus, job defs), and a bare entry is a malli registry lookup, not `any?`: the owning namespace must `sm/register!` it at load, or the backend crashes at boot with `:malli.core/invalid-schema` instead of asserting. Unit tests never build those keys, so `backend-tests.worker-test` compiles the three worker schemas to catch it. +- Integrant params schemas name injected components as bare `::ns/key` entries (pool, metrics, msgbus, job defs), and a bare entry is a malli registry lookup, not `any?`: the owning namespace must `sm/register!` it at load, or the backend crashes at boot with `:malli.core/invalid-schema` instead of asserting. Unit tests never build those keys, so `backend-tests.worker-test` compiles the three worker schemas and `backend-tests.http-websocket-test` compiles the revocation-watcher one to catch it. The assertion only runs with `PENPOT_FLAGS=enable-backend-asserts`, so without it a broken schema reaches production and only crashes at boot. +- Whether a bare entry works has nothing to do with its position or with how many of them there are: it is a registry lookup, so it works only if that exact keyword was `sm/register!`'d. `[:map ::db/pool ::metrics/metrics]` is fine, `[:map :app.aaa/unregistered :app.ccc/unregistered]` throws. When a component is **not** registered as a malli type, spell it as an explicit pair instead of registering a type just to satisfy the lookup — `[:app.nitrate/client {:optional true} [:maybe :map]]` is how `auth.oidc/schema:routes-params` and `loggers.audit.archive-task/schema:audit-log-archive-params` declare it. `{:optional true}` matters because `init-key` yields `nil` when the owning feature flag (`:admin-console`) is off, and a required key would then fail the assert on a perfectly normal boot. +- `sm/valid?` catches every `Throwable` and returns `false`, so an uncompilable schema and a genuinely invalid value both surface as a bare `Assert failed:` with no cause. When an `assert-key` fires with no message, compile the schema by hand to see the real exception. ## Config and HTTP/session middleware @@ -92,21 +94,21 @@ ## File data persistence -- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data. -- `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob. +- File data backends for writing are `db` and `storage`; `legacy-db` is a read-only alias for old payloads. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores nil data and points at the object with the `storage_object_id` column (FK `NO ACTION DEFERRABLE`). +- `fdata/upsert!` touches the storage object the persisted row pointed at before replacing it; the pointer travels in the params (`file->file-data-params` from the read file map, `nil` for a new row, the row for the offload) and the params of `update-file!` cannot override it. `binfile.common/update-file!` merges the read file map with the fields to change, so the pointer survives; `insert-file!` drops it because an INSERT never replaces a persisted object. The column is the only source of truth and the metadata no longer carries `storage-ref-id` (the migration removes it from the backfilled rows and the write path never stores it); the release is a coordinated restart when the storage backend is in use (the previous code cannot read the new rows), while the default `db` backend reads the same data column and tolerates a rolling deploy. Responses built from a file map go through `bfc/file->response`, which strips the pointer and a residual `storage-ref-id` from the metadata. - Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. A pointer keeps its id while nothing changes; content (`assoc`/`without`) and metadata (`with-meta`) changes produce a fresh id, so a fragment row is never rewritten after it is written. Re-assoc with an identical value returns the same map in array and hash maps alike, so a save keeps the fragment id of every page it does not change. - `enable-objects-map` skips unloaded pointer pages: their fragment was already wrapped when it was written, and loading it only to wrap it again costs one page read per untouched page on every save. A page with a plain fragment is wrapped when it is first loaded or edited; a data migration that enables `fdata/objects-map` must rewrite the data itself. - Snapshots share fragment rows with the live file (they are stored with pointers, not materialized). `file-gc` marks every fragment referenced neither by the current main file nor by an unpurged snapshot; the file is read unrealized and unchanged pages keep their ids via reattach, so a pass over an unchanged file writes no new fragments. - A save attaches its automatic snapshot before applying pending file migrations: the snapshot keeps the pre-migration pointers, data and migration set, and restoring it leaves the next read to migrate the file. This also holds for files without the feature. The preview (`get-file-snapshot`) applies the pending migrations in memory with the live file's libraries and never persists them: the stored version keeps its data, the payload carries the snapshot's `:migrations` (an ordered set) and `fdata/objects-map` is realized for clients that do not declare it. The historical feature set is deliberately not checked against the enabled one: a version may predate a feature change. The window is narrow: the `get-file` RPC reads with `migrate?` on and its ETag hashes `available-migrations`, so a post-deploy read cannot be a 304 from the pre-deploy state and migrates before any save in the normal flow; the pre-migration snapshot only appears when the save is the first request to touch the file after the deploy (an open tab that saves without re-reading, or a non-UI writer). `create-file-snapshot` and the restore backup read with `migrate?` on, so those versions are stored already migrated. - `:migrations` is always a `d/ordered-set`: `file-migrations/resolve-applied-migrations` builds one, `fmg/migrate` re-wraps the union after applying, and `file-snapshots/decode-snapshot` converts the `file_change` pgarray (as a plain vector, `set/difference` and `contains?` would compare indexes instead of names). A snapshot with NULL `:migrations` and an old `:version` predates the `file_change.migrations` column (SQL migration 0140): the preview regenerates the set from the stored version and applies that full chain, while `restore!` uses a fixed 67 baseline, so preview and restore can disagree for very old snapshots. -- The migration read path (`binfile.common/migrate-file`) captures `fdata/original-pointers` before realizing, reattaches unchanged positions after applying the migrations, and persists the reattached file; it returns the realized file because reads expose plain data. A migration that adds `fdata/objects-map` skips the reattach so the pages get wrapped. +- The migration read path (`binfile.common/get-file*`) reads the `file` row once (`sql:get-file`, with its data columns; with the storage backend the blob lives in the object, not in the column) plus an explicit `resolve-applied-migrations`, because a row read does not resolve them. A read with no pending migration is completed by `attach-file-data` on that map: one row read and one migrations resolution. Only when a migration is pending is the row re-read under `FOR UPDATE` on the `file` row, where `need-migration?` is re-checked on the fresh map and the migrated file persisted (`binfile.common/migrate-file`); that re-read resolves the migrations again, and `migrate-file` once more after persisting, to return the applied set. The migration path holds two raw copies of the blob (the first read and the locked re-read) but a single decoded one: the alternative, a row-only probe before the read, avoided the first raw copy at the cost of an extra row read on every open, and was removed. `get-file` runs the read inside `db/tx-run!` (joining the caller's transaction when the cfg carries a connection) so the persist commits with the read; a read-only read (`:read-only?`) takes no lock and writes nothing, so it runs on a plain connection (`db/run!`) instead of opening a transaction of its own, and with a cfg that already carries a connection it still joins the caller's. It accepts a cfg map only and throws an `IllegalArgumentException` otherwise (`assert` is elided without `:backend-asserts`); a data read rejects `:lock-for-share?` (only row-only reads accept it), because the migration branch re-reads with `FOR UPDATE`, and a lock option needs a read-write transaction from the caller because PostgreSQL rejects row locks in a read-only transaction. A row-only read (`load-data?` false) resolves the applied migrations by default too (`:load-migrations?`), because the row can be completed later and the caller may need them; a caller that does not want them passes `:load-migrations? false`. Persisting is an optimization of the read (the read already returns migrated data): a row that changed between the first read and the locked read is not overwritten, and a pending migration discovered on the unlocked read is applied in memory only (the next read persists it under the lock). This was the only writer of the file row outside the lock protocol (save, library absorption, restore, `file-gc` and offload already take it); without it, a save that committed in between was overwritten by the stale map (lost content, `revn` moving back so the next save fails with `:revn-conflict`, and the object the save left as a frozen orphan). `migrate-file` captures `fdata/original-pointers` before realizing, reattaches unchanged positions after applying the migrations, and persists the reattached file; it returns that reattached map when it persists (the pages the migration does not change keep their pointers and fragment ids) and the realized map when it does not, so a read is plain only when the caller asks with `:realize?`, and a migration that adds `fdata/objects-map` skips the reattach so the pages get wrapped. - `fdata/load-pointer` reads rows that a previous `file-gc` pass marked on purpose (explicit `::db/remove-deleted false`): the fragment is only gone once `objects-gc` purges it. The `get-file-fragment` RPC keeps rejecting marked rows. - Bind the fragment loader through `fdata/with-pointers` (reads), `fdata/with-tracked-pointers` (writes with a loader) or `fdata/with-tracked` (writes with nothing to load yet, e.g. file creation), and `fdata/with-untracked` (validation passes that must track nothing) instead of hand-rolled `binding` forms. Two direct bindings remain on purpose: the fine-grained tracked toggles of `app.rpc.commands.files-update` (validation/repair turn tracking off, restore the existing atom and read it before repairing), and `binfile.common/encode-file` (a fresh tracked set inheriting the current one). Converge them only if atom-scoped macros (`with-tracking`, `with-inherited-tracking`) are added; today the extra indirection buys nothing. -- Snapshot creation takes `FOR SHARE` on the file row: both `create-file-snapshot` and the backup read of `restore-file-snapshot` hold it from the read to the insert, so they serialize with a `file-gc` pass (`FOR UPDATE ... SKIP LOCKED`) instead of pinning fragments the pass is about to mark. `get-file-snapshot` realizes the snapshot pointers with the live file id (fragment rows are keyed by file, not by snapshot). +- Snapshot creation takes `FOR UPDATE` on the file row: both `create-file-snapshot` and the backup read of `restore-file-snapshot` hold it from the read to the insert, so they serialize with a `file-gc` pass (`FOR UPDATE ... SKIP LOCKED`) and with a concurrent save, and never pin fragments the pass is about to mark. The lock is exclusive and not `FOR SHARE` because the migration-on-read path may re-read the same row with `FOR UPDATE`: a shared lock that asks for the exclusive one deadlocks two concurrent readers (one creating a snapshot, the other saving, both seeing a pending migration). `get-file-snapshot` realizes the snapshot pointers with the live file id (fragment rows are keyed by file, not by snapshot). - `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. - A `file_data` row is one of three classes, told apart by `type`: `main` is the file's current state and carries `id = file_id`; `snapshot` is a point-in-time copy whose `id` is the one its `file_change` row shares; `fragment` is a piece of the pointer-map, referenced from the file's data. - Their lifecycles differ: `main` and `snapshot` live with the file (dated when it is deleted, undated when it is restored), while a `fragment` carries no date and is collected by `file_gc` once the file stops referencing it. Neither is a session buffer, so do not read a `fragment` as an ephemeral revision. -- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only written by the `legacy-db` backend, plus the rows written before the 2025-07 file storage refactor. +- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only read by the `legacy-db` backend (it is no longer written), plus the rows written before the 2025-07 file storage refactor. - Tell a snapshot apart by `label IS NOT NULL` (with `created_by` / `deleted_at` as needed), never by `file_change.data IS NOT NULL`: the label holds with every file data backend, and the payload column stopped meaning "this is a snapshot" when the payload moved out. Queries that identify versions must use the label. `0157-migrate-legacy-file-data` moved the labelled leftovers to `file_data` and deleted the unlabelled payload rows, so from 2.19 on the label is the only marker there is. - The version quota counts of `app.rpc.quotes` (`sql:get-snapshots-per-file`, `sql:get-snapshots-per-team`) are served by the partial index `file_change__user_snapshots__idx`, whose predicate mirrors their where clause; changing one without the other drops the index only scan and the count walks the whole table again. - Revisions live only in `file_change` and have no `file_data` counterpart. They are born already scheduled for deletion one hour later (`app.rpc.commands.files-update`) and the snapshot paths cannot reach one, because they resolve their row through a `label IS NOT NULL` filter. diff --git a/.serena/memories/backend/testing.md b/.serena/memories/backend/testing.md index e06f9ace32..61c92d8fed 100644 --- a/.serena/memories/backend/testing.md +++ b/.serena/memories/backend/testing.md @@ -3,12 +3,15 @@ JVM `clojure.test` (kaocha runner) under `backend/test/backend_tests/`. - READ `mem:testing` FIRST — it defines the execution discipline (no piping, tee to file, preferred commands) that applies to all JVM test runs. +- Test DB/Redis come from the `PENPOT_TEST_*` env vars already set in the shell (`PENPOT_TEST_DATABASE_URI`, `PENPOT_TEST_REDIS_URI`, ...); per instance they point at `penpot_test` / `penpot_test_wsN` and valkey DB `6+N` (`mem:devenv/core`). Do not override them with values copied from CI logs or other sources unless the user asks (e.g. "use a temporary database"); the CI env (`redis://redis/1`, `postgresql://postgres/penpot_test`) only works inside the CI docker network. +- The test database may have migrations applied from newer versions (it is shared dev infra). Reinitializing it — `dropdb`/`createdb` of the `PENPOT_TEST_*` database only — is explicitly permitted when that blocks a test run; never drop the main `penpot` database or another instance's test database. - All CLI commands must be executed from the `backend/` subdirectory. - Tests are invoked directly via `clojure -M:dev:test` (kaocha) — there is no pnpm wrapper. Kaocha auto-discovers test namespaces, so no runner registration is needed. - **Preferred way to run them: `scripts/ci --test`**, which runs lint, tests and format per module and writes a log per task under `.ci-logs/`. It exports `PENPOT_FLAGS` with `enable-backend-asserts` appended, so use it rather than calling `clojure -M:dev:test` by hand. - **Why the flag matters:** `clojure.core/assert` is elided at compile time when `*assert*` is false, and `app.config` only sets that var from the flags when it loads. The test fixture redefines the flags after that, so they never reach it: without the flag the suite runs with every assertion compiled out, and a test that depends on an assert only means something with it set. `scripts/ci` and `tests-backend.yml` export the flag (GitHub Actions sets it too). When running the suite directly — including a `--focus` run — the command is `PENPOT_FLAGS="enable-backend-asserts" clojure -M:dev:test`. - Coverage: if code is added or modified in `src/`, corresponding tests in `test/backend_tests/` must be added or updated. -- Isolated run: `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace, or `clojure -M:dev:test --focus backend-tests.my-ns-test/my-test-var` for a specific test var. +- Isolated run: `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace, or `clojure -M:dev:test --focus backend-tests.my-ns-test/my-test-var` for a specific test var. Keep the `PENPOT_FLAGS="enable-backend-asserts"` prefix on these too; a focused run without it hides assert-dependent failures exactly like a full one. +- A **whole-suite** crash reported as `Execution error ... Assert failed: ...` with no `FAIL in` line is not a test failure: an `ig/assert-key` ran while the test system was being built, which happens outside `clojure.test`, so kaocha reports it as a load/build error and the summary has no counts. Fix the offending schema (see `mem:backend/subtleties`) rather than hunting for a test. - Multiple focus symbols are supported by REPEATING the flag (kaocha accumulates them, last-wins does not apply): `clojure -M:dev:test --focus backend-tests.ns-one --focus backend-tests.ns-two` — preferred over a shell loop of isolated runs. - Regression run: `clojure -M:dev:test` to ensure no regressions in related functional areas. - If you need to filter output, tee to a temp file first: `clojure -M:dev:test 2>&1 | tee /tmp/penpot-test-output.txt`. diff --git a/.serena/memories/backend/websocket-subscriptions.md b/.serena/memories/backend/websocket-subscriptions.md new file mode 100644 index 0000000000..ab56219f81 --- /dev/null +++ b/.serena/memories/backend/websocket-subscriptions.md @@ -0,0 +1,44 @@ +# Backend WebSocket Subscriptions + +## Registry + +- `app.http.websocket/state` holds `{:connections {conn-id → wsp} :by-profile {profile-id → #{conn-id}}}` in **one** atom, so a single `swap!` keeps both consistent; `register-connection`/`unregister-connection` maintain them together and drop a profile from the index once its last connection leaves. +- Each connection's own `::ws/state` atom holds its subscriptions (`::file-subscription`, `::team-subscription`). `state` is shadowed by `::ws/state` in every `handle-message` destructuring, so only `on-connect`, `on-disconnect` and the `repl-*` helpers touch the registry — reading the registry from a handler is a bug. +- `::profile-id` and `::session-id` on a connection live in the `app.http.websocket` namespace, while `::ws/id`, `::ws/state`, `::ws/output-ch`, `::ws/session-id` live in `app.util.websocket`. Mixing them up yields silently `nil` values (a `nil` `session-id` makes the channel's `:xf` drop **every** message). +- The registry is local to one backend instance. The message bus is shared (`mem:prod-infra/core`). Anything that must reach connections owned by another instance has to go through the bus — closing a connection from an RPC handler only ever works in a single-backend deployment. + +## Revocation model + +A subscription is authorized once, at `:subscribe-file`/`:subscribe-team`, and that decision used to be trusted for the whole connection life (GHSA-m53j-2766-6jqw, medium). Two independent mechanisms now bound it: + +- **Announcement (primary).** A mutation that can change access publishes on `app.rpc.notifications/internal-revocation-topic` (`notify-permissions-changed` for one profile, `notify-team-permissions-changed` for one team id — the watcher resolves the members itself, so a large team is one event, not N). The `::revocation-watcher` Integrant component in `app.http.websocket` subscribes and calls `revalidate-profile-subscriptions`, which looks the profile up in `:by-profile` and re-checks every subscription with `file-readable?`/`team-readable?`: existence (the file gate is the same `sql:get-minimal-file` as the `:subscribe-file` gate, made non-throwing via `{:db/throw-if-not-exists false}`; the team gate is `db/get*` with `::sql/columns [:id :deleted-at]` — `deleted_at` must be in the projection or the soft-delete filter is a no-op) AND the `has-read-permissions?` predicates — and `sql:file-permissions` additionally filters `deleted_at` on file, project and team, as do `sql:get-team-id-for-file`/`-project` plus a final team-row check in `resolve-team-id`, so the organization-owner fallback grants nothing through soft-deleted file/project/team. The existence gate closes that fallback (expelled org owner, deleted resource). The event is only the trigger; the fresh check is the decision, which is what keeps a downgrade to `viewer` (still readable) and a non-member organization owner on a live resource from being mistaken for a revocation. Both publishers defer through `db/after-commit`, because the watcher checks on another connection and would otherwise see the pre-commit state. `:subscribe-team` deliberately has no existence gate of its own; only the re-checks add one, matching the file side (user decision, PR #12189 review). +- **Interval (safety net).** `start-relay` runs the relay as a `go-loop` that waits on the subscription channel and a one-shot tick channel together via `sp/alts!` with `:priority true` on the tick: when both hold something, the tick wins, so a steady stream of messages can never push the re-check away. The cadence is `subscription-revalidation-interval` (default `default-revalidation-interval`, 5 min, a code constant in `app.http.websocket` and **not** in the `config.clj` default map, same rule as session lifetimes). The deadline is fixed per cadence: a single tick channel covers a whole cadence and message turns reuse it (minting a fresh timeout per turn would both churn timers and starve under load exactly like an in-loop timer does); a new one is minted only after a tick fires. It covers what no announcement can: Nitrate transferring organization ownership (an HTTP call to another service) and rows deleted straight over SQL. Closing the channel ends the loop either way, because a take on a closed channel is always ready with nil — every way a subscription can end closes it, so a finished relay never keeps polling. Cost is three or four queries per subscription per interval (existence row + predicates), plus one Nitrate call per team per interval when the `:admin-console` flag is on (cached 30 s); one `timeout-chan` per subscription per cadence on promesa's default scheduler (one or two threads shared with every other scheduled task) — do not shorten the interval without sizing that. A tick abandoned at teardown (the subscription closed while it was pending) still fires its scheduled close harmlessly. Two places fail open, deliberately: `still-authorized?` (a transient DB error logs and keeps the subscription; `InterruptedException` is rethrown) and the `catch Throwable` in `watch-revocations`. + +Which mutations announce: `delete-team-member`, `leave-team`, `update-team-member-role` (member only), `delete-team`, `delete-file`, `delete-project`, `move-files`, `move-project` (one team event each). Anything not listed here is covered only by the interval. `delete-share-link` deliberately does not: a share-link visitor never passes the websocket subscribe check, because `check-read-permissions!` is called without a `share-id` and that arity only considers team membership. + +`close-file-subscription`/`close-team-subscription` are idempotent: no-op on a mismatched id, and they `dissoc` the subscription so a repeat call and `:pointer-update` do not publish to a dead topic. Closing the channel is what ends the relay (`take!` on a closed channel returns `nil`). + +## Topic and layering + +- `internal-revocation-topic` is a fixed uuid that is never a real resource id, so it cannot collide with a file, team or profile topic — and deliberately not the `uuid/zero` system topic, which is `sp/pipe`d straight into the client output and anything published there reaches every client. No client can subscribe to the internal topic: `:subscribe-file`/`:subscribe-team` only accept ids that already passed a permission check, and `:broadcast`/`:pointer-update` only republish to ids the server already holds. +- The topic and the publish helpers live in `app.rpc.notifications`, not in `app.http.websocket`: the command namespaces cannot require the watcher, because `app.http.websocket` already requires `app.rpc.commands.files`/`teams`. The watcher keeps the registry, which is the part it owns. + +## Time and channel traps in this namespace + +- The relay waits with `sp/alts!` over `[tick-ch channel]` with `:priority true`, following the `msgbus.clj` pattern (`start-io-loop` destructures the same `[val port]` shape and compares ports with `identical?`). A take on a closed channel is always ready with nil, which is what ends the loop — and what makes closing the channel from any teardown path prompt even while parked in `alts!`. `sp/timeout-chan` is one-shot: it closes once after the delay, and one abandoned at teardown still fires its scheduled close harmlessly. +- `ct/diff a b` returns **b − a** (`Duration/between`), so "time until a future instant" is `(inst-ms (ct/diff (ct/now) future))`. +- `ct/duration` unit keywords are singular: `{:millis 150}`, `{:minutes 5}`, never `:milliseconds`/`:days`. +- `still-authorized?` takes the check itself: `(boolean (check-access))`. Calling `(boolean pred)` on the closure is always true and silently disables the interval — this exact bug shipped once and no test caught it until the resubscribe-counting test existed. Naming rule here: no `?` for the action that queries and can fail (`check-access`), `?` only for the safe yes/no answer (`still-authorized?`). + +## Tests + +`backend/test/backend_tests/http_websocket_test.clj` drives the real Redis-backed msgbus, so the revocation tests are end-to-end: register the connection in the real registry, run the real command, wait for the subscription to drop, then assert no content arrives. A connection built without `register-connection` is invisible to the watcher, so the watcher-based tests must go through `open-registered-connection`. Interval tests shorten the config via `with-redefs` on `cf/get` rather than waiting minutes. `with-clean-registry` runs as an `:each` fixture, so no test can leak a live relay loop into the next one. `captured-events` forwards the keyword args through to the real `mbus/pub!` — rebuilding them positionally publishes to the topic `:message` with a `nil` payload and silently untests every other notice. Interval tests take access away with a direct row delete, never with a command: a command announces, so the test would pass even with the interval removed. The same rule covers the two counting/fan-out tests: `revocation-closes-subscriptions-that-are-no-longer-authorized` takes access away with `delete-team-membership!` (raw row delete, publishes nothing) because a command announcement lets the live watcher close both subscriptions before the test's own `revalidate-profile-subscriptions` call, and its `(= 2 …)` count then reads 0; `team-event-closes-every-member-subscription` soft-deletes the team row (`sql:team-members` joins `team_profile_rel` and `profile` only, so the watcher still resolves every member) and asserts nothing closed before one synthetic `notify-team-permissions-changed`, leaving the command path to `deleting-a-team-announces-once` and `deleting-a-team-closes-open-subscriptions`. The resubscribe-counting test is the one that proves finished relays stop polling. The soft-delete + org-owner re-check tests pair a direct row soft-delete with `with-redefs` on `nitrate/organization-owner-of-team?`: before the existence gate, the org-owner fallback kept the subscription open on a deleted resource. The soft-deleted-project/team-closes-org-owner-file-subscription tests pin the root fix in `app.rpc.permissions` (`resolve-team-id` refuses dead file/project/team for every caller of the fallback). `watcher-consumes-revocation-events-from-the-bus` takes access away with a direct row delete and drives the watcher with a synthetic bus event — never a command and never a second `notify-*` — so the "still subscribed" assertion cannot race an after-commit hook and a broken command announcement cannot mask a broken watcher. `moving-a-project-closes-source-team-subscriptions` pins the move-project asymmetry: the file subscription drops, the source-team subscription survives (the member is still a member). + +`revocation-watcher-schema-resolves-every-ref` covers the `::revocation-watcher` params schema, which no other test compiles. It existed broken for a while: the three injected keys were written as bare keywords, which is a valid malli *property* position and not a key, so the schema raised `:malli.core/invalid-schema` — invisible unless `PENPOT_FLAGS=enable-backend-asserts` was set, and a crash of the whole system build rather than one test (see `mem:backend/testing` on the flag). + +## See also + +- `mem:backend/auth-permissions-product-domains` for the permission predicates and the not-found semantics that the re-checks reuse. +- `mem:prod-infra/core` for why the bus crosses backends and the registry does not. +- `mem:frontend/routing-app-shell-subtleties` for the client side: the frontend already reacts to `team-membership-change` by navigating away, which is why revocation needs no new client message. +- `.agents/plans/2026-10-07-websocket-subscription-revocation.md` for the original plan, and the sibling `.agents/plans/2026-10-05-leave-team-file-access-cleanup.md` whose hardened permission queries this re-verification depends on (without live team membership required in `sql:file-permissions`, the re-check would keep reporting access for a member who already left). diff --git a/.serena/memories/clojure/design-rules.md b/.serena/memories/clojure/design-rules.md index 2b439ee038..e39facedc3 100644 --- a/.serena/memories/clojure/design-rules.md +++ b/.serena/memories/clojure/design-rules.md @@ -8,6 +8,7 @@ How to shape a function in this codebase. Each rule came from a review round; do - **Functions that obtain data start with `get-` or `fetch-`.** A reader must see at a glance that the function reads (`get-file`, `get-snapshot`, `get-file-json`); a bare noun hides it (`file-json` says nothing about reading). New helpers that fetch by id follow this, never a `current-` prefix: the id parameter already says which one it is. - **`?` marks a predicate, not a data prop.** A var or function that answers a question takes `?` (`export-error?`, `(defn editable? …)`); a map key that names a property does not (`:export-error`, never `:export-error?`), even when its value is a boolean. Existing `?` keys are legacy to remove gradually; do not add a new one. - **Do not shape production code for tests.** No return values added just to assert them; assert on the observable effect. When preconditions move, move the tests with them. +- **All code-facing text MUST be in English** — identifiers, comments, docstrings, test descriptions. No exceptions; see the hard rule in `AGENTS.md`. Reference implementation: the http metrics samplers in `app.http` (`sample-worker-metrics` / `sample-connector-metrics` are guard-free; `sample-http-metrics` guards at the boundary). diff --git a/.serena/memories/clojure/idioms.md b/.serena/memories/clojure/idioms.md index 3fa120aa62..e8c7bc299e 100644 --- a/.serena/memories/clojure/idioms.md +++ b/.serena/memories/clojure/idioms.md @@ -11,4 +11,5 @@ Behaviors confirmed against the language/stdlib — do not re-derive from assump - Inside an `^:async` fn, a nested call that needs a temp binding (e.g. calling a local fn `(g (f x))`, whose arg gets a `G__` binding) compiles to `(await (async function(){…})())`: `await` unwraps a returned promise, so `(settled (upload (make-ctx)))` blocks until the upload promise settles instead of passing the promise on (a test that pushes the answer later hangs, idle). Bind each step in its own `let` binding. - `!` suffix allowed cases (only exceptions to the no-new-`!` rule in `mem:clojure/design-rules`): mutating public `app.db` API (`insert!`, `update!`, `delete!`, `run!`, `tx-run!`, `exec!`, ...) and fns whose contract is to work over transients (stdlib `conj!`, `assoc!`, `dissoc!`, `disj!`, `persistent!` and custom helpers over them); any further case must be registered here with justification before use. - Penpot extends `clojure.core/Inst` in `app.common.time` ( JVM-only section), so `inst-ms` also accepts `Duration` (returns `.toMillis`, NOT epoch millis), `FileTime` and `OffsetDateTime`. It only works where that namespace is loaded; on a bare `Duration`, prefer explicit `.toMillis`, which reads as what it is (an amount, not an instant). +- The threading macros insert the threaded value at DIFFERENT positions: `->` and `some->` put it FIRST (`(-> x (f a))` → `(f x a)`), `->>` and `some->>` put it LAST (`(->> x (f a))` → `(f a x)`). `some->`/`some->>` also short-circuit on nil. With unary calls both nest the same way (`(-> x (f) (g))` and `(->> x (f) (g))` are both `(g (f x))`): what changes between them is the position inside each call, not the order of the steps. A pipeline that calls a fn whose first argument is a config/context (`app.features.fdata/resolve-file-data` is `(cfg, file)`) must use `->>`/`some->>`; with `->`/`some->` the arguments are swapped and the call silently returns the wrong value (it cost a broken `get-snapshot` in `app.features.file-snapshots`). - `cuerdas.core/index-of` returns `nil` (not `-1`) when the substring is absent, like `clojure.string/index-of`. `(pos? nil)` **throws on the JVM** but evaluates to `false` in CLJS (JS coercion), so `(if (pos? (str/index-of s "\n")) …)` works in the browser and crashes the backend. Guard with `some?` before `pos?` (this was `app.common.exceptions/first-line`). diff --git a/.serena/memories/clojure/logging.md b/.serena/memories/clojure/logging.md new file mode 100644 index 0000000000..56fba6a1b3 --- /dev/null +++ b/.serena/memories/clojure/logging.md @@ -0,0 +1,7 @@ +# Clojure Logging Conventions + +Structured logging through `app.common.logging` (`l/trc`/`l/dbg`/`l/info`/`l/wrn`/`l/error`/`l/fatal`): `:hint` carries the human-readable message, every other datum goes in its own keyword field (`:id`, `:container`, `:removed`, ...). + +- `:hint` values start lowercase: `"fix dangling children"`, not `"Fix dangling children"`. Uppercase-first hints exist as legacy (~57 occurrences, the densest in `backend/src`; also `l/trc` in migration `legacy-28` and `log/error` in `app.common.files.repair`) — do not copy them, and do not mass-rewrite them unless the surrounding code needs the change. +- A name that requires uppercase (an env var like `PENPOT_MEDIA_PROCESSING_SERVICE_URI`, an acronym like `MCP` or `LDAP`) stays uppercase inside an otherwise lowercase sentence. +- Repair work done by a file migration logs at `l/trc`, only when it changes something, and with the ids it touched: the house style of `migrations.cljc` (`legacy-28` precedes; the file has no `warn`/`info` lines). The shipped `log4j2.xml` pins the `app` logger to `info` (static file, `monitorInterval="60"`), so those trace lines appear only after the operator raises the level; the durable record of a repaired file is the migration id in its `:migrations`. diff --git a/.serena/memories/common/decimals-and-coordinates.md b/.serena/memories/common/decimals-and-coordinates.md index 0a4e8992ad..1cc966f452 100644 --- a/.serena/memories/common/decimals-and-coordinates.md +++ b/.serena/memories/common/decimals-and-coordinates.md @@ -52,3 +52,8 @@ its own drift. Comparing `:selrect.width` from shape A to `:selrect.width` from shape B is comparing two values that "semantically" should be equal but were computed via different operation chains — exact equality will often be false. + +## Library sync comparison + +- `update-attrs` (`app.common.logic.libraries`) compares main vs copy attrs exactly first, then with tolerance for: `drifting-geometry-attrs` (`:x :y :width :height :selrect :points`, via `gsh/close-attrs?`); path and bool `:content` (`segment/content-close?`; bool shapes keep their path in `:content`); and text `:position-data` (`position-data-close?` over `mth/close-map?`: every numeric field of every entry within `mth/close?`, other fields such as fills, fonts and text exact). All other attrs (`:opacity`, radii, `:transform`, `:rotation`, text `:content`) stay exact: tolerance there hides real changes. +- `close-attrs?` pairs lists with `d/zip`, which truncates to the shorter list and treats nil vs vector as equal for `:points`/`:position-data`. Do not use it for variable-length data. That is why `:position-data` has its own helper, and why `attr-equal?` checks the `:points` count before calling it. diff --git a/.serena/memories/common/file-change-validation-migration-subtleties.md b/.serena/memories/common/file-change-validation-migration-subtleties.md index 75d9d80b4c..fab22a45fd 100644 --- a/.serena/memories/common/file-change-validation-migration-subtleties.md +++ b/.serena/memories/common/file-change-validation-migration-subtleties.md @@ -33,4 +33,5 @@ - `migrate-file` synthesizes legacy migration ids from old numeric versions when `:migrations` is absent, migrates legacy features, and records feature flags created through `cfeat/*new*`. - When a file had no previous `:migrations`, `migrate-file` marks all migrations as migrated in metadata so callers persist the complete migration set, not only transformations that changed data. - Closed schemas (`schema:stroke-attrs` and others) reject a key that was dropped from the declaration, so removing a declared attr needs a migration in the same change, for data already written with it. -- Rebase collision on a migration number: upstream keeps the id. Renumber the branch migration to the next free one, keep it last in `available-migrations`, and rename everything that spells the number — its test file and namespace (`files_migrations_00NN_test.cljc`), the test names, and comments in production code or memories that name it. \ No newline at end of file +- Rebase collision on a migration number: upstream keeps the id. Renumber the branch migration to the next free one, keep it last in `available-migrations`, and rename everything that spells the number — its test file and namespace (`files_migrations_00NN_test.cljc`), the test names, and comments in production code or memories that name it. +- Withdrawing a migration id: remove both the `defmethod` and the id from `available-migrations`; the replacement migration subsumes it and files that recorded the old id re-migrate through the replacement (persisted `:migrations` keep the old id via union — do not rewrite them, including playwright fixture JSONs). Idempotent/pure-cleanup migrations only; never withdraw one whose transform must not run again unless the replacement is safe for those files. Precedents: "0021-fix-shape-svg-attrs" -> "0035-fix-shape-svg-attrs-and-defs"; "legacy-36" -> "0036-clear-nil-keys-in-objects". diff --git a/.serena/memories/critical-info.md b/.serena/memories/critical-info.md index f1c5a2ba4a..ca15de08db 100644 --- a/.serena/memories/critical-info.md +++ b/.serena/memories/critical-info.md @@ -6,6 +6,7 @@ You are working on the GitHub project `penpot/penpot`, a monorepo. - A section's top-level memory is `
/core`. When a section is relevant, read the core memory before focused memories. - Edits/stale refs/duplication cleanup: `mem:memory-maintenance`. - Cross-cutting testing principles, TDD workflow, and anti-patterns: `mem:testing`. +- Structured logging conventions (`:hint` casing and fields, log levels): `mem:clojure/logging` — read before adding or changing any log call. - Verified Clojure language behaviors that contradict common assumptions (e.g. `int?` covers `Long`; `integer?` is the general predicate): `mem:clojure/idioms` — read before assuming stdlib predicate semantics. - Read `mem:clojure/design-rules` (function shape, boundary checks, naming) before writing or modifying Clojure/ClojureScript code — not only when designing a solution or an API. - Promises: which call style to use where (`fmap`/`mcat`/`merr` vs `then`/`catch`) and how they bridge to rx: `mem:clojure/promesa` — read before chaining promises or mixing them with observables. diff --git a/.serena/memories/devenv/core.md b/.serena/memories/devenv/core.md index 3507a5d6e0..a6645ed04f 100644 --- a/.serena/memories/devenv/core.md +++ b/.serena/memories/devenv/core.md @@ -15,6 +15,7 @@ Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Par - For ws1+, `instance-env-overrides` computes the per-instance overrides (container/volume names, host ports offset `10000·N`, `PENPOT_PUBLIC_URI`, `PENPOT_REDIS_URI`, `PENPOT_TEST_DATABASE_URI`, `PENPOT_TEST_REDIS_URI`) and `instance-compose` injects them as env vars at compose time — never written to disk, recomputed each call so they can't drift. ws0 uses `defaults.env` as-is. - Per-instance test databases are created by `ensure-instance-test-database` (called from `start-instance`): `CREATE DATABASE` guarded by a `pg_database` existence check, so it is idempotent. Needed because `postgresql_init.sql` only runs on first volume creation. The backend applies migrations itself on first use, so an empty database is enough. - `backend/scripts/_env`: backend-internal only — secret keys, `PENPOT_FLAGS` (with `enable-backend-worker` gated on `PENPOT_BACKEND_WORKER`), `JAVA_OPTS`, `setup_s3_bucket()`. Never duplicates `defaults.env`. +- Frontend admin-console is off by default in devenv: `docker-compose.main.yml` sets `PENPOT_DEVENV=true`, which makes `app.config` (`goog-define devenv-mode`) prepend `disable-admin-console` ahead of the user `config.js` flags; flags parse in order and user flags win, so `enable-admin-console` in `frontend/resources/public/js/config.js` re-enables it. Outside devenv (prod builds) the default is unset and the admin-console stays on, matching `flags/default`. - Compose files use pure `${VAR}` substitution; missing var = compose fails. ## Invariants diff --git a/.serena/memories/frontend/core.md b/.serena/memories/frontend/core.md index 139859bc98..e0e84e4d9f 100644 --- a/.serena/memories/frontend/core.md +++ b/.serena/memories/frontend/core.md @@ -12,7 +12,7 @@ Frontend: CLJS SPA; React/Rumext; Potok; RxJS; okulary refs; SCSS modules; share - `app.render_wasm.*`: frontend bridge to Rust/WASM renderer. - `app.util.*`: DOM, HTTP, i18n, keyboard, codegen, and general frontend utilities. - `frontend/packages/*` and `frontend/text-editor`: JS/TS workspace packages consumed by the app. -- Nitrate subscription/organization UI and flows live under `app.main.data.nitrate` and `app.main.ui.nitrate*`; backend/API behavior is covered by backend memories, and shared permission rules are in `common/src/app/common/types/nitrate_permissions.cljc`. +- Nitrate subscription/organization UI and flows live under `app.main.data.nitrate` and `app.main.ui.nitrate*`; backend/API behavior is covered by backend memories, and shared organization permission rules (`allowed?`, `can-send-invitations?`) are in `common/src/app/common/types/organization.cljc`. - `app.config` external bridges (`external-feature-flag`, `external-session-id`, `external-context-info`, `external-notify-register-success`, `initialize-external-context-info`) delegate to `globalThis` hooks injected by the SaaS host. Never delete `external-feature-flag`, even with zero call sites: it is the seam for future A/B tests. diff --git a/.serena/memories/frontend/dashboard-viewer-subtleties.md b/.serena/memories/frontend/dashboard-viewer-subtleties.md index 14471e7195..148f5307e8 100644 --- a/.serena/memories/frontend/dashboard-viewer-subtleties.md +++ b/.serena/memories/frontend/dashboard-viewer-subtleties.md @@ -5,9 +5,11 @@ - Dashboard initialization fetches projects and fonts for the team, then listens to websocket messages only for global topic `uuid/zero` or the current profile id. - Project fetch replaces each project map completely instead of merging, so fields such as `deleted-at` can disappear cleanly. - Dashboard file/project mutations are often optimistic local updates with fire-and-forget RPC watchers. Bulk permanent delete/restore paths use SSE progress and progress notifications. -- The import wizard runs the package as a job of the backend: it reads the manifest of the file in the main thread (`app.main.data.imports`, no web worker) and follows the job, so its header paints the milestone of the job (the stage and its counter) instead of the messages the old SSE stream sent; every file of a container reports the same milestone, so `jp/current-milestone-text` writes it once and the rows never repeat it. Before the job exists, the header paints a client-side `:upload` milestone built from the chunked upload (`uploads/upload-blob-chunked` `:on-progress`, chunks done/total), which never comes from the backend; the chunks go one at a time and it writes them as a percentage of the file, not as a counter. Once created and still waiting for a worker, no row says anything: the header says the import waits for a slot while no job of it has started. The bar of the header measures the whole import, never a file: indeterminate while checking, waiting for a slot or rebuilding, because the client cannot know a global index and total of operations. The upload is the exception — the chunked upload knows its chunks, so the bar is determinate while the containers are sent, weighted by their size (`imp/upload-pct` over `:container-size`, the only percentage the client really knows). The export modal follows the same one-job-per-file model, but every file is queued up front (even the ones whose job is created only after the previous one is over) and its queue ends only with a milestone: `:start`/`:retry` do not end it there. Neither modal paints a generic "…ing file…" line on the rows any more: the rows of the import wizard carry only what is not progress — the name, the size, `Ready` once their container finished uploading, and the error text —, and the button that drops a file only shows while the wizard still offers its import step (the confirmation screen). Above the rows, the import wizard paints a stepper (Check/Upload/Import) plus one phase line: `imp/import-step` derives the step from the dialog status and the entries (upload holds while any file is pending, either `:import-ready` or carrying its own `:upload` milestone), so Check covers analyze, Upload covers the chunked client-side upload, and Import covers queue plus job. A file of a version 3 package shows what it occupies inside its container, computed while analyzing (`imp/file-sizes`): its own `files/…` entries plus the storage objects its media and thumbnails reference, whose metadata is the only link to `objects/…`. The container's own size stays in `:container-size` (what the dropped file weighs) and is what the upload bar weighs, never what a row shows. A file the manifest lists but the container holds no entry of has no size at all, and its row then shows no subtitle: what the package weighs never stands in for it. The subtitle is the size alone, with no extension. While analyzing with no failures nothing renders below the phase header — no rows, no spinner (`imp/awaiting-analysis?`); the header text is the only "Checking…" message. Rows appear once the analysis resolves (one per file inside the package), and a failed analysis takes the error-summary path instead. The ready step and its Continue wait for the end of the analysis (`imp/ready-to-import?`, fed with the completion of `imp/analyze`): a ready file is not enough, because the files still to arrive would never be imported. While rebuilding, the phase title counts the files and `imp/imported-count` counts them done with the job's own `:files` counter, so the number moves while a container rebuilds its files one by one. The `:error` of an import entry is user-facing text (the backend hint, or a translated label), never an i18n key: the wizard renders it as-is, it does not translate it again. The link-later step takes the file id from the `:id` of its resolution entry, never from the key of the resolution map: the map comes from the job result, so what a key holds depends on the result schema of the job-def that wrote it, while the wizard indexes its selection by the id of the entry. The legacy SSE import stays in the backend for other clients. +- The v3 format carries no size per file: the size of one is the sum of its own entries plus the `objects/` its media and thumbnail metadata reference, the only link between a file and that shared storage. +- Those metadata entries are read one at a time on purpose: a container holds thousands and every read opens its own decompression stream, which makes Chromium fail the analysis — zip.js then masks the cause behind a `Cannot close a ERRORED writable stream` — while Firefox tolerates it. Do not turn this back into a parallel read. The read is best-effort: a reference that cannot be read leaves that file without size, never the whole package. +- The `:error` of an import entry is user-facing text, never an i18n key. The legacy SSE import stays in the backend for other clients. - File creation/duplication strips file `:data` before putting file summaries into dashboard state. -- The pending invitations are a team-admin only list (backend `:is-admin` already includes the owner). The dashboard hides the invitations entry from the header tabs and the team-options dropdown for non-admins, `invitations-section-content` keys the section on `:is-admin`, and `fetch-invitations` answers `insufficient-permissions` by emitting `invitations-fetched` with `[]`, so a demoted admin does not keep the cached emails and the header does not paint the invite button from a stale list. The organization `send-invitations` policy only gates inviting and resending, not the list. +- The pending invitations are a team-admin only list (backend `:is-admin` already includes the owner). The dashboard hides the invitations entry from the header tabs and the team-options dropdown for non-admins, `invitations-section-content` keys the section on `:is-admin`, and `fetch-invitations` answers `insufficient-permissions` by emitting `invitations-fetched` with `[]`, so a demoted admin does not keep the cached emails and the header does not paint the invite button from a stale list. The organization `send-invitations` policy (`can-invite?`) does not gate the list, but it gates every action on it: inviting (header and empty-state buttons render disabled with the `dashboard.invite-profile-disabled` tooltip), the role selector (read-only label), row selection (so resend and delete never show) and copy link (`invitation-link-copyable?`). The backend enforces the same rule. ## Viewer diff --git a/.serena/memories/frontend/plugin-api-to-cljs-binding.md b/.serena/memories/frontend/plugin-api-to-cljs-binding.md index 080c4b009c..31ceb4b10c 100644 --- a/.serena/memories/frontend/plugin-api-to-cljs-binding.md +++ b/.serena/memories/frontend/plugin-api-to-cljs-binding.md @@ -15,6 +15,9 @@ ## Runtime initialization and permissions - The frontend initializes `@penpot/plugins-runtime` only after `features/initialize` and only when feature `plugins/runtime` is active. It also installs the runtime `isPluginError` predicate into frontend error handling. +- `app.plugins/init-plugins-runtime` returns whether `ɵloadPlugin` was installed. It calls `preg/signal-runtime-ready` even on failure: `workspace.cljs` waits on that promise before `mcp/init`, so skipping it hangs MCP initialization forever. +- When SES bootstrap fails, `app.plugins` emits one `plugins-runtime-unavailable` telemetry event (with the SES error message), and `app.main.data.plugins/load-plugin!` shows the lazy `plugins.runtime-unavailable` toast when the user opens a plugin. `start-plugin!` (MCP path) stays silent by design. +- Runtime availability is detected by the existing `(fn? (unchecked-get global "ɵloadPlugin"))` check; do not add a second mechanism. - Manifest parsing expands write permissions to read permissions (`content:write` => `content:read`, etc.). Permission checks also allow the all-zero plugin id and the hard-coded MCP plugin id. - Manifest URL origin differs by manifest version: v1 clears the path; v2 joins `.` to the plugin URL. Existing plugin ids are reused by matching manifest name and host. - The MCP plugin id is defined in `app.plugins.register` to avoid a circular dependency with workspace MCP code. diff --git a/.serena/memories/frontend/routing-app-shell-subtleties.md b/.serena/memories/frontend/routing-app-shell-subtleties.md index 8c43a30161..ff7f334484 100644 --- a/.serena/memories/frontend/routing-app-shell-subtleties.md +++ b/.serena/memories/frontend/routing-app-shell-subtleties.md @@ -7,6 +7,8 @@ - `set-token!` and `replace-token!` are deliberately asymmetric. `set-token!` is a total no-op when the token already is the current one (no URL write, no report), so navigating to the screen you are on cannot grow the history stack. `replace-token!` always writes and always reports, because it is how the app corrects the URL in place (legacy `#/…` translation, cleaned share links) and that correction must reach the router even when the token did not change. This matches `Html5History`, which did the same; do not "fix" the asymmetry without deciding what navigates to the current screen. - `app.util.dom/replace-history-state!` is the deliberate bypass used by `app.main.ui.dashboard`/`app.main.ui.static` to rewrite the URL **without** routing. Keep using it there instead of `replace-token!`, which would re-enter the router. - Route params live entirely in the query map under the reserved `screen` key; duplicate query params can become vectors, so use `rt/get-query-param` when a scalar is required. +- The organization SSO flow sends the **query string** (`rt/get-current-query-string`), not the URL and not the token: the backend builds the destination URL from `public-uri` + `?` + those params, so the client never chooses the origin of the redirect. The history token (`rt/get-current-token`, `location.search`) keeps its leading `?` because every other consumer concatenates it over the base path (`str cf/public-uri token`, `dom/append-query-param`, `dom/remove-query-param`); `rt/token->query-string` is the only bridge between the two shapes, so do not trim at a call site by hand; `rt/token->url` is the single client builder of an absolute destination from either shape (used by the SSO retry and error flow), so do not assemble `public-uri + "?" + params` at call sites. The gate is not the only caller: `dtm/resolve-login-team-id` runs from the root branch of `on-navigate` too, and there the URL carries its params instead of a screen (`?template=…`) — that call only reads `:authorized`, so it never follows the destination, which is exactly why a wrong destination there goes unnoticed. Do not add a rule that requires `screen`. See `mem:backend/auth-permissions-product-domains`. +- The router events expose their payload through `@event` (`IDeref`): `rt/navigate` returns `{:id :params :options}` and `rt/nav-raw` returns its `:href`/`:uri`. `rt/nav-raw` is the way out of the router (raw URL, outside the token scheme) and nothing in the app listens to `::rt/nav-raw`; the destination travels in the effect, so `@event` is the only way to tell where the user is being sent (tests included). - Legacy `#/…` hash URLs translate client-side to the query format (one-version compat; see `legacy-routes` in `app.main.ui.routes`, TODO(next-version) to delete). - Unknown/empty routes trigger an extra `get-profile`/`get-teams` check before redirecting. This avoids invitation and root-route race conditions. - The root app has two error layers, and they reset differently. `:exception` in the store renders `static/exception-page*` directly and is cleared by `rt/navigated`; a React render error is caught by `app.main.ui.error-boundary` (`react-error-boundary`), whose error lives in internal React state that only `reset-error-boundary` clears. Because every exit from the error page (the logo, the no-permission dialogs, the SSO dialog, the feedback link) only *navigates*, the boundary is given `:resetKeys [(eb/reset-key route)]` — the screen name — so a navigation counts as a fresh attempt at rendering. Three traps in that `resetKeys` value, all learned the hard way: it must be a real JS array (`#js [...]`), never a ClojureScript vector, because `react-error-boundary` reads `.length` and `.some` and a `PersistentVector` has neither, so `componentDidUpdate` throws `some is not a function`; it must hold the screen name as a *string*, because the library compares entries with `Object.is` and ClojureScript does not intern keywords, so two equal keywords are two objects and a keyword key would read as new on every route rebuilt, resetting the boundary on a param change inside the same screen; and it must be the screen rather than the params, so a param change does not replay the screen that just crashed. Navigating to the *same* screen therefore does not reset, and a reload is the only way out of that case. @@ -22,6 +24,9 @@ - `flash-persistence` only delegates to another handler for `:authentication`, `:not-found` and `:code :vern-conflict`. Everything else it reports itself, and the report type is derived from the cause: `#{:validation :server-error :internal}` are application defects and report as `unhandled-exception` so they reach the internal reports and the alerts; the rest stay `handled-exception`. `:cause-type` wins over `:type` when classifying, because `persistence-failed` wraps the original cause under `:type :persistence`. Environment failures are deliberately kept handled: they land here after the retry budget is spent and the user's connection is not our bug. `:assertion` is also kept handled, matching its own handler. - The report governor is global and stateful across tests: two reports with the same fingerprint inside `report-window-ms` collapse. Call `errors/reset-report-governor!` between assertions that expect an emission. - Error messages are bounded at the source: `app.common.exceptions/get-hint` clamps to `max-message-length` (1000 chars) and the CLJS `format-throwable` clamps the message line of the stack trace and honors `:trace-length`. A message can embed a whole payload (a serialized file, for example) and reach megabytes; the bound keeps the audit `:hint` prop and the `:report` string small. The `Context: Hint:` header in `generate-report` prints `get-hint` (not the raw `:hint` from `ex-data`) for the same reason. +- A team scoped navigation carries a team whenever any of the four tiers exists, and `app.main.data.common/resolve-team-id` decides it for every team scoped `go-to-*` helper of that namespace: the UUID the caller passes, `:current-team-id`, the team of the route already on screen (`?team-id`), then the profile `:default-team-id`. `go-to-feedback` and `go-to-viewer` are not team scoped and do not go through it, and `:team-id :default` keeps meaning the profile default, resolved before the rest. The route goes before the profile default because an emitter that runs from a team screen knows which team the user is looking at, and the personal team would open their file under another team. Two `rt/nav` emitters outside that namespace still decide their own id and can still emit nil, and they are the residual: `app.main.ui.nitrate.entry/nitrate-entry*` (the profile default) and the root branch of `app.main.ui.routes/on-query-navigate` (`dtm/resolve-login-team-id`, which answers nil when the profile has no default team). When none of the tiers exists the helper still fires the navigation with a nil id, so the not-found page appears instead of the click doing nothing. +- `app.main.ui/team-container*` raises `:not-found :invalid-team-id` for a dashboard or workspace route without a valid `team-id`; the raise is what keeps a broken URL from painting a blank shell, so it stays after the hook fix of #12728. `exception-section*` skips `errors/submit-report` for `:not-found`, so this failure never reaches PostHog and cannot be measured from the error dashboard. The exit from that page is the `error-container*` header (logo plus `not-found.no-permission.go-dashboard`), which emits `go-to-dashboard-recent` with the profile default and falls back to `rt/nav-root`; every not-found and authentication page already carries it, so there is no button to add there. +- A click cannot reach a team screen before its team is resolved: `team-container*` paints its children only when the route id matches the store team, and `refs/team` derives from `:current-team-id`, so with that key empty the dashboard and the workspace paint nothing clickable. The nil-id emissions come from events that run on their own (login, invitation acceptance, membership change, the sidebar logo, the workspace shortcuts) and from typed or bookmarked URLs. ## Store and websocket diff --git a/.serena/memories/frontend/testing.md b/.serena/memories/frontend/testing.md index 37f5304395..45e6bff9e4 100644 --- a/.serena/memories/frontend/testing.md +++ b/.serena/memories/frontend/testing.md @@ -17,7 +17,8 @@ Frontend testing is async-first: everything essentially asynchronous is modeled - `mock/with-mocks` (callback style, legacy compat): installs with `set!` so mocks survive async boundaries; bodies run deferred past the current tick via `asap`, so only done-chained (`t/async`) contexts are allowed; `done'` restores and completes exactly once (twice only warns; never calling it stalls the run and leaks the mocks). Prefer `mock/with-mocks*` for new tests. - `mock/with-mocks*` (direction): body forms wrapped in a generated `^:async` fn, evaluates to a promise — `await` it, `await` nested scopes too, no `done` in test code. Rejections and non-promise returns report as `:error` via `run-mocked`. - `mock/stub` wraps fns for arities 0-6 (the `:esm` test build dispatches multi-arity vars as `cljs$core$IFn$_invoke$arity$N`); when the mocked var is variadic-defined (or called with more than 6 args), the call compiles to variadic dispatch, so use a plain variadic `fn` — the stub does not forward variadic. A mock must not call the mocked var again (self-delegation inside a multi-arity function recurses). -- Helpers in `frontend-tests.helpers.async`: `->promise` (single-value observable → promise; beicon has no `to-promise`), `await-response` (subscribe→push→await, atomic), `settle`, `wait-for` (immediate check + bounded poll, fails instead of hanging), `observe` (stream → termination promise; asserts provided, timeout rejects). +- Helpers in `frontend-tests.helpers.async`: `->promise` (single-value observable → promise; beicon has no `to-promise`), `await-response` (subscribe→push→await, atomic), `settle`, `wait-for`, `wait-for-store`, `observe` (stream → termination promise; asserts provided, timeout rejects). +- `wait-for` waits on a **signal**, not on turns: `:signal` (an observable) re-checks the predicate on every emission and resolves on the event that changes the state; `:timeout-ms` (default 5000) only bounds failure then. Without a signal it polls against a real-time deadline — 50 turns were ~54 ms idle, and turns are not a unit of time. `wait-for-store` passes `ptk/input-stream` as the signal (the store notifies after applying the update, so the predicate sees the new state); the persistence fixture exposes `requests$` for `(= N (count @requests))`. A wait that resolves on an event consumes no turn. The waits with nothing to listen to stay on the poll: test atoms filled outside the store (`@cycles`, `@reports`, `@done`), `@failures`, and predicates over JS objects or closures. - Fixtures return promises (`with-watchdog`, `with-persistence`); `await` them from `^:async` tests. `main_errors` and `fonts` are fully migrated (no legacy `with-mocks` left). - Valid `^:async` placements: `mem:clojure/idioms`. @@ -37,9 +38,14 @@ To assert over emitted event sequences, observe termination: subscribe through ` - `st/emit!` doubles collect heterogeneous events: audit `DataEvent`s deref, toast reify-objects do not — discriminate with `ptk/type` (total, never throws), never blind `deref`. - Subscription order decides delivery order: never resolve settlement from a pre-subscribed branch racing the pipeline. - Auto-answering mocks lose deadline expressiveness (can't time answers), need teardown timer-cancellation, and post-teardown deliveries hit real implementations — explicit pushes + async delivery + `wait-for` won on every axis. +- A wait that observes an effect the test asserts takes a signal. A wait that means "the side effect advanced before the next emit" has no event to listen to: keep the poll (or an explicit `settle`) and say so, because a signal would resolve inside the same `emit!` and the next step would overtake the pipeline (`persistence-rollback-test` drove its four emits this way). +- With `rp/cmd!` mocked, a store-state wait can resolve before the flow that changes it ends, and the mock scope then closes in flight: the next call hits the real HTTP layer (malformed URL from a nil `cf/public-uri`) and the error is reported on a later test. `unlink-file-from-library` dissocs the library before its async chain, so the only proof the chain ran inside the mock is its second RPC call; wait for that, through the mock. - Teardown belongs to the terminal continuation, never to `finally`-around-triggers (it would dispose in-flight flows). - A body that awaits must be `(^:async fn …)` even if the rest is sync; sync sequences are atomic vs the event loop. +- `app.main.errors` prints every handled cause with `ex/print-throwable` before notifying, and the app logger does not gate it. A test that runs a real handler end to end stubs that var (`main-errors-test/quiet-throwable`): without it, the 10 000-error governor test alone wrote 190 k of the 192 k lines of a run log (13,8 MB), and GitHub truncates a step log past ~4 MB, cutting the summary and any failing assertion. Same for a production `console.error` a test provokes on purpose: `code-gen-markup-svg-wasm-test` silences `(.-error js/console)` in a `try/finally` while two WASM panics go through `get-svg-markup` (2 blocks, 70 lines, and the exception dumps its own `data`/`arr`). - The `:esm` test build cannot load the CommonJS `sax` module behind `tubax`, so SVG parsing fails in tests (`dwm/valid-svg-string?` answers false, `dwm/svg->clj` errors): mock both with the parsed data. +- A hook-order violation in a mounted component reaches neither the root's `onUncaughtError`/`onCaughtError`/`onRecoverableError` nor the markup: React logs an internal error to `console.error` and recovers, so the callbacks stay empty and the tree repaints. Assert on a captured `console.error` too. It only fires when a render *adds* hooks (nil → uuid); a render that calls none at all slips through React's bookkeeping, so the regression test needs the "hooks arrive" direction. `frontend-tests.ui.team-container-test` does both, with a jsdom document. +- `beicon.v2.core` has no `never`. A stream that must **not emit** is `(rx/empty)`: it completes at once, which is enough for `merge`, `filter` and `take` where only the emission matters. A stream that must **stay open** is a `(rx/subject)` nobody pushes to; `rx/empty` would complete the flow behind it. An undeclared `rx/never` compiles to an undefined var and the test dies as a rejected body, which `cljs.test` reports as a test with zero assertions instead of a failure. From `frontend/`: - Full unit test run (always builds, suppressed output): `pnpm run test:quiet`. @@ -55,7 +61,7 @@ New frontend test namespaces must be required/listed in `frontend_tests/runner.c ## Playwright integration tests -Do not add, modify, or run Playwright integration tests under `frontend/playwright` unless explicitly asked. When explicitly asked, use `pnpm run test:e2e` or `pnpm run test:e2e --grep "pattern"` from `frontend/`; ensure dependencies are installed through `./scripts/setup` if the environment is not prepared. +Do not add, modify, or run Playwright integration tests under `frontend/playwright` unless explicitly asked. When explicitly asked, run the whole suite with a worker count sized to the machine: `pnpm run test:e2e --workers=N` from `frontend/`, about one worker per two hardware threads, because each keeps two or three busy and holds about 2 GB of browser memory. The config's default of one worker leaves it slow; four take a 16-thread host from 48 minutes to 35 and fit a 64 GB machine beside a watching devenv, while a 32 GB machine takes two, or four with a slim devenv. `--grep "pattern"` narrows the run. Ensure dependencies are installed through `./scripts/setup` if the environment is not prepared. Integration tests fake backend behavior by intercepting network/websocket traffic, so every RPC or websocket the page needs must be mocked. Use existing Page Object Models: - `BasePage.mockRPC` intercepts RPC calls and already prefixes `/api/rpc/command/`; pass command names such as `get-profile`, not full URLs. diff --git a/.serena/memories/plugins/core.md b/.serena/memories/plugins/core.md index 64f4643860..16ec5165bf 100644 --- a/.serena/memories/plugins/core.md +++ b/.serena/memories/plugins/core.md @@ -18,6 +18,7 @@ ## Sandbox and global cleanup +- `src/index.ts` calls `bootstrapSES()` (in `src/lib/bootstrap.ts`) at module load. If `repairIntrinsics` throws (e.g. SES 2.2+ rejects a `Temporal` wrapped by a browser extension), the error is caught, `getPluginsRuntimeError()` exposes it, and `initPluginsRuntime` installs no `ɵ*` globals: the frontend app still boots, plugins and MCP are unavailable in that environment. On failure, `Function.prototype.toString` has already been replaced but no intrinsics are hardened and no compartment/sandbox is created — never run plugins on that path. - The runtime uses SES compartments. Public API return values are passed through `ses.safeReturn` before crossing back to plugin code. - Plugin `fetch` is sanitized: credentials are omitted and Authorization is blanked. The exposed response only includes ok/status/statusText/url/text/json. - Timer callbacks are wrapped to mark plugin-originated errors, and timeout/interval IDs are tracked so plugin close can clear them. diff --git a/.serena/memories/prod-infra/core.md b/.serena/memories/prod-infra/core.md index d8ced132ca..09a58d9784 100644 --- a/.serena/memories/prod-infra/core.md +++ b/.serena/memories/prod-infra/core.md @@ -43,7 +43,13 @@ Production runs the deployed toolbox, not the devenv one: - **Start from the ids, not from an inventory.** The ids come from the error or the logs. Unbounded queries over tables with tens of millions of rows (`storage_object`, `file_media_object`) do not finish in a reasonable time; bounded to an id list they are index lookups. - **One query per reference column.** Two columns in one `IN (col_a, col_b)` lose the index and scan the whole table; split them. - The read-only replica cuts long queries: see `mem:prod-infra/grafana`. -- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table (file saves write to `file_change` on every save) build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block); the shipped migration uses `IF NOT EXISTS`, finds it and does nothing. A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_indexes` and `pg_index.indisvalid`). +- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block). A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_index` and `pg_index.indisvalid`). +- A partitioned parent cannot take `CREATE INDEX CONCURRENTLY` (`cannot create index on partitioned table ... concurrently`). Build one index per partition with `CONCURRENTLY`, then `CREATE INDEX ... ON ONLY` on the parent and `ALTER INDEX ... ATTACH PARTITION` for each child; the parent index only becomes valid when every partition is attached. +- Write the migration as plain, idempotent SQL: `ADD COLUMN IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, and `DROP CONSTRAINT IF EXISTS` followed by `ADD CONSTRAINT`. Do not encode shape checks as `pg_catalog` introspection: they are easy to get wrong (a missing `NOT condeferred` accepts an `INITIALLY DEFERRED` foreign key) and they duplicate what the runbook verifies by hand. `IF NOT EXISTS` matches the name, not the definition, so the out-of-band procedure verifies the index shape and completeness. +- The runbook is the contract for a large instance: it builds and verifies the out-of-band work and then marks the migration step as applied (`INSERT INTO migrations (module, step) ...`), so the body does not run at startup. Verify the column, the pending rows, the index (valid, no predicate or expression, every partition attached) and the constraint shape before inserting the row; the marker is the only thing that skips the migration. +- A reference that a GC task resolves must be `DEFERRABLE INITIALLY IMMEDIATE`, never `INITIALLY DEFERRED`: the immediate check fires inside the transaction that catches the violation and retries the chunk; the deferred one fires at commit, outside the catch. A validated foreign key belongs out of band: `ADD FOREIGN KEY` runs as an `ALTER TABLE` and it scans the whole referencing table; `NOT VALID` on a partitioned table only exists since PostgreSQL 18. +- A backfill that removes a key an old reader uses (for example `metadata.storage-ref-id`, which the previous storage GC scans) must run with the old code stopped. Otherwise the old reader loses the reference and can delete a live object, and the rows it writes in the meantime miss the backfill. +- `ADD COLUMN` takes a brief `ACCESS EXCLUSIVE` even when `IF NOT EXISTS` skips it, and `ADD FOREIGN KEY` takes `ACCESS EXCLUSIVE` on the referencing table and `SHARE ROW EXCLUSIVE` on the referenced one. A migration accepts those locks; a deployment that cannot take them runs the runbook and marks the step as applied. ## See also diff --git a/.serena/memories/scripts/changelog.md b/.serena/memories/scripts/changelog.md index 7134377fa5..2f2d582145 100644 --- a/.serena/memories/scripts/changelog.md +++ b/.serena/memories/scripts/changelog.md @@ -8,7 +8,8 @@ - `boom [--changes CHANGES.md --write]` — generate `### :boom: Breaking changes & Deprecations` from the closed, non-excluded milestone issues labelled `breaking change`, sorted by issue number. Each issue becomes a `#### ` block: content = body of the issue's last `## Breaking change note` comment (whole markdown, headings demoted to `#####` outside code fences; fallback = heading plus refs with a stderr warning), refs = issue link plus merged closing PRs on their own line, with `(by @author)` for `community contribution`. `--write` replaces only that subsection of the version (inserts it after the version header when missing, keeps the heading when there are no entries; idempotent). Print-only by default. Uses `gh.py comments`. - `security [--changes CHANGES.md --write]` — generate `### :lock: Security fixes` from the advisories whose `patched_versions` include the version (via `gh.py advisories --patched-version`), sorted by GHSA id. Entry: `- Fix () (PR: [#N](...))`; description = `Fix` + advisory summary, first word decapitalized unless it starts with an acronym; PRs = merged milestone PRs labelled `pr pt` whose body contains the GHSA id, comma-separated, and no PR is legal. Warns on stderr when a `pr pt` PR references an advisory with no matching `patched_versions`. `--write` replaces only that subsection, placed after `:rocket:` (or `:boom:`, or the version header when neither exists); empty entries remove the subsection; idempotent. Print-only by default. - `cross-ref [--changes CHANGES.md]` — fetch milestone PRs with `--state all` once, report merged PRs missing from the version section plus CLOSED (unmerged) warnings. PR links inside `:lock:` entries count as referenced; advisory URLs never appear here by design. -- `report [--changes CHANGES.md --output CHANGES-ISSUES.md]` — overwrite the anomaly report: A/B milestone mismatches, E `:boom:` entry without the label, F `:rocket:` sync, G `:boom:` sync, plus the C/D `:rocket:` gaps (warnings). `:boom:`/`:rocket:` rule violations are fixed by regenerating or relabelling, not by editing entries in place. +- `dev-only-bugs [--apply]` — list closed milestone bugs that never reached production, without `no changelog` (bugs carrying `release blocker` are analyzed too). Sure: parent issue is a `Feature`/`Enhancement` of the same milestone; `--apply` adds `no changelog` to these only. Candidates (never labelled automatically): parent is an epic of the milestone, `regression` label, or the "Affected versions" / "Environment" body field names only hourly, develop, staging, PRE or the milestone version itself (`only_unreleased_envs`; any production hint or other version keeps the bug in — versions compare as full semver with missing patch = 0, so for milestone 2.19.2 the released 2.19.0 keeps the bug in). Reads `body` and `parent` from one batched `gh.py issue` lookup. +- `report [--changes CHANGES.md --output CHANGES-ISSUES.md]` — overwrite the anomaly report: A/B milestone mismatches, E `:boom:` entry without the label, F `:rocket:` sync, G `:boom:` sync, plus the C/D `:rocket:` gaps and the `dev-only-bugs` lists (warnings). `:boom:`/`:rocket:` rule violations are fixed by regenerating or relabelling, not by editing entries in place. ## Conventions @@ -18,4 +19,4 @@ - PR/issue reference parsing covers standard `[#N](.../pull/N)` / `[#N](.../issues/N)` links plus legacy `PR:[N]` / `[Github #N]` forms. - Shared parsing helpers (`extract_version_section`, `extract_subsection`, `collect_changelog_prs/issues`) live at module top so future subcommands reuse them instead of duplicating regexes. - Report anomaly types: A/B milestone mismatches, E `:boom:` entry whose issue lacks the `breaking change` label (counted, entries preserved — human labels the issue or moves the entry), G `breaking change` issue missing from `:boom:` (counted, same contract as F); C/D are `:rocket:` gaps (warnings, never counted). -- `issue_excluded` (closed, non-Task, non-Rejected, no `release blocker`/`no changelog` label) is module-level and shared by `report` and `boom` so the exclusions cannot drift. +- `issue_excluded` (closed, non-Task, non-Rejected, no `no changelog` label) is module-level and shared by `report` and `boom` so the exclusions cannot drift. diff --git a/.serena/memories/scripts/gh.md b/.serena/memories/scripts/gh.md index c2a10d8f6b..d9775f3d82 100644 --- a/.serena/memories/scripts/gh.md +++ b/.serena/memories/scripts/gh.md @@ -40,7 +40,7 @@ python3 scripts/gh.py issues "2.16.0" --label "bug" python3 scripts/gh.py issues "2.16.0" --label "bug,regression" # Exclude by label -python3 scripts/gh.py issues "2.16.0" --exclude "release blocker,no changelog" +python3 scripts/gh.py issues "2.16.0" --exclude "no changelog" # Show only issues NOT yet in CHANGES.md python3 scripts/gh.py issues "2.16.0" --compare CHANGES.md @@ -81,7 +81,7 @@ python3 scripts/gh.py prs --milestone "2.16.0" --state all Fetch issue details by number in batches of 50 (same input styles as `prs`: numbers, `--file`, `--stdin`). No filters. A dead number fails its whole batch (unlike PRs, issues error instead of resolving to null), so the batch falls back to one-by-one lookups and reports misses as `{"number", "error": "not_found"}`. Any other failure aborts. -**Output**: JSON array to stdout; progress to stderr. +**Output**: JSON array to stdout; progress to stderr. Each issue carries the shared issue fields plus `body` and `parent` (`number`, `issue_type`, `milestone`, or null); `changelog.py dev-only-bugs` reads both. ### `comments` diff --git a/.serena/memories/testing.md b/.serena/memories/testing.md index ce301ff5bb..75b775d0ec 100644 --- a/.serena/memories/testing.md +++ b/.serena/memories/testing.md @@ -73,6 +73,8 @@ a test that reproduces it: - **One Assertion Per Concept** — each test verifies one behavior; split compound assertions - **Descriptive Test Names** — names read like specifications +- **English Only** — test names, `testing` strings, comments, and every + code-facing text MUST be in English; see the hard rule in `AGENTS.md`. ## Prefer Real Implementations Over Mocks diff --git a/.serena/memories/workflow/creating-commits.md b/.serena/memories/workflow/creating-commits.md index cf08358aaf..d225217867 100644 --- a/.serena/memories/workflow/creating-commits.md +++ b/.serena/memories/workflow/creating-commits.md @@ -56,6 +56,14 @@ that breaks them is wrong, even if the code is right. `:bug:` bug fix · `:sparkles:` enhancement · `:tada:` new feature · `:recycle:` refactor · `:lipstick:` cosmetic · `:ambulance:` critical fix · `:books:` docs · `:construction:` WIP · `:boom:` breaking · `:wrench:` config · `:zap:` perf · `:whale:` docker · `:paperclip:` other · `:arrow_up:` dep upgrade · `:arrow_down:` dep downgrade · `:fire:` removal · `:globe_with_meridians:` translations · `:rocket:` epic/highlight +## Atomic Commits + +A PR squash-merges into one commit by default, so the split inside the branch +does not reach `main` and nothing depends on it. When the PR asks for a +regular merge (`mem:workflow/creating-prs` → Merge Strategy), every commit +must stand alone: it builds, passes the linter, and needs no later commit on +the same branch to make sense. + ## Referencing Issues Use `Closes #NNNN` (not `Fixes #NNNN`) in commit and PR body text for readable context. The explicit issue-to-PR link is assigned with `python3 scripts/gh.py link-issue ` as described in `mem:workflow/creating-prs`; do not rely on the keyword to create that link. diff --git a/.serena/memories/workflow/creating-prs.md b/.serena/memories/workflow/creating-prs.md index 87f10f2121..e8e34181e6 100644 --- a/.serena/memories/workflow/creating-prs.md +++ b/.serena/memories/workflow/creating-prs.md @@ -32,6 +32,19 @@ PR titles follow commit title conventions: See `mem:workflow/creating-commits` for emoji codes. Squash merge uses the PR title as the final commit subject, so title format matters. +## Merge Strategy + +Policy: `CONTRIBUTING.md` → "Merging", under "Pull Requests". + +- Squash merge is the default, so the PR's own commits never reach `main` + and the PR title becomes the landed subject. +- A regular merge is possible on request only. The request is the **first + line** of the PR body, above the `**Note:**` line: + `> **Merge:** do not squash; each commit is self-contained.` +- Ask for it only when every commit on the branch is self-contained (builds + and passes lint on its own). Default to squash unless the user asks for + the opposite. + ## Description Body Include concise sections covering: @@ -60,6 +73,8 @@ PR descriptions follow this structure: ``` The "Note:" line is required at the top. Adjust if this is a manual (non-AI) PR. +When the PR requests a regular merge instead of a squash merge, its +`**Merge:**` line goes above the `**Note:**` line (see Merge Strategy). ## Writing Principles diff --git a/.serena/memories/workflow/updating-changelog.md b/.serena/memories/workflow/updating-changelog.md index 54ca604790..380631aba3 100644 --- a/.serena/memories/workflow/updating-changelog.md +++ b/.serena/memories/workflow/updating-changelog.md @@ -24,11 +24,12 @@ Editing `CHANGES.md` happens only on explicit request or as part of the release | Is it a breaking change? | `breaking change` label | | Is it a highlight? | `release highlight` label | -- **Issue Type decides the section**: `Bug` → `:bug:`, `Feature`/`Enhancement` → `:sparkles:`, `Task` is excluded (internal chores). Labels `bug`/`enhancement` and title emoji prefixes are often wrong or missing; do not use them. +- **Issue Type decides the section**: `Feature` → `:tada:`, `Enhancement` → `:sparkles:`, `Bug` → `:bug:`, `Task` is excluded (internal chores). Labels `bug`/`enhancement` and title emoji prefixes are often wrong or missing; do not use them. - **Security fixes come from advisories**: an advisory whose `patched_versions` include the version becomes a `:lock:` entry (section 7). `triage`, `draft` and `published` are all valid states; the advisory may still be unpublished when the changelog is written. A security fix may also carry a regular bug issue; both entries are legal and independent (the issue lives in `:bug:` as any other bug). - **Two labels override the Issue Type**: `breaking change` → an entry only under `:boom:`, never in its regular section; `release highlight` → an extra entry under `:rocket:` on top of its regular entry. - **Only closed issues**. An open issue in the milestone is not generated, even if someone added a provisional entry by hand before the merge. -- **Excluded issues**: `no changelog`, `release blocker`, `Task` issue type, `Rejected` project status. +- **Excluded issues**: `no changelog`, `Task` issue type, `Rejected` project status. `release blocker` is a workflow label (the issue blocks the release), not a changelog exclusion: bugs carrying it are analyzed by `dev-only-bugs` (section 8, step 3), and closed non-bug issues carrying it are reviewed by hand (section 8, step 3). +- **Bugs that never reached production** stay out: a bug found and fixed while building a feature of the same, unreleased version never affected users. The team rarely labels them, so the flow finds them with `dev-only-bugs` and labels them `no changelog` (section 8, step 3). - **Excluded PRs**: a PR with `release blocker` or `no issue required` does not bring an entry of its own. - **An issue with no milestone** belongs to another (probably private) project: it is neither changelog candidate nor anomaly. @@ -39,7 +40,7 @@ The newest version goes at the top, right after the `# CHANGELOG` header. Inside ```markdown # CHANGELOG -## 2.19.0 (Unreleased) +## 2.20.0 (Unreleased) ### :boom: Breaking changes & Deprecations @@ -57,21 +58,27 @@ The newest version goes at the top, right after the `# CHANGELOG` header. Inside - Fix () (PR: [#PR](https://github.com/penpot/penpot/pull/PR)) -### :bug: Bugs fixed - -- Fix [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR)) - -### :sparkles: New features & Enhancements +### :tada: Features - Add [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR)) + +### :sparkles: Enhancements + +- Improve [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR)) + +### :bug: Bugs + +- Fix [#ISSUE](https://github.com/penpot/penpot/issues/ISSUE) (PR: [#PR](https://github.com/penpot/penpot/pull/PR)) ``` -- `:boom:` first, then `:rocket:`, `:lock:`, `:bug:`, `:sparkles:`. Other headings exist in old versions (`:heart:` community contributions); do not add them to new versions. +- `:boom:` first, then `:rocket:`, `:lock:`, `:tada:`, `:sparkles:`, `:bug:`. Features come before enhancements so they stand out to readers deciding whether to update. +- **This layout applies from 2.20.0.** Versions up to 2.19.x use the old one: `:bug: Bugs fixed` before `:sparkles: New features & Enhancements`, which mixes features and enhancements. Keep the old layout in those sections, including new 2.19.x patches, so a minor line reads the same. The in-app update dialog only reads `:rocket:`, so the change does not affect it. +- Other headings exist in old versions (`:heart:` community contributions); do not add them to new versions. - Include a section only when it has entries. Historical versions may have an empty `:boom:` or `:rocket:` heading; `boom --write` keeps the `:boom:` heading even when there are no entries. `:lock:` is omitted (and removed on `--write`) when the version has no security advisories. - One blank line between the last entry of a section and the next heading. - No manual line wrapping: one line per bullet or paragraph, however long. -## 4. Regular entries (`:bug:`, `:sparkles:`) +## 4. Regular entries (`:tada:`, `:sparkles:`, `:bug:`) The entry shape is: @@ -152,16 +159,20 @@ Generated from the advisories that declare the version in `patched_versions`; ne ## 8. Flow 1. **Confirm the version** (a semver string such as `2.19.0`) with the user when it is not obvious. -2. **Fetch the milestone issues**: `python3 scripts/gh.py issues "" --exclude "release blocker,no changelog"` (closed by default; add `--state all` only to inspect open issues). `gh.py` already drops `Task` and `Rejected`; pass `--include-tasks` / `--include-rejected` only to inspect them. To find what is missing from an existing section, add `--compare CHANGES.md`. -3. **Fetch PR details when needed**: `python3 scripts/gh.py prs ` (also `--milestone`). Use it for the PR author, labels and merge state. -4. **Categorize** by Issue Type and the two labels (section 2), and **build the descriptions** (section 4). -5. **Verify PR merge status** before writing: `python3 scripts/changelog.py check-merged `. Only merged PRs go in the changelog; if a closing PR was closed unmerged, look for the PR that superseded it. -6. **Reconcile the existing text** (section 9). -7. **Generate `:boom:`**: `python3 scripts/changelog.py boom ""` prints the section; add `--write` to replace it in `CHANGES.md`. The write replaces only that subsection, inserts it after the version header when missing, keeps the heading when there are no entries, and is idempotent. -8. **Generate `:rocket:`** from the `release highlight` label (section 6). -9. **Generate `:lock:`**: `python3 scripts/changelog.py security ""` prints the section; add `--write` to replace it in `CHANGES.md`. The contract lives in section 7. -10. **Cross-check PRs**: `python3 scripts/changelog.py cross-ref ""`. It lists merged milestone PRs missing from the section and warns about closed (unmerged) PRs. PRs linked from `:lock:` count as referenced; an advisory with no PR is invisible here by design. -11. **Generate the anomaly report**: `python3 scripts/changelog.py report ""`. It overwrites `CHANGES-ISSUES.md` with the current state (section 10). +2. **Fetch the milestone issues**: `python3 scripts/gh.py issues "" --exclude "no changelog"` (closed by default; add `--state all` only to inspect open issues). `gh.py` already drops `Task` and `Rejected`; pass `--include-tasks` / `--include-rejected` only to inspect them. To find what is missing from an existing section, add `--compare CHANGES.md`. +3. **Keep out bugs that never reached production**: `python3 scripts/changelog.py dev-only-bugs ""` lists them; add `--apply` to label the sure ones `no changelog` on GitHub, then re-fetch step 2. The label puts the decision on the issue, so it survives a regeneration. Bugs carrying `release blocker` are analyzed like any other bug. + - **Sure**: a bug whose parent issue is a `Feature` or `Enhancement` of the same milestone. `--apply` labels these. + - **Candidates**: a bug whose parent is an epic of the same milestone, that carries the `regression` label, or whose "Affected versions" / "Environment" field names only hourly, develop, staging, PRE or the milestone version itself. `regression` also marks regressions users hit in production, and the field says where the bug was seen, not since when it exists, so never label these yourself: show them to the user, who labels the ones they confirm. The anomaly report lists them too (section 10). + - **Closed non-bug issues carrying `release blocker`** have no automatic check: `python3 scripts/gh.py issues "" --label "release blocker" --exclude "no changelog"` lists them (closed, non-Task). Show them to the user: the internal ones get the `no changelog` label, the rest keep their entry. +4. **Fetch PR details when needed**: `python3 scripts/gh.py prs ` (also `--milestone`). Use it for the PR author, labels and merge state. +5. **Categorize** by Issue Type and the two labels (section 2), and **build the descriptions** (section 4). +6. **Verify PR merge status** before writing: `python3 scripts/changelog.py check-merged `. Only merged PRs go in the changelog; if a closing PR was closed unmerged, look for the PR that superseded it. +7. **Reconcile the existing text** (section 9). +8. **Generate `:boom:`**: `python3 scripts/changelog.py boom ""` prints the section; add `--write` to replace it in `CHANGES.md`. The write replaces only that subsection, inserts it after the version header when missing, keeps the heading when there are no entries, and is idempotent. +9. **Generate `:rocket:`** from the `release highlight` label (section 6). +10. **Generate `:lock:`**: `python3 scripts/changelog.py security ""` prints the section; add `--write` to replace it in `CHANGES.md`. The contract lives in section 7. +11. **Cross-check PRs**: `python3 scripts/changelog.py cross-ref ""`. It lists merged milestone PRs missing from the section and warns about closed (unmerged) PRs. PRs linked from `:lock:` count as referenced; an advisory with no PR is invisible here by design. +12. **Generate the anomaly report**: `python3 scripts/changelog.py report ""`. It overwrites `CHANGES-ISSUES.md` with the current state (section 10). ## 9. Pre-flight: reconcile before writing @@ -169,10 +180,10 @@ Re-fetch the milestone (do not trust cached data) and fix these in `CHANGES.md` 1. **Duplicate across versions** → remove it from the current section; the earlier version wins for backports. The same holds after merging a patch branch into `staging`: the fix keeps its released patch entry and leaves the newer, unreleased section (the merge conflict in `CHANGES.md` is resolved by keeping both sections, newest first, minus the duplicate). 2. **Stale milestone assignment** (the issue moved out of this milestone) → remove the entry. -3. **Newly applied exclusion labels** (`no changelog`, `release blocker`) → remove the entry. +3. **Newly applied exclusion labels** (`no changelog`) → remove the entry. (`release blocker` is not an exclusion label; it never removes an entry by itself.) 4. **Issue no longer closed, deleted, or now `Rejected`** → remove the entry. 5. **Unmerged or moved PR reference** → fix the reference or remove the entry. A PR merged in a different milestone is an anomaly (section 10); do not remove it silently. -6. **Issue Type changed to `Task`** → remove the entry. +6. **Issue Type changed** → to `Task`: remove the entry; to another type: move the entry to that type's section. 7. **Breaking change out of sync** (a `breaking change` issue outside `:boom:`, or a `:boom:` entry without the label) → regenerate the section; a labelled issue has no regular entry. 8. **Highlight out of sync** (`:rocket:` entry without the label, or a labelled issue missing) → remove it or add it. 9. **Missing valid issues** (closed, non-excluded, referenced nowhere) → add them to their section. @@ -189,6 +200,8 @@ Re-fetch the milestone (do not trust cached data) and fix these in `CHANGES.md` - **C — released X.Y.0 version without a `:rocket:` section** (gap; patches never carry highlights). - **D — `:rocket:` entry without issue AND PR references** (gap). +**Bugs that may never have reached production** are warnings too (section 8, step 3): sub-issues of a milestone feature still unlabelled (run `dev-only-bugs --apply`) and candidates for the user to confirm. They never count toward the total. + Anything else is a rule violation, not a report item: fix it in the pre-flight and re-run. ## 11. Pitfalls @@ -203,6 +216,6 @@ Anything else is a rule violation, not a report item: fix it in the pre-flight a ## 12. Where the code is - `scripts/gh.py`: `issues`, `issue`, `comments`, `prs`, `link-issue`, `advisories` (with `--patched-version`). Details: `mem:scripts/gh`. -- `scripts/changelog.py`: `check-merged`, `boom`, `security`, `cross-ref`, `report`. Details: `mem:scripts/changelog`. +- `scripts/changelog.py`: `check-merged`, `boom`, `security`, `cross-ref`, `dev-only-bugs`, `report`. Details: `mem:scripts/changelog`. - Tests: `scripts/test_gh.py`, `scripts/test_changelog.py`. - The note contract for issue authors: `mem:workflow/creating-issues`, section **Breaking changes**. diff --git a/AGENTS.md b/AGENTS.md index 4e4bf1fec1..261d099e68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,12 @@ - Before `gh pr create` / `gh pr edit` → `mem:workflow/creating-prs` (title format, body structure, AI note) Don't infer format from the title of a previous commit/issue/PR — the memory is the source of truth. +- **Everything in code MUST be in English.** Identifiers (functions, + variables, types, namespaces), code comments, docstrings, test + descriptions, log hints — always English, regardless of the language the + author works in. Tests are not an exception. The only place another + language may appear in the codebase is product copy rendered through the + i18n system. ## CRITICAL: Read module memories BEFORE writing any code diff --git a/CHANGES.md b/CHANGES.md index e1aa8dc7b9..6943b9a29e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,116 @@ # CHANGELOG +## 2.21.0 + +### :boom: Breaking changes & Deprecations + +### :lock: Security fixes + +- Fix real-time collaboration WebSocket never re-validates file access after `:subscribe-file`, letting revoked users keep receiving live edit content (https://github.com/penpot/penpot/security/advisories/GHSA-m53j-2766-6jqw) (PR: [#12189](https://github.com/penpot/penpot/pull/12189)) + +### :tada: Features + +- Auto-link libraries during import based on slugified name [#9263](https://github.com/penpot/penpot/issues/9263) (PR: [#9958](https://github.com/penpot/penpot/pull/9958)) +- Implement basic bullet lists [#12630](https://github.com/penpot/penpot/issues/12630) (PR: [#12720](https://github.com/penpot/penpot/pull/12720)) + +### :sparkles: Enhancements + +- Improve performance of payload decoding in the workspace (by @lukel97) [#12916](https://github.com/penpot/penpot/issues/12916) (PR: [#12704](https://github.com/penpot/penpot/pull/12704)) +- Keep the search when changing library in the Swap component menu (by @desty) [#12335](https://github.com/penpot/penpot/issues/12335) (PR: [#12873](https://github.com/penpot/penpot/pull/12873)) + +### :bug: Bugs + +- Fix workspace stuttering when switching pages in a large file (by @lukel97) [#12917](https://github.com/penpot/penpot/issues/12917) (PR: [#12811](https://github.com/penpot/penpot/pull/12811)) +- Fix workspace freezes when undoing or redoing a move of a board with many components (by @lukel97) [#12913](https://github.com/penpot/penpot/issues/12913) (PR: [#12870](https://github.com/penpot/penpot/pull/12870)) +- Fix dashboard freezes for about a second after duplicating a large file (by @lukel97) [#12914](https://github.com/penpot/penpot/issues/12914) (PR: [#12911](https://github.com/penpot/penpot/pull/12911)) +- Fix not being able to collapse a token set group after reordering it [#12625](https://github.com/penpot/penpot/issues/12625) (PR: [#12671](https://github.com/penpot/penpot/pull/12671)) +- Fix slow variant switch with the WebGL renderer [#12968](https://github.com/penpot/penpot/issues/12968) (PR: [#12985](https://github.com/penpot/penpot/pull/12985)) +- Fix login page GitHub logo showing with low contrast in the light theme [#12938](https://github.com/penpot/penpot/issues/12938) (PR: [#12943](https://github.com/penpot/penpot/pull/12943)) +- Fix leak and black screen on wasm-critical errors [#12977](https://github.com/penpot/penpot/issues/12977) (PR: [#12978](https://github.com/penpot/penpot/pull/12978)) + ## 2.20.0 -### :bug: Bugs fixed +### :boom: Breaking changes & Deprecations +#### Add table partitioning for file-change + +The file history table (`file_change`) is now partitioned by file, with the same 16 partitions as the file payload table (`file_data`). Queries that filter by file, and the join between the two tables, resolve against a single partition, so they stay fast as the table grows. The rewrite also removes the dead columns and stores the table in its final layout; the space the old table held comes back when it is dropped. A save no longer writes the file features on every revision, and the team snapshot quota is a single lookup. + +The four snapshot RPC methods (`update-file-snapshot`, `delete-file-snapshot`, `lock-file-snapshot` and `unlock-file-snapshot`) now require `file-id`. The frontend already sends it; API clients that call them must add it. + +On small and medium instances the table is rewritten automatically on boot; there is nothing to do. On large instances (hundreds of GB in `file_change`) it does not fit in the startup transaction: run it by hand **before upgrading to 2.20.0** following the [large-instance runbook](https://github.com/penpot/penpot/blob/staging/operations/2.20.0-file-change-partition.md). The copy runs while the instance is online, followed by a cutover of a few minutes with the backend and the workers stopped. The old table is kept for rollback; check the disk headroom before starting, and drop the old table days later to get the space back. + +[#12416](https://github.com/penpot/penpot/issues/12416) (PR: [#12417](https://github.com/penpot/penpot/pull/12417)) + +#### Protect the file data storage objects with a foreign key in the backend + +This release moves the file data storage pointer out of `file_data.metadata->>'storage-ref-id'` and into a new `file_data.storage_object_id` column with a foreign key to `storage_object(id)`. The metadata key is gone: nothing writes or reads it. + +What it means for self-hosted instances: + +- The release is a coordinated restart. The new code reads the column; the migration drops `storage-ref-id` from the rows it backfills. Rolling back to the previous binary is not supported after this release runs. +- A fresh database or a small instance needs nothing special: the startup migration does the whole job. +- A large instance should not let the startup migration run. Build the column, the index and the foreign key online beforehand, stop the instances, run the backfill only if a row still carries `metadata.storage-ref-id`, verify the end state, and mark the migration as applied. The full procedure is documented in [`operations/2.20.0-file-data-storage-object-id.md`](https://github.com/penpot/penpot/blob/main/operations/2.20.0-file-data-storage-object-id.md). +- If the storage backend was never enabled, there is nothing to backfill and the stopped window is just the restart. + +If you upgrade blindly: on fresh databases and small instances the startup migration completes the change. On a large instance, the foreign key validation scan can block writes for a long time; follow the runbook instead. + +[#12768](https://github.com/penpot/penpot/issues/12768) (PR: [#12769](https://github.com/penpot/penpot/pull/12769)) + +### :rocket: Epics and highlights + +- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759) +- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761) + +### :tada: Features + +- Add a feedback channel to the MCP server [#12490](https://github.com/penpot/penpot/issues/12490) (PR: [#12551](https://github.com/penpot/penpot/pull/12551)) +- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759) +- Implement full text format paste within Penpot [#12047](https://github.com/penpot/penpot/issues/12047) (PR: [#12137](https://github.com/penpot/penpot/pull/12137)) +- Add team, project and file management to the Plugin API for global plugins [#12170](https://github.com/penpot/penpot/issues/12170) (PR: [#12169](https://github.com/penpot/penpot/pull/12169)) +- Serve Penpot under a subpath [#3194](https://github.com/penpot/penpot/issues/3194) (PR: [#12358](https://github.com/penpot/penpot/pull/12358)) +- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761) +- Add OAuth support for MCP client connections [#12034](https://github.com/penpot/penpot/issues/12034) (PR: [#12480](https://github.com/penpot/penpot/pull/12480)) + +### :sparkles: Enhancements + +- Auto-scroll sortable lists when dragging an item near their top or bottom edge (by @ShreyashAgare26) [#10478](https://github.com/penpot/penpot/issues/10478) (PR: [#11595](https://github.com/penpot/penpot/pull/11595)) +- Make WebGL the default renderer for new accounts [#11855](https://github.com/penpot/penpot/issues/11855) (PR: [#12880](https://github.com/penpot/penpot/pull/12880)) +- Prevent the hosted MCP server from running execute_code twice when the plugin is open in two tabs [#12008](https://github.com/penpot/penpot/issues/12008) (PR: [#11927](https://github.com/penpot/penpot/pull/11927)) +- Allow detaching typography tokens [#11887](https://github.com/penpot/penpot/issues/11887) (PR: [#11969](https://github.com/penpot/penpot/pull/11969)) +- Complete the Russian translations [#11955](https://github.com/penpot/penpot/issues/11955) (PR: [#11956](https://github.com/penpot/penpot/pull/11956)) +- Run .penpot file import and export as durable background jobs [#12194](https://github.com/penpot/penpot/issues/12194) (PR: [#12056](https://github.com/penpot/penpot/pull/12056), [#12192](https://github.com/penpot/penpot/pull/12192)) +- Open files and projects in a new tab with Ctrl/Cmd+click in the dashboard [#12228](https://github.com/penpot/penpot/issues/12228) (PR: [#12230](https://github.com/penpot/penpot/pull/12230)) +- Add an RPC method to read file-level plugin data [#12258](https://github.com/penpot/penpot/issues/12258) (PR: [#10171](https://github.com/penpot/penpot/pull/10171)) +- Add a get-file-tokens RPC method to read a file's design tokens [#12260](https://github.com/penpot/penpot/issues/12260) (PR: [#12172](https://github.com/penpot/penpot/pull/12172)) +- Show the progress and the size of each file in the import and export dialogs [#12340](https://github.com/penpot/penpot/issues/12340) (PR: [#12192](https://github.com/penpot/penpot/pull/12192)) +- Focus the search field when opening the color tokens tab of the color picker [#12344](https://github.com/penpot/penpot/issues/12344) (PR: [#12345](https://github.com/penpot/penpot/pull/12345)) +- Harden the file data offload job with a row lock, per-file transactions, throttle and metrics [#12398](https://github.com/penpot/penpot/issues/12398) (PR: [#12403](https://github.com/penpot/penpot/pull/12403)) +- Render email templates inside the mail job in the backend [#12445](https://github.com/penpot/penpot/issues/12445) (PR: [#12448](https://github.com/penpot/penpot/pull/12448)) +- Run local WASM exports for shapes and boards without freezing the UI [#12512](https://github.com/penpot/penpot/issues/12512) (PR: [#12515](https://github.com/penpot/penpot/pull/12515)) +- Use WASM to generate SVG markup in the code inspector when render-wasm is active [#12567](https://github.com/penpot/penpot/issues/12567) (PR: [#12608](https://github.com/penpot/penpot/pull/12608)) +- Close the workspace search with the Escape key (by @filipsajdak) [#9540](https://github.com/penpot/penpot/issues/9540) (PR: [#12374](https://github.com/penpot/penpot/pull/12374)) +- Restrict management RPC methods to their declared shared keys [#12730](https://github.com/penpot/penpot/issues/12730) (PR: [#12736](https://github.com/penpot/penpot/pull/12736)) +- Reduce memory used by duplicate shape ids in the workspace [#12791](https://github.com/penpot/penpot/issues/12791) (PR: [#12792](https://github.com/penpot/penpot/pull/12792)) +- Reduce memory consumption on shape decoding [#12789](https://github.com/penpot/penpot/issues/12789) (PR: [#12790](https://github.com/penpot/penpot/pull/12790)) +- Notify other users when a snapshot is restored [#12425](https://github.com/penpot/penpot/issues/12425) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Normalize the configured tenant to lowercase [#12842](https://github.com/penpot/penpot/issues/12842) (PR: [#12843](https://github.com/penpot/penpot/pull/12843)) +- Speed up saves on large files by checking only the pages a save changes [#12602](https://github.com/penpot/penpot/issues/12602) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Retry saves after short server failures and keep the workspace open when a save fails [#12601](https://github.com/penpot/penpot/issues/12601) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Switch the workspace to read-only mode when the file can't be saved [#12507](https://github.com/penpot/penpot/issues/12507) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Validate upload content against real file bytes in media uploads [#12796](https://github.com/penpot/penpot/issues/12796) (PR: [#12801](https://github.com/penpot/penpot/pull/12801)) +- Add the file and MCP session to audit events sent from plugins and the MCP [#12902](https://github.com/penpot/penpot/issues/12902) (PR: [#12972](https://github.com/penpot/penpot/pull/12972)) +- Add skip-if-active option to job submission in the backend jobs substrate [#12265](https://github.com/penpot/penpot/issues/12265) (PR: [#12266](https://github.com/penpot/penpot/pull/12266)) +- Add HTML paste from clipboard in text editor v3 [#10465](https://github.com/penpot/penpot/issues/10465) +- Skip loading the thumbnail rasterizer in WebGL renderer sessions to cut workspace memory [#13029](https://github.com/penpot/penpot/issues/13029) (PR: [#12835](https://github.com/penpot/penpot/pull/12835)) +- Speed up uploading page shapes to WASM [#13030](https://github.com/penpot/penpot/issues/13030) (PR: [#12854](https://github.com/penpot/penpot/pull/12854)) + +### :bug: Bugs + +- Fix font upload failing with a regex error [#10436](https://github.com/penpot/penpot/issues/10436) (PR: [#11951](https://github.com/penpot/penpot/pull/11951)) - Fix dashboard file thumbnails loading slow for large multi-page files [#12053](https://github.com/penpot/penpot/issues/12053) (PR: [#12054](https://github.com/penpot/penpot/pull/12054)) - Fix Plugin API resolving a font family token to its separate words when the family name has several words [#11748](https://github.com/penpot/penpot/issues/11748) (PR: [#11750](https://github.com/penpot/penpot/pull/11750)) +- Fix file import recreating circular library references when the exported libraries use each other [#12052](https://github.com/penpot/penpot/issues/12052) (PR: [#12062](https://github.com/penpot/penpot/pull/12062)) - Fix asset redirect responses advertising a cache lifetime of about 1000 days instead of 24 hours (by @ners) [#12761](https://github.com/penpot/penpot/issues/12761) (PR: [#12758](https://github.com/penpot/penpot/pull/12758)) - Fix an internal error on typography tokens due to circular dependencies on other token types [#9580](https://github.com/penpot/penpot/issues/9580) - Fix the MCP server rejecting request bodies over 100kb (express.json default) with PayloadTooLargeError (by @GitNimay) [#12080](https://github.com/penpot/penpot/issues/12080) (PR: [#12450](https://github.com/penpot/penpot/pull/12450)) @@ -22,40 +127,24 @@ - Fix token not showing when selecting two text layers (by @ShreyashAgare26) [#11924](https://github.com/penpot/penpot/issues/11924) (PR: [#11944](https://github.com/penpot/penpot/pull/11944)) - Fix inner shadow fill not working in the SVG render while it works in WebGL [#11967](https://github.com/penpot/penpot/issues/11967) (PR: [#12141](https://github.com/penpot/penpot/pull/12141)) - Fix @penpot/plugin-styles loading Work Sans from a third-party CDN (by @ump45nose) [#12023](https://github.com/penpot/penpot/issues/12023) (PR: [#12087](https://github.com/penpot/penpot/pull/12087)) +- Fix schema error on swatch component for image fills [#12135](https://github.com/penpot/penpot/issues/12135) (PR: [#12136](https://github.com/penpot/penpot/pull/12136)) +- Fix worker storing more shape data than needed [#12740](https://github.com/penpot/penpot/issues/12740) (PR: [#12840](https://github.com/penpot/penpot/pull/12840)) +- Fix worker not freeing pages [#12738](https://github.com/penpot/penpot/issues/12738) (PR: [#12751](https://github.com/penpot/penpot/pull/12751)) +- Fix editing the L value in HSL mode resetting H and S to 0 (by @makesomethingshit) [#12615](https://github.com/penpot/penpot/issues/12615) (PR: [#12629](https://github.com/penpot/penpot/pull/12629)) +- Restrict dashboard plugins to the MCP plugin and localhost plugins [#12948](https://github.com/penpot/penpot/issues/12948) (PR: [#12949](https://github.com/penpot/penpot/pull/12949)) +- Fix workspace memory growing on every page switch with the WebGL renderer [#13028](https://github.com/penpot/penpot/issues/13028) (PR: [#12877](https://github.com/penpot/penpot/pull/12877)) +- Fix workspace showing React error #310 when the team container renders without a team id (by @filipsajdak) [#12728](https://github.com/penpot/penpot/issues/12728) (PR: [#12920](https://github.com/penpot/penpot/pull/12920)) -### :sparkles: New features & Enhancements +## 2.19.2 -- Add a feedback channel to the MCP server [#12490](https://github.com/penpot/penpot/issues/12490) (PR: [#12551](https://github.com/penpot/penpot/pull/12551)) -- Auto-scroll sortable lists when dragging an item near their top or bottom edge (by @ShreyashAgare26) [#10478](https://github.com/penpot/penpot/issues/10478) (PR: [#11595](https://github.com/penpot/penpot/pull/11595)) -- Make WebGL the default renderer for new accounts [#11855](https://github.com/penpot/penpot/issues/11855) (PR: [#12880](https://github.com/penpot/penpot/pull/12880)) -- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759) -- Complete the Russian translations [#11955](https://github.com/penpot/penpot/issues/11955) (PR: [#11956](https://github.com/penpot/penpot/pull/11956)) -- Implement full text format paste within Penpot [#12047](https://github.com/penpot/penpot/issues/12047) (PR: [#12137](https://github.com/penpot/penpot/pull/12137)) -- Prevent the hosted MCP server from running execute_code twice when the plugin is open in two tabs [#12008](https://github.com/penpot/penpot/issues/12008) (PR: [#11927](https://github.com/penpot/penpot/pull/11927)) -- Allow detaching typography tokens [#11887](https://github.com/penpot/penpot/issues/11887) (PR: [#11969](https://github.com/penpot/penpot/pull/11969)) -- Add team, project and file management to the Plugin API for global plugins [#12170](https://github.com/penpot/penpot/issues/12170) (PR: [#12169](https://github.com/penpot/penpot/pull/12169)) -- Run .penpot file import and export as durable background jobs [#12194](https://github.com/penpot/penpot/issues/12194) (PR: [#12056](https://github.com/penpot/penpot/pull/12056), [#12192](https://github.com/penpot/penpot/pull/12192)) -- Open files and projects in a new tab with Ctrl/Cmd+click in the dashboard [#12228](https://github.com/penpot/penpot/issues/12228) (PR: [#12230](https://github.com/penpot/penpot/pull/12230)) -- Add an RPC method to read file-level plugin data [#12258](https://github.com/penpot/penpot/issues/12258) (PR: [#10171](https://github.com/penpot/penpot/pull/10171)) -- Add a get-file-tokens RPC method to read a file's design tokens [#12260](https://github.com/penpot/penpot/issues/12260) (PR: [#12172](https://github.com/penpot/penpot/pull/12172)) -- Show the progress and the size of each file in the import and export dialogs [#12340](https://github.com/penpot/penpot/issues/12340) (PR: [#12192](https://github.com/penpot/penpot/pull/12192)) -- Focus the search field when opening the color tokens tab of the color picker [#12344](https://github.com/penpot/penpot/issues/12344) (PR: [#12345](https://github.com/penpot/penpot/pull/12345)) -- Harden the file data offload job with a row lock, per-file transactions, throttle and metrics [#12398](https://github.com/penpot/penpot/issues/12398) (PR: [#12403](https://github.com/penpot/penpot/pull/12403)) -- Render email templates inside the mail job in the backend [#12445](https://github.com/penpot/penpot/issues/12445) (PR: [#12448](https://github.com/penpot/penpot/pull/12448)) -- Serve Penpot under a subpath [#3194](https://github.com/penpot/penpot/issues/3194) (PR: [#12358](https://github.com/penpot/penpot/pull/12358)) -- Run local WASM exports for shapes and boards without freezing the UI [#12512](https://github.com/penpot/penpot/issues/12512) (PR: [#12515](https://github.com/penpot/penpot/pull/12515)) -- Use WASM to generate SVG markup in the code inspector when render-wasm is active [#12567](https://github.com/penpot/penpot/issues/12567) (PR: [#12608](https://github.com/penpot/penpot/pull/12608)) -- Close the workspace search with the Escape key (by @filipsajdak) [#9540](https://github.com/penpot/penpot/issues/9540) (PR: [#12374](https://github.com/penpot/penpot/pull/12374)) -- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761) -- Add OAuth support for MCP client connections [#12034](https://github.com/penpot/penpot/issues/12034) (PR: [#12480](https://github.com/penpot/penpot/pull/12480)) -- Restrict management RPC methods to their declared shared keys [#12730](https://github.com/penpot/penpot/issues/12730) (PR: [#12736](https://github.com/penpot/penpot/pull/12736)) -- Reduce memory used by duplicate shape ids in the workspace [#12791](https://github.com/penpot/penpot/issues/12791) (PR: [#12792](https://github.com/penpot/penpot/pull/12792)) -- Reduce memory consumption on shape decoding [#12789](https://github.com/penpot/penpot/issues/12789) (PR: [#12790](https://github.com/penpot/penpot/pull/12790)) -- Notify other users when a snapshot is restored [#12425](https://github.com/penpot/penpot/issues/12425) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) -- Normalize the configured tenant to lowercase [#12842](https://github.com/penpot/penpot/issues/12842) (PR: [#12843](https://github.com/penpot/penpot/pull/12843)) -- Speed up saves on large files by checking only the pages a save changes [#12602](https://github.com/penpot/penpot/issues/12602) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) -- Retry saves after short server failures and keep the workspace open when a save fails [#12601](https://github.com/penpot/penpot/issues/12601) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) -- Switch the workspace to read-only mode when the file can't be saved [#12507](https://github.com/penpot/penpot/issues/12507) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +### :bug: Bugs fixed + +- Fix referential integrity error when copying component variants with nested components across libraries and teams [#10704](https://github.com/penpot/penpot/issues/10704) (PR: [#12928](https://github.com/penpot/penpot/pull/12928)) +- Fix migration 0033 inserting nil shape entries for dangling child references that blocked file opening [#12936](https://github.com/penpot/penpot/issues/12936) (PR: [#12979](https://github.com/penpot/penpot/pull/12979)) +- Fix switching a variant and adding a component instance taking over 10 seconds in files with many token-styled shapes [#13019](https://github.com/penpot/penpot/issues/13019) (PR: [#13021](https://github.com/penpot/penpot/pull/13021)) +- Fix autosave failing after uploading an SVG as a component in the assets panel, and on other mains named unlike their component [#12773](https://github.com/penpot/penpot/issues/12773) (PR: [#12794](https://github.com/penpot/penpot/pull/12794)) +- Fix `enable-oidc-registration` not working when password login is disabled [#12964](https://github.com/penpot/penpot/issues/12964) (PR: [#12974](https://github.com/penpot/penpot/pull/12974)) +- Fix self-hosted backend opening 60 database connections at startup and keeping them open while idle (by @UsamaSadiq) [#12622](https://github.com/penpot/penpot/issues/12622) (PR: [#13052](https://github.com/penpot/penpot/pull/13052)) ## 2.19.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b23ae0fa9..27ec89998b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,7 @@ Center](https://help.penpot.app/). - [Title format](#title) - [Description](#description) - [Review process](#review-process) + - [Merging](#merging) - [What we won't accept](#what-we-wont-accept) - [Good first issues](#good-first-issues) - [Commit Guidelines](#commit-guidelines) @@ -111,8 +112,8 @@ Branch names are not enforced, but we recommend the following: - Otherwise, use a short, descriptive name with words separated by hyphens and no slashes (e.g. `fix-ellipse-icon-typo`, `feat-auto-link-libraries`). -Since PRs are squash-merged, the branch name does not survive into the -commit history — what matters is the [PR title](#title). +Since PRs are squash-merged by default, the branch name does not survive +into the commit history — what matters is the [PR title](#title). ### Format @@ -156,9 +157,27 @@ for more concrete information. - Address review feedback by **pushing new commits** — do not force-push during review, as it breaks comment threads. - PRs require at least **one approval** before merge. -- We use **squash-merge** by default. The PR title becomes the final +- We **squash-merge** by default; see [Merging](#merging) for the reason + and for how to ask for a regular merge. The PR title becomes the final commit message, so follow the [title format](#title) above. +### Merging + +PRs are **squash-merged** by default. One PR lands as one commit, which keeps +the change atomic: the history stays readable, and `git bisect` never stops +on an intermediate commit that fails the tests or the linter. + +This is the default, not a hard rule. If you keep your commits atomic and +self-contained — each one builds, passes the linter and makes sense on its +own — you may ask for a regular merge instead. Put the request in the first +line of the PR description, above everything else: + +> **Merge:** do not squash; each commit is self-contained. + +Maintainers honor the request when the commits hold up, and squash-merge +otherwise. After a squash merge the PR title becomes the commit subject on +the main branch; see [Title format](#title). + ### What we won't accept To save time on both sides, please avoid submitting PRs that: diff --git a/backend/package.json b/backend/package.json index ff67eed620..47d2c28a66 100644 --- a/backend/package.json +++ b/backend/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en España SL", "private": true, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "repository": { "type": "git", "url": "https://github.com/penpot/penpot" diff --git a/backend/pnpm-lock.yaml b/backend/pnpm-lock.yaml index 503657d9a7..ba33015523 100644 --- a/backend/pnpm-lock.yaml +++ b/backend/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/backend/src/app/auth/oidc.clj b/backend/src/app/auth/oidc.clj index 55aeff66fc..0de9163f84 100644 --- a/backend/src/app/auth/oidc.clj +++ b/backend/src/app/auth/oidc.clj @@ -690,15 +690,113 @@ (assoc :query (u/map->query-string params)))] (redirect-response uri)))) +(defn- public-uri + "The configured public URI, parsed, with the app base path normalized with + a trailing slash." + [] + (u/ensure-path-slash (u/uri (cf/get :public-uri)))) + +(defn- public-uri-base + "The public URI as a string, with the app base path normalized with a + trailing slash. Every URL the server builds starts here, so the app base + path is never lost." + [] + (str (public-uri))) + (defn- redirect-with-organization-sso-error [{:keys [dest-url organization-id organization-name]}] - (-> (str (or dest-url (cf/get :public-uri))) + (-> (str (or dest-url (public-uri-base))) (u/append-query-param :sso-error true) (u/append-query-param :organization-id organization-id) (cond-> organization-name (u/append-query-param :organization-name organization-name)) (redirect-response))) +(defn- routable-params? + "True when `params` are params of a route: at least one, and no key carries + a scheme, a path or a fragment. `screen=workspace&team-id=…` and the root + params of the application (`template=…`) qualify; `https://evil.example/` + does not." + [params] + (and (seq params) + (every? #(nil? (re-find #"[/:?#]" (name %))) (keys params)))) + +(defn build-dest-url + "Return the absolute URL for `query-string` under the current public URI. + + The query string is the route of the client (`screen=…¶ms`, and the root + of the application may carry only its params, like `template=…`): it does not + carry the `?` that separates it from the path, which is put back here, so the + destination is always built by the server and the client only picks the route. + A value that still comes with the separator is trimmed, so a client bundled + before that change keeps its destination. + + The params are parsed and rendered back, so a value that is not a route never + becomes a destination: it is ignored with a warning and the user lands on the + public URI, which is also where a blank one lands." + [query-string] + (try + (let [base (public-uri-base) + query (str query-string) + route (if (str/starts-with? query "?") + (subs query 1) + query) + params (u/query-string->map route)] + (cond + (str/blank? query) + base + + (not (routable-params? params)) + (do (l/warn :hint "ignoring a redirect target that is not a route" + :query-string (subs route 0 (min 200 (count route)))) + base) + + :else + (str base "?" (u/map->query-string params)))) + (catch Exception e + ;; The destination must never break the SSO flow: a query string the + ;; server cannot parse lands on the public URI instead. + (l/warn :hint "could not build the redirect target" :cause e) + (public-uri-base)))) + +(defn- same-origin? + "True when `uri` has the same origin as `base`: scheme, host (case + insensitive) and port, with the default port made explicit." + [uri base] + (let [default-port (fn [scheme] (if (= "https" (str/lower (str scheme))) 443 80)) + port (fn [u] (or (:port u) (default-port (:scheme u))))] + (and (= (str/lower (str (:scheme uri))) + (str/lower (str (:scheme base)))) + (= (str/lower (str (:host uri))) + (str/lower (str (:host base)))) + (= (port uri) (port base))))) + +(defn- under-base-path? + "True when the path of `uri` is under the base path of `base`." + [uri base] + (str/starts-with? (str (:path uri)) (:path base))) + +(defn- trusted-dest-url + "Return `dest-url` when it points inside the current public URI, nil + otherwise. + + The destination is built by the server (see `build-dest-url`), but state + tokens minted before that was the case may carry any URL, so the callback + checks the destination before redirecting to it. The check compares the + parsed origin, so neither a userinfo prefix + (`https://penpot.example.com@evil.example/`) nor a look-alike host + (`https://penpot.example.com.evil.example/`) can pass, and it requires + the path to be under the base path of the public URI, so the destination + stays inside the application." + [dest-url] + (when (some? dest-url) + (let [dest (try (u/uri (str dest-url)) (catch Exception _ nil)) + base (public-uri)] + (when (and (some? dest) + (same-origin? dest base) + (under-base-path? dest base)) + (str dest-url))))) + (defn- redirect-to-register [cfg info provider] (let [info (assoc info @@ -915,11 +1013,18 @@ (defn build-organization-sso-auth-redirect-uri "Build the OIDC authorization redirect URI for an organization SSO config. - Raises if the config is incomplete or OIDC discovery fails." - [cfg sso & {:keys [dest-url organization-id provider]}] + Raises if the config is incomplete or OIDC discovery fails. + + `query-string` is the route of the client (`screen=…¶ms`, the query + string proper, without its separator): the destination is built here, so it + can never point outside the public URI. + `profile-id` is the profile that started the flow; it travels in the + state token so the callback only grants the organization SSO entry to that + very session." + [cfg sso & {:keys [query-string organization-id provider profile-id]}] (let [organization-id (or organization-id (:organization-id sso)) issuer (organization-sso-discovery-uri sso) - dest-url (or dest-url (str (cf/get :public-uri)))] + dest-url (build-dest-url query-string)] (when-not issuer (raise-invalid-sso-config :hint "missing issuer" @@ -929,6 +1034,7 @@ state-token (tokens/generate cfg {:iss "oidc" :dest-url dest-url :organization-id organization-id + :profile-id profile-id :issuer issuer :exp (ct/in-future "4h")})] (build-auth-redirect-uri oidc-provider state-token)) @@ -1024,25 +1130,39 @@ (defn- organization-sso-callback-handler "Handle the organization-SSO branch of the OIDC callback: state carries :dest-url — exchange the authorization code with the OIDC provider to - verify authentication actually occurred, then redirect back to dest-url." + verify authentication actually occurred, then redirect back to dest-url. + + The state also carries the :profile-id that started the flow: only that + session is granted the organization SSO entry, so a state prepared by + someone else cannot mark the session of the one who completes it." [cfg request state code] - (let [dest-url (:dest-url state)] + (let [dest-url (trusted-dest-url (:dest-url state))] (try (let [organization-id (:organization-id state) sso (nitrate/call cfg :get-organization-sso {:organization-id organization-id}) provider (prepare-organization-sso-provider cfg sso) _info (get-info cfg provider state code) session (session/get-session request) - exp (ct/in-future {:hours 4})] - (when (and session organization-id) - (let [props (-> (or (:props session) {}) - (update :sso assoc organization-id exp))] - (session/update-session (::session/manager cfg) (assoc session :props props)) + exp (ct/in-future {:hours 4}) + ;; A state minted before the profile claim existed never + ;; matches, and neither does the session of another profile. + same-profile? (and (some? session) + (some? (:profile-id state)) + (= (:profile-id state) (:profile-id session)))] + (if (and (some? session) (not same-profile?)) + (submit-organization-sso-auth-event + cfg request (:profile-id session) organization-id "organization-sso-auth-failed" + :failure-reason "session-mismatch") + (do + (when (and session organization-id) + (let [props (-> (or (:props session) {}) + (update :sso assoc organization-id exp))] + (session/update-session (::session/manager cfg) (assoc session :props props)) + (submit-organization-sso-auth-event + cfg request (:profile-id session) organization-id "organization-sso-token-issued"))) (submit-organization-sso-auth-event - cfg request (:profile-id session) organization-id "organization-sso-token-issued"))) - (submit-organization-sso-auth-event - cfg request (:profile-id session) organization-id "organization-sso-auth-succeeded") - (redirect-response dest-url)) + cfg request (:profile-id session) organization-id "organization-sso-auth-succeeded"))) + (redirect-response (or dest-url (public-uri-base)))) (catch Throwable cause (let [{:keys [code]} (ex-data cause)] (binding [l/*context* (errors/request->context request)] @@ -1060,10 +1180,36 @@ (let [organization-id (:organization-id state) organization-name (:name (nitrate/call cfg :get-organization-summary {:organization-id organization-id}))] (redirect-with-organization-sso-error - {:dest-url dest-url + {:dest-url (or dest-url (public-uri-base)) :organization-id organization-id :organization-name organization-name})))))) +(def oidc-provider-types + "Backends authenticating through an OIDC-compatible provider. + Only these backends take the SSO registration gate." + #{"oidc" "google" "github" "gitlab"}) + +(defn sso-registration-allowed? + "SSO registration is allowed when public registration or explicit + OIDC registration is enabled. Shared by the callback gate and the + `register-profile` token gate — keep one definition." + [] + (or (contains? cf/flags :registration) + (contains? cf/flags :oidc-registration))) + +(defn- valid-invitation-for? + "Returns true when `invitation-token` is a verifiable + `:team-invitation` JWT issued for `email`. Never throws: a garbage + token means there is no valid invitation." + [cfg invitation-token email] + (try + (boolean + (when (and (some? invitation-token) (some? email)) + (let [invitation (tokens/verify cfg {:token invitation-token :iss :team-invitation})] + (= (profile/clean-email (:member-email invitation)) + (profile/clean-email email))))) + (catch Throwable _ false))) + (defn- callback-handler [cfg {:keys [params] :as request}] (if-let [error (get params :error)] @@ -1096,8 +1242,8 @@ (redirect-with-error "email-domain-not-allowed") :else - (if (or (contains? cf/flags :registration) - (contains? cf/flags :oidc-registration)) + (if (or (sso-registration-allowed?) + (valid-invitation-for? cfg (:invitation-token info) (:email info))) (redirect-to-register cfg info provider) (redirect-with-error "registration-disabled"))) diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 37d9edb548..0bed77adc1 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -45,7 +45,6 @@ (def ^:dynamic *state* nil) (def ^:dynamic *options* nil) -(def ^:dynamic *reference-file* nil) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; DEFAULTS @@ -161,7 +160,7 @@ [index obj & attrs] (reduce (fn [index attr-fn] (let [old-id (attr-fn obj) - new-id (if (::overwrite *options*) old-id (uuid/next))] + new-id (uuid/next)] (assoc index old-id new-id))) index attrs)) @@ -212,6 +211,7 @@ p.team_id, coalesce(fd.backend, 'legacy-db') AS backend, fd.metadata AS metadata, + fd.storage_object_id AS storage_object_id, fd.data AS data FROM file AS f LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id) @@ -221,27 +221,37 @@ (str sql:files-with-data " WHERE f.id = ?")) (def sql:get-file-without-data - (str "WITH files AS (" sql:files-with-data ")" - "SELECT f.id, - f.project_id, - f.created_at, - f.modified_at, - f.deleted_at, - f.name, - f.is_shared, - f.has_media_trimmed, - f.revn, - f.ignore_sync_until, - f.comment_thread_seqn, - f.features, - f.version, - f.vern, - f.team_id - FROM files AS f - WHERE f.id = ?")) + "SELECT f.id, + f.project_id, + f.created_at, + f.modified_at, + f.deleted_at, + f.name, + f.is_shared, + f.has_media_trimmed, + f.revn, + f.ignore_sync_until, + f.comment_thread_seqn, + f.features, + f.version, + f.vern, + p.team_id + FROM file AS f + INNER JOIN project AS p ON (p.id = f.project_id) + WHERE f.id = ?") (defn- migrate-file - [{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}] + "Apply the pending migrations to the file. + + With `persist-after-migration?` the migrated map is also persisted, which is + only safe from a map read under the `file` row lock (see `get-file*`). The + returned map is then the reattached one: decoded data where the pages the + migration did not change keep their pointers and their fragment ids, plus + the `:storage-object-id` of the row the write left, so a caller that + persists again touches the current object. Without `persist-after-migration?` + the realized map is returned, which is what a read-only context can do. + `get-file*` realizes the result when the caller asks for it." + [cfg {:keys [id] :as file} {:keys [persist-after-migration?]}] (fdata/with-tracked-pointers cfg id (let [libs (delay (get-resolved-file-libraries cfg file)) ;; Capture the pointer layout before realizing: the migration @@ -255,17 +265,93 @@ objects-map-added? (and (not objects-map-before?) (contains? (:features realized) "fdata/objects-map"))] - (if (or read-only? (db/read-only? conn)) - realized - (let [file (if objects-map-added? - realized - (fdata/reattach-unchanged original realized))] + (if persist-after-migration? + (let [file (if objects-map-added? + realized + (fdata/reattach-unchanged original realized)) + written (update-file! cfg file)] ;; When the file is migrated, we break the rule of not ;; performing mutations on get operations and update the file - ;; with all migrations applied. The caller still receives the - ;; realized file: the read contract is plain data. - (update-file! cfg file) - (fmigr/resolve-applied-migrations cfg realized)))))) + ;; with all migrations applied. The reattached map is also what + ;; the read returns: the caller realizes it only when it asks. + (assoc (fmigr/resolve-applied-migrations cfg file) + :storage-object-id (:storage-object-id written))) + realized)))) + +(defn- fetch-row + "Read the file row. With `load-data?` the file data columns come along, but + they are not resolved nor decoded (see `attach-file-data`). With + `load-migrations?` the applied migrations are resolved, which is what + `need-migration?` expects. A skip-locked read answers nil instead of raising + when the row is gone." + [{:keys [::db/conn] :as cfg} id + {:keys [include-deleted? + skip-locked? + lock-for-update? + lock-for-share? + throw-if-not-exists? + load-migrations? + load-data?]}] + + (let [sql (if load-data? + sql:get-file + sql:get-file-without-data) + sql (cond + lock-for-update? + (str sql " FOR UPDATE of f") + + lock-for-share? + (str sql " FOR SHARE of f") + + :else + sql) + + sql (if (and skip-locked? (or lock-for-share? lock-for-update?)) + (str sql " SKIP LOCKED") + sql) + + file (db/get-with-sql conn [sql id] {::db/throw-if-not-exists false + ::db/remove-deleted (not include-deleted?)}) + file (some-> file + (d/update-when :features db/decode-pgarray #{}) + (d/update-when :metadata fdata/decode-metadata)) + file (if (and file load-migrations?) + (fmigr/resolve-applied-migrations cfg file) + file)] + + (or file + (when-not (or skip-locked? (not throw-if-not-exists?)) + (ex/raise :type :not-found + :code :object-not-found + :hint "database object not found" + :table :file + :file-id id))))) + + +(defn- attach-file-data + "Resolve the data of an already read row and decode it when `decode?`. It + completes the map it receives instead of reading the row again. It does not + realize: `get-file*` owns that step, once per path." + [cfg file {:keys [decode?]}] + (let [file (fdata/resolve-file-data cfg file)] + (if decode? + (fdata/decode-file-data cfg file) + file))) + +(defn- fetch-file + "Read the row with its data resolved and decoded, under the requested lock." + [cfg id options] + (when-let [file (fetch-row cfg id options)] + (attach-file-data cfg file options))) + +(def ^:private + default-get-file-options + {:load-migrations? true + :throw-if-not-exists? true + :persist-after-migration? true + :load-data? true + :migrate? true + :decode? true}) ;; --- Branch files: derived data -------------------------------------- ;; @@ -561,110 +647,123 @@ :branch-oplog-depth depth :branch-oplog-batches (count rows)})))) +(defn- attach-branch-file-data + "Complete a branch row read with its data columns: the branch counterpart + of `attach-file-data`. A branch file derives its data from the pinned base + snapshot and the op log (see `branch-file-data`), so data migrations apply + in memory only and the derived state stays current without persisting a + data payload. Realizes when `realize?`, as `get-file*` does on every path." + [cfg file {:keys [migrate? realize? decode? include-base-data?] :as options}] + (let [file (cond->> (branch-file-data cfg file decode? include-base-data?) + decode? + (fdata/decode-file-data cfg))] + (cond-> file + (and migrate? decode? (fmg/need-migration? file)) + (as-> $ (migrate-file cfg $ (assoc options :persist-after-migration? false))) + + realize? + (->> (fdata/realize cfg))))) + (defn- get-file* - [{:keys [::db/conn] :as cfg} id - {:keys [migrate? - realize? - decode? - skip-locked? - include-deleted? - load-data? - throw-if-not-exists? - lock-for-update? - lock-for-share? - include-base-data?] - :or {lock-for-update? false - lock-for-share? false - load-data? true - migrate? true - decode? true - include-deleted? false - throw-if-not-exists? true - realize? false - include-base-data? false} - :as options}] + [{:keys [::db/conn] :as cfg} id options] (assert (db/connection? conn) "expected cfg with valid connection") - (when (and (not load-data?) - (or lock-for-share? lock-for-update? skip-locked?)) - (throw (IllegalArgumentException. "locking is incompatible when `load-data?` is false"))) - (let [sql - (if load-data? - sql:get-file - sql:get-file-without-data) + (let [{:keys [migrate? + realize? + decode? + load-data? + lock-for-share? + read-only?] + :as options} + (merge default-get-file-options options) - sql + read-only? + (or read-only? (db/read-only? conn))] + + (when (and realize? (or (not load-data?) (not decode?))) + (throw (IllegalArgumentException. "invalid options, you only can realize when loading data and decoding"))) + + (when (and lock-for-share? load-data?) + (throw (IllegalArgumentException. "invalid options, `:lock-for-share?` is only allowed on row-only reads"))) + + (cond + ;; The caller only wants the row: that read is the answer. + (not load-data?) + (fetch-row cfg id options) + + (and migrate? decode? (not read-only?)) + (when-let [file (fetch-row cfg id options)] (cond - lock-for-update? - (str sql " FOR UPDATE of f") + (:is-branch file) + (attach-branch-file-data cfg file options) - lock-for-share? - (str sql " FOR SHARE of f") + (fmg/need-migration? file) + (when-let [file (fetch-file cfg id (assoc options :lock-for-update? true))] + (cond-> file + (fmg/need-migration? file) + (as-> $ (migrate-file cfg $ options)) + + realize? + (->> (fdata/realize cfg)))) :else - sql) + (let [file (attach-file-data cfg file options)] + (cond-> file + realize? + (->> (fdata/realize cfg)))))) - sql - (if skip-locked? - (str sql " SKIP LOCKED") - sql) + :else + (when-let [file (fetch-row cfg id options)] + (if (:is-branch file) + (attach-branch-file-data cfg file options) + (let [file (attach-file-data cfg file options)] + (cond-> file + (and migrate? decode? (fmg/need-migration? file)) + (as-> $ (migrate-file cfg $ (assoc options :persist-after-migration? false))) - file - (db/get-with-sql conn [sql id] - {::db/throw-if-not-exists false - ::db/remove-deleted (not include-deleted?)}) - - file - (-> file - (d/update-when :features db/decode-pgarray #{}) - (d/update-when :metadata fdata/decode-metadata))] - - (if file - (if load-data? - (let [branch? (:is-branch file) - - file - (if branch? - (branch-file-data cfg file decode? include-base-data?) - (->> file - (fmigr/resolve-applied-migrations cfg) - (fdata/resolve-file-data cfg))) - - ;; a branch file derives its data from the pinned base - ;; snapshot and the op log; data migrations apply in - ;; memory only (read-only) so the derived state stays - ;; current without persisting a data payload - will-migrate? - (and migrate? (fmg/need-migration? file))] - - (if decode? - (cond->> (fdata/decode-file-data cfg file) - (and realize? (not will-migrate?)) - (fdata/realize cfg) - - will-migrate? - (migrate-file cfg (cond-> options - branch? (assoc :read-only? true)))) - - file)) - file) - - (when-not (or skip-locked? (not throw-if-not-exists?)) - (ex/raise :type :not-found - :code :object-not-found - :hint "database object not found" - :table :file - :file-id id))))) + realize? + (->> (fdata/realize cfg))))))))) (defn get-file "Get file, resolve all features and apply migrations. + Runs inside a transaction: it joins the caller's when the cfg already + carries a connection and opens one otherwise, so the migration write + commits with the read. A read-only read (`:read-only?`) takes no lock and + writes nothing, so it runs on a plain connection instead of opening a + transaction of its own; with a cfg that already carries a connection it + still joins the caller's. Accepts a cfg map only: a bare pool or a + connection raises `IllegalArgumentException`. + Useful when you have plan to apply massive or not surgical operations on file, because it removes the overhead of lazy fetching - and decoding." + and decoding. + + Options: + - `:realize?` returns plain data instead of fragment pointers; it + requires `:load-data?` and `:decode?`. + - `:decode?` decodes the stored blob (default true). + - `:load-data?` reads the data columns (default true). A row-only read + (`false`) is the only mode that accepts `:lock-for-share?`. + - `:migrate?` applies the pending migrations on read (default true). + - `:load-migrations?` resolves the applied migrations on a row read + (default true). + - `:read-only?` applies the pending migrations in memory only and skips + the transaction. It does not take a lock. + - `:lock-for-update?`, `:skip-locked?`, `:include-deleted?` and + `:throw-if-not-exists?` are the usual read knobs. A lock needs a + read-write transaction from the caller: PostgreSQL rejects row locks in + a read-only transaction. + - `:persist-after-migration?` is internal: `get-file*` turns it off + for the in-memory migration of a read-only context. Do not pass it." [cfg file-id & {:as opts}] - (db/run! cfg get-file* file-id opts)) + (when-not (map? cfg) + (throw (IllegalArgumentException. + "expected cfg map, not a pool or a connection"))) + (if (:read-only? opts) + (db/run! cfg get-file* file-id opts) + (db/tx-run! cfg get-file* file-id opts))) (defn clean-file-features [file] @@ -692,19 +791,27 @@ inner join project as p on (p.id = f.project_id) where fpr.file_id = ? and fpr.profile_id = ? + and f.deleted_at is null + and p.deleted_at is null and exists (select 1 from team_profile_rel as tpr + inner join team as t on (t.id = tpr.team_id) where tpr.team_id = p.team_id - and tpr.profile_id = fpr.profile_id) + and tpr.profile_id = fpr.profile_id + and t.deleted_at is null) union all select tpr.is_owner, tpr.is_admin, tpr.can_edit from team_profile_rel as tpr + inner join team as t on (t.id = tpr.team_id) inner join project as p on (p.team_id = tpr.team_id) inner join file as f on (p.id = f.project_id) where f.id = ? and tpr.profile_id = ? + and t.deleted_at is null + and p.deleted_at is null + and f.deleted_at is null union all select ppr.is_owner, ppr.is_admin, @@ -714,10 +821,14 @@ inner join project as p on (p.id = ppr.project_id) where f.id = ? and ppr.profile_id = ? + and f.deleted_at is null + and p.deleted_at is null and exists (select 1 from team_profile_rel as tpr + inner join team as t on (t.id = tpr.team_id) where tpr.team_id = p.team_id - and tpr.profile_id = ppr.profile_id)") + and tpr.profile_id = ppr.profile_id + and t.deleted_at is null)") (defn- get-file-permissions* [conn profile-id file-id] @@ -796,10 +907,13 @@ ON fls.file_id = flr.file_id AND fls.library_file_id = flr.library_file_id WHERE flr.file_id = ANY(?) - AND l.deleted_at IS NULL;") + AND l.deleted_at IS NULL + ORDER BY flr.file_id, flr.library_file_id;") (defn get-files-rels - "Given a set of file-id's, return all matching relations with the libraries" + "Given a set of file-id's, return all matching relations with the libraries. + Rows come back in stable `(file-id, library-file-id)` order so export + and import resolve cycles deterministically." [cfg ids] (assert (set? ids) "expected a set of uuids") @@ -842,6 +956,20 @@ (->> (db/exec! conn [sql:get-libraries ids']) (into #{} xf-map-id)))))) +(defn library-link-status + "Check whether linking `file-id` to `library-id` is safe. Returns + `:ok` when the link can be created, `:self-link` when both ids are + the same, and `:circular` when the link would close a dependency + cycle (i.e. `file-id` is already among the transitive libraries of + `library-id`). Shared by the UI command (`link-file-to-library`, + which raises) and the v3 importer (`import-file-relations`, which + skips and logs), so both stay consistent." + [cfg file-id library-id] + (cond + (= file-id library-id) :self-link + (contains? (get-libraries cfg [library-id]) file-id) :circular + :else :ok)) + (defn get-file-object-thumbnails "Return all file object thumbnails for a given file." [cfg file-id] @@ -1017,7 +1145,12 @@ (let [library-ids (get-libraries cfg [file-id]) data (reduce (fn [data library-id] (if-let [library (get-file cfg library-id :include-deleted? true)] - (cond-> (ctf/absorb-assets data (:data library)) + ;; The library is read without `:realize?`, so its + ;; `:components` may be a pointer map. Bind the + ;; library's own loader: the absorb walks the + ;; components and would raise without one. + (cond-> (fdata/with-pointers cfg library-id + (ctf/absorb-assets data (:data library))) (= library-id (cfo/get-tokens-source data)) (cfo/absorb-tokens-lib (:data library))) data)) @@ -1102,18 +1235,33 @@ (dissoc :metadata) (dissoc :migrations))) +(defn file->response + "Strip the internal keys of a file map before it leaves the backend in a + response. The file data storage pointer is bookkeeping for the write path: + a `file-data` object is authentication-only, so a leaked id would let any + logged-in profile fetch the blob from `/assets/by-id`. The metadata key of + the old pointer is stripped too: a row the migration does not backfill keeps + it, and the metadata schema is open, so it would travel in the response." + [file] + (-> file + (dissoc :storage-object-id) + (d/update-when :metadata #(dissoc % :storage-ref-id)))) + (defn- file->file-data-params - [{:keys [id] :as file} & {:as opts}] + [{:keys [id] :as file}] (let [created-at (or (:created-at file) (ct/now)) modified-at (or (:modified-at file) created-at)] - (d/without-nils - {:id id - :type "main" - :file-id id - :data (:data file) - :metadata (:metadata file) - :created-at created-at - :modified-at modified-at}))) + (-> (d/without-nils + {:id id + :type "main" + :file-id id + :data (:data file) + :metadata (:metadata file) + :created-at created-at + :modified-at modified-at}) + ;; The storage object the persisted row pointed at (nil for a new or + ;; imported file). `upsert!` touches it before replacing the row. + (assoc :storage-object-id (:storage-object-id file))))) (defn insert-file! "Insert a new file into the database table. Expectes a not-encoded file. @@ -1137,34 +1285,63 @@ :cause cause) (throw cause)))) - (->> (file->file-data-params file) + ;; An INSERT never replaces a persisted pointer: `duplicate-file` hands + ;; the source file map here, and touching the source object would be a + ;; write on a live object. + (->> (file->file-data-params (dissoc file :storage-object-id)) (fdata/upsert! cfg)) nil)) (defn update-file! - "Update an existing file on the database. Expects not encoded file." - [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}] + "Update an existing file on the database. Expects a not-encoded file map, + as returned by `get-file`, and optionally the fields to change as keyword + args; they are merged into the file map before persisting. - (if (::reset-migrations? opts false) - (fmigr/reset-migrations! conn file) - (fmigr/upsert-migrations! conn file)) + The storage object the persisted row pointed at travels in the map + (`:storage-object-id`), and `upsert!` touches it before replacing the row. + That is why the caller must pass the map it read instead of building a + partial one: nothing is queried here. The params only change mutable fields + of the file: the pointer and the identity (`:id`, `:file-id`) always come + from the read map. - (let [file - (encode-file cfg file) + Returns the map `fdata/upsert!` answers with: `:written` and the + `:storage-object-id` of the object this write created, so a caller that + persists again touches the current object instead of the replaced one." + [{:keys [::db/conn] :as cfg} file & {:as params}] - file-params - (file->params (dissoc file :id)) + ;; A map read with `:load-data? false` has no `:data`; persisting it would + ;; wipe the file content. `ex/raise` instead of `assert`: the guard must + ;; exist in the deployed binary, where `clojure.core/assert` is compiled + ;; out. + (when-not (contains? file :data) + (ex/raise :type :internal + :code :missing-file-data + :hint "expected a file map read with data" + :file-id (:id file))) - file-data-params - (file->file-data-params file)] + (let [reset? (::reset-migrations? params false) + ;; The pointer and the identity are bookkeeping of the persisted row, + ;; not editable fields: they can only come from the read map. A pointer + ;; in the params would skip the touch and leak the replaced blob; an + ;; `:id` would retarget the UPDATE. + file (merge file (dissoc params ::reset-migrations? + :storage-object-id + :id + :file-id))] - (db/update! conn :file file-params - {:id id} - {::db/return-keys false}) + (if reset? + (fmigr/reset-migrations! conn file) + (fmigr/upsert-migrations! conn file)) - (fdata/upsert! cfg file-data-params) - nil)) + (let [file (encode-file cfg file) + file-params (file->params (dissoc file :id))] + + (db/update! conn :file file-params + {:id (:id file)} + {::db/return-keys false}) + + (fdata/upsert! cfg (file->file-data-params file))))) (defn insert-file-row! "Insert only the `file` row of a new file, without writing the @@ -1223,8 +1400,7 @@ (let [file (-> file (assoc :created-at timestamp) (assoc :modified-at timestamp) - (cond-> (not (::overwrite cfg)) - (assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5})))) + (assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5}))) (update :features (fn [features] (-> (::features cfg #{}) @@ -1241,9 +1417,7 @@ (when (ex/exception? result) (l/error :hint "file schema validation error" :cause result)))) - (if (::overwrite cfg) - (update-file! cfg file (assoc opts ::reset-migrations? true)) - (insert-file! cfg file opts)))) + (insert-file! cfg file opts))) (def ^:private sql:get-file-libraries "WITH RECURSIVE libs AS ( diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index 76fc870f5d..9b9bc5af27 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -482,9 +482,11 @@ (bfm/apply-pending-migrations! cfg) ;; Knowing that the ids of the created files are in index, - ;; just lookup them and return them in the same shape as v3. + ;; just lookup them and return them in the same shape as v3. The + ;; metadata of an old package may repeat a file id, so the answer + ;; carries one id per imported file. (let [files (-> bfc/*state* deref :files)] - {:file-ids (into [] (keep #(get-in @bfc/*state* [:index %])) files)}))))) + {:file-ids (into [] (comp (keep #(get-in @bfc/*state* [:index %])) (distinct)) files)}))))) (defmethod read-import :v1 [options] @@ -510,89 +512,114 @@ (defmethod read-section :v1/files [{:keys [::bfc/input ::bfc/project-id ::bfc/name] :as system}] - (doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))] - (let [file (read-obj! input) - media (read-obj! input) + (let [imported (volatile! #{})] + (doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))] + (let [file (read-obj! input) + media (read-obj! input) + file-id (:id file)] - file-id (:id file) - file-id' (bfc/lookup-index file-id) + (when (not= file-id expected-file-id) + (ex/raise :type :validation + :code :inconsistent-penpot-file + :found-id file-id + :expected-id expected-file-id + :hint "the penpot file seems corrupt, found unexpected uuid (file-id)")) - file (bfc/clean-file-features file) - thumbnails (:thumbnails file)] + (if (contains? @imported file-id) + ;; Old exports could repeat a file id in the metadata (the file was + ;; also one of its own libraries). The entry is read above to keep + ;; the stream in sync, but the file is imported only once. + (l/warn :hint "ignoring repeated file entry" + :file-id (str file-id) + ::l/sync? true) - (when (not= file-id expected-file-id) - (ex/raise :type :validation - :code :inconsistent-penpot-file - :found-id file-id - :expected-id expected-file-id - :hint "the penpot file seems corrupt, found unexpected uuid (file-id)")) + (let [file-id' (bfc/lookup-index file-id) + file (bfc/clean-file-features file) + thumbnails (:thumbnails file)] - (l/dbg :hint "processing file" - :id (str file-id) - :features (:features file) - :version (-> file :data :version) - :media (count media) - :thumbnails (count thumbnails) - ::l/sync? true) + (vswap! imported conj file-id) - (when (seq thumbnails) - (let [thumbnails (remap-thumbnails thumbnails file-id')] - (l/dbg :hint "updated index with thumbnails" - :total (count thumbnails) - ::l/sync? true) - (vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails))) + (l/dbg :hint "processing file" + :id (str file-id) + :features (:features file) + :version (-> file :data :version) + :media (count media) + :thumbnails (count thumbnails) + ::l/sync? true) - (when (seq media) - ;; Update index with media - (l/dbg :hint "update index with media" :total (count media) ::l/sync? true) - (vswap! bfc/*state* update :index bfc/update-index (map :id media)) + (when (seq thumbnails) + (let [thumbnails (remap-thumbnails thumbnails file-id')] + (l/dbg :hint "updated index with thumbnails" + :total (count thumbnails) + ::l/sync? true) + (vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails))) - ;; Store file media for later insertion - (l/dbg :hint "update media references" ::l/sync? true) - (vswap! bfc/*state* update :media into (map #(update % :id bfc/lookup-index)) media)) + (when (seq media) + ;; Update index with media + (l/dbg :hint "update index with media" :total (count media) ::l/sync? true) + (vswap! bfc/*state* update :index bfc/update-index (map :id media)) - (let [file (-> file - (assoc :id file-id') - (cond-> (and (= idx 0) (some? name)) - (assoc :name name)) - (assoc :project-id project-id) - (dissoc :thumbnails)) - file (bfc/process-file system file)] + ;; Store file media for later insertion + (l/dbg :hint "update media references" ::l/sync? true) + (vswap! bfc/*state* update :media into (map #(update % :id bfc/lookup-index)) media)) - ;; All features that are enabled and requires explicit migration are - ;; added to the state for a posterior migration step. - (doseq [feature (-> (::bfc/features system) - (set/difference cfeat/no-migration-features) - (set/difference (:features file)))] - (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id'])) + (let [file (-> file + (assoc :id file-id') + (cond-> (and (= idx 0) (some? name)) + (assoc :name name)) + (assoc :project-id project-id) + (dissoc :thumbnails)) + file (bfc/process-file system file)] - (l/dbg :hint "create file" :id (str file-id') ::l/sync? true) - (bfc/save-file! system file ::db/return-keys false) + ;; All features that are enabled and requires explicit migration + ;; are added to the state for a posterior migration step. + (doseq [feature (-> (::bfc/features system) + (set/difference cfeat/no-migration-features) + (set/difference (:features file)))] + (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id'])) - file-id')))) + (l/dbg :hint "create file" :id (str file-id') ::l/sync? true) + (bfc/save-file! system file ::db/return-keys false) + + file-id'))))))) (defmethod read-section :v1/rels - [{:keys [::db/conn ::bfc/input ::bfc/timestamp]}] - (let [rels (read-obj! input) + [{:keys [::db/conn ::bfc/input ::bfc/timestamp] :as cfg}] + (let [rels (->> (read-obj! input) + (sort-by (fn [{:keys [file-id library-file-id]}] + [(str file-id) (str library-file-id)]))) ids (into #{} (-> bfc/*state* deref :files))] - ;; Insert all file relations (doseq [{:keys [library-file-id] :as rel} rels] (let [rel (-> rel (update :file-id bfc/lookup-index) (update :library-file-id bfc/lookup-index))] (if (contains? ids library-file-id) - (do - (l/dbg :hint "create file library link" - :file-id (:file-id rel) - :lib-id (:library-file-id rel) - ::l/sync? true) - (let [rel-params (dissoc rel :synced-at)] - (db/insert! conn :file-library-rel rel-params) - (bfc/upsert-file-library-sync! conn {:file-id (:file-id rel-params) - :library-file-id (:library-file-id rel-params) - :synced-at (or (:synced-at rel) - timestamp)}))) + (case (bfc/library-link-status cfg (:file-id rel) (:library-file-id rel)) + :self-link + (l/warn :hint "ignoring self-referencing library link" + :file-id (:file-id rel) + :lib-id (:library-file-id rel) + ::l/sync? true) + + :circular + (l/warn :hint "ignoring circular library link" + :file-id (:file-id rel) + :lib-id (:library-file-id rel) + ::l/sync? true) + + :ok + (do + (l/dbg :hint "create file library link" + :file-id (:file-id rel) + :lib-id (:library-file-id rel) + ::l/sync? true) + (let [rel-params (dissoc rel :synced-at)] + (db/insert! conn :file-library-rel rel-params) + (bfc/upsert-file-library-sync! conn {:file-id (:file-id rel-params) + :library-file-id (:library-file-id rel-params) + :synced-at (or (:synced-at rel) + timestamp)})))) (l/warn :hint "ignoring file library link" :file-id (:file-id rel) diff --git a/backend/src/app/binfile/v2.clj b/backend/src/app/binfile/v2.clj index b339505ef8..06bb22b7c8 100644 --- a/backend/src/app/binfile/v2.clj +++ b/backend/src/app/binfile/v2.clj @@ -319,6 +319,11 @@ (db/insert! conn :file-tagged-object-thumbnail thumbnail {::db/return-keys false}))) + ;; NOTE: v2 packages are only produced and imported through the + ;; developer REPL (`app.srepl.binfile`), never through the product, so + ;; this path keeps inserting the relations as they come: unlike v1 and + ;; v3 it does not skip self-referencing or circular links (see + ;; `bfc/library-link-status`). (doseq [rel (read-obj cfg :file-rels file-id)] (let [rel (-> rel (update :file-id bfc/lookup-index) diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 243cff1781..ee629ee86d 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -988,8 +988,7 @@ :old-id (str (:id item)) ::l/sync? true) - (db/insert! conn :file-media-object params - ::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))) + (db/insert! conn :file-media-object params))) (doseq [[index item] (d/enumerate thumbnails)] (let [media-id (bfc/lookup-index (:media-id item)) @@ -1038,26 +1037,49 @@ file-id'))) (defn- import-file-relations + "Import library relations from the manifest, ignoring relations whose + endpoints are not both part of the package. Of the rest, skip edges that + would close a dependency cycle or link a file to itself (same check as + `link-file-to-library`, see `bfc/library-link-status`). Relations are + processed in sorted manifest order and the first edge wins: an edge + that would close a cycle with previously imported relations is skipped + and logged." [{:keys [::db/conn ::manifest ::bfc/timestamp] :as cfg}] - (let [relations (:relations manifest) + (let [relations (sort-by (fn [[file-id libr-id]] [(str file-id) (str libr-id)]) + (:relations manifest)) file-ids (into #{} (map :id) (:files manifest))] (doseq [[index [file-id libr-id]] (d/enumerate relations)] (tap-progress cfg {:section :relations - :current (inc index) :total (count relations)}) + :current (inc index) + :total (count relations)}) - (if (and (not= file-id libr-id) - (contains? file-ids file-id) + (if (and (contains? file-ids file-id) (contains? file-ids libr-id)) (let [file-id (bfc/lookup-index file-id) libr-id (bfc/lookup-index libr-id)] - (l/dbg :hint "create file library link" - :file-id (str file-id) - :lib-id (str libr-id) - ::l/sync? true) - (let [rel-params {:file-id file-id - :library-file-id libr-id}] - (db/insert! conn :file-library-rel rel-params) - (bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp)))) + (case (bfc/library-link-status cfg file-id libr-id) + :self-link + (l/warn :hint "ignoring self-referencing library link" + :file-id (str file-id) + :lib-id (str libr-id) + ::l/sync? true) + + :circular + (l/warn :hint "ignoring circular library link" + :file-id (str file-id) + :lib-id (str libr-id) + ::l/sync? true) + + :ok + (do + (l/dbg :hint "create file library link" + :file-id (str file-id) + :lib-id (str libr-id) + ::l/sync? true) + (let [rel-params {:file-id file-id + :library-file-id libr-id}] + (db/insert! conn :file-library-rel rel-params) + (bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp)))))) (l/warn :hint "ignoring invalid file library relation" :file-id (str file-id) :lib-id (str libr-id)))))) @@ -1282,70 +1304,6 @@ {:file-ids file-ids :resolution resolution}))) -(defn- invalidate-thumbnails - [cfg file-id] - (let [storage (sto/resolve cfg ::db/reuse-conn true) - - sql-1 - (str "update file_tagged_object_thumbnail " - " set deleted_at = now() " - " where file_id=? returning media_id") - - sql-2 - (str "update file_thumbnail " - " set deleted_at = now() " - " where file_id=? returning media_id")] - - (run! #(sto/touch-object! storage %) - (sequence - (keep :media-id) - (concat - (db/exec! cfg [sql-1 file-id]) - (db/exec! cfg [sql-2 file-id])))))) - -(def ^:private sql:is-branch-file - "SELECT f.is_branch - FROM file AS f - WHERE f.id = ?") - -(defn- import-file-and-overwrite* - [{:keys [::manifest ::bfc/file-id] :as cfg}] - - (when (not= 1 (count (:files manifest))) - (ex/raise :type :validation - :code :invalid-condition - :hint "unable to perform in-place update with binfile containing more than 1 file" - :manifest manifest)) - - ;; A branch file's data payload is derived from the branch base and - ;; the operation log; overwriting it would persist a `file_data` row - ;; no read path consults. - (when (:is-branch (db/exec-one! cfg [sql:is-branch-file file-id])) - (ex/raise :type :validation - :code :branch-file-cant-be-overwritten - :hint "branch file data is derived from the branch base and the operation log and can't be overwritten by an import" - :file-id file-id)) - - (bfc/configure-import-transaction cfg) - - (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true) - file (first (get manifest :files)) - cfg (assoc cfg ::bfc/overwrite true)] - - (vswap! bfc/*state* update :index assoc (:id file) file-id) - - (binding [bfc/*options* cfg - bfc/*reference-file* ref-file] - - (import-storage-objects cfg) - (import-file cfg file {:current 1 :total 1}) - - (invalidate-thumbnails cfg file-id) - (bfm/apply-pending-migrations! cfg) - - {:file-ids [file-id] - :resolution {}}))) - (defn- import-files [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}] @@ -1396,9 +1354,7 @@ (tap-progress cfg {:section :manifest}) (binding [bfc/*state* (volatile! {:media [] :index {}})] - (if (::bfc/file-id cfg) - (db/tx-run! cfg import-file-and-overwrite*) - (db/tx-run! cfg import-files*))))) + (db/tx-run! cfg import-files*)))) ;; --- PUBLIC API @@ -1467,6 +1423,12 @@ (defn import-files! [{:keys [::bfc/input] :as cfg}] + (when (::bfc/file-id cfg) + (ex/raise :type :validation + :code :not-supported + :hint "in-place import is not supported" + :file-id (::bfc/file-id cfg))) + (assert (and (uuid? (::bfc/profile-id cfg)) (uuid? (::bfc/project-id cfg))) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index e2a969c6c1..e1d00c902a 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -10,6 +10,7 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.flags :as flags] + [app.common.logging :as l] [app.common.schema :as sm] [app.common.time :as ct] [app.common.uri :as u] @@ -168,6 +169,10 @@ [:http-server-io-threads {:optional true} ::sm/int] [:http-server-max-worker-threads {:optional true} ::sm/int] + ;; How often a websocket subscription re-checks that its access still + ;; holds. The default lives in `app.http.websocket`, not here. + [:subscription-revalidation-interval {:optional true} ::ct/duration] + ;; Explicit CORS allowlist used when the :cors flag is enabled. ;; Configured via PENPOT_ALLOWED_ORIGINS as a comma/whitespace ;; separated list of origins (e.g. "https://plugins.example.com"). @@ -431,6 +436,22 @@ (def explain-config (sm/explainer schema:config)) +(defn- normalize-file-data-backend + "The file data moved to `file_data` (migration `0157-migrate-legacy-file-data`), so + `file.data` and `file_change.data` are read-only leftovers and the `legacy-db` + back end is read-only: a write against it fails on every save. Accepting it would + start a healthy instance that cannot save a single file, so it reads as `db`, + which serves the legacy rows too. Warn instead of aborting: the operator may + still be halfway through the migration." + [config] + (if (= "legacy-db" (c/get config :file-data-backend)) + (do + (l/warn :hint "file-data-backend=legacy-db is read-only, using db" + :file-data-backend "legacy-db" + :replacement "db") + (assoc config :file-data-backend "db")) + config)) + (defn read-config "Reads the configuration from environment variables and decodes all known values." @@ -438,7 +459,8 @@ (->> (read-env prefix) (merge default) (normalize-tenant) - (decode-config))) + (decode-config) + (normalize-file-data-backend))) (def version (v/parse (or (some-> (io/resource "version.txt") diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index c9998b49b8..bfce3b2274 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -70,6 +70,7 @@ (def defaults {::name :main + ::min-size 0 ::max-size 60 ::connection-timeout 30000 ::validation-timeout 10000 @@ -92,8 +93,7 @@ :read-only read-only :credentials (and (contains? cfg ::username) (contains? cfg ::password)) - :min-size (or (::min-size cfg) - (::max-size cfg)) + :min-size (::min-size cfg) :max-size (::max-size cfg)) (create-pool cfg)))) @@ -108,15 +108,16 @@ (def initsql (str "SET statement_timeout = 300000;\n" - "SET idle_in_transaction_session_timeout = 300000;")) + "SET idle_in_transaction_session_timeout = 300000;\n" + ;; lets the planner resolve a join between two partitioned tables + ;; pairwise, when both are partitioned with the same bounds + "SET enable_partitionwise_join = on;")) (defn- create-datasource-config [{:keys [::uri] :as cfg}] ;; (app.common.pprint/pprint cfg) - (let [config (HikariConfig.) - max-size (::max-size cfg) - min-size (or (::min-size cfg) max-size)] + (let [config (HikariConfig.)] (doto config (.setJdbcUrl (str "jdbc:" uri)) (.setPoolName (d/name (::name cfg))) @@ -126,8 +127,8 @@ (.setValidationTimeout (::validation-timeout cfg)) (.setIdleTimeout (::idle-timeout cfg)) (.setMaxLifetime (::max-lifetime cfg)) - (.setMinimumIdle min-size) - (.setMaximumPoolSize max-size) + (.setMinimumIdle (::min-size cfg)) + (.setMaximumPoolSize (::max-size cfg)) (.setConnectionInitSql initsql) (.setInitializationFailTimeout -1)) diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index 0b14d34bf5..2279e2e5f7 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -105,11 +105,16 @@ (dissoc file :legacy-data)) (defmethod resolve-file-data "storage" - [cfg {:keys [metadata] :as file}] + [cfg {:keys [storage-object-id] :as file}] (let [storage (sto/resolve cfg ::db/reuse-conn true) - ref-id (:storage-ref-id metadata) - data (->> (sto/get-object storage ref-id) - (sto/get-object-bytes storage))] + sobject (or (sto/get-object storage storage-object-id) + (ex/raise :type :internal + :code :missing-storage-object + :hint "file data row has no readable storage object" + :file-id (or (:file-id file) (:id file)) + :id (:id file) + :storage-object-id storage-object-id)) + data (->> (sto/get-object-bytes storage sobject))] (-> file (assoc :data data) (dissoc :legacy-data)))) @@ -122,8 +127,8 @@ (def ^:private sql:insert-file-data "INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at, - type, backend, metadata, data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") + type, backend, metadata, data, storage_object_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") (def ^:private sql:upsert-file-data (str sql:insert-file-data @@ -132,105 +137,103 @@ deleted_at=?, backend=?, metadata=?, - data=?")) + data=?, + storage_object_id=? + WHERE file_data.storage_object_id IS NOT DISTINCT FROM ?")) (defn- upsert-in-database - [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata]}] + [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata storage-object-id]} expected-storage-object-id] (let [created-at (or created-at (ct/now)) - metadata (some-> metadata db/json) - modified-at (or modified-at created-at)] + ;; The metadata pointer is dead (the column is the only source of + ;; truth). Clean it here, the single point where every branch + ;; serializes the metadata, so no write can resurrect it. + metadata (some-> metadata (dissoc :storage-ref-id) (db/json)) + modified-at (or modified-at created-at) - (db/exec-one! cfg [sql:upsert-file-data - file-id id - created-at - modified-at - deleted-at - type - backend - metadata - data - modified-at - deleted-at - backend - metadata - data]))) + result + (db/exec-one! cfg [sql:upsert-file-data + file-id id + created-at + modified-at + deleted-at + type + backend + metadata + data + storage-object-id + modified-at + deleted-at + backend + metadata + data + storage-object-id + expected-storage-object-id])] + + ;; A zero count means the row exists with another pointer: the caller + ;; read it before someone else replaced it. Every writer holds the `file` + ;; row lock, so this is a protocol violation, not a race to retry. + (when-not (pos? (db/get-update-count result)) + (ex/raise :type :internal + :code :stale-file-data-pointer + :hint "the file data row changed since it was read" + :file-id file-id + :id id)) + + result)) (defn- handle-persistence - [cfg {:keys [type backend id file-id data] :as params}] - - (cond - (= backend "storage") - ;; IMPORTANT: we strongly do not reuse the main connection that can - ;; run inside a transaction because the storage upload process can - ;; fail in the middle of uploading and leave garbage on the underlying - ;; backend, if we participate in the main transaction and it aborts - ;; we will lose all registry of the pending to reconcile blobs - ;; what the storage subsystem registers in other parallel - ;; transaction - (let [storage (sto/resolve cfg) - content (sto/content data) - sobject (sto/put-object! storage - {::sto/content content - ::sto/touch true - :bucket "file-data" - :content-type "application/octet-stream" - :file-id file-id - :id id}) - metadata (-> (:metadata params) - (assoc :storage-ref-id (:id sobject))) - params (-> params - (assoc :metadata metadata) - (assoc :data nil))] - (upsert-in-database cfg params)) - - (= backend "db") - (let [metadata (dissoc (:metadata params) :storage-ref-id) - params (assoc params :metadata metadata)] - (upsert-in-database cfg params)) - - (= backend "legacy-db") + [cfg {:keys [backend id file-id data] :as params}] + (let [expected (:storage-object-id params)] (cond - (= type "main") - (do - (db/delete! cfg :file-data - {:id id :file-id file-id :type "main"} - {::db/return-keys false}) - (db/update! cfg :file - {:data data} - {:id file-id} - {::db/return-keys false})) + (= backend "storage") + ;; IMPORTANT: we strongly do not reuse the main connection that can + ;; run inside a transaction because the storage upload process can + ;; fail in the middle of uploading and leave garbage on the underlying + ;; backend, if we participate in the main transaction and it aborts + ;; we will lose all registry of the pending to reconcile blobs + ;; what the storage subsystem registers in other parallel + ;; transaction + (let [storage (sto/resolve cfg) + content (sto/content data) + sobject (sto/put-object! storage + {::sto/content content + ::sto/touch true + :bucket "file-data" + :content-type "application/octet-stream" + :file-id file-id + :id id}) + params (-> params + (assoc :storage-object-id (:id sobject)) + (assoc :data nil))] + {:result (upsert-in-database cfg params expected) + :storage-object-id (:id sobject)}) - (= type "snapshot") - (do - (db/delete! cfg :file-data - {:id id :file-id file-id :type "snapshot"} - {::db/return-keys false}) - (db/update! cfg :file-change - {:data data} - {:file-id file-id :id id} - {::db/return-keys false})) - - (= type "fragment") - (upsert-in-database cfg - (-> (dissoc params :metadata) - (assoc :backend "db"))) + (= backend "db") + {:result (upsert-in-database cfg (assoc params :storage-object-id nil) expected) + :storage-object-id nil} :else - (throw (RuntimeException. "not implemented"))) + (throw (IllegalArgumentException. + (str "backend '" backend "' not supported")))))) - :else - (throw (IllegalArgumentException. - (str "backend '" backend "' not supported"))))) - -(defn process-metadata - [cfg metadata] - (when-let [storage-id (:storage-ref-id metadata)] - (let [storage (sto/resolve cfg ::db/reuse-conn true)] - (sto/touch-object! storage storage-id)))) +(defn touch-storage-object + "Mark a storage object as touched so the storage GC can reclaim it once + nothing references it anymore. Used when a file data row replaces or + deletes the object it pointed at. Expects a non-nil object id; callers + guard with `some->`/`some->>`." + [cfg storage-object-id] + (-> (sto/resolve cfg ::db/reuse-conn true) + (sto/touch-object! storage-object-id))) (defn- default-backend [backend] - (or backend (cf/get :file-data-backend))) + (let [backend (or backend (cf/get :file-data-backend))] + (when (= "legacy-db" backend) + (ex/raise :type :validation + :code :legacy-backend-read-only + :hint "the legacy-db file data backend is read-only; set the file data backend to db or storage" + :backend backend)) + backend)) (defn decode-metadata [metadata] @@ -243,8 +246,12 @@ [:id ::sm/uuid] [:type [:enum "main" "snapshot" "fragment"]] [:file-id ::sm/uuid] - [:backend {:optional true} [:enum "db" "legacy-db" "storage"]] + [:backend {:optional true} [:enum "db" "storage"]] [:metadata {:optional true} ctf/schema:file-metadata] + ;; The storage object the persisted row pointed at before this write; it is + ;; the object to touch when the row replaces it. Required so a new caller + ;; must decide it explicitly (nil for a new row) instead of leaking it. + [:storage-object-id [:maybe ::sm/uuid]] [:data {:optional true} bytes?] [:created-at {:optional true} ::ct/inst] [:modified-at {:optional true} [:maybe ::ct/inst]] @@ -254,17 +261,28 @@ (sm/check-fn schema:update-params :hint "invalid params received for update")) (defn upsert! - "Create or update file data" + "Create or update a file data row. + + `:storage-object-id` is the storage object the persisted row pointed at + before this write (nil for a new row or a row that had none), not the + object the new content will live in: the write resolves that itself. The + previous object is marked as touched before the row is replaced, so the + storage GC can reclaim the blob once nothing references it. + + The function does not query the row: the caller must hand the pointer it + read. Returns a map with `:written` (whether a row was written) and the + `:storage-object-id` of the object this write created, nil for the `db` + and `legacy-db` backends." [cfg params & {:as opts}] (let [params (-> (check-update-params params) (update :backend default-backend))] - (some->> (:metadata params) - (process-metadata cfg)) + (some->> (:storage-object-id params) + (touch-storage-object cfg)) - (-> (handle-persistence cfg params) - (db/get-update-count) - (pos?)))) + (let [{:keys [result storage-object-id]} (handle-persistence cfg params)] + {:written (pos? (db/get-update-count result)) + :storage-object-id storage-object-id}))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; POINTER-MAP @@ -278,7 +296,7 @@ ;; previous pass marked must still load until `objects-gc` purges it. (let [fragment (some-> (db/get* cfg :file-data {:id id :file-id file-id :type "fragment"} - {::sql/columns [:data :backend :id :metadata] + {::sql/columns [:data :backend :file_id :id :metadata :storage_object_id] ::db/remove-deleted false}) (update :metadata decode-metadata))] @@ -305,9 +323,12 @@ (when (pmap/modified? item) (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id)) (let [content (-> item deref blob/encode)] + ;; A modified pointer has a fresh id, so the fragment row is always a + ;; new insert: there is no previous storage object to touch. (upsert! cfg {:id id :file-id file-id :type "fragment" + :storage-object-id nil :data content}))))) (defmacro with-pointers diff --git a/backend/src/app/features/file_changes.clj b/backend/src/app/features/file_changes.clj new file mode 100644 index 0000000000..f01b7bfe6c --- /dev/null +++ b/backend/src/app/features/file_changes.clj @@ -0,0 +1,21 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.features.file-changes + "The kinds of the rows of the `file_change` table. + + A snapshot is the state of a file at a point: it has a label and its + data lives in `file_data` (old `legacy-db` payloads live in + `file_change.data`). A revision is the delta of a save: it has + no label and it holds the changes.") + +(def snapshot-kind + "The kind of a snapshot row." + 1) + +(def revision-kind + "The kind of a revision row." + 2) diff --git a/backend/src/app/features/file_snapshots.clj b/backend/src/app/features/file_snapshots.clj index 4460c84fd6..78897dcdf2 100644 --- a/backend/src/app/features/file_snapshots.clj +++ b/backend/src/app/features/file_snapshots.clj @@ -19,6 +19,7 @@ [app.db :as db] [app.db.sql :as-alias sql] [app.features.fdata :as fdata] + [app.features.file-changes :as fchanges] [app.jobs :as jobs] [app.storage :as sto] [app.util.blob :as blob] @@ -41,7 +42,8 @@ c.data AS legacy_data, fd.data AS data, coalesce(fd.backend, 'legacy-db') AS backend, - fd.metadata AS metadata + fd.metadata AS metadata, + fd.storage_object_id AS storage_object_id FROM file_change AS c LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id AND fd.id = c.id @@ -64,68 +66,88 @@ (d/update-when :features db/decode-pgarray #{})))) (def ^:private sql:get-minimal-file + ;; `fd.storage_object_id` is load-bearing: `restore!` hands this map to + ;; `update-file!` so the previous main object is touched before the row is + ;; replaced. Removing it would leave the old blob unreclaimed with no error. "SELECT f.id, f.revn, f.modified_at, f.deleted_at, f.is_branch, fd.backend AS backend, - fd.metadata AS metadata + fd.metadata AS metadata, + fd.storage_object_id AS storage_object_id FROM file AS f LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id) WHERE f.id = ?") +(def ^:private sql:snapshot-columns + "c.id, + c.label, + c.revn, + c.created_at, + c.modified_at, + c.deleted_at, + c.profile_id, + c.created_by, + c.locked_by, + c.features, + c.metadata, + c.migrations, + c.version, + c.file_id") + +(def ^:private sql:snapshot-visible + "AND CASE WHEN c.created_by = 'user' + THEN c.deleted_at IS NULL + WHEN c.created_by = 'system' + THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz + END") + (def ^:private sql:get-snapshot-without-data (str "WITH snapshots AS (" sql:snapshots ")" - "SELECT c.id, - c.label, - c.revn, - c.created_at, - c.modified_at, - c.deleted_at, - c.profile_id, - c.created_by, - c.locked_by, - c.features, - c.metadata, - c.migrations, - c.version, - c.file_id - FROM snapshots AS c - WHERE c.id = ? - AND CASE WHEN c.created_by = 'user' - THEN c.deleted_at IS NULL - WHEN c.created_by = 'system' - THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz - END")) + "SELECT " sql:snapshot-columns + " FROM snapshots AS c + WHERE c.file_id = ? + AND c.id = ? " sql:snapshot-visible)) (defn get-minimal-snapshot - [cfg snapshot-id] + "Get the minimal snapshot info (without the data blob). The query filters + by `file_id`, which lets the planner prune the partitions of `file_change` + when it chooses an index scan. Raises `:snapshot-not-found` when there is + no matching snapshot." + [cfg file-id snapshot-id] (let [now (ct/now)] - (-> (db/get-with-sql cfg [sql:get-snapshot-without-data snapshot-id now] - {::db/remove-deleted false}) - (decode-snapshot)))) + (or (-> (db/get-with-sql cfg [sql:get-snapshot-without-data file-id snapshot-id now] + {::db/remove-deleted false + ::db/throw-if-not-exists false}) + (decode-snapshot)) + (ex/raise :type :not-found + :code :snapshot-not-found + :hint "unable to find snapshot with the provided id" + :snapshot-id snapshot-id + :file-id file-id)))) (def ^:private sql:get-snapshot (str sql:snapshots " AND c.file_id = ? - AND c.id = ? - AND CASE WHEN c.created_by = 'user' - THEN (c.deleted_at IS NULL) - WHEN c.created_by = 'system' - THEN (c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz) - END")) + AND c.id = ? " + sql:snapshot-visible)) (defn get-snapshot "Get a fully decoded snapshot for read-only preview or restoration. - Returns the snapshot map with decoded :data field." + Returns the snapshot map with decoded :data field, or nil when the file has + no snapshot with that id. The internal storage pointer of the row is not + part of the result." [cfg file-id snapshot-id] (let [now (ct/now)] - (->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now] - {::db/remove-deleted false}) - (decode-snapshot) - (fdata/resolve-file-data cfg) - (fdata/decode-file-data cfg)))) + (some->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now] + {::db/remove-deleted false + ::db/throw-if-not-exists false}) + (decode-snapshot) + (fdata/resolve-file-data cfg) + (fdata/decode-file-data cfg) + (bfc/file->response)))) (def ^:private sql:get-visible-snapshots (str "WITH " @@ -268,6 +290,7 @@ (-> snapshot (update :features into-array) (update :migrations into-array) + (assoc :kind fchanges/snapshot-kind) (assoc :updated-at created-at) (assoc :profile-id profile-id) (assoc :session-id session-id) @@ -278,6 +301,7 @@ {:id snapshot-id :file-id (:id file) :type "snapshot" + :storage-object-id nil :data (blob/encode (:data file)) :created-at created-at :deleted-at deleted-at}) @@ -314,7 +338,10 @@ (-> (db/update! cfg :file-change {:label label :created-by "user" - :updated-at modified-at + ;; the partition runbook uses updated_at as the + ;; watermark of its delta passes: a rename has to move + ;; it, unlike file_data.modified_at above + :updated-at (ct/now) :deleted-at nil} {:file-id file-id :id id} @@ -360,7 +387,7 @@ (when-not snapshot (ex/raise :type :not-found :code :snapshot-not-found - :hint "unable to find snapshot with the provided label" + :hint "unable to find snapshot with the provided id" :snapshot-id snapshot-id :file-id file-id)) @@ -440,7 +467,8 @@ :file-id file-id :id id}}) (db/update! cfg :file-change - {:deleted-at deleted-at} + {:deleted-at deleted-at + :updated-at (ct/now)} {:id id :file-id file-id} {::db/return-keys false}) true) @@ -449,19 +477,25 @@ (str sql:snapshots " AND c.file_id = ?")) (defn lock-by! - [conn id profile-id] + [conn file-id id profile-id] + (assert (uuid? file-id) "missing file-id") + (assert (uuid? id) "missing id") (-> (db/update! conn :file-change - {:locked-by profile-id} - {:id id} + {:locked-by profile-id + :updated-at (ct/now)} + {:id id :file-id file-id} {::db/return-keys false}) (db/get-update-count) (pos?))) (defn unlock! - [conn id] + [conn file-id id] + (assert (uuid? file-id) "missing file-id") + (assert (uuid? id) "missing id") (-> (db/update! conn :file-change - {:locked-by nil} - {:id id} + {:locked-by nil + :updated-at (ct/now)} + {:id id :file-id file-id} {::db/return-keys false}) (db/get-update-count) (pos?))) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index d62520706f..768e358176 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -123,7 +123,8 @@ (defn- get-resolved-file [cfg file-id] - (bfc/get-file cfg file-id :migrate? false :decode? false)) + (-> (bfc/get-file cfg file-id :migrate? false :decode? false) + (bfc/file->response))) (defn prepare-download [file filename] @@ -372,7 +373,6 @@ :profile-id profile-id :project-id project-id) cfg (assoc cfg - ::bfc/overwrite false ::bfc/profile-id profile-id ::bfc/project-id project-id ::bfc/team-id (:id team) diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index 4e9ce3bd77..12fd7353b1 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -222,7 +222,8 @@ (defn invalidate-all "Delete all sessions for a given profile. Used when a profile is deleted - to ensure immediate access revocation across all devices." + or a password is recovered, to ensure immediate access revocation across + all devices." [cfg profile-id] (let [sql "delete from http_session_v2 where profile_id = ?"] (-> (db/exec-one! cfg [sql profile-id]) diff --git a/backend/src/app/http/websocket.clj b/backend/src/app/http/websocket.clj index 50bd64f115..597c115c1e 100644 --- a/backend/src/app/http/websocket.clj +++ b/backend/src/app/http/websocket.clj @@ -15,13 +15,16 @@ [app.common.schema :as sm] [app.common.time :as ct] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] + [app.db.sql :as sql] [app.http.session :as session] [app.metrics :as mtx] [app.msgbus :as mbus] [app.nitrate :as nitrate] [app.rpc.commands.files :as files] [app.rpc.commands.teams :as teams] + [app.rpc.notifications :as notifications] [app.util.websocket :as ws] [integrant.core :as ig] [promesa.exec.csp :as sp] @@ -37,31 +40,79 @@ ;; WEBSOCKET HOOKS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(def state (atom {})) +(def state + "Registry of the websocket connections held by this backend instance. + + Holds `:connections` (connection id -> connection data) and + `:by-profile` (profile id -> set of connection ids). Both live in the + same atom so a single `swap!` keeps them consistent, and so the + profile index never has to be rebuilt by scanning every connection. + + NOTE: the registry is local to one backend instance, unlike the + message bus, which is shared. Code that must reach connections + regardless of which instance owns them has to go through the bus." + (atom {:connections {} + :by-profile {}})) + +(defn register-connection + "Adds `wsp` to the registry under `id`, indexing it by its profile." + [id wsp] + (let [profile-id (::profile-id wsp)] + (swap! state + (fn [st] + (-> st + (update :connections assoc id wsp) + (cond-> profile-id + (update-in [:by-profile profile-id] + (fnil conj #{}) id))))))) + +(defn unregister-connection + "Removes the connection `id` from the registry and from the profile + index." + [id] + (swap! state + (fn [{:keys [connections] :as st}] + (let [owner (::profile-id (get connections id)) + indexed (disj (get-in st [:by-profile owner] #{}) id)] + (cond-> (assoc st :connections (dissoc connections id)) + (and owner (seq indexed)) + (update :by-profile assoc owner indexed) + (and owner (empty? indexed)) + (update :by-profile dissoc owner)))))) + +(defn get-connection + "Returns the connection data registered under `id`, or nil." + [id] + (get-in @state [:connections id])) + +(defn connections-for-profile + "Returns the set of connection ids currently held by `profile-id`." + [profile-id] + (get-in @state [:by-profile profile-id] #{})) ;; REPL HELPERS (defn repl-get-connections-for-file [file-id] - (->> (vals @state) + (->> (vals (:connections @state)) (filter #(= file-id (-> % ::ws/state deref ::file-subscription :file-id))) (map ::ws/id))) (defn repl-get-connections-for-team [team-id] - (->> (vals @state) + (->> (vals (:connections @state)) (filter #(= team-id (-> % ::ws/state deref ::team-subscription :team-id))) (map ::ws/id))) (defn repl-close-connection [id] - (when-let [{:keys [::ws/close-ch] :as wsp} (get @state id)] + (when-let [{:keys [::ws/close-ch]} (get-connection id)] (sp/put! close-ch [8899 "closed from server"]) (sp/close! close-ch))) (defn repl-get-connection-info [id] - (when-let [wsp (get @state id)] + (when-let [wsp (get-connection id)] (let [subs (some-> wsp ::ws/state deref)] {:id id :created-at (::created-at wsp) @@ -85,6 +136,160 @@ (pp/pprint))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; SUBSCRIPTIONS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A subscription is authorized once, when it is opened, and that +;; single decision is then trusted for as long as the connection +;; lives. Two things bound that: a mutation that changes access +;; announces it (see the revocation section below), and the relay +;; re-checks on an interval. + +(def default-revalidation-interval + "How long a subscription may rely on the authorization it was granted + with before its relay loop re-checks it. + + This is the safety net for what the announcement cannot cover: an + access change made outside a Penpot command. Nitrate transferring the + ownership of an organization is an HTTP call to another service that + fires no RPC here, and rows deleted straight over SQL fire nothing + either. Both are rare, so the interval is generous; the cost is one + permission query per subscription per interval." + (ct/duration {:minutes 5})) + +(defn revalidation-interval + [] + (cf/get :subscription-revalidation-interval default-revalidation-interval)) + +(defn- still-authorized? + "Runs the `check-access` permission check, reporting instead of + throwing when the lookup itself fails. + + A transient database error must not close a subscription that is still + legitimate, and the next tick tries again. This is one of two places + that fail open, deliberately: the announcement path (`watch-revocations` + below) also logs and moves on when a re-check throws. An interrupt is + not a lookup failure and is rethrown." + [check-access] + (try + (boolean (check-access)) + (catch InterruptedException cause + (throw cause)) + (catch Throwable cause + (l/error :hint "cannot re-check a websocket subscription" + :cause cause) + true))) + +(defn- file-readable? + "Non-throwing mirror of the `:subscribe-file` gates: the file must + still exist (a soft-deleted row grants nothing) and the profile must + read it. Same `sql:get-minimal-file` as the subscribe gate, with the + existence failure turned into a boolean — the fresh answer is the + decision." + [cfg profile-id file-id] + (and (some? (bfc/get-minimal-file cfg file-id + {::db/throw-if-not-exists false})) + (files/has-read-permissions? cfg profile-id file-id))) + +(defn- team-readable? + "Same shape for teams: the team must still exist (a soft-deleted row + grants nothing) and the profile must read it. `:subscribe-team` has + no existence gate of its own; the re-check adds one, matching the + file side. Selects only `:id` and `:deleted-at` — `db/get*` filters + soft-deleted rows through the `deleted_at` column, so it must be + in the projection." + [cfg profile-id team-id] + (and (some? (db/get* cfg :team {:id team-id} + {::sql/columns [:id :deleted-at]})) + (teams/has-read-permissions? cfg profile-id team-id))) + +(defn close-file-subscription + "Tears down the file subscription held by the connection `wsp`, if it + is subscribed to `file-id`. + + Announces the departure so the remaining participants drop the + presence of this session, closes the relay channel (which in turn + stops the `:subscribe-file` go-loop, because `take!` on a closed + channel returns nil) and removes the subscription from the bus. + + Does nothing when the connection is subscribed to a different file, + so it is safe to call for a subscription that is already gone." + [{:keys [::mbus/msgbus]} {:keys [::ws/state ::session-id ::profile-id]} file-id] + (let [subs (::file-subscription @state)] + (when (= (:file-id subs) file-id) + (mbus/pub! msgbus + :topic file-id + :message {:type :leave-file + :file-id file-id + :session-id session-id + :profile-id profile-id}) + (let [ch (:channel subs)] + (sp/close! ch) + (mbus/purge! msgbus [ch]) + (swap! state dissoc ::file-subscription))))) + +(defn close-team-subscription + "Tears down the team subscription held by the connection `wsp`, if it + is subscribed to `team-id`. + + Closing the channel is what stops the relay, so no further team or + organization traffic reaches this connection. + + Does nothing when the connection is subscribed to a different team, + so it is safe to call for a subscription that is already gone." + [{:keys [::mbus/msgbus]} {:keys [::ws/state]} team-id] + (let [subs (::team-subscription @state)] + (when (= (:team-id subs) team-id) + (let [ch (:channel subs)] + (sp/close! ch) + (mbus/purge! msgbus [ch]) + (swap! state dissoc ::team-subscription))))) + + +(defn- start-relay + "Forwards `channel` into the client output channel of `wsp`, and keeps + a re-check of `check-access` running on a fixed cadence. When access no + longer holds, `close-fn` tears the subscription down, which closes the + channel and ends the loop. + + The loop waits on the subscription channel and a one-shot tick channel + together, with priority on the tick: when both hold something, the tick + wins, so a steady stream of messages can never push the re-check away. + A single tick channel covers a whole cadence and message turns reuse + it, so busy files don't churn timers; the cadence holds under load + because nothing moves the tick. Closing the channel ends the loop + either way, because a take on a closed channel is always ready + with nil. + + `on-forward` runs for every forwarded message, which is how the file + relay announces presence." + [{:keys [::ws/output-ch] :as wsp} channel check-access close-fn on-forward] + (let [interval-ms (inst-ms (revalidation-interval))] + (sp/go-loop [tick-ch (sp/timeout-chan interval-ms)] + (let [[message port] (sp/alts! [tick-ch channel] :priority true)] + (cond + (identical? port tick-ch) + (do + (when-not (still-authorized? check-access) + (l/info :hint + "closing websocket subscription on re-check" + :profile-id (::profile-id wsp)) + (close-fn)) + (when-not (sp/closed? channel) + (recur (sp/timeout-chan interval-ms)))) + + (nil? message) + ;; The channel was closed, which means the subscription is gone + ;; and the relay ends. + nil + + :else + (do + (sp/put! output-ch message) + (when on-forward (on-forward message)) + (recur tick-ch))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; WEBSOCKET HANDLER ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -151,10 +356,10 @@ nil))) (defmethod handle-message :subscribe-team - [cfg {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [team-id] :as params}] + [cfg {:keys [::ws/id ::ws/state ::session-id ::profile-id] :as wsp} {:keys [team-id] :as params}] (l/trace :fn "handle-message" :event "subscribe-team" :team-id team-id :conn-id id) (teams/check-read-permissions! cfg profile-id team-id) - (let [prev-subs (get @state ::team-subscription) + (let [prev-subs (::team-subscription @state) organization-id (get-team-organization-id cfg team-id) ;; Resolved server-side so a client only hears its readable team's org topics (cond-> [team-id] @@ -163,23 +368,29 @@ channel (sp/chan :buf (sp/dropping-buffer 64) :xf (remove #(= (:session-id %) session-id)))] - (sp/pipe channel output-ch false) - (mbus/sub! (::mbus/msgbus cfg) :topics topics :chan channel) - + ;; The team topic also carries library change diffs, so a + ;; subscription that outlives access to the team leaks content, not + ;; just presence. (let [subs {:team-id team-id :organization-id organization-id :channel channel :topic team-id}] (swap! state assoc ::team-subscription subs)) + (start-relay wsp channel + #(team-readable? cfg profile-id team-id) + #(close-team-subscription cfg wsp team-id) + nil) + + (mbus/sub! (::mbus/msgbus cfg) :topics topics :chan channel) + ;; Close previous subscription if exists (when-let [ch (:channel prev-subs)] (sp/close! ch) (mbus/purge! (::mbus/msgbus cfg) [ch])))) - (defmethod handle-message :subscribe-file - [cfg {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [file-id] :as params}] + [cfg {:keys [::ws/id ::ws/state ::session-id ::profile-id] :as wsp} {:keys [file-id] :as params}] (l/trace :fn "handle-message" :event "subscribe-file" :file-id file-id :conn-id id) (bfc/check-file-exists cfg file-id) (files/check-read-permissions! cfg profile-id file-id) @@ -190,51 +401,40 @@ (let [subs {:file-id file-id :channel fch :topic file-id}] (swap! state assoc ::file-subscription subs)) + (start-relay wsp fch + #(file-readable? cfg profile-id file-id) + #(close-file-subscription cfg wsp file-id) + (fn [{:keys [type]}] + (when (or (= :join-file type) + (= :leave-file type) + (= :disconnect type)) + (mbus/pub! (::mbus/msgbus cfg) + :topic file-id + :message {:type :presence + :file-id file-id + :session-id session-id + :profile-id profile-id})))) + ;; Close previous subscription if exists (when-let [ch (:channel psub)] (sp/close! ch) (mbus/purge! (::mbus/msgbus cfg) [ch])) - (sp/go-loop [] - (when-let [{:keys [type] :as message} (sp/take! fch)] - (sp/put! output-ch message) - (when (or (= :join-file type) - (= :leave-file type) - (= :disconnect type)) - (let [message {:type :presence - :file-id file-id - :session-id session-id - :profile-id profile-id}] - (mbus/pub! (::mbus/msgbus cfg) - :topic file-id - :message message))) - (recur))) - ;; Subscribe to file topic (mbus/sub! (::mbus/msgbus cfg) :topic file-id :chan fch) ;; Notifify the rest of participants of the new connection. - (let [message {:type :join-file - :file-id file-id - :session-id session-id - :profile-id profile-id}] - (mbus/pub! (::mbus/msgbus cfg) :topic file-id :message message)))) + (mbus/pub! (::mbus/msgbus cfg) + :topic file-id + :message {:type :join-file + :file-id file-id + :session-id session-id + :profile-id profile-id}))) (defmethod handle-message :unsubscribe-file - [{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::session-id ::profile-id]} {:keys [file-id] :as params}] + [cfg {:keys [::ws/id] :as wsp} {:keys [file-id] :as params}] (l/trace :fn "handle-message" :event "unsubscribe-file" :file-id file-id :conn-id id) - - (let [subs (::file-subscription @state) - message {:type :leave-file - :file-id file-id - :session-id session-id - :profile-id profile-id}] - - (when (= (:file-id subs) file-id) - (mbus/pub! msgbus :topic file-id :message message) - (let [ch (:channel subs)] - (sp/close! ch) - (mbus/purge! msgbus [ch]))))) + (close-file-subscription cfg wsp file-id)) (defmethod handle-message :keepalive [_ _ _] @@ -265,6 +465,110 @@ :message message :conn-id id)) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; SUBSCRIPTION REVOCATION +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A subscription is authorized once, when it is opened, and that single +;; decision is then trusted for as long as the connection lives. The +;; mutation that changes access announces it on +;; `app.rpc.notifications/internal-revocation-topic`, and every backend +;; re-verifies the subscriptions it owns for that profile. +;; +;; The announcement travels over the message bus precisely because this +;; registry is local: the RPC that revokes access runs on whichever +;; instance received the request, which is usually not the one holding +;; the socket. + +(defn revalidate-profile-subscriptions + "Re-checks every subscription held by the connections of `profile-id` + and closes the ones that are no longer authorized. + + The event is the trigger, the fresh check is the decision. + `file-readable?` and `team-readable?` mirror the gates of + `:subscribe-file` and extend them to teams: resource exists + (a soft-deleted row grants nothing) and the profile reads it. + The organization-owner fallback keeps a downgrade to viewer and a + non-member owner from being mistaken for revocations — but not a + soft-deleted resource. + + Returns the number of subscriptions it closed." + [cfg profile-id] + (let [closed (volatile! 0)] + (doseq [id (connections-for-profile profile-id)] + (when-let [wsp (get-connection id)] + (let [subs (some-> wsp ::ws/state deref) + fsub (get subs ::file-subscription) + tsub (get subs ::team-subscription)] + (when (and fsub + (not (file-readable? cfg profile-id + (:file-id fsub)))) + (close-file-subscription cfg wsp (:file-id fsub)) + (vswap! closed inc)) + (when (and tsub + (not (team-readable? cfg profile-id + (:team-id tsub)))) + (close-team-subscription cfg wsp (:team-id tsub)) + (vswap! closed inc))))) + @closed)) + +(defn- watch-revocations + "Consumes revocation events and applies them to the connections this + instance owns. Returns the subscription channel, which `ig/halt-key!` + closes to stop the loop." + [{:keys [::mbus/msgbus] :as cfg}] + (let [ch (sp/chan :buf (sp/dropping-buffer 64))] + (mbus/sub! msgbus + :topic notifications/internal-revocation-topic + :chan ch) + (sp/go-loop [] + (when-let [{:keys [type profile-id team-id]} (sp/take! ch)] + (try + (cond + (= :profile-permissions-changed type) + (when (pos? (revalidate-profile-subscriptions cfg profile-id)) + (l/debug :hint "revoked websocket subscriptions" + :profile-id profile-id)) + + (= :team-permissions-changed type) + (let [revoked (->> (teams/get-team-members cfg team-id) + (map :id) + (map #(revalidate-profile-subscriptions cfg %)) + (reduce + 0))] + (when (pos? revoked) + (l/debug :hint "revoked websocket subscriptions" + :team-id team-id + :count revoked)))) + (catch Throwable cause + (l/error :hint "cannot revalidate websocket subscriptions" + :profile-id profile-id + :team-id team-id + :cause cause))) + (recur))) + ch)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; INTEGRANT +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:private schema:revocation-watcher + [:map + ::mbus/msgbus + ::db/pool + [:app.nitrate/client {:optional true} [:maybe :map]]]) + +(defmethod ig/assert-key ::revocation-watcher + [_ params] + (assert (sm/valid? schema:revocation-watcher params))) + +(defmethod ig/init-key ::revocation-watcher + [_ cfg] + (watch-revocations cfg)) + +(defmethod ig/halt-key! ::revocation-watcher + [_ ch] + (sp/close! ch)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; HTTP HANDLER ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -273,7 +577,7 @@ [{:keys [::mtx/metrics]} {:keys [::ws/id] :as wsp}] (let [created-at (ct/now)] (l/trace :fn "on-connect" :conn-id id) - (swap! state assoc id wsp) + (register-connection id wsp) (mtx/run! metrics :id :websocket-active-connections :inc 1) @@ -281,7 +585,7 @@ (assoc wsp ::ws/on-disconnect (fn [] (l/trace :fn "on-disconnect" :conn-id id) - (swap! state dissoc id) + (unregister-connection id) (mtx/run! metrics :id :websocket-active-connections :dec 1) (mtx/run! metrics :id :websocket-session-timing diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 71ddafd9bf..26b6552e92 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -571,6 +571,11 @@ ::session/manager (ig/ref ::session/manager) :app.nitrate/client (ig/ref :app.nitrate/client)} + ::http.ws/revocation-watcher + {::db/pool (ig/ref ::db/pool) + ::mbus/msgbus (ig/ref ::mbus/msgbus) + :app.nitrate/client (ig/ref :app.nitrate/client)} + :app.http.assets/routes {::http.assets/path (cf/get :assets-path) ::http.assets/cache-max-age (ct/duration {:hours 24}) diff --git a/backend/src/app/media/local.clj b/backend/src/app/media/local.clj index f3811e83aa..6cd9b0c866 100644 --- a/backend/src/app/media/local.clj +++ b/backend/src/app/media/local.clj @@ -241,8 +241,12 @@ foutput (fs/path (str finput ".ttf"))] (try (io/write* finput data) + ;; CID-keyed fonts (common in CJK) load as several sub-fonts and + ;; Generate only writes the active one unless they are merged. (let [res (exec-font! system ["fontforge" "-lang=ff" "-c" - (str/fmt "Open('%s'); Generate('%s')" + (str/fmt (str "Open('%s'); " + "if ($cidfontname != \"\"); CIDFlatten(); endif; " + "Generate('%s')") (str finput) (str foutput))])] (when (zero? (:exit res)) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 0f68a807f9..95a2ebfb92 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -541,6 +541,12 @@ {:name "0158-del-forbidden-file-library-rels" :fn (mg/resource "app/migrations/sql/0158-del-forbidden-file-library-rels.sql")} + {:name "0158-partition-file-change" + :fn (mg/resource "app/migrations/sql/0158-partition-file-change.sql")} + + {:name "0159-add-file-data-storage-object-id" + :fn (mg/resource "app/migrations/sql/0159-add-file-data-storage-object-id.sql")} + {:name "0800-add-file-branch-table" :fn (mg/resource "app/migrations/sql/0800-add-file-branch-table.sql")} diff --git a/backend/src/app/migrations/sql/0158-partition-file-change.sql b/backend/src/app/migrations/sql/0158-partition-file-change.sql new file mode 100644 index 0000000000..e8abe5523c --- /dev/null +++ b/backend/src/app/migrations/sql/0158-partition-file-change.sql @@ -0,0 +1,258 @@ +-- Partition `file_change` by `file_id`. +-- +-- The table grows with every file change and its TOAST holds the blobs of +-- the changes, so it is the biggest relation of the database. Partitioning it +-- by `file_id` lets the queries that filter by file prune to a single +-- partition, and lets the join with `file_data` (already partitioned with the +-- same modulus) be resolved partition by partition. +-- +-- The primary key has to include the partition key, so it becomes +-- `(file_id, id)`. Every access by id alone would lose its index, so the +-- access paths of the application filter by `file_id` too. +-- +-- The new table is created with the final schema: the columns that nobody +-- used anymore are gone (`data_ref_id` and `data_backend`, dead since the +-- file storage refactor), the `data` column is kept as-is (the `legacy-db` +-- file data backend reads old payloads from it: migration +-- 0157-migrate-legacy-file-data moved the existing blobs), and the new +-- `kind` column tells the snapshots and the revisions apart. +-- +-- The `kind` column is a reserve: the queries still tell a snapshot from a +-- revision by its label (`label IS NOT NULL`), and `kind` takes over when +-- they are rewritten. That is why no constraint ties the two together: the +-- day `kind` is the source of truth, the constraint would have to go. +-- +-- It stays nullable on purpose: the previous version inserts without it, +-- and a rollback to that version has to keep writing. The `NOT NULL` comes +-- in a later migration, once the rollback window is closed. +-- +-- The old table is kept aside as `file_change_legacy`: a frozen copy of the +-- previous layout. It is dropped by hand, days later (or right here, when it +-- is empty: on a new instance there is nothing to keep). +-- +-- This migration is not meant to run on a big instance: the copy of the +-- rows of the table happens inside the transaction of the migration, and +-- the pool sets a 5 minute statement timeout. On those instances mark the +-- migration as applied and partition the table by hand, with the procedure +-- of `operations/2.20.0-file-change-partition.md`. +-- +-- It assumes the service is stopped (a stop-start upgrade): it renames the +-- table, and a process of the previous version would keep writing to +-- `file_change_legacy` through its server-side prepared statements, which +-- hold the OID of the table and survive the rename. A rolling deployment is +-- the scenario of the runbook, not of this path. +-- +-- The GRANTs of the table are not carried over: this creates a new table, +-- so an instance with custom grants on `file_change` has to re-apply them +-- after the upgrade. + +-- 0. The guard for the big instances, before anything is touched: a copy +-- that does not fit keeps the backend down in a restart loop, and the +-- operator has to learn it from a clear message instead of from a +-- `statement timeout`. Both numbers are catalog reads, not table scans. +-- +-- The row count comes from `pg_class.reltuples`, which is -1 on a +-- relation that was never analyzed (a restore without `--analyze`, for +-- instance). Analyze first so the number is real; on a table that fails +-- the guard this costs seconds per boot attempt, not a table scan. +-- +-- The copy, the index builds and the `ANALYZE` of an instance that +-- passes the guard can take longer than the 5 minutes of the pool. The +-- timeout is bounded, so a wrong estimate still fails instead of hanging +-- the boot, and it covers the `ANALYZE`: on a cold, large table it can +-- outlast the pool too, and the guard has to be the one that speaks. +SET LOCAL statement_timeout = '30min'; + +ANALYZE file_change; + +DO $$ +BEGIN + IF pg_total_relation_size('file_change'::regclass) > 10::bigint * 1024 * 1024 * 1024 + OR (SELECT reltuples FROM pg_class WHERE oid = 'file_change'::regclass) > 2000000 + THEN + RAISE EXCEPTION + 'file_change is too large for the automatic partition migration: mark 0158-partition-file-change as applied and follow operations/2.20.0-file-change-partition.md'; + END IF; +END $$; + +-- 1. The legacy blobs of the snapshots were already moved to `file_data` +-- by migration 0157-migrate-legacy-file-data (on the big instances, by +-- hand, with its runbook), so there is nothing to copy here. The `data` +-- column travels with the row in step 4 below: the `legacy-db` backend +-- still reads old payloads from it. + +-- 2. The current table becomes the legacy one. The names of its indexes and +-- constraints live in the schema, not in the table, so they are renamed +-- with a `_legacy` suffix to free the canonical names. +ALTER TABLE file_change RENAME TO file_change_legacy; + +ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_pkey + TO file_change_pkey_legacy; +ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_file_id_fkey + TO file_change_file_id_fkey_legacy; +ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_locked_by_fkey + TO file_change_locked_by_fkey_legacy; +ALTER TABLE file_change_legacy RENAME CONSTRAINT file_change_profile_id_fkey + TO file_change_profile_id_fkey_legacy; + +ALTER INDEX file_change__data_ref_id__idx + RENAME TO file_change__data_ref_id__idx_legacy; +ALTER INDEX file_change__deleted_at__idx + RENAME TO file_change__deleted_at__idx_legacy; +ALTER INDEX file_change__file_id__revn__idx + RENAME TO file_change__file_id__revn__idx_legacy; +ALTER INDEX file_change__locked_by__idx + RENAME TO file_change__locked_by__idx_legacy; +ALTER INDEX file_change__profile_id__idx + RENAME TO file_change__profile_id__idx_legacy; +ALTER INDEX file_change__system_snapshots__idx + RENAME TO file_change__system_snapshots__idx_legacy; + +-- This one comes from another branch, so an instance may not have it. +ALTER INDEX IF EXISTS file_change__user_snapshots__idx + RENAME TO file_change__user_snapshots__idx_legacy; + +-- 3. The new table, partitioned with the same bounds as `file_data`, so both +-- relations can be joined partition by partition. +CREATE TABLE file_change ( + file_id uuid NOT NULL, + id uuid NOT NULL DEFAULT gen_random_uuid(), + created_at timestamptz NOT NULL DEFAULT now(), + session_id uuid, + revn bigint NOT NULL DEFAULT 0, + data bytea, + changes bytea, + profile_id uuid, + features text[], + label text, + version integer, + created_by text NOT NULL DEFAULT 'system', + updated_at timestamptz DEFAULT now(), + deleted_at timestamptz, + migrations text[], + locked_by uuid, + -- Nullable on purpose: the previous version inserts without `kind`, and + -- a rollback to it has to keep writing. The `NOT NULL` comes in a later + -- migration, once the rollback window is closed. + kind smallint, + CONSTRAINT file_change_kind_check CHECK (kind IN (1, 2)) +) PARTITION BY HASH (file_id); + +-- the columns keep the storage of PostgreSQL by default (migration +-- 0156-reset-pg-storage-defaults dropped the explicit EXTERNAL overrides) + +COMMENT ON COLUMN file_change.kind IS + 'The kind of the row: 1 is a snapshot (it has a label and its data lives in file_data) and 2 is a revision (it has no label and it holds the changes).'; +COMMENT ON COLUMN file_change.locked_by IS + 'Profile ID of user who has locked this version. Only the creator can lock/unlock their own versions. Locked versions cannot be deleted by others.'; + +CREATE TABLE file_change_00 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 0); +CREATE TABLE file_change_01 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 1); +CREATE TABLE file_change_02 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 2); +CREATE TABLE file_change_03 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 3); +CREATE TABLE file_change_04 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 4); +CREATE TABLE file_change_05 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 5); +CREATE TABLE file_change_06 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 6); +CREATE TABLE file_change_07 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 7); +CREATE TABLE file_change_08 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 8); +CREATE TABLE file_change_09 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 9); +CREATE TABLE file_change_10 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 10); +CREATE TABLE file_change_11 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 11); +CREATE TABLE file_change_12 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 12); +CREATE TABLE file_change_13 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 13); +CREATE TABLE file_change_14 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 14); +CREATE TABLE file_change_15 PARTITION OF file_change + FOR VALUES WITH (MODULUS 16, REMAINDER 15); + +-- 4. The rows move to the partitioned table. The column list is explicit +-- because the legacy table has columns that the new one does not have +-- (`data_ref_id` and `data_backend`). +-- The kind comes from the label: a row with a label is a snapshot, a row +-- without one is a revision. +INSERT INTO file_change (id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, kind) +SELECT id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, + CASE WHEN label IS NOT NULL THEN 1 ELSE 2 END + FROM file_change_legacy; + +-- 5. The primary key includes the partition key now. +ALTER TABLE file_change ADD PRIMARY KEY (file_id, id); + +-- 6. The foreign keys. +ALTER TABLE file_change ADD FOREIGN KEY (file_id) + REFERENCES file(id) DEFERRABLE; +ALTER TABLE file_change ADD FOREIGN KEY (locked_by) + REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE; +ALTER TABLE file_change ADD FOREIGN KEY (profile_id) + REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE; + +-- 7. The indexes, with the canonical names. The one of the deleted rows now +-- carries the `file_id`, so the garbage collector can read the rows it has +-- to delete without touching the table. +-- +-- There is no index for the system snapshots: the old one filtered them by +-- `data IS NOT NULL`, and with the payload living in `file_data` the +-- column no longer tells snapshots apart. The new one could only filter +-- them by `deleted_at IS NULL`, but the system snapshots always carry a +-- deletion date in the future, so it would never match a row (and +-- the query that would use it filters that date with an "or", which no +-- partial index can serve). +CREATE INDEX file_change__deleted_at__idx + ON file_change (deleted_at, file_id, id) WHERE deleted_at IS NOT NULL; +CREATE INDEX file_change__file_id__revn__idx + ON file_change (file_id, revn); +CREATE INDEX file_change__locked_by__idx + ON file_change (locked_by) WHERE locked_by IS NOT NULL; +CREATE INDEX file_change__profile_id__idx + ON file_change (profile_id) WHERE profile_id IS NOT NULL; +CREATE INDEX file_change__user_snapshots__idx + ON file_change (file_id) + WHERE label IS NOT NULL + AND created_by = 'user' + AND deleted_at IS NULL; + +-- 8. Statistics for the new partitions. The `ANALYZE` of step 0 ran against +-- the old table, before the copy: without this one the planner starts +-- with empty statistics on the partitions and picks plans by guesswork +-- until autovacuum gets to them. The backend starts serving right after +-- this migration, so it is the worst moment to leave them unanalyzed. +ANALYZE file_change; + +-- 9. The legacy table is kept as a frozen copy of the previous layout. It is +-- dropped by hand, days after the cutover: +-- DROP TABLE file_change_legacy; +-- +-- Its foreign key on `file_id` goes away with the copy: it is `NO +-- ACTION`, so while the legacy table has rows, the objects GC cannot +-- delete a file (it deletes the rows of the file first, and the legacy +-- ones are not its business). The legacy table is frozen: nothing writes +-- to it again. +ALTER TABLE file_change_legacy + DROP CONSTRAINT file_change_file_id_fkey_legacy; + +-- On a new instance there is nothing to go back to, so the empty table +-- is dropped here. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM file_change_legacy) THEN + DROP TABLE file_change_legacy; + END IF; +END $$; diff --git a/backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql b/backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql new file mode 100644 index 0000000000..3a92172edf --- /dev/null +++ b/backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql @@ -0,0 +1,65 @@ +-- Turn the file data storage pointer into a column with a foreign key. +-- +-- Until now the pointer lived in `file_data.metadata->>'storage-ref-id'`. +-- The storage GC parsed the JSON, and nothing stopped it from deleting the +-- blob of a live row. With the column and the foreign key the database +-- refuses that delete. +-- +-- With the storage backend the release is a coordinated restart: the column is +-- the only source of truth, and the previous code cannot read the rows this +-- release writes. With the default `db` backend the new code writes the same +-- `file_data.data` column the previous code reads, so a rolling deploy is +-- safe. +-- +-- This migration runs on a fresh database and on small instances. A large +-- instance builds the column and the index out of band, runs this same +-- backfill and foreign key with the instances stopped, and marks this step as +-- applied; the end state is identical. See +-- `operations/2.20.0-file-data-storage-object-id.md`. +-- +-- `DEFERRABLE` follows the schema convention (all foreign keys of the project +-- are deferrable, so `th/database-reset` can delete table by table with +-- `SET CONSTRAINTS ALL DEFERRED`). It is `INITIALLY IMMEDIATE`, not +-- `DEFERRED`: the storage GC catches the violation inside the chunk +-- transaction and retries the chunk, and a deferred check would fire at +-- commit, outside that transaction, so the GC would never move past it. + +ALTER TABLE file_data + ADD COLUMN IF NOT EXISTS storage_object_id uuid; + +-- The CTE is MATERIALIZED so the uuid cast only sees rows that passed the +-- regex. A plain subquery can be inlined, and a corrupt value would reach the +-- cast and abort the migration. +WITH candidates AS MATERIALIZED ( + SELECT file_id, + id, + (metadata->>'storage-ref-id')::uuid AS soid + FROM file_data + WHERE backend = 'storage' + AND storage_object_id IS NULL + AND (metadata->>'storage-ref-id') ~* + '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$' +) +UPDATE file_data AS fd + SET storage_object_id = c.soid, + metadata = fd.metadata - 'storage-ref-id' + FROM candidates AS c + WHERE fd.file_id = c.file_id + AND fd.id = c.id + AND EXISTS (SELECT 1 FROM storage_object AS so WHERE so.id = c.soid); + +-- The deletion check of the foreign key looks a row up by this column for +-- every deleted object; without the index it scans all partitions. +CREATE INDEX IF NOT EXISTS file_data__storage_object_id__idx + ON file_data (storage_object_id); + +-- Rebuild the constraint so the definition is always the one above. This is +-- also how a constraint created by hand with the wrong shape (CASCADE, not +-- deferrable, or `INITIALLY DEFERRED`) gets repaired. +ALTER TABLE file_data + DROP CONSTRAINT IF EXISTS file_data_storage_object_id_fkey; + +ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE; diff --git a/backend/src/app/nitrate.clj b/backend/src/app/nitrate.clj index d0d8c3e219..6fb33b6ebf 100644 --- a/backend/src/app/nitrate.clj +++ b/backend/src/app/nitrate.clj @@ -398,6 +398,20 @@ (generate-nitrate-uri "api/subscriptions/" profile-id) schema:subscription params)) +(def ^:private schema:profile-subscription + [:map {:title "ProfileSubscription"} + [:profile-id ::sm/uuid] + [:type ::sm/text] + [:status ::sm/text]]) + +(defn- get-subscriptions-api + [cfg {:keys [profile-ids] :as params}] + (let [params (assoc params :request-params {:profile-ids profile-ids})] + (request-to-nitrate cfg :post + (generate-nitrate-uri "api/subscriptions/bulk") + [:vector schema:profile-subscription] + params))) + (def ^:private schema:subscription-warning [:maybe [:map {:title "SubscriptionWarning"} @@ -533,6 +547,7 @@ :delete-team (partial delete-team-api cfg) :remove-team-from-organization (partial remove-team-from-organization-api cfg) :get-subscription (partial get-subscription-api cfg) + :get-subscriptions (partial get-subscriptions-api cfg) :get-subscription-warning (partial get-subscription-warning-api cfg) :connectivity (partial get-connectivity-api cfg) :get-identity (partial get-identity-api cfg) @@ -699,6 +714,47 @@ team)) teams)))) +(def ^:private valid-licence-statuses + "The statuses `app.main.data.nitrate/is-valid-license?` accepts on the + frontend." + #{"active" "past_due" "trialing"}) + +(defn- get-valid-licence-statuses + "Returns a map of profile id to the status of its valid Nitrate licence. + Returns an empty map when Nitrate does not answer: the licences of team + owners only decide which plan a team shows." + [cfg profile-ids params] + (try + (let [params (assoc (or params {}) :profile-ids (vec profile-ids)) + subscriptions (call cfg :get-subscriptions params)] + (into {} + (comp (filter #(contains? valid-licence-statuses (:status %))) + (map (juxt :profile-id :status))) + subscriptions)) + (catch Throwable cause + (l/wrn :hint "failed to get team owner licences" + :cause cause) + {}))) + +(defn add-owner-licence-to-teams + "Sets type \"nitrate\" in the `:subscription` of each team outside an + organization whose owner has a valid Nitrate licence, so every member + sees the plan of the owner, not only the owner. The database only knows + the Stripe subscription of the owner, which teams keep when Nitrate does + not answer. Teams in an organization take their plan from it." + [cfg teams params] + (let [owner-id #(when (nil? (:organization %)) + (get-in % [:subscription :owner-id])) + owner-ids (into #{} (keep owner-id) teams)] + (if (empty? owner-ids) + teams + (let [statuses (get-valid-licence-statuses cfg owner-ids params)] + (mapv (fn [team] + (if-let [status (get statuses (owner-id team))] + (update team :subscription assoc :type "nitrate" :status status) + team)) + teams))))) + (defn set-team-organization "Associates a team with an organization in Nitrate. Requires organization-id and is-default in params. @@ -716,12 +772,19 @@ :organization-id (:organization-id params)})) team)) +(defn- organization-membership + "The membership of `profile-id` in `organization-id`, as Nitrate reports it. + The profile travels in the path of the Nitrate endpoint, so no acting + profile is sent." + [cfg profile-id organization-id] + (call cfg :get-organization-membership {:profile-id profile-id + :organization-id organization-id})) + (defn assert-membership "Verifies that the user is a member of the organization. Raises an exception if the organization doesn't exist or the user is not a member." [cfg profile-id organization-id] - (let [membership (call cfg :get-organization-membership {:profile-id profile-id - :organization-id organization-id})] + (let [membership (organization-membership cfg profile-id organization-id)] (when-not (:organization-id membership) (ex/raise :type :validation :code :organization-does-not-exist)) @@ -729,3 +792,18 @@ (when-not (:is-member membership) (ex/raise :type :validation :code :user-doesnt-belong-organization)))) + +(defn organization-access? + "True when `profile-id` may act for `organization-id`: it is a member of + the organization or its owner. + + The organization id reaches the backend from the client, so the flows that + take one have to check it: without this check any authenticated profile + could start a flow for an organization it does not belong to." + [cfg profile-id organization-id] + (let [membership (organization-membership cfg profile-id organization-id)] + (if (:is-member membership) + true + (= (:owner-id (call cfg :get-organization-summary {:organization-id organization-id + ::rpc/profile-id profile-id})) + profile-id)))) diff --git a/backend/src/app/rpc/commands/auth.clj b/backend/src/app/rpc/commands/auth.clj index 272391a104..11d32a636f 100644 --- a/backend/src/app/rpc/commands/auth.clj +++ b/backend/src/app/rpc/commands/auth.clj @@ -46,6 +46,10 @@ (def schema:token [::sm/word-string {:max 6000}]) +(def ^:private penpot-backend + "Auth backend identifier for the password flow." + "penpot") + (defn- elapsed-verify-threshold? [profile] (let [elapsed (ct/diff (:modified-at profile) (ct/now)) @@ -201,6 +205,11 @@ (let [profile (some->> (validate-token token) (update-password conn))] (when profile + ;; Recovery must revoke every existing session: a previously + ;; captured cookie otherwise keeps working after the password + ;; change (GHSA-3j2p-qr45-q365). Runs inside the same transaction + ;; as the password update (::db/transaction true). + (session/invalidate-all cfg (:id profile)) (login-lockout/clear-attempts! cfg (:id profile)) (eml/send-template cfg {::eml/reuse-conn true ::eml/template "password-changed" @@ -226,25 +235,72 @@ ;; ---- COMMAND: Prepare Register -(defn- validate-registration-flags! - ([cfg invitation] - (validate-registration-flags! cfg invitation false)) - ([cfg invitation lock-invitation?] - (when-not (contains? cf/flags :login-with-password) - (ex/raise :type :restriction - :code :registration-disabled - :hint "registration disabled")) +(defn- check-registration-flags + "The `:backend` is the one embedded in the verified + `:prepared-register` token: the password flow backend, one of the + `oidc/oidc-provider-types` backends for SSO flows, or any other + value, which is rejected loudly." + [backend invitation] + (cond + (= backend penpot-backend) + (do + (when-not (contains? cf/flags :login-with-password) + (ex/raise :type :restriction + :code :registration-disabled + :hint "registration disabled")) - (when-not (contains? cf/flags :registration) - (if (nil? invitation) - (ex/raise :type :restriction - :code :registration-disabled - :hint "registration disabled") - (let [opts (when lock-invitation? {::db/for-update true})] - (when (nil? (teams-invitations/active-invitation cfg invitation opts)) - (ex/raise :type :validation - :code :invalid-token - :hint "no active invitation associated with the token"))))))) + (when-not (or (contains? cf/flags :registration) + (some? invitation)) + (ex/raise :type :restriction + :code :registration-disabled + :hint "registration disabled"))) + + (contains? oidc/oidc-provider-types backend) + ;; SSO backends share the OIDC callback gate + ;; (`oidc/sso-registration-allowed?`): registration is allowed with + ;; `:registration` or `:oidc-registration`, without requiring + ;; password login. An invitation token is a substitute for the + ;; flags, as in the password flow. + (when-not (or (oidc/sso-registration-allowed?) + (some? invitation)) + (ex/raise :type :restriction + :code :registration-disabled + :hint "registration disabled")) + + :else + (ex/raise :type :validation + :code :invalid-token + :hint "unsupported registration backend"))) + +(defn- check-registration-invitation + "Verifies that the invitation is still active. No-op unless a live + invitation is required: a backend whose registration flags do not + allow public registration, with an invitation at hand." + [cfg backend invitation lock-invitation?] + (let [flag-allows? (if (= backend penpot-backend) + (contains? cf/flags :registration) + (oidc/sso-registration-allowed?))] + (when (and (some? invitation) + (not flag-allows?)) + (let [opts (when lock-invitation? {::db/for-update true})] + (when (nil? (teams-invitations/active-invitation cfg invitation opts)) + (ex/raise :type :validation + :code :invalid-token + :hint "no active invitation associated with the token")))))) + +(defn- check-registration + "Shared registration gate (register-profile and prepare-register): + validates the flags, the invitation email and the team invitation + row in order." + [cfg backend invitation email lock-invitation?] + (check-registration-flags backend invitation) + (when (and (some? invitation) + (not= (profile/clean-email (:member-email invitation)) + (profile/clean-email email))) + (ex/raise :type :restriction + :code :email-does-not-match-invitation + :hint "email should match the invitation")) + (check-registration-invitation cfg backend invitation lock-invitation?)) (defn- validate-register-attempt! [cfg params] @@ -254,23 +310,7 @@ {:token (:invitation-token params) :iss :team-invitation}))] - (when (or (not (contains? cf/flags :login-with-password)) - (and (not (contains? cf/flags :registration)) - (not invitation?))) - (ex/raise :type :restriction - :code :registration-disabled - :hint "registration disabled")) - - (when (and invitation? - (not= (profile/clean-email (:email params)) - (profile/clean-email (:member-email invitation)))) - (ex/raise :type :restriction - :code :email-does-not-match-invitation - :hint "email should match the invitation")) - - (when (and invitation? - (not (contains? cf/flags :registration))) - (validate-registration-flags! cfg invitation))) + (check-registration cfg penpot-backend invitation (:email params) false)) (when (and (email.blacklist/enabled? cfg) (email.blacklist/contains? cfg (:email params))) @@ -342,7 +382,7 @@ :fullname fullname :password (:password params) :invitation-token (:invitation-token params) - :backend "penpot" + :backend penpot-backend :iss :prepared-register :exp (ct/in-future {:days 7}) :props props} @@ -419,7 +459,7 @@ locale (when (and (string? locale) (not (str/blank? locale))) locale) - backend (:backend params "penpot") + backend (:backend params penpot-backend) is-demo (:is-demo params false) is-muted (:is-muted params false) is-active (:is-active params false) @@ -505,12 +545,25 @@ :token vtoken :extra-data ptoken})))) +(def ^:private schema:prepared-register-claims + [:map {:title "PreparedRegisterClaims"} + [:backend ::sm/text] + [:email ::sm/email]]) + +(def ^:private valid-prepared-register-claims? + (sm/lazy-validator schema:prepared-register-claims)) + (defn register-profile [{:keys [::db/conn] :as cfg} {:keys [token] :as params}] (let [claims (tokens/verify cfg {:token token :iss :prepared-register}) + _ (when-not (valid-prepared-register-claims? claims) + (ex/raise :type :validation + :code :invalid-token + :hint "prepared-register token contains unexpected data")) + backend (:backend claims) invitation (when-let [token (:invitation-token claims)] (tokens/verify cfg {:token token :iss :team-invitation})) - _ (validate-registration-flags! cfg invitation true) + _ (check-registration cfg backend invitation (:email claims) true) params (cond-> claims (:accept-newsletter-updates params) (update :props assoc :newsletter-updates true)) diff --git a/backend/src/app/rpc/commands/comments.clj b/backend/src/app/rpc/commands/comments.clj index a0103086e5..9a94b205e6 100644 --- a/backend/src/app/rpc/commands/comments.clj +++ b/backend/src/app/rpc/commands/comments.clj @@ -195,7 +195,10 @@ (-> file (assoc :page-name (dm/get-in data [:pages-index page-id :name])) (assoc :page-id page-id) - (dissoc :data))))) + (dissoc :data) + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (bfc/file->response))))) ;; FIXME: rename (defn- get-comment-thread diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 0a787fbeb4..87f6c3b6d8 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -42,8 +42,8 @@ [app.rpc.cond :as-alias cond] [app.rpc.doc :as-alias doc] [app.rpc.helpers :as rph] + [app.rpc.notifications :as ntf] [app.rpc.permissions :as perms] - [app.util.blob :as blob] [app.util.events :as events] [app.util.pointer-map :as pmap] [app.util.services :as sv] @@ -222,7 +222,11 @@ (if (and (contains? (:features file) "fdata/objects-map") (not (contains? (:features params) "fdata/objects-map"))) (feat.fdata/realize-objects cfg file) - file))))) + file) + + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (bfc/file->response file))))) ;; --- COMMAND QUERY: get-file-fragment (by id) @@ -231,7 +235,7 @@ [:id ::sm/uuid] [:file-id ::sm/uuid] [:created-at ::ct/inst] - [:content ::sm/any]]) + [:data ::sm/any]]) (def schema:get-file-fragment [:map {:title "get-file-fragment"} @@ -241,8 +245,17 @@ (defn- get-file-fragment [cfg file-id fragment-id] - (some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"}) - (update :data blob/decode))) + ;; Resolve through the file data backend: a fragment can live on cold + ;; storage, and reading its row directly would hand `nil` to `blob/decode`. + ;; Only the public fields leave the backend: the row also carries the + ;; internal storage pointer and the backend metadata. + (when-let [row (db/get cfg :file-data {:file-id file-id + :id fragment-id + :type "fragment"})] + (let [fragment (->> row + (feat.fdata/resolve-file-data cfg) + (feat.fdata/decode-file-data cfg))] + (select-keys fragment (sm/keys schema:file-fragment))))) (defn- check-fragment-scope! "Checks that the fragment is reachable from the pages authorized by @@ -561,17 +574,23 @@ (let [file (bfc/get-file cfg id :load-data? false) summary (t/decode-str result)] (-> (assoc file :library-summary summary) - (dissoc :data))))) + (dissoc :data :migrations) + (bfc/file->response))))) calculate-from-db (fn [] (let [file (bfc/get-file cfg id) result (feat.fdata/with-pointers cfg id (calculate-library-summary file))] + ;; The storage pointer is internal bookkeeping: this summary is an + ;; RPC response and must not expose it. The applied migrations go + ;; for the same reason: this endpoint is not where a client + ;; completes a read, and `schema:file` declares the key optional. (-> file (assoc :library-summary result) (dissoc :legacy-data) - (dissoc :data)))) + (dissoc :data :migrations) + (bfc/file->response)))) persist-to-cache (fn [{:keys [::rds/conn]} data cache-key] @@ -1039,13 +1058,10 @@ :library-id (str (:id ldata)) :file-id (str file-id)) - (bfc/update-file! cfg {:id file-id - :features (:features file) - :migrations (:migrations file) - :revn (inc (:revn file)) - :data (:data file) - :modified-at (ct/now) - :has-media-trimmed false})))) + (bfc/update-file! cfg file + :revn (inc (:revn file)) + :modified-at (ct/now) + :has-media-trimmed false)))) (defn- get-referenced-files [cfg library-id] @@ -1200,6 +1216,12 @@ :file-id id :profile-id profile-id}) + ;; Deleting the file ends read access for whoever had it open, and + ;; which profiles those are is only known to the connections + ;; themselves, so the whole team is announced and the watcher + ;; re-checks each of them. + (ntf/notify-team-permissions-changed cfg (:id team)) + (rph/with-meta (rph/wrap) {::audit/props {:team-id (:id team) :project-id (:project-id file) @@ -1272,11 +1294,10 @@ (check-edition-permissions! conn profile-id library-id) (check-library-team-ownership! conn file-id library-id) - (let [transitive-deps (bfc/get-libraries cfg [library-id])] - (when (contains? transitive-deps file-id) - (ex/raise :type :validation - :code :circular-library-reference - :hint "linking this library would create a circular dependency"))) + (when (= :circular (bfc/library-link-status cfg file-id library-id)) + (ex/raise :type :validation + :code :circular-library-reference + :hint "linking this library would create a circular dependency")) (link-file-to-library conn params) (bfc/get-libraries cfg [library-id])) @@ -1434,7 +1455,9 @@ ;; for ever. Rows that still carry a legacy payload without a label are ;; garbage that no read path reaches; 0157-migrate-legacy-file-data deletes ;; them, so they must keep their date and go with the collector. - "UPDATE file_change SET deleted_at = null + ;; Every write moves `updated_at`: the delta passes copy the rows whose + ;; `updated_at` changed, so a restore that left it stale could be missed. + "UPDATE file_change SET deleted_at = null, updated_at = now() WHERE file_id = ANY(?::uuid[]) AND label IS NOT NULL") @@ -1452,8 +1475,8 @@ (def ^:private sql:restore-storage-objects ;; Collect referenced storage ids first (UUID set), then join by PK. - ;; Casting `storage_object.id` to text (or OR-ing that with `id IN`) forces - ;; a scan of deleted rows; cast the JSON side instead. + ;; `file_data` points at its object with the `storage_object_id` column, so + ;; the reference is already a uuid and no JSON cast is needed. "WITH ids AS (SELECT unnest(?::uuid[]) AS file_id), refs AS ( SELECT fmo.media_id AS id @@ -1480,11 +1503,11 @@ WHERE fot.deleted_at IS NULL AND fot.media_id IS NOT NULL UNION - SELECT (fd.metadata->>'storage-ref-id')::uuid + SELECT fd.storage_object_id FROM file_data AS fd JOIN ids ON (ids.file_id = fd.file_id) WHERE fd.deleted_at IS NULL - AND (fd.metadata->>'storage-ref-id') IS NOT NULL + AND fd.storage_object_id IS NOT NULL ) UPDATE storage_object AS so SET deleted_at = null diff --git a/backend/src/app/rpc/commands/files_branch.clj b/backend/src/app/rpc/commands/files_branch.clj index 7af39af58b..98fc3384d2 100644 --- a/backend/src/app/rpc/commands/files_branch.clj +++ b/backend/src/app/rpc/commands/files_branch.clj @@ -295,12 +295,12 @@ fragment pointers, and `file` is the same document realized and checked against the size limits. - The row is read FOR SHARE. File-gc takes it FOR UPDATE and marks every - fragment that neither main nor a committed snapshot names, so a - snapshot that copies main's pointers must commit before the next pass - can read the row." + The row is read FOR UPDATE, since a data read cannot take FOR SHARE. + File-gc takes it FOR UPDATE too and marks every fragment that neither + main nor a committed snapshot names, so a snapshot that copies main's + pointers must commit before the next pass can read the row." [cfg main-id operation] - (let [stored (bfc/get-file cfg main-id :lock-for-share? true) + (let [stored (bfc/get-file cfg main-id :lock-for-update? true) file (-> (fdata/realize cfg stored) (check-file-size-limits! operation))] [stored file])) @@ -590,10 +590,10 @@ ;; GC while the branch is open (it is released when the ;; branch is deleted or its base moves). The snapshot ;; stores main's fragment pointers, not its pages, so - ;; main's row is read FOR SHARE: file-gc takes it FOR - ;; UPDATE, and a pass that ran between this read and - ;; the commit could mark a fragment the base names. - (let [file (bfc/get-file cfg file-id :lock-for-share? true) + ;; main's row is read FOR UPDATE: file-gc takes it FOR + ;; UPDATE as well, and a pass that ran between this read + ;; and the commit could mark a fragment the base names. + (let [file (bfc/get-file cfg file-id :lock-for-update? true) _ (fdata/with-pointers cfg file-id (check-file-size-limits! file :create-branch)) base (fsnap/create! cfg file diff --git a/backend/src/app/rpc/commands/files_create.clj b/backend/src/app/rpc/commands/files_create.clj index 368f480ec7..3d1fac497a 100644 --- a/backend/src/app/rpc/commands/files_create.clj +++ b/backend/src/app/rpc/commands/files_create.clj @@ -68,7 +68,10 @@ {:modified-at (ct/now)} {:id project-id}) - (bfc/get-file cfg (:id file)))))) + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (-> (bfc/get-file cfg (:id file)) + (bfc/file->response)))))) (def ^:private schema:create-file [:map {:title "create-file" :closed true} diff --git a/backend/src/app/rpc/commands/files_snapshot.clj b/backend/src/app/rpc/commands/files_snapshot.clj index 012c43acc6..bdf514d020 100644 --- a/backend/src/app/rpc/commands/files_snapshot.clj +++ b/backend/src/app/rpc/commands/files_snapshot.clj @@ -52,6 +52,7 @@ "Retrieve a file bundle with data from a specific snapshot for read-only preview. Does not modify any database state." {::doc/added "2.16" + ::doc/changes [["2.20" "An unknown snapshot is reported with the :snapshot-not-found error code"]] ::sm/params schema:get-file-snapshot ::sm/result files/schema:file-with-permissions ::db/transaction true} @@ -125,12 +126,15 @@ ;; every unpurged snapshot as a liveness root, and readers ;; (get-file-snapshot, restore!, reduce-snapshots) resolve pointers ;; on demand. - ;; NOTE: the file is read with FOR SHARE inside this transaction so + ;; NOTE: the file is read with FOR UPDATE inside this transaction so ;; snapshot creation serializes with a concurrent :file-gc pass - ;; (which takes FOR UPDATE + SKIP LOCKED on the same row). Without - ;; the lock, a version pinned mid-pass could reference fragment rows - ;; the pass is recreating. - (let [file (bfc/get-file cfg file-id :lock-for-share? true) + ;; (which takes FOR UPDATE + SKIP LOCKED on the same row) and with a + ;; concurrent save. Without the lock, a version pinned mid-pass could + ;; reference fragment rows the pass is recreating. The lock is + ;; exclusive, not shared, because the migration-on-read path may + ;; re-read the same row with FOR UPDATE; holding a shared lock while + ;; asking for the exclusive one deadlocks two concurrent readers. + (let [file (bfc/get-file cfg file-id :lock-for-update? true) project (db/get-by-id cfg :project (:project-id file))] (-> cfg @@ -166,12 +170,14 @@ delay (ldel/get-deletion-delay team) ;; Create backup snapshot of the current state (committed immediately - ;; independent of the restore outcome). The read takes FOR SHARE so a + ;; independent of the restore outcome). The read takes FOR UPDATE so a ;; concurrent :file-gc pass (which holds FOR UPDATE + SKIP LOCKED on the ;; file row) either finishes first -- and we snapshot the new fragments -- ;; or skips the file and retries later. Without the lock, the backup ;; could reference fragments the pass is about to mark as deleted, - ;; breaking the backup permanently. + ;; breaking the backup permanently. The lock is exclusive because the + ;; migration-on-read path may re-read the row with FOR UPDATE: a shared + ;; lock that upgrades deadlocks two concurrent readers. ;; ;; The backup read also provides the revn the guard below compares ;; against. An edit committed before this read is captured in the @@ -180,7 +186,7 @@ backup-revn (db/tx-run! cfg (fn [cfg] - (let [file (bfc/get-file cfg file-id :lock-for-share? true)] + (let [file (bfc/get-file cfg file-id :lock-for-update? true)] ;; `fsnap/restore!` refuses a branch file too. The ;; command refuses before it takes the backup ;; snapshot, which is committed whatever the @@ -201,7 +207,7 @@ (ex/raise :type :conflict :code :file-modified :hint "the file was modified during the restore process, please retry"))) - (let [snapshot (fsnap/get-minimal-snapshot cfg id) + (let [snapshot (fsnap/get-minimal-snapshot cfg file-id id) vern (fsnap/restore! cfg file-id id)] ;; Peers name the restored version by its label when a ;; user named it, else by its date @@ -219,15 +225,17 @@ (def ^:private schema:update-file-snapshot [:map {:title "update-file-snapshot"} + [:file-id ::sm/uuid] [:id ::sm/uuid] [:label ::sm/text]]) (sv/defmethod ::update-file-snapshot {::doc/added "1.20" + ::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]] ::sm/params schema:update-file-snapshot ::db/transaction true} - [{:keys [::db/conn]} {:keys [::rpc/profile-id id label]}] - (let [snapshot (fsnap/get-minimal-snapshot conn id)] + [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id id label]}] + (let [snapshot (fsnap/get-minimal-snapshot conn file-id id)] (files/check-edition-permissions! conn profile-id (:file-id snapshot)) (when (not= (:created-by snapshot) "user") @@ -241,14 +249,16 @@ (def ^:private schema:remove-file-snapshot [:map {:title "remove-file-snapshot"} + [:file-id ::sm/uuid] [:id ::sm/uuid]]) (sv/defmethod ::delete-file-snapshot {::doc/added "1.20" + ::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]] ::sm/params schema:remove-file-snapshot ::db/transaction true} - [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id]}] - (let [snapshot (fsnap/get-minimal-snapshot conn id)] + [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id id]}] + (let [snapshot (fsnap/get-minimal-snapshot conn file-id id)] (files/check-edition-permissions! conn profile-id (:file-id snapshot)) (when (not= (:created-by snapshot) "user") @@ -276,14 +286,16 @@ (def ^:private schema:lock-file-snapshot [:map {:title "lock-file-snapshot"} + [:file-id ::sm/uuid] [:id ::sm/uuid]]) (sv/defmethod ::lock-file-snapshot {::doc/added "1.20" + ::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]] ::sm/params schema:lock-file-snapshot ::db/transaction true} - [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}] - (let [snapshot (fsnap/get-minimal-snapshot conn id)] + [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id id]}] + (let [snapshot (fsnap/get-minimal-snapshot conn file-id id)] (files/check-edition-permissions! conn profile-id (:file-id snapshot)) (when (not= (:created-by snapshot) "user") @@ -311,18 +323,20 @@ :profile-id profile-id :locked-by (:locked-by snapshot))) - (fsnap/lock-by! conn id profile-id))) + (fsnap/lock-by! conn (:file-id snapshot) id profile-id))) (def ^:private schema:unlock-file-snapshot [:map {:title "unlock-file-snapshot"} + [:file-id ::sm/uuid] [:id ::sm/uuid]]) (sv/defmethod ::unlock-file-snapshot {::doc/added "1.20" + ::doc/changes [["2.20" "The :file-id param is now required and an unknown snapshot answers :snapshot-not-found"]] ::sm/params schema:unlock-file-snapshot ::db/transaction true} - [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}] - (let [snapshot (fsnap/get-minimal-snapshot conn id)] + [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id id]}] + (let [snapshot (fsnap/get-minimal-snapshot conn file-id id)] (files/check-edition-permissions! conn profile-id (:file-id snapshot)) (when (not= (:created-by snapshot) "user") @@ -349,4 +363,4 @@ :snapshot-id id :profile-id profile-id)) - (fsnap/unlock! conn id))) + (fsnap/unlock! conn (:file-id snapshot) id))) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index f73670bac2..07deb4cec7 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -23,6 +23,7 @@ [app.config :as cf] [app.db :as db] [app.features.fdata :as fdata] + [app.features.file-changes :as fchanges] [app.features.file-snapshots :as fsnap] [app.features.logical-deletion :as ldel] [app.http.errors :as errors] @@ -469,6 +470,7 @@ (when repair (db/insert! conn :file-change {:id (uuid/next) + :kind fchanges/revision-kind :session-id repair-session-id :profile-id profile-id :created-at timestamp @@ -477,7 +479,6 @@ :file-id (:id file) :revn (:revn repair) :version (:version file) - :features (into-array (:features file)) :changes (blob/encode (:changes repair))} {::db/return-keys false}) (l/wrn :hint "file auto-repaired" @@ -489,8 +490,13 @@ ;; Insert change (xlog) with deleted_at in a future data for ;; make them automatically eleggible for GC once they expires + ;; + ;; The revision carries only the delta of the save: the features of the + ;; file are not repeated here (nobody reads them from a revision, and + ;; they are duplicated on every save). (db/insert! conn :file-change {:id (uuid/next) + :kind fchanges/revision-kind :session-id session-id :profile-id profile-id :created-at change-created-at @@ -499,7 +505,6 @@ :file-id (:id file) :revn (:revn file) :version (:version file) - :features (into-array (:features file)) :changes (blob/encode changes)} {::db/return-keys false}) @@ -572,9 +577,15 @@ :branch-duration-ms duration))}))))) (defn get-file - "Get not-decoded file, only decodes the features set." + "Get not-decoded file, only decodes the features set. + + The read takes the `file` row with FOR UPDATE: the save persists the + map it read, so it must hold the row from the read to the write. The + lock is exclusive, not shared, because the migration-on-read path may + re-read the same row with FOR UPDATE, and a shared lock that upgrades + deadlocks two concurrent readers." [cfg id] - (bfc/get-file cfg id :decode? false :lock-for-share? true)) + (bfc/get-file cfg id :decode? false :lock-for-update? true)) (defn persist-file! "Function responsible of persisting already encoded file. Should be diff --git a/backend/src/app/rpc/commands/fonts.clj b/backend/src/app/rpc/commands/fonts.clj index f2a5ffa126..de498e006c 100644 --- a/backend/src/app/rpc/commands/fonts.clj +++ b/backend/src/app/rpc/commands/fonts.clj @@ -112,6 +112,7 @@ [:font-family types.font/schema:font-family] [:font-weight [::sm/one-of {:format "number"} valid-weight]] [:font-style [::sm/one-of {:format "string"} valid-style]] + [:variant-name {:optional true} [:maybe [::sm/text {:max 250}]]] [:uploads [:map-of ::sm/text ::sm/uuid]]]) (defn- prepare-font-data-from-uploads @@ -139,7 +140,8 @@ chunked-upload API)." {::doc/added "1.18" ::doc/changes [["2.16" "Add :uploads param for chunked upload support"] - ["2.18" "Remove :data param, use :uploads exclusively"]] + ["2.18" "Remove :data param, use :uploads exclusively"] + ["2.20" "Add optional :variant-name param"]] ::climit/id [[:process-font/by-profile ::rpc/profile-id] [:process-font/global]] ::webhooks/event? true diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index b896d585a5..8ddc0174b1 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -25,6 +25,8 @@ [app.rpc.commands.projects :as proj] [app.rpc.commands.teams :as teams] [app.rpc.doc :as-alias doc] + [app.rpc.notifications :as ntf] + [app.rpc.permissions :as perms] [app.setup :as-alias setup] [app.setup.templates :as tmpl] [app.storage.tmp :as tmp] @@ -88,7 +90,9 @@ fmeds)] (db/insert! conn :file-media-object params ::db/return-keys false)) - file))) + ;; The storage pointer of the source file is internal bookkeeping: the + ;; new file does not inherit it, and it never leaves the backend. + (bfc/file->response file)))) (def ^:private schema:duplicate-file @@ -272,6 +276,17 @@ ;; --- COMMAND: Move file +(defn- notify-team-access-changed + "Announces a possible read-access change for `team-ids`. + + Moving content can take it out of the reach of a whole team, and the + only place that knows who was relying on it is the set of open + connections. One event per team: the watcher resolves the members + itself, so a large team cannot overflow the bounded publish buffers." + [cfg team-ids] + (doseq [team-id (into #{} (remove nil?) team-ids)] + (ntf/notify-team-permissions-changed cfg team-id))) + (def sql:get-files "select id, features, project_id, is_branch from file where id = ANY(?)") @@ -356,6 +371,14 @@ (recur (rest project-ids) (ct/plus modified-at 10)))) + ;; Moving files can take them out of the reach of the team they came + ;; from. Only the open connections know who was relying on them, so + ;; the source teams are announced wholesale and the watcher re-checks + ;; each member against the subscriptions they actually hold. + (notify-team-access-changed + cfg + (keep (partial perms/get-team-id-for-project conn) source)) + nil)) (def ^:private @@ -425,6 +448,10 @@ ;; delete possible broken relations on moved files (db/exec-one! conn [sql:delete-broken-relations pids]) + ;; The project, and everything under it, leaves the source team. The + ;; destination team only gains access, so it is not announced. + (notify-team-access-changed cfg [(:team-id project)]) + nil)) (def ^:private diff --git a/backend/src/app/rpc/commands/nitrate.clj b/backend/src/app/rpc/commands/nitrate.clj index af52bede6f..61243719c8 100644 --- a/backend/src/app/rpc/commands/nitrate.clj +++ b/backend/src/app/rpc/commands/nitrate.clj @@ -704,7 +704,7 @@ [:map {:title "CheckNitrateSsoParams"} [:team-id {:optional true} ::sm/uuid] [:organization-id {:optional true} ::sm/uuid] - [:url ::sm/uri]] + [:query-string [:string {:max 2048}]]] [::sm/contains-any #{:team-id :organization-id}]]) (sv/defmethod ::check-nitrate-sso @@ -714,40 +714,57 @@ session already holds a valid entry for the organization, and {:authorized true :reason :no-team-access} when the gate was skipped because the user cannot access the team; the reason lets the client tell a usable session - apart from a plain permission failure. + apart from a plain permission failure. When an organization id comes with the + request, the gate is also skipped with + {:authorized true :reason :no-organization-access} when the profile neither + belongs to that organization nor owns it. Returns {:authorized false :redirect-uri } when SSO is active; the client must redirect there. The OIDC provider itself handles re-authentication transparently if the user already has an active SSO session. - A nil :redirect-uri means SSO is required but the provider is not usable." + A nil :redirect-uri means SSO is required but the provider is not usable. + + The :query-string is the route of the client as a query string + (`screen=…¶ms`, without the `?` that separates it from the path; the root + of the application may carry only its params, like `template=…`): the server + builds the destination URL from it, so it can never point outside the public + URI. A value without params of its own is not a route, is ignored with a + warning, and the user lands on the home page." {::rpc/auth true ::doc/added "2.18" ::sm/params schema:check-nitrate-sso ::nitrate/sso false} - [cfg {:keys [::rpc/profile-id team-id organization-id url] :as params}] + [cfg {:keys [::rpc/profile-id team-id organization-id query-string] :as params}] (if (contains? cf/flags :admin-console) (if (and team-id (not (teams/has-read-permissions? cfg profile-id team-id))) ;; Let the destination RPC enforce its own permissions. Starting SSO before ;; access is established sends unrelated users through the organization's IdP. {:authorized true :reason :no-team-access} - (let [request (rph/get-request params) - {:keys [authorized sso]} (nitrate/sso-session-authorized? cfg organization-id team-id request)] - (if authorized - {:authorized true :reason :sso-satisfied} - (if (oidc/organization-sso-discovery-uri sso) - (try - (let [redirect-uri (oidc/build-organization-sso-auth-redirect-uri - cfg sso - :dest-url url - :organization-id organization-id) - organization-id (or organization-id (:organization-id sso))] - (oidc/submit-organization-sso-auth-started-event - cfg request profile-id organization-id) - {:authorized false :redirect-uri redirect-uri}) - (catch Throwable cause - (oidc/submit-organization-sso-auth-failed-event - cfg request profile-id (or organization-id (:organization-id sso)) cause) - (throw cause))) - {:authorized false - :redirect-uri nil})))) + (if (and organization-id + (not (nitrate/organization-access? cfg profile-id organization-id))) + ;; The organization id comes from the client: without this check any + ;; authenticated profile could start a flow for an organization it does + ;; not belong to. + {:authorized true :reason :no-organization-access} + (let [request (rph/get-request params) + {:keys [authorized sso]} (nitrate/sso-session-authorized? cfg organization-id team-id request)] + (if authorized + {:authorized true :reason :sso-satisfied} + (if (oidc/organization-sso-discovery-uri sso) + (try + (let [redirect-uri (oidc/build-organization-sso-auth-redirect-uri + cfg sso + :query-string query-string + :organization-id organization-id + :profile-id profile-id) + organization-id (or organization-id (:organization-id sso))] + (oidc/submit-organization-sso-auth-started-event + cfg request profile-id organization-id) + {:authorized false :redirect-uri redirect-uri}) + (catch Throwable cause + (oidc/submit-organization-sso-auth-failed-event + cfg request profile-id (or organization-id (:organization-id sso)) cause) + (throw cause))) + {:authorized false + :redirect-uri nil}))))) {:authorized true :reason :sso-satisfied})) diff --git a/backend/src/app/rpc/commands/profile.clj b/backend/src/app/rpc/commands/profile.clj index 6d5c90ba13..35b4eff985 100644 --- a/backend/src/app/rpc/commands/profile.clj +++ b/backend/src/app/rpc/commands/profile.clj @@ -70,6 +70,7 @@ [:onboarding-questions {:optional true} [:map-of :keyword :string]] [:onboarding-questions-answered {:optional true} ::sm/boolean] [:nitrate-onboarding-viewed {:optional true} ::sm/boolean] + [:promotional-banner-collapsed {:optional true} ::sm/boolean] [:v2-info-shown {:optional true} ::sm/boolean] [:release-notes-viewed {:optional true} [::sm/text {:max 100}]] diff --git a/backend/src/app/rpc/commands/projects.clj b/backend/src/app/rpc/commands/projects.clj index 3b77b9ba29..ca46503410 100644 --- a/backend/src/app/rpc/commands/projects.clj +++ b/backend/src/app/rpc/commands/projects.clj @@ -22,6 +22,7 @@ [app.rpc.commands.teams :as teams] [app.rpc.doc :as-alias doc] [app.rpc.helpers :as rph] + [app.rpc.notifications :as ntf] [app.rpc.permissions :as perms] [app.rpc.quotes :as quotes] [app.util.services :as sv])) @@ -321,6 +322,12 @@ :profile-id profile-id :project-id id) project (delete-project cfg team id)] + + ;; Deleting the project ends read access for whoever had one of its + ;; files open, so the whole team is announced and the watcher + ;; re-checks each member against the subscriptions they hold. + (ntf/notify-team-permissions-changed cfg (:id team)) + (rph/with-meta (rph/wrap) {::audit/props {:team-id (:team-id project) :name (:name project) diff --git a/backend/src/app/rpc/commands/teams.clj b/backend/src/app/rpc/commands/teams.clj index 85b7e6d2fb..5fb0ade7a7 100644 --- a/backend/src/app/rpc/commands/teams.clj +++ b/backend/src/app/rpc/commands/teams.clj @@ -31,6 +31,7 @@ [app.rpc.commands.profile :as profile] [app.rpc.doc :as-alias doc] [app.rpc.helpers :as rph] + [app.rpc.notifications :as ntf] [app.rpc.permissions :as perms] [app.rpc.quotes :as quotes] [app.setup :as-alias setup] @@ -161,7 +162,8 @@ WHEN 'professional' THEN 'active' ELSE COALESCE(p.props->'~:subscription'->>'~:status', 'incomplete') END, - '~:seats', p.props->'~:subscription'->'~:quantity' + '~:seats', p.props->'~:subscription'->'~:quantity', + '~:owner-id', '~u' || p.id::text ) AS subscription FROM team_profile_rel AS tp JOIN team AS t ON (t.id = tp.team_id) @@ -211,8 +213,9 @@ (dm/with-open [conn (db/open pool)] (let [teams (get-teams conn profile-id)] (if (contains? cf/flags :admin-console) - (->> (nitrate/add-organization-info-to-teams cfg teams params) - (remove #(get-in % [:organization :expired-license]))) + (as-> (nitrate/add-organization-info-to-teams cfg teams params) teams + (remove #(get-in % [:organization :expired-license]) teams) + (nitrate/add-owner-licence-to-teams cfg teams params)) teams)))) (def ^:private sql:get-owned-teams @@ -258,7 +261,8 @@ [cfg {:keys [::rpc/profile-id id file-id] :as params}] (let [team (get-team cfg :profile-id profile-id :team-id id :file-id file-id)] (if (contains? cf/flags :admin-console) - (nitrate/add-organization-info-to-team cfg team params) + (let [team (nitrate/add-organization-info-to-team cfg team params)] + (first (nitrate/add-owner-licence-to-teams cfg [team] params))) team))) (defn- get-organization-owner-viewer-team @@ -268,13 +272,14 @@ [cfg profile-id default-team-id params] (when-let [team-id (perms/resolve-team-id cfg params)] (when (nitrate/organization-owner-of-team? cfg profile-id team-id) + ;; resolve-team-id already proved the team row live; the guard + ;; that used to live here (when-not is-row-deleted?) was dead. (when-let [team (db/get* cfg :team {:id team-id})] - (when-not (db/is-row-deleted? team) - (-> team - (decode-row) - (merge perms/viewer-role-flags) - (assoc :is-default (= team-id default-team-id)) - (process-permissions))))))) + (-> team + (decode-row) + (merge perms/viewer-role-flags) + (assoc :is-default (= team-id default-team-id)) + (process-permissions)))))) (defn get-team [cfg & {:keys [profile-id team-id project-id file-id] :as params}] @@ -822,9 +827,10 @@ :team-id team-id})) (defn leave-team - [{:keys [::db/conn ::mbus/msgbus]} {:keys [profile-id id reassign-to]}] + [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [profile-id id reassign-to]}] (let [perms (get-permissions conn profile-id id) - members (get-team-members conn id)] + members (get-team-members conn id) + team (db/get conn :team {:id id})] (cond ;; we can only proceed if there are more members in the team @@ -873,6 +879,16 @@ (delete-team-rels conn profile-id id) + ;; The leaver is told so, exactly as a removed member is, so the + ;; session stops asking for content it can no longer read. + (mbus/pub! msgbus + :topic profile-id + :message {:type :team-membership-change + :change :removed + :team-id id + :team-name (:name team)}) + (ntf/notify-permissions-changed cfg profile-id) + nil)) (def ^:private schema:leave-team @@ -938,6 +954,10 @@ ::jobs/params {:object :team :deleted-at (:deleted-at team) :id team-id}}) + ;; The team is soft-deleted, so every subscription its members hold + ;; loses its authorization at once. One event: the watcher resolves + ;; the members itself. + (ntf/notify-team-permissions-changed cfg team-id) (rph/with-meta (rph/wrap team) {::audit/props {:organization-id organization-id :team-id team-id @@ -970,7 +990,7 @@ ;; --- Mutation: Team Update Role (defn update-team-member-role - [{:keys [::db/conn ::mbus/msgbus]} {:keys [profile-id team-id member-id role] :as params}] + [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [profile-id team-id member-id role] :as params}] ;; We retrieve all team members instead of query the ;; database for a single member. This is just for ;; convenience, if this becomes a bottleneck or problematic, @@ -1020,6 +1040,12 @@ params {:team-id team-id :profile-id member-id}) + + ;; A role change never removes read access on its own, but the open + ;; subscriptions are re-checked rather than assumed: the watcher + ;; decides from fresh data, which is also what keeps a downgrade to + ;; viewer from being mistaken for a revocation. + (ntf/notify-permissions-changed cfg member-id) nil))) (def ^:private schema:update-team-member-role @@ -1093,6 +1119,11 @@ :team-id team-id :team-name (:name team)})) + ;; Whatever this member may still read through another path (an + ;; organization owner keeps read-only access), the subscriptions they + ;; already hold have to be re-checked, not trusted from here on. + (ntf/notify-permissions-changed cfg member-id) + nil)) ;; --- Mutation: Update Team Photo diff --git a/backend/src/app/rpc/commands/teams_invitations.clj b/backend/src/app/rpc/commands/teams_invitations.clj index 75e5ce2c87..e60f8f26c8 100644 --- a/backend/src/app/rpc/commands/teams_invitations.clj +++ b/backend/src/app/rpc/commands/teams_invitations.clj @@ -474,6 +474,31 @@ "The number of invitations can be sent in a single rpc request" 25) +(defn- check-send-invitations-policy + "Raises `:insufficient-permissions` when the organization of the team does + not let the profile send invitations. The policy covers the pending + invitations too: changing their role, deleting them and minting their token + manage the same access that sending one grants. + + Only applies with the `:admin-console` flag. Then it returns the team with + its organization info, so the caller reuses the same Nitrate answer." + [cfg profile-id team-id perms] + (when (contains? cf/flags :admin-console) + (let [team (db/get-by-id cfg :team team-id) + ;; Strict lookup: a failed Nitrate call must not skip the + ;; organization policy by looking like a team without one. + team (nitrate/add-organization-info-to-team-strict cfg team {::rpc/profile-id profile-id}) + organization (:organization team)] + (when (and organization + (not (cto/allowed? :send-invitations + {:organization-perms organization + :profile-id profile-id + :team-perms perms}))) + (ex/raise :type :validation + :code :insufficient-permissions + :hint "Organization policy does not allow you to send invitations")) + team))) + (sv/defmethod ::create-team-invitations "A rpc call that allows to send single or multiple invitations to join the team. @@ -486,12 +511,10 @@ [cfg {:keys [::rpc/profile-id team-id role emails] :as params}] (let [perms (teams/check-invitation-permissions cfg profile-id team-id) profile (db/get-by-id cfg :profile profile-id) - team (db/get-by-id cfg :team team-id) - ;; Strict lookup: a failed Nitrate call must not skip the - ;; organization policy below by looking like a team without one. - team-with-org (when (contains? cf/flags :admin-console) - (nitrate/add-organization-info-to-team-strict cfg team {::rpc/profile-id profile-id})) - organization (:organization team-with-org) + ;; Reuse the organization lookup of the policy check, so every + ;; organization rule below sees the same Nitrate answer. + team (or (check-send-invitations-policy cfg profile-id team-id perms) + (db/get-by-id cfg :team team-id)) ;; Determine which format is being used using-emails-format? (and emails role) ;; Handle both parameter formats @@ -503,16 +526,6 @@ (count emails) (count (:invitations params)))] - (when (and (contains? cf/flags :admin-console) - organization - (not (cto/allowed? :send-invitations - {:organization-perms organization - :profile-id profile-id - :team-perms perms}))) - (ex/raise :type :validation - :code :insufficient-permissions - :hint "Organization policy does not allow you to send invitations")) - ;; Don't allow promote to owner to admin users. (when (and (not (:is-owner perms)) (or (= role :owner) @@ -536,10 +549,7 @@ ;; Check if the current profile is allowed to send emails (teams/check-profile-muted cfg profile) - (let [;; Reuse the strict organization lookup made for the policy - ;; check above, so every rule sees the same Nitrate answer. - team (or team-with-org team) - ;; NOTE: Is important pass RPC method params down to the + (let [;; NOTE: Is important pass RPC method params down to the ;; `create-team-invitations` because it uses the implicit ;; RPC properties from params for fill necessary data on ;; emiting an entry to the audit-log @@ -647,6 +657,9 @@ :code :cant-promote-to-owner :hint "only the team owner can get an owner invitation token")) + ;; Handing out the link invites as much as sending the email does. + (check-send-invitations-policy cfg profile-id team-id perms) + (let [member (profile/get-profile-by-email pool (:email-to invit)) token (create-invitation-token cfg {:team-id (:team-id invit) ;; The inviter is the creator of @@ -678,17 +691,20 @@ (sv/defmethod ::update-team-invitation-role {::doc/added "1.17" ::doc/module :teams - ::sm/params schema:update-team-invitation-role - ::db/transaction true} - [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email role] :as params}] - (let [perms (teams/check-invitation-permissions conn profile-id team-id)] + ::sm/params schema:update-team-invitation-role} + [cfg {:keys [::rpc/profile-id team-id email role] :as params}] + (let [perms (teams/check-invitation-permissions cfg profile-id team-id)] ;; Don't allow promote to owner to admin users. (when (and (not (:is-owner perms)) (= role :owner)) (ex/raise :type :validation :code :cant-promote-to-owner)) - (db/update! conn :team-invitation + ;; Checked outside a transaction: it may call Nitrate over HTTP, and the + ;; update below is a single statement. + (check-send-invitations-policy cfg profile-id team-id perms) + + (db/update! cfg :team-invitation {:role (name role) :updated-at (ct/now)} {:team-id team-id :email-to (profile/clean-email email)}) @@ -703,16 +719,19 @@ (sv/defmethod ::delete-team-invitation {::doc/added "1.17" - ::sm/params schema:delete-team-invition - ::db/transaction true} - [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email] :as params}] - (teams/check-invitation-permissions conn profile-id team-id) + ::sm/params schema:delete-team-invition} + [cfg {:keys [::rpc/profile-id team-id email] :as params}] + (let [perms (teams/check-invitation-permissions cfg profile-id team-id)] - (let [invitation (db/delete! conn :team-invitation - {:team-id team-id - :email-to (profile/clean-email email)} - {::db/return-keys true})] - (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}}))) + ;; Checked outside a transaction: it may call Nitrate over HTTP, and the + ;; delete below is a single statement. + (check-send-invitations-policy cfg profile-id team-id perms) + + (let [invitation (db/delete! cfg :team-invitation + {:team-id team-id + :email-to (profile/clean-email email)} + {::db/return-keys true})] + (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}})))) ;; --- Mutation: Request Team Invitation @@ -770,6 +789,9 @@ (-> file (dissoc :data) (dissoc :deleted-at) + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (bfc/file->response) (assoc :page-id (-> file :data :pages first))))) (def ^:private schema:create-team-access-request diff --git a/backend/src/app/rpc/commands/viewer.clj b/backend/src/app/rpc/commands/viewer.clj index 55c2d908a3..b9497b79b0 100644 --- a/backend/src/app/rpc/commands/viewer.clj +++ b/backend/src/app/rpc/commands/viewer.clj @@ -135,7 +135,10 @@ (defn- get-view-only-bundle [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id share-id ::perms] :as params}] - (let [file (bfc/get-file cfg file-id) + (let [;; The storage pointer is internal bookkeeping: this bundle is an RPC + ;; response and must not expose it. + file (-> (bfc/get-file cfg file-id) + (bfc/file->response)) ;; pull request review sandbox: the viewer shows the pinned ;; review snapshot of the branch, not its live state @@ -175,7 +178,8 @@ libs (->> (bfc/get-file-libraries conn file-id) (mapv (fn [{:keys [id] :as lib}] - (merge lib (bfc/get-file cfg id))))) + (-> (merge lib (bfc/get-file cfg id)) + (bfc/file->response))))) ;; A share link never grants pages of a linked library, so trim ;; each library to the components the allowed pages reference. diff --git a/backend/src/app/rpc/notifications.clj b/backend/src/app/rpc/notifications.clj index 00d6b8548e..b3ffd1be65 100644 --- a/backend/src/app/rpc/notifications.clj +++ b/backend/src/app/rpc/notifications.clj @@ -6,6 +6,8 @@ (ns app.rpc.notifications (:require + [app.common.uuid :as uuid] + [app.db :as db] [app.msgbus :as mbus])) (defn notify-team-change @@ -49,3 +51,68 @@ :topic organization-id :message {:type :organization-change-sso :organization-id organization-id}))) + +;; --- Subscription revocation + +;; A websocket subscription is authorized once, when it is opened, and +;; that single decision is trusted for as long as the connection lives. +;; A mutation that can take access away announces it here, and every +;; backend re-verifies the subscriptions it owns for that profile. +;; +;; The announcement travels over the message bus precisely because the +;; connection registry is local to one backend instance, while the RPC +;; that revokes access runs on whichever instance received the request, +;; which is usually not the one holding the socket. + +(def internal-revocation-topic + "Message-bus topic carrying backend-internal revocation events. + + A fixed uuid that is never a real resource id, so it cannot collide + with a file, team or profile topic — and deliberately not the + `uuid/zero` system topic, which is piped straight into the client + output channel, so anything published there would reach every + connected client. + + No client can subscribe here. `:subscribe-file` and `:subscribe-team` + only accept ids that first pass a permission check, so a client can + never make the bus deliver this topic to it." + (uuid/uuid "ffffffff-ffff-ffff-ffff-ffffffffffff")) + +(defn notify-permissions-changed + "Asks every backend to re-verify the websocket subscriptions currently + held by `profile-id`. + + Deferred past the commit: the watcher decides from a fresh permission + check on another connection, so announcing before the transaction + commits would let it see the old state and close nothing. Outside a + transaction the announcement goes out immediately. + + The watcher always decides from a fresh permission check, so an extra + announcement is safe: it closes nothing whose access still holds." + [cfg profile-id] + (db/after-commit + (fn [] + (let [msgbus (::mbus/msgbus cfg)] + (mbus/pub! msgbus + :topic internal-revocation-topic + :message {:type :profile-permissions-changed + :profile-id profile-id}))))) + +(defn notify-team-permissions-changed + "Asks every backend to re-verify the subscriptions of every member of + `team-id`. + + Used when a change can move a resource out of the reach of a whole + team (moving, deleting or soft-deleting files, projects and teams). One + event per team: the watcher resolves the members itself, so a large + team cannot overflow the bounded publish buffers on the way out. + + Like `notify-permissions-changed`, deferred past the commit." + [cfg team-id] + (db/after-commit + (fn [] + (let [msgbus (::mbus/msgbus cfg)] + (mbus/pub! msgbus + :topic internal-revocation-topic + :message {:type :team-permissions-changed + :team-id team-id}))))) diff --git a/backend/src/app/rpc/permissions.clj b/backend/src/app/rpc/permissions.clj index 77653017db..8dcf79c604 100644 --- a/backend/src/app/rpc/permissions.clj +++ b/backend/src/app/rpc/permissions.clj @@ -108,13 +108,16 @@ :can-edit false}) (def ^:private sql:get-team-id-for-project - "SELECT team_id FROM project WHERE id = ?") + "SELECT team_id FROM project WHERE id = ? + AND deleted_at IS NULL") (def ^:private sql:get-team-id-for-file "SELECT p.team_id FROM file AS f JOIN project AS p ON (p.id = f.project_id) - WHERE f.id = ?") + WHERE f.id = ? + AND f.deleted_at IS NULL + AND p.deleted_at IS NULL") (defn get-team-id-for-project [cfg project-id] @@ -127,11 +130,18 @@ (:team-id))) (defn resolve-team-id + "Resolves the team that owns the project/file `params` references, or + `params`' own `:team-id`. A soft-deleted file/project/team resolves to + nil: no caller may rely on a dead team for access decisions. The team + row is checked for all branches — the SQL filters cover file and + project, this final check covers the team itself." [cfg {:keys [team-id project-id file-id]}] - (cond - (some? team-id) team-id - (some? project-id) (get-team-id-for-project cfg project-id) - (some? file-id) (get-team-id-for-file cfg file-id))) + (let [resolved-team-id (cond (some? team-id) team-id + (some? project-id) (get-team-id-for-project cfg project-id) + (some? file-id) (get-team-id-for-file cfg file-id))] + (when (and resolved-team-id + (some? (db/get* cfg :team {:id resolved-team-id}))) + resolved-team-id))) (defn get-organization-owner-permissions "When `profile-id` is a non-member owner of the organization that owns diff --git a/backend/src/app/rpc/quotes.clj b/backend/src/app/rpc/quotes.clj index 4372007f53..eb742ce718 100644 --- a/backend/src/app/rpc/quotes.clj +++ b/backend/src/app/rpc/quotes.clj @@ -423,6 +423,8 @@ (sm/lazy-validator schema:snapshots-per-file)) (def ^:private sql:get-snapshots-per-file + ;; A version is told apart by its label, and the partial index with the same + ;; predicate answers the count without touching the table. "SELECT count(*) AS total FROM file_change AS fc WHERE fc.file_id = ? @@ -454,11 +456,16 @@ (sm/lazy-validator schema:snapshots-per-team)) (def ^:private sql:get-snapshots-per-team + ;; The files of the team are resolved first, so the count filters by + ;; `file_id`: the planner can prune the partitions of `file_change` when it + ;; chooses an index scan (with a sequential scan it reads all of them). A + ;; version is told apart by its label. "SELECT count(*) AS total FROM file_change AS fc - JOIN file AS f ON (f.id = fc.file_id) - JOIN project AS p ON (p.id = f.project_id) - WHERE p.team_id = ? + WHERE fc.file_id = ANY (SELECT f.id + FROM file AS f + JOIN project AS p ON (p.id = f.project_id) + WHERE p.team_id = ?) AND fc.created_by = 'user' AND fc.deleted_at IS NULL AND fc.label IS NOT NULL") diff --git a/backend/src/app/scripts/decode_file_data.clj b/backend/src/app/scripts/decode_file_data.clj index b7d775eafc..a732a34fc1 100644 --- a/backend/src/app/scripts/decode_file_data.clj +++ b/backend/src/app/scripts/decode_file_data.clj @@ -39,6 +39,7 @@ `[.data[\"pages-index\"][] | {id, name}]`, etc." (:require [app.binfile.common :as bfc] + [app.common.exceptions :as ex] [app.common.json :as json] [app.db :as db] [app.features.file-snapshots :as fsnap] @@ -183,16 +184,23 @@ (defn get-snapshot-json "Return a snapshot as JSON (realized). For nREPL. - Takes the same `format` and `output` as `get-file-json`." + Takes the same `format` and `output` as `get-file-json`. It raises when the + file has no snapshot with that id." [system file-id snapshot-id & {:keys [format output] :or {format :full}}] (let [snapshot (db/run! system (fn [cfg] (fsnap/get-snapshot cfg file-id snapshot-id))) - data (case format - :full snapshot - :data (:data snapshot) - :summary (summarize-data (:data snapshot)) - :pages (:pages (summarize-data (:data snapshot))))] + data (if snapshot + (case format + :full snapshot + :data (:data snapshot) + :summary (summarize-data (:data snapshot)) + :pages (:pages (summarize-data (:data snapshot)))) + (ex/raise :type :not-found + :code :snapshot-not-found + :hint "unable to find snapshot with the provided id" + :snapshot-id snapshot-id + :file-id file-id))] (write-output (data->json data) output))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index bae8f2f8ca..25a1265c0a 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -192,5 +192,5 @@ :created-by "system"})) (let [file' (update file' :revn inc)] - (bfc/update-file! system file' opts) + (bfc/update-file! system file' ::bfc/reset-migrations? (::bfc/reset-migrations? opts)) true)))) diff --git a/backend/src/app/storage/gc_deleted.clj b/backend/src/app/storage/gc_deleted.clj index aacc2d7ac3..378649d3c6 100644 --- a/backend/src/app/storage/gc_deleted.clj +++ b/backend/src/app/storage/gc_deleted.clj @@ -118,6 +118,16 @@ [(ct/now) chunk-size]])] (db/exec! conn (into [sql] params)))) +(defn- server-error + "Return the PostgreSQL server error of a foreign key violation, or nil. It + carries the constraint and the referencing table, which is what the operator + needs: any `NO ACTION` foreign key on `storage_object` can block the delete, + not only the `file_data` one." + [cause] + (when (and (instance? org.postgresql.util.PSQLException cause) + (= "23503" (.getSQLState ^java.sql.SQLException cause))) + (.getServerErrorMessage ^org.postgresql.util.PSQLException cause))) + (defn- clean-deleted "Processes the pending objects chunk by chunk with keyset pagination: each chunk is one short transaction that reads and locks the next `chunk-size` @@ -127,11 +137,16 @@ marked as deleted. A chunk that fails aborts its transaction: nothing of the chunk is deleted, - its locks are released, and it is logged and skipped. The cursor advances - to the `(deleted_at, id)` of the last row of each read chunk regardless of - the deletion result, so a failed chunk is not visited again in the same - run. There is no long-lived read transaction: each transaction lives only - as long as one chunk, letting the vacuum proceed between chunks. Rows + its locks are released, and it is logged and skipped. A chunk blocked by a + foreign key (SQLState 23503) keeps its log at error level with a hint and the + constraint and table the database reports, and is retried on the next run; + the reference must be fixed for the chunk to make progress. Any `NO ACTION` + foreign key on `storage_object` can block the delete, not only the one of + `file_data`. The cursor advances to the `(deleted_at, id)` of + the last row of each read chunk regardless of the deletion result, so a + failed chunk is not visited again in the same run. There is no long-lived + read transaction: each transaction lives only as long as one chunk, letting + the vacuum proceed between chunks. Rows locked by a concurrent run are skipped and deferred to the next run; a chunk that fails after a partial backend delete self-heals on the next run (physical deletes are idempotent). Every `throttle-every` deleted @@ -155,9 +170,22 @@ ;; afterwards is a JDBC no-op on the already ;; rolled-back connection. (db/rollback! conn) - (l/err :hint "unexpected error on chunk deletion" - :ids (map :id rows) - :cause cause) + (if-let [serr (server-error cause)] + ;; A NO ACTION foreign key refused a physical + ;; delete: a live row of another table still + ;; references an object marked for deletion. + ;; That is a real anomaly (the reference scan + ;; should have frozen it), so it stays an + ;; error and the chunk is retried on the next + ;; run until the reference is fixed. + (l/err :hint "chunk deletion blocked by a live reference; fix the reference and let the next run reclaim the chunk" + :ids (map :id rows) + :constraint (.getConstraint ^org.postgresql.util.ServerErrorMessage serr) + :table (.getTable ^org.postgresql.util.ServerErrorMessage serr) + :cause cause) + (l/err :hint "unexpected error on chunk deletion" + :ids (map :id rows) + :cause cause)) [rows {}])))))] ;; Reported only after the chunk commits: a rolled-back chunk leaves diff --git a/backend/src/app/storage/gc_touched.clj b/backend/src/app/storage/gc_touched.clj index 80e411d9da..c4e4c463e9 100644 --- a/backend/src/app/storage/gc_touched.clj +++ b/backend/src/app/storage/gc_touched.clj @@ -108,16 +108,13 @@ (def sql:exists-file-data-refs "SELECT EXISTS ( SELECT 1 FROM file_data - WHERE file_id = ? - AND id = ? - AND metadata->>'storage-ref-id' = ?::text + WHERE storage_object_id = ? ) AS has_refs") (defn- has-file-data-refs? [conn sobject] - (let [{:keys [file-id id]} (:metadata sobject)] - (-> (db/exec-one! conn [sql:exists-file-data-refs file-id id (:id sobject)]) - (get :has-refs)))) + (-> (db/exec-one! conn [sql:exists-file-data-refs (:id sobject)]) + (get :has-refs))) (def ^:private sql:mark-freeze-in-bulk "UPDATE storage_object diff --git a/backend/src/app/tasks/delete_object.clj b/backend/src/app/tasks/delete_object.clj index 31fb998b5d..2f35824420 100644 --- a/backend/src/app/tasks/delete_object.clj +++ b/backend/src/app/tasks/delete_object.clj @@ -53,7 +53,8 @@ :deleted-at (ct/format-inst deleted-at)) (db/update! conn :file-change - {:deleted-at deleted-at} + {:deleted-at deleted-at + :updated-at (ct/now)} {:id id :file-id file-id} {::db/return-keys false}) @@ -139,7 +140,8 @@ ;; lifetime; the collector would still take it, just much later than it ;; should. (db/update! conn :file-change - {:deleted-at deleted-at} + {:deleted-at deleted-at + :updated-at (ct/now)} {:file-id id :deleted-at nil} {::db/return-keys false}) diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 1fca7bf321..8f4ff44a98 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -613,7 +613,7 @@ ORDER BY deleted_at ASC") (def ^:private sql:lock-file-data - "SELECT file_id, id, type, deleted_at, metadata, backend + "SELECT file_id, id, type, deleted_at, backend, storage_object_id FROM file_data WHERE (file_id, id) IN (SELECT file_id, id FROM unnest(?::uuid[], ?::uuid[]) AS u(file_id, id)) AND deleted_at IS NOT NULL @@ -630,11 +630,10 @@ (let [file-ids (db/create-array conn "uuid" (map :file-id rows)) ids (db/create-array conn "uuid" (map :id rows))] (->> (db/exec! conn [sql:lock-file-data file-ids ids threshold]) - (reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}] + (reduce (fn [total {:keys [file-id id type deleted-at backend storage-object-id]}] - (some->> metadata - (fdata/decode-metadata) - (fdata/process-metadata cfg)) + (some->> storage-object-id + (fdata/touch-storage-object cfg)) (l/trc :obj "file-data" :id (str id) @@ -673,41 +672,48 @@ {:fetch-size 5})))))) (def ^:private sql:get-file-change - "SELECT id FROM file_change + "SELECT id, file_id FROM file_change WHERE deleted_at IS NOT NULL AND deleted_at <= ? ORDER BY deleted_at ASC") (def ^:private sql:lock-file-change - "SELECT id, file_id, deleted_at FROM file_change - WHERE id = ANY(?::uuid[]) + "SELECT file_id, id, deleted_at FROM file_change + WHERE file_id = ? + AND id = ANY(?::uuid[]) AND deleted_at IS NOT NULL AND deleted_at <= ? FOR UPDATE SKIP LOCKED") (defn- delete-file-changes-chunk - [cfg ids] + [cfg rows] (try (db/tx-run! cfg (fn [{:keys [::db/conn ::threshold]}] (db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"]) - (let [ids (db/create-array conn "uuid" ids)] - (->> (db/exec! conn [sql:lock-file-change ids threshold]) - (reduce (fn [total {:keys [id file-id deleted-at]}] - (l/trc :obj "file-change" - :id (str id) - :file-id (str file-id) - :deleted-at (ct/format-inst deleted-at)) + ;; The rows are locked per file, with a constant `file_id`, + ;; so the planner prunes to the partition that holds each + ;; one instead of scanning all of them for a tuple `IN`. + (->> (group-by :file-id rows) + (mapcat (fn [[file-id file-rows]] + (let [ids (db/create-array conn "uuid" (map :id file-rows))] + (db/exec! conn [sql:lock-file-change file-id ids threshold])))) + (reduce (fn [total {:keys [id file-id deleted-at]}] + (l/trc :obj "file-change" + :id (str id) + :file-id (str file-id) + :deleted-at (ct/format-inst deleted-at)) - (let [affected (-> (db/delete! conn :file-change {:id id}) - (db/get-update-count))] - (+ total affected))) - 0))))) + (let [affected (-> (db/delete! conn :file-change + {:id id :file-id file-id}) + (db/get-update-count))] + (+ total affected))) + 0)))) (catch Throwable cause (l/err :obj "file-change" :hint "error on chunk deletion, skipping it" - :chunk ids + :chunk rows :cause cause) 0))) @@ -720,7 +726,10 @@ (+ total deleted))))] (db/tx-run! (assoc cfg ::db/read-only true) (fn [{:keys [::db/conn]}] - (transduce (comp (map :id) (partition-all chunk-size)) rf 0 + (transduce (comp (map (fn [row] {:id (:id row) + :file-id (:file-id row)})) + (partition-all chunk-size)) + rf 0 (db/plan conn [sql:get-file-change threshold] {:fetch-size 5})))))) diff --git a/backend/src/app/tasks/offload_file_data.clj b/backend/src/app/tasks/offload_file_data.clj index e8e3eb12ff..816a272ccb 100644 --- a/backend/src/app/tasks/offload_file_data.clj +++ b/backend/src/app/tasks/offload_file_data.clj @@ -28,18 +28,33 @@ (def ^:private sql:lock-file ;; NOTE: SKIP LOCKED only serializes against writers that also lock - ;; the file row. Every file_data writer must therefore hold at least - ;; FOR SHARE on it (the update RPC via lock-for-share?, file-gc via - ;; FOR UPDATE); a writer skipping that lock would reopen the - ;; lost-update race this lock closes. + ;; the file row. Every file_data writer must therefore lock it too: + ;; the update RPC and the snapshot paths via `lock-for-update?`, and + ;; file-gc via FOR UPDATE. A writer skipping that lock would reopen + ;; the lost-update race this lock closes. "SELECT id FROM file WHERE id = ? AND deleted_at IS NULL FOR UPDATE SKIP LOCKED") +(defn- decode-row + "Normalize a `file_data` row read with `SELECT fd.*`: NULL columns are + absent (the db layer uses `next.jdbc.optional`) and jsonb metadata arrives + as a `PGobject`, while `fdata/upsert!` expects a decoded metadata map. A + JSON `null` decodes to nil, and the key has to disappear rather than travel + as `:metadata nil`, which the schema rejects." + [row] + (let [metadata (some-> (:metadata row) fdata/decode-metadata)] + (cond-> (dissoc row :metadata) + (some? metadata) (assoc :metadata metadata)))) + (defn- offload-file-data "Offload one hot-storage row to cold storage. Returns the row type and the migrated bytes for the caller to aggregate." [cfg {:keys [id file-id type data] :as fdata}] (jobs/heartbeat cfg) - (fdata/upsert! cfg (assoc fdata :backend "storage")) + ;; The pointer key is normalized here because a NULL `storage_object_id` + ;; is omitted by the row builder; `decode-row` already normalized metadata. + (fdata/upsert! cfg (assoc fdata + :backend "storage" + :storage-object-id (:storage-object-id fdata))) (let [size (if (bytes? data) (alength ^bytes data) 0)] (l/trc :hint "file data row offloaded" :file-id (str file-id) @@ -61,16 +76,19 @@ [{:keys [::db/conn] :as cfg} file-id] (if (db/exec-one! conn [sql:lock-file file-id]) (let [{:keys [rows bytes by-type]} - (reduce (fn [stats fdata] - (let [{:keys [type size]} (offload-file-data cfg fdata)] - (-> stats - (update :rows inc) - (update :bytes + size) - (update-in [:by-type type] - (fn [{r :rows b :bytes :or {r 0 b 0}}] - {:rows (inc r) :bytes (+ b size)}))))) - {:rows 0 :bytes 0 :by-type {}} - (db/plan conn [sql:get-file-data file-id]))] + (transduce + (map decode-row) + (completing + (fn [stats fdata] + (let [{:keys [type size]} (offload-file-data cfg fdata)] + (-> stats + (update :rows inc) + (update :bytes + size) + (update-in [:by-type type] + (fn [{r :rows b :bytes :or {r 0 b 0}}] + {:rows (inc r) :bytes (+ b size)})))))) + {:rows 0 :bytes 0 :by-type {}} + (db/plan conn [sql:get-file-data file-id]))] (l/trc :hint "file data offloaded" :file-id (str file-id) :offloaded rows diff --git a/backend/test/backend_tests/auth_oidc_test.clj b/backend/test/backend_tests/auth_oidc_test.clj index ea249d3144..8b624a6fbb 100644 --- a/backend/test/backend_tests/auth_oidc_test.clj +++ b/backend/test/backend_tests/auth_oidc_test.clj @@ -10,6 +10,7 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.time :as ct] + [app.common.uri :as u] [app.common.uuid :as uuid] [app.config :as cf] [app.http.session :as session] @@ -448,6 +449,76 @@ loc (redirect-location result)] (t/is (.contains loc "error=registration-disabled"))))))) +(t/deftest callback-with-invitation-and-no-flags-redirects-to-register + ;; Review of #12974: a verifiable invitation issued for the OIDC + ;; email acts as a substitute for the SSO registration flags, so the + ;; user reaches the register screen instead of the + ;; registration-disabled error. The JWT is verified and the email is + ;; compared here; the invitation row is revalidated under a row lock + ;; in `register-profile`. + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + invitation (tokens/generate cfg {:iss :team-invitation + :exp (ct/in-future {:hours 1}) + :member-email "u@e.com"}) + state (make-state-token cfg {:invitation-token invitation}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "http://localhost:3449"} + cf/flags #{}] + (with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"}) + app.auth.oidc/get-info (constantly {:email "u@e.com" :fullname "U" + :backend "oidc" :email-verified false + :invitation-token invitation + :props {}}) + app.auth.oidc/get-profile (constantly nil)] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result) + claims (tokens/decode cfg (location-token loc))] + (t/is (.contains loc "screen=auth-register-validate")) + (t/is (.contains loc "token=")) + (t/is (= :prepared-register (:iss claims)) + "the redirect token must be a :prepared-register token") + (t/is (= invitation (:invitation-token claims)) + "the invitation token must travel inside the prepared-register claims")))))) + +(t/deftest callback-with-invitation-email-mismatch-redirects-to-registration-disabled + ;; Review of #12974: an invitation issued for another email must not + ;; unlock registration with closed registration. + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + invitation (tokens/generate cfg {:iss :team-invitation + :exp (ct/in-future {:hours 1}) + :member-email "alice@example.com"}) + state (make-state-token cfg {:invitation-token invitation}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "http://localhost:3449"} + cf/flags #{}] + (with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"}) + app.auth.oidc/get-info (constantly {:email "attacker@example.com" :fullname "A" + :backend "oidc" :email-verified false + :invitation-token invitation + :props {}}) + app.auth.oidc/get-profile (constantly nil)] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result)] + (t/is (.contains loc "error=registration-disabled"))))))) + +(t/deftest callback-with-garbage-invitation-redirects-to-registration-disabled + ;; Review of #12974: raw presence is not enough — an unverifiable + ;; invitation string must not reach the register screen. + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + state (make-state-token cfg {:invitation-token "inv-123"}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "http://localhost:3449"} + cf/flags #{}] + (with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"}) + app.auth.oidc/get-info (constantly {:email "u@e.com" :fullname "U" + :backend "oidc" :email-verified false + :invitation-token "inv-123" + :props {}}) + app.auth.oidc/get-profile (constantly nil)] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result)] + (t/is (.contains loc "error=registration-disabled"))))))) + (t/deftest callback-profile-blocked (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) state (make-state-token cfg {}) @@ -578,69 +649,73 @@ (t/deftest organization-sso-callback-success-emits-succeeded ;; No session on the request → auth-succeeded only (no token-issued). (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) - state (make-state-token cfg {:dest-url "https://penpot.example.com/#/workspace" + state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace" :organization-id test-organization-id}) request (default-request cfg :state state) events (atom [])] - (with-redefs [app.nitrate/call (constantly {:active true}) - app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) - app.auth.oidc/get-info (constantly {}) - app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] - (let [result (#'oidc/callback-handler cfg request)] - (t/is (= "https://penpot.example.com/#/workspace" (redirect-location result))) - (t/is (= ["organization-sso-auth-succeeded"] (mapv :name @events))) - (t/is (= test-organization-id (get-in (first @events) [:props :organization-id]))))))) + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (with-redefs [app.nitrate/call (constantly {:active true}) + app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + app.auth.oidc/get-info (constantly {}) + app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] + (let [result (#'oidc/callback-handler cfg request)] + (t/is (= "https://penpot.example.com/?screen=workspace" (redirect-location result))) + (t/is (= ["organization-sso-auth-succeeded"] (mapv :name @events))) + (t/is (= test-organization-id (get-in (first @events) [:props :organization-id])))))))) (t/deftest organization-sso-callback-success-with-session-emits-token-issued (let [manager (session/inmemory-manager) cfg (assoc (dissoc base-cfg :app.email/blacklist :app.email/whitelist) ::session/manager manager) - state (make-state-token cfg {:dest-url "https://penpot.example.com/#/workspace" - :organization-id test-organization-id}) + state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace" + :organization-id test-organization-id + :profile-id test-profile-id}) request (default-request cfg :state state) events (atom []) sess {:id (uuid/next) :profile-id test-profile-id :props {}}] - (with-redefs [app.nitrate/call (constantly {:active true}) - app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) - app.auth.oidc/get-info (constantly {}) - app.http.session/get-session (constantly sess) - app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] - (let [result (#'oidc/callback-handler cfg request) - names (mapv :name @events) - stored (session/read-session manager (:id sess))] - (t/is (= "https://penpot.example.com/#/workspace" (redirect-location result))) - (t/is (= ["organization-sso-token-issued" - "organization-sso-auth-succeeded"] - names)) - (t/is (= test-organization-id - (get-in (first @events) [:props :organization-id]))) - (t/is (= test-profile-id (:profile-id (first @events)))) - (t/is (= #{:organization-id} - (set (keys (:props (first @events)))))) - (t/is (contains? (get-in stored [:props :sso]) test-organization-id)))))) + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (with-redefs [app.nitrate/call (constantly {:active true}) + app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + app.auth.oidc/get-info (constantly {}) + app.http.session/get-session (constantly sess) + app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] + (let [result (#'oidc/callback-handler cfg request) + names (mapv :name @events) + stored (session/read-session manager (:id sess))] + (t/is (= "https://penpot.example.com/?screen=workspace" (redirect-location result))) + (t/is (= ["organization-sso-token-issued" + "organization-sso-auth-succeeded"] + names)) + (t/is (= test-organization-id + (get-in (first @events) [:props :organization-id]))) + (t/is (= test-profile-id (:profile-id (first @events)))) + (t/is (= #{:organization-id} + (set (keys (:props (first @events)))))) + (t/is (contains? (get-in stored [:props :sso]) test-organization-id))))))) (t/deftest organization-sso-callback-error-emits-failed (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) - state (make-state-token cfg {:dest-url "https://penpot.example.com/#/workspace" + state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace" :organization-id test-organization-id}) request (default-request cfg :state state) events (atom [])] - (with-redefs [app.nitrate/call (fn [_cfg method _params] - (case method - :get-organization-sso {:active true} - :get-organization-summary {:name "Organization"})) - app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) - app.auth.oidc/get-info (fn [& _] - (ex/raise :type :internal - :code :unable-to-retrieve-user-info)) - app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] - (#'oidc/callback-handler cfg request) - (t/is (= ["organization-sso-auth-failed"] (mapv :name @events))) - (t/is (= {:organization-id test-organization-id - :failure-reason "user-info-failed"} - (:props (first @events))))))) + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (with-redefs [app.nitrate/call (fn [_cfg method _params] + (case method + :get-organization-sso {:active true} + :get-organization-summary {:name "Organization"})) + app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + app.auth.oidc/get-info (fn [& _] + (ex/raise :type :internal + :code :unable-to-retrieve-user-info)) + app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] + (#'oidc/callback-handler cfg request) + (t/is (= ["organization-sso-auth-failed"] (mapv :name @events))) + (t/is (= {:organization-id test-organization-id + :failure-reason "user-info-failed"} + (:props (first @events)))))))) (t/deftest organization-sso-oauth-error-emits-failed-without-changing-redirect (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) @@ -659,6 +734,155 @@ :failure-reason "access-denied"} (:props (first @events))))))))) +(t/deftest build-dest-url-builds-the-destination-on-the-public-uri + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (t/testing "the route is appended to the public uri, separator included" + (t/is (= "https://penpot.example.com/?screen=workspace&team-id=0e5f" + (oidc/build-dest-url "screen=workspace&team-id=0e5f")))) + + (t/testing "the app base path of the public uri is kept" + (binding [cf/config {:public-uri "https://host/penpot"}] + (t/is (= "https://host/penpot/?screen=workspace" + (oidc/build-dest-url "screen=workspace"))))) + + (t/testing "a blank query string lands on the public uri" + (t/is (= "https://penpot.example.com/" (oidc/build-dest-url ""))) + (t/is (= "https://penpot.example.com/" (oidc/build-dest-url nil)))) + + (t/testing "a value without params of its own is not a route" + (doseq [query-string ["https://evil.example/" + "//evil.example/x"]] + (t/is (= "https://penpot.example.com/" + (oidc/build-dest-url query-string)) query-string))) + + (t/testing "the root of the app keeps its params (invitation, template)" + (t/is (= "https://penpot.example.com/?template=xyz" + (oidc/build-dest-url "template=xyz")))) + + (t/testing "a value that still carries the separator is trimmed" + (t/is (= "https://penpot.example.com/?screen=workspace" + (oidc/build-dest-url "?screen=workspace")))) + + (t/testing "control characters never reach the destination" + (let [dest (oidc/build-dest-url "screen=x\r\nSet-Cookie: a=b")] + (t/is (not (.contains dest "\r"))) + (t/is (not (.contains dest "\n"))))) + + (t/testing "repeated params are kept" + (t/is (= "https://penpot.example.com/?a=1&a=2" + (oidc/build-dest-url "a=1&a=2")))))) + +(def ^:private foreign-dest-urls + "Destinations that a naive prefix check would let through: the first one + parses the prefix as userinfo, the second one is a look-alike host." + ["https://evil.example/" + "https://penpot.example.com@evil.example/" + "https://penpot.example.com.evil.example/"]) + +(t/deftest build-organization-sso-auth-redirect-uri-seals-a-server-built-destination + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + provider {:id "oidc" + :type "oidc" + :client-id "client" + :auth-uri "https://idp.example.com/authorize" + :scopes ["openid"]} + sso {:active true + :issuer "https://idp.example.com" + :organization-id test-organization-id}] + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (doseq [[query-string expected] + [["screen=workspace" "https://penpot.example.com/?screen=workspace"] + ["template=xyz" "https://penpot.example.com/?template=xyz"] + ["https://evil.example/" "https://penpot.example.com/"] + ["" "https://penpot.example.com/"]]] + (t/testing (str "the destination sealed for " (pr-str query-string)) + (let [uri (oidc/build-organization-sso-auth-redirect-uri + cfg sso + :query-string query-string + :organization-id test-organization-id + :profile-id test-profile-id + :provider provider) + claims (tokens/decode cfg (-> (u/uri uri) + :query + (u/query-string->map) + :state))] + ;; The destination is always composed by the server: the route + ;; of the client, never a URL of its own. + (t/is (= expected (:dest-url claims))) + (t/is (= test-profile-id (:profile-id claims))))))))) + +(t/deftest organization-sso-callback-ignores-a-destination-outside-the-public-uri + (doseq [dest-url foreign-dest-urls] + (t/testing (str "the success path ignores " dest-url) + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + state (make-state-token cfg {:dest-url dest-url + :organization-id test-organization-id}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (with-redefs [app.nitrate/call (constantly {:active true}) + app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + app.auth.oidc/get-info (constantly {})] + (let [result (#'oidc/callback-handler cfg request)] + (t/is (= "https://penpot.example.com/" (redirect-location result)))))))) + + (t/testing (str "the error path ignores " dest-url " as well") + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + state (make-state-token cfg {:dest-url dest-url + :organization-id test-organization-id}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (with-redefs [app.nitrate/call (fn [_cfg method _params] + (case method + :get-organization-sso {:active true} + :get-organization-summary {:name "Organization"})) + app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + app.auth.oidc/get-info (fn [& _] + (ex/raise :type :internal + :code :unable-to-retrieve-user-info))] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result)] + (t/is (not (.contains loc "evil.example"))) + (t/is (.contains loc "https://penpot.example.com/"))))))))) + +(t/deftest trusted-dest-url-requires-the-base-path + (binding [cf/config {:public-uri "https://host/penpot"}] + (t/testing "a destination under the app base path is trusted" + (t/is (= "https://host/penpot/?screen=workspace" + (#'oidc/trusted-dest-url "https://host/penpot/?screen=workspace")))) + + (t/testing "a destination in another path of the same host is not" + (t/is (nil? (#'oidc/trusted-dest-url "https://host/otra/?screen=workspace")))) + + (t/testing "a destination in another host is not" + (t/is (nil? (#'oidc/trusted-dest-url "https://otro.example/penpot/?screen=workspace")))))) + +(t/deftest organization-sso-callback-does-not-grant-the-sso-entry-to-another-session + (let [manager (session/inmemory-manager) + cfg (assoc (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + ::session/manager manager) + ;; The state was prepared by another profile: the session that + ;; completes the flow must not be granted the organization entry. + state (make-state-token cfg {:dest-url "https://penpot.example.com/?screen=workspace" + :organization-id test-organization-id + :profile-id (uuid/random)}) + request (default-request cfg :state state) + events (atom []) + sess {:id (uuid/next) + :profile-id test-profile-id + :props {}}] + (binding [cf/config {:public-uri "https://penpot.example.com"}] + (with-redefs [app.nitrate/call (constantly {:active true}) + app.auth.oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + app.auth.oidc/get-info (constantly {}) + app.http.session/get-session (constantly sess) + app.loggers.audit/submit (fn [_cfg event] (swap! events conj event))] + (let [result (#'oidc/callback-handler cfg request) + stored (session/read-session manager (:id sess))] + (t/is (= "https://penpot.example.com/?screen=workspace" (redirect-location result))) + (t/is (= ["organization-sso-auth-failed"] (mapv :name @events))) + (t/is (= "session-mismatch" (get-in (first @events) [:props :failure-reason]))) + (t/is (nil? (get-in stored [:props :sso])))))))) + (t/deftest prepare-organization-sso-provider-does-not-skip-ssrf-check (t/testing "organization SSO provider must use SSRF protection" (let [captured-params (atom nil)] @@ -800,7 +1024,7 @@ {:client-id "test-client" :client-secret "test-secret" :issuer "https://wrong-idp.example.com"} - :dest-url "https://localhost:3449/#/dashboard" + :query-string "screen=dashboard" :organization-id #uuid "00000000-0000-0000-0000-000000000001") (catch Throwable t t))] (t/is (ex/error? e)) diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index e6a64f85d7..140a466154 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -801,6 +801,95 @@ (t/is (cfo/token-applied? token-radius shape [:r1 :r2 :r3 :r4])) (t/is (cfo/token-applied? token-opacity shape [:opacity])))))) +(t/deftest embed-assets-loads-the-library-components + ;; The library is read without `:realize?`, so its `:components` is an + ;; unloaded pointer map: `absorb-assets` walks it with `vals`, which calls + ;; `load!`, and that raises when no loader is bound. The absorb has to run + ;; under the library's own loader. This is a pre-existing failure, not a + ;; regression of the migration-on-read work: the migration path returns the + ;; pointer already loaded by the reattach comparison. + (let [profile (th/create-profile* 1) + library (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true}) + file (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + lib-page (first (get-in library [:data :pages])) + file-page (first (get-in file [:data :pages])) + component-id (uuid/random) + main-id (uuid/random) + copy-id (uuid/random)] + + (th/link-file-to-library* {:file-id (:id file) :library-id (:id library)}) + + ;; The library owns one component. + (update-file! + :file-id (:id library) + :profile-id (:id profile) + :changes + [{:type :add-obj + :page-id lib-page + :id main-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id main-id + :name "Board" + :frame-id uuid/zero + :parent-id uuid/zero + :type :frame + :main-instance true + :component-root true + :component-file (:id library) + :component-id component-id})} + {:type :add-component + :path "" + :name "Board" + :main-instance-id main-id + :main-instance-page lib-page + :id component-id + :anotation nil}]) + + ;; The file instances it, so the absorb has a usage to embed. + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :changes + [{:type :add-obj + :page-id file-page + :id copy-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id copy-id + :name "Board" + :frame-id uuid/zero + :parent-id uuid/zero + :type :frame + :main-instance false + :component-root true + :component-file (:id library) + :component-id component-id})}]) + + (t/testing "a plain library is absorbed" + (let [file-data (-> (bfc/get-file th/*system* (:id file) :realize? true) :data) + result (bfc/embed-assets th/*system* file-data (:id file))] + (t/is (contains? (:components result) component-id)))) + + (t/testing "a library that needed a migration is absorbed too" + ;; The scenario the review reported. It works even without the loader, + ;; because `reattach-unchanged` loads the pointers it puts back; it + ;; stays pinned so a future reattach that stops loading does not + ;; silently break the absorb. + (th/db-delete! :file-migration + {:file-id (:id library) + :name "0031-migrate-stroke-width-token-attr"}) + (let [file-data (-> (bfc/get-file th/*system* (:id file) :realize? true) :data) + result (bfc/embed-assets th/*system* file-data (:id file))] + (t/is (contains? (:components result) component-id)))))) + (t/deftest export-detach-libraries-preserves-own-tokens-and-status (let [profile (th/create-profile* 1) {:keys [library file page-id shape-id token-radius token-opacity]} @@ -1410,26 +1499,23 @@ (t/is (some? file-res)) (t/is (= 1 (count (:done file-res)))))))) -(t/deftest link-later-overwrite-import-no-resolution +(t/deftest import-rejects-an-in-place-import + ;; The in-place import was removed: it never carried the previous storage + ;; pointer, so it could leave the replaced object unreclaimed. A caller that + ;; still sets `::bfc/file-id` fails instead of importing as new files. (let [{:keys [profile file-id team-id]} (import-sample-file) - output (tmp/tempfile :suffix ".zip")] - ;; Export file with link-later - (v3/export-files! - (-> th/*system* - (assoc ::bfc/file-ids #{file-id}) - (assoc ::bfc/export-type :link-later) - (assoc ::bfc/output (io/output-stream output)))) - - ;; Import with overwrite (file-id set) - (let [result (-> th/*system* + output (tmp/tempfile :suffix ".zip") + cause (try + (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) (assoc ::bfc/profile-id (:id profile)) (assoc ::bfc/team-id team-id) (assoc ::bfc/file-id file-id) (assoc ::bfc/input output) - (v3/import-files!))] - ;; Overwrite should have empty resolution - (t/is (= {} (:resolution result)))))) + (v3/import-files!)) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + (t/is (= :not-supported (:code (ex-data cause)))))) ;; ----------------------------------------------------------------------------- ;; Category 2: Cross-Team Migration @@ -3082,17 +3168,51 @@ (:object-id thumb))) (t/is (some? (:media-id thumb))))))) -(t/deftest get-file-without-data-rejects-locks - ;; Locking is incompatible with `load-data? false`: the guard fires - ;; before any query, so no DB row is needed. Regression test for the - ;; `lock-for-share?`/`lock-for-share?` typo that left `lock-for-update?` - ;; unchecked. NOTE: the guard is an unconditional `when`/`throw` in - ;; `get-file*` (not an `assert`), so this test is not flag-dependent: - ;; it behaves the same with or without `enable-backend-asserts`. - (doseq [opt [:lock-for-share? :lock-for-update? :skip-locked?]] +(t/deftest get-file-without-data-can-lock + ;; Locking is legal without loading the data: the row-only query selects + ;; directly from `file`, so `FOR UPDATE`/`FOR SHARE` apply to the row and + ;; the lock lives until the enclosing transaction ends. `SKIP LOCKED` only + ;; modifies a locking read, so on its own it is ignored. The SQL is pinned + ;; because a plain `SELECT` also returns the row: the lock clause is what + ;; serializes with the other writers. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + (doseq [[opt sql] {:lock-for-share? (str bfc/sql:get-file-without-data " FOR SHARE of f") + :lock-for-update? (str bfc/sql:get-file-without-data " FOR UPDATE of f") + :skip-locked? bfc/sql:get-file-without-data}] + (let [[row sqls] (th/read-recording-sql + #(bfc/get-file th/*system* (:id file) :load-data? false opt true))] + (t/is (= (:id file) (:id row))) + (t/is (nil? (:data row))) + (t/is (= [sql] sqls)))))) + +(t/deftest get-file-rejects-realize-without-data + ;; `:realize?` walks the decoded data; asking for it without loading or + ;; decoding the data is a caller bug. The guard fires before any query, so + ;; no file row is needed. + (let [id (uuid/random)] (t/is (thrown? IllegalArgumentException - (bfc/get-file th/*system* (uuid/random) - :load-data? false opt true))))) + (bfc/get-file th/*system* id :realize? true :decode? false))) + (t/is (thrown? IllegalArgumentException + (bfc/get-file th/*system* id :realize? true :load-data? false))))) + +(t/deftest get-file-rejects-a-shared-lock-with-data + ;; The probe inherits the caller's lock and the migration branch re-reads + ;; with FOR UPDATE, so a shared lock on a data read is a lock upgrade. The + ;; guard fires before any query, so no row is needed. + (t/is (thrown? IllegalArgumentException + (bfc/get-file th/*system* (uuid/random) :lock-for-share? true)))) + +(t/deftest get-file-rejects-a-bare-pool-or-connection + ;; The read opens a transaction (unless it is read-only) and resolves the + ;; cfg, so it takes a cfg map only: a bare pool or connection is a caller + ;; bug. The guard is an `IllegalArgumentException`, not an `assert`, which + ;; is elided when the backend runs without `:backend-asserts`. + (let [id (uuid/random)] + (t/is (thrown? IllegalArgumentException (bfc/get-file th/*pool* id))) + (with-open [conn (db/open th/*system*)] + (t/is (thrown? IllegalArgumentException (bfc/get-file conn id)))))) ;; ----------------------------------------------------------------------------- ;; Branch target protection @@ -3101,7 +3221,8 @@ (t/deftest import-overwrite-refuses-branch-file ;; A branch file's data payload is derived from its merge base and op ;; log, so an in-place import would write a `file_data` main row that - ;; no read path consults. + ;; no read path consults. `import-files!` refuses every in-place import, + ;; and this keeps a branch file among the files it refuses. (with-redefs [cf/flags (conj cf/flags :branching)] (let [profile (th/create-profile* 1) file (prepare-simple-file profile) @@ -3134,7 +3255,7 @@ (t/is (uuid? branch-id)) (t/is (= :validation (:type error))) - (t/is (= :branch-file-cant-be-overwritten (:code error))) + (t/is (= :not-supported (:code error))) (t/is (empty? (th/db-query :file-data {:file-id branch-id :type "main"}))))))) ;; --- Library relations security tests @@ -3197,7 +3318,8 @@ [[(str (:id file)) (str (:id file))]]) [file-id :as result] (import-bundle profile evil)] (t/is (= 1 (count result))) - (t/is (empty? (th/db-query :file-library-rel {:file-id file-id}))))) + (t/is (empty? (th/db-query :file-library-rel {:file-id file-id}))) + (t/is (empty? (th/db-query :file-library-sync {:file-id file-id}))))) (t/deftest import-v3-keeps-valid-library-relation (let [profile (th/create-profile* 1) @@ -3216,6 +3338,251 @@ (t/is (= 1 (count (th/db-query :file-library-rel {:file-id (:id new-file)}))))))) +(t/deftest import-skips-circular-library-relations + ;; https://github.com/penpot/penpot/issues/12052: two libraries that use + ;; each other (possible before the 2.17.1 guard) export both relations in + ;; the manifest; the import must skip the one closing the cycle instead + ;; of recreating it. The cycle is built with direct inserts because the + ;; UI command `link-file-to-library` already rejects it. Relations are + ;; processed in sorted manifest order and the first edge wins. + (let [profile (th/create-profile* 1) + lib-a (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library A"}) + lib-b (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library B"})] + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-a) + :library-file-id (:id lib-b)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-b) + :library-file-id (:id lib-a)}) + + (let [output (tmp/tempfile :suffix ".zip")] + (v3/export-files! + (-> th/*system* + (assoc ::bfc/file-ids #{(:id lib-a) (:id lib-b)}) + (assoc ::bfc/export-type :include-libraries) + (assoc ::bfc/output (io/output-stream output)))) + + (let [result (import-bundle profile output) + rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %}) + result) + syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %}) + result)] + (t/is (= 2 (count result))) + ;; only one of the two cycle edges survives the import + (t/is (= 1 (count rels))) + ;; the skipped edge leaves no sync row behind + (t/is (= 1 (count syncs))) + (t/is (= (set (map (juxt :file-id :library-file-id) syncs)) + (set (map (juxt :file-id :library-file-id) rels)))) + ;; no imported file lists itself among its transitive libraries + (doseq [file-id result] + (t/is (not (contains? (bfc/get-libraries th/*system* [file-id]) + file-id)))))))) + +(t/deftest import-skips-transitive-cycle + ;; A three-node cycle (A -> B -> C -> A) only breaks when the check uses + ;; the transitive closure: a direct-edge-only check would keep all three + ;; edges. Built with direct inserts because the UI command rejects it. + (let [profile (th/create-profile* 1) + lib-a (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library A"}) + lib-b (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library B"}) + lib-c (th/create-file* 3 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library C"})] + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-a) + :library-file-id (:id lib-b)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-b) + :library-file-id (:id lib-c)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-c) + :library-file-id (:id lib-a)}) + + (let [output (tmp/tempfile :suffix ".zip")] + (v3/export-files! + (-> th/*system* + (assoc ::bfc/file-ids #{(:id lib-a) (:id lib-b) (:id lib-c)}) + (assoc ::bfc/export-type :include-libraries) + (assoc ::bfc/output (io/output-stream output)))) + + (let [result (import-bundle profile output) + rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %}) + result) + syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %}) + result)] + (t/is (= 3 (count result))) + ;; only two of the three cycle edges survive the import + (t/is (= 2 (count rels))) + ;; the skipped edge leaves no sync row behind + (t/is (= 2 (count syncs))) + (t/is (= (set (map (juxt :file-id :library-file-id) syncs)) + (set (map (juxt :file-id :library-file-id) rels)))) + ;; no imported file lists itself among its transitive libraries + (doseq [file-id result] + (t/is (not (contains? (bfc/get-libraries th/*system* [file-id]) + file-id)))))))) + +(t/deftest import-keeps-acyclic-library-relations + ;; Control test: a chain without cycles must be imported untouched, so + ;; the new guard only skips edges that would close a cycle. + (let [profile (th/create-profile* 1) + lib-a (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library A"}) + lib-b (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library B"}) + lib-c (th/create-file* 3 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library C"})] + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-a) + :library-file-id (:id lib-b)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-b) + :library-file-id (:id lib-c)}) + + (let [output (tmp/tempfile :suffix ".zip")] + (v3/export-files! + (-> th/*system* + (assoc ::bfc/file-ids #{(:id lib-a) (:id lib-b) (:id lib-c)}) + (assoc ::bfc/export-type :include-libraries) + (assoc ::bfc/output (io/output-stream output)))) + + (let [result (import-bundle profile output) + rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %}) + result) + syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %}) + result)] + (t/is (= 3 (count result))) + (t/is (= 2 (count rels))) + ;; every kept relation has its sync row + (t/is (= 2 (count syncs))) + (t/is (= (set (map (juxt :file-id :library-file-id) syncs)) + (set (map (juxt :file-id :library-file-id) rels)))))))) + +(t/deftest import-v1-skips-circular-library-relations + ;; Issue #12052 also names the version 1 importer: importing a package + ;; whose libraries use each other must not recreate the cycle. The cycle + ;; is built with direct inserts because `link-file-to-library` already + ;; rejects it. The exported file only *uses* one of the libraries, so the + ;; package carries the cycle the same way a real export does. + (let [profile (th/create-profile* 1) + file (prepare-simple-file profile) + lib-a (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library A"}) + lib-b (th/create-file* 3 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library B"})] + (th/link-file-to-library* {:file-id (:id file) + :library-id (:id lib-a)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-a) + :library-file-id (:id lib-b)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-b) + :library-file-id (:id lib-a)}) + + (let [output (tmp/tempfile :suffix ".penpot")] + (v1/export-files! + (-> th/*system* + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/include-libraries true)) + (io/output-stream output)) + + (let [result (:file-ids (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output) + (v1/import-files!))) + rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %}) + result) + syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %}) + result)] + (t/is (= 3 (count result))) + ;; the relation of the file to its library, plus one of the two + ;; cycle edges; the edge that would close the cycle is skipped + (t/is (= 2 (count rels))) + ;; the skipped edge leaves no sync row behind + (t/is (= 2 (count syncs))) + (t/is (= (set (map (juxt :file-id :library-file-id) syncs)) + (set (map (juxt :file-id :library-file-id) rels)))) + ;; no imported file lists itself among its transitive libraries + (doseq [file-id result] + (t/is (not (contains? (bfc/get-libraries th/*system* [file-id]) + file-id)))))))) + +(t/deftest import-v1-tolerates-repeated-file-entries + ;; Old v1 exports could list the same file twice in the package metadata: + ;; the exporter unioned the requested files with their libraries without + ;; deduplicating, and a file inside a library cycle is one of its own + ;; libraries. The import must read every entry and import each file once. + (let [profile (th/create-profile* 1) + lib-a (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library A"}) + lib-b (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true + :name "Library B"})] + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-a) + :library-file-id (:id lib-b)}) + (db/insert! th/*system* :file-library-rel + {:file-id (:id lib-b) + :library-file-id (:id lib-a)}) + + (let [output (tmp/tempfile :suffix ".penpot")] + ;; exporting only lib-a repeats its id in the metadata, because it is + ;; among its own transitive libraries + (v1/export-files! + (-> th/*system* + (assoc ::bfc/file-ids #{(:id lib-a)}) + (assoc ::bfc/include-libraries true)) + (io/output-stream output)) + + (let [result (:file-ids (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output) + (v1/import-files!))) + rels (mapcat #(db/query th/*system* :file-library-rel {:file-id %}) + result) + syncs (mapcat #(db/query th/*system* :file-library-sync {:file-id %}) + result)] + ;; two files: the repeated entry is imported once + (t/is (= 2 (count result))) + ;; one of the two cycle edges survives the import + (t/is (= 1 (count rels))) + (t/is (= 1 (count syncs))) + (t/is (= (set (map (juxt :file-id :library-file-id) syncs)) + (set (map (juxt :file-id :library-file-id) rels)))) + ;; no imported file lists itself among its transitive libraries + (doseq [file-id result] + (t/is (not (contains? (bfc/get-libraries th/*system* [file-id]) + file-id)))))))) + (t/deftest export-multi-file-includes-package-relations (let [profile (th/create-profile* 1) file (prepare-simple-file profile) @@ -3314,3 +3681,45 @@ (t/is (empty? (th/db-query :file-library-sync {:library-file-id (:id victim)}))) (t/is (empty? (th/db-query :file-library-rel {:library-file-id (:id file)}))) (t/is (empty? (th/db-query :file-library-sync {:library-file-id (:id file)})))))) + +(t/deftest file-permissions-ignore-soft-deleted + ;; sql:file-permissions must not grant anything through a file, + ;; project or team that is soft-deleted (GHSA-m53j-2766-6jqw: the + ;; websocket re-check relies on this query, and the subscribe handler + ;; additionally refuses deleted files via check-file-exists). + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (let [project (th/create-project* 1 {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project)}) + perms? #(some? (bfc/get-file-permissions th/*system* (:id editor) (:id file))) + undelete (fn [table id] + (db/update! th/*system* table {:deleted-at nil} {:id id})) + soft-delete (fn [table id] + (db/update! th/*system* table {:deleted-at (ct/now)} {:id id}))] + + (t/testing "a live member can read the live file" + (t/is (perms?))) + + (t/testing "a soft-deleted file grants nothing" + (soft-delete :file (:id file)) + (t/is (nil? (bfc/get-file-permissions th/*system* (:id editor) (:id file)))) + (undelete :file (:id file)) + (t/is (perms?))) + + (t/testing "a soft-deleted project grants nothing" + (soft-delete :project (:id project)) + (t/is (nil? (bfc/get-file-permissions th/*system* (:id editor) (:id file)))) + (undelete :project (:id project)) + (t/is (perms?))) + + (t/testing "a soft-deleted team grants nothing" + (soft-delete :team (:id team)) + (t/is (nil? (bfc/get-file-permissions th/*system* (:id editor) (:id file)))) + (undelete :team (:id team)) + (t/is (perms?)))))) diff --git a/backend/test/backend_tests/climit_test.clj b/backend/test/backend_tests/climit_test.clj index 9a40857d55..317d3454a6 100644 --- a/backend/test/backend_tests/climit_test.clj +++ b/backend/test/backend_tests/climit_test.clj @@ -452,10 +452,9 @@ (t/is (thrown? clojure.lang.ExceptionInfo (invoke! cfg (fn [_ _] :never) {})))) - ;; the count came back down instead of staying up for good. Only the - ;; last publication is asserted, not the exact list: the sampler - ;; republishes the same count when its first tick lands inside the - ;; window. + ;; the count came back down instead of staying up for good. The + ;; republisher daemon thread can add publications inside the window, + ;; so what is asserted is the last value, not the whole sequence. (t/is (= 0 (waiting-count climit name))) (t/is (= 0 (waiting-last name))) diff --git a/backend/test/backend_tests/config_test.clj b/backend/test/backend_tests/config_test.clj index 74a159a703..fa168708ef 100644 --- a/backend/test/backend_tests/config_test.clj +++ b/backend/test/backend_tests/config_test.clj @@ -6,9 +6,11 @@ (ns backend-tests.config-test (:require + [app.common.logging :as l] [app.common.schema :as sm] [app.common.time :as ct] [app.config :as cf] + [clojure.core :as c] [clojure.test :as t])) (t/deftest tenant-is-normalized-to-lowercase @@ -86,3 +88,51 @@ (t/testing "and falls back to a week" (with-redefs [cf/config (dissoc cf/config :jobs-user-ttl)] (t/is (= (ct/duration {:days 7}) (cf/get-jobs-user-ttl)))))) + +;; `legacy-db` is read-only: the file data moved to `file_data`, and +;; `file.data` / `file_change.data` are read-only leftovers. An instance +;; that keeps it starts healthy and then fails on every save. It is read +;; as `db` instead, which also serves the legacy rows. + +(defn- read-config + "`read-config` with the environment replaced by `env`." + [env & {:keys [default] :or {default {}}}] + (with-redefs [cf/read-env (constantly env)] + (cf/read-config :default default))) + +(defn- read-config-with-warning + "Reads the configuration and collects the warnings it emits. Logging is + emitted asynchronously, so the record is awaited (bounded)." + [env] + (let [records (atom [])] + (add-watch l/log-record ::capture + (fn [_ _ _ record] (swap! records conj record))) + (try + (let [config (read-config env)] + [(loop [remaining 100] + (let [warns (filter #(= :warn (::l/level %)) @records)] + (cond + (seq warns) warns + (zero? remaining) warns + :else (do (Thread/sleep 10) (recur (dec remaining)))))) + config]) + (finally + (remove-watch l/log-record ::capture))))) + +(t/deftest read-config-replaces-the-read-only-legacy-db + (let [[warns config] (read-config-with-warning {:file-data-backend "legacy-db"})] + (t/testing "the configuration writes through db" + (t/is (= "db" (c/get config :file-data-backend)))) + (t/testing "the operator is warned" + (t/is (contains? (into #{} (map #(c/get (::l/props %) :hint)) warns) + "file-data-backend=legacy-db is read-only, using db"))))) + +(t/deftest read-config-keeps-the-writable-back-ends + (doseq [backend ["db" "storage"]] + (t/testing backend + (t/is (= backend (c/get (read-config {:file-data-backend backend}) + :file-data-backend)))))) + +(t/deftest read-config-takes-the-default-back-end + (t/is (= "db" (c/get (read-config {} :default cf/default) + :file-data-backend)))) diff --git a/backend/test/backend_tests/db_test.clj b/backend/test/backend_tests/db_test.clj index aa6732ead5..b0f20cb256 100644 --- a/backend/test/backend_tests/db_test.clj +++ b/backend/test/backend_tests/db_test.clj @@ -80,3 +80,11 @@ (t/deftest uuid->hash-code-stable-for-zero-uuid (t/is (= 3659997967308761462 (db/uuid->hash-code uuid/zero)))) + +(t/deftest pool-enables-the-partitionwise-join + ;; The pool sets the GUC on every connection, so the planner can resolve a + ;; join between two partitioned tables pairwise, partition by partition. + (let [row (db/exec-one! th/*pool* + ["SELECT current_setting('enable_partitionwise_join') AS partitionwise_join"])] + + (t/is (= "on" (:partitionwise-join row))))) diff --git a/backend/test/backend_tests/fdata_resolve_test.clj b/backend/test/backend_tests/fdata_resolve_test.clj new file mode 100644 index 0000000000..e9d777d900 --- /dev/null +++ b/backend/test/backend_tests/fdata_resolve_test.clj @@ -0,0 +1,42 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.fdata-resolve-test + "Resolution of file data by backend (`app.features.fdata/resolve-file-data`). + + A row without a `file_data` twin carries no backend: the read queries mark + it with `coalesce(fd.backend, 'legacy-db')` and hand its blob over as + `:legacy-data`. The `legacy-db` backend is still supported, so that case + has to resolve to the blob instead of nil." + (:require + [app.features.fdata :as fdata] + [clojure.test :as t])) + +(t/deftest legacy-db-resolves-the-legacy-blob + (let [blob (vec (.getBytes "legacy-blob" "UTF-8")) + file {:backend "legacy-db" + :legacy-data (byte-array blob)} + out (fdata/resolve-file-data nil file)] + (t/is (= blob (vec (:data out)))) + (t/is (not (contains? out :legacy-data))))) + +(t/deftest missing-backend-defaults-to-legacy-db + ;; The queries leave `:backend` nil when the `LEFT JOIN` with `file_data` + ;; finds no twin; the dispatch default keeps that row on `legacy-db`. + (let [blob (vec (.getBytes "legacy-blob" "UTF-8")) + file {:legacy-data (byte-array blob)} + out (fdata/resolve-file-data nil file)] + (t/is (= blob (vec (:data out)))) + (t/is (not (contains? out :legacy-data))))) + +(t/deftest db-backend-drops-the-legacy-key + (let [data (vec (.getBytes "db-blob" "UTF-8")) + file {:backend "db" + :data (byte-array data) + :legacy-data (byte-array (vec (.getBytes "stale" "UTF-8")))} + out (fdata/resolve-file-data nil file)] + (t/is (= data (vec (:data out)))) + (t/is (not (contains? out :legacy-data))))) diff --git a/backend/test/backend_tests/fdata_upsert_test.clj b/backend/test/backend_tests/fdata_upsert_test.clj new file mode 100644 index 0000000000..c2fa034da3 --- /dev/null +++ b/backend/test/backend_tests/fdata_upsert_test.clj @@ -0,0 +1,127 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.fdata-upsert-test + "Covers the write contract of `fdata/upsert!` after the `legacy-db` backend + became read-only." + + (:require + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.features.fdata :as fdata] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(defn- upsert-cause + [params] + (try + (fdata/upsert! th/*system* params) + nil + (catch Exception cause + cause))) + +(t/deftest legacy-db-param-is-rejected + ;; `legacy-db` is not a write backend anymore: the schema refuses it before + ;; touching the database. + (let [cause (upsert-cause {:id (uuid/next) + :file-id (uuid/next) + :type "snapshot" + :backend "legacy-db" + :storage-object-id nil + :data (byte-array [1 2 3])})] + (t/is (some? cause)) + (t/is (= :assertion (:type (ex-data cause)))) + (t/is (= :data-validation (:code (ex-data cause)))))) + +(t/deftest upsert-reports-the-pointer-it-created + ;; The write reports the object it created, so a caller that persists again + ;; touches the current pointer instead of the replaced one. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + data (byte-array [1 2 3])] + + (let [out (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg {:id (:id file) + :file-id (:id file) + :type "main" + :backend "storage" + :storage-object-id nil + :data data})))] + (t/testing "a storage write reports the object it created" + (t/is (true? (:written out))) + (t/is (uuid? (:storage-object-id out))) + (t/is (= (:storage-object-id out) + (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id (:id file)})))))) + + (let [previous (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id (:id file)})) + out (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg {:id (:id file) + :file-id (:id file) + :type "main" + :backend "db" + :storage-object-id previous + :data data})))] + (t/testing "a db write has no object to report" + (t/is (true? (:written out))) + (t/is (nil? (:storage-object-id out))) + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id (:id file)})))))))) + +(t/deftest upsert-rejects-a-stale-pointer + ;; The write only replaces the row whose pointer the caller read. A stale + ;; map must fail instead of touching the wrong object and leaking the blob. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + data (byte-array [1 2 3]) + before (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + cause (try + (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg {:id (:id file) + :file-id (:id file) + :type "main" + :backend "db" + :storage-object-id (uuid/next) + :data data}))) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + + (t/is (= :stale-file-data-pointer (:code (ex-data cause)))) + + (let [after (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/testing "the row is untouched" + (t/is (= (:backend before) (:backend after))) + (t/is (= (:storage-object-id before) (:storage-object-id after))) + (t/is (= (vec (:data before)) (vec (:data after)))))))) + +(t/deftest legacy-db-config-is-rejected-as-read-only + ;; The configuration value is kept for reading old data, but a write with + ;; it as the default backend fails with a clear message. + (with-redefs [cf/config (assoc cf/config :file-data-backend "legacy-db")] + (let [cause (upsert-cause {:id (uuid/next) + :file-id (uuid/next) + :type "snapshot" + :storage-object-id nil + :data (byte-array [1 2 3])})] + (t/is (some? cause)) + (t/is (= :validation (:type (ex-data cause)))) + (t/is (= :legacy-backend-read-only (:code (ex-data cause))))))) + +(t/deftest file-data-backend-defaults-to-db + (t/is (= "db" (cf/get :file-data-backend)))) diff --git a/backend/test/backend_tests/file_change_retention_test.clj b/backend/test/backend_tests/file_change_retention_test.clj index 3929e0b129..83246e9226 100644 --- a/backend/test/backend_tests/file_change_retention_test.clj +++ b/backend/test/backend_tests/file_change_retention_test.clj @@ -10,7 +10,7 @@ A file has two kinds of row. Durable history: the snapshots, which a restore brings back. And ephemeral revisions: the xlog of an edit, born already scheduled for deletion one hour later, which nobody needs once the sessions - that were going to read them are gone. + that were going to read them are gone. The `kind` column tells them apart. The invariant under test is that a deletion date is assigned once and never moved: restoring a file must not unmark a revision, and deleting a file must @@ -21,6 +21,7 @@ [app.common.time :as ct] [app.common.uuid :as uuid] [app.db :as db] + [app.features.file-changes :as fchanges] [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] [app.util.blob :as blob] @@ -64,6 +65,7 @@ :version 0 :created-by (or created-by "system") :label label + :kind (if label fchanges/snapshot-kind fchanges/revision-kind) :data data :changes changes} {::db/return-keys false}) @@ -170,6 +172,7 @@ out (th/command! {::th/type :update-file-snapshot ::rpc/profile-id (:id profile) + :file-id file-id :id revision-id :label "renamed"})] diff --git a/backend/test/backend_tests/file_change_schema_test.clj b/backend/test/backend_tests/file_change_schema_test.clj new file mode 100644 index 0000000000..a40871ddde --- /dev/null +++ b/backend/test/backend_tests/file_change_schema_test.clj @@ -0,0 +1,115 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.file-change-schema-test + (:require + [app.db :as db] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(defn- query + "Runs a query that returns a single value and returns it." + [sql & params] + (-> (db/exec! th/*pool* (into [sql] params)) + first + (vals) + first)) + +(defn- partition-names + [] + (->> (db/exec! th/*pool* ["SELECT c.relname AS name + FROM pg_class c + JOIN pg_partition_tree('file_change') t ON t.relid = c.oid + WHERE t.isleaf + ORDER BY c.relname"]) + (mapv :name))) + +(t/deftest file-change-is-partitioned-in-sixteen-named-partitions + (t/is (= (mapv #(format "file_change_%02d" %) (range 16)) + (partition-names)))) + +(t/deftest file-change-primary-key-is-file-id-and-id + (t/is (= "PRIMARY KEY (file_id, id)" + (query "SELECT pg_get_constraintdef(oid) AS def + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'p'")))) + +(t/deftest file-change-has-three-foreign-keys-and-one-check + (t/is (= 3 (query "SELECT count(*) AS total + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'f'"))) + (t/is (= 1 (query "SELECT count(*) AS total + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'c'"))) + (t/is (re-find #"kind" + (query "SELECT pg_get_constraintdef(oid) AS def + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'c'")))) + +(t/deftest file-change-has-the-six-canonical-indexes + (t/is (= ["file_change__deleted_at__idx" + "file_change__file_id__revn__idx" + "file_change__locked_by__idx" + "file_change__profile_id__idx" + "file_change__user_snapshots__idx" + "file_change_pkey"] + (->> (db/exec! th/*pool* ["SELECT indexname AS name + FROM pg_indexes + WHERE tablename = 'file_change' + AND schemaname = current_schema() + ORDER BY indexname"]) + (mapv :name))))) + +(t/deftest file-change-keeps-the-data-column-and-a-kind-column + ;; The `data` column is kept: the `legacy-db` backend reads old payloads + ;; from it (migration 0157 moved the existing blobs). The dead + ;; `data_ref_id`/`data_backend` columns stay gone: no code read them since + ;; the file storage refactor. + (t/is (= [] + (->> (db/exec! th/*pool* ["SELECT attname AS name + FROM pg_attribute + WHERE attrelid = 'file_change'::regclass + AND attnum > 0 AND NOT attisdropped + AND attname IN ('data_ref_id', 'data_backend')"]) + (mapv :name)))) + (t/is (= ["bytea"] + (->> (db/exec! th/*pool* ["SELECT pg_catalog.format_type(atttypid, atttypmod) AS type + FROM pg_attribute + WHERE attrelid = 'file_change'::regclass + AND attname = 'data'"]) + (mapv :type)))) + ;; Nullable on purpose: a rollback to the previous version inserts + ;; without `kind` and has to keep writing (migration 0158). + (t/is (false? (query "SELECT attnotnull AS notnull + FROM pg_attribute + WHERE attrelid = 'file_change'::regclass + AND attname = 'kind'")))) + +(t/deftest inserting-a-row-without-a-kind-succeeds + ;; The column is nullable on purpose: the previous version inserts without + ;; it, and a rollback to that version has to keep writing. The backfill of + ;; a later NOT NULL migration turns these NULLs into 1/2. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + (db/exec! th/*pool* ["INSERT INTO file_change (file_id, id) + VALUES (?, gen_random_uuid())" + (:id file)]) + (t/is (nil? (query "SELECT kind AS k + FROM file_change + WHERE file_id = ?" (:id file)))))) + +(t/deftest inserting-a-row-with-an-unknown-kind-fails + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + (t/is (thrown? java.sql.SQLException + (db/exec! th/*pool* ["INSERT INTO file_change (file_id, id, kind) + VALUES (?, gen_random_uuid(), 3)" + (:id file)]))))) diff --git a/backend/test/backend_tests/file_data_test.clj b/backend/test/backend_tests/file_data_test.clj new file mode 100644 index 0000000000..ff37dda751 --- /dev/null +++ b/backend/test/backend_tests/file_data_test.clj @@ -0,0 +1,323 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.file-data-test + (:require + [app.binfile.common :as bfc] + [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.features.fdata :as fdata] + [app.storage :as sto] + [app.util.blob :as blob] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +;; The storage pointer of a `file_data` row is a column with a foreign key and +;; it is the only source of truth: the release is a coordinated restart and +;; the metadata no longer carries `storage-ref-id`. + +(defn- find-object + [id] + (th/db-get :storage-object {:id id} {::db/remove-deleted false})) + +(defn- put-object! + ([bucket metadata content] + (sto/put-object! (::sto/storage th/*system*) + (merge {::sto/content content + :content-type "application/octet-stream" + :bucket bucket} + metadata))) + ([bucket metadata] + (put-object! bucket metadata (sto/content "sobject")))) + +(defn- create-file! + [] + (let [profile (th/create-profile* 1)] + (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}))) + +(defn- upsert! + [params] + (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg params)))) + +(defn- resolve! + [row] + (db/tx-run! th/*system* + (fn [cfg] + (fdata/resolve-file-data cfg row)))) + +(t/deftest upsert-to-storage-writes-the-column + (let [file (create-file!) + data (blob/encode {:pages [] :pages-index {}})] + + ;; `main` always carries `id = file_id`: update the row the file already + ;; has instead of inventing a second one. + (upsert! {:id (:id file) + :file-id (:id file) + :type "main" + :backend "storage" + :storage-object-id nil + :data data + :metadata {:generated-by "test"}}) + + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + metadata (fdata/decode-metadata (:metadata row)) + sobject (:storage-object-id row)] + + (t/testing "the row points at the object with the column" + (t/is (= "storage" (:backend row))) + (t/is (nil? (:data row))) + (t/is (uuid? sobject))) + + (t/testing "the metadata no longer carries the pointer" + (t/is (nil? (:storage-ref-id metadata))) + (t/is (= "test" (:generated-by metadata)))) + + (t/testing "the object is stored in the file-data bucket" + (t/is (= "file-data" (:bucket (meta (sto/get-object (::sto/storage th/*system*) + sobject)))))) + + (t/testing "the row resolves to the stored bytes" + (t/is (= (vec data) (vec (:data (resolve! row))))))))) + +(t/deftest upsert-back-to-db-clears-the-column-and-touches-the-object + (let [file (create-file!)] + + (upsert! {:id (:id file) + :file-id (:id file) + :type "main" + :backend "storage" + :storage-object-id nil + :data (blob/encode {:pages [] :pages-index {}})}) + + (let [sobject (:storage-object-id (th/db-get :file-data + {:file-id (:id file) :id (:id file)}))] + + ;; The caller passes the object the persisted row points at; `upsert!` + ;; touches it before replacing the row. The dead metadata pointer must + ;; not survive the write either. + (upsert! {:id (:id file) + :file-id (:id file) + :type "main" + :backend "db" + :storage-object-id sobject + :data (blob/encode {:pages [] :pages-index {}}) + :metadata {:generated-by "test" + :storage-ref-id (uuid/next)}}) + + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + metadata (fdata/decode-metadata (:metadata row))] + + (t/testing "the hot row no longer references the object" + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row))) + (t/is (nil? (:storage-ref-id metadata))) + (t/is (= "test" (:generated-by metadata)))) + + (t/testing "the replaced object is touched for the storage GC" + (t/is (some? (:touched-at (find-object sobject))))))))) + +(t/deftest resolve-file-data-uses-the-column-and-ignores-the-metadata + (let [file (create-file!) + data (blob/encode {:pages [] :pages-index {}}) + obj (put-object! "file-data" + {:file-id (:id file) :id (:id file)} + (sto/content data)) + ;; A different, well-formed pointer in the metadata: it must be + ;; ignored now that the column is the only source of truth. + other (put-object! "file-data" + {:file-id (uuid/next) :id (uuid/next)})] + + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id obj) + :metadata (db/json {"storage-ref-id" (str (:id other))})} + {:file-id (:id file) :id (:id file)}) + + (t/testing "the column wins over the metadata" + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (= (:id obj) (:storage-object-id row))) + (t/is (= (vec data) (vec (:data (resolve! row))))))))) + +(t/deftest gc-touched-freezes-an-object-referenced-only-by-the-column + (let [file (create-file!) + ;; The object metadata points at other ids: the freeze must come from + ;; the column, not from the old metadata lookup. + sobject (put-object! "file-data" + {:file-id (uuid/next) :id (uuid/next)})] + + (th/db-update! :storage-object + {:touched-at (ct/in-past {:hours 3})} + {:id (:id sobject)}) + + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id sobject)} + {:file-id (:id file) :id (:id file)}) + + (let [result (th/run-task! :storage-gc-touched {})] + (t/is (= 1 (:freeze result))) + (t/is (= 0 (:delete result))) + (t/is (nil? (:touched-at (find-object (:id sobject)))))))) + +(t/deftest update-file!-ignores-the-pointer-in-the-params + ;; `:storage-object-id` is bookkeeping of the persisted row, not an editable + ;; field: a param must not be able to drop the pointer and skip the touch. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + (th/run-task! :storage-gc-touched {:skip-delay true}) + + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file) + :id (:id file)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (find-object sobject)))) + + (db/tx-run! th/*system* + (fn [cfg] + (let [file (bfc/get-file cfg (:id file) + :migrate? false + :lock-for-update? true)] + (bfc/update-file! cfg file + :storage-object-id nil + :modified-at (ct/now))))) + + (t/testing "the previous object is still touched" + (t/is (some? (:touched-at (find-object sobject)))))))) + +(t/deftest resolve-file-data-reports-a-missing-storage-object + (let [file (create-file!)] + + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id nil} + {:file-id (:id file) :id (:id file)}) + + (let [cause (try + (db/tx-run! th/*system* + (fn [cfg] + (->> (th/db-get :file-data {:file-id (:id file) + :id (:id file)}) + (fdata/resolve-file-data cfg)))) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + (t/is (some? cause)) + (t/is (= :missing-storage-object (:code (ex-data cause)))) + (t/is (= (:id file) (:file-id (ex-data cause))))))) + +(t/deftest load-pointer-reports-the-file-when-the-object-is-gone + (let [file (create-file!) + fragment (first (th/db-query :file-data {:file-id (:id file) + :type "fragment"})) + sobject (put-object! "file-data" + {:file-id (:id file) :id (:id fragment)})] + + (t/is (uuid? (:id fragment))) + + ;; The foreign key accepts the row (the object exists); a soft-deleted + ;; object is simply unreadable, which is the case the error reports. + (th/db-update! :storage-object + {:deleted-at (ct/in-past "1h")} + {:id (:id sobject)}) + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id sobject)} + {:file-id (:id file) :id (:id fragment)}) + + (let [cause (try + (db/tx-run! th/*system* + (fn [cfg] + (fdata/load-pointer cfg (:id file) (:id fragment)))) + nil + (catch clojure.lang.ExceptionInfo cause cause)) + data (ex-data cause)] + (t/is (some? cause)) + (t/is (= :missing-storage-object (:code data))) + (t/is (= (:id file) (:file-id data))) + (t/is (= (:id fragment) (:id data)))))) + +(t/deftest update-file!-rejects-a-map-without-data + ;; A map read with `:load-data? false` has no `:data` and no storage pointer; + ;; persisting it would wipe the content and skip the touch. The guard is an + ;; explicit raise, not an assert, so it also exists in production. + (let [file (create-file!) + cause (try + (db/tx-run! th/*system* + (fn [cfg] + (bfc/update-file! cfg (bfc/get-file cfg (:id file) + :load-data? false)))) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + (t/is (some? cause)) + (t/is (= :missing-file-data (:code (ex-data cause)))))) + +(t/deftest update-file!-ignores-the-identity-in-the-params + ;; The params change mutable fields only: `:id` cannot retarget the UPDATE. + (let [file (create-file!) + other (uuid/next) + when (ct/inst "2020-01-01T00:00:00Z")] + + (db/tx-run! th/*system* + (fn [cfg] + (let [file (bfc/get-file cfg (:id file) + :migrate? false + :lock-for-update? true)] + (bfc/update-file! cfg file :id other :modified-at when)))) + + (t/testing "the read file is updated" + (t/is (= when (:modified-at (th/db-get :file {:id (:id file)}))))) + + (t/testing "the id in the params is ignored" + (t/is (nil? (th/db-get :file {:id other})))))) + +(t/deftest update-file!-touches-the-previous-object-on-a-storage-write + ;; With `:file-data-backend` = "storage", every ordinary save replaces the + ;; object of the row: the previous one must be touched so the storage GC can + ;; reclaim it. The storage -> db and db -> storage directions are covered + ;; above; this pins the configuration where storage is the default. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + previous (:storage-object-id (th/db-get :file-data + {:file-id (:id file) :id (:id file)}))] + + (t/is (uuid? previous)) + + ;; A new object is born touched; clear the mark so the assertion below + ;; proves the save is what touches it. + (th/db-update! :storage-object {:touched-at nil} {:id previous}) + (t/is (nil? (:touched-at (find-object previous)))) + + (db/tx-run! th/*system* + (fn [cfg] + (let [file (bfc/get-file cfg (:id file) + :migrate? false + :lock-for-update? true)] + (bfc/update-file! cfg file :modified-at (ct/now))))) + + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + current (:storage-object-id row)] + + (t/testing "the row points at a new object" + (t/is (uuid? current)) + (t/is (not= previous current))) + + (t/testing "the previous object is touched for the storage GC" + (t/is (some? (:touched-at (find-object previous))))))))) diff --git a/backend/test/backend_tests/file_migration_read_test.clj b/backend/test/backend_tests/file_migration_read_test.clj new file mode 100644 index 0000000000..da5821d21f --- /dev/null +++ b/backend/test/backend_tests/file_migration_read_test.clj @@ -0,0 +1,365 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.file-migration-read-test + "Persisting the migrations applied on read is an optimization of the read: + the read already returns migrated data, so the write must never overwrite a + row written after that read, and it must keep a single decoded copy of the + file in the heap." + (:require + [app.binfile.common :as bfc] + [app.common.features :as cfeat] + [app.common.files.migrations :as fmg] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.features.fdata :as fdata] + [app.features.file-migrations :as fmigr] + [app.rpc :as-alias rpc] + [app.util.pointer-map :as pmap] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(def ^:private migration-name + "0031-migrate-stroke-width-token-attr") + +(defn- create-file! + [] + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false})] + {:profile-id (:id profile) + :file-id (:id file) + :page-id (first (:pages (:data (bfc/get-file th/*system* (:id file)))))})) + +(defn- force-migration! + "Make the next read see a pending migration, the way the snapshot tests do: + the version matches, so only a missing row triggers the migration." + [file-id] + (th/db-delete! :file-migration + {:file-id file-id + :name migration-name})) + +(defn- file-data-row + [file-id] + (th/db-get :file-data {:file-id file-id :id file-id})) + +(defn- storage-object + [id] + (th/db-get :storage-object {:id id} {::db/remove-deleted false})) + +(defn- save! + "Run a real save request so the test exercises the same write path a client + takes, not a hand-built row." + [profile-id file-id revn changes] + (th/command! {::th/type :update-file + ::rpc/profile-id profile-id + :id file-id + :session-id (uuid/random) + :revn revn + :vern 0 + :features (cfeat/get-enabled-features cf/flags) + :changes changes})) + +(t/deftest migrate-on-read-does-not-overwrite-a-concurrent-save + ;; A save commits after the read that saw the pending migration. The read + ;; must decide again on the map it takes under the `file` row lock, so it + ;; writes nothing: the save survives, `revn` never goes back (a client whose + ;; revn is higher than the stored one cannot save again) and the object the + ;; save left stays referenced instead of becoming a frozen orphan. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [profile-id file-id page-id]} (create-file!) + shape-id (uuid/random) + fired? (atom false) + replaced (atom nil) + saved-at (atom nil) + saved-revn (atom nil)] + + (force-migration! file-id) + + (with-redefs [fmigr/resolve-applied-migrations + (let [orig fmigr/resolve-applied-migrations] + (fn [cfg file] + (let [result (orig cfg file)] + (when (compare-and-set! fired? false true) + ;; The save commits once the read has already + ;; seen the pending migration. It applies the + ;; migration on its own and records it. + (let [out (save! profile-id file-id 0 + [(th/add-rect-change page-id shape-id)])] + (t/is (nil? (:error out)))) + + (let [row (file-data-row file-id)] + (reset! replaced (:storage-object-id row)) + (reset! saved-revn (:revn (th/db-get :file {:id file-id}))) + (reset! saved-at (:modified-at (th/db-get :file {:id file-id})))) + + ;; Freeze the object the save left, as the storage + ;; GC would if it saw it before a stale write + ;; replaced the row again: a write that does not + ;; touch it leaks the blob forever. + (th/db-update! :storage-object + {:touched-at nil} + {:id @replaced})) + result)))] + + (let [returned (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + row (th/db-get :file {:id file-id}) + persisted (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id + :migrate? false + :realize? true)))] + + (t/testing "the save survives the read" + (t/is (= @saved-revn (:revn returned))) + (t/is (= @saved-revn (:revn row))) + (t/is (= @saved-at (:modified-at row))) + (t/is (some? (get-in persisted [:data :pages-index page-id :objects shape-id])))) + + (t/testing "the read writes nothing and the file ends migrated" + (t/is (not (fmg/need-migration? returned))) + (t/is (= @replaced (:storage-object-id (file-data-row file-id)))) + (t/is (seq (th/db-query :file-migration + {:file-id file-id + :name migration-name})))) + + (t/testing "the next save of the same client does not conflict" + (let [out (save! profile-id file-id @saved-revn + [(th/add-rect-change page-id (uuid/random))])] + (t/is (nil? (:error out)))))))))) + +(t/deftest migration-already-applied-by-another-reader-is-not-persisted + ;; Two readers see the migration pending; the first to get the lock migrates + ;; and persists. The second must re-check on its locked read and write + ;; nothing (a second write would create another object for the same data). + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id]} (create-file!) + fired? (atom false) + pointer (atom nil)] + + (force-migration! file-id) + + (with-redefs [fmg/need-migration? + (let [orig fmg/need-migration?] + (fn [file] + (let [result (orig file)] + (when (and result (compare-and-set! fired? false true)) + ;; The other reader wins the lock, migrates and + ;; persists before this one gets it. + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + (reset! pointer (:storage-object-id (file-data-row file-id)))) + result)))] + + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + + (t/testing "the reader that lost the race does not write" + (t/is (uuid? @pointer)) + (t/is (= @pointer (:storage-object-id (file-data-row file-id))))) + + (t/testing "the file ends migrated" + (t/is (seq (th/db-query :file-migration + {:file-id file-id + :name migration-name})))))))) + +(t/deftest migration-read-loads-the-file-data-once + ;; The probe decides with a row read without data, so the migration path + ;; keeps a single decoded copy of the file in the heap (a large file can be + ;; hundreds of MB and a second copy can exhaust it). + (let [{:keys [file-id]} (create-file!) + reads (atom 0)] + + (force-migration! file-id) + + (with-redefs [fdata/resolve-file-data + (let [orig fdata/resolve-file-data] + (fn [cfg file] + (when (= file-id (:id file)) + (swap! reads inc)) + (orig cfg file)))] + + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + + (t/is (= 1 @reads))))) + +(t/deftest realize-runs-once-per-read-path + ;; `fdata/realize` is a full walk of the decoded data. The read owns it once + ;; per path; the migration path pays it twice on purpose, once inside + ;; `migrate-file` (the chain needs plain data) and once on its result. + (let [{:keys [file-id]} (create-file!) + calls (atom 0) + read! (fn [opts] + (reset! calls 0) + (with-redefs [fdata/realize + (let [orig fdata/realize] + (fn [cfg file] + (swap! calls inc) + (orig cfg file)))] + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id opts)))))] + (t/testing "a plain read realizes once" + (read! {:realize? true}) + (t/is (= 1 @calls))) + + (t/testing "a read-only read realizes once" + (read! {:realize? true :read-only? true}) + (t/is (= 1 @calls))) + + (t/testing "the migration path realizes twice" + (force-migration! file-id) + (read! {:realize? true}) + (t/is (= 2 @calls))))) + +(t/deftest row-only-read-resolves-the-applied-migrations-on-demand + ;; A row-only read resolves the applied migrations by default, because the + ;; row can be completed later and the caller may need them; the explicit + ;; opt-out leaves them out. + (let [{:keys [file-id]} (create-file!) + with-migrations (bfc/get-file th/*system* file-id :load-data? false) + without (bfc/get-file th/*system* file-id + :load-data? false + :load-migrations? false)] + (t/is (seq (:migrations with-migrations))) + (t/is (nil? (:migrations without))))) + +(t/deftest migrating-read-returns-the-pointer-it-persisted + ;; The map a migrating read returns must carry the pointer of the row it + ;; left: a caller that writes with it touches that object instead of the + ;; replaced one. The storage GC freezes a referenced object, so a write + ;; that misses it leaks the blob forever. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id]} (create-file!)] + (force-migration! file-id) + (let [file (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + left (:storage-object-id (file-data-row file-id))] + + (t/testing "the read reports the pointer of the persisted row" + (t/is (uuid? left)) + (t/is (= left (:storage-object-id file)))) + + ;; Freeze the object while the row still references it, as the storage + ;; GC does; the later write must touch it (see the concurrent-save + ;; test for the same technique). + (th/db-update! :storage-object {:touched-at nil} {:id left}) + + (db/tx-run! th/*system* + (fn [cfg] (bfc/update-file! cfg file))) + + (let [row (file-data-row file-id)] + (t/testing "the write replaces the row and touches the object it left" + (t/is (not= left (:storage-object-id row))) + (t/is (some? (:touched-at (storage-object left)))))))))) + +(t/deftest read-completes-with-a-single-row-read + ;; The common read is one row read (with its data columns) and + ;; `attach-file-data` completes it. + (let [{:keys [file-id page-id]} (create-file!) + [file sqls] (th/read-recording-sql + #(db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))))] + (t/is (= [bfc/sql:get-file] sqls)) + (t/is (some? (get-in file [:data :pages-index page-id]))))) + +(t/deftest read-resolves-the-applied-migrations-once + ;; The common read resolves the applied migrations once. The migration path + ;; resolves them again on the locked re-read (it must decide on the fresh + ;; row) and once more after persisting (to return the applied set); no path + ;; resolves them twice without a reason. + (let [{:keys [file-id]} (create-file!) + calls (atom 0) + read! (fn [] + (reset! calls 0) + (with-redefs [fmigr/resolve-applied-migrations + (let [orig fmigr/resolve-applied-migrations] + (fn [cfg file] + (swap! calls inc) + (orig cfg file)))] + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id)))))] + + (read!) + (t/is (= 1 @calls)) + + (force-migration! file-id) + (read!) + (t/is (= 3 @calls)))) + +(t/deftest migration-read-rereads-the-row-under-lock + ;; The first read decides; only when a migration is pending the row is read + ;; again, and that second read takes the `file` row lock. + (let [{:keys [file-id]} (create-file!)] + (force-migration! file-id) + (let [[_ sqls] (th/read-recording-sql + #(db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))))] + (t/is (= [bfc/sql:get-file + (str bfc/sql:get-file " FOR UPDATE of f")] + sqls))))) + +(t/deftest read-only-migration-never-writes + ;; A read-only context migrates in memory: no lock, no row write and no + ;; storage write. The pending migration stays pending. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id page-id]} (create-file!) + pointer (:storage-object-id (file-data-row file-id))] + + (force-migration! file-id) + + (let [returned (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id :read-only? true)))] + + (t/testing "the read returns migrated, realized data" + (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-id]))))) + + (t/testing "nothing is persisted" + (t/is (= pointer (:storage-object-id (file-data-row file-id)))) + (t/is (empty? (th/db-query :file-migration + {:file-id file-id + :name migration-name})))))))) + +(t/deftest read-only-cfg-migration-never-writes + ;; A cfg marked `::db/read-only` migrates in memory too: `db/tx-run!` opens + ;; the transaction read-only, `get-file*` sees the connection read-only and + ;; takes the no-persist branch. The config flag that fills this key is + ;; legacy, but the key is set by hand (objects-gc, the REPL), so the read + ;; must keep working under it. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id page-id]} (create-file!) + pointer (:storage-object-id (file-data-row file-id))] + + (force-migration! file-id) + + (let [returned (db/tx-run! (assoc th/*system* ::db/read-only true) + (fn [cfg] (bfc/get-file cfg file-id)))] + + (t/testing "the read returns migrated data" + (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-id]))))) + + (t/testing "nothing is persisted" + (t/is (= pointer (:storage-object-id (file-data-row file-id)))) + (t/is (empty? (th/db-query :file-migration + {:file-id file-id + :name migration-name})))))))) + +(t/deftest read-only-read-does-not-open-a-transaction + ;; A read-only read takes no lock and writes nothing, so `get-file` runs it + ;; on a plain connection instead of opening a transaction of its own. The + ;; redefinition proves both directions: the same call without the option + ;; does open one. + (let [{:keys [file-id]} (create-file!)] + (with-redefs [db/tx-run! (fn [& _] + (throw (ex-info "opened a transaction" {})))] + (t/is (some? (bfc/get-file th/*system* file-id :read-only? true))) + (t/is (thrown? clojure.lang.ExceptionInfo + (bfc/get-file th/*system* file-id)))))) diff --git a/backend/test/backend_tests/file_pointer_snapshot_test.clj b/backend/test/backend_tests/file_pointer_snapshot_test.clj index 15262eb93e..fd5f305f24 100644 --- a/backend/test/backend_tests/file_pointer_snapshot_test.clj +++ b/backend/test/backend_tests/file_pointer_snapshot_test.clj @@ -856,7 +856,8 @@ (t/deftest migration-keeps-unchanged-page-fragments ;; Applying a pending migration on read keeps the fragment ids of the ;; pages the migration does not change; a page it changes gets a new - ;; id. The caller still receives realized data, not pointers. + ;; id. The read returns decoded data with pointers for the pages the + ;; migration does not change; `:realize?` is what returns it plain. (let [profile (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile) @@ -884,7 +885,9 @@ (let [returned (db/tx-run! th/*system* (fn [cfg] (bfc/get-file cfg file-id)))] - (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-a])))) + ;; The migration changed page-b, so it comes plain; page-a keeps + ;; its pointer (and its fragment id) until someone realizes it. + (t/is (pmap/pointer-map? (get-in returned [:data :pages-index page-a]))) (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-b])))) (t/is (not (fmg/need-migration? returned))) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 1051d749ea..23a0024ded 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -373,8 +373,8 @@ (let [tpoint (ct/tpoint)] (-> system (assoc ::files.update/timestamp (ct/now)) - (db/tx-run! (fn [{:keys [::db/conn] :as system}] - (let [file (files.update/get-file conn file-id)] + (db/tx-run! (fn [system] + (let [file (files.update/get-file system file-id)] (#'files.update/update-file* system {:id file-id :revn revn @@ -848,6 +848,19 @@ [& params] (apply db/get* *pool* params)) +(defn read-recording-sql + "Run `f` and return [result sqls], where `sqls` are the SQL statements of + every `db/get-with-sql` call. Those are exactly the file reads of + `binfile.common/get-file`: the fragment loads go through `db/get*`." + [f] + (let [sqls (atom [])] + (with-redefs [db/get-with-sql + (let [orig db/get-with-sql] + (fn [ds sql & opts] + (swap! sqls conj (first sql)) + (apply orig ds sql opts)))] + [(f) @sqls]))) + (defn sleep [ms-or-duration] (Thread/sleep (inst-ms (ct/duration ms-or-duration)))) diff --git a/backend/test/backend_tests/http_debug_test.clj b/backend/test/backend_tests/http_debug_test.clj index 9c874601f4..d2ac1ee20d 100644 --- a/backend/test/backend_tests/http_debug_test.clj +++ b/backend/test/backend_tests/http_debug_test.clj @@ -10,7 +10,9 @@ resolve the pointer-mapped pages with the file's own loader." (:require [app.http.debug :as debug] + [app.storage :as sto] [backend-tests.helpers :as th] + [clojure.string :as str] [clojure.test :as t] [yetti.response :as-alias yres])) @@ -27,3 +29,28 @@ (t/is (= 200 (::yres/status result))) (t/is (= "NO VALIDATION ERRORS FOUND" (::yres/body result))))) + +(t/deftest debug-download-does-not-expose-the-storage-object-id + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + sobject (sto/put-object! (::sto/storage th/*system*) + {::sto/content (sto/content "sobject") + :content-type "application/octet-stream" + :bucket "file-data" + :file-id (:id file) + :id (:id file)}) + _ (th/db-update! :file-data {:storage-object-id (:id sobject)} + {:file-id (:id file) + :id (:id file)}) + handler (deref #'debug/download-file-data) + result (handler th/*system* {:params {:file-id (str (:id file)) + :download true}}) + body (String. ^bytes (::yres/body result))] + + (t/is (= 200 (::yres/status result))) + (t/is (str/includes? (get-in result [::yres/headers "content-disposition"]) + "attachment")) + (t/is (not (str/includes? body "storage-object-id"))) + (t/is (not (str/includes? body "storage_object_id"))))) diff --git a/backend/test/backend_tests/http_websocket_test.clj b/backend/test/backend_tests/http_websocket_test.clj index 6ffc10e744..64c8b20cbb 100644 --- a/backend/test/backend_tests/http_websocket_test.clj +++ b/backend/test/backend_tests/http_websocket_test.clj @@ -6,7 +6,10 @@ (ns backend-tests.http-websocket-test (:require + [app.common.schema :as sm] + [app.common.time :as ct] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.http.websocket :as ws] [app.msgbus :as mbus] @@ -14,13 +17,23 @@ [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] [app.rpc.commands.teams :as teams] + [app.rpc.notifications :as ntf] [app.util.websocket :as util-ws] [backend-tests.helpers :as th] [clojure.test :as t] [promesa.exec.csp :as sp])) +(defn- with-clean-registry + "Runs `f` against an empty connection registry, restoring the previous + contents afterwards so tests cannot leak connections into each other." + [f] + (let [saved @ws/state] + (reset! ws/state {:connections {} :by-profile {}}) + (try (f) + (finally (reset! ws/state saved))))) + (t/use-fixtures :once th/state-init) -(t/use-fixtures :each th/database-reset) +(t/use-fixtures :each th/database-reset with-clean-registry) (defn make-wsp [profile-id state output-ch] @@ -145,3 +158,1137 @@ (t/is (nil? ((get-method ws/handle-message :pointer-update) cfg empty-wsp msg))))))) + +;; --- SUBSCRIPTION REVOCATION +;; +;; GHSA-m53j-2766-6jqw: a websocket subscription checks read permission +;; once, at subscribe time. Once the connection is subscribed to the +;; message-bus topic it keeps receiving live content for the rest of its +;; life, even after access is revoked. + +(def ^:private revocation-timeout-ms + "How long to wait for a message that should never arrive. Long enough + to absorb the round trip through redis, short enough to keep the test + fast." + 500) + +(defn- drain-window! + "Blocks for up to `ms` draining `ch`, and returns the first message + whose `:type` is `type`, or nil when the window passes without one." + [ch type ms] + (let [deadline (+ (System/currentTimeMillis) ms)] + (loop [] + (let [remain (- deadline (System/currentTimeMillis))] + (when (pos? remain) + (let [[msg _] (sp/alts! [ch (sp/timeout-chan remain)])] + (if (= type (:type msg)) + msg + (recur)))))))) + +(defn- poll-msg! + "Drains `ch` for up to `ms`, returning the first message whose `:type` + is `type`, or nil when none arrives. + + Draining rather than reading a single message matters for the negative + assertions: a live subscription also carries presence traffic, so + \"no `:file-change` arrived\" is the property under test, not \"the + channel stayed empty\"." + [ch type ms] + (let [result (promise)] + (sp/go + (when-let [msg (drain-window! ch type ms)] + (deliver result msg))) + (deref result (inc ms) nil))) + +(def ^:private delivery-attempts + "How many times a positive delivery assertion publishes before giving + up. The second attempt only happens when the subscription was not known + to redis yet; more than that means the traffic is not flowing at all." + 3) + +(defn- poll-published! + "Publishes with `publish!` and returns the first message of `type` that + reaches `ch`, publishing again when a window of `ms` passes without one. + + The subscription reaches redis through an asynchronous action, so a + message published right after subscribing can be lost before redis knows + the subscription, and a single publish makes the test bet on winning + that race. Repeating it after a window that saw nothing removes the + race without weakening the assertion: the test still proves that the + client receives a message it published, and a publish only repeats when + the window saw nothing." + [ch publish! type ms] + (let [result (promise)] + (sp/go + (loop [attempt 1] + (when (<= attempt delivery-attempts) + (publish!) + (if-let [msg (drain-window! ch type ms)] + (deliver result msg) + (recur (inc attempt)))))) + (deref result (+ (* (inc delivery-attempts) ms) 100) nil))) + +(defn- open-registered-connection + "Registers a connection for `profile-id` in the real registry, so the + revocation watcher can find it, and returns the pieces a test needs to + drive it and inspect what it did." + [profile-id] + (let [id (uuid/next) + output (sp/chan :buf (sp/dropping-buffer 64)) + ws-state (atom {}) + wsp (make-wsp profile-id ws-state output)] + (ws/register-connection id wsp) + {:id id :wsp wsp :state ws-state :output output})) + +(defn- open-subscription! + "Opens a registered subscription for `profile-id` and returns the + output channel that receives whatever it forwards. + + The connection goes through the real registry on purpose: these tests + assert the end-to-end path, where a mutation announces the change and + the watcher closes the subscription." + [handler profile-id params] + (let [{:keys [wsp output]} (open-registered-connection profile-id)] + ((get-method ws/handle-message handler) th/*system* wsp params) + output)) + +(defn- wait-until + "Polls `pred` until it holds or `ms` elapses. Returns whether it held." + [pred ms] + (let [deadline (+ (System/currentTimeMillis) ms)] + (loop [] + (cond + (pred) true + (>= (System/currentTimeMillis) deadline) false + :else (do (Thread/sleep 25) (recur)))))) + +(defn- subscribe + [{:keys [wsp] :as conn} handler params] + ((get-method ws/handle-message handler) th/*system* wsp params) + conn) + +(defn- subscribed-file [conn] (-> @(:state conn) ::ws/file-subscription :file-id)) +(defn- subscribed-team [conn] (-> @(:state conn) ::ws/team-subscription :team-id)) + +(defn- publisher! + "Returns a fn that publishes `message` on `topic` through the system + msgbus, the same path `send-notifications!` uses." + [] + (let [msgbus (::mbus/msgbus th/*system*)] + (fn [topic message] + (mbus/pub! msgbus :topic topic :message message)))) + +;; --- CONNECTION REGISTRY + +(t/deftest registry-indexes-connections-by-profile + (let [profile-a (uuid/next) + profile-b (uuid/next) + conn-a (uuid/next) + conn-b (uuid/next) + conn-c (uuid/next)] + + (t/testing "a profile with no connections has an empty index" + (t/is (= #{} (ws/connections-for-profile profile-a)))) + + (ws/register-connection conn-a {::ws/profile-id profile-a}) + (ws/register-connection conn-b {::ws/profile-id profile-a}) + (ws/register-connection conn-c {::ws/profile-id profile-b}) + + (t/testing "connections are indexed under their profile" + (t/is (= #{conn-a conn-b} (ws/connections-for-profile profile-a))) + (t/is (= #{conn-c} (ws/connections-for-profile profile-b)))) + + (t/testing "connections are retrievable by id" + (t/is (= profile-a (::ws/profile-id (ws/get-connection conn-a)))) + (t/is (nil? (ws/get-connection (uuid/next))))) + + (t/testing "unregistering removes it from both the map and the index" + (ws/unregister-connection conn-b) + (t/is (= #{conn-a} (ws/connections-for-profile profile-a))) + (t/is (nil? (ws/get-connection conn-b)))) + + (t/testing "a profile whose last connection left is dropped from the index" + (ws/unregister-connection conn-a) + (t/is (= #{} (ws/connections-for-profile profile-a))) + (t/is (not (contains? (:by-profile @ws/state) profile-a))) + (t/is (= #{conn-c} (ws/connections-for-profile profile-b)))) + + (t/testing "unregistering an unknown connection is a no-op" + (ws/unregister-connection (uuid/next)) + (t/is (= #{conn-c} (ws/connections-for-profile profile-b))) + (t/is (= conn-c (-> (:connections @ws/state) keys first)))))) + +(t/deftest close-file-subscription-drops-only-the-matching-file + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + other (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + output (sp/chan :buf (sp/dropping-buffer 64)) + relay (sp/chan :buf (sp/dropping-buffer 64)) + ws-state (atom {::ws/file-subscription + {:file-id (:id file) + :channel relay + :topic (:id file)}}) + wsp (assoc (make-wsp (:id profile) ws-state output) + ::ws/state ws-state)] + + (t/testing "closing a file the connection is not subscribed to does nothing" + (with-redefs [mbus/pub! (fn [& _] (throw ::unexpected-publish)) + mbus/purge! (fn [& _] (throw ::unexpected-purge))] + (t/is (nil? (ws/close-file-subscription th/*system* wsp (:id other)))) + (t/is (some? (::ws/file-subscription @ws-state))))) + + (t/testing "closing the subscribed file purges the bus" + (let [purged (atom [])] + (with-redefs [mbus/purge! (fn [_ chans] (swap! purged conj chans))] + (ws/close-file-subscription th/*system* wsp (:id file)) + (t/is (= 1 (count @purged)))))) + + (t/testing "the relay channel is closed so the go-loop stops" + (t/is (nil? (sp/poll! relay))) + (t/is (true? (sp/closed? relay)))) + + (t/testing "the subscription is forgotten, so closing again is a no-op" + (t/is (nil? (::ws/file-subscription @ws-state))) + (with-redefs [mbus/pub! (fn [& _] (throw ::unexpected-publish))] + (t/is (nil? (ws/close-file-subscription th/*system* wsp (:id file)))))))) + +(t/deftest revoked-member-stops-receiving-file-changes + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + + (let [project (th/create-project* 1 {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project)}) + file-id (:id file) + publish (publisher!) + change {:type :file-change + :file-id file-id + :revn 1 + :changes [{:type :add :id (uuid/next)}]}] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id file-id})) + out (:output conn)] + + (t/testing "member receives file changes before revocation" + (t/is (some? (poll-published! out #(publish file-id change) + :file-change revocation-timeout-ms)))) + + (t/testing "owner removes the member from the team" + (let [result (th/command! {::th/type :delete-team-member + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor)})] + (t/is (th/success? result)))) + + (t/testing "the mutation alone drops the subscription" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000))) + + (t/testing "revoked member stops receiving file changes" + (publish file-id (assoc change :revn 2)) + (t/is (nil? (poll-msg! out :file-change + revocation-timeout-ms)))))))) +(t/deftest revoked-member-stops-receiving-library-changes + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + + (let [team-id (:id team) + publish (publisher!) + change {:type :library-change + :team-id team-id + :file-id (uuid/next) + :revn 1 + :changes [{:type :add :id (uuid/next)}]}] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-team {:team-id team-id})) + out (:output conn)] + + (t/testing "member receives library changes before revocation" + (t/is (some? (poll-published! out #(publish team-id change) + :library-change revocation-timeout-ms)))) + + (t/testing "member leaves the team" + (let [result (th/command! {::th/type :leave-team + ::rpc/profile-id (:id editor) + :id team-id})] + (t/is (th/success? result)))) + + (t/testing "leaving drops the team subscription" + (t/is (wait-until #(nil? (subscribed-team conn)) 3000))) + + (t/testing "revoked member stops receiving library changes" + (publish team-id (assoc change :revn 2)) + (t/is (nil? (poll-msg! out :library-change + revocation-timeout-ms)))))))) +(t/deftest role-downgrade-keeps-revocation-irrelevant + "A role change that does not remove read access must not be treated as + a revocation: `:viewer` can still read, so the subscription stays." + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + + (let [project (th/create-project* 1 {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project)}) + file-id (:id file) + publish (publisher!) + change {:type :file-change :file-id file-id :revn 1 :changes []} + out (open-subscription! :subscribe-file (:id editor) + {:file-id file-id})] + + (t/testing "editor is downgraded to viewer" + (let [result (th/command! {::th/type :update-team-member-role + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor) + :role :viewer})] + (t/is (th/success? result)))) + + (t/testing "viewer keeps receiving file changes" + (t/is (some? (poll-published! out #(publish file-id change) + :file-change revocation-timeout-ms))))))) + +;; --- REVOCATION WATCHER +;; +;; The registry is local to a backend instance while the message bus is +;; shared, so a revocation travels over the bus and each instance +;; re-verifies the connections it owns. + +(defn- editor-in-team! + "Builds an owner, an editor with read access to `team`, and a project + and file inside it." + [owner-i editor-i] + (let [owner (th/create-profile* owner-i {:is-active true}) + editor (th/create-profile* editor-i {:is-active true}) + team (th/create-team* owner-i {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (let [project (th/create-project* owner-i {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* owner-i {:profile-id (:id editor) + :project-id (:id project)})] + {:owner owner :editor editor :team team :file file}))) + +(defn- delete-team-membership! + "Takes `profile-id` out of `team-id` the way an external change would: + a direct row delete. Fires no command and publishes no announcement, + so no `db/after-commit` hook can race the test." + [team-id profile-id] + (db/exec! th/*system* + ["delete from team_profile_rel where team_id = ? and profile_id = ?" + team-id profile-id])) + +(t/deftest revocation-closes-subscriptions-that-are-no-longer-authorized + (let [{:keys [editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}) + (subscribe :subscribe-team {:team-id (:id team)}))] + + (t/testing "both subscriptions are open before the revocation" + (t/is (= (:id file) (subscribed-file conn))) + (t/is (= (:id team) (subscribed-team conn)))) + + ;; A direct row delete fires no command and publishes no + ;; announcement, so the explicit revalidation below is the only + ;; thing that can close these subscriptions and the count is + ;; deterministic. A command would announce, and the live watcher + ;; could close them before the count is taken. + (t/testing "the member is removed straight over SQL, so nothing announces it" + (delete-team-membership! (:id team) (:id editor)) + (t/is (= (:id file) (subscribed-file conn))) + (t/is (= (:id team) (subscribed-team conn)))) + + (t/testing "revalidation closes the file and the team subscription" + (t/is (= 2 (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + + (t/testing "nothing is left subscribed to relay content" + (t/is (nil? (subscribed-file conn))) + (t/is (nil? (subscribed-team conn)))) + + (t/testing "revalidating again is a no-op" + (t/is (zero? (ws/revalidate-profile-subscriptions + th/*system* (:id editor)))))))) + +(t/deftest revocation-keeps-subscriptions-that-are-still-authorized + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}) + (subscribe :subscribe-team {:team-id (:id team)}))] + + (t/testing "the editor is downgraded to viewer, which can still read" + (let [result (th/command! {::th/type :update-team-member-role + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor) + :role :viewer})] + (t/is (th/success? result)))) + + (t/testing "revalidation closes nothing" + (t/is (zero? (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + + (t/testing "both subscriptions survive" + (t/is (= (:id file) (subscribed-file conn))) + (t/is (= (:id team) (subscribed-team conn))))))) + +(t/deftest revocation-leaves-other-profiles-alone + (let [{:keys [owner editor team file]} (editor-in-team! 1 2) + other-owner (th/create-profile* 3 {:is-active true}) + other-editor (th/create-profile* 4 {:is-active true}) + other-team (th/create-team* 3 {:profile-id (:id other-owner)})] + (th/create-team-role* {:team-id (:id other-team) + :profile-id (:id other-editor) + :role :editor}) + (let [other-file (th/create-file* 3 {:profile-id (:id other-editor) + :project-id (:default-project-id + other-editor)})] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)})) + bystander (-> (open-registered-connection (:id other-editor)) + (subscribe :subscribe-file {:file-id (:id other-file)}))] + + (th/command! {::th/type :delete-team-member + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor)}) + + (ws/revalidate-profile-subscriptions th/*system* (:id editor)) + + (t/testing "the revoked member's subscription is closed" + (t/is (nil? (subscribed-file conn)))) + + (t/testing "a member of another team keeps their subscription" + (t/is (= (:id other-file) (subscribed-file bystander)))) + + (t/testing "revalidating a profile with no connections is safe" + (t/is (zero? (ws/revalidate-profile-subscriptions + th/*system* (uuid/next))))))))) + +(t/deftest watcher-consumes-revocation-events-from-the-bus + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + ;; Access is taken away the way an external change would take + ;; it: a direct row delete fires no command and publishes no + ;; announcement, so the "still subscribed" check cannot race an + ;; after-commit hook, and the command's own announcement cannot + ;; mask a broken watcher. + (delete-team-membership! (:id team) (:id editor)) + + (t/testing "the event is not applied until it is announced" + (t/is (= (:id file) (subscribed-file conn)))) + + (ntf/notify-permissions-changed th/*system* (:id editor)) + + (t/testing "the watcher closes the subscription off the bus" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000)))))) + +(t/deftest internal-revocation-events-are-not-delivered-to-clients + (let [profile (th/create-profile* 1 {:is-active true}) + id (uuid/next) + output (sp/chan :buf (sp/dropping-buffer 64)) + wsp (assoc (make-wsp (:id profile) (atom {}) output) + ::ws/id id)] + + (ws/register-connection id wsp) + ((get-method ws/handle-message :open) th/*system* wsp nil) + + (t/testing "the client does receive its own profile traffic" + (t/is (some? (poll-published! + output + #(mbus/pub! (::mbus/msgbus th/*system*) + :topic (:id profile) + :message {:type :notification :text "hello"}) + :notification + revocation-timeout-ms)))) + + (t/testing "an internal revocation event never reaches the client" + (ntf/notify-permissions-changed th/*system* (:id profile)) + (t/is (nil? (poll-msg! output :profile-permissions-changed + revocation-timeout-ms)))))) + + +;; --- MUTATIONS THAT ANNOUNCE THE CHANGE +;; +;; The watcher only helps if the mutations that can revoke access +;; actually announce it. Each of these asserts the announcement reaches +;; the bus, and that a change which does not remove access announces +;; nothing harmful. + +(defn- captured-events + "Runs `f` collecting the revocation events published on the internal + topic, while letting every other publish reach the real bus untouched." + [f] + (let [events (atom []) + real mbus/pub!] + (with-redefs [mbus/pub! (fn [bus & {:keys [topic message] :as opts}] + (when (= ntf/internal-revocation-topic topic) + (swap! events conj message)) + (apply real bus (mapcat identity opts)))] + (f)) + @events)) + +(defn- event-for + [events profile-id] + (first (filter #(= profile-id (:profile-id %)) events))) + +(t/deftest removing-a-member-announces-the-revocation + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (let [events (captured-events + #(th/command! {::th/type :delete-team-member + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor)}))] + (t/is (= {:type :profile-permissions-changed + :profile-id (:id editor)} + (event-for events (:id editor))))))) + +(t/deftest a-role-change-announces-and-still-closes-nothing + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (let [project (th/create-project* 1 {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project)}) + events (captured-events + #(th/command! {::th/type :update-team-member-role + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor) + :role :viewer}))] + + (t/testing "the change is announced so subscriptions get re-checked" + (t/is (some? (event-for events (:id editor))))) + + (t/testing "a viewer can still read, so the subscription survives" + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + (t/is (zero? (ws/revalidate-profile-subscriptions + th/*system* (:id editor)))) + (t/is (some? (subscribed-file conn)))))))) + +(t/deftest deleting-a-file-announces-to-the-whole-team + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (let [project (th/create-project* 1 {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project)}) + events (captured-events + #(th/command! {::th/type :delete-file + ::rpc/profile-id (:id owner) + :id (:id file)}))] + + ;; One event per team: the watcher resolves the members itself, so + ;; a large team cannot overflow the bounded publish buffers. + (t/is (= [{:type :team-permissions-changed + :team-id (:id team)}] + events))))) + +(t/deftest deleting-a-team-announces-once + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (let [events (captured-events + #(th/command! {::th/type :delete-team + ::rpc/profile-id (:id owner) + :id (:id team)}))] + (t/is (= [{:type :team-permissions-changed + :team-id (:id team)}] + events))))) + +(t/deftest team-event-closes-every-member-subscription + (let [{:keys [editor team file]} (editor-in-team! 1 2) + other (th/create-profile* 3 {:is-active true})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id other) + :role :editor}) + (let [other-project (th/create-project* 3 {:profile-id (:id other) + :team-id (:id team)}) + other-file (th/create-file* 3 {:profile-id (:id other) + :project-id (:id other-project)})] + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)})) + conn2 (-> (open-registered-connection (:id other)) + (subscribe :subscribe-file {:file-id (:id other-file)}))] + + ;; Same shape as the watcher test above: a direct soft-delete + ;; publishes nothing, so this test owns the fan-out of one event + ;; and the announcement of `delete-team` is left to + ;; `deleting-a-team-announces-once`. A second notify here would + ;; make the test pass even if that announcement disappeared. + (db/update! th/*system* :team {:deleted-at (ct/now)} {:id (:id team)}) + + (t/testing "nothing closes until the event arrives" + (t/is (some? (subscribed-file conn))) + (t/is (some? (subscribed-file conn2)))) + + (t/testing "one team event reaches the watcher off the bus" + (ntf/notify-team-permissions-changed th/*system* (:id team)) + (t/is (wait-until #(nil? (subscribed-file conn)) 3000)) + (t/is (wait-until #(nil? (subscribed-file conn2)) 3000))))))) + +(t/deftest rolled-back-revocation-announces-nothing + (let [editor (th/create-profile* 2 {:is-active true}) + events (captured-events + #(try + (db/tx-run! th/*system* + (fn [_cfg] + (ntf/notify-permissions-changed th/*system* (:id editor)) + (throw (ex-info "boom" {})))) + (catch clojure.lang.ExceptionInfo _)))] + (t/is (empty? events)))) + +(t/deftest leaving-a-team-announces-and-tells-only-the-leaver + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + other (th/create-profile* 3 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team) + :profile-id (:id editor) + :role :editor}) + (th/create-team-role* {:team-id (:id team) + :profile-id (:id other) + :role :editor}) + (let [published (atom []) + real mbus/pub!] + (with-redefs [mbus/pub! (fn [bus & args] + (let [opts (apply hash-map args)] + (swap! published conj opts)) + (apply real bus args))] + (th/command! {::th/type :leave-team + ::rpc/profile-id (:id editor) + :id (:id team)})) + + (t/testing "the leaver is told they were removed, so the client leaves" + (let [msg (->> @published + (filter #(= (:id editor) (:topic %))) + (map :message) + first)] + (t/is (= :team-membership-change (:type msg))) + (t/is (= :removed (:change msg))) + (t/is (= (:id team) (:team-id msg))))) + + (t/testing "no other member is told they were removed" + (t/is (empty? (filter #(= (:id other) (:topic %)) @published))))))) + +(t/deftest deleting-a-share-link-announces-nothing + ;; A share-link visitor never passes the websocket subscribe check: + ;; `check-read-permissions!` is called without a share-id, and that + ;; arity only considers team membership. There is no subscription to + ;; revoke, so announcing would be noise. + (let [owner (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id owner) + :project-id (:default-project-id owner)})] + (th/command! {::th/type :create-share-link + ::rpc/profile-id (:id owner) + :file-id (:id file) + :who-comment "none" + :who-inspect "none" + :pages #{}}) + (let [link-id (:id (db/exec-one! th/*system* + ["select id from share_link where file_id = ?" + (:id file)])) + events (captured-events + #(th/command! {::th/type :delete-share-link + ::rpc/profile-id (:id owner) + :id link-id}))] + (t/is (some? link-id)) + (t/is (empty? events))))) + +;; --- PERIODIC REVALIDATION +;; +;; The announcement path is the primary mechanism. This interval is the +;; safety net for access changes that fire no command here, such as +;; Nitrate transferring the ownership of an organization, or rows +;; deleted straight over SQL. + +(t/deftest revalidation-interval-defaults-to-five-minutes + (t/is (= (ct/duration {:minutes 5}) ws/default-revalidation-interval)) + (t/is (= (ct/duration {:minutes 5}) (ws/revalidation-interval)))) + +(t/deftest revalidation-is-configurable + (with-redefs [cf/get (fn [k default] (if (= :subscription-revalidation-interval k) + (ct/duration {:seconds 30}) + default))] + (t/is (= (ct/duration {:seconds 30}) (ws/revalidation-interval))))) + +(t/deftest interval-recheck-drops-a-subscription-without-any-announcement + (let [{:keys [editor team file]} (editor-in-team! 1 2)] + + (with-redefs [cf/get (fn [k default] + (if (= :subscription-revalidation-interval k) + (ct/duration {:millis 150}) + default))] + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)})) + out (:output conn)] + + (t/testing "the subscription is live at first" + (t/is (some? (subscribed-file conn)))) + + ;; Access is taken away the way an external change would take + ;; it: a direct row delete fires no command and publishes no + ;; announcement, so the interval is the only thing that can + ;; close the subscription. + (delete-team-membership! (:id team) (:id editor)) + + (t/testing "the interval re-check drops it anyway" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000))) + + (t/testing "and nothing reaches the client afterwards" + (mbus/pub! (::mbus/msgbus th/*system*) + :topic (:id file) + :message {:type :file-change :file-id (:id file) :revn 9}) + (t/is (nil? (poll-msg! out :file-change + revocation-timeout-ms)))))))) + +(t/deftest interval-recheck-keeps-a-subscription-that-still-holds + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (with-redefs [cf/get (fn [k default] + (if (= :subscription-revalidation-interval k) + (ct/duration {:millis 150}) + default))] + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)})) + out (:output conn)] + + (t/testing "a downgrade to viewer still allows reading" + (t/is (th/success? (th/command! {::th/type :update-team-member-role + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor) + :role :viewer})))) + + ;; Wait for at least two interval ticks, so a re-check that + ;; wrongly closed the subscription would have run. + (Thread/sleep 500) + + (t/testing "the subscription survives the interval" + (t/is (some? (subscribed-file conn)))) + + (t/testing "and traffic still flows" + (t/is (some? (poll-published! + out + #(mbus/pub! (::mbus/msgbus th/*system*) + :topic (:id file) + :message {:type :file-change + :file-id (:id file) + :revn 3}) + :file-change + revocation-timeout-ms)))))))) + +(t/deftest interval-recheck-covers-the-team-subscription-too + (let [{:keys [editor team]} (editor-in-team! 1 2)] + + (with-redefs [cf/get (fn [k default] + (if (= :subscription-revalidation-interval k) + (ct/duration {:millis 150}) + default))] + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-team {:team-id (:id team)}))] + + (t/is (some? (subscribed-team conn))) + + ;; Same external-change shape as above: no command, no + ;; announcement, only the interval. + (delete-team-membership! (:id team) (:id editor)) + + (t/testing "the team subscription is dropped by the interval" + (t/is (wait-until #(nil? (subscribed-team conn)) 3000))))))) + +(t/deftest a-false-check-closes-the-subscription + (let [{:keys [editor file]} (editor-in-team! 1 2)] + + (with-redefs [cf/get (fn [k default] + (if (= :subscription-revalidation-interval k) + (ct/duration {:millis 150}) + default))] + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + (with-redefs [files/has-read-permissions? (fn [& _] false)] + (t/testing "a check that returns false closes it" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000)))))))) + +(t/deftest resubscribe-stops-previous-revalidation + (let [profile (th/create-profile* 1 {:is-active true}) + file1 (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + file2 (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (with-redefs [cf/get (fn [k default] + (if (= :subscription-revalidation-interval k) + (ct/duration {:millis 150}) + default))] + (let [counts (atom {}) + real files/has-read-permissions?] + (with-redefs [files/has-read-permissions? + (fn [cfg pid fid] + (swap! counts update fid (fnil inc 0)) + (real cfg pid fid))] + (let [conn (-> (open-registered-connection (:id profile)) + (subscribe :subscribe-file {:file-id (:id file1)}))] + + (t/testing "the first subscription is re-checked on the interval" + (t/is (wait-until #(pos? (get @counts (:id file1) 0)) 3000))) + + (t/testing "resubscribing moves the relay to the new file" + (subscribe conn :subscribe-file {:file-id (:id file2)}) + (t/is (wait-until #(>= (get @counts (:id file2) 0) 2) 3000))) + + (t/testing "the previous subscription stops revalidating" + (let [old (get @counts (:id file1) 0)] + (Thread/sleep 400) + (t/is (= old (get @counts (:id file1) 0))))))))))) + +;; --- EFFECT TESTS +;; +;; Announcing is not the same as revoking: these assert the subscription +;; actually drops and the content stops flowing. + +(t/deftest deleting-a-file-closes-open-subscriptions + (let [{:keys [owner editor file]} (editor-in-team! 1 2) + publish (publisher!)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)})) + out (:output conn)] + + (th/command! {::th/type :delete-file + ::rpc/profile-id (:id owner) + :id (:id file)}) + + (t/testing "the file subscription drops without any manual publish" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000))) + + (t/testing "no file content reaches the client afterwards" + (publish (:id file) {:type :file-change :file-id (:id file) :revn 9}) + (t/is (nil? (poll-msg! out :file-change revocation-timeout-ms))))))) + +(t/deftest deleting-a-project-closes-open-subscriptions + (let [{:keys [owner editor file]} (editor-in-team! 1 2) + project-id (:project-id file)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)})) + out (:output conn)] + + (th/command! {::th/type :delete-project + ::rpc/profile-id (:id owner) + :id project-id}) + + (t/testing "the file subscription drops" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000))) + + (t/testing "no file content reaches the client afterwards" + (mbus/pub! (::mbus/msgbus th/*system*) + :topic (:id file) + :message {:type :file-change :file-id (:id file) :revn 9}) + (t/is (nil? (poll-msg! out :file-change revocation-timeout-ms))))))) + +(t/deftest deleting-a-team-closes-open-subscriptions + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}) + (subscribe :subscribe-team {:team-id (:id team)}))] + + (th/command! {::th/type :delete-team + ::rpc/profile-id (:id owner) + :id (:id team)}) + + (t/testing "both subscriptions drop" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000)) + (t/is (wait-until #(nil? (subscribed-team conn)) 3000)))))) + +(t/deftest moving-a-file-closes-source-team-subscriptions + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team-a (th/create-team* 1 {:profile-id (:id owner)}) + team-b (th/create-team* 2 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team-a) + :profile-id (:id editor) + :role :editor}) + (let [project-a (th/create-project* 1 {:profile-id (:id owner) + :team-id (:id team-a)}) + project-b (th/create-project* 2 {:profile-id (:id owner) + :team-id (:id team-b)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project-a)}) + conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + (th/command! {::th/type :move-files + ::rpc/profile-id (:id owner) + :ids #{(:id file)} + :project-id (:id project-b)}) + + (t/testing "a member left behind in the source team loses the subscription" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000)))))) + +(t/deftest moving-a-project-closes-source-team-subscriptions + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team-a (th/create-team* 1 {:profile-id (:id owner)}) + team-b (th/create-team* 2 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id team-a) + :profile-id (:id editor) + :role :editor}) + (let [project-a (th/create-project* 1 {:profile-id (:id owner) + :team-id (:id team-a)}) + project-b (th/create-project* 2 {:profile-id (:id owner) + :team-id (:id team-b)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project-a)}) + conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}) + (subscribe :subscribe-team {:team-id (:id team-a)}))] + + (th/command! {::th/type :move-project + ::rpc/profile-id (:id owner) + :team-id (:id team-b) + :project-id (:id project-a)}) + + (t/testing "the file subscription drops: the file only lives in team-b now" + (t/is (wait-until #(nil? (subscribed-file conn)) 3000))) + + (t/testing "the team subscription survives: the editor is still a member of team-a" + (t/is (= (:id team-a) (subscribed-team conn))))))) + +(t/deftest close-team-subscription-is-idempotent + (let [profile (th/create-profile* 1 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id profile)}) + other (th/create-team* 2 {:profile-id (:id profile)}) + output (sp/chan :buf (sp/dropping-buffer 64)) + relay (sp/chan :buf (sp/dropping-buffer 64)) + ws-state (atom {::ws/team-subscription + {:team-id (:id team) + :channel relay + :topic (:id team)}}) + wsp (assoc (make-wsp (:id profile) ws-state output) + ::ws/state ws-state)] + + (t/testing "closing a team the connection is not subscribed to does nothing" + (with-redefs [mbus/purge! (fn [& _] (throw ::unexpected-purge))] + (t/is (nil? (ws/close-team-subscription th/*system* wsp (:id other)))) + (t/is (some? (::ws/team-subscription @ws-state))))) + + (t/testing "closing the subscribed team purges the bus and the relay stops" + (let [purged (atom [])] + (with-redefs [mbus/purge! (fn [_ chans] (swap! purged conj chans))] + (ws/close-team-subscription th/*system* wsp (:id team)) + (t/is (= 1 (count @purged))))) + (t/is (nil? (sp/poll! relay))) + (t/is (true? (sp/closed? relay)))) + + (t/testing "the subscription is forgotten, so closing again is a no-op" + (t/is (nil? (::ws/team-subscription @ws-state))) + (t/is (nil? (ws/close-team-subscription th/*system* wsp (:id team))))))) + +(t/deftest losing-a-team-keeps-the-other-file-subscription + (let [{:keys [owner editor team]} (editor-in-team! 1 2) + other-team (th/create-team* 3 {:profile-id (:id owner)})] + (th/create-team-role* {:team-id (:id other-team) + :profile-id (:id editor) + :role :editor}) + (let [other-project (th/create-project* 3 {:profile-id (:id editor) + :team-id (:id other-team)}) + other-file (th/create-file* 3 {:profile-id (:id editor) + :project-id (:id other-project)}) + conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-team {:team-id (:id team)}) + (subscribe :subscribe-file {:file-id (:id other-file)}))] + + (th/command! {::th/type :delete-team-member + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor)}) + + (t/testing "the lost team subscription drops" + (t/is (wait-until #(nil? (subscribed-team conn)) 3000))) + + (t/testing "the file subscription on the other team survives" + (t/is (some? (subscribed-file conn))))))) + +;; The revocation watcher is the only integrant key here, so it is the +;; only one that compiles this schema at boot: a ref with no registered +;; schema crashes the boot with :malli.core/invalid-schema instead of +;; asserting. The test system builds the component, but assert-key only +;; runs its check with *assert* set, so pin the compilation here. +(t/deftest revocation-watcher-schema-resolves-every-ref + (t/testing "the schema compiles" + (t/is (sm/schema? (sm/schema @#'ws/schema:revocation-watcher)))) + + (t/testing "every ref the key injects is registered" + (t/is (= #{::db/pool ::mbus/msgbus} + (disj (set (sm/keys @#'ws/schema:revocation-watcher)) + :app.nitrate/client)))) + (t/testing "the nitrate client is optional, so a nil one passes" + (t/is (sm/valid? @#'ws/schema:revocation-watcher + {::db/pool (:app.db/pool th/*system*) + ::mbus/msgbus (:app.msgbus/msgbus th/*system*)}))) + (t/testing "a non-map nitrate client is rejected" + (t/is (not (sm/valid? @#'ws/schema:revocation-watcher + {::db/pool (:app.db/pool th/*system*) + ::mbus/msgbus (:app.msgbus/msgbus th/*system*) + :app.nitrate/client "not-a-client"}))))) + +(t/deftest organization-owner-keeps-read-only-access + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + ;; The admin-console flag is off in tests, so Nitrate never grants + ;; this; stand in for an organization owner expelled from the team. + (with-redefs [nitrate/organization-owner-of-team? (fn [& _] true)] + (th/command! {::th/type :delete-team-member + ::rpc/profile-id (:id owner) + :team-id (:id team) + :member-id (:id editor)}) + + (t/testing "revalidation closes nothing for an organization owner" + (t/is (zero? (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + + (t/testing "the read-only subscription survives" + (t/is (some? (subscribed-file conn)))))))) + +(t/deftest soft-deleted-file-closes-org-owner-subscription + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + ;; The admin-console flag is off in tests, so Nitrate never grants + ;; this; stand in for an organization owner expelled from the team. + (with-redefs [nitrate/organization-owner-of-team? (fn [& _] true)] + ;; Access is taken away the way an external change would take + ;; it: a direct row soft-delete fires no command and publishes no + ;; announcement. Membership SQL grants nothing through a deleted + ;; file, but the org-owner fallback does not look at deleted_at, + ;; so only the existence gate can close the subscription. + (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id file)}) + + (t/testing "revalidation closes the subscription although Nitrate still grants read-only" + (t/is (= 1 (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + (t/testing "the read-only subscription is gone" + (t/is (nil? (subscribed-file conn)))))))) + +(t/deftest soft-deleted-team-closes-org-owner-subscription + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-team {:team-id (:id team)}))] + + ;; Same shape as the file case: the org-owner fallback grants + ;; read-only access without looking at the team's deleted_at. + (with-redefs [nitrate/organization-owner-of-team? (fn [& _] true)] + (db/update! th/*system* :team {:deleted-at (ct/now)} {:id (:id team)}) + + (t/testing "revalidation closes the subscription although Nitrate still grants read-only" + (t/is (= 1 (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + (t/testing "the read-only subscription is gone" + (t/is (nil? (subscribed-team conn)))))))) + +(t/deftest soft-deleted-project-closes-org-owner-file-subscription + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + ;; The admin-console flag is off in tests, so Nitrate never grants + ;; this; stand in for an organization owner expelled from the team. + (with-redefs [nitrate/organization-owner-of-team? (fn [& _] true)] + ;; A direct row soft-delete fires no command and publishes no + ;; announcement. Membership SQL denies through a deleted project, + ;; but the org-owner fallback resolves the team without any + ;; deleted_at filter, so only the root fix can close the file + ;; subscription. + (db/update! th/*system* :project {:deleted-at (ct/now)} + {:id (:project-id file)}) + + (t/testing "revalidation closes the file subscription although Nitrate still grants read-only" + (t/is (= 1 (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + (t/testing "the read-only subscription is gone" + (t/is (nil? (subscribed-file conn)))))))) + +(t/deftest soft-deleted-team-closes-org-owner-file-subscription + (let [{:keys [owner editor team file]} (editor-in-team! 1 2)] + + (let [conn (-> (open-registered-connection (:id editor)) + (subscribe :subscribe-file {:file-id (:id file)}))] + + ;; Same shape with the ancestor unavailable at the team level: + ;; membership denies (t.deleted_at is null), the org-owner + ;; fallback grants, and only resolve-team-id's team-row check + ;; can close the file subscription. + (with-redefs [nitrate/organization-owner-of-team? (fn [& _] true)] + (db/update! th/*system* :team {:deleted-at (ct/now)} {:id (:id team)}) + + (t/testing "revalidation closes the file subscription although Nitrate still grants read-only" + (t/is (= 1 (ws/revalidate-profile-subscriptions + th/*system* (:id editor))))) + (t/testing "the read-only subscription is gone" + (t/is (nil? (subscribed-file conn)))))))) + +;; --- POLLING HELPERS + +(t/deftest a-delivery-assertion-republishes-when-the-subscription-lands-late + ;; The subscription reaches redis through an asynchronous action, so the + ;; first publish of a delivery assertion can be lost before redis knows + ;; the subscription. The assertion republishes: this pins that it does, + ;; and that it stops as soon as a message arrives. + (let [ch (sp/chan :buf (sp/dropping-buffer 8)) + published (atom 0) + publish! (fn [] + (when (= 2 (swap! published inc)) + (sp/put! ch {:type :notification :text "hello"})))] + (t/is (some? (poll-published! ch publish! :notification 50))) + (t/is (= 2 @published) + "the second publish is the last one, once the message arrived"))) diff --git a/backend/test/backend_tests/media_test.clj b/backend/test/backend_tests/media_test.clj index ddc8c6ca3d..aac2cd6c98 100644 --- a/backend/test/backend_tests/media_test.clj +++ b/backend/test/backend_tests/media_test.clj @@ -12,7 +12,10 @@ [backend-tests.helpers :as th] [clojure.test :as t] [cuerdas.core :as str] - [datoteka.fs :as fs])) + [datoteka.fs :as fs] + [datoteka.io :as io]) + (:import + java.nio.ByteBuffer)) (t/use-fixtures :once th/state-init) @@ -554,3 +557,23 @@ :height 200})] (t/is (= :webp (:format thumb))) (t/is (= "image/webp" (:mtype thumb)))))) + +(defn- sfnt-num-glyphs + "Returns numGlyphs from the maxp table of the sfnt font in `data`." + [^bytes data] + (let [buf (ByteBuffer/wrap data) + num-tables (.getShort buf 4)] + (some (fn [i] + (let [rec (+ 12 (* i 16))] + (when (= "maxp" (String. data rec 4 "ISO-8859-1")) + (.getShort buf (+ (.getInt buf (+ rec 8)) 4))))) + (range num-tables)))) + +(t/deftest generate-fonts-cid-keyed-otf + (t/testing "otf->ttf keeps the glyphs of every CID sub-font" + ;; font-cid.otf has 5 glyphs split across two CID sub-fonts + (let [data (io/read* (io/resource "backend_tests/test_files/font-cid.otf")) + result (media/run th/*system* {:cmd :generate-fonts + :input {"font/otf" data}}) + ttf (io/read* (get result "font/ttf"))] + (t/is (<= 5 (sfnt-num-glyphs ttf)))))) diff --git a/backend/test/backend_tests/migrations_legacy_file_data_test.clj b/backend/test/backend_tests/migrations_legacy_file_data_test.clj index e2a31ce2bf..5fcd1874ec 100644 --- a/backend/test/backend_tests/migrations_legacy_file_data_test.clj +++ b/backend/test/backend_tests/migrations_legacy_file_data_test.clj @@ -13,6 +13,7 @@ [app.common.time :as ct] [app.common.uuid :as uuid] [app.db :as db] + [app.features.file-changes :as fchanges] [app.util.blob :as blob] [backend-tests.helpers :as th] [clojure.java.io :as io] @@ -58,6 +59,7 @@ :revn 1 :data snap-data :label "v1" + :kind fchanges/snapshot-kind :created-at created-at :updated-at updated-at :deleted-at deleted-at}) @@ -69,7 +71,8 @@ {:id rev-id :file-id fid :revn 2 - :data rev-data}) + :data rev-data + :kind fchanges/revision-kind}) ;; Live revision (changes, no data, no label): untouched. live-id (uuid/next) @@ -77,7 +80,8 @@ {:id live-id :file-id fid :revn 3 - :changes (blob/encode [{:type :dummy}])})] + :changes (blob/encode [{:type :dummy}]) + :kind fchanges/revision-kind})] (run-migration) @@ -168,7 +172,8 @@ :file-id fid :revn 1 :label "v1" - :data stale})] + :data stale + :kind fchanges/snapshot-kind})] (run-migration) diff --git a/backend/test/backend_tests/migrations_partition_file_change_test.clj b/backend/test/backend_tests/migrations_partition_file_change_test.clj new file mode 100644 index 0000000000..b75310a5e8 --- /dev/null +++ b/backend/test/backend_tests/migrations_partition_file_change_test.clj @@ -0,0 +1,225 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.migrations-partition-file-change-test + "Covers the data copy of 0158-partition-file-change: the rows of the old + table move to the partitioned one with their columns in place, the `kind` + comes from the label (1 snapshot, 2 revision), it stays nullable so a + rollback keeps writing, and the legacy table keeps the rows." + + (:require + [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.db :as db] + [app.util.blob :as blob] + [backend-tests.helpers :as th] + [clojure.java.io :as io] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(def ^:private sql:old-schema + "CREATE TABLE file (id uuid PRIMARY KEY); + CREATE TABLE profile (id uuid PRIMARY KEY); + + CREATE TABLE file_change ( + file_id uuid NOT NULL, + id uuid NOT NULL DEFAULT gen_random_uuid(), + created_at timestamptz NOT NULL DEFAULT now(), + session_id uuid, + revn bigint NOT NULL DEFAULT 0, + data bytea, + changes bytea, + profile_id uuid, + features text[], + label text, + version integer, + created_by text NOT NULL DEFAULT 'system', + updated_at timestamptz DEFAULT now(), + deleted_at timestamptz, + migrations text[], + locked_by uuid, + data_ref_id uuid, + data_backend text, + CONSTRAINT file_change_pkey PRIMARY KEY (id), + CONSTRAINT file_change_file_id_fkey FOREIGN KEY (file_id) + REFERENCES file(id) DEFERRABLE, + CONSTRAINT file_change_locked_by_fkey FOREIGN KEY (locked_by) + REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE, + CONSTRAINT file_change_profile_id_fkey FOREIGN KEY (profile_id) + REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE); + + CREATE INDEX file_change__data_ref_id__idx + ON file_change (data_ref_id); + CREATE INDEX file_change__deleted_at__idx + ON file_change (deleted_at) WHERE deleted_at IS NOT NULL; + CREATE INDEX file_change__file_id__revn__idx + ON file_change (file_id, revn); + CREATE INDEX file_change__locked_by__idx + ON file_change (locked_by) WHERE locked_by IS NOT NULL; + CREATE INDEX file_change__profile_id__idx + ON file_change (profile_id) WHERE profile_id IS NOT NULL; + CREATE INDEX file_change__system_snapshots__idx + ON file_change (file_id, created_at) + WHERE created_by = 'system' AND deleted_at IS NULL; + CREATE INDEX file_change__user_snapshots__idx + ON file_change (file_id) + WHERE label IS NOT NULL AND created_by = 'user' AND deleted_at IS NULL;") + +(defn- run-migration! + [conn] + (let [sql (slurp (io/resource "app/migrations/sql/0158-partition-file-change.sql"))] + (db/exec! conn [sql]))) + +(t/deftest copies-the-rows-into-the-partitioned-table + ;; The migration runs in its own schema, with the shape of the table before + ;; it: the run below is the only place where its INSERT..SELECT runs with + ;; rows, so a swapped column or a wrong `kind` fails here and not on a real + ;; instance. + (let [schema (str "test_0158_" (subs (str (uuid/next)) 0 8)) + fid (uuid/next) + snap-id (uuid/next) + rev-id (uuid/next) + created-at (ct/in-past "2h") + updated-at (ct/in-past "1h") + deleted-at (ct/in-past "30m") + snap-data (blob/encode {:snapshot 1}) + rev-changes (blob/encode [{:type :add-obj :id (uuid/next)}])] + + (db/tx-run! th/*pool* + (fn [{:keys [::db/conn]}] + ;; the schema is dropped before the commit, and a failure + ;; rolls the whole transaction back: nothing is left behind + (db/exec! conn [(str "CREATE SCHEMA " schema)]) + (db/exec! conn [(str "SET LOCAL search_path TO " schema)]) + (db/exec! conn [sql:old-schema]) + + (db/insert! conn :file {:id fid}) + + (db/insert! conn :file-change + {:id snap-id + :file-id fid + :revn 1 + :data snap-data + :label "v1" + :version 1 + :created-by "user" + :created-at created-at + :updated-at updated-at + :deleted-at deleted-at + ;; a row that still carries the dead columns + :data-ref-id (uuid/next) + :data-backend "legacy-db"}) + + (db/insert! conn :file-change + {:id rev-id + :file-id fid + :revn 2 + :changes rev-changes + :created-at created-at + :updated-at updated-at}) + + (run-migration! conn) + + (let [rows (db/exec! conn ["SELECT id, revn, kind, label, data, changes, + created_at, updated_at, deleted_at, created_by + FROM file_change + ORDER BY revn"]) + snapshot (first rows) + revision (second rows) + legacy (db/exec! conn ["SELECT id, revn, data, changes + FROM file_change_legacy + ORDER BY revn"])] + + (t/testing "both rows are in the partitioned table" + (t/is (= 2 (count rows))) + (t/is (= [1 2] (mapv :revn rows)))) + + (t/testing "the labelled row is a snapshot with its payload" + (t/is (= 1 (:kind snapshot))) + (t/is (= "v1" (:label snapshot))) + (t/is (= "user" (:created-by snapshot))) + (t/is (= (seq snap-data) (seq ^bytes (:data snapshot)))) + (t/is (nil? (:changes snapshot))) + (t/is (= (inst-ms created-at) (inst-ms (:created-at snapshot)))) + (t/is (= (inst-ms updated-at) (inst-ms (:updated-at snapshot)))) + (t/is (= (inst-ms deleted-at) (inst-ms (:deleted-at snapshot))))) + + (t/testing "the unlabelled row is a revision with its changes" + (t/is (= 2 (:kind revision))) + (t/is (nil? (:label revision))) + (t/is (= (seq rev-changes) (seq ^bytes (:changes revision)))) + (t/is (nil? (:data revision))) + (t/is (= (inst-ms created-at) (inst-ms (:created-at revision))))) + + (t/testing "the legacy table keeps the rows for the rollback" + (t/is (= 2 (count legacy))) + (t/is (= [snap-id rev-id] (mapv :id legacy))) + (t/is (= (seq snap-data) + (seq ^bytes (:data (first legacy))))) + (t/is (= (seq rev-changes) + (seq ^bytes (:changes (second legacy))))))) + + (t/testing "the legacy table no longer points at file" + ;; Its foreign key on `file_id` is `NO ACTION`: while the table has + ;; rows, the objects GC cannot delete the files they point to. The + ;; migration drops that one and keeps the two `ON DELETE SET NULL` ones. + (t/is (= ["file_change_locked_by_fkey_legacy" + "file_change_profile_id_fkey_legacy"] + (->> (db/exec! conn ["SELECT con.conname AS name + FROM pg_constraint con + WHERE con.conrelid = 'file_change_legacy'::regclass + AND con.contype = 'f' + ORDER BY con.conname"]) + (mapv :name))))) + + (t/testing "the dead columns are not carried over" + ;; The old row carried `data_ref_id` and `data_backend`; + ;; the new table has no such columns. + (t/is (= [] + (->> (db/exec! conn ["SELECT attname AS name + FROM pg_attribute + WHERE attrelid = 'file_change'::regclass + AND attnum > 0 AND NOT attisdropped + AND attname IN ('data_ref_id', 'data_backend')"]) + (mapv :name))))) + + (t/testing "kind stays nullable so a rollback keeps writing" + ;; The previous version inserts without `kind`: the column + ;; must accept NULL, like the runbook's staging column. + (t/is (false? (:notnull (db/exec-one! conn ["SELECT attnotnull AS notnull + FROM pg_attribute + WHERE attrelid = 'file_change'::regclass + AND attname = 'kind'"])))) + (let [rb-id (uuid/next)] + (db/insert! conn :file-change {:id rb-id :file-id fid :revn 99}) + (t/is (nil? (:kind (db/exec-one! conn ["SELECT kind + FROM file_change + WHERE file_id = ? AND id = ?" + fid rb-id])))))) + + (db/exec! conn [(str "DROP SCHEMA " schema " CASCADE")]))))) + + +(t/deftest drops-the-legacy-table-when-it-is-empty + ;; On a new instance there is nothing to keep: the migration leaves no + ;; legacy table behind. The relation is asserted on its own, so a missing + ;; table can never pass for a clean one. + (let [schema (str "test_0158_empty_" (subs (str (uuid/next)) 0 8))] + (db/tx-run! th/*pool* + (fn [{:keys [::db/conn]}] + (db/exec! conn [(str "CREATE SCHEMA " schema)]) + (db/exec! conn [(str "SET LOCAL search_path TO " schema)]) + (db/exec! conn [sql:old-schema]) + + (run-migration! conn) + + (t/is (-> (db/exec-one! conn ["SELECT to_regclass('file_change_legacy') IS NULL AS empty"]) + :empty) + "the empty legacy table is dropped") + + (db/exec! conn [(str "DROP SCHEMA " schema " CASCADE")]))))) diff --git a/backend/test/backend_tests/migrations_test.clj b/backend/test/backend_tests/migrations_test.clj index 12597a4968..9a93eaecac 100644 --- a/backend/test/backend_tests/migrations_test.clj +++ b/backend/test/backend_tests/migrations_test.clj @@ -6,7 +6,9 @@ (ns backend-tests.migrations-test (:require + [app.common.uuid :as uuid] [app.db :as db] + [app.features.fdata :as fdata] [app.migrations :as migrations] [app.util.migrations :as mg] [backend-tests.helpers :as th] @@ -15,6 +17,7 @@ [clojure.test :as t])) (t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) (t/deftest migrations-registry-satisfies-steps-spec (t/testing "every entry is a {:name :fn} step (guards against stray forms)" @@ -51,6 +54,18 @@ (map :conname) set)) +(defn- foreign-key-flags + "The deferral and delete-action flags of one foreign key. `condeferred` is + true only for `INITIALLY DEFERRED`." + [table name] + (-> (th/db-exec! ["SELECT condeferrable, condeferred, confdeltype, convalidated + FROM pg_constraint + WHERE contype = 'f' + AND conname = ? + AND conrelid = ?::regclass" + name table]) + first)) + (t/deftest job-table-exists-with-expected-columns (t/is (= #{; dispatch/lifecycle columns "id" "name" "tenant" "queue" "label" "priority" "scheduled_at" @@ -216,3 +231,205 @@ (t/is (not (contains? triggers "job__modified_at__tgr")))) (t/testing "deletion protection is the only trigger on job" (t/is (= #{"deletion_protection__tgr"} triggers))))) + +;; --- File data storage reference (migration 0159) + +(def ^:private file-data-migration + (mg/resource "app/migrations/sql/0159-add-file-data-storage-object-id.sql")) + +(defn- apply-file-data-migration! + [] + (db/tx-run! (::db/pool th/*system*) + (fn [{:keys [::db/conn]}] + (file-data-migration conn)))) + +(t/deftest file-data-storage-object-reference-is-a-deferrable-no-action-fk + (t/testing "the pointer column exists" + (t/is (contains? (table-columns "file_data") "storage_object_id"))) + (t/testing "the index that serves the FK check and the GC scan exists" + (t/is (contains? (table-indexes "file_data") + "file_data__storage_object_id__idx"))) + (t/testing "the reference does not cascade" + (t/is (contains? (table-foreign-keys "file_data") + ["file_data_storage_object_id_fkey" "a"]))) + (t/testing "the reference is DEFERRABLE but checked immediately" + ;; The storage GC catches the violation inside the chunk transaction. An + ;; `INITIALLY DEFERRED` check fires at commit, outside the catch, and the + ;; sweep would never move past the blocked chunk. + (t/is (= {:condeferrable true + :condeferred false + :confdeltype "a" + :convalidated true} + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey"))))) + +(t/deftest deleting-a-referenced-storage-object-fails + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (:id (th/db-insert! :storage-object {:backend "fs"})) + data-id (uuid/next)] + + (th/db-insert! :file-data + {:file-id (:id file) + :id data-id + :type "snapshot" + :backend "storage" + :storage-object-id sobject}) + + (t/testing "a live reference refuses the physical delete" + (let [cause (try + (th/db-exec! ["DELETE FROM storage_object WHERE id = ?" sobject]) + nil + (catch java.sql.SQLException cause cause))] + (t/is (some? cause)) + (t/is (= "23503" (.getSQLState cause))))) + + (th/db-delete! :file-data {:file-id (:id file) :id data-id}) + + (t/testing "without the reference the delete succeeds" + (th/db-exec! ["DELETE FROM storage_object WHERE id = ?" sobject]) + (t/is (nil? (th/db-get :storage-object {:id sobject})))))) + +(t/deftest file-data-migration-backfills-and-is-idempotent + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (:id (th/db-insert! :storage-object {:backend "fs"})) + data-id (uuid/next)] + + (th/db-insert! :file-data + {:file-id (:id file) + :id data-id + :type "snapshot" + :backend "storage" + :metadata (db/json {"storage-ref-id" (str sobject)})}) + + (t/testing "the backfill copies the metadata pointer into the column" + (apply-file-data-migration!) + (t/is (= sobject (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id data-id}))))) + + (t/testing "the backfill removes the dead metadata pointer" + (let [row (th/db-get :file-data {:file-id (:id file) :id data-id}) + metadata (fdata/decode-metadata (:metadata row))] + (t/is (nil? (:storage-ref-id metadata))))) + + (t/testing "a second application changes nothing" + (let [before (th/db-get :file-data {:file-id (:id file) :id data-id})] + (apply-file-data-migration!) + (t/is (= before (th/db-get :file-data {:file-id (:id file) + :id data-id}))))))) + +(t/deftest file-data-migration-guards-corrupt-and-orphan-pointers + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (:id (th/db-insert! :storage-object {:backend "fs"})) + valid-id (uuid/next) + corrupt-id (uuid/next) + orphan-id (uuid/next) + other-id (uuid/next)] + + (th/db-insert! :file-data + {:file-id (:id file) :id valid-id :type "snapshot" :backend "storage" + :metadata (db/json {"storage-ref-id" (str sobject)})}) + (th/db-insert! :file-data + {:file-id (:id file) :id corrupt-id :type "snapshot" :backend "storage" + :metadata (db/json {"storage-ref-id" "not-a-uuid"})}) + (th/db-insert! :file-data + {:file-id (:id file) :id orphan-id :type "snapshot" :backend "storage" + :metadata (db/json {"storage-ref-id" (str (uuid/next))})}) + (th/db-insert! :file-data + {:file-id (:id file) :id other-id :type "snapshot" :backend "db" + :metadata (db/json {"storage-ref-id" (str sobject)})}) + + (t/testing "the migration applies without aborting" + (t/is (true? (apply-file-data-migration!)))) + + (t/testing "a valid pointer is backfilled" + (t/is (= sobject (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id valid-id}))))) + + (t/testing "a non-uuid value stays NULL" + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id corrupt-id}))))) + + (t/testing "a pointer to a missing object stays NULL" + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id orphan-id}))))) + + (t/testing "rows of other backends are not touched" + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id other-id}))))))) + +(t/deftest file-data-migration-creates-a-missing-index + ;; In the test database the index already exists, so the creation branch is + ;; only reachable by dropping it first. It is the branch a fresh deployment + ;; runs. + (th/db-exec! ["DROP INDEX file_data__storage_object_id__idx"]) + (t/is (not (contains? (table-indexes "file_data") + "file_data__storage_object_id__idx"))) + + (t/is (true? (apply-file-data-migration!))) + + (t/is (contains? (table-indexes "file_data") + "file_data__storage_object_id__idx"))) + +(t/deftest file-data-migration-rebuilds-a-wrong-foreign-key + ;; The migration owns the definition. A constraint created by hand with + ;; CASCADE would delete the referencing rows instead of refusing the delete, + ;; so it is replaced with the NO ACTION one. + (th/db-exec! ["ALTER TABLE file_data + DROP CONSTRAINT file_data_storage_object_id_fkey"]) + (try + (th/db-exec! ["ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE CASCADE DEFERRABLE"]) + (t/is (true? (apply-file-data-migration!))) + (t/is (= "a" (:confdeltype + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey")))) + (finally + (apply-file-data-migration!)))) + +(t/deftest file-data-migration-rebuilds-a-non-deferrable-foreign-key + ;; Without DEFERRABLE the constraint breaks `SET CONSTRAINTS ALL DEFERRED` + ;; and `database-reset`; the migration recreates it deferrable. + (th/db-exec! ["ALTER TABLE file_data + DROP CONSTRAINT file_data_storage_object_id_fkey"]) + (try + (th/db-exec! ["ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION"]) + (t/is (true? (apply-file-data-migration!))) + (t/is (true? (:condeferrable + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey")))) + (finally + (apply-file-data-migration!)))) + +(t/deftest file-data-migration-rebuilds-a-deferred-foreign-key + ;; An `INITIALLY DEFERRED` check fires at commit, outside the + ;; `storage-gc-deleted` chunk transaction, so the sweep would never move past + ;; a blocked chunk. The migration must leave the check immediate. + (th/db-exec! ["ALTER TABLE file_data + DROP CONSTRAINT file_data_storage_object_id_fkey"]) + (try + (th/db-exec! ["ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"]) + (t/is (true? (apply-file-data-migration!))) + (t/is (false? (:condeferred + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey")))) + (finally + (apply-file-data-migration!)))) diff --git a/backend/test/backend_tests/objects_gc_test.clj b/backend/test/backend_tests/objects_gc_test.clj index 6696a7f2f0..462cbc5f5a 100644 --- a/backend/test/backend_tests/objects_gc_test.clj +++ b/backend/test/backend_tests/objects_gc_test.clj @@ -8,8 +8,10 @@ (:require [app.common.logging :as l] [app.common.time :as ct] + [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] + [app.features.file-changes :as fchanges] [app.rpc :as-alias rpc] [app.storage :as sto] [app.tasks.objects-gc :as ogc] @@ -73,6 +75,69 @@ (t/is (pos? (:processed result))) (t/is (nil? (find-file (:id file)))))) +(t/deftest deletes-the-pending-file-changes + ;; The pending file changes are locked per file, with a constant `file_id`, + ;; so the rows of a file that is pending deletion are gone after the run. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + + _ (th/db-insert! :file-change + {:file-id (:id file) + :kind fchanges/revision-kind + :deleted-at (ct/in-past "10m")}) + + result (th/run-task! :objects-gc {})] + + (t/is (pos? (:processed result))) + (t/is (nil? (th/db-get :file-change {:file-id (:id file)} + {::db/remove-deleted false}))))) + +(t/deftest deletes-the-pending-file-changes-of-several-chunks + ;; More pending rows than the chunk size: the plan walks the cursor and each + ;; chunk locks its own rows, one query per file, with a constant `file_id` + ;; so the planner prunes the partition. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + ids (repeatedly 5 uuid/next) + _ (doseq [id ids] + (th/db-insert! :file-change + {:id id + :file-id (:id file) + :kind fchanges/revision-kind + :deleted-at (ct/in-past "10m")})) + result (th/run-task! :objects-gc {:chunk-size 2})] + + (t/is (pos? (:processed result))) + (t/is (empty? (th/db-query :file-change {:file-id (:id file)} + {::db/remove-deleted false}))))) + +(t/deftest a-chunk-spanning-several-files-deletes-all-of-them + ;; The rows of one chunk can belong to different files. Each group is locked + ;; with its own `file_id`, and every row is still deleted; buffering the + ;; groups instead of locking them would leave them behind. + (let [profile (th/create-profile* 1) + opts {:profile-id (:id profile) + :project-id (:default-project-id profile)} + files [(th/create-file* 1 opts) + (th/create-file* 2 opts)] + + _ (doseq [file files] + (dotimes [_ 2] + (th/db-insert! :file-change + {:file-id (:id file) + :kind fchanges/revision-kind + :deleted-at (ct/in-past "10m")}))) + + ;; one chunk holds the rows of both files + result (th/run-task! :objects-gc {:chunk-size 4})] + + (t/is (pos? (:processed result))) + (doseq [file files] + (t/is (empty? (th/db-query :file-change {:file-id (:id file)} + {::db/remove-deleted false})))))) + (t/deftest profile-cascade-keeps-profile-deletion-date ;; The cascade marks the owned rows with the deletion date of the profile, ;; like the RPC delete-profile flow does, not with the timestamp of the run. @@ -222,6 +287,34 @@ (t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false}))) (t/is (empty? (:call-args-list @audit-mock)))))) +(t/deftest gc-deletes-file-data-and-touches-its-storage-object + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (sto/put-object! (::sto/storage th/*system*) + {::sto/content (sto/content "file-data") + :content-type "application/octet-stream" + :bucket "file-data" + :file-id (:id file) + :id (:id file)})] + + ;; `main` always carries `id = file_id`: mark the real row as deleted + ;; instead of inventing a second main row. + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id sobject) + :deleted-at (ct/in-past "30m")} + {:file-id (:id file) :id (:id file)}) + + (th/run-task! :objects-gc {}) + + (t/testing "the row is purged" + (t/is (nil? (th/db-get :file-data {:file-id (:id file) :id (:id file)})))) + + (t/testing "the storage object is touched for the storage GC" + (t/is (some? (:touched-at (th/db-get :storage-object {:id (:id sobject)}))))))) + (t/deftest gc-emits-events-only-for-committed-chunks (with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}] (with-redefs [cf/flags (conj cf/flags :audit-log)] diff --git a/backend/test/backend_tests/offload_file_data_test.clj b/backend/test/backend_tests/offload_file_data_test.clj index e7158f0567..8f4fad28a9 100644 --- a/backend/test/backend_tests/offload_file_data_test.clj +++ b/backend/test/backend_tests/offload_file_data_test.clj @@ -62,6 +62,42 @@ (t/is (every? #(= "storage" (:backend %)) after)) (t/is (every? #(nil? (:data %)) after))))) +(t/deftest offloads-a-file-data-row-with-metadata + ;; `SELECT fd.*` returns jsonb metadata as a `PGobject`; the offload has to + ;; decode it before `fdata/upsert!` validates the row. Imported files carry + ;; metadata, so without the normalization the job never offloads them. + (let [profile (th/create-profile* 1) + file (create-file 1 profile)] + + (th/db-update! :file-data + {:metadata (db/json {"generated-by" "offload-test"})} + {:file-id (:id file) :id (:id file)}) + + (let [result (th/run-task! :offload-file-data {:file-id (:id file)}) + row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (pos? (:offloaded result))) + (t/is (= "storage" (:backend row))) + (t/is (uuid? (:storage-object-id row))) + (t/is (= {:generated-by "offload-test"} + (fdata/decode-metadata (:metadata row))))))) + +(t/deftest offloads-a-file-data-row-with-a-json-null-metadata + ;; A jsonb `null` decodes to nil; the key must disappear from the row instead + ;; of travelling as `:metadata nil`, which the update schema rejects. + (let [profile (th/create-profile* 1) + file (create-file 1 profile)] + + (th/db-exec! ["UPDATE file_data SET metadata = 'null'::jsonb + WHERE file_id = ? AND id = ?" + (:id file) (:id file)]) + + (let [result (th/run-task! :offload-file-data {:file-id (:id file)}) + row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (pos? (:offloaded result))) + (t/is (= "storage" (:backend row))) + (t/is (uuid? (:storage-object-id row))) + (t/is (nil? (:metadata row)))))) + (t/deftest skips-file-locked-by-another-transaction (let [profile (th/create-profile* 1) file (create-file 1 profile) @@ -385,36 +421,65 @@ (let [result (th/run-task! :offload-file-data {:file-id (:id file)})] (t/is (pos? (:offloaded result)))) - (t/testing "live snapshot rows moved to cold storage" - (let [snapshots (th/db-query :file-data {:file-id (:id file) - :type "snapshot" - :deleted-at nil})] - (t/is (pos? (count snapshots))) - (t/is (every? #(= "storage" (:backend %)) snapshots)) - (t/is (every? #(nil? (:data %)) snapshots)))) + (th/run-task! :storage-gc-touched {:skip-delay true}) - (t/testing "snapshots are still listed" - (let [out (th/command! {::th/type :get-file-snapshots - ::rpc/profile-id (:id profile) - :file-id (:id file)})] - (t/is (nil? (:error out))) - (t/is (some #(= "before-offload" (:label %)) (:result out))))) + ;; The main row points at a cold object, and the storage GC froze it + ;; because the row still references it. The restore must touch it before + ;; replacing the row, or the blob is never reclaimed. + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file) + :id (:id file)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))) + + (t/testing "live snapshot rows moved to cold storage" + (let [snapshots (th/db-query :file-data {:file-id (:id file) + :type "snapshot" + :deleted-at nil})] + (t/is (pos? (count snapshots))) + (t/is (every? #(= "storage" (:backend %)) snapshots)) + (t/is (every? #(nil? (:data %)) snapshots)))) + + (t/testing "snapshots are still listed" + (let [out (th/command! {::th/type :get-file-snapshots + ::rpc/profile-id (:id profile) + :file-id (:id file)})] + (t/is (nil? (:error out))) + (t/is (some #(= "before-offload" (:label %)) (:result out))))) - (t/testing "snapshot restores from cold storage" (let [snap-id (->> (:result (th/command! {::th/type :get-file-snapshots ::rpc/profile-id (:id profile) :file-id (:id file)})) (filter #(= "before-offload" (:label %))) first - :id) - out (th/command! {::th/type :restore-file-snapshot + :id)] + (t/is (some? snap-id)) + + (t/testing "snapshot preview does not expose the storage pointer" + (let [out (th/command! {::th/type :get-file-snapshot ::rpc/profile-id (:id profile) :file-id (:id file) :id snap-id})] - (t/is (some? snap-id)) - (t/is (nil? (:error out))) - (t/is (some? (get-in (get-file-data profile (:id file)) - [:pages-index page-id :objects shape-id]))))))) + (t/is (nil? (:error out))) + (t/is (not (contains? (:result out) :storage-object-id))))) + + (t/testing "snapshot restores from cold storage" + (let [out (th/command! {::th/type :restore-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :id snap-id})] + (t/is (nil? (:error out))) + (t/is (some? (get-in (get-file-data profile (:id file)) + [:pages-index page-id :objects shape-id])))))) + + (t/testing "the replaced main object is touched for the storage GC" + (t/is (some? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false}))))) + + (t/testing "the main row is back on hot storage without the old pointer" + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row)))))))) (t/deftest write-after-offload-returns-to-hot-storage (let [profile (th/create-profile* 1 {:is-active true}) @@ -483,11 +548,7 @@ (let [result (th/run-task! :offload-file-data {:file-id (:id file)})] (t/is (pos? (:offloaded result)))) - (let [ref-ids (into #{} - (comp (map :metadata) - (map fdata/decode-metadata) - (keep :storage-ref-id)) - (db-rows (:id file)))] + (let [ref-ids (into #{} (keep :storage-object-id) (db-rows (:id file)))] (t/is (pos? (count ref-ids))) (t/is (every? some? (map #(sto/get-object storage %) ref-ids))) diff --git a/backend/test/backend_tests/restore_storage_objects_test.clj b/backend/test/backend_tests/restore_storage_objects_test.clj index 5986170b20..ba19a17c67 100644 --- a/backend/test/backend_tests/restore_storage_objects_test.clj +++ b/backend/test/backend_tests/restore_storage_objects_test.clj @@ -9,6 +9,7 @@ [app.common.time :as ct] [app.common.uuid :as uuid] [app.db :as db] + [app.features.file-changes :as fchanges] [app.rpc.commands.files :as files] [app.srepl.main :as srepl] [app.storage :as sto] @@ -90,8 +91,8 @@ file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile)}) data-id (uuid/next) - ;; file-data objects carry the ids of their row: the GC resolves the - ;; reference with that pair. + ;; the row points at its object through `storage_object_id`; the + ;; metadata ids are informational for diagnostics sobject (:id (put-object! "file-data" {:file-id (:id file) :id data-id}))] @@ -102,7 +103,7 @@ :type "main" :backend "fs" :deleted-at (ct/in-past {:days 1}) - :metadata (db/json {"storage-ref-id" (str sobject)})}) + :storage-object-id sobject}) (mark-object! sobject) (restore-file! (:id file)) @@ -130,3 +131,29 @@ (srepl/restore-team! (:id team))) (t/is (nil? (:deleted-at (find-object sobject)))))) + +(t/deftest restoring-a-file-updates-its-file-changes + ;; Restoring a file brings its file changes back to life, and that mutation + ;; has to be observable through `updated_at`, the watermark used by the + ;; online partitioning copy of `file_change`. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + + stale-at #inst "2020-01-01T00:00:00.000Z" + + _ (th/db-insert! :file-change + {:file-id (:id file) + :label "v1" + :kind fchanges/snapshot-kind + :deleted-at (ct/in-past {:days 1}) + :updated-at stale-at})] + + (restore-file! (:id file)) + + (let [change (th/db-get :file-change {:file-id (:id file)} + {::db/remove-deleted false})] + (t/is (nil? (:deleted-at change))) + (t/is (pos? (- (inst-ms (:updated-at change)) + (inst-ms stale-at))) + "updated_at must be bumped by the restore")))) diff --git a/backend/test/backend_tests/rpc_auth_test.clj b/backend/test/backend_tests/rpc_auth_test.clj index e2a4a54f8c..32eda259d2 100644 --- a/backend/test/backend_tests/rpc_auth_test.clj +++ b/backend/test/backend_tests/rpc_auth_test.clj @@ -10,6 +10,7 @@ [app.auth.login-lockout :as lol] [app.common.flags :as flags] [app.common.generic-pool :as gpool] + [app.common.time :as ct] [app.common.uuid :as uuid] [app.http.session :as session] [app.loggers.audit :as-alias audit] @@ -17,6 +18,7 @@ [app.rpc.commands.auth :as auth] [app.rpc.commands.ldap :as ldap-cmd] [app.storage :as sto] + [app.tokens :as tokens] [backend-tests.helpers :as th] [clojure.test :as t] [datoteka.io :as io] @@ -28,6 +30,80 @@ (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) +(t/deftest recover-profile-invalidates-existing-sessions + "Regression: password recovery must delete the victim's server-side + sessions, otherwise a previously captured cookie keeps working + (GHSA-3j2p-qr45-q365)." + (let [prof (th/create-profile* 1) + manager (::session/manager th/*system*) + sid1 (uuid/random) + sid2 (uuid/random) + _ (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, user_agent) VALUES (?, ?, ?)" + sid1 (:id prof) "agent-1"]) + _ (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, user_agent) VALUES (?, ?, ?)" + sid2 (:id prof) "agent-2"]) + token (tokens/generate th/*system* + {:iss :password-recovery + :exp (ct/in-future "15m") + :profile-id (:id prof)})] + (with-mocks [_ {:target 'app.email/send-template :return nil}] + (let [out (th/command! {::th/type :recover-profile + :token token + :password "Foobar12!"})] + (t/is (nil? (:error out))))) + (t/is (nil? (session/read-session manager sid1)) + "recover-profile must delete the victim's sessions (GHSA-3j2p-qr45-q365)") + (t/is (nil? (session/read-session manager sid2)) + "recover-profile must delete the victim's sessions (GHSA-3j2p-qr45-q365)"))) + +(t/deftest recover-profile-keeps-other-profiles-sessions + (let [prof-a (th/create-profile* 1) + prof-b (th/create-profile* 2) + manager (::session/manager th/*system*) + sid-a (uuid/random) + sid-b (uuid/random) + _ (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, user_agent) VALUES (?, ?, ?)" + sid-a (:id prof-a) "agent-a"]) + _ (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, user_agent) VALUES (?, ?, ?)" + sid-b (:id prof-b) "agent-b"]) + token (tokens/generate th/*system* + {:iss :password-recovery + :exp (ct/in-future "15m") + :profile-id (:id prof-a)})] + (with-mocks [_ {:target 'app.email/send-template :return nil}] + (let [out (th/command! {::th/type :recover-profile + :token token + :password "Foobar12!"})] + (t/is (nil? (:error out))))) + (t/is (nil? (session/read-session manager sid-a))) + (t/is (some? (session/read-session manager sid-b)) + "sessions of other profiles must survive the recovery"))) + +(t/deftest recover-profile-rollback-keeps-password-and-sessions + "A failure after the password UPDATE rolls the transaction back, so the + captured session and the old password both survive + (GHSA-3j2p-qr45-q365)." + (let [prof (th/create-profile* 1) + manager (::session/manager th/*system*) + sid (uuid/random) + _ (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, user_agent) VALUES (?, ?, ?)" + sid (:id prof) "agent-1"]) + before (:password (th/db-exec-one! ["SELECT password FROM profile WHERE id = ?" (:id prof)])) + token (tokens/generate th/*system* + {:iss :password-recovery + :exp (ct/in-future "15m") + :profile-id (:id prof)})] + (with-mocks [_ {:target 'app.email/send-template + :throw (ex-info "smtp down" {})}] + (let [out (th/command! {::th/type :recover-profile + :token token + :password "Foobar12!"})] + (t/is (some? (:error out))))) + (t/is (some? (session/read-session manager sid)) + "a rolled-back recovery must not delete sessions") + (t/is (= before (:password (th/db-exec-one! ["SELECT password FROM profile WHERE id = ?" (:id prof)]))) + "a rolled-back recovery must not change the password"))) + (t/deftest logout-invalidates-current-session (let [prof (th/create-profile* 1) manager (::session/manager th/*system*) diff --git a/backend/test/backend_tests/rpc_comment_test.clj b/backend/test/backend_tests/rpc_comment_test.clj index 3447e5aad8..9b29853c75 100644 --- a/backend/test/backend_tests/rpc_comment_test.clj +++ b/backend/test/backend_tests/rpc_comment_test.clj @@ -15,6 +15,7 @@ [app.rpc.commands.comments :as comments] [app.rpc.cond :as cond] [app.rpc.quotes :as-alias quotes] + [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] [datoteka.fs :as fs] @@ -1058,3 +1059,28 @@ :team-id (:id team)})] (t/is (th/success? out)) (t/is (= 0 (count (:result out)))))))) + +(t/deftest internal-file-map-for-comments-omits-the-storage-object-id + ;; The comments module builds its file map through `comments/get-file`; the + ;; storage pointer is internal bookkeeping and must not travel in it. The + ;; RPC responses never embed the file map, so this pins the internal choke + ;; point instead. + (let [profile (th/create-profile* 1 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id profile)}) + project (th/create-project* 1 {:team-id (:id team) + :profile-id (:id profile)}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project)}) + page-id (get-in file [:data :pages 0]) + sobject (sto/put-object! (::sto/storage th/*system*) + {::sto/content (sto/content "sobject") + :content-type "application/octet-stream" + :bucket "file-data" + :file-id (:id file) + :id (:id file)}) + _ (th/db-update! :file-data {:storage-object-id (:id sobject)} + {:file-id (:id file) + :id (:id file)}) + result ((deref #'comments/get-file) th/*system* (:id file) page-id)] + + (t/is (not (contains? result :storage-object-id))))) diff --git a/backend/test/backend_tests/rpc_file_snapshot_test.clj b/backend/test/backend_tests/rpc_file_snapshot_test.clj index a4bc17a76c..34d415f52a 100644 --- a/backend/test/backend_tests/rpc_file_snapshot_test.clj +++ b/backend/test/backend_tests/rpc_file_snapshot_test.clj @@ -88,6 +88,19 @@ (t/is (= 0 (:revn row))) (t/is (= (:id profile) (:profile-id row)))))) + (t/testing "update snapshot with file-id" + (let [params {::th/type :update-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :id @snapshot-id + :label "label1-updated"} + out (th/command! params)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (t/is (true? (:result out))) + (t/is (= "label1-updated" + (:label (th/db-get :file-change {:id @snapshot-id})))))) + (t/testing "restore snapshot" (let [params {::th/type :restore-file-snapshot ::rpc/profile-id (:id profile) @@ -102,9 +115,10 @@ ;; (th/print-result! out) (t/is (nil? (:error out))) (t/is (nil? (:result out))) - ;; Peers learn who restored which version + ;; Peers learn who restored which version, named by the label the + ;; snapshot has now (the previous block renamed it) (t/is (= (:id profile) (:profile-id sent))) - (t/is (= "label1" (:label sent))) + (t/is (= "label1-updated" (:label sent))) (t/is (ct/inst? (:created-at sent)))) (let [[row1 row2 :as rows] @@ -277,10 +291,11 @@ nil {:data (:data preview)}))] (t/is (= preview-data live))))) -(t/deftest user-snapshot-of-file-needing-migration-is-materialized - ;; A file with pending migrations cannot share fragments: the - ;; migration path realizes the data, so the stored snapshot holds - ;; plain maps. This pins the shape so it cannot drift silently. +(t/deftest user-snapshot-of-file-needing-migration-shares-fragments + ;; A file with pending migrations is migrated on read before the snapshot + ;; is taken. The read keeps the pointers of the pages the migration does + ;; not change, so the snapshot shares those fragments with the live file + ;; like any other snapshot instead of storing a materialized copy. (let [profile (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile) @@ -295,7 +310,11 @@ snap-id (:id (:result out)) snap (fsnap/get-snapshot th/*system* file-id snap-id)] (t/is (uuid? snap-id)) - (t/is (empty? (fdata/get-used-pointer-ids (:data snap)))) + (let [snap-pointers (fdata/get-used-pointer-ids (:data snap)) + live-pointers (fdata/get-used-pointer-ids + (:data (bfc/get-file th/*system* file-id)))] + (t/is (seq snap-pointers)) + (t/is (every? live-pointers snap-pointers))) ;; ...while preview keeps working on the migrated content. (let [preview (th/command! {::th/type :get-file-snapshot ::rpc/profile-id (:id profile) @@ -419,6 +438,7 @@ (t/testing "renaming the branch base pin is refused" (let [out (th/command! {::th/type :update-file-snapshot ::rpc/profile-id (:id profile) + :file-id (:id file) :id (:id pin) :label "stolen-label"})] (t/is (some? (:error out))) @@ -459,6 +479,7 @@ {::db/remove-deleted false}) out (th/command! {::th/type :update-file-snapshot ::rpc/profile-id (:id profile) + :file-id (:id file2) :id (:id backup) :label "stolen-backup"})] (t/is (uuid? (:id backup))) @@ -503,29 +524,69 @@ ;; would be committed whatever the restore's outcome (t/is (empty? (th/db-query :file-change {:file-id branch-id :created-by "system"})))))) +(t/deftest snapshot-creation-reads-the-file-row-for-update + ;; The read inherits the caller's lock, and the migration-on-read path may + ;; re-read the same row with FOR UPDATE. A shared lock would be an upgrade, + ;; and two concurrent readers can deadlock there. The read of the command + ;; takes the exclusive lock. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + [out sqls] (th/read-recording-sql + #(th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "pinned"}))] + (t/is (nil? (:error out))) + (t/is (uuid? (get-in out [:result :id]))) + (t/is (= [(str bfc/sql:get-file " FOR UPDATE of f")] + sqls)))) + +(t/deftest restore-backup-reads-the-file-row-for-update + ;; The backup read is the same reader as the snapshot creation, so it also + ;; takes the exclusive lock: the migration-on-read re-read cannot upgrade a + ;; shared one and deadlock with a concurrent reader. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + snap-id (:id (:result (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "v1"}))) + [out sqls] (th/read-recording-sql + #(th/command! {::th/type :restore-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :id snap-id}))] + (t/is (nil? (:error out))) + (t/is (some #(= (str bfc/sql:get-file " FOR UPDATE of f") %) sqls)) + (t/is (not-any? #(str/includes? % "FOR SHARE") sqls)))) + (t/deftest snapshot-creation-serializes-with-file-gc ;; A user snapshot shares fragment rows with the live file, so its ;; creation must serialize with a concurrent :file-gc pass (which ;; takes FOR UPDATE + SKIP LOCKED on the file row): creation holds - ;; FOR SHARE, the GC skips locked files instead of racing it. + ;; FOR UPDATE, the GC skips locked files instead of racing it. (let [profile (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile) :is-shared false}) file-id (:id file)] - (t/testing "gc skips a file locked for share" + (t/testing "gc skips a file locked for update" (db/tx-run! th/*system* (fn [cfg] - ;; Hold FOR SHARE to commit, like create-file-snapshot does. - (bfc/get-file cfg file-id :lock-for-share? true) + ;; Hold FOR UPDATE to commit, like create-file-snapshot does. + (bfc/get-file cfg file-id :lock-for-update? true) ;; The GC uses SKIP LOCKED on a different connection, so it ;; must skip instead of blocking or marking. (t/is (not (th/run-task! :file-gc {:file-id file-id})))))) (t/testing "creation cannot slip between gc scan and mark" ;; Deterministic interleaving: pause the GC right after its - ;; snapshot scan, then create a snapshot. With the FOR SHARE lock + ;; snapshot scan, then create a snapshot. With the FOR UPDATE lock ;; the creation waits at the read and sees post-GC fragment ids; ;; without it, it reads pre-GC ids and pins a fragment the pass ;; is about to mark. @@ -552,7 +613,7 @@ (t/is (= true (deref scan-started 10000 ::timeout))) (let [created (future (do-create))] ;; The creation must not reach the insert while the GC - ;; pass holds the file row: with FOR SHARE it waits at the + ;; pass holds the file row: with FOR UPDATE it waits at the ;; read instead of snapshotting pre-GC fragment ids. (t/is (= ::timeout (deref reached-insert 10000 ::timeout))) (deliver release-scan true) @@ -588,7 +649,7 @@ (t/deftest restore-backup-serializes-with-file-gc ;; The backup snapshot taken by restore shares fragment rows with ;; the live file, so its creation must serialize with a concurrent - ;; :file-gc pass, like ::create-file-snapshot does: with FOR SHARE + ;; :file-gc pass, like ::create-file-snapshot does: with FOR UPDATE ;; held by the GC pass, the restore waits at the locked read instead ;; of snapshotting pre-GC fragment ids that the pass is about to ;; mark as deleted. @@ -863,3 +924,231 @@ (let [stored (fsnap/get-snapshot th/*system* file-id snap-id)] (t/is (nil? (:migrations stored))) (t/is (= 60 (:version stored))))))) +(t/deftest snapshots-require-file-id + ;; Since version 2.20 the snapshot commands require the `file-id` + ;; parameter, so partition pruning can always be applied on the + ;; `file_change` table. + (let [profile (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id profile) + proj-id (:default-project-id profile) + + file (th/create-file* 1 {:profile-id (:id profile) + :project-id proj-id + :is-shared false}) + + snapshot + (let [params {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "label1"} + out (th/command! params)] + ;; (th/print-result! out) + + (t/is (nil? (:error out))) + (:result out))] + + (doseq [type [:update-file-snapshot :lock-file-snapshot + :unlock-file-snapshot :delete-file-snapshot]] + (let [params (merge {::th/type type + ::rpc/profile-id (:id profile) + :id (:id snapshot)} + (when (= type :update-file-snapshot) + {:label "label2"})) + out (th/command! params)] + + (t/testing (str (name type) " without file-id") + (t/is (th/ex-of-code? (:error out) :params-validation))))))) + +(t/deftest snapshots-reject-a-file-id-that-does-not-match + ;; The snapshot commands filter by `file_id` so that, once `file_change` is + ;; hash-partitioned by it, the lookup prunes to a single partition. Sending + ;; a `file-id` of another file must not find nor touch the snapshot. + (let [profile (th/create-profile* 1 {:is-active true}) + file-a (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + file-b (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + + snapshot (-> (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file-a) + :label "label1"}) + :result)] + + (doseq [type [:update-file-snapshot :lock-file-snapshot + :unlock-file-snapshot :delete-file-snapshot]] + (let [params (merge {::th/type type + ::rpc/profile-id (:id profile) + :file-id (:id file-b) + :id (:id snapshot)} + (when (= type :update-file-snapshot) + {:label "label2"})) + out (th/command! params)] + + (t/testing (name type) + (t/is (th/ex-info? (:error out))) + (t/is (= :not-found (:type (ex-data (:error out))))) + (t/is (= :snapshot-not-found (:code (ex-data (:error out))))) + (t/is (= "label1" + (:label (th/db-get :file-change {:id (:id snapshot)} + {::db/remove-deleted false}))) + "the snapshot of file-a stays intact")))))) + +(t/deftest file-change-mutations-maintain-updated-at + ;; `updated_at` is the per-row watermark used by the online partitioning + ;; copy of `file_change` (hash partitioning by file_id): every mutation + ;; must bump it, otherwise the copy misses the row. Lock, unlock and the + ;; soft-delete used to leave it stale. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + + snapshot + (let [params {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "label1"} + out (th/command! params)] + ;; (th/print-result! out) + + (t/is (nil? (:error out))) + (:result out)) + + stale-at #inst "2020-01-01T00:00:00.000Z"] + + (doseq [type [:lock-file-snapshot :unlock-file-snapshot + :update-file-snapshot :delete-file-snapshot]] + (t/testing (name type) + ;; make the row look old, so the mutation has to bump it + (th/db-update! :file-change {:updated-at stale-at} + {:id (:id snapshot)} + {::db/remove-deleted false}) + + (let [params (cond-> {::th/type type + ::rpc/profile-id (:id profile) + :file-id (:id file) + :id (:id snapshot)} + (= :update-file-snapshot type) + (assoc :label "renamed")) + out (th/command! params) + row (th/db-get :file-change {:id (:id snapshot)} + {::db/remove-deleted false})] + + (t/is (nil? (:error out))) + (t/is (true? (:result out))) + (t/is (pos? (- (inst-ms (:updated-at row)) (inst-ms stale-at))) + "updated_at must be bumped by the mutation") + (t/is (<= (inst-ms (:updated-at row)) (inst-ms (ct/now))) + "updated_at must not be in the future") + (t/is (<= (- (inst-ms (ct/now)) (inst-ms (:updated-at row))) 60000) + "updated_at must be the moment of the mutation") + (when (= type :delete-file-snapshot) + (t/is (< (inst-ms (:updated-at row)) (inst-ms (:deleted-at row))) + "the soft delete must not stamp updated_at with the future deleted_at"))))) + + (t/testing "the deleted snapshot is marked as deleted" + (t/is (some? (:deleted-at (th/db-get :file-change {:id (:id snapshot)} + {::db/remove-deleted false}))))) + + ;; process the delete-object job enqueued by the snapshot delete + (th/run-pending-jobs))) + +(t/deftest unknown-snapshot-is-reported-as-not-found + ;; The error points at the id that was not found: the snapshots are looked + ;; up by id, not by label. Both commands that resolve a snapshot report it + ;; the same way. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + + params {::rpc/profile-id (:id profile) + :file-id (:id file) + :id (uuid/random)}] + + (doseq [type [:restore-file-snapshot :get-file-snapshot]] + (t/testing (name type) + (let [out (th/command! (assoc params ::th/type type))] + + (t/is (th/ex-of-code? (:error out) :snapshot-not-found)) + (t/is (= "unable to find snapshot with the provided id" + (:hint (ex-data (:error out)))))))))) + +(t/deftest revision-and-snapshot-carry-different-content + ;; A revision is the delta of a save: it carries the changes and nothing + ;; else. A snapshot is the state of the file at a point: it carries the + ;; features and the migrations. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + page-id (uuid/random)] + + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :changes + [{:type :add-page + :name "test" + :id page-id}]) + + (t/testing "the revision carries the changes and not the features" + (let [rows (th/db-query :file-change {:file-id (:id file)}) + revision (first (filter #(nil? (:label %)) rows))] + + (t/is (some? revision)) + (t/is (some? (:changes revision))) + (t/is (nil? (:features revision))))) + + (t/testing "the snapshot carries the migrations and the features" + (let [out (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "label1"})] + + (t/is (nil? (:error out))) + + (let [snapshot (th/db-get :file-change {:id (get-in out [:result :id])})] + (t/is (= "label1" (:label snapshot))) + (t/is (some? (:migrations snapshot))) + (t/is (some? (:features snapshot)))))))) + +(t/deftest file-change-rows-have-a-kind + ;; The kind tells the two kinds of rows of `file_change` apart: a snapshot + ;; (1) and a revision (2). The values are the contract with the database. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + page-id (uuid/random)] + + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :changes + [{:type :add-page + :name "test" + :id page-id}]) + + (t/testing "the revision is a revision" + (let [rows (th/db-query :file-change {:file-id (:id file)}) + revision (first (filter #(nil? (:label %)) rows))] + + (t/is (some? revision)) + (t/is (= 2 (:kind revision))))) + + (t/testing "the snapshot is a snapshot" + (let [out (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "label1"})] + + (t/is (nil? (:error out))) + + (let [snapshot (th/db-get :file-change {:id (get-in out [:result :id])})] + (t/is (= 1 (:kind snapshot)))))))) diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 593c769546..e45794a739 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -23,6 +23,7 @@ [app.features.fdata :as fdata] [app.http :as http] [app.loggers.audit :as-alias audit] + [app.redis :as rds] [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] [app.storage :as sto] @@ -68,7 +69,8 @@ (let [result (:result out)] (t/is (uuid? file-id)) (t/is (= (:name data) (:name result))) - (t/is (= proj-id (:project-id result))))) + (t/is (= proj-id (:project-id result))) + (t/is (not (contains? result :storage-object-id))))) (t/testing "rename file" (let [data {::th/type :rename-file @@ -923,6 +925,68 @@ (t/is (nil? (:error out))) (t/is (false? (-> out :result :is-shared)))))) +(t/deftest unsharing-a-library-touches-the-consumer-storage-object + ;; Regression: the absorption used to build a partial file map that dropped + ;; `:storage-object-id`, so the offloaded blob of the consumer was never + ;; touched and the storage GC could not reclaim it. + (let [profile (th/create-profile* 1 {:is-active true}) + library (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true}) + consumer (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (th/link-file-to-library* {:file-id (:id consumer) + :library-id (:id library)}) + + (th/run-task! :offload-file-data {:file-id (:id consumer)}) + (th/run-task! :storage-gc-touched {:skip-delay true}) + + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id consumer) + :id (:id consumer)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))) + + (let [out (th/command! {::th/type :set-file-shared + ::rpc/profile-id (:id profile) + :id (:id library) + :is-shared false})] + (t/is (nil? (:error out))) + + (t/testing "the absorbed file rewrites its data and touches the old object" + (let [row (th/db-get :file-data {:file-id (:id consumer) :id (:id consumer)})] + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row))) + (t/is (some? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))))))))) + +(t/deftest get-file-does-not-expose-the-storage-pointer + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + (t/is (uuid? (:storage-object-id (th/db-get :file-data {:file-id (:id file) + :id (:id file)})))) + + ;; A row the migration does not backfill keeps the old pointer in the + ;; metadata. The metadata schema is open, so it would travel in the + ;; response unless `file->response` strips it. + (th/db-update! :file-data + {:metadata (db/json {"storage-ref-id" (str (uuid/next))})} + {:file-id (:id file) :id (:id file)}) + + (let [out (th/command! {::th/type :get-file + ::rpc/profile-id (:id profile) + :id (:id file)})] + (t/is (nil? (:error out))) + (t/is (= (:id file) (-> out :result :id))) + (t/testing "the internal storage object id never reaches the client" + (t/is (not (contains? (:result out) :storage-object-id)))) + (t/testing "a residual metadata pointer does not reach the client" + (t/is (not (contains? (:metadata (:result out)) :storage-ref-id))))))) + (t/deftest permissions-checks-link-to-library-1 (let [profile1 (th/create-profile* 1) profile2 (th/create-profile* 2) @@ -1606,14 +1670,13 @@ (t/is (every? #(nil? (:data %)) rows)) (t/is (every? #(= "storage" (:backend %)) rows))) - (let [file (-> (th/db-get :file-data {:id (:id file) :type "main"}) - (update :metadata fdata/decode-metadata)) + (let [file (th/db-get :file-data {:id (:id file) :type "main"}) storage (sto/resolve th/*system*)] ;; (pp/pprint file) (t/is (= "storage" (:backend file))) (t/is (nil? (:data file))) - (let [sobj (sto/get-object storage (-> file :metadata :storage-ref-id))] + (let [sobj (sto/get-object storage (:storage-object-id file))] (vreset! sobject sobj) ;; (pp/pprint (meta sobj)) (t/is (= "file-data" (:bucket (meta sobj)))) @@ -2749,6 +2812,43 @@ (let [edata (-> out :error ex-data)] (t/is (= :circular-library-reference (:code edata))))))) +(t/deftest link-file-to-library-self-reference + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + (let [out (th/command! {::th/type :link-file-to-library + ::rpc/profile-id (:id profile) + :file-id (:id file) + :library-id (:id file)})] + (t/is (th/ex-info? (:error out))) + (t/is (th/ex-of-type? (:error out) :validation)) + (t/is (= :invalid-library (:code (ex-data (:error out)))))))) + +(t/deftest permissions-check-precedes-circular-library-check + ;; A caller without edition permission must get `:not-found`, not the + ;; circular error: the permission checks run before the cycle check, so + ;; the dependency structure of files the caller cannot read never leaks. + (let [profile1 (th/create-profile* 1) + profile2 (th/create-profile* 2) + file1 (th/create-file* 1 {:profile-id (:id profile1) + :project-id (:default-project-id profile1) + :is-shared true}) + file2 (th/create-file* 2 {:profile-id (:id profile1) + :project-id (:default-project-id profile1) + :is-shared true}) + file3 (th/create-file* 3 {:profile-id (:id profile1) + :project-id (:default-project-id profile1) + :is-shared true})] + ;; file3 -> file2 -> file1 makes linking file1 -> file3 circular + (th/link-file-to-library* {:file-id (:id file3) :library-id (:id file2)}) + (th/link-file-to-library* {:file-id (:id file2) :library-id (:id file1)}) + (let [out (th/command! {::th/type :link-file-to-library + ::rpc/profile-id (:id profile2) + :file-id (:id file1) + :library-id (:id file3)})] + (t/is (th/ex-info? (:error out))) + (t/is (th/ex-of-type? (:error out) :not-found))))) + (t/deftest get-file-etag-includes-deleted-at (let [profile-id (uuid/random) file1 {:modified-at (ct/now) @@ -3012,6 +3112,36 @@ :id slink-id})] (t/is (nil? (:error out))))))) +(t/deftest get-file-fragment-returns-only-public-fields + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + fragment (first (th/db-query :file-data {:file-id (:id file) + :type "fragment"}))] + + (t/is (uuid? (:id fragment))) + + (t/testing "a fragment on hot storage resolves" + (let [out (th/command! {::th/type :get-file-fragment + ::rpc/profile-id (:id profile) + :file-id (:id file) + :fragment-id (:id fragment)})] + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (contains? result :data)) + (t/is (= #{:id :file-id :created-at :data} + (set (keys result))))))) + + (t/testing "a fragment moved to cold storage still resolves" + (th/run-task! :offload-file-data {:file-id (:id file)}) + (let [out (th/command! {::th/type :get-file-fragment + ::rpc/profile-id (:id profile) + :file-id (:id file) + :fragment-id (:id fragment)})] + (t/is (nil? (:error out))) + (t/is (contains? (:result out) :data)) + (t/is (not (contains? (:result out) :storage-object-id))))))) + (t/deftest share-link-fragment-access-denied (let [owner (th/create-profile* 1 {:is-active true}) viewer (th/create-profile* 2 {:is-active true}) @@ -3235,3 +3365,71 @@ :team-id (:default-team-id profile)}))] (t/is (th/success? out)) (t/is (= #{(:id file2)} (into #{} (map :id) (:result out)))))) + +(t/deftest get-team-shared-files-does-not-expose-the-storage-pointer + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true})] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + + (let [out (th/command! {::th/type :get-team-shared-files + ::rpc/profile-id (:id profile) + :team-id (:default-team-id profile)}) + result (:result out)] + (t/is (nil? (:error out))) + (t/is (= #{(:id file)} (into #{} (map :id) result))) + (t/is (every? #(not (contains? % :storage-object-id)) result))))) + +(t/deftest save-reads-the-file-row-for-update + ;; The save persists the map it read, so the read holds the `file` row from + ;; the read to the write, with the exclusive lock: the migration-on-read + ;; re-read cannot upgrade a shared one and deadlock with another reader. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + page-id (first (get-in file [:data :pages])) + [_ sqls] (th/read-recording-sql + #(update-file! :profile-id (:id profile) + :file-id (:id file) + :revn 0 + :changes [(th/add-rect-change page-id (uuid/random))]))] + (t/is (some #(= (str bfc/sql:get-file " FOR UPDATE of f") %) sqls)) + (t/is (not-any? #(str/includes? % "FOR SHARE") sqls)))) + +(t/deftest get-team-shared-files-does-not-return-the-applied-migrations + ;; The applied migrations are read bookkeeping for the write path: this + ;; summary is not where a client completes a read, so both cache paths strip + ;; the key. The recorded SQL proves which path ran: a miss reads the file, a + ;; hit reads the row only. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true}) + shared (fn [] + (th/command! {::th/type :get-team-shared-files + ::rpc/profile-id (:id profile) + :team-id (:default-team-id profile)}))] + ;; Redis is not reset between tests and the helper builds deterministic + ;; file ids: drop the entry so the first call is a real cache miss. + (rds/run! th/*system* + (fn [{:keys [::rds/conn]}] + (rds/del conn (files/file-summary-cache-key (:id file))))) + (with-redefs [cf/flags (conj cf/flags :redis-cache)] + (let [[miss sqls-miss] (th/read-recording-sql shared) + [hit sqls-hit] (th/read-recording-sql shared)] + (t/is (= [bfc/sql:get-file] sqls-miss)) + (t/is (= [bfc/sql:get-file-without-data] sqls-hit)) + + (t/testing "the cache miss does not carry the applied migrations" + (t/is (nil? (:error miss))) + (t/is (= #{(:id file)} (into #{} (map :id) (:result miss)))) + (t/is (every? #(not (contains? % :migrations)) (:result miss)))) + + (t/testing "the cache hit does not carry the applied migrations" + (t/is (nil? (:error hit))) + (t/is (= (into #{} (map :id) (:result miss)) + (into #{} (map :id) (:result hit)))) + (t/is (every? #(not (contains? % :migrations)) (:result hit)))))))) diff --git a/backend/test/backend_tests/rpc_file_update_repair_test.clj b/backend/test/backend_tests/rpc_file_update_repair_test.clj index f8eba29f53..82201bbb30 100644 --- a/backend/test/backend_tests/rpc_file_update_repair_test.clj +++ b/backend/test/backend_tests/rpc_file_update_repair_test.clj @@ -346,6 +346,10 @@ (t/is (= 2 (count new))) (t/is (= (inc n) (:revn (first new)))) (t/is (= files-update/repair-session-id (:session-id (first new)))) + ;; the repair is a revision like the save: no features, a changes blob + (t/is (= 2 (:kind (first new)))) + (t/is (nil? (:features (first new)))) + (t/is (some? (:changes (first new)))) (t/is (some? (:deleted-at (first new)))) (t/is (= (:created-at (first new)) (:updated-at (first new)))) (t/is (= (+ n 2) (:revn (second new)))) diff --git a/backend/test/backend_tests/rpc_font_test.clj b/backend/test/backend_tests/rpc_font_test.clj index 3da67f3951..7512d9030c 100644 --- a/backend/test/backend_tests/rpc_font_test.clj +++ b/backend/test/backend_tests/rpc_font_test.clj @@ -403,6 +403,51 @@ (t/is (nil? (:error out))) (assert-font-variant-result params (:result out))))) +(t/deftest create-font-variant-persists-variant-name + "The optional :variant-name is stored and returned by get-font-variants." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 34) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024)) + out (th/command! {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "variant-name-font" + :font-weight 600 + :font-style "normal" + :variant-name "SemiBold" + :uploads {"font/ttf" session-id}})] + (t/is (th/success? out)) + (t/is (= "SemiBold" (-> out :result :variant-name))) + + (let [variants (-> (th/command! {::th/type :get-font-variants + ::rpc/profile-id (:id prof) + :team-id team-id}) + :result) + variant (first (filter #(= font-id (:font-id %)) variants))] + (t/is (= "SemiBold" (:variant-name variant))))) + + ;; An explicit nil must be accepted (the param is optional). + (let [prof (th/create-profile* 2 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 35) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024)) + out (th/command! {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "variant-name-nil" + :font-weight 400 + :font-style "normal" + :variant-name nil + :uploads {"font/ttf" session-id}})] + (t/is (th/success? out)) + (t/is (nil? (-> out :result :variant-name)))))) + ;; ----------------------------------------------------------------------- ;; Error cases ;; ----------------------------------------------------------------------- diff --git a/backend/test/backend_tests/rpc_management_test.clj b/backend/test/backend_tests/rpc_management_test.clj index 3197a0026b..76466b0161 100644 --- a/backend/test/backend_tests/rpc_management_test.clj +++ b/backend/test/backend_tests/rpc_management_test.clj @@ -171,6 +171,39 @@ (let [rows (th/db-query :file {:project-id (:id project)})] (t/is (= 3 (count rows)))))))) +(t/deftest duplicate-file-does-not-touch-the-source-storage-object + (let [profile (th/create-profile* 1 {:is-active true}) + project (th/create-project* 1 {:team-id (:default-team-id profile) + :profile-id (:id profile)}) + file1 (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project)})] + + (th/run-task! :offload-file-data {:file-id (:id file1)}) + (th/run-task! :storage-gc-touched {:skip-delay true}) + + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file1) + :id (:id file1)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))) + + (let [out (th/command! {::th/type :duplicate-file + ::rpc/profile-id (:id profile) + :file-id (:id file1)})] + (t/is (nil? (:error out))) + (t/is (not (contains? (:result out) :storage-object-id))) + + (t/testing "duplication does not write to the source object" + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false}))))) + + (let [file-id (:id (:result out)) + row (th/db-get :file-data {:file-id file-id :id file-id})] + (t/testing "the new file does not inherit the source pointer" + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row))) + (t/is (bytes? (:data row))))))))) + (t/deftest duplicate-file-with-deleted-relations (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) diff --git a/backend/test/backend_tests/rpc_nitrate_test.clj b/backend/test/backend_tests/rpc_nitrate_test.clj index abbfc679fc..26f0de65ec 100644 --- a/backend/test/backend_tests/rpc_nitrate_test.clj +++ b/backend/test/backend_tests/rpc_nitrate_test.clj @@ -87,6 +87,15 @@ :issuer "https://idp.example.com" :organization-id organization-id} + :get-organization-membership + {:is-member true + :organization-id organization-id} + + :get-organization-summary + {:id organization-id + :owner-id organization-owner-id + :teams []} + nil))) (defn- unauthorized-sso-mock @@ -127,7 +136,7 @@ {::th/type :check-nitrate-sso ::rpc/profile-id (:id external-profile) :team-id (:id team) - :url "https://penpot.example.com/#/workspace"} + :query-string "screen=workspace&team-id=0e5f-1a2b"} {::http/request {}})] (binding [cf/flags (conj cf/flags :admin-console)] (with-redefs [nitrate/call @@ -154,7 +163,7 @@ {::th/type :check-nitrate-sso ::rpc/profile-id (:id team-owner) :team-id (:id team) - :url "https://penpot.example.com/#/workspace"} + :query-string "screen=workspace&team-id=0e5f-1a2b"} {::http/request {}})] (binding [cf/flags (conj cf/flags :admin-console)] (with-redefs [nitrate/call @@ -175,8 +184,9 @@ (t/is (= {:authorized false :redirect-uri redirect-uri} (:result out))) - (t/is (= #{:dest-url :organization-id} (set (keys @redirect-options)))) - (t/is (= "https://penpot.example.com/#/workspace" (str (:dest-url @redirect-options)))) + (t/is (= #{:query-string :organization-id :profile-id} (set (keys @redirect-options)))) + (t/is (= "screen=workspace&team-id=0e5f-1a2b" (:query-string @redirect-options))) + (t/is (= (:id team-owner) (:profile-id @redirect-options))) (t/is (nil? (:organization-id @redirect-options))) (t/is (= {:profile-id (:id team-owner) :organization-id organization-id} @@ -191,10 +201,16 @@ {::th/type :check-nitrate-sso ::rpc/profile-id (:id profile) :organization-id organization-id - :url "https://penpot.example.com/#/workspace"} + :query-string "screen=workspace&team-id=0e5f-1a2b"} {::http/request {}})] (binding [cf/flags (conj cf/flags :admin-console)] (with-redefs [nitrate/sso-session-authorized? (unauthorized-sso-mock organization-id) + nitrate/call + (fn [_cfg method _params] + (case method + :get-organization-membership {:is-member true + :organization-id organization-id} + nil)) oidc/build-organization-sso-auth-redirect-uri (fn [& _] (throw cause)) oidc/submit-organization-sso-auth-failed-event (fn [_cfg _request profile-id received-organization-id received-cause] @@ -218,7 +234,7 @@ {::th/type :check-nitrate-sso ::rpc/profile-id (:id organization-owner) :team-id (:id team) - :url "https://penpot.example.com/#/workspace"} + :query-string "screen=workspace&team-id=0e5f-1a2b"} {::http/request {}})] (binding [cf/flags (conj cf/flags :admin-console)] (with-redefs [nitrate/organization-owner-of-team? @@ -246,7 +262,7 @@ {::th/type :check-nitrate-sso ::rpc/profile-id (:id team-owner) :team-id (:id team) - :url "https://penpot.example.com/#/workspace"} + :query-string "screen=workspace&team-id=0e5f-1a2b"} {::http/request {}})] (binding [cf/flags (conj cf/flags :admin-console)] (with-redefs [nitrate/sso-session-authorized? @@ -259,6 +275,153 @@ (t/is (th/success? out)) (t/is (= {:authorized true :reason :sso-satisfied} (:result out)))))))) +(t/deftest check-nitrate-sso-denies-an-organization-the-profile-does-not-belong-to + (let [profile (th/create-profile* 1 {:is-active true}) + organization-id (uuid/random) + params (with-meta + {::th/type :check-nitrate-sso + ::rpc/profile-id (:id profile) + :organization-id organization-id + :query-string "screen=workspace"} + {::http/request {}})] + (binding [cf/flags (conj cf/flags :admin-console)] + (with-redefs [nitrate/call + (fn [_cfg method _params] + (case method + :get-organization-membership {:is-member false + :organization-id organization-id} + :get-organization-summary {:id organization-id + :owner-id (uuid/random) + :teams []} + :get-organization-sso {:active true + :issuer "https://idp.example.com" + :organization-id organization-id} + nil)) + oidc/build-organization-sso-auth-redirect-uri + (constantly "https://idp.example.com/authorize")] + (let [out (th/command! params)] + (t/is (th/success? out)) + ;; The organization id comes from the client: a profile that does + ;; not belong to the organization must not start a flow for it. + (t/is (= {:authorized true :reason :no-organization-access} (:result out)))))))) + +(t/deftest check-nitrate-sso-denies-an-organization-the-profile-does-not-belong-to-even-with-a-team + (let [profile (th/create-profile* 1 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id profile)}) + organization-id (uuid/random) + params (with-meta + {::th/type :check-nitrate-sso + ::rpc/profile-id (:id profile) + :team-id (:id team) + :organization-id organization-id + :query-string "screen=workspace"} + {::http/request {}})] + (binding [cf/flags (conj cf/flags :admin-console)] + (with-redefs [nitrate/call + (fn [_cfg method _params] + (case method + :get-organization-membership {:is-member false + :organization-id organization-id} + :get-organization-summary {:id organization-id + :owner-id (uuid/random) + :teams []} + :get-organization-sso {:active true + :issuer "https://idp.example.com" + :organization-id organization-id} + nil)) + oidc/build-organization-sso-auth-redirect-uri + (constantly "https://idp.example.com/authorize")] + (let [out (th/command! params)] + (t/is (th/success? out)) + ;; The organization id comes from the client too when the team is + ;; also sent, so the check applies to both shapes. + (t/is (= {:authorized true :reason :no-organization-access} (:result out)))))))) + +(t/deftest check-nitrate-sso-survives-an-organization-that-does-not-exist + (let [profile (th/create-profile* 1 {:is-active true}) + organization-id (uuid/random) + params (with-meta + {::th/type :check-nitrate-sso + ::rpc/profile-id (:id profile) + :organization-id organization-id + :query-string "screen=workspace"} + {::http/request {}})] + (binding [cf/flags (conj cf/flags :admin-console)] + (with-redefs [nitrate/call + (fn [_cfg _method _params] nil) + oidc/build-organization-sso-auth-redirect-uri + (constantly "https://idp.example.com/authorize")] + (let [out (th/command! params)] + (t/is (th/success? out)) + ;; Nitrate knows nothing about the organization: that is a denial, + ;; not a crash. + (t/is (= {:authorized true :reason :no-organization-access} (:result out)))))))) + +(t/deftest check-nitrate-sso-ignores-a-value-that-is-not-a-route + (let [profile (th/create-profile* 1 {:is-active true}) + organization-id (uuid/random) + redirect-uri "https://idp.example.com/authorize"] + (doseq [query-string ["https://evil.example/" ""]] + (let [params (with-meta + {::th/type :check-nitrate-sso + ::rpc/profile-id (:id profile) + :organization-id organization-id + :query-string query-string} + {::http/request {}})] + (binding [cf/flags (conj cf/flags :admin-console)] + (with-redefs [nitrate/call + (fn [_cfg method _params] + (case method + :get-organization-membership {:is-member true + :organization-id organization-id} + :get-organization-summary {:id organization-id + :owner-id (:id profile) + :teams []} + :get-organization-sso {:active true + :issuer "https://idp.example.com" + :organization-id organization-id} + nil)) + oidc/prepare-organization-sso-provider (constantly {:type "oidc"}) + oidc/build-auth-redirect-uri (constantly redirect-uri)] + (let [out (th/command! params)] + (t/is (th/success? out)) + ;; A destination the server cannot render must not break the + ;; flow: the user lands on the public uri instead. + (t/is (= {:authorized false + :redirect-uri redirect-uri} + (:result out)))))))))) + +(t/deftest check-nitrate-sso-allows-the-owner-of-the-organization + (let [profile (th/create-profile* 1 {:is-active true}) + organization-id (uuid/random) + redirect-uri "https://idp.example.com/authorize" + params (with-meta + {::th/type :check-nitrate-sso + ::rpc/profile-id (:id profile) + :organization-id organization-id + :query-string "screen=workspace"} + {::http/request {}})] + (binding [cf/flags (conj cf/flags :admin-console)] + (with-redefs [nitrate/call + (fn [_cfg method _params] + (case method + :get-organization-membership {:is-member false + :organization-id organization-id} + :get-organization-summary {:id organization-id + :owner-id (:id profile) + :teams []} + :get-organization-sso {:active true + :issuer "https://idp.example.com" + :organization-id organization-id} + nil)) + oidc/build-organization-sso-auth-redirect-uri + (constantly redirect-uri)] + (let [out (th/command! params)] + (t/is (th/success? out)) + (t/is (= {:authorized false + :redirect-uri redirect-uri} + (:result out)))))))) + (t/deftest nitrate-sso-required-error-resolves-the-team-from-the-file (t/testing "the workspace path, where the file id arrives as :id, still reports the team" (let [profile (th/create-profile* 1 {:is-active true}) diff --git a/backend/test/backend_tests/rpc_organization_owner_permissions_test.clj b/backend/test/backend_tests/rpc_organization_owner_permissions_test.clj index 13c7f8120f..6967dddac6 100644 --- a/backend/test/backend_tests/rpc_organization_owner_permissions_test.clj +++ b/backend/test/backend_tests/rpc_organization_owner_permissions_test.clj @@ -6,11 +6,14 @@ (ns backend-tests.rpc-organization-owner-permissions-test (:require + [app.common.time :as ct] [app.common.uuid :as uuid] [app.config :as cf] + [app.db :as db] [app.msgbus :as mbus] [app.nitrate :as nitrate] [app.rpc :as-alias rpc] + [app.rpc.permissions :as perms] [backend-tests.helpers :as th] [clojure.test :as t])) @@ -247,3 +250,36 @@ :id (:id target-team)})] (t/is (th/ex-info? (:error out))) (t/is (th/ex-of-type? (:error out) :not-found))))))) + +(t/deftest resolve-team-id-refuses-soft-deleted-ancestors + (let [owner (th/create-profile* 1 {:is-active true}) + editor (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)}) + project (th/create-project* 1 {:profile-id (:id editor) + :team-id (:id team)}) + file (th/create-file* 1 {:profile-id (:id editor) + :project-id (:id project)})] + + (t/testing "a live file/project/team resolves in every branch" + (t/is (= (:id team) (perms/resolve-team-id th/*system* {:file-id (:id file)}))) + (t/is (= (:id team) (perms/resolve-team-id th/*system* {:project-id (:id project)}))) + (t/is (= (:id team) (perms/resolve-team-id th/*system* {:team-id (:id team)})))) + + (t/testing "a soft-deleted file refuses the file branch" + (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id file)}) + (t/is (nil? (perms/resolve-team-id th/*system* {:file-id (:id file)}))) + (t/is (= (:id team) (perms/resolve-team-id th/*system* {:team-id (:id team)}))) + (db/update! th/*system* :file {:deleted-at nil} {:id (:id file)})) + + (t/testing "a soft-deleted project refuses the file and the project branch" + (db/update! th/*system* :project {:deleted-at (ct/now)} {:id (:id project)}) + (t/is (nil? (perms/resolve-team-id th/*system* {:file-id (:id file)}))) + (t/is (nil? (perms/resolve-team-id th/*system* {:project-id (:id project)}))) + (t/is (= (:id team) (perms/resolve-team-id th/*system* {:team-id (:id team)}))) + (db/update! th/*system* :project {:deleted-at nil} {:id (:id project)})) + + (t/testing "a soft-deleted team refuses every branch" + (db/update! th/*system* :team {:deleted-at (ct/now)} {:id (:id team)}) + (t/is (nil? (perms/resolve-team-id th/*system* {:file-id (:id file)}))) + (t/is (nil? (perms/resolve-team-id th/*system* {:project-id (:id project)}))) + (t/is (nil? (perms/resolve-team-id th/*system* {:team-id (:id team)})))))) diff --git a/backend/test/backend_tests/rpc_profile_test.clj b/backend/test/backend_tests/rpc_profile_test.clj index 2b4a3469c1..25542b9845 100644 --- a/backend/test/backend_tests/rpc_profile_test.clj +++ b/backend/test/backend_tests/rpc_profile_test.clj @@ -987,6 +987,10 @@ (t/is (= :email-does-not-match-invitation (:code edata)))))) (t/deftest prepare-register-with-invitation-and-disabled-registration-and-wrong-email + ;; The email check runs before the invitation row lookup, so a + ;; mismatched email is reported even when the invitation row is gone. + ;; Flags stay `#{:login-with-password}`: password registration is + ;; allowed only via invitation. (with-redefs [app.config/flags #{:login-with-password}] (let [itoken (tokens/generate th/*system* {:iss :team-invitation @@ -994,7 +998,6 @@ :role :editor :team-id uuid/zero :member-email "user2@example.com"}) - data {::th/type :prepare-register-profile :invitation-token itoken :email "user@example.com" @@ -1046,6 +1049,216 @@ (t/is (= :registration-disabled (-> out :error ex-data :code))) (t/is (nil? (th/db-get :profile {:email email}))))))))) +(defn- create-oidc-register-token + "Mint a `:prepared-register` token shaped like the ones the OIDC + callback produces in `redirect-to-register` (no real provider needed)." + [email extra] + (tokens/generate th/*system* + (merge {:iss :prepared-register + :exp (ct/in-future "48h") + :backend "oidc" + :email email + :fullname "OIDC User" + :email-verified true + :props {}} + extra))) + +(t/deftest register-profile-allows-oidc-registration-with-password-login-disabled + ;; Issue #12964: with password login and public registration disabled, + ;; a new user arriving from OIDC must still be able to register when + ;; `oidc-registration` is enabled. + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [email "oidc-user@example.com" + token (create-oidc-register-token email nil) + out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + ;; A session is minted: the user lands on the dashboard. + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile))) + (t/is (true? (:is-active profile)))) + ;; No password fallback while password login is disabled. + (let [login-out (th/command! {::th/type :login-with-password + :email email + :password "Whatever1!"})] + (t/is (not (th/success? login-out))) + (t/is (= :restriction (-> login-out :error ex-data :type))) + (t/is (= :login-disabled (-> login-out :error ex-data :code))))))) + +(t/deftest register-profile-rejects-oidc-without-registration-flags + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [email "oidc-rejected@example.com" + token (create-oidc-register-token email nil) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :registration-disabled (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email email})))))) + +(t/deftest register-profile-allows-oidc-with-public-registration-and-password-login-disabled + (with-redefs [app.config/flags #{:login-with-oidc :registration}] + (let [email "oidc-public@example.com" + token (create-oidc-register-token email nil) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (th/success? out)) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile))))))) + +(t/deftest register-profile-allows-oidc-with-invitation-and-registration-disabled + (let [email "oidc-invited@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken}) + out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + ;; Invitation branch: the token is echoed back and a session is + ;; minted so the frontend can complete the team-invitation flow. + (t/is (string? (:invitation-token result))) + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile)))))))) + +(t/deftest register-profile-allows-oidc-with-canceled-invitation + ;; The invitation row is only consulted when no SSO registration + ;; flag is set: with `:oidc-registration` (as here), registration + ;; succeeds even when the invitation was canceled, and team + ;; membership fails later at `verify-token`. + (let [email "oidc-canceled@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken})] + (th/db-delete! :team-invitation + {:team-id (:id (:team fixture)) + :email-to email}) + (let [out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + (t/is (string? (:invitation-token result))) + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile)))) + ;; The team is not joined: with the invitation row gone, + ;; accepting it fails. + (let [vout (th/command! {::th/type :verify-token + :token (:invitation-token result)})] + (t/is (not (th/success? vout))) + (t/is (= :validation (-> vout :error ex-data :type))) + (t/is (= :invalid-token (-> vout :error ex-data :code))))))))) + +(t/deftest register-profile-rejects-prepared-token-without-backend + ;; The `:backend` claim is mandatory: a prepared-register token + ;; without it is rejected loudly instead of falling into any + ;; registration branch. + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [token (tokens/generate th/*system* + {:iss :prepared-register + :exp (ct/in-future "48h") + :email "no-backend@example.com" + :fullname "No Backend" + :email-verified true + :props {}}) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :invalid-token (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email "no-backend@example.com"})))))) + +(t/deftest register-profile-rejects-non-oidc-backend-token + ;; The SSO registration gate only applies to backends of OIDC type; + ;; a token carrying any other backend is rejected loudly instead of + ;; silently taking the SSO branch. + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [token (tokens/generate th/*system* + {:iss :prepared-register + :exp (ct/in-future "48h") + :backend "ldap" + :email "unknown-backend@example.com" + :fullname "Unknown Backend" + :email-verified true + :props {}}) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :invalid-token (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email "unknown-backend@example.com"})))))) + +(t/deftest register-profile-allows-oidc-with-invitation-and-no-registration-flags + ;; Review of #12974: an invitation token is a substitute for the + ;; SSO registration flags, as it is for the password flow. With no + ;; `:registration` nor `:oidc-registration` flag, a user arriving + ;; through a team invitation can still register via OIDC. + (let [email "oidc-invited-noflags@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken}) + out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + ;; Invitation branch: the token is echoed back and a session is + ;; minted so the frontend can complete the team-invitation flow. + (t/is (string? (:invitation-token result))) + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile)))))))) + +(t/deftest register-profile-rejects-oidc-invitation-email-mismatch-with-no-registration-flags + ;; Review of #12974: the invitation is a substitute for the SSO + ;; registration flags only when it was issued for the registering + ;; email. A forwarded invitation for another email must not unlock + ;; registration with closed registration. + (let [invited-email "oidc-invited-other@example.com" + attacker-mail "oidc-attacker-noflags@example.com" + fixture (create-invitation-fixture invited-email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [itoken (create-test-invitation-token fixture invited-email) + token (create-oidc-register-token attacker-mail {:invitation-token itoken}) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :restriction (-> out :error ex-data :type))) + (t/is (= :email-does-not-match-invitation (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email attacker-mail}))))))) + +(t/deftest register-profile-rejects-oidc-with-canceled-invitation-and-no-registration-flags + ;; Review of #12974: with no registration flag set, the invitation + ;; row is revalidated under a row lock before creating the profile. + ;; A canceled invitation stops signup and no account is created. + (let [email "oidc-canceled-noflags@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken})] + (th/db-delete! :team-invitation + {:team-id (:id (:team fixture)) + :email-to email}) + (let [out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :invalid-token (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email email})))))))) + (t/deftest verify-token-redirects-invitation-based-on-member-id (with-redefs [app.config/flags #{:login-with-password}] (let [owner (th/create-profile* 1 {:is-active true}) diff --git a/backend/test/backend_tests/rpc_quotes_test.clj b/backend/test/backend_tests/rpc_quotes_test.clj index cd49f85042..8bc7c78fee 100644 --- a/backend/test/backend_tests/rpc_quotes_test.clj +++ b/backend/test/backend_tests/rpc_quotes_test.clj @@ -563,6 +563,7 @@ ;; deleting a version takes it out of the count right away (let [out (th/command! {::th/type :delete-file-snapshot ::rpc/profile-id (:id profile) + :file-id (:id file) :id (:id snapshot)})] (t/is (nil? (:error out))) (t/is (true? (:result out))) diff --git a/backend/test/backend_tests/rpc_team_test.clj b/backend/test/backend_tests/rpc_team_test.clj index 51a1e746e1..a59d2759da 100644 --- a/backend/test/backend_tests/rpc_team_test.clj +++ b/backend/test/backend_tests/rpc_team_test.clj @@ -1093,6 +1093,112 @@ (t/is (nil? (:result out))) (t/is (nil? res))))) +;; The organization `send-invitations` policy covers the pending invitations +;; too: changing their role, deleting them and getting their link manage the +;; same access that sending one grants. + +(def ^:private pending-invitation-email "pending@example.com") + +(defn- reset-pending-invitation + "Leaves one pending `editor` invitation in the team, whatever the previous + scenario did to it." + [team-id] + (db/delete! th/*pool* :team-invitation + {:team-id team-id :email-to pending-invitation-email}) + (db/insert! th/*pool* :team-invitation + {:team-id team-id + :email-to pending-invitation-email + :role "editor" + :valid-until (ct/in-future "48h")})) + +(defn- get-pending-invitation-role + [team-id] + (:role (db/get* th/*pool* :team-invitation + {:team-id team-id :email-to pending-invitation-email}))) + +(defn- organization-with-policy + "A team organization lookup that answers with an organization whose + `send-invitations` value and attribute restriction answer are the given + ones." + [owner-id send-invitations restrictions-met] + (fn [_params] + {:organization {:id (uuid/random) + :name "Acme" + :slug "acme" + :owner-id owner-id + :avatar-bg-url "https://example.com/avatar.svg" + :permissions {:send-invitations send-invitations} + :restrictions-met {:send-invitations restrictions-met}} + :is-your-penpot false})) + +(defn- organization-lookup-failure + [status] + (fn [_params] + (throw (ex-info "nitrate error" {:type :nitrate-http-error + :status status})))) + +(defn- check-follows-send-invitations-policy + "Runs the invitation `command` (a params map without profile, team and + email) under each organization policy and checks who may go through." + [command] + (let [owner (th/create-profile* 440 {:is-active true}) + admin (th/create-profile* 441 {:is-active true}) + team (th/create-team* 440 {:profile-id (:id owner)}) + _ (th/create-team-role* {:team-id (:id team) + :profile-id (:id admin) + :role :admin}) + team-id (:id team) + + execute + (fn [profile get-team-organization] + (reset-pending-invitation team-id) + (with-redefs [cf/flags (conj cf/flags :admin-console) + nitrate/call (fn [_cfg method params] + (case method + :get-team-organization (get-team-organization params) + nil))] + (th/command! (assoc command + ::rpc/profile-id (:id profile) + :team-id team-id + :email pending-invitation-email))))] + + (t/testing "an admin is denied when only team owners may invite" + (let [out (execute admin (organization-with-policy (:id owner) "owners" true))] + (t/is (= :insufficient-permissions (th/ex-code (:error out)))) + (t/is (= "editor" (get-pending-invitation-role team-id))))) + + (t/testing "an admin is denied when an attribute restriction leaves them out" + (let [out (execute admin (organization-with-policy (:id owner) "ownersAndAdmins" false))] + (t/is (= :insufficient-permissions (th/ex-code (:error out)))) + (t/is (= "editor" (get-pending-invitation-role team-id))))) + + (t/testing "a failed organization lookup denies the admin" + (let [out (execute admin (organization-lookup-failure 403))] + (t/is (= :nitrate-http-error (th/ex-type (:error out)))) + (t/is (= "editor" (get-pending-invitation-role team-id))))) + + (t/testing "a team owner is allowed when only team owners may invite" + (t/is (th/success? (execute owner (organization-with-policy (:id owner) "owners" true))))) + + (t/testing "an admin is allowed when the policy and the restriction include them" + (t/is (th/success? (execute admin (organization-with-policy (:id owner) "ownersAndAdmins" true))))) + + (t/testing "an admin is allowed in a team without organization" + (t/is (th/success? (execute admin (organization-lookup-failure 404))))))) + +(t/deftest update-team-invitation-role-follows-send-invitations-policy + (check-follows-send-invitations-policy + {::th/type :update-team-invitation-role + :role :admin})) + +(t/deftest delete-team-invitation-follows-send-invitations-policy + (check-follows-send-invitations-policy + {::th/type :delete-team-invitation})) + +(t/deftest get-team-invitation-token-follows-send-invitations-policy + (check-follows-send-invitations-policy + {::th/type :get-team-invitation-token})) + (t/deftest get-owned-teams (let [profile1 (th/create-profile* 1 {:is-active true}) @@ -1210,6 +1316,127 @@ (t/is (= :get-teams-organizations (ffirst @calls))))))) +(defn- find-team + [teams team-id] + (d/seek #(= team-id (:id %)) teams)) + +(t/deftest get-teams-shows-owner-nitrate-licence-to-every-member + (let [member (th/create-profile* 1 {:is-active true}) + owner (th/create-profile* 2 {:is-active true}) + org-owner (th/create-profile* 3 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)}) + organization-team (th/create-team* 2 {:profile-id (:id org-owner)}) + _ (th/create-team-role* {:team-id (:id team) + :profile-id (:id member) + :role :editor}) + _ (th/create-team-role* {:team-id (:id organization-team) + :profile-id (:id member) + :role :editor}) + calls (atom []) + nitrate-call (fn [_cfg method params] + (swap! calls conj [method params]) + (case method + :get-teams-organizations + [{:id (:id organization-team) + :is-your-penpot false + :organization {:id (uuid/random) + :name "Acme" + :slug "acme" + :owner-id (:id org-owner)}}] + :get-subscriptions + [{:profile-id (:id owner) :type "nitrate" :status "trialing"} + {:profile-id (:id org-owner) :type "nitrate" :status "active"}])) + params {::th/type :get-teams + ::rpc/profile-id (:id member)}] + (with-redefs [cf/flags (conj cf/flags :admin-console :subscriptions) + nitrate/call nitrate-call] + (let [out (th/command! params) + teams (:result out)] + (t/is (th/success? out)) + (t/is (= [:get-teams-organizations :get-subscriptions] (mapv first @calls))) + ;; One request with the owners of the teams outside an organization, + ;; the member included as owner of its default team. + (t/is (= #{(:id owner) (:id member)} + (-> @calls second second :profile-ids set))) + (t/is (= {:type "nitrate" :status "trialing"} + (-> (find-team teams (:id team)) + :subscription + (select-keys [:type :status])))) + ;; A team in an organization keeps the Stripe plan of its owner. + (t/is (= "professional" + (-> (find-team teams (:id organization-team)) :subscription :type))) + (t/is (= "professional" + (-> (find-team teams (:default-team-id member)) :subscription :type))))))) + +(t/deftest get-teams-ignores-owner-nitrate-licence-that-is-not-valid + (let [member (th/create-profile* 1 {:is-active true}) + owner (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)}) + _ (th/create-team-role* {:team-id (:id team) + :profile-id (:id member) + :role :editor}) + nitrate-call (fn [_cfg method _params] + (case method + :get-teams-organizations [] + :get-subscriptions [{:profile-id (:id owner) + :type "nitrate" + :status "canceled"}])) + params {::th/type :get-teams + ::rpc/profile-id (:id member)}] + (with-redefs [cf/flags (conj cf/flags :admin-console :subscriptions) + nitrate/call nitrate-call] + (let [out (th/command! params)] + (t/is (th/success? out)) + (t/is (= "professional" + (-> (find-team (:result out) (:id team)) :subscription :type))))))) + +(t/deftest get-teams-keeps-stripe-plan-when-owner-licences-fail + (let [member (th/create-profile* 1 {:is-active true}) + owner (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)}) + _ (th/create-team-role* {:team-id (:id team) + :profile-id (:id member) + :role :editor}) + nitrate-call (fn [_cfg method _params] + (case method + :get-teams-organizations [] + :get-subscriptions (throw (ex-info "nitrate is down" + {:type :nitrate-unavailable})))) + params {::th/type :get-teams + ::rpc/profile-id (:id member)}] + (with-redefs [cf/flags (conj cf/flags :admin-console :subscriptions) + nitrate/call nitrate-call] + (let [out (th/command! params)] + (t/is (th/success? out)) + (t/is (= "professional" + (-> (find-team (:result out) (:id team)) :subscription :type))))))) + +(t/deftest get-team-shows-owner-nitrate-licence-to-a-member + (let [member (th/create-profile* 1 {:is-active true}) + owner (th/create-profile* 2 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id owner)}) + _ (th/create-team-role* {:team-id (:id team) + :profile-id (:id member) + :role :editor}) + calls (atom []) + nitrate-call (fn [_cfg method params] + (swap! calls conj [method params]) + (case method + :get-team-organization nil + :get-subscriptions [{:profile-id (:id owner) + :type "nitrate" + :status "active"}])) + params {::th/type :get-team + ::rpc/profile-id (:id member) + :id (:id team)}] + (with-redefs [cf/flags (conj cf/flags :admin-console :subscriptions) + nitrate/call nitrate-call] + (let [out (th/command! params)] + (t/is (th/success? out)) + (t/is (= [(:id owner)] (-> (last @calls) second :profile-ids))) + (t/is (= "nitrate" (-> out :result :subscription :type))))))) + + (t/deftest team-deletion-1 (let [profile1 (th/create-profile* 1 {:is-active true}) team (th/create-team* 1 {:profile-id (:id profile1)}) @@ -1371,6 +1598,10 @@ file (th/create-file* 1 {:profile-id (:id owner) :project-id (:id proj)}) + ;; Cold storage makes the file map carry the internal pointer, so the + ;; response assertion below is meaningful. + _ (th/run-task! :offload-file-data {:file-id (:id file)}) + data {::th/type :create-team-access-request ::rpc/profile-id (:id requester) :file-id (:id file)}] @@ -1383,7 +1614,8 @@ :requester-id (:id requester)})] (t/is (th/success? out)) (t/is (= 1 (:call-count @mock))) - (t/is (= 1 (count requests)))) + (t/is (= 1 (count requests))) + (t/is (not (contains? (:result out) :storage-object-id)))) ;; request again fails (th/reset-mock! mock) diff --git a/backend/test/backend_tests/rpc_viewer_test.clj b/backend/test/backend_tests/rpc_viewer_test.clj index ac09c1e67f..58e568ef02 100644 --- a/backend/test/backend_tests/rpc_viewer_test.clj +++ b/backend/test/backend_tests/rpc_viewer_test.clj @@ -910,3 +910,38 @@ (t/is (= #{comp-b} (set (keys (get-in by-id [(:id lib2) :data :components]))))) (t/is (= [] (get-in by-id [(:id lib2) :data :pages]))))))) + +(t/deftest view-only-bundle-does-not-expose-the-storage-pointer + ;; Both the file and its libraries are moved to cold storage, so their file + ;; maps carry the internal pointer unless the response strips it. + (let [owner (th/create-profile* 1 {:is-active true}) + proj-id (:default-project-id owner) + file (th/create-file* 1 {:profile-id (:id owner) + :project-id proj-id + :is-shared false}) + lib (th/create-file* 2 {:profile-id (:id owner) + :project-id proj-id + :is-shared true}) + page-id (get-in file [:data :pages 0]) + _ (th/link-file-to-library* {:file-id (:id file) + :library-id (:id lib)}) + slink (th/command! {::th/type :create-share-link + ::rpc/profile-id (:id owner) + :file-id (:id file) + :pages #{page-id} + :who-comment "team" + :who-inspect "all"}) + slink-id (get-in slink [:result :id])] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + (th/run-task! :offload-file-data {:file-id (:id lib)}) + + (let [out (th/command! {::th/type :get-view-only-bundle + :share-id slink-id + :file-id (:id file)}) + result (:result out)] + (t/is (nil? (:error out))) + (t/is (seq (:libraries result))) + (t/is (not (contains? (:file result) :storage-object-id))) + (t/is (every? #(not (contains? % :storage-object-id)) + (:libraries result)))))) diff --git a/backend/test/backend_tests/scripts_decode_file_data_test.clj b/backend/test/backend_tests/scripts_decode_file_data_test.clj index e14618542f..3a6484b4ed 100644 --- a/backend/test/backend_tests/scripts_decode_file_data_test.clj +++ b/backend/test/backend_tests/scripts_decode_file_data_test.clj @@ -10,11 +10,14 @@ [app.common.uuid :as uuid] [app.scripts.decode-file-data :as dfd] [app.util.blob :as blob] + [backend-tests.helpers :as th] [clojure.java.io :as io] [clojure.test :as t]) (:import java.util.Base64)) +(t/use-fixtures :once th/state-init) + (defn- sample-file-data [] (let [p1 (uuid/next) @@ -80,3 +83,11 @@ parsed (json/decode (slurp tmp-out))] (t/is (== 2 (get-in parsed ["stats" "pages"]))) (t/is (= ["Accueil1" "Contact"] (mapv #(get % "name") (get parsed "pages")))))) + +(t/deftest get-snapshot-json-fails-when-the-snapshot-does-not-exist + ;; The snapshot lookup returns nil when there is no match, and a nil + ;; snapshot used to be printed as an empty JSON instead of failing. + (let [file-id (uuid/next) + snapshot-id (uuid/next)] + (t/is (thrown? clojure.lang.ExceptionInfo + (dfd/get-snapshot-json th/*system* file-id snapshot-id))))) diff --git a/backend/test/backend_tests/storage_gc_deleted_test.clj b/backend/test/backend_tests/storage_gc_deleted_test.clj index ecd805af73..f2924d681c 100644 --- a/backend/test/backend_tests/storage_gc_deleted_test.clj +++ b/backend/test/backend_tests/storage_gc_deleted_test.clj @@ -6,6 +6,7 @@ (ns backend-tests.storage-gc-deleted-test (:require + [app.common.logging :as l] [app.common.time :as ct] [app.common.uuid :as uuid] [app.db :as db] @@ -155,3 +156,75 @@ (t/testing "the objects after the failed chunk are gone" (t/is (every? #(nil? (find-object (:id %))) recent))))) + +(defn- blocked-by-file-data! + "Puts an object referenced by a `file_data` row, so the `NO ACTION` + foreign key of `storage_object_id` refuses the physical delete." + [file-id] + (let [blocked (put-object! (ct/in-past "30m"))] + ;; `main` always carries `id = file_id`: point the real row at the object + ;; instead of inventing a second main row. + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id blocked)} + {:file-id file-id :id file-id}) + blocked)) + +(def ^:private blocked-chunk-hint + "chunk deletion blocked by a live reference; fix the reference and let the next run reclaim the chunk") + +(defn- blocked-chunk-log + "The first captured error record with the blocked-chunk hint, waiting for it + because logging is asynchronous." + [records] + (let [deadline (+ (System/currentTimeMillis) 30000)] + (loop [] + (let [record (first (filter (fn [record] + (and (= :error (::l/level record)) + (= blocked-chunk-hint + (get (::l/props record) :hint)))) + @records))] + (cond + record record + (> (System/currentTimeMillis) deadline) nil + :else (do (Thread/sleep 10) (recur))))))) + +(t/deftest file-data-reference-blocks-its-chunk + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + + ;; The blocked object goes first in `deleted_at` order, so it lands in + ;; the first chunk with the objects that fill it up. The last object + ;; lands in the next chunk: it proves the run kept going. + blocked (blocked-by-file-data! (:id file)) + others (mapv (fn [_] (put-object! (ct/in-past "20m"))) (range (dec chunk-size))) + last-id (:id (put-object! (ct/in-past "5m"))) + + records (atom []) + _ (add-watch l/log-record ::capture + (fn [_ _ _ record] (swap! records conj record))) + result (try + (th/run-task! :storage-gc-deleted {}) + (finally + (remove-watch l/log-record ::capture)))] + + (t/testing "the run finishes and only the object of the second chunk is deleted" + (t/is (= 1 (:deleted result)))) + + (t/testing "the object referenced by the file data row is still alive" + (t/is (some? (find-object (:id blocked))))) + + (t/testing "the other objects of the failed chunk are still alive" + (t/is (every? #(some? (find-object (:id %))) others))) + + (t/testing "the blocked chunk names the constraint that blocks it" + (let [record (blocked-chunk-log records)] + (t/is (some? record)) + (t/is (= "file_data_storage_object_id_fkey" + (get (::l/props record) :constraint))) + (t/is (= "file_data" (get (::l/props record) :table))))) + + (t/testing "the object of the next chunk is gone" + (t/is (nil? (find-object last-id)))))) diff --git a/backend/test/backend_tests/test_files/font-cid.otf b/backend/test/backend_tests/test_files/font-cid.otf new file mode 100644 index 0000000000..d9fc9d18ca Binary files /dev/null and b/backend/test/backend_tests/test_files/font-cid.otf differ diff --git a/common/deps.edn b/common/deps.edn index e7ed617f90..ba6e28cc89 100644 --- a/common/deps.edn +++ b/common/deps.edn @@ -48,6 +48,11 @@ frankiesardo/linked {:mvn/version "1.3.0"} + ;; Resolves token values, reads and writes token files. + penpot/tokenscript-interpreter + {:git/sha "321061dbddaf78270319136eb41cfbff172fba02" + :git/url "https://github.com/penpot/tokenscript-interpreter.git"} + com.sun.mail/jakarta.mail {:mvn/version "2.0.2"} org.la4j/la4j {:mvn/version "0.6.0"} diff --git a/common/package.json b/common/package.json index c4c8c89efe..d1840df4f3 100644 --- a/common/package.json +++ b/common/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en España SL", "private": true, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "type": "module", "repository": { "type": "git", diff --git a/common/pnpm-lock.yaml b/common/pnpm-lock.yaml index 92e614fc48..703b542a72 100644 --- a/common/pnpm-lock.yaml +++ b/common/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/common/src/app/common/attrs.cljc b/common/src/app/common/attrs.cljc index 205b02ebd4..95e43d268c 100644 --- a/common/src/app/common/attrs.cljc +++ b/common/src/app/common/attrs.cljc @@ -128,5 +128,7 @@ (merge defaults (get-attrs-multi (->> (txt/node-seq txt/is-root-node? content)) root-attrs) - (get-attrs-multi (->> (txt/node-seq txt/is-paragraph-node? content)) paragraph-attrs) + (get-attrs-multi (->> (txt/node-seq txt/is-paragraph-node? content) + (map #(merge txt/default-list-attrs %))) + paragraph-attrs) (get-attrs-multi (->> (txt/node-seq txt/is-text-node? content)) text-node-attrs)))) diff --git a/common/src/app/common/files/builder.cljc b/common/src/app/common/files/builder.cljc index d9b86ce36d..fdf5ee37e6 100644 --- a/common/src/app/common/files/builder.cljc +++ b/common/src/app/common/files/builder.cljc @@ -86,7 +86,7 @@ (defn- unique-name [name state] (let [container-id (::current-page-id state) - unames (dm/get-in state [:unames container-id])] + unames (dm/get-in state [::unames container-id])] (d/unique-name name (or unames #{})))) (defn- clear-names [file] @@ -469,12 +469,24 @@ frame-id (or frame-id (get state ::current-frame-id)) + name + (or name "anonymous") + + path + (d/nilv path "") + + ;; The main instance root name must match the component's path+name, so + ;; the frame is renamed if needed. This keeps the builder output valid + ;; under the :invalid-main-instance-name integrity check. + normalized-name + (cpn/merge-path-item path name) + change1 (d/without-nils {:type :add-component :id component-id - :name (or name "anonmous") - :path (d/nilv path "") + :name name + :path path :main-instance-id frame-id :main-instance-page page-id :variant-id variant-id @@ -488,11 +500,13 @@ [{:type :set :attr :component-root :val true} {:type :set :attr :main-instance :val true} {:type :set :attr :component-id :val component-id} - {:type :set :attr :component-file :val file-id}]}] + {:type :set :attr :component-file :val file-id} + {:type :set :attr :name :val normalized-name}]}] (-> state (commit-change change1) - (commit-change change2)))) + (commit-change change2) + (track-used-name normalized-name)))) (defn add-tokens-lib [state tokens-lib] diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index ccd0900284..1e364407f1 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -793,17 +793,6 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "legacy-36" - [data _] - (letfn [(update-container [container] - (d/update-when container :objects (fn [objects] - (if (contains? objects nil) - (dissoc objects nil) - objects))))] - (-> data - (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container)))) - (defmethod migrate-data "legacy-37" [data _] (d/without-nils data)) @@ -2310,44 +2299,49 @@ ;; been added to enforce this from now on. ;; Main instances in variants are a special case because they need to match the variant name. (letfn [(update-shape [container shape-id] - (let [shape (ctn/get-shape container shape-id) + (let [shape (ctn/get-shape container shape-id)] + ;; A shape may reference a child that does not exist in the + ;; container (a dangling reference). There is nothing to update + ;; on it, and storing it would add a nil entry to the objects + ;; map, making the whole file invalid. + (if (nil? shape) + container + (let [shape (cond + ;; Variant container must have a normalized name + (ctk/is-variant-container? shape) + (let [name (:name shape) + clean-name (cpn/clean-path name) + clean-name (if (str/blank? clean-name) + "(empty)" ;; this may occur if the name contains only separators + clean-name)] + (if (= name clean-name) + shape + (assoc shape :name clean-name))) - shape (cond - ;; Variant container must have a normalized name - (ctk/is-variant-container? shape) - (let [name (:name shape) - clean-name (cpn/clean-path name) - clean-name (if (str/blank? clean-name) - "(empty)" ;; this may occur if the name contains only separators - clean-name)] - (if (= name clean-name) - shape - (assoc shape :name clean-name))) + ;; Main instance inside a variant must have the same name as the container + (ctk/main-instance? shape) + (if (ctk/is-variant? shape) + (let [variant-container (ctn/get-shape container (:parent-id shape))] + (if (or (= (:name shape) (:name variant-container)) + (nil? variant-container)) + shape + (assoc shape :name (:name variant-container)))) - ;; Main instance inside a variant must have the same name as the container - (ctk/main-instance? shape) - (if (ctk/is-variant? shape) - (let [variant-container (ctn/get-shape container (:parent-id shape))] - (if (or (= (:name shape) (:name variant-container)) - (nil? variant-container)) - shape - (assoc shape :name (:name variant-container)))) + ;; Other main instances must have the same name as the path+name of the component + (let [component (ctkl/get-component data (:component-id shape) true) + expected-name (when component + (cpn/merge-path-item (:path component) (:name component)))] + (if (or (= expected-name (:name shape)) + (nil? expected-name)) + shape + (assoc shape :name expected-name)))) - ;; Other main instances must have the same name as the path+name of the component - (let [component (ctkl/get-component data (:component-id shape) true) - expected-name (when component - (cpn/merge-path-item (:path component) (:name component)))] - (if (or (= expected-name (:name shape)) - (nil? expected-name)) - shape - (assoc shape :name expected-name)))) + ;; Any other shape does not need processing + :else shape)] - ;; Any other shape does not need processing - :else shape)] - - (reduce update-shape - (ctt/set-shape container shape) - (:shapes shape)))) + (reduce update-shape + (ctt/set-shape container shape) + (:shapes shape)))))) (update-container [container] @@ -2399,6 +2393,35 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container))))) +(defmethod migrate-data "0035-fix-dangling-children" + [data _] + ;; A shape may list a child id that does not exist in the objects map of + ;; the page or component that owns it (a dangling reference). Such a + ;; reference is invalid: any operation that walks the shape tree through + ;; the children lists trips on it, and the file cannot be migrated nor + ;; opened. Here we remove the dangling ids from the children lists, + ;; keeping every shape that really exists. + (letfn [(fix-shape [container-id objects shape] + (d/update-when shape :shapes + (fn [shapes] + (let [shapes' (filterv #(contains? objects %) shapes)] + (when (not= shapes shapes') + (l/trc :hint "fix dangling children" + :container (str container-id) + :shape-id (str (:id shape)) + :removed (remove #(contains? objects %) shapes))) + shapes')))) + + (fix-objects [container-id objects] + (d/update-vals objects (partial fix-shape container-id objects))) + + (update-container [container] + (d/update-when container :objects #(fix-objects (:id container) %)))] + + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + (defmethod migrate-data "0035-fix-shape-svg-attrs-and-defs" ;; Stored svg attribute keys are kebab-case (issue #11947): the v3 reader ;; already outputs kebab and birth normalizes any outside spelling, so @@ -2445,6 +2468,26 @@ (update :pages-index d/update-vals migrate-container) (d/update-when :components d/update-vals migrate-container)))) +(defmethod migrate-data "0036-clear-nil-keys-in-objects" + [data _] + ;; Soft-delete can leave a nil key in `:objects` (gsh/move on a missing + ;; main, then index-by :id). That fails the file schema on duplicate-file + ;; (#12336). "legacy-36" did the same cleanup but its id was withdrawn + ;; from `available-migrations` below: files that recorded it migrate + ;; again through this one. + (letfn [(update-container [container] + (d/update-when container :objects + (fn [objects] + (if (contains? objects nil) + (do + (l/trc :hint "clear nil key in objects" + :container-id (:id container)) + (dissoc objects nil)) + objects))))] + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + (def available-migrations (into (d/ordered-set) ["legacy-2" @@ -2472,7 +2515,6 @@ "legacy-32" "legacy-33" "legacy-34" - "legacy-36" "legacy-37" "legacy-38" "legacy-39" @@ -2536,4 +2578,6 @@ "0032-fix-children-not-in-parent" "0033-normalize-main-instance-names" "0034-fix-path-content-offset" - "0035-fix-shape-svg-attrs-and-defs"])) + "0035-fix-dangling-children" + "0035-fix-shape-svg-attrs-and-defs" + "0036-clear-nil-keys-in-objects"])) diff --git a/common/src/app/common/files/shapes_builder.cljc b/common/src/app/common/files/shapes_builder.cljc index ad2d90f610..efa1d4c27b 100644 --- a/common/src/app/common/files/shapes_builder.cljc +++ b/common/src/app/common/files/shapes_builder.cljc @@ -18,6 +18,7 @@ [app.common.geom.shapes.common :as gsc] [app.common.geom.shapes.transforms :as gst] [app.common.math :as mth] + [app.common.path-names :as cpn] [app.common.schema :as sm :refer [max-safe-int min-safe-int]] [app.common.svg :as csvg] [app.common.types.color :as clr] @@ -200,7 +201,7 @@ (let [[vb-x vb-y vb-width vb-height] (svg-dimensions svg-data) unames (cfh/get-used-names objects) - svg-name (str/replace (:name svg-data) ".svg" "") + svg-name (cpn/remove-extension (:name svg-data)) svg-data (-> svg-data (assoc :x (mth/round diff --git a/common/src/app/common/files/variant.cljc b/common/src/app/common/files/variant.cljc index 989a80a58b..80dde18d72 100644 --- a/common/src/app/common/files/variant.cljc +++ b/common/src/app/common/files/variant.cljc @@ -28,6 +28,25 @@ (map #(ctcl/get-component data % true)) reverse))) +(defn collect-variant-properties + "Map of component-id → variant-properties for clipboard copy. + Variant container: every related component. Variant main: that + component only. Anything else: empty map." + [data objects shape] + (cond + (ctc/is-variant-container? shape) + (into {} + (map (juxt :id :variant-properties)) + (find-variant-components data objects (:id shape))) + + (ctc/is-variant-main? shape) + (if-let [component (ctcl/get-component data (:component-id shape) true)] + {(:id component) (:variant-properties component)} + {}) + + :else + {})) + (defn extract-properties-names [shape data] (->> shape @@ -40,7 +59,7 @@ [data objects variant-id] (->> (find-variant-components data objects variant-id) (mapcat :variant-properties) - (group-by :name) + (ctv/group-properties-by-name) (mapv (fn [[k v]] (let [mdata (reduce merge {} (map meta v))] (with-meta {:name k diff --git a/common/src/app/common/flags.cljc b/common/src/app/common/flags.cljc index cf08f554b1..4439a5b316 100644 --- a/common/src/app/common/flags.cljc +++ b/common/src/app/common/flags.cljc @@ -191,6 +191,8 @@ :available-viewer-wasm :stroke-path :stroke-per-side + ;; Enables bullet lists in the v3 (WASM) text editor. + :text-editor-wasm-v1-lists :custom-shortcuts :remote-media-processing diff --git a/common/src/app/common/logic/libraries.cljc b/common/src/app/common/logic/libraries.cljc index fee7e718c2..a8dce1eca8 100644 --- a/common/src/app/common/logic/libraries.cljc +++ b/common/src/app/common/logic/libraries.cljc @@ -58,6 +58,44 @@ ;; We don't do automatic update of the `layout-grid-cells` property. (remove #(= :layout-grid-cells %)))) +;; Attrs whose value drifts by float noise when a shape is moved; they +;; are compared with tolerance during sync. The rest compare exactly. +(def ^:private drifting-geometry-attrs + #{:x :y :width :height :selrect :points}) + +(defn- position-data-close? + "Compares text position-data entry by entry: numbers with tolerance, + every other field (fills, fonts, text) exactly." + [a b] + (let [a (vec a) + b (vec b)] + (and (= (count a) (count b)) + (every? true? (map mth/close-map? a b))))) + +(defn- attr-equal? + [attr origin-shape dest-shape] + (let [origin-val (get origin-shape attr) + dest-val (get dest-shape attr)] + (or (= origin-val dest-val) + (cond + ;; close-attrs? zips :points and would ignore a length mismatch + (= :points attr) + (and (= (count origin-val) (count dest-val)) + (gsh/close-attrs? attr origin-val dest-val)) + + (contains? drifting-geometry-attrs attr) + (gsh/close-attrs? attr origin-val dest-val) + + (and (= :content attr) + (or (cfh/path-shape? origin-shape) (cfh/bool-shape? origin-shape))) + (segment/content-close? origin-val dest-val) + + (= :position-data attr) + (position-data-close? origin-val dest-val) + + :else + false)))) + ;; Flex-child layout attrs synced between main and copy (interactions are only kept on swap) (def ^:private flex-child-attrs (disj ctk/swap-keep-attrs :interactions)) @@ -1146,26 +1184,69 @@ initial-root? (:component-root shape-inst) + ;; The component name is normalized (path separators with one space at + ;; each side), so the name synced to the main instance root must be + ;; normalized too. Otherwise the root would not match the component's + ;; path+name and the file would fail :invalid-main-instance-name. + rename? (and (some? component) + (contains? (:touched shape-inst) :name-group)) + original-name (:name shape-inst) + clean-name (when rename? + (cpn/clean-path original-name)) + shape-inst (cond-> shape-inst + (some? clean-name) + (assoc :name clean-name)) + root-inst shape-inst root-main (ctf/get-component-root library component) - changes (cond-> changes - (and component (contains? (:touched shape-inst) :name-group)) - (generate-rename-component (:component-id shape-inst) (:name shape-inst) library))] + ;; When the synced shape is itself the component's main instance root, + ;; `find-remote-shape` returns nil and the recursive sync never reaches + ;; it, so it must be renamed explicitly to keep it matching the + ;; component's path+name. + main-root? (and (some? component) + (= (:id shape-inst) (:main-instance-id component))) - (if component - (generate-sync-shape-inverse-recursive changes - container - shape-inst - component - library - file - libraries - shape-main - root-inst - root-main - initial-root?) - changes))) + changes (cond-> changes + (some? clean-name) + (generate-rename-component (:component-id shape-inst) clean-name library)) + + changes (if component + (generate-sync-shape-inverse-recursive changes + container + shape-inst + component + library + file + libraries + shape-main + root-inst + root-main + initial-root?) + changes) + + changes (cond-> changes + (and (some? clean-name) + main-root? + (not= clean-name original-name)) + (-> (update :redo-changes conj + (make-change container + {:type :mod-obj + :id (:id shape-inst) + :local-change? true + :operations [{:type :set + :attr :name + :val clean-name}]})) + (update :undo-changes conj + (make-change container + {:type :mod-obj + :id (:id shape-inst) + :local-change? true + :operations [{:type :set + :attr :name + :val original-name}]}))))] + + changes)) (defn- generate-sync-shape-inverse-recursive [changes container shape-inst component library file libraries shape-main root-inst root-main initial-root?] @@ -1993,7 +2074,7 @@ reset-pos-data? (and (cfh/text-shape? origin-shape) (= attr :position-data) - (not= (:position-data origin-shape) (:position-data dest-shape)) + (not (attr-equal? :position-data origin-shape dest-shape)) (touched :geometry-group)) ;; On texts, when we want to omit the touched attrs, both text (the actual letters) @@ -2007,7 +2088,7 @@ (touched sync-group)) skip-operations? - (or (= (get origin-shape attr) (get dest-shape attr)) + (or (attr-equal? attr origin-shape dest-shape) (and (touched sync-group) omit-touched? ;; When it is a text-partial-change, we should generate operations @@ -2716,10 +2797,20 @@ [path name] (cpn/split-group-name name) + ;; The name of the main instance root must match the path+name of the + ;; component. The component name is normalized here (via split-group-name), + ;; so the root shape name must be normalized too. Otherwise creating a + ;; component from a shape with an unnormalized name (e.g. "Group/Subgroup") + ;; would leave a main instance failing :invalid-main-instance-name. + normalized-name (cpn/merge-path-item path name) + [root-shape updated-shapes] - (ctn/convert-shape-in-component root objects file-id) + (ctn/convert-shape-in-component (assoc root :name normalized-name) objects file-id) changes (-> changes + (cond-> (not= normalized-name (:name root)) + (pcb/update-shapes [(:id root)] #(assoc % :name normalized-name) + {:ignore-touched true})) (pcb/add-component (:id root-shape) path name @@ -2728,7 +2819,8 @@ page-id nil variant-id - props))] + props + {:apply-changes-local-library? true}))] [root-shape changes])) (defn generate-add-component @@ -3165,6 +3257,7 @@ is-component-main? (ctk/main-instance? obj) subinstance-head? (ctk/subinstance-head? obj) instance-root? (ctk/instance-root? obj) + parent-not-in-copy (not (ctk/in-component-copy? parent)) into-component? (and duplicating-component? (ctn/in-any-component? objects parent)) @@ -3174,6 +3267,8 @@ (ctn/get-nesting-level-delta objects obj parent)) new-shape-ref (ctf/advance-shape-ref nil page libraries obj level-delta {:include-deleted? true}) + + regenerate-component (fn [changes shape] (let [[_ changes] (generate-add-component-changes changes shape objects file-id (:id page) variant-props)] @@ -3188,7 +3283,8 @@ (cond-> (and (not instance-root?) subinstance-head? - remove-swap-slot?) + remove-swap-slot? + parent-not-in-copy) (ctk/remove-swap-slot)) (dissoc :shapes @@ -3240,6 +3336,25 @@ (and is-component-root? is-component-main?) (regenerate-component new-obj)) + ;; Unknown-component paste into a variant set: align props/names + ;; after regenerate. Reload the shape so we use the new component-id. + ;; Prefit the component name/path to the container so + ;; create-new-properties-from-variant does not treat the component + ;; full name as a former set name and inflate properties (add-name?). + changes (cond-> changes + (and is-component-root? + is-component-main? + (ctk/is-variant-container? parent)) + (as-> $ + (let [shape (get (pcb/get-objects $) (:id new-obj)) + [cpath cname] (cpn/split-group-name (:name parent)) + changes (pcb/update-component + $ + (:component-id shape) + #(assoc % :name cname :path cpath) + {:apply-changes-local-library? true})] + (clvp/generate-make-shapes-variant changes [shape] parent)))) + ;; This is needed for the recursive call to find the new object as parent page' (ctst/add-shape (:id new-obj) new-obj diff --git a/common/src/app/common/logic/variant_properties.cljc b/common/src/app/common/logic/variant_properties.cljc index 02490a3c81..141cbd2279 100644 --- a/common/src/app/common/logic/variant_properties.cljc +++ b/common/src/app/common/logic/variant_properties.cljc @@ -266,10 +266,20 @@ (first (reduce (fn [[changes used-values] shape] - (let [component (ctcl/get-component data (:component-id shape) true)] - (if (or (zero? num-shapes) ;; do nothing if there are no shapes - (and (= variant-id (:variant-id shape)) ;; or we are only moving the shape inside its parent (it is - (not (:deleted component)))) ;; the same parent and the component isn't deleted) + (let [component (ctcl/get-component data (:component-id shape) true) + already-here? (= variant-id (:variant-id shape)) + prop-names (cfv/extract-properties-names shape data) + base-names (map :name base-props) + props-ok? (= prop-names base-names) + names-ok? (= (cpn/merge-path-item (:path component) (:name component)) + container-name) + variant-ok? (= (:variant-name shape) + (ctv/properties-to-name (:variant-properties component))) + aligned? (and props-ok? names-ok? variant-ok?)] + (if (or (zero? num-shapes) ;; do nothing if there are no shapes + (and already-here? ;; or already a fully aligned child of this container + aligned? + (not (:deleted component)))) [changes used-values] (let [props (-> (create-new-properties shape total-props) (fill-blank-properties used-values)) diff --git a/common/src/app/common/math.cljc b/common/src/app/common/math.cljc index 41e3047880..c924446c15 100644 --- a/common/src/app/common/math.cljc +++ b/common/src/app/common/math.cljc @@ -208,6 +208,19 @@ ([num1 num2 precision] (<= (abs (- num1 num2)) precision))) +(defn close-map? + "Equality for maps: same keys, numeric values within `close?`, any other + value compared exactly." + [a b] + (and (= (count a) (count b)) + (every? (fn [[k v]] + (and (contains? b k) + (let [w (get b k)] + (if (number? v) + (and (number? w) (close? v w)) + (= v w))))) + a))) + (defn lerp "Calculates a the linear interpolation between two values and a given percent" [v0 v1 t] diff --git a/common/src/app/common/media.cljc b/common/src/app/common/media.cljc index a82759a7bd..f881684cbe 100644 --- a/common/src/app/common/media.cljc +++ b/common/src/app/common/media.cljc @@ -98,36 +98,90 @@ (let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)$"] (str/replace filename image-extensions-re ""))) +(def ^:private font-variant-tokens + "Weight and style tokens stripped from a font name to derive its family. + Keep the alternatives in sync with `parse-font-weight` and + `parse-font-style`." + ["extra\\s*black" "ultra\\s*black" "extra\\s*bold" "ultra\\s*bold" + "semi\\s*bold" "demi\\s*bold" "extra\\s*light" "ultra\\s*light" + "hairline" "thin" "light" "normal" "regular" "medium" "bold" "black" + "heavy" "solid" "italic" "oblique"]) + +(def ^:private font-variant-tokens-re + ;; The trailing boundary is a lookahead, so two tokens separated by a single + ;; space are both stripped in one pass (the separator is not consumed). + (re-pattern (str "(?i)(^|[-_\\s])(" (str/join "|" font-variant-tokens) ")(?=[-_\\s]|$)"))) + +(def ^:private font-concatenated-style-re + ;; Separate a style token glued to a weight token, e.g. "BoldItalic" -> + ;; "Bold Italic". Only a known weight token followed by a style token is + ;; split, so families such as "OpenSans" are left untouched. The trailing + ;; boundary is a lookahead, so a glued style token followed by a technical + ;; suffix (e.g. "Italic_v2") is split too. + #"(?i)(black|bold|heavy|solid|medium|light|thin|hairline|regular|normal)(italic|oblique)(?=[-_\s]|$)") + +(def ^:private font-technical-suffix-re + ;; Trailing foundry/build suffixes, e.g. "Roboto-Bold-VF" or "Inter_v2". + ;; Requires a separator; stripped before token detection so the parsers + ;; interpret the name the same way. + #"(?i)[-_\s]+(?:vf|v[0-9]+)$") + +(defn- normalize-font-name + "Normalize a font name or variant string before token detection: + split a style token glued to a weight token, then strip a trailing + technical suffix. Applied by `parse-font-family`, `parse-font-weight` + and `parse-font-style` so all three interpret the name the same way." + [s] + (-> s + (str/replace font-concatenated-style-re "$1 $2") + (str/replace font-technical-suffix-re ""))) + +(defn parse-font-family + "Derive a font family name from a base name (a filename without its + extension), stripping the known weight/style tokens." + [base-name] + (let [normalized (normalize-font-name base-name) + stripped (-> normalized + (str/replace font-variant-tokens-re "$1") + (str/replace #"[-_\s]+" " ") + (str/trim))] + (if (str/blank? stripped) base-name stripped))) + (defn parse-font-weight "Parses a font variant/subfamily string into a numeric weight. A nil or non-string variant means the font carries no subfamily metadata, so it defaults to 400 (regular), the same as a blank string." [variant] - (let [variant (if (string? variant) variant "")] + (let [variant (normalize-font-name (if (string? variant) variant ""))] (cond - (re-seq #"(?i)(?:^|[-_\s])(hairline|thin)(?=(?:[-_\s]|$|italic\b))" variant) 100 - (re-seq #"(?i)(?:^|[-_\s])(extra\s*light|ultra\s*light)(?=(?:[-_\s]|$|italic\b))" variant) 200 - (re-seq #"(?i)(?:^|[-_\s])(light)(?=(?:[-_\s]|$|italic\b))" variant) 300 - (re-seq #"(?i)(?:^|[-_\s])(normal|regular)(?=(?:[-_\s]|$|italic\b))" variant) 400 - (re-seq #"(?i)(?:^|[-_\s])(medium)(?=(?:[-_\s]|$|italic\b))" variant) 500 - (re-seq #"(?i)(?:^|[-_\s])(semi\s*bold|demi\s*bold)(?=(?:[-_\s]|$|italic\b))" variant) 600 - (re-seq #"(?i)(?:^|[-_\s])(extra\s*bold|ultra\s*bold)(?=(?:[-_\s]|$|italic\b))" variant) 800 - (re-seq #"(?i)(?:^|[-_\s])(bold)(?=(?:[-_\s]|$|italic\b))" variant) 700 - (re-seq #"(?i)(?:^|[-_\s])(extra\s*black|ultra\s*black)(?=(?:[-_\s]|$|italic\b))" variant) 950 - (re-seq #"(?i)(?:^|[-_\s])(black|heavy|solid)(?=(?:[-_\s]|$|italic\b))" variant) 900 - :else 400))) + (re-seq #"(?i)(?:^|[-_\s])(hairline|thin)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 100 + (re-seq #"(?i)(?:^|[-_\s])(extra\s*light|ultra\s*light)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 200 + (re-seq #"(?i)(?:^|[-_\s])(light)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 300 + (re-seq #"(?i)(?:^|[-_\s])(normal|regular)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 400 + (re-seq #"(?i)(?:^|[-_\s])(medium)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 500 + (re-seq #"(?i)(?:^|[-_\s])(semi\s*bold|demi\s*bold)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 600 + (re-seq #"(?i)(?:^|[-_\s])(extra\s*bold|ultra\s*bold)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 800 + (re-seq #"(?i)(?:^|[-_\s])(bold)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 700 + (re-seq #"(?i)(?:^|[-_\s])(extra\s*black|ultra\s*black)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 950 + (re-seq #"(?i)(?:^|[-_\s])(black|heavy|solid)(?=(?:[-_\s]|$|italic\b|oblique\b))" variant) 900 + :else 400))) (defn parse-font-style "Parses a font variant/subfamily string into \"italic\" or \"normal\". A nil or non-string variant means the font carries no subfamily metadata, so it defaults to \"normal\"." [variant] - (let [variant (if (string? variant) variant "")] - (if (or (re-seq #"(?i)(?:^|[-_\s])(italic)(?:[-_\s]|$)" variant) - (re-seq #"(?i)italic$" variant)) + (let [variant (normalize-font-name (if (string? variant) variant ""))] + (if (or (re-seq #"(?i)(?:^|[-_\s])(italic|oblique)(?:[-_\s]|$)" variant) + (re-seq #"(?i)(?:italic|oblique)$" variant)) "italic" "normal"))) +(defn first-nonblank-string + "Return the first nonblank string from the candidates, or nil." + [& candidates] + (first (filter #(and (string? %) (not (str/blank? %))) candidates))) + (defn font-weight->name [weight] (case (long weight) diff --git a/common/src/app/common/path_names.cljc b/common/src/app/common/path_names.cljc index 3f5fd28603..a81b6cfaee 100644 --- a/common/src/app/common/path_names.cljc +++ b/common/src/app/common/path_names.cljc @@ -147,6 +147,20 @@ Some naming conventions: [(subs s 0 (inc last-period)) (subs s (inc last-period))] [s ""])) +(defn remove-extension + "Removes the extension (the text after the last period, including the + period itself) from a name: + 'photo.svg' -> 'photo' + 'icon.SVG' -> 'icon' + 'my.svg.icon.svg' -> 'my.svg.icon' + 'archive.tar.gz' -> 'archive.tar' + 'design' -> 'design'" + [name] + (let [[base ext] (split-by-last-period name)] + (if (str/empty? ext) + name + (subs base 0 (dec (count base)))))) + ;; Tree building functions -------------------------------------------------- (defn- attach-remaining-path diff --git a/common/src/app/common/render_wasm/README.md b/common/src/app/common/render_wasm/README.md index b153832e52..6517399427 100644 --- a/common/src/app/common/render_wasm/README.md +++ b/common/src/app/common/render_wasm/README.md @@ -5,12 +5,12 @@ layouts, memory helpers and serializers that turn Penpot shapes into the buffers `render-wasm` consumes. The workspace drives it from `app.render-wasm.*`, the headless exporter from -`app.wasm.*` — same code underneath, so the two cannot drift. +`app.wasm.*`, the renderer benchmarking code from `benches.render-wasm.*`. Font knowledge is *not* here even though both hosts need it for rendering: it is not specific to the wasm backend, so the google fonts catalog (baked from `common/resources/fonts/gfonts.*.json`), the bundled builtin family and the -emoji/script fallback tables live in `app.common.fonts`. Likewise the image-id +emoji/script fallback tables live in `app.common.fonts`. Likewise, the image-id enumeration lives in `app.common.types.shape.images`. `shared.js` is not here: it is a per-build artifact, so each host compiles diff --git a/common/src/app/common/render_wasm/api/modifiers.cljs b/common/src/app/common/render_wasm/api/modifiers.cljs new file mode 100644 index 0000000000..4aa3ac288f --- /dev/null +++ b/common/src/app/common/render_wasm/api/modifiers.cljs @@ -0,0 +1,171 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.common.render-wasm.api.modifiers + "Modifier operations for an initialized WASM renderer. + + Shared by the frontend and the renderer benchmarking code. + + Calls share one payload slot and must run without overlap. The caller owns + renderer initialization and render scheduling." + (:require + [app.common.data :as d] + [app.common.geom.matrix :as gmt] + [app.common.render-wasm.helpers :as h] + [app.common.render-wasm.mem :as mem] + [app.common.render-wasm.mem.heap32 :as mem.h32] + [app.common.render-wasm.serializers :as sr] + [app.common.render-wasm.wasm :as wasm] + [app.common.uuid :as uuid])) + +(def ^:private ^:const MODIFIER-U8-SIZE 40) +(def ^:private ^:const MODIFIER-U32-SIZE (/ MODIFIER-U8-SIZE 4)) +(def ^:private ^:const INPUT-MODIFIER-U8-SIZE 44) +(def ^:private ^:const STRUCTURE-MODIFIER-U8-SIZE 44) + +(def ^:private pixel-precision + {:disabled 0 + :both 1 + :only-x 2 + :only-y 3}) + +(defn- pixel-precision-mode + [snap-pixel? snap-ignore-axis] + (pixel-precision + (cond + (not snap-pixel?) :disabled + (= :x snap-ignore-axis) :only-y + (= :y snap-ignore-axis) :only-x + :else :both))) + +(defn- get-modifier-entry + [heapu32 heapf32 offset] + (let [id1 (aget heapu32 (+ offset 0)) + id2 (aget heapu32 (+ offset 1)) + id3 (aget heapu32 (+ offset 2)) + id4 (aget heapu32 (+ offset 3)) + a (aget heapf32 (+ offset 4)) + b (aget heapf32 (+ offset 5)) + c (aget heapf32 (+ offset 6)) + d (aget heapf32 (+ offset 7)) + e (aget heapf32 (+ offset 8)) + f (aget heapf32 (+ offset 9))] + (d/vec2 (uuid/from-unsigned-parts id1 id2 id3 id4) + (gmt/matrix a b c d e f)))) + +(defn- get-modifiers + [pointer] + (let [offset (mem/->offset-32 pointer) + heapu32 (mem/get-heap-u32) + heapf32 (mem/get-heap-f32) + length (aget heapu32 offset) + start (inc offset) + end (+ start (* length MODIFIER-U32-SIZE))] + (mapv (partial get-modifier-entry heapu32 heapf32) + (range start end MODIFIER-U32-SIZE)))) + +(defn propagate-modifiers + "Propagates geometry transforms through the shape tree. + + Rounds geometry to the pixel grid when snapping is enabled, except on the + ignored axis. Requires an initialized renderer and enum bindings. + + Args: + entries: Counted collection of [uuid {:transform matrix :kind kind}], + where kind is :parent or :child. + snap-pixel?: Whether to round geometry to the pixel grid. + snap-ignore-axis: Axis to leave unsnapped, :x, :y or nil. + + Returns: + A vector of [uuid matrix] transforms, or [] for empty entries." + [entries snap-pixel? snap-ignore-axis] + (if (empty? entries) + [] + (let [size (mem/get-alloc-size entries INPUT-MODIFIER-U8-SIZE) + offset (mem/alloc->offset-32 size) + heapf32 (mem/get-heap-f32) + heapu32 (mem/get-heap-u32)] + (reduce (fn [offset [id {:keys [transform kind]}]] + (-> offset + (mem.h32/write-uuid heapu32 id) + (mem.h32/write-matrix heapf32 transform) + (mem.h32/write-u32 heapu32 (sr/translate-transform-entry-kind kind)))) + offset + entries) + (let [precision (pixel-precision-mode snap-pixel? snap-ignore-axis) + pointer (h/call wasm/internal-module "_propagate_modifiers" precision)] + (try + (get-modifiers pointer) + (finally + (mem/free))))))) + +(defn set-modifiers + "Sets temporary geometry transforms for rendering. + + Empty modifiers leave the current preview unchanged. + + Args: + modifiers: Vector of [uuid matrix] transforms." + [modifiers] + (when (seq modifiers) + (let [size (mem/get-alloc-size modifiers MODIFIER-U8-SIZE) + offset (mem/alloc->offset-32 size) + heapu32 (mem/get-heap-u32) + heapf32 (mem/get-heap-f32)] + (reduce (fn [offset [id transform]] + (-> offset + (mem.h32/write-uuid heapu32 id) + (mem.h32/write-matrix heapf32 transform))) + offset + modifiers) + (h/call wasm/internal-module "_set_modifiers"))) + nil) + +(defn set-structure-modifiers + "Sets temporary changes to shape structure and content scaling. + + Empty entries leave the current structure modifiers unchanged. + + Args: + entries: Counted collection of maps with :type, :parent, :id, :index + and :value. A missing index defaults to zero." + [entries] + (when (seq entries) + (let [size (mem/get-alloc-size entries STRUCTURE-MODIFIER-U8-SIZE) + offset (mem/alloc->offset-32 size) + heapu32 (mem/get-heap-u32) + heapf32 (mem/get-heap-f32)] + (reduce (fn [offset {:keys [type parent id index value]}] + (-> offset + (mem.h32/write-u32 heapu32 (sr/translate-structure-modifier-type type)) + (mem.h32/write-u32 heapu32 (d/nilv index 0)) + (mem.h32/write-uuid heapu32 parent) + (mem.h32/write-uuid heapu32 id) + (mem.h32/write-f32 heapf32 value))) + offset + entries) + (h/call wasm/internal-module "_set_structure_modifiers"))) + nil) + +(defn set-modifiers-start + "Enables interactive rendering for shape transforms." + [] + (h/call wasm/internal-module "_set_modifiers_start") + nil) + +(defn set-modifiers-end + "Ends interactive rendering for shape transforms. + + A subsequent render produces the final full-quality image." + [] + (h/call wasm/internal-module "_set_modifiers_end") + nil) + +(defn clean-modifiers + "Clears temporary geometry, structure and content-scaling modifiers." + [] + (h/call wasm/internal-module "_clean_modifiers") + nil) diff --git a/common/src/app/common/render_wasm/api/upload.cljs b/common/src/app/common/render_wasm/api/upload.cljs index 8f53145b05..146de90edf 100644 --- a/common/src/app/common/render_wasm/api/upload.cljs +++ b/common/src/app/common/render_wasm/api/upload.cljs @@ -27,6 +27,7 @@ (def ^:const BASE-PROPS-SIZE 104) (def ^:const FLAG-CLIP-CONTENT 0x01) (def ^:const FLAG-HIDDEN 0x02) +(def ^:const FLAG-HIDE-FILL-ON-EXPORT 0x04) (def ^:const SECTION-CHILDREN 0x01) (def ^:const SECTION-BLUR-LAYER 0x02) @@ -61,9 +62,11 @@ (not (get shape :show-content)) false) hidden (get shape :hidden false) + hide-fill (get shape :hide-fill-on-export false) flags (cond-> 0 clip-content (bit-or FLAG-CLIP-CONTENT) - hidden (bit-or FLAG-HIDDEN)) + hidden (bit-or FLAG-HIDDEN) + hide-fill (bit-or FLAG-HIDE-FILL-ON-EXPORT)) blend-mode (sr/translate-blend-mode (get shape :blend-mode)) constraint-h (sr/translate-constraint-h (or (get shape :constraints-h) :none)) constraint-v (sr/translate-constraint-v (or (get shape :constraints-v) :none)) diff --git a/common/src/app/common/render_wasm/enums.clj b/common/src/app/common/render_wasm/enums.clj index 0be29aacf9..0e73f76142 100644 --- a/common/src/app/common/render_wasm/enums.clj +++ b/common/src/app/common/render_wasm/enums.clj @@ -38,6 +38,7 @@ ["text-direction" "RawTextDirection"] ["text-decoration" "RawTextDecoration"] ["text-transform" "RawTextTransform"] + ["list-style-type" "RawListStyleType"] ["multiple-state" "MultipleState"] ["transform-entry-kind" "RawTransformEntryKind"] ["segment-data" "RawSegmentData"] diff --git a/common/src/app/common/render_wasm/helpers.cljc b/common/src/app/common/render_wasm/helpers.cljc index d96cc9f990..7a73d6f94e 100644 --- a/common/src/app/common/render_wasm/helpers.cljc +++ b/common/src/app/common/render_wasm/helpers.cljc @@ -5,34 +5,42 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.common.render-wasm.helpers - #?(:cljs (:require-macros [app.common.render-wasm.helpers])) - (:require [app.common.data :as d])) + #?(:cljs (:require-macros [app.common.render-wasm.helpers]))) (def error-code "WASM error code constants (must match render-wasm/src/error.rs and mem.rs)." {0x01 :non-blocking 0x02 :panic}) +#?(:cljs + (defn on-call-error + "Restores the stack pointer after a failed WASM call and throws the + classified error." + [module fn-name stack cause] + (when (some? stack) + ((unchecked-get module "stackRestore") stack)) + (let [read-code-fn (unchecked-get module "_read_error_code") + code-num (when read-code-fn (read-code-fn)) + code (get error-code code-num :wasm-critical) + hint (str "WASM Error (" (name code) ")")] + (throw (ex-info hint {:type :wasm-error :code code :hint hint :fn fn-name} cause))))) + (defmacro call "A helper for calling a wasm function. Catches any exception thrown by the WASM function, reads the error code from WASM when available, and routes it based on the error type: - :wasm-non-blocking: call app.main.errors/on-error (eventually, shows a toast and logs the error) - - :wasm-critical or unknown: throws an exception to be handled by the global error handler (eventually, shows the internal error page)" + - :wasm-critical or unknown: throws an exception to be handled by the global error handler (eventually, shows the internal error page) + Restores the stack pointer on failure, since a trap leaves its shadow-stack frames behind." [module name & params] - (let [fn-sym (with-meta (gensym "fn-") {:tag 'function}) - cause-sym (gensym "cause")] - `(let [~fn-sym (cljs.core/unchecked-get ~module ~name)] + (let [fn-sym (with-meta (gensym "fn-") {:tag 'function}) + module-sym (gensym "module") + stack-sym (gensym "stack")] + `(let [~module-sym ~module + ~fn-sym (cljs.core/unchecked-get ~module-sym ~name) + save-fn# (cljs.core/unchecked-get ~module-sym "stackSave") + ~stack-sym (when save-fn# (save-fn#))] (try (~fn-sym ~@params) - (catch :default ~cause-sym - (let [read-code-fn# (cljs.core/unchecked-get ~module "_read_error_code") - code-num# (when read-code-fn# (read-code-fn#)) - code# (get error-code code-num# :wasm-critical) - hint# (str "WASM Error (" (d/name code#) ")") - context# {:type :wasm-error - :code code# - :hint hint# - :fn ~name} - cause# (ex-info hint# context# ~cause-sym)] - (throw cause#))))))) + (catch :default cause# + (on-call-error ~module-sym ~name ~stack-sym cause#)))))) diff --git a/common/src/app/common/render_wasm/serializers.cljs b/common/src/app/common/render_wasm/serializers.cljs index 02fe6632d2..b3db247e10 100644 --- a/common/src/app/common/render_wasm/serializers.cljs +++ b/common/src/app/common/render_wasm/serializers.cljs @@ -302,6 +302,16 @@ [text-direction] (untranslate "text-direction" text-direction "ltr")) +(defn translate-list-style-type + [list-style-type] + (let [values (unchecked-get wasm/serializers "list-style-type") + default (unchecked-get values "none")] + (d/nilv (unchecked-get values (d/name list-style-type)) default))) + +(defn untranslate-list-style-type + [list-style-type] + (untranslate "list-style-type" list-style-type "none")) + (defn translate-font-style [font-style] diff --git a/common/src/app/common/render_wasm/text_content.cljs b/common/src/app/common/render_wasm/text_content.cljs index c98c00b4c4..a19fb47c47 100644 --- a/common/src/app/common/render_wasm/text_content.cljs +++ b/common/src/app/common/render_wasm/text_content.cljs @@ -24,7 +24,7 @@ [app.common.uuid :as uuid] [cuerdas.core :as str])) -(def ^:const PARAGRAPH-ATTR-U8-SIZE 12) +(def ^:const PARAGRAPH-ATTR-U8-SIZE 16) (def ^:const SPAN-ATTR-U8-SIZE 64) (def ^:const MAX-TEXT-FILLS types.fills.impl/MAX-FILLS) @@ -109,7 +109,8 @@ text-decoration (sr/translate-text-decoration (get paragraph :text-decoration)) text-transform (sr/translate-text-transform (get paragraph :text-transform)) line-height (serialize-line-height (get paragraph :line-height)) - letter-spacing (serialize-letter-spacing (get paragraph :letter-spacing))] + letter-spacing (serialize-letter-spacing (get paragraph :letter-spacing)) + list-style-type (sr/translate-list-style-type (get paragraph :list-style-type))] (-> offset (mem/write-u8 dview text-align) (mem/write-u8 dview text-direction) @@ -117,6 +118,11 @@ (mem/write-u8 dview text-transform) (mem/write-f32 dview line-height) (mem/write-f32 dview letter-spacing) + (mem/write-u8 dview list-style-type) + ;; padding + (mem/write-u8 dview 0) + (mem/write-u8 dview 0) + (mem/write-u8 dview 0) (mem/assert-written offset PARAGRAPH-ATTR-U8-SIZE)))) (defn- write-spans diff --git a/common/src/app/common/test_helpers/render_wasm.cljs b/common/src/app/common/test_helpers/render_wasm.cljs new file mode 100644 index 0000000000..454db679d8 --- /dev/null +++ b/common/src/app/common/test_helpers/render_wasm.cljs @@ -0,0 +1,131 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.common.test-helpers.render-wasm + (:require + [app.common.geom.matrix :as gmt] + [app.common.uuid :as uuid])) + +(def serializers + #js {"transform-entry-kind" #js {"parent" 0 "child" 1}}) + +(defn- get-uuid + [view offset] + (uuid/from-unsigned-parts + (.getUint32 view offset true) + (.getUint32 view (+ offset 4) true) + (.getUint32 view (+ offset 8) true) + (.getUint32 view (+ offset 12) true))) + +(defn- get-matrix + [view offset] + (apply gmt/matrix + (mapv #(.getFloat32 view (+ offset (* 4 %)) true) (range 6)))) + +(defn- write-transform + [view offset [id matrix]] + (doseq [[index value] (map-indexed vector (uuid/get-u32 id))] + (.setUint32 view (+ offset (* 4 index)) value true)) + (doseq [[index key] (map-indexed vector [:a :b :c :d :e :f])] + (.setFloat32 view (+ offset 16 (* 4 index)) (get matrix key) true))) + +(defn create-module + "Creates a fake module implementing modifier payload exchanges. + + Args: + options: Map with :output transforms, :grow-on-alloc, :grow-on-propagate, + :propagate-error or :set-error code, and :invalid-output-pointer. + + Returns: + A map with :module and an atom :state containing decoded native inputs, + the active payload slot, previews and interactive mode." + [{:keys [output grow-on-alloc grow-on-propagate propagate-error set-error invalid-output-pointer] + :or {output []}}] + (let [module #js {} + state (atom {:slot nil :propagated nil :preview [] :structure [] + :interactive false :error-code 0}) + resize (fn [size] + (let [buffer (js/ArrayBuffer. size)] + (set! (.-HEAPU8 module) (js/Uint8Array. buffer)) + (set! (.-HEAPU32 module) (js/Uint32Array. buffer)) + (set! (.-HEAPF32 module) (js/Float32Array. buffer)))) + alloc (fn [size] + (when (or (:slot @state) (not (pos? size))) + (throw (js/Error. "Payload slot is unavailable"))) + (when grow-on-alloc + (resize (* 2 (.-byteLength (.-HEAPU8 module))))) + (swap! state assoc :slot {:offset 64 :size size}) + 64) + take-input + (fn [entry-size decode] + (let [{:keys [offset size]} (:slot @state)] + (when (or (nil? offset) (not (zero? (mod size entry-size)))) + (throw (js/Error. "Invalid modifier payload"))) + (swap! state assoc :slot nil :error-code 0) + (let [view (js/DataView. (.-buffer (.-HEAPU8 module)))] + (mapv #(decode view (+ offset %)) (range 0 size entry-size))))) + get-transform + (fn [view offset] + [(get-uuid view offset) (get-matrix view (+ offset 16))])] + (resize 4096) + (set! (.-_alloc_bytes module) alloc) + (set! (.-_free_bytes module) + (fn [] + (swap! state assoc :slot nil :error-code 0) + js/undefined)) + (set! (.-_read_error_code module) (fn [] (:error-code @state))) + (set! (.-_propagate_modifiers module) + (fn [precision] + (let [entries (take-input + 44 + (fn [view offset] + [(get-uuid view offset) + {:transform (get-matrix view (+ offset 16)) + :kind (case (.getUint32 view (+ offset 40) true) + 0 :parent + 1 :child)}]))] + (swap! state assoc :propagated {:precision precision :entries entries}) + (when propagate-error + (swap! state assoc :error-code propagate-error) + (throw (js/Error. "Native propagation failed"))) + (when grow-on-propagate + (resize (* 2 (.-byteLength (.-HEAPU8 module))))) + (let [pointer (alloc (+ 4 (* 40 (count output)))) + view (js/DataView. (.-buffer (.-HEAPU8 module)))] + (.setUint32 view pointer (count output) true) + (doseq [[index entry] (map-indexed vector output)] + (write-transform view (+ pointer 4 (* 40 index)) entry)) + (if invalid-output-pointer + #js {"valueOf" (fn [] (throw (js/Error. "Invalid output pointer")))} + pointer))))) + (set! (.-_set_modifiers module) + (fn [] + (let [transforms (take-input 40 get-transform)] + (when set-error + (swap! state assoc :error-code set-error) + (throw (js/Error. "Native preview failed"))) + (swap! state assoc :preview transforms)))) + (set! (.-_set_structure_modifiers module) + (fn [] + (swap! state assoc :structure + (take-input + 44 + (fn [view offset] + {:type (case (.getUint32 view offset true) + 1 :remove-children + 2 :add-children + 3 :scale-content) + :index (.getUint32 view (+ offset 4) true) + :parent (get-uuid view (+ offset 8)) + :id (get-uuid view (+ offset 24)) + :value (.getFloat32 view (+ offset 40) true)}))))) + (set! (.-_set_modifiers_start module) + (fn [] (swap! state assoc :interactive true))) + (set! (.-_set_modifiers_end module) + (fn [] (swap! state assoc :interactive false))) + (set! (.-_clean_modifiers module) + (fn [] (swap! state assoc :preview [] :structure []))) + {:module module :state state})) diff --git a/common/src/app/common/types/color.cljc b/common/src/app/common/types/color.cljc index 96e71a5fcb..0979f1214f 100644 --- a/common/src/app/common/types/color.cljc +++ b/common/src/app/common/types/color.cljc @@ -535,9 +535,10 @@ (if (= max norm-g) (+ 120 (* 60 (/ (- norm-b norm-r) (- max min)))) (+ 240 (* 60 (/ (- norm-r norm-g) (- max min))))))) - s (if (and (> l 0) (<= l 0.5)) - (/ (- max min) (* 2 l)) - (/ (- max min) (- 2 (* 2 l))))] + s (cond + (= max min) 0 + (<= l 0.5) (/ (- max min) (* 2 l)) + :else (/ (- max min) (- 2 (* 2 l))))] [(mod (+ h 360) 360) s l])) (defn hex->hsv diff --git a/common/src/app/common/types/container.cljc b/common/src/app/common/types/container.cljc index 849467d7f4..7c61356cbc 100644 --- a/common/src/app/common/types/container.cljc +++ b/common/src/app/common/types/container.cljc @@ -196,7 +196,7 @@ ;; Calculate how many parent heads share in common the original ;; shape and the new parent. pairs (map vector orig-heads dest-heads) - common-count (count (take-while (fn [a b] (= a b)) pairs))] + common-count (count (take-while (fn [[a b]] (= a b)) pairs))] (- (count orig-heads) common-count))) diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index 444484b68f..359cbd3d6d 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -95,7 +95,6 @@ (def schema:file-metadata [:map {:title "Metadata"} - [:storage-ref-id {:optional true} ::sm/uuid] [:generated-by {:optional true} :string] [:referer {:optional true} :string]]) @@ -616,16 +615,28 @@ ([file-data component] (load-component-objects file-data component (gpt/point 0 0))) ([file-data component delta] - (if (and component (empty? (:objects component))) ;; This operation may be called twice, e.g. in an idempotent change + ;; nil? not empty?: a missing-main soft-delete stores {}, which must + ;; still count as already loaded so render/library callers do not re-enter. + (if (and component (nil? (:objects component))) ;; This operation may be called twice, e.g. in an idempotent change (let [component-page (get-component-page file-data component) page-objects (:objects component-page) - objects (->> (cons (:main-instance-id component) - (cfh/get-children-ids page-objects (:main-instance-id component))) - (map #(get page-objects %)) + ids (into [] + (cons (:main-instance-id component) + (cfh/get-children-ids page-objects (:main-instance-id component)))) + ;; Skip ids already gone from the page. Calling gsh/move on nil + ;; builds {:selrect nil :points []} and index-by :id stores it + ;; under nil, which later fails duplicate-file schema checks. + shapes (into [] (keep #(get page-objects %)) ids) + objects (->> shapes ;; when it is an undo of a cut-paste, we need to undo the movement ;; of the shapes so we need to move them delta (map #(gsh/move % delta)) (d/index-by :id))] + (when (not= (count shapes) (count ids)) + (l/wrn :hint "skipping missing shapes when loading component objects" + :component-id (str (:id component)) + :main-instance-id (str (:main-instance-id component)) + :file-id (str (:id file-data)))) (assoc component :objects objects)) component))) diff --git a/common/src/app/common/types/path/segment.cljc b/common/src/app/common/types/path/segment.cljc index 561dd4e44b..dbfc1a44bc 100644 --- a/common/src/app/common/types/path/segment.cljc +++ b/common/src/app/common/types/path/segment.cljc @@ -1355,6 +1355,20 @@ (grc/points->rect extremities) (grc/make-rect)))) +(defn- segment-close? + [a b] + (and (= (:command a) (:command b)) + (mth/close-map? (:params a) (:params b)))) + +(defn content-close? + "True when both path contents have the same segments and every param + is within `mth/close?` of its counterpart." + [content-a content-b] + (let [a (vec content-a) + b (vec content-b)] + (and (= (count a) (count b)) + (every? true? (map segment-close? a b))))) + (defn content-center [content] (-> content diff --git a/common/src/app/common/types/shape/attrs.cljc b/common/src/app/common/types/shape/attrs.cljc index eb8dc69193..400538033d 100644 --- a/common/src/app/common/types/shape/attrs.cljc +++ b/common/src/app/common/types/shape/attrs.cljc @@ -357,6 +357,8 @@ :text-direction + :list-style-type + :line-height :letter-spacing diff --git a/common/src/app/common/types/text.cljc b/common/src/app/common/types/text.cljc index 4e53c9b753..fbeb80695c 100644 --- a/common/src/app/common/types/text.cljc +++ b/common/src/app/common/types/text.cljc @@ -43,6 +43,9 @@ (def text-direction-attrs [:text-direction]) +(def text-list-attrs + [:list-style-type]) + (def text-spacing-attrs [:line-height :letter-spacing]) @@ -102,7 +105,8 @@ (def paragraph-attrs (d/concat-vec text-align-attrs - text-direction-attrs)) + text-direction-attrs + text-list-attrs)) (def text-node-attrs (d/concat-vec @@ -138,6 +142,15 @@ {:text-align "left" :text-direction "ltr"}) +(def default-list-attrs + "Older paragraphs have no list attrs; read them as no list." + {:list-style-type "none"}) + +(defn list-paragraph? + "True when a paragraph (or its merged style map) is a list item." + [node] + (= "disc" (:list-style-type node))) + (def default-text-attrs {:typography-ref-file nil :typography-ref-id nil diff --git a/common/src/app/common/types/tokens_lib.cljc b/common/src/app/common/types/tokens_lib.cljc index 082da2d43e..d045e93e55 100644 --- a/common/src/app/common/types/tokens_lib.cljc +++ b/common/src/app/common/types/tokens_lib.cljc @@ -25,7 +25,10 @@ [clojure.pprint :as pp] [clojure.set :as set] [clojure.walk :as walk] - [cuerdas.core :as str])) + [cuerdas.core :as str] + [tokenscript.document.dtcg :as tsd] + [tokenscript.document.read :as tsr] + [tokenscript.document.write :as tsw])) ;; === Common @@ -1460,127 +1463,18 @@ active-tokens))) ;; === Import / Export from JSON format - -;; Supported formats: -;; - Legacy: for tokens files prior to DTCG second draft -;; - DTCG: for tokens files conforming to the DTCG second draft (current for now) -;; https://www.w3.org/community/design-tokens/2022/06/14/call-to-implement-the-second-editors-draft-and-share-feedback/ ;; -;; - Single set: for files that comply with the base DTCG format, that contain a single tree of tokens. -;; - Multi sets: for files with the Tokens Studio extension, that may contain several sets, and also themes and other $metadata. +;; Token files are read and written by `tokenscript.document.read` and +;; `tokenscript.document.write`, which know the formats: DTCG (the 2022 +;; drafts and the Format Module 2025.10), legacy and multi-set files. +;; This section maps its documents to a TokensLib and back: penpot's +;; token types, the keys of composite values, and the activation of +;; themes and sets. ;; ;; Small glossary: -;; * json data: a json-encoded string -;; * decode: convert a json string into a plain clojure nested map -;; * parse: build a TokensLib (or a fragment) from a decoded json data -;; * export: generate from a TokensLib a plain clojure nested map, suitable to be encoded as a json string - -(def ^:private legacy-node? - (sm/validator - [:or - [:map - ["value" :string] - ["type" :string]] - [:map - ["value" [:sequential [:map ["type" :string]]]] - ["type" :string]] - [:map - ["value" :map] - ["type" :string]]])) - -(def ^:private schema:dtcg-node - ;; Per the DTCG Community Group Final Report (W3C, 2025-10-28): - ;; "The presence of a $value property definitively identifies an object as a token." - ;; "A token's type can be specified by the optional $type property [...] - ;; Furthermore, the $type property on a group applies to all tokens nested - ;; within that group." - ;; So $value is the only required property; $type may be inherited from a - ;; parent group and is therefore optional on the token itself. - [:schema {:registry - {::simple-value - [:or :string :int :double ::sm/boolean] - ::value - [:or - [:ref ::simple-value] - [:vector ::simple-value] - [:vector [:map-of :string ::simple-value]] - [:map-of :string [:or - [:ref ::simple-value] - [:vector ::simple-value]]]]}} - [:map - ["$type" {:optional true} :string] - ["$value" [:ref ::value]]]]) - -(def ^:private dtcg-node? - (sm/validator schema:dtcg-node)) - -(defn- get-json-format - "Searches through decoded token file and returns: - - `:json-format/legacy` when first node satisfies `legacy-node?` predicate - - `:json-format/dtcg` when first node satisfies `dtcg-node?` predicate - - If neither combination is found, return dtcg format by default (we assume that - the file does not contain any token, so the format is irrelevan)." - ([decoded-json] - (get-json-format decoded-json legacy-node? dtcg-node?)) - ([decoded-json legacy-node? dtcg-node?] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (let [branch? map? - children (fn [node] (vals node)) - check-node (fn [node] - (cond - (legacy-node? node) :json-format/legacy - (dtcg-node? node) :json-format/dtcg - :else nil)) - walk (fn walk [node] - (lazy-seq - (cons - (check-node node) - (when (branch? node) - (mapcat walk (children node))))))] - (d/nilv (->> (walk decoded-json) - (filter some?) - first) - :json-format/dtcg)))) - -(defn- legacy-json->dtcg-json - "Converts a decoded json file in legacy format into DTCG format." - [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (walk/postwalk - (fn [node] - (cond-> node - ;; Handle sequential values that are objects with type - (and (map? node) - (contains? node "value") - (sequential? (get node "value")) - (map? (first (get node "value")))) - (update "value" - (fn [seq-value] - (map #(set/rename-keys % {"type" "$type"}) seq-value))) - - ;; Keep array of font families - (and (map? node) - (contains? node "type") - (= "fontFamilies" (get node "type")) - (contains? node "value") - (sequential? (get node "value")) - (not (map? (first (get node "value"))))) - identity - - ;; Rename keys for all token nodes - (and (map? node) - (and (contains? node "type") - (contains? node "value"))) - (set/rename-keys {"value" "$value" - "type" "$type"}))) - decoded-json)) - -(defn- single-set? - "Check if the decoded json file conforms to basic DTCG format with a single set." - [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (and (not (contains? decoded-json "$metadata")) - (not (contains? decoded-json "$themes")))) +;; * decoded json: a json file decoded into a plain map with string keys +;; * parse / import: build a TokensLib from decoded json +;; * export: build decoded json from a TokensLib (defn convert-dtcg-font-family "Convert font-family token value from DTCG format to internal format. @@ -1605,255 +1499,218 @@ ;; Reference value value)) -(defn convert-dtcg-shadow-composite - "Convert shadow token value from DTCG format to internal format." +(def ^:private dtcg-shadow-keys + {"offsetX" :offset-x + "offsetY" :offset-y + "blur" :blur + "spread" :spread + "color" :color + "inset" :inset}) + +(def ^:private dtcg-shadow-keys-inverted + (set/map-invert dtcg-shadow-keys)) + +(defn- shadow-with-dtcg-keys + [shadow] + (if (map? shadow) + (set/rename-keys shadow dtcg-shadow-keys-inverted) + shadow)) + +(defn- shadows-with-penpot-keys + "Shadows read by `tokenscript.document.dtcg/read-shadow` with penpot's + keys." [value] - (let [process-shadow (fn [shadow] - (if (map? shadow) - (let [legacy-shadow-type (get shadow "type")] - (-> shadow - (set/rename-keys {"x" :offset-x - "offsetX" :offset-x - "y" :offset-y - "offsetY" :offset-y - "blur" :blur - "spread" :spread - "color" :color - "inset" :inset}) - (update :inset #(cond - (boolean? %) % - (= "true" %) true - (= "false" %) false - (= legacy-shadow-type "innerShadow") true - :else false)) - (select-keys [:offset-x :offset-y :blur :spread :color :inset]))) - shadow))] - (cond - ;; Reference value - keep as string - (string? value) - value + (if (sequential? value) + (mapv (fn [shadow] + (if (map? shadow) + (-> (set/rename-keys shadow dtcg-shadow-keys) + (select-keys (vals dtcg-shadow-keys))) + shadow)) + value) + value)) - ;; Array of shadows - process each - (sequential? value) - (mapv process-shadow value) +(defn convert-dtcg-shadow-composite + "Convert shadow token value from DTCG format to internal format. The + value is read by `tokenscript.document.dtcg/read-shadow` first, which + takes single shadows and the legacy `x`, `y` and `innerShadow`. A value + already in the internal format, as plugins set it, is kept." + [value] + (shadows-with-penpot-keys + (tsd/read-shadow (if (sequential? value) + (mapv shadow-with-dtcg-keys value) + (shadow-with-dtcg-keys value))))) - ;; Single shadow object - wrap in vector - (map? value) - [(process-shadow value)] +(defn- file-value->token-value + "The token value of a value read from a file. The reader has already + read shadows with `read-shadow`, so only their keys change." + [token-type value] + (case token-type + :font-family (convert-dtcg-font-family value) + :typography (convert-dtcg-typography-composite value) + :shadow (shadows-with-penpot-keys value) + value)) - ;; Fallback - keep as is - :else - value))) +(defn- document-token + "A penpot token from the token data of a document, or nil when penpot + has no type for it." + [token-name {:keys [type value description]}] + (when-let [token-type (cto/dtcg-token-type->token-type type)] + (make-token :name token-name + :type token-type + :value (file-value->token-value token-type value) + :description description))) -(defn- flatten-nested-tokens-json - "Convert a tokens tree in the decoded json fragment into a flat map, - being the keys the token paths after joining the keys with '.'. +(defn- document-token-path + [{:keys [document]} set-name token-name] + (if (= :multi-set document) + (str set-name "." token-name) + token-name)) - Per the DTCG spec, a node is a token iff it carries a `$value` - property; every other map is a group. A group may declare a `$type` - that is inherited by every nested token that does not declare its - own `$type`, so this walker propagates the nearest enclosing group - `$type` down through the recursion." - ([decoded-json-tokens parent-path] - (flatten-nested-tokens-json decoded-json-tokens parent-path nil)) - ([decoded-json-tokens parent-path inherited-type] - (reduce-kv - (fn [tokens k v] - (let [child-path (if (empty? parent-path) - (name k) - (str parent-path "." k))] - (if (and (map? v) - (not (contains? v "$value"))) - ;; Group: recurse, letting the group's `$type` (if any) replace - ;; the inherited type for descendants that don't declare their - ;; own `$type`. - (merge tokens - (flatten-nested-tokens-json - v - child-path - (or (get v "$type") inherited-type))) - ;; Token: resolve the type from the token's own `$type`, - ;; falling back to the inherited group `$type`. - (let [type-key (or (get v "$type") inherited-type) - token-type (cto/dtcg-token-type->token-type type-key)] - (if token-type - (assoc tokens child-path (make-token - :name child-path - :type token-type - :value - (let [token-value (get v "$value")] - (case token-type - :font-family (convert-dtcg-font-family token-value) - :typography (convert-dtcg-typography-composite token-value) - :shadow (convert-dtcg-shadow-composite token-value) - token-value)) - :description (get v "$description"))) - ;; Discard unknown / un-typeable tokens - tokens))))) - {} - decoded-json-tokens))) +(defn- dropped-properties + "The properties of a token or group that penpot does not store; token + descriptions are stored, group descriptions are not. The reader only + keeps a token's `$extensions` when it is an object, so a malformed one + goes unreported on a token, unlike on a group." + [{:keys [description deprecated extensions properties]} group?] + (cond-> (vec (keys properties)) + (and group? (some? description)) (conj "$description") + (some? deprecated) (conj "$deprecated") + (d/not-empty? extensions) (conj "$extensions"))) -(defn- parse-single-set-dtcg-json - "Parse a decoded json file with a single set of tokens in DTCG format into a TokensLib." - [set-name decoded-json-tokens] - (assert (map? decoded-json-tokens) "expected a plain clojure map for `decoded-json-tokens`") - (assert (= (get-json-format decoded-json-tokens) :json-format/dtcg) "expected a dtcg format for `decoded-json-tokens`") +(defn- import-warnings + "What penpot drops from a document: `:unknown-types` maps the tokens of + types penpot does not have to their type, `:unsupported` lists the + properties penpot does not store (`$extensions`, `$deprecated`, group + descriptions, ...) and the tokens the reader dropped, as + `{:property :path}`." + [{:keys [sets issues] :as document}] + (let [path (partial document-token-path document)] + {:unknown-types + (not-empty + (into {} + (for [{set-name :name :keys [tokens]} sets + [token-name {:keys [type]}] tokens + :when (nil? (cto/dtcg-token-type->token-type type))] + [(path set-name token-name) type]))) - (let [set-name (normalize-set-name set-name) - tokens (flatten-nested-tokens-json decoded-json-tokens "")] + :unsupported + (not-empty + (vec (concat + (for [{set-name :name :keys [tokens groups]} sets + [node-name data group?] (concat (map (fn [[k v]] [k v false]) tokens) + (map (fn [[k v]] [k v true]) groups)) + property (dropped-properties data group?)] + {:property property :path (path set-name node-name)}) + (for [{:keys [code data] :as issue} issues + :let [[property node-name] + (case code + (:dtcg/invalid-value :dtcg/invalid-node) ["$value" (:token-name data)] + :dtcg/missing-type ["$type" (:token-name data)] + :dtcg/invalid-name ["$root" (:token-name data)] + (:dtcg/invalid-extends :dtcg/circular-extends) ["$extends" (:group data)] + nil)] + :when property] + {:property property :path (path (:set issue) node-name)}))))})) - (when (empty? tokens) +(defn- activate-theme-path + "Activates the theme at `theme-path` in `library`, deactivating the + other themes of its group. The hidden theme, which holds the active + sets of a file without themes, stays active." + [library theme-path] + (let [[group name] (split-theme-path theme-path)] + (if-let [theme (get-theme-by-name library group name)] + (let [group-paths (into #{} + (comp (filter #(= group (:group %))) + (remove hidden-theme?) + (map get-theme-path)) + (get-themes library))] + (TokensLib. (.-sets ^TokensLib library) + (.-themes ^TokensLib library) + (-> (set/difference (.-active-themes ^TokensLib library) group-paths) + (conj (get-theme-path theme))))) + library))) + +(defn- document->tokens-lib + "Builds a TokensLib from a document read by + `tokenscript.document.read/read`. Sets a theme marks `:disabled` are + not part of the theme." + [{doc-sets :sets :keys [document themes active-themes active-sets]}] + (let [xf-normalize-set-name (map normalize-set-name) + + sets + (reduce (fn [sets {:keys [name tokens]}] + (assoc sets + (normalize-set-name name) + (into {} + (keep (fn [[token-name data]] + (when-let [token (document-token token-name data)] + [token-name token]))) + tokens))) + (d/ordered-map) + doc-sets) + + themes + (->> themes + (map (fn [{:keys [id name group description is-source] theme-sets :sets}] + (make-token-theme + :id (or (uuid/parse* id) (uuid/next)) + :name name + :group group + :description description + :is-source is-source + :external-id id + :sets (into #{} + (comp (remove (comp #{:disabled} val)) + (map key) + xf-normalize-set-name + (filter #(contains? sets %))) + theme-sets)))) + (not-empty)) + + active-set-names + (into #{} xf-normalize-set-name active-sets) + + active-theme-names + (or (not-empty (vec (distinct active-themes))) + [hidden-theme-path])] + + (when (if (= :single-set document) + (every? empty? (vals sets)) + (and (empty? sets) (empty? themes))) (throw (ex-info "the file doesn't contain any tokens" {:error/code :error.import/invalid-json-data}))) - (-> (make-tokens-lib) - (add-set (make-token-set :name set-name - :tokens tokens))))) + (as-> (make-tokens-lib) library + (reduce-kv (fn [library name tokens] + (add-set library (make-token-set :name name :tokens tokens))) + library + sets) + (update-theme library hidden-theme-id #(assoc % :sets active-set-names)) + (reduce add-theme library themes) + (reduce activate-theme-path library active-theme-names)))) -(defn- parse-single-set-legacy-json - "Parse a decoded json file with a single set of tokens in legacy format into a TokensLib." - [set-name decoded-json-tokens] - (assert (map? decoded-json-tokens) "expected a plain clojure map for `decoded-json-tokens`") - (assert (= (get-json-format decoded-json-tokens) :json-format/legacy) "expected a legacy format for `decoded-json-tokens`") - (parse-single-set-dtcg-json set-name (legacy-json->dtcg-json decoded-json-tokens))) +(defn import-decoded-json + "Reads a decoded token file into a TokensLib, guessing its format. The + `file-name` is the set name when the file contains a single set. + Returns `{:tokens-lib lib :unknown-types .. :unsupported ..}`, see + `import-warnings`." + [decoded-json file-name] + (let [document (tsr/read decoded-json {:set-name file-name})] + (assoc (import-warnings document) + :tokens-lib (document->tokens-lib document)))) -(def ^:private schema:multi-set-dtcg - "Schema for penpot multi-set dtcg json decoded data/ - - Mainly used for validate the structure of the incoming data before - proceed to parse it to our internal data structures." - [:schema {:registry - {::node - [:or - [:map-of :string [:ref ::node]] - schema:dtcg-node]}} - [:map - ["$themes" {:optional true} - [:vector - [:map {:title "Theme"} - ["id" {:optional true} :string] - ["name" :string] - ["description" :string] - ["isSource" {:optional true} :boolean] - ["selectedTokenSets" - [:map-of :string [:enum "enabled" "disabled"]]]]]] - ["$metadata" {:optional true} - [:map {:title "Metadata"} - ["tokenSetOrder" {:optional true} [:vector :string]] - ["activeThemes" {:optional true} [:vector :string]] - ["activeSets" {:optional true} [:vector :string]]]] - - [:malli.core/default - [:map-of :string [:ref ::node]]]]]) - -(def ^:private check-multi-set-dtcg-data - (sm/check-fn schema:multi-set-dtcg)) - -(def ^:private decode-multi-set-dtcg-data - (sm/decoder schema:multi-set-dtcg - sm/json-transformer)) +;; FIXME: remove `-json` suffix +(defn parse-decoded-json + "Like `import-decoded-json`, the TokensLib only." + [decoded-json file-name] + (:tokens-lib (import-decoded-json decoded-json file-name))) ;; FIXME: remove `-json` suffix (defn parse-multi-set-dtcg-json - "Parse a decoded json file with multi sets in DTCG format into a TokensLib." + "Parse a decoded json file with multi sets into a TokensLib, also when + it has no `$themes` and no `$metadata`." [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (assert (= (get-json-format decoded-json) :json-format/dtcg) "expected a dtcg format for `decoded-json`") - - (let [metadata (get decoded-json "$metadata") - - xf-normalize-set-name - (map normalize-set-name) - - sets - (dissoc decoded-json "$themes" "$metadata") - - ordered-set-names - (-> (d/ordered-set) - (into xf-normalize-set-name (get metadata "tokenSetOrder")) - (into xf-normalize-set-name (keys sets))) - - active-set-names - (or (->> (get metadata "activeSets") - (into #{} xf-normalize-set-name) - (not-empty)) - #{}) - - active-theme-names - (or (->> (get metadata "activeThemes") - (into #{}) - (not-empty)) - #{hidden-theme-path}) - - themes - (->> (get decoded-json "$themes") - (map (fn [theme] - (make-token-theme - :id (or (uuid/parse* (get theme "id")) - (uuid/next)) - :name (get theme "name") - :group (get theme "group") - :is-source (or (get theme "isSource") - ;; NOTE: backward compatibility - (get theme "is-source")) - :external-id (get theme "id") - :sets (into #{} - (comp (map key) - xf-normalize-set-name - (filter #(contains? ordered-set-names %))) - (get theme "selectedTokenSets"))))) - (not-empty)) - - library - (make-tokens-lib) - - sets - (reduce-kv (fn [result name tokens] - (assoc result - (normalize-set-name name) - (flatten-nested-tokens-json tokens ""))) - {} - sets) - - library - (reduce (fn [library name] - (if-let [tokens (get sets name)] - (add-set library (make-token-set :name name :tokens tokens)) - library)) - library - ordered-set-names) - - library - (update-theme library hidden-theme-id - #(assoc % :sets active-set-names)) - - library - (reduce add-theme library themes) - - ;; Activate themes by directly manipulating the internal active-themes field - ;; (since activate-theme is being removed from the protocol) - library - (reduce (fn [library theme-path] - (let [[group name] (split-theme-path theme-path) - theme (get-theme-by-name library group name)] - (if theme - (let [group-themes (->> (get themes group) - (map (comp get-theme-path val)) - (into #{})) - active-themes' (-> (set/difference (.-active-themes ^TokensLib library) group-themes) - (conj (get-theme-path theme)))] - (TokensLib. (.-sets ^TokensLib library) - (.-themes ^TokensLib library) - active-themes')) - library))) - library - active-theme-names)] - - (when (and (empty? sets) (empty? themes)) - (throw (ex-info "the file doesn't contain any tokens" - {:error/code :error.import/invalid-json-data}))) - - library)) + (document->tokens-lib (tsr/read decoded-json {:multi-set? true}))) (defn read-multi-set-dtcg "Read penpot multi-set dctg tokens. Accepts string or JSON decoded @@ -1868,45 +1725,8 @@ data #?(:cljs (if (object? data) (json/->clj data :key-fn identity) data) - :clj data) - - data (decode-multi-set-dtcg-data data)] - (-> (check-multi-set-dtcg-data data) - (parse-multi-set-dtcg-json))))) - -(defn- parse-multi-set-legacy-json - "Parse a decoded json file with multi sets in legacy format into a TokensLib." - [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (assert (= (get-json-format decoded-json) :json-format/legacy) "expected a legacy format for `decoded-json`") - - (let [sets-data (dissoc decoded-json "$themes" "$metadata") - other-data (select-keys decoded-json ["$themes" "$metadata"]) - dtcg-sets-data (legacy-json->dtcg-json sets-data)] - (parse-multi-set-dtcg-json (merge other-data - dtcg-sets-data)))) - -;; FIXME: remove `-json` suffix -(defn parse-decoded-json - "Guess the format and content type of the decoded json file and parse it into a TokensLib. - The `file-name` is used to determine the set name when the json file contains a single set." - [decoded-json file-name] - (let [single-set? (single-set? decoded-json) - json-format (get-json-format decoded-json)] - (cond - (and single-set? - (= :json-format/legacy json-format)) - (parse-single-set-legacy-json file-name decoded-json) - - (and single-set? - (= :json-format/dtcg json-format)) - (parse-single-set-dtcg-json file-name decoded-json) - - (= :json-format/legacy json-format) - (parse-multi-set-legacy-json decoded-json) - - :else - (parse-multi-set-dtcg-json decoded-json)))) + :clj data)] + (parse-multi-set-dtcg-json data)))) (defn- typography-token->dtcg-token [value] @@ -1926,234 +1746,98 @@ (mapv (fn [shadow] (if (map? shadow) (-> shadow - (set/rename-keys {:offset-x "offsetX" - :offset-y "offsetY" - :blur "blur" - :spread "spread" - :color "color" - :inset "inset"}) - (select-keys ["offsetX" "offsetY" "blur" "spread" "color" "inset"])) + (set/rename-keys dtcg-shadow-keys-inverted) + (select-keys (keys dtcg-shadow-keys))) shadow)) value) value)) -(defn- token->dtcg-token [token] - (cond-> {"$value" (cond-> (:value token) - ;; Transform typography token values - (= :typography (:type token)) - typography-token->dtcg-token - ;; Transform shadow token values - (= :shadow (:type token)) - shadow-token->dtcg-token) - "$type" (cto/token-type->dtcg-token-type (:type token))} - (:description token) (assoc "$description" (:description token)))) +(defn- token->document-token + [token] + {:value (cond-> (:value token) + (= :typography (:type token)) typography-token->dtcg-token + (= :shadow (:type token)) shadow-token->dtcg-token) + :type (cto/token-type->dtcg-token-type (:type token)) + :description (:description token)}) -(defn- dtcg-export-themes - "Extract themes for a dtcg json export." - ([tokens-lib] - ;; Backward-compatible version for print/serialization without tokens-status - (let [themes-xform - (comp - (filter #(and (instance? TokenTheme %) - (not (hidden-theme? %)))) - (map (fn [token-theme] - ;; NOTE: this probaly can be implemented as type method - (d/without-nils - {"id" (:external-id token-theme) - "name" (:name token-theme) - "group" (:group token-theme) - "description" (:description token-theme) - "isSource" (:is-source token-theme) - "selectedTokenSets" (reduce #(assoc %1 %2 "enabled") {} (:sets token-theme))})))) +(defn- token-theme->document-theme + [token-theme] + (d/without-nils + {:id (:external-id token-theme) + :name (:name token-theme) + :group (:group token-theme) + :description (:description token-theme) + :is-source (:is-source token-theme) + :sets (zipmap (:sets token-theme) (repeat :enabled))})) - themes - (->> (get-theme-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (into [] themes-xform)) +(defn- legacy-activation + "Active theme paths and set names from the lib's own active themes, + for printing and serialization without a TokensStatus." + [tokens-lib] + (let [active-themes (.-active-themes ^TokensLib tokens-lib)] + [(disj active-themes hidden-theme-path) + (into #{} + (comp (filter #(contains? active-themes (get-theme-path %))) + (mapcat :sets)) + (get-themes tokens-lib))])) - ;; Active themes without exposing hidden penpot theme - active-themes - (-> (.-active-themes ^TokensLib tokens-lib) - (disj hidden-theme-path))] - [themes active-themes])) - ([tokens-lib active-theme-ids] - ;; Version with explicit active-theme-ids from TokensStatus - (let [themes-xform - (comp - (filter #(and (instance? TokenTheme %) - (not (hidden-theme? %)))) - (map (fn [token-theme] - (d/without-nils - {"id" (:external-id token-theme) - "name" (:name token-theme) - "group" (:group token-theme) - "description" (:description token-theme) - "isSource" (:is-source token-theme) - "selectedTokenSets" (reduce #(assoc %1 %2 "enabled") {} (:sets token-theme))})))) +(defn- status-activation + "Active theme paths and set names from a TokensStatus." + [tokens-lib tokens-status] + (let [active-set-ids (ctos/get-active-set-ids tokens-status)] + [(into #{} + (comp (map #(get-theme tokens-lib %)) + (filter some?) + (map get-theme-path) + (remove #(= % hidden-theme-path))) + (ctos/get-active-theme-ids tokens-status)) + (into #{} + (comp (filter #(contains? active-set-ids (get-id %))) + (map get-name)) + (get-sets tokens-lib))])) - themes - (->> (get-theme-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (into [] themes-xform)) - - ;; Convert theme ids to theme paths, excluding hidden theme - active-themes - (into #{} - (comp (map #(get-theme tokens-lib %)) - (filter some?) - (map get-theme-path) - (filter #(not= % hidden-theme-path))) - active-theme-ids)] - [themes active-themes]))) +(defn- tokens-lib->document + [tokens-lib [active-themes active-sets]] + {:sets (mapv (fn [token-set] + {:name (get-name token-set) + :tokens (d/update-vals (get-tokens- token-set) token->document-token)}) + (get-sets tokens-lib)) + :themes (into [] + (comp (filter #(and (instance? TokenTheme %) (not (hidden-theme? %)))) + (map token-theme->document-theme)) + (tree-seq d/ordered-map? vals (get-theme-tree tokens-lib))) + :active-themes active-themes + :active-sets active-sets}) (defn export-dtcg-multi-file - "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi json files each encoded in DTCG format." + "Convert a TokensLib into a plain clojure map, suitable to be encoded + as multiple json files, one per set plus `$themes.json` and + `$metadata.json`. Options: `:profile`, `:default` or `:dtcg` for + DTCG 2025.10 values and types, see `tokenscript.document.write/write`. + A `:dtcg` file is not meant to import back unchanged: tokens come back + with their DTCG type and value, a spacing token as a dimension, `4` as + `4px` and `rgb(255, 0, 0)` as `#ff0000`." ([tokens-lib] - ;; Backward-compatible version for print/serialization without tokens-status - (let [[themes active-themes] - (dtcg-export-themes tokens-lib) - - sets - (->> (get-sets tokens-lib) - (map (fn [token-set] - (let [name (get-name token-set) - tokens (get-tokens- token-set)] - [(str name ".json") (tokens-tree tokens :update-token-fn token->dtcg-token)]))) - (into {}))] - - (-> sets - (assoc "$themes.json" themes) - (assoc "$metadata.json" - {"tokenSetOrder" (get-set-names tokens-lib) - "activeThemes" active-themes - ;; For backward compat, use internal active-themes field - "activeSets" (into #{} - (comp (filter #(and (instance? TokenTheme %) - (contains? (.-active-themes ^TokensLib tokens-lib) - (get-theme-path %)))) - (mapcat :sets)) - (get-themes tokens-lib))})))) + (tsw/write-files (tokens-lib->document tokens-lib (legacy-activation tokens-lib)))) ([tokens-lib tokens-status] - ;; Version with tokens-status for real exports - (let [active-theme-ids (ctos/get-active-theme-ids tokens-status) - [themes active-themes] - (dtcg-export-themes tokens-lib active-theme-ids) - - sets - (->> (get-sets tokens-lib) - (map (fn [token-set] - (let [name (get-name token-set) - tokens (get-tokens- token-set)] - [(str name ".json") (tokens-tree tokens :update-token-fn token->dtcg-token)]))) - (into {})) - - active-set-ids (ctos/get-active-set-ids tokens-status) - active-set-names (into #{} - (comp (filter #(contains? active-set-ids (get-id %))) - (map get-name)) - (get-sets tokens-lib))] - - (-> sets - (assoc "$themes.json" themes) - (assoc "$metadata.json" - {"tokenSetOrder" (get-set-names tokens-lib) - "activeThemes" active-themes - "activeSets" active-set-names}))))) + (export-dtcg-multi-file tokens-lib tokens-status nil)) + ([tokens-lib tokens-status opts] + (tsw/write-files (tokens-lib->document tokens-lib (status-activation tokens-lib tokens-status)) + opts))) (defn export-dtcg-json - "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi sets json string in DTCG format." + "Convert a TokensLib into a plain clojure map, suitable to be encoded + as a multi sets json string, or nil for an empty lib. Options as in + `export-dtcg-multi-file`." ([tokens-lib] - ;; Backward-compatible version for print/serialization without tokens-status - (let [[themes active-themes] - (dtcg-export-themes tokens-lib) - - name-set-tuples - (->> (get-set-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (filter (partial instance? TokenSet)) - (map (fn [set] - [(get-name set) - (tokens-tree (get-tokens- set) :update-token-fn token->dtcg-token)]))) - - ordered-set-names - (mapv first name-set-tuples) - - sets - (into {} name-set-tuples) - - ;; For backward compat, use internal active-themes field - active-set-names (into #{} - (comp (filter #(and (instance? TokenTheme %) - (contains? (.-active-themes ^TokensLib tokens-lib) - (get-theme-path %)))) - (mapcat :sets)) - (get-themes tokens-lib))] - - (when-not (empty-lib? tokens-lib) - (-> sets - (assoc "$themes" themes) - (assoc "$metadata" {"tokenSetOrder" ordered-set-names - "activeThemes" active-themes - "activeSets" active-set-names}))))) + (when-not (empty-lib? tokens-lib) + (tsw/write (tokens-lib->document tokens-lib (legacy-activation tokens-lib))))) ([tokens-lib tokens-status] - ;; Version with tokens-status for real exports - (let [active-theme-ids (ctos/get-active-theme-ids tokens-status) - [themes active-themes] - (dtcg-export-themes tokens-lib active-theme-ids) - - name-set-tuples - (->> (get-set-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (filter (partial instance? TokenSet)) - (map (fn [set] - [(get-name set) - (tokens-tree (get-tokens- set) :update-token-fn token->dtcg-token)]))) - - ordered-set-names - (mapv first name-set-tuples) - - sets - (into {} name-set-tuples) - - active-set-ids (ctos/get-active-set-ids tokens-status) - active-set-names (into #{} - (comp (filter #(contains? active-set-ids (get-id %))) - (map get-name)) - (get-sets tokens-lib))] - - (when-not (empty-lib? tokens-lib) - (-> sets - (assoc "$themes" themes) - (assoc "$metadata" {"tokenSetOrder" ordered-set-names - "activeThemes" active-themes - "activeSets" active-set-names})))))) - -(defn get-tokens-of-unknown-type - "Search for all tokens in the decoded json file that have a type that is not currently - supported by Penpot. Returns a map token-path -> token type." - [decoded-json {:keys [json-format parent-path process-token-type] - :or {json-format (get-json-format decoded-json) - parent-path "" - process-token-type identity} - :as opts}] - (let [type-key (if (= json-format :json-format/dtcg) "$type" "type")] - (reduce-kv - (fn [unknown-tokens k v] - (let [child-path (if (empty? parent-path) - (name k) - (str parent-path "." k))] - (if (and (map? v) - (not (contains? v type-key))) - (let [nested-unknown-tokens (get-tokens-of-unknown-type v (assoc opts :parent-path child-path))] - (merge unknown-tokens nested-unknown-tokens)) - (let [token-type-str (get v type-key) - token-type (-> (cto/dtcg-token-type->token-type token-type-str) - (process-token-type))] - (if (and (not (some? token-type)) (some? token-type-str)) - (assoc unknown-tokens child-path token-type-str) - unknown-tokens))))) - nil - decoded-json))) + (export-dtcg-json tokens-lib tokens-status nil)) + ([tokens-lib tokens-status opts] + (when-not (empty-lib? tokens-lib) + (tsw/write (tokens-lib->document tokens-lib (status-activation tokens-lib tokens-status)) + opts)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; SERIALIZATION (TRANSIT) diff --git a/common/src/app/common/types/variant.cljc b/common/src/app/common/types/variant.cljc index f91075a170..0469c3c674 100644 --- a/common/src/app/common/types/variant.cljc +++ b/common/src/app/common/types/variant.cljc @@ -315,6 +315,16 @@ props2) :props))) +(defn group-properties-by-name + "Group properties by :name, preserving first-seen name order. + clojure.core/group-by loses order once maps grow past ArrayMap's + 8-key threshold." + [props] + (reduce (fn [m p] + (update m (:name p) (fnil conj []) p)) + (d/ordered-map) + props)) + (defn compare-properties "Compares vectors of properties keeping the value if it is the same for all or setting a custom value where their values do not coincide" @@ -322,7 +332,7 @@ (compare-properties props-list nil)) ([props-list distinct-mark] - (let [grouped (group-by :name (apply concat props-list)) + (let [grouped (group-properties-by-name (apply concat props-list)) check-values (fn [values] (let [vals (map :value values)] (if (apply = vals) diff --git a/common/test/common_tests/attrs_test.cljc b/common/test/common_tests/attrs_test.cljc index 340a89e0da..8d06d1126b 100644 --- a/common/test/common_tests/attrs_test.cljc +++ b/common/test/common_tests/attrs_test.cljc @@ -148,4 +148,26 @@ {:x -153}] result (attrs/get-attrs-multi objs [:x])] (t/is (= {:x :multiple} result) - "Different positions should be :multiple")))) \ No newline at end of file + "Different positions should be :multiple")))) +(defn- text-shape-with-paragraphs + [& paragraphs] + {:content {:type "root" + :children [{:type "paragraph-set" + :children (mapv #(merge {:type "paragraph" + :children [{:text "item"}]} + %) + paragraphs)}]}}) + +(t/deftest get-text-attrs-multi-list-style-type + (t/testing "paragraphs with the same list style return it" + (let [shape (text-shape-with-paragraphs {:list-style-type "disc"} + {:list-style-type "disc"})] + (t/is (= "disc" (:list-style-type (attrs/get-text-attrs-multi shape {} [:list-style-type])))))) + + (t/testing "a paragraph without list style counts as no list" + (let [shape (text-shape-with-paragraphs {})] + (t/is (= "none" (:list-style-type (attrs/get-text-attrs-multi shape {} [:list-style-type])))))) + + (t/testing "a bullet paragraph next to one without list style is mixed" + (let [shape (text-shape-with-paragraphs {:list-style-type "disc"} {})] + (t/is (= :multiple (:list-style-type (attrs/get-text-attrs-multi shape {} [:list-style-type]))))))) diff --git a/common/test/common_tests/files/shapes_builder_test.cljc b/common/test/common_tests/files/shapes_builder_test.cljc index 0c9418ffde..8b7270c14f 100644 --- a/common/test/common_tests/files/shapes_builder_test.cljc +++ b/common/test/common_tests/files/shapes_builder_test.cljc @@ -324,3 +324,20 @@ (t/is (some? rect)) (t/is (= (clr/parse "#0000ff") (:stroke-color stroke)) "child style beats group attr") (t/is (nil? (get-in rect [:svg-attrs :stroke])) "group attr is not inherited over child style"))) + +;; Regression for https://github.com/penpot/penpot/issues/12773: the SVG +;; root shape name must have any extension stripped (not only ".svg"), +;; so the component name and the main instance root name match. +(t/deftest create-svg-shapes-strips-name-extension + (let [svg-data {:name "my.svg.icon.SVG" + :tag :svg + :attrs {:xmlns "http://www.w3.org/2000/svg" + :width "24" :height "24" + :viewBox "0 0 24 24"} + :content [{:tag :rect + :attrs {:x "0" :y "0" :width "24" :height "24"} + :content []}]} + [root _children] (sb/create-svg-shapes svg-data {:x 0 :y 0} {} nil nil #{} false)] + + (t/is (= "my.svg.icon" (:name root))))) + diff --git a/common/test/common_tests/files/tokens_test.cljc b/common/test/common_tests/files/tokens_test.cljc index 8d6a28b393..540a7b9625 100644 --- a/common/test/common_tests/files/tokens_test.cljc +++ b/common/test/common_tests/files/tokens_test.cljc @@ -7,6 +7,7 @@ (ns common-tests.files.tokens-test (:require [app.common.files.tokens :as cfo] + [app.common.schema :as sm] [app.common.test-helpers.files :as thf] [app.common.test-helpers.ids-map :as thi] [app.common.test-helpers.tokens :as tht] @@ -1709,3 +1710,42 @@ (t/is (not (contains? (:data detached) :tokens-source))) (t/is (not (contains? (:data detached) :tokens-status))) (t/is (= detached (ctf/check-file detached))))) + +;; Token name schema + +(t/deftest test-make-token-name-schema + (t/testing "rejects a name that already exists as a token" + (let [schema (cfo/make-token-name-schema {"foo" {:name "foo"}})] + (t/is (false? (sm/validate schema "foo"))))) + + (t/testing "rejects a token inside an existing token" + (let [schema (cfo/make-token-name-schema {"foo" {:name "foo"}})] + (t/is (false? (sm/validate schema "foo.bar"))))) + + (t/testing "rejects a token name that collides with a group" + (let [schema (cfo/make-token-name-schema {"foo" {"child" {:name "foo.child"}}})] + (t/is (false? (sm/validate schema "foo"))))) + + (t/testing "rejects a name that exists as a child token" + (let [schema (cfo/make-token-name-schema {"foo" {"bar" {:name "foo.bar"}}})] + (t/is (false? (sm/validate schema "foo.bar"))))) + + (t/testing "accepts a valid sibling name" + (let [schema (cfo/make-token-name-schema {"foo" {:name "foo"}})] + (t/is (true? (sm/validate schema "bar"))) + (t/is (true? (sm/validate schema "bar.baz"))))) + + (t/testing "rejects every name when there is no tokens tree" + (let [schema (cfo/make-token-name-schema nil)] + (t/is (false? (sm/validate schema "foo"))))) + + (t/testing "rejects names that do not match the token name format" + (let [schema (cfo/make-token-name-schema {})] + (t/is (false? (sm/validate schema ""))) + (t/is (false? (sm/validate schema "$foo"))) + (t/is (false? (sm/validate schema "foo."))) + (t/is (false? (sm/validate schema "foo..bar"))))) + + (t/testing "rejects names longer than 255 characters" + (let [schema (cfo/make-token-name-schema {})] + (t/is (false? (sm/validate schema (apply str (repeat 256 "a")))))))) diff --git a/common/test/common_tests/files/variant_test.cljc b/common/test/common_tests/files/variant_test.cljc new file mode 100644 index 0000000000..fda97d565e --- /dev/null +++ b/common/test/common_tests/files/variant_test.cljc @@ -0,0 +1,49 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files.variant-test + (:require + [app.common.files.variant :as cfv] + [app.common.test-helpers.components :as thc] + [app.common.test-helpers.files :as thf] + [app.common.test-helpers.ids-map :as thi] + [app.common.test-helpers.shapes :as ths] + [app.common.test-helpers.variants :as thv] + [clojure.test :as t])) + +(t/use-fixtures :each thi/test-fixture) + +(t/deftest test-collect-variant-properties-for-main + (let [file (-> (thf/sample-file :file1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02)) + page (thf/current-page file) + main (ths/get-shape file :m01) + comp (thc/get-component file :c01) + collected (cfv/collect-variant-properties (:data file) (:objects page) main)] + + (t/is (= {(:id comp) (:variant-properties comp)} collected)))) + +(t/deftest test-collect-variant-properties-for-container + (let [file (-> (thf/sample-file :file1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02)) + page (thf/current-page file) + container (ths/get-shape file :v01) + c01 (thc/get-component file :c01) + c02 (thc/get-component file :c02) + collected (cfv/collect-variant-properties (:data file) (:objects page) container)] + + (t/is (= {(:id c01) (:variant-properties c01) + (:id c02) (:variant-properties c02)} + collected)))) + +(t/deftest test-collect-variant-properties-for-plain-shape + (let [file (-> (thf/sample-file :file1) + (ths/add-sample-shape :rect1 :type :rect)) + page (thf/current-page file) + shape (ths/get-shape file :rect1) + collected (cfv/collect-variant-properties (:data file) (:objects page) shape)] + + (t/is (= {} collected)))) diff --git a/common/test/common_tests/files_builder_test.cljc b/common/test/common_tests/files_builder_test.cljc index db0242ea93..9658f51b28 100644 --- a/common/test/common_tests/files_builder_test.cljc +++ b/common/test/common_tests/files_builder_test.cljc @@ -7,6 +7,8 @@ (ns common-tests.files-builder-test (:require [app.common.files.builder :as fb] + [app.common.files.validate :as cfv] + [app.common.path-names :as cpn] [app.common.uuid :as uuid] [clojure.test :as t])) @@ -120,3 +122,30 @@ :height 10})) shape (fb/get-shape state shape-id)] (t/is (= "Group / Subgroup / Name" (:name shape))))) + +(t/deftest add-component-renames-frame-to-match-component + (t/testing "the main instance frame name is aligned with the component's path+name" + (let [file-id (uuid/next) + page-id (uuid/next) + frame-id (uuid/next) + component-id (uuid/next) + state (-> (fb/create-state) + (fb/add-file {:id file-id :name "Test file"}) + (fb/add-page {:id page-id :name "Page 1"}) + (fb/add-board {:id frame-id + :name "Wrong / Frame / Name" + :x 0 + :y 0 + :width 10 + :height 10}) + (fb/add-component {:component-id component-id + :name "Component" + :path "Group / Subgroup"})) + shape (fb/get-shape state frame-id) + file-data (get-in state [::fb/files file-id :data]) + component (get-in file-data [:components component-id]) + errors (cfv/validate-file {:id file-id :data file-data} {})] + + (t/is (= "Group / Subgroup / Component" (:name shape))) + (t/is (= (:name shape) (cpn/merge-path-item (:path component) (:name component)))) + (t/is (nil? errors))))) diff --git a/common/test/common_tests/files_migrations_0033_test.cljc b/common/test/common_tests/files_migrations_0033_test.cljc new file mode 100644 index 0000000000..6f56602b77 --- /dev/null +++ b/common/test/common_tests/files_migrations_0033_test.cljc @@ -0,0 +1,138 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0033-test + "Regression coverage for the shape tree traversal of the + `0033-normalize-main-instance-names` migration. + + The migration walks a container from its root down, following the + `:shapes` vectors. A file can hold a `:shapes` entry pointing to a + shape that is not in the `:objects` map (a dangling reference). The + traversal must skip it instead of storing a nil shape into the + `:objects` map, and must keep normalizing the names of the shapes + that do exist." + (:require + [app.common.files.migrations :as cfm] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +(def ^:private migration "0033-normalize-main-instance-names") + +(defn- make-root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- make-main-instance + "A frame declared as the main instance of a component." + [main-id file-id component-id name] + (-> (cts/setup-shape {:id main-id :type :frame :name name}) + (assoc :main-instance true + :component-id component-id + :component-file file-id))) + +(defn- page-data + "File data with a single page owning `objects`." + [page-id objects] + {:pages-index {page-id {:id page-id :objects objects}}}) + +(defn- component-data + "File data with a single component owning `objects`." + [component-id objects] + {:components {component-id {:id component-id :objects objects}}}) + +;; --- The guard: a missing child must not become a nil shape + +(t/deftest missing-child-of-a-page-does-not-create-a-nil-shape + (let [page-id (uuid/next) + child-id (uuid/next) + root (make-root [child-id]) + data (page-data page-id {uuid/zero root}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= 1 (count objects')) + "the objects map keeps only the shapes that exist") + (t/is (= root (get objects' uuid/zero)) + "the root is left untouched"))) + +(t/deftest missing-child-of-a-component-does-not-create-a-nil-shape + (let [component-id (uuid/next) + child-id (uuid/next) + root (make-root [child-id]) + data (component-data component-id {uuid/zero root}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= 1 (count objects')) + "the objects map keeps only the shapes that exist") + (t/is (= root (get objects' uuid/zero)) + "the component root is left untouched"))) + +(t/deftest missing-child-deep-in-the-tree-does-not-create-a-nil-shape + (let [page-id (uuid/next) + frame-id (uuid/next) + missing-id (uuid/next) + frame (-> (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + (assoc :shapes [missing-id])) + data (page-data page-id {uuid/zero (make-root [frame-id]) + frame-id frame}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= frame (get objects' frame-id)) + "the parent of the missing child is left untouched"))) + +;; --- The behavior: names keep being normalized next to a missing child + +(t/deftest main-instance-of-a-page-is-renamed-next-to-a-missing-child + (let [file-id (uuid/next) + page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + missing-id (uuid/next) + main-shape (make-main-instance main-id file-id component-id "Group/Name") + data (assoc (page-data page-id {uuid/zero (make-root [missing-id main-id]) + main-id main-shape}) + :components {component-id {:id component-id + :name "Name" + :path "Group" + :main-instance-id main-id}}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= "Group / Name" (get-in objects' [main-id :name])) + "the main instance takes the path and name of its component"))) + +(t/deftest main-instance-of-a-component-is-renamed-next-to-a-missing-child + (let [file-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + missing-id (uuid/next) + main-shape (make-main-instance main-id file-id component-id "Group/Name") + data (component-data component-id {uuid/zero (make-root [missing-id main-id]) + main-id main-shape}) + data (assoc-in data [:components component-id :name] "Name") + data (assoc-in data [:components component-id :path] "Group") + data (assoc-in data [:components component-id :main-instance-id] main-id) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= "Group / Name" (get-in objects' [main-id :name])) + "the main instance takes the path and name of its component"))) diff --git a/common/test/common_tests/files_migrations_0035_test.cljc b/common/test/common_tests/files_migrations_0035_test.cljc index 8dd421d614..350b669fed 100644 --- a/common/test/common_tests/files_migrations_0035_test.cljc +++ b/common/test/common_tests/files_migrations_0035_test.cljc @@ -10,10 +10,186 @@ [app.common.geom.matrix :as gmt] [app.common.geom.rect :as grc] [app.common.types.file :as ctf] + [app.common.types.objects-map :as omap] [app.common.types.shape :as cts] [app.common.uuid :as uuid] [clojure.test :as t])) +;; ====================================================================== +;; 0035-fix-dangling-children +;; A shape can list a child id that is not present in the `:objects` map +;; of the page or component owning it (a dangling reference). Any +;; traversal of the shape tree trips on it, and the file becomes +;; unopenable. The migration drops those ids and keeps every shape that +;; really exists. +;; ====================================================================== + +(def ^:private migration "0035-fix-dangling-children") + +(defn- make-root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- make-frame + [frame-id child-ids] + (-> (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + (assoc :shapes child-ids))) + +(defn- page-data + [page-id objects] + {:pages-index {page-id {:id page-id :objects objects}}}) + +(defn- component-data + [component-id objects] + {:components {component-id {:id component-id :objects objects}}}) + +(t/deftest removes-the-dangling-children-of-a-page + (let [page-id (uuid/next) + frame-id (uuid/next) + child-id (uuid/next) + frame (make-frame frame-id [child-id]) + data (page-data page-id {uuid/zero (make-root [frame-id child-id]) + frame-id frame}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling id is removed from the root children list") + (t/is (= [] (get-in objects' [frame-id :shapes])) + "the dangling id is removed from the frame children list") + (t/is (= 2 (count objects')) + "every shape that exists is kept") + (t/is (some? (get objects' frame-id)) + "the frame itself is kept"))) + +(t/deftest removes-the-dangling-children-of-a-component + (let [component-id (uuid/next) + frame-id (uuid/next) + child-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + data (component-data component-id {uuid/zero (make-root [frame-id child-id]) + frame-id frame}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling id is removed from the children list") + (t/is (= 2 (count objects')) + "every shape that exists is kept") + (t/is (= frame (get objects' frame-id)) + "the existing shape is left untouched"))) + +(t/deftest migration-works-with-an-objects-map + ;; The backend stores the page :objects as an ObjectsMap (the lazy + ;; representation enabled by the fdata/objects-map feature). It is an + ;; IPersistentMap that does not implement IFn on the JVM: a migration + ;; must consult it with contains?/get, never by calling it as a + ;; function. + (let [page-id (uuid/next) + frame-id (uuid/next) + child-id (uuid/next) + frame (make-frame frame-id [child-id]) + objects (omap/wrap {uuid/zero (make-root [frame-id child-id]) + frame-id frame}) + data (page-data page-id objects) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling id is removed with the ObjectsMap representation") + (t/is (= [] (get-in objects' [frame-id :shapes])) + "the frame children list is pruned the same way"))) + +(t/deftest keeps-the-order-and-the-shapes-of-a-valid-tree + (let [page-id (uuid/next) + frame-id (uuid/next) + rect-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + rect (cts/setup-shape {:id rect-id :type :rect :name "Rect"}) + data (page-data page-id {uuid/zero (make-root [frame-id rect-id]) + frame-id frame + rect-id rect}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (= [frame-id rect-id] (get-in objects' [uuid/zero :shapes])) + "the children list is kept as it was") + (t/is (= data migrated) + "a tree without dangling references is not modified"))) + +(t/deftest leaves-shapes-with-empty-children-alone + (let [page-id (uuid/next) + frame (cts/setup-shape {:id (uuid/next) :type :frame :name "Frame"}) + data (page-data page-id {uuid/zero (make-root []) + (:id frame) frame}) + migrated (cfm/migrate-data data migration)] + + (t/is (= data migrated) + "empty children vectors are left alone"))) + +(t/deftest leaves-shapes-without-children-key-alone + (let [page-id (uuid/next) + rect-id (uuid/next) + rect (cts/setup-shape {:id rect-id :type :rect :name "Rect"}) + data (page-data page-id {uuid/zero (make-root []) + rect-id rect}) + migrated (cfm/migrate-data data migration)] + + (t/is (= data migrated) + "shapes without a children key are left alone") + (t/is (not (contains? (get-in migrated [:pages-index page-id :objects rect-id]) :shapes)) + "no children key is created on the shape"))) + +(t/deftest migration-is-idempotent + (let [page-id (uuid/next) + child-id (uuid/next) + data (page-data page-id {uuid/zero (make-root [child-id])}) + migrated (cfm/migrate-data data migration)] + + (t/is (= migrated (cfm/migrate-data migrated migration)) + "applying the migration twice does not change the result"))) + +(t/deftest file-with-dangling-children-passes-the-whole-migration + (let [file-id (uuid/next) + page-id (uuid/next) + frame-id (uuid/next) + missing-id (uuid/next) + frame (make-frame frame-id [missing-id]) + base (ctf/make-file-data file-id page-id) + data (-> base + (assoc-in [:pages-index page-id :objects uuid/zero] + (make-root [frame-id missing-id])) + (assoc-in [:pages-index page-id :objects frame-id] frame)) + ;; The file reached the state before 0033 ran, so 0033, 0034 and + ;; 0035 are all pending, exactly as in the reported failure. + file {:id file-id + :data data + :migrations (disj cfm/available-migrations + "0033-normalize-main-instance-names" + "0034-fix-path-content-offset" + migration)}] + + (t/is (cfm/need-migration? file) + "the file is scheduled for migration") + + (let [migrated (cfm/migrate-file file {}) + objects' (get-in migrated [:data :pages-index page-id :objects])] + + (t/is (contains? (:migrations migrated) migration) + "the migration is recorded as applied") + (t/is (not (contains? objects' nil)) + "the migrated data has no nil shape entry") + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling reference of the root is removed") + (t/is (= [] (get-in objects' [frame-id :shapes])) + "the dangling reference of the frame is removed") + (t/is (some? (get objects' frame-id)) + "the frame that exists is kept")))) + +;; ====================================================================== ;; 0035-fix-shape-svg-attrs-and-defs ;; Stored svg attribute keys are kebab-case (issue #11947). Old birth and ;; the cleaner era stored camelCase React prop names, which the v3 reader @@ -22,6 +198,7 @@ ;; through the whitelist+kebab transform, svg-raw `:content` attrs are ;; kebab-ized spelling-only. It replaces "0021-fix-shape-svg-attrs" (id ;; withdrawn): files that recorded it migrate again through this one. +;; ====================================================================== (def ^:private migration-id "0035-fix-shape-svg-attrs-and-defs") diff --git a/common/test/common_tests/files_migrations_0036_test.cljc b/common/test/common_tests/files_migrations_0036_test.cljc new file mode 100644 index 0000000000..d9789ea072 --- /dev/null +++ b/common/test/common_tests/files_migrations_0036_test.cljc @@ -0,0 +1,130 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0036-test + (:require + [app.common.files.migrations :as cfm] + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; ====================================================================== +;; 0036-clear-nil-keys-in-objects +;; Soft-delete can leave a nil key in a deleted component's `:objects` +;; (`{nil {:selrect nil :points []}}`). That fails the file schema on +;; duplicate-file (#12336). It replaces `legacy-36` (id withdrawn from +;; `available-migrations`): files that recorded it migrate again +;; through this one. +;; ====================================================================== + +(def ^:private migration "0036-clear-nil-keys-in-objects") + +(def ^:private nil-placeholder + {:selrect nil :points []}) + +(defn- root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(t/deftest removes-nil-key-from-component-objects + (let [component-id (uuid/next) + data {:components + {component-id + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page (uuid/next) + :objects {nil nil-placeholder}}}} + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "nil key is removed from the deleted component :objects") + (t/is (= {} objects') + "only the nil placeholder was present, so objects is empty"))) + +(t/deftest removes-nil-key-from-page-objects + (let [page-id (uuid/next) + frame-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame" :shapes []}) + data {:pages-index + {page-id + {:id page-id + :name "Page" + :objects {uuid/zero (root [frame-id]) + frame-id frame + nil nil-placeholder}}}} + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "nil key is removed from the page :objects") + (t/is (some? (get objects' frame-id)) + "real shapes are kept"))) + +(t/deftest migrated-file-data-passes-schema + ;; Same failure mode duplicate-file hits: full schema check after migrate. + (let [file-id (uuid/next) + page-id (uuid/next) + component-id (uuid/next) + data (-> (ctf/make-file-data file-id page-id) + (assoc-in [:components component-id] + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page page-id + :objects {nil nil-placeholder}}))] + + (t/is (thrown? #?(:clj Exception :cljs js/Error) + (ctf/check-file-data data)) + "corrupt trash objects fail the file schema") + + (let [data' (cfm/migrate-data data migration)] + (t/is (not (contains? (get-in data' [:components component-id :objects]) nil))) + (t/is (= data' (ctf/check-file-data data')) + "after clearing nil keys the file data passes the schema")))) + +(t/deftest migration-0036-runs-through-file-migration + ;; A file that recorded the withdrawn "legacy-36" (and everything else + ;; current) still needs migration: legacy-36 left + ;; `available-migrations`, which re-arms it for 0036. + (let [component-id (uuid/next) + file (ctf/make-file {:name "Legacy nil objects"}) + page-id (first (get-in file [:data :pages])) + file (-> file + (assoc :migrations (-> cfm/available-migrations + (disj migration) + (conj "legacy-36"))) + (assoc-in [:data :components component-id] + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page page-id + :objects {nil nil-placeholder}})) + file' (cfm/migrate-file file {})] + + (t/is (cfm/need-migration? file) + "the withdrawn legacy-36 re-arms the cleanup for this file") + (t/is (not (cfm/need-migration? file')) + "new migration recorded") + (t/is (contains? (:migrations file') migration) + "0036 migration id persisted") + (t/is (contains? (:migrations file') "legacy-36") + "the old id stays in the recorded list") + (t/is (not (contains? cfm/available-migrations "legacy-36")) + "the withdrawn legacy-36 id stays out of available-migrations") + (t/is (not (contains? (get-in file' [:data :components component-id :objects]) nil)) + "nil key is cleared through the full migrate-file path"))) diff --git a/common/test/common_tests/logic/comp_creation_test.cljc b/common/test/common_tests/logic/comp_creation_test.cljc index 95f158e2a3..eb0fb36d01 100644 --- a/common/test/common_tests/logic/comp_creation_test.cljc +++ b/common/test/common_tests/logic/comp_creation_test.cljc @@ -9,6 +9,7 @@ [app.common.data :as d] [app.common.files.changes-builder :as pcb] [app.common.files.shapes-helpers :as cfsh] + [app.common.files.validate :as cfv] [app.common.geom.point :as gpt] [app.common.logic.libraries :as cll] [app.common.logic.shapes :as cls] @@ -91,6 +92,37 @@ (t/is (ctk/main-instance? root)) (t/is (ctk/main-instance-of? (:id root) (:id page) component)))) +(t/deftest test-add-component-normalizes-root-name + (t/testing "main instance root name matches the component even if the source name is not normalized" + (let [;; ==== Setup + file (-> (thf/sample-file :file1) + (ths/add-sample-shape :frame1 :type :frame :name "Group/Subgroup / Name")) + + page (thf/current-page file) + frame1 (ths/get-shape file :frame1) + + ;; ==== Action + [_ component-id changes] + (cll/generate-add-component (pcb/empty-changes) + [frame1] + (:objects page) + (:id page) + (:id file) + nil) + + file' (thf/apply-changes file changes) + + ;; ==== Get + component (thc/get-component-by-id file' component-id) + root (ths/get-shape-by-id file' (:main-instance-id component)) + errors (cfv/validate-file file' {})] + + ;; ==== Check + (t/is (= "Group / Subgroup" (:path component))) + (t/is (= "Name" (:name component))) + (t/is (= "Group / Subgroup / Name" (:name root))) + (t/is (nil? errors))))) + (t/deftest test-add-component-from-several-shapes (let [;; ==== Setup file (-> (thf/sample-file :file1) diff --git a/common/test/common_tests/logic/comp_remove_swap_slots_test.cljc b/common/test/common_tests/logic/comp_remove_swap_slots_test.cljc index da766cd71d..bf62f42c72 100644 --- a/common/test/common_tests/logic/comp_remove_swap_slots_test.cljc +++ b/common/test/common_tests/logic/comp_remove_swap_slots_test.cljc @@ -567,3 +567,62 @@ ;; blue2' has not swap-id after move (t/is (some? blue2')) (t/is (nil? (ctk/get-swap-slot blue2'))))) + +(defn- setup-nested-swap-file + [] + ;; {:frame-icon-a} [:name Frame1] # [Component :icon-a] + ;; {:frame-icon-b} [:name Frame1] # [Component :icon-b] + ;; {:frame-card} [:name Frame1] # [Component :card] + ;; :frame-inner [:name Frame1] + ;; :icon-copy [:name Frame1] @--> :frame-icon-a + ;; :card-copy [:name Frame1] @--> :frame-card + ;; :inner-copy [:name Frame1] @--> :frame-inner + ;; :icon-in-card-copy [:name Frame1] @--> :frame-icon-a + ;; swapped to :icon-b, it becomes + ;; :icon-b-copy [:name Frame1, swap-slot] @--> :frame-icon-b + (-> (thf/sample-file :file1) + (tho/add-frame :frame-icon-a) + (thc/make-component :icon-a :frame-icon-a) + (tho/add-frame :frame-icon-b) + (thc/make-component :icon-b :frame-icon-b) + (tho/add-frame :frame-card) + (thc/make-component :card :frame-card) + (tho/add-frame :frame-inner :parent-label :frame-card) + (thc/instantiate-component :icon-a :icon-copy :parent-label :frame-inner) + (thc/instantiate-component :card :card-copy + :children-labels [:inner-copy :icon-in-card-copy]) + (thc/component-swap :icon-in-card-copy :icon-b :icon-b-copy))) + +(t/deftest test-keep-swap-slot-duplicating-swapped-nested-copy-inside-copy + (let [;; ==== Setup + file (setup-nested-swap-file) + + page (thf/current-page file) + icon-b-copy (ths/get-shape file :icon-b-copy) + + ;; ==== Action + changes (-> (pcb/empty-changes nil) + (cll/generate-duplicate-changes (:objects page) ;; objects + page ;; page + #{(:id icon-b-copy)} ;; ids + (gpt/point 0 0) ;; delta + {(:id file) file} ;; libraries + (:data file) ;; library-data + (:id file)) ;; file-id + (cll/generate-duplicate-changes-update-indices (:objects page) ;; objects + #{(:id icon-b-copy)})) ;; ids + + file' (thf/apply-changes file changes) + + ;; ==== Get + page' (thf/current-page file') + duplicated-icon-b' (find-duplicated-shape icon-b-copy page')] + + ;; ==== Check + + ;; icon-b-copy has swap-slot + (t/is (some? (ctk/get-swap-slot icon-b-copy))) + + ;; duplicated icon-b copy keeps its swap-slot + (t/is (some? duplicated-icon-b')) + (t/is (some? (ctk/get-swap-slot duplicated-icon-b'))))) diff --git a/common/test/common_tests/logic/comp_sync_test.cljc b/common/test/common_tests/logic/comp_sync_test.cljc index 02cd219779..034a575bcf 100644 --- a/common/test/common_tests/logic/comp_sync_test.cljc +++ b/common/test/common_tests/logic/comp_sync_test.cljc @@ -8,17 +8,21 @@ (:require [app.common.data :as d] [app.common.files.changes-builder :as pcb] + [app.common.files.validate :as cfv] [app.common.geom.point :as gpt] [app.common.geom.shapes :as gsh] [app.common.logic.libraries :as cll] [app.common.logic.shapes :as cls] + [app.common.path-names :as cpn] [app.common.test-helpers.components :as thc] [app.common.test-helpers.compositions :as tho] [app.common.test-helpers.files :as thf] [app.common.test-helpers.ids-map :as thi] [app.common.test-helpers.shapes :as ths] [app.common.types.component :as ctk] + [app.common.types.components-list :as ctkl] [app.common.types.container :as ctn] + [app.common.types.modifiers :as ctm] [app.common.types.shape-tree :as ctst] [clojure.test :as t])) @@ -552,6 +556,330 @@ {(:id file) file} (:id file))) +(defn- changes-attrs + "Set of attrs touched by the :mod-obj changes of a changes map." + [changes] + (->> (:redo-changes changes) + (filter #(= :mod-obj (:type %))) + (mapcat :operations) + (map :attr) + (into #{}))) + +(defn- file-with-path-component + [] + (let [content [{:command :move-to :params {:x 10.3 :y 10.7}} + {:command :line-to :params {:x 24.3 :y 10.7}} + {:command :line-to :params {:x 17.3 :y 24.7}} + {:command :close-path}]] + (-> (thf/sample-file :file1) + (tho/add-frame :main-root {:width 107.3 :height 41.7}) + (ths/add-sample-shape :main-path :type :path :parent-label :main-root :content content) + (thc/make-component :component1 :main-root) + (thc/instantiate-component :component1 :copy-root {:children-labels [:copy-path]})))) + +(t/deftest test-no-sync-changes-when-position-changes-with-float-drift + ;; Regression (#12719): geometry in production carries sub-pixel drift + ;; and non-integer sizes, so move(+delta) followed by reposition(-delta) + ;; does not cancel exactly. A position-only move of the main must still + ;; produce no sync changes, and so no "library updated" notification. + (let [;; ==== Setup + file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root + :main-root-params {:width 107.3 :height 41.7} + :main-child-params {:width 33.3 :height 17.7} + :copy-root-params {:children-labels [:copy-child]})) + page (thf/current-page file) + main-root (ths/get-shape file :main-root) + main-child (ths/get-shape file :main-child) + copy-child (ths/get-shape file :copy-child) + + ;; Override on the copy, so it carries touched state + changes0 (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(:id copy-child)} + (fn [shape] + (assoc shape :fills (ths/sample-fills-color :fill-color "#fabada"))) + (:objects page) + {}) + file (thf/apply-changes file changes0) + page (thf/current-page file) + + ;; ==== Action + ;; Move the whole main by 1 px plus sub-pixel noise + delta (gpt/point 1.0000123 0.0000071) + changes1 (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(:id main-root) (:id main-child)} + (fn [shape] (gsh/move shape delta)) + (:objects page) + {}) + + updated-file (thf/apply-changes file changes1) + + sync-changes (cll/generate-sync-file-changes (pcb/empty-changes) + nil + :components + (:id updated-file) + (thi/id :component1) + (:id updated-file) + {(:id updated-file) updated-file} + (:id updated-file))] + + ;; ==== Check + (t/is (empty? (:redo-changes sync-changes))))) + +(t/deftest test-sync-size-change-of-main-child + ;; The drift tolerance must not swallow real size edits (>> 0.001). + (let [;; ==== Setup + file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root + :main-root-params {:width 107.3 :height 41.7} + :main-child-params {:width 33.3 :height 17.7} + :copy-root-params {:children-labels [:copy-child]})) + page (thf/current-page file) + + ;; ==== Action + updated-file (thf/apply-changes file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(thi/id :main-child)} + #(gsh/transform-shape % (ctm/change-size % (+ (:width %) 1) nil)) + (:objects page) + {})) + attrs (changes-attrs (sync-file-changes updated-file))] + + ;; ==== Check + (t/is (contains? attrs :width)) + (t/is (contains? attrs :selrect)))) + +(t/deftest test-sync-text-position-data-change-with-same-xy + ;; A change in a text run (here its fill) that keeps x/y must still be + ;; synced: position-data is not a drifting geometry attr. + (let [;; ==== Setup + run (fn [color] + [{:x 0 :y 0 :width 50 :height 20 :text "hello" + :fills [{:fill-color color :fill-opacity 1}]}]) + file (-> (thf/sample-file :file1) + (tho/add-frame-with-text :main-root :main-child "hello") + (thc/make-component :component1 :main-root) + (thc/instantiate-component :component1 :copy-root {:children-labels [:copy-child]})) + set-pd (fn [file color] + (let [page (thf/current-page file) + main-child (ths/get-shape file :main-child)] + (thf/apply-changes + file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(:id main-child)} + #(assoc % :position-data (run color)) + (:objects page) + {})))) + + file1 (set-pd file "#000000") + file2 (thf/apply-changes file1 (sync-file-changes file1)) + file3 (set-pd file2 "#ff0000") + + ;; ==== Action + sync-changes (sync-file-changes file3)] + + ;; ==== Check + (t/is (contains? (changes-attrs sync-changes) :position-data)))) + +(t/deftest test-sync-small-opacity-change + ;; The drift tolerance must not swallow real non-geometry changes. + (let [;; ==== Setup + file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root + :copy-root-params {:children-labels [:copy-child]})) + set-opacity (fn [file opacity] + (let [page (thf/current-page file) + main-child (ths/get-shape file :main-child)] + (thf/apply-changes + file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(:id main-child)} + #(assoc % :opacity opacity) + (:objects page) + {})))) + + file1 (set-opacity file 1) + file2 (thf/apply-changes file1 (sync-file-changes file1)) + file3 (set-opacity file2 0.9995) + + ;; ==== Action + sync-changes (sync-file-changes file3)] + + ;; ==== Check + (t/is (contains? (changes-attrs sync-changes) :opacity)))) + +(t/deftest test-no-sync-changes-when-path-moves-with-float-drift + ;; Path content stores absolute coordinates, so it drifts like the other + ;; geometry when the main is moved. + (let [;; ==== Setup + file (file-with-path-component) + page (thf/current-page file) + main-root (ths/get-shape file :main-root) + main-path (ths/get-shape file :main-path) + + ;; ==== Action + changes1 (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(:id main-root) (:id main-path)} + #(gsh/move % (gpt/point 1.0000123 0.0000071)) + (:objects page) + {}) + updated-file (thf/apply-changes file changes1)] + + ;; ==== Check + (t/is (empty? (:redo-changes (sync-file-changes updated-file)))))) + +(t/deftest test-sync-path-moved-inside-main + ;; Moving only the path inside its main is a real change and must sync. + (let [;; ==== Setup + file (file-with-path-component) + page (thf/current-page file) + main-path (ths/get-shape file :main-path) + + ;; ==== Action + changes1 (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(:id main-path)} + #(gsh/move % (gpt/point 1 0)) + (:objects page) + {}) + updated-file (thf/apply-changes file changes1)] + + ;; ==== Check + (t/is (contains? (changes-attrs (sync-file-changes updated-file)) :content)))) + +(defn- file-with-bool-component + [] + (let [content [{:command :move-to :params {:x 10.3 :y 10.7}} + {:command :line-to :params {:x 24.3 :y 10.7}} + {:command :line-to :params {:x 17.3 :y 24.7}} + {:command :close-path}]] + (-> (thf/sample-file :file1) + (tho/add-frame :main-root {:width 107.3 :height 41.7}) + (ths/add-sample-shape :main-bool :type :bool :bool-type :union :parent-label :main-root :content content) + (ths/add-sample-shape :main-p1 :type :path :parent-label :main-bool :content content) + (ths/add-sample-shape :main-p2 :type :path :parent-label :main-bool :content content) + (thc/make-component :component1 :main-root) + (thc/instantiate-component :component1 :copy-root {:children-labels [:copy-bool :copy-p1 :copy-p2]})))) + +(defn- move-shapes + [file labels delta] + (let [page (thf/current-page file)] + (thf/apply-changes file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + (into #{} (map #(thi/id %)) labels) + #(gsh/move % delta) + (:objects page) + {})))) + +(t/deftest test-no-sync-changes-when-text-moves-with-float-drift + ;; Text position-data moves with the shape and drifts like geometry, so + ;; a position-only move of the main must produce no sync changes. + (let [;; ==== Setup + position-data [{:x 10.3 :y 10.7 :width 33.3 :height 17.7 :text "hello" + :fills [{:fill-color "#000000" :fill-opacity 1}]}] + file (-> (thf/sample-file :file1) + (tho/add-frame-with-text :main-root :main-child "hello") + (thc/make-component :component1 :main-root) + (thc/instantiate-component :component1 :copy-root {:children-labels [:copy-child]})) + page (thf/current-page file) + file (thf/apply-changes file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(thi/id :main-child)} + #(assoc % :position-data position-data) + (:objects page) + {})) + file (thf/apply-changes file (sync-file-changes file)) + + ;; ==== Action + updated-file (move-shapes file [:main-root :main-child] (gpt/point 1.0000123 0.0000071)) + sync-changes (sync-file-changes updated-file)] + + ;; ==== Check + (t/is (= #{} (changes-attrs sync-changes))) + (t/is (empty? (:redo-changes sync-changes))))) + +(t/deftest test-sync-text-moved-inside-main + ;; Moving only the text inside its main is a real change beyond the + ;; tolerance and must sync its position-data. + (let [;; ==== Setup + position-data [{:x 10.3 :y 10.7 :width 33.3 :height 17.7 :text "hello" + :fills [{:fill-color "#000000" :fill-opacity 1}]}] + file (-> (thf/sample-file :file1) + (tho/add-frame-with-text :main-root :main-child "hello") + (thc/make-component :component1 :main-root) + (thc/instantiate-component :component1 :copy-root {:children-labels [:copy-child]})) + page (thf/current-page file) + file (thf/apply-changes file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(thi/id :main-child)} + #(assoc % :position-data position-data) + (:objects page) + {})) + file (thf/apply-changes file (sync-file-changes file)) + + ;; ==== Action + updated-file (move-shapes file [:main-child] (gpt/point 5 0))] + + ;; ==== Check + (t/is (contains? (changes-attrs (sync-file-changes updated-file)) :position-data)))) + +(t/deftest test-sync-text-position-data-entry-count-change + ;; A text that gains a line keeps its first entry untouched but has one + ;; more entry: a length mismatch is a real change and must sync. + (let [;; ==== Setup + entry {:x 10.3 :y 10.7 :width 33.3 :height 17.7 :text "hello" + :fills [{:fill-color "#000000" :fill-opacity 1}]} + set-pd (fn [file position-data] + (let [page (thf/current-page file)] + (thf/apply-changes file + (cls/generate-update-shapes (pcb/empty-changes nil (:id page)) + #{(thi/id :main-child)} + #(assoc % :position-data position-data) + (:objects page) + {})))) + file (-> (thf/sample-file :file1) + (tho/add-frame-with-text :main-root :main-child "hello") + (thc/make-component :component1 :main-root) + (thc/instantiate-component :component1 :copy-root {:children-labels [:copy-child]})) + file (set-pd file [entry]) + file (thf/apply-changes file (sync-file-changes file)) + + ;; ==== Action + updated-file (set-pd file [entry (assoc entry :y 28.7 :text "world")])] + + ;; ==== Check + (t/is (contains? (changes-attrs (sync-file-changes updated-file)) :position-data)))) + +(t/deftest test-no-sync-changes-when-bool-moves-with-float-drift + ;; Bool content stores absolute coordinates, so it drifts like path content. + (let [;; ==== Action + file (file-with-bool-component) + updated-file (move-shapes file + [:main-root :main-bool :main-p1 :main-p2] + (gpt/point 1.0000123 0.0000071)) + sync-changes (sync-file-changes updated-file)] + + ;; ==== Check + (t/is (= #{} (changes-attrs sync-changes))) + (t/is (empty? (:redo-changes sync-changes))))) + +(t/deftest test-sync-bool-content-change-inside-main + ;; Moving only the bool inside its main is a real change and must sync. + (let [;; ==== Action + file (file-with-bool-component) + updated-file (move-shapes file [:main-bool :main-p1 :main-p2] (gpt/point 1 0))] + + ;; ==== Check + (t/is (contains? (changes-attrs (sync-file-changes updated-file)) :content)))) + (t/deftest test-sync-many-updated-shapes ;; The undo changes of a sync used to grow one lazy level per updated ;; shape (#12576). Overflowing the stack needs thousands of shapes, which @@ -795,3 +1123,64 @@ (t/is (not= (geometry main-group) (geometry main-group'))) (t/is (= (:shapes main-group) (:shapes main-group''))) (t/is (= (geometry main-group) (geometry main-group''))))) + +(defn- run-sync-inverse + "Run the inverse sync on `shape-tag`, which has been renamed to an + unnormalized name, and return the resulting main root and component + names, and the updated file." + [shape-tag] + (let [file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root) + (tho/change-shape shape-tag #(update % :touched (fnil conj #{}) :name-group)) + (ths/update-shape shape-tag :name "Group/Subgroup")) + page (thf/current-page file) + container (ctn/make-container page :page) + changes (-> (pcb/empty-changes) + (pcb/with-container container) + (cll/generate-sync-shape-inverse (:data file) + {(:id file) file} + container + (thi/id shape-tag))) + file' (thf/apply-changes file changes) + main-root' (ths/get-shape file' :main-root) + component' (ctkl/get-component (:data file') (thi/id :component1))] + [(:name main-root') + (cpn/merge-path-item (:path component') (:name component')) + file'])) + +(t/deftest test-sync-inverse-normalizes-copy-name + (t/testing "syncing an unnormalized copy name keeps the component and its main root matching" + (let [file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root) + (tho/change-shape :copy-root #(update % :touched (fnil conj #{}) :name-group)) + (ths/update-shape :copy-root :name "Group/Subgroup")) + page (thf/current-page file) + container (ctn/make-container page :page) + changes (-> (pcb/empty-changes) + (pcb/with-container container) + (cll/generate-sync-shape-inverse (:data file) + {(:id file) file} + container + (thi/id :copy-root))) + file' (thf/apply-changes file changes) + main-root' (ths/get-shape file' :main-root) + component' (ctkl/get-component (:data file') (thi/id :component1))] + + ;; ==== Check + (t/is (= "Group / Subgroup" (:name main-root'))) + (t/is (= "Group / Subgroup" (cpn/merge-path-item (:path component') (:name component')))) + (t/is (nil? (cfv/validate-file file' {})))))) + +(t/deftest test-sync-inverse-normalizes-main-name + (t/testing "syncing an unnormalized main name keeps the component and the main root matching" + (let [[main-root-name component-name file'] (run-sync-inverse :main-root)] + ;; ==== Check + (t/is (= "Group / Subgroup" main-root-name)) + (t/is (= "Group / Subgroup" component-name)) + (t/is (nil? (cfv/validate-file file' {})))))) diff --git a/common/test/common_tests/logic/variants_paste_test.cljc b/common/test/common_tests/logic/variants_paste_test.cljc new file mode 100644 index 0000000000..020b8dcd7e --- /dev/null +++ b/common/test/common_tests/logic/variants_paste_test.cljc @@ -0,0 +1,202 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.logic.variants-paste-test + (:require + [app.common.data :as d] + [app.common.files.changes-builder :as pcb] + [app.common.files.helpers :as cfh] + [app.common.files.validate :as cfv] + [app.common.geom.point :as gpt] + [app.common.logic.libraries :as cll] + [app.common.logic.variant-properties :as clvp] + [app.common.test-helpers.components :as thc] + [app.common.test-helpers.compositions :as tho] + [app.common.test-helpers.files :as thf] + [app.common.test-helpers.ids-map :as thi] + [app.common.test-helpers.shapes :as ths] + [app.common.test-helpers.variants :as thv] + [app.common.types.component :as ctk] + [app.common.types.variant :as ctv] + [clojure.test :as t])) + +(t/use-fixtures :each thi/test-fixture) + +(defn- paste-variants-into-container + "Paste individual variants from file-a into file-b's variant container. + Omit file-a from `libraries` to hit the unknown-component path; include + it to hit `duplicate-variant`. Optional `variant-props` mirrors clipboard + collection (Task 2)." + [file-a file-b source-labels target-container-label + & {:keys [libraries variant-props] + :or {libraries {(:id file-b) file-b}}}] + (let [page-a (thf/current-page file-a) + page-b (thf/current-page file-b) + container-b (ths/get-shape file-b target-container-label) + parent-id (:id container-b) + + source-ids (mapv thi/id source-labels) + tree-ids (into [] + (mapcat #(cfh/get-children-ids-with-self (:objects page-a) %)) + source-ids) + + pasted (->> tree-ids + (map (d/getf (:objects page-a))) + (map (fn [shape] + (cond-> shape + (contains? (set source-ids) (:id shape)) + (assoc :parent-id parent-id + :frame-id parent-id)))) + (d/index-by :id)) + + all-objects (merge (:objects page-b) pasted)] + (-> (pcb/empty-changes nil) + (pcb/with-page-id (:id page-b)) + (pcb/with-library-data (:data file-b)) + (pcb/with-objects all-objects) + (cll/generate-duplicate-changes all-objects + page-b + (set source-ids) + (gpt/point 0 0) + libraries + (:data file-b) + (:id file-b) + {:variant-props variant-props})))) + +(defn- property-names + [component] + (mapv :name (:variant-properties component))) + +(defn- variant-mains-in + [file container-label] + (let [container (ths/get-shape file container-label)] + (->> (vals (:objects (thf/current-page file))) + (filter ctk/main-instance?) + (filter #(= (:id container) (:variant-id %)))))) + +(defn- other-set-file + [] + (-> (thf/sample-file :file-b) + (thv/add-variant-two-properties :vb :cb1 :mb1 :cb2 :mb2) + (ths/update-shape :vb :name "Other") + (ths/update-shape :mb1 :name "Other") + (ths/update-shape :mb2 :name "Other") + (thc/update-component :cb1 {:name "Other"}) + (thc/update-component :cb2 {:name "Other"}))) + +(t/deftest test-paste-unknown-variant-into-container-validates + ;; #10704: paste individual variants from file A into file B's set + ;; when A is not in libraries must leave a valid file (desired behavior). + (let [file-a (-> (thf/sample-file :file-a) + (thv/add-variant :va :ca1 :ma1 :ca2 :ma2)) + file-b (other-set-file) + changes (paste-variants-into-container file-a file-b [:ma1 :ma2] :vb) + file' (thf/apply-changes file-b changes :validate? false) + errors (cfv/validate-file file' {}) + pasted (->> (variant-mains-in file' :vb) + (remove #(#{(thi/id :mb1) (thi/id :mb2)} (:id %))))] + + (t/is (nil? errors)) + (doseq [shape pasted] + (let [comp (thc/get-component-by-id file' (:component-id shape))] + (t/is (= ["Property 1" "Property 2"] (property-names comp))) + (t/is (= "Other" (:name shape))) + (t/is (= (:variant-name shape) + (ctv/properties-to-name (:variant-properties comp)))))))) + +(t/deftest test-duplicate-variant-in-own-container-bumps-last-value + ;; Same-container duplicate via duplicate-variant: originals keep their + ;; props; the new main gets a bumped last-property value. Does not cover + ;; generate-make-shapes-variant's already-aligned early-exit (new mains + ;; enter with a nil variant-id). + (let [file (-> (thf/sample-file :file1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02)) + c01-props (:variant-properties (thc/get-component file :c01)) + c02-props (:variant-properties (thc/get-component file :c02)) + file' (tho/duplicate-shape file :m01) + mains (variant-mains-in file' :v01) + new-main (->> mains + (remove #(#{(thi/id :m01) (thi/id :m02)} (:id %))) + first) + new-comp (thc/get-component-by-id file' (:component-id new-main))] + + (thf/validate-file! file') + (t/is (= 3 (count mains))) + (t/is (= c01-props (:variant-properties (thc/get-component file' :c01)))) + (t/is (= c02-props (:variant-properties (thc/get-component file' :c02)))) + (t/is (= ["Property 1"] (property-names new-comp))) + (t/is (= (str ctv/value-prefix 3) + (-> new-comp :variant-properties last :value))))) + +(t/deftest test-make-shapes-variant-skips-when-already-aligned + ;; Locks the narrowed early-exit: a fully aligned child must not have its + ;; props or variant-name rewritten. + (let [file (-> (thf/sample-file :file1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02)) + page (thf/current-page file) + container (ths/get-shape file :v01) + shape (ths/get-shape file :m01) + before (:variant-properties (thc/get-component file :c01)) + before-vn (:variant-name shape) + changes (-> (pcb/empty-changes nil) + (pcb/with-page page) + (pcb/with-objects (:objects page)) + (pcb/with-library-data (:data file)) + (clvp/generate-make-shapes-variant [shape] container)) + file' (thf/apply-changes file changes) + shape' (ths/get-shape-by-id file' (:id shape))] + + (t/is (= before (:variant-properties (thc/get-component file' :c01)))) + (t/is (= before-vn (:variant-name shape'))))) + +(t/deftest test-paste-known-variant-into-other-container-aligns + ;; file A present in libraries → duplicate-variant path; props must match + ;; the target set and the file must validate. + (let [file-a (-> (thf/sample-file :file-a) + (thv/add-variant :va :ca1 :ma1 :ca2 :ma2)) + file-b (other-set-file) + libraries {(:id file-a) file-a + (:id file-b) file-b} + changes (paste-variants-into-container file-a file-b [:ma1 :ma2] :vb + :libraries libraries) + file' (thf/apply-changes file-b changes :validate? false) + errors (cfv/validate-file file' {}) + pasted (->> (variant-mains-in file' :vb) + (remove #(#{(thi/id :mb1) (thi/id :mb2)} (:id %))))] + + (t/is (nil? errors)) + (t/is (= 2 (count pasted))) + (doseq [shape pasted] + (let [comp (thc/get-component-by-id file' (:component-id shape))] + (t/is (= ["Property 1" "Property 2"] (property-names comp))) + (t/is (= "Other" (:name shape))) + (t/is (= (:variant-name shape) + (ctv/properties-to-name (:variant-properties comp)))))))) + +(t/deftest test-paste-unknown-preserves-matching-property-values + ;; Clipboard props (Task 2) whose names exist on the target survive regenerate. + (let [file-a (-> (thf/sample-file :file-a) + (thv/add-variant :va :ca1 :ma1 :ca2 :ma2)) + file-b (other-set-file) + props {(:id (thc/get-component file-a :ca1)) + (:variant-properties (thc/get-component file-a :ca1)) + (:id (thc/get-component file-a :ca2)) + (:variant-properties (thc/get-component file-a :ca2))} + changes (paste-variants-into-container file-a file-b [:ma1 :ma2] :vb + :variant-props props) + file' (thf/apply-changes file-b changes :validate? false) + pasted (->> (variant-mains-in file' :vb) + (remove #(#{(thi/id :mb1) (thi/id :mb2)} (:id %))) + (map #(thc/get-component-by-id file' (:component-id %)))) + values (->> pasted + (mapcat :variant-properties) + (filter #(= "Property 1" (:name %))) + (map :value) + set)] + + (thf/validate-file! file') + (t/is (contains? values "Value1")) + (t/is (contains? values "Value2")))) diff --git a/common/test/common_tests/logic/variants_test.cljc b/common/test/common_tests/logic/variants_test.cljc index 8162cf5598..a2ba965283 100644 --- a/common/test/common_tests/logic/variants_test.cljc +++ b/common/test/common_tests/logic/variants_test.cljc @@ -7,6 +7,7 @@ (ns common-tests.logic.variants-test (:require [app.common.files.changes-builder :as pcb] + [app.common.files.variant :as cfv] [app.common.geom.point :as gpt] [app.common.logic.libraries :as cll] [app.common.logic.shapes :as cls] @@ -722,3 +723,16 @@ (cpn/merge-path-item (:path comp2') (:name comp2')))) (t/is (some? (:variant-id comp1'))) (t/is (some? (:variant-id comp2'))))) + +(t/deftest test-extract-properties-values-preserves-order-past-eight + (let [names (mapv #(str "Property " %) (range 1 10)) + props1 (mapv (fn [n] {:name n :value (str n "-a")}) names) + props2 (mapv (fn [n] {:name n :value (str n "-b")}) names) + file (-> (thf/sample-file :file1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02) + (thc/update-component :c01 {:variant-properties props1}) + (thc/update-component :c02 {:variant-properties props2})) + page (thf/current-page file) + v-id (:id (ths/get-shape file :v01)) + result (cfv/extract-properties-values (:data file) (:objects page) v-id)] + (t/is (= names (mapv :name result))))) diff --git a/common/test/common_tests/math_test.cljc b/common/test/common_tests/math_test.cljc index 524963df9b..a8c3188a13 100644 --- a/common/test/common_tests/math_test.cljc +++ b/common/test/common_tests/math_test.cljc @@ -57,3 +57,21 @@ :clj (t/testing "finite? returns false for Double/NaN (CLJ)" (t/is (false? (mth/finite? Double/NaN)))))) + +(t/deftest close-map?-test + (t/testing "numbers within tolerance are close" + (t/is (mth/close-map? {:x 1.0 :y 2.0} {:x 1.0004 :y 2.0}))) + + (t/testing "numbers beyond tolerance are not close" + (t/is (not (mth/close-map? {:x 1.0} {:x 1.01})))) + + (t/testing "non-numeric values compare exactly" + (t/is (mth/close-map? {:text "a"} {:text "a"})) + (t/is (not (mth/close-map? {:text "a"} {:text "b"})))) + + (t/testing "different key sets are not close" + (t/is (not (mth/close-map? {:x 1.0} {:x 1.0 :y 1.0}))) + (t/is (not (mth/close-map? {:x 1.0 :y nil} {:x 1.0 :z nil})))) + + (t/testing "a number is not close to a non-number" + (t/is (not (mth/close-map? {:x 1.0} {:x "1.0"}))))) diff --git a/common/test/common_tests/media_test.cljc b/common/test/common_tests/media_test.cljc index f94525cae7..18ad0fc9fd 100644 --- a/common/test/common_tests/media_test.cljc +++ b/common/test/common_tests/media_test.cljc @@ -37,6 +37,11 @@ (t/is (= 400 (media/parse-font-weight "Thinker"))) (t/is (= 400 (media/parse-font-weight "Mediaeval")))) + (t/testing "matches a weight glued to a style token" + (t/is (= 700 (media/parse-font-weight "Roboto-BoldItalic"))) + (t/is (= 700 (media/parse-font-weight "Roboto-BoldOblique"))) + (t/is (= 600 (media/parse-font-weight "Roboto-SemiBoldItalic")))) + (t/testing "defaults to regular when the variant metadata is missing" (t/is (= 400 (media/parse-font-weight nil))) (t/is (= 400 (media/parse-font-weight ""))) @@ -54,11 +59,79 @@ (t/is (= "normal" (media/parse-font-style "Italica"))) (t/is (= "normal" (media/parse-font-style "Roboto-Regular")))) + (t/testing "treats oblique as italic" + (t/is (= "italic" (media/parse-font-style "Roboto-Oblique"))) + (t/is (= "italic" (media/parse-font-style "Roboto-BoldOblique"))) + (t/is (= "italic" (media/parse-font-style "RobotoOblique")))) + (t/testing "defaults to normal when the variant metadata is missing" (t/is (= "normal" (media/parse-font-style nil))) (t/is (= "normal" (media/parse-font-style ""))) (t/is (= "normal" (media/parse-font-style " "))))) +(t/deftest test-parse-font-family + (t/testing "strips weight and style tokens" + (t/is (= "Roboto" (media/parse-font-family "Roboto-Bold"))) + (t/is (= "Roboto" (media/parse-font-family "Roboto_Bold"))) + (t/is (= "Roboto" (media/parse-font-family "Roboto Bold"))) + (t/is (= "Roboto" (media/parse-font-family "Roboto-Oblique"))) + (t/is (= "OpenSans" (media/parse-font-family "OpenSans-SemiBold"))) + (t/is (= "Lato" (media/parse-font-family "Lato-Light")))) + + (t/testing "separates a style token glued to a weight token" + (t/is (= "Roboto" (media/parse-font-family "Roboto-BoldItalic"))) + (t/is (= "Roboto" (media/parse-font-family "Roboto-SemiBoldItalic"))) + (t/is (= "Roboto" (media/parse-font-family "Roboto-LightOblique"))) + (t/is (= "Roboto" (media/parse-font-family "Roboto-BlackOblique")))) + + (t/testing "does not strip tokens embedded in words" + (t/is (= "Boldini" (media/parse-font-family "Boldini-Bold"))) + (t/is (= "Lighthaus" (media/parse-font-family "Lighthaus"))) + (t/is (= "Blackwood" (media/parse-font-family "Blackwood"))) + (t/is (= "Mediaeval" (media/parse-font-family "Mediaeval")))) + + (t/testing "keeps unknown tokens in the family" + (t/is (= "Satoshi Variable" (media/parse-font-family "Satoshi-Variable")))) + + (t/testing "falls back to the base name when everything is stripped" + (t/is (= "BoldItalic" (media/parse-font-family "BoldItalic"))))) + +(t/deftest test-font-glued-tokens + (t/testing "a glued style token is recognized consistently by all three parsers" + ;; From EvaMarco's review on #11951: without a shared interpretation of + ;; the name, the bold-italic face collided with the bold-only one and + ;; was silently overwritten on upload. + (t/is (= "opensans webfont" (media/parse-font-family "opensans-bolditalic-webfont"))) + (t/is (= 700 (media/parse-font-weight "opensans-bolditalic-webfont"))) + (t/is (= "italic" (media/parse-font-style "opensans-bolditalic-webfont")))) + + (t/testing "a glued style token followed by a technical suffix" + (t/is (= "Roboto" (media/parse-font-family "Roboto-BoldItalic-VF"))) + (t/is (= 700 (media/parse-font-weight "Roboto-BoldItalic-VF"))) + (t/is (= "italic" (media/parse-font-style "Roboto-BoldItalic-VF")))) + + (t/testing "numeric build suffix after a glued style token" + (t/is (= "Roboto" (media/parse-font-family "Roboto-LightItalic_v2"))) + (t/is (= 300 (media/parse-font-weight "Roboto-LightItalic_v2"))) + (t/is (= "italic" (media/parse-font-style "Roboto-LightItalic_v2")))) + + (t/testing "the bold-only control case is unchanged" + (t/is (= "opensans webfont" (media/parse-font-family "opensans-bold-webfont"))) + (t/is (= 700 (media/parse-font-weight "opensans-bold-webfont"))) + (t/is (= "normal" (media/parse-font-style "opensans-bold-webfont"))))) + +(t/deftest test-first-nonblank-string + (t/testing "returns the first nonblank string" + (t/is (= "Bold" (media/first-nonblank-string " " "Bold"))) + (t/is (= "Bold" (media/first-nonblank-string "Bold" "Light"))) + (t/is (= "Bold" (media/first-nonblank-string nil "Bold"))) + (t/is (= "Inter" (media/first-nonblank-string "" "Inter")))) + + (t/testing "returns nil when every candidate is blank or not a string" + (t/is (nil? (media/first-nonblank-string nil " "))) + (t/is (nil? (media/first-nonblank-string))) + (t/is (nil? (media/first-nonblank-string 42 " "))))) + (t/deftest test-strip-image-extension (t/testing "removes extension from supported image files" (t/is (= (media/strip-image-extension "foo.png") "foo")) diff --git a/common/test/common_tests/path_names_test.cljc b/common/test/common_tests/path_names_test.cljc index 04cf0650b8..b6dde08be0 100644 --- a/common/test/common_tests/path_names_test.cljc +++ b/common/test/common_tests/path_names_test.cljc @@ -9,6 +9,14 @@ [app.common.path-names :as cpn] [clojure.test :as t])) +(t/deftest remove-extension + (t/is (= "photo" (cpn/remove-extension "photo.svg"))) + (t/is (= "icon" (cpn/remove-extension "icon.SVG"))) + (t/is (= "my.svg.icon" (cpn/remove-extension "my.svg.icon.svg"))) + (t/is (= "archive.tar" (cpn/remove-extension "archive.tar.gz"))) + (t/is (= "design" (cpn/remove-extension "design"))) + (t/is (= "" (cpn/remove-extension ".svg")))) + (t/deftest split-group-name (t/is (= ["foo" "bar"] (cpn/split-group-name "foo/bar"))) (t/is (= ["" "foo"] (cpn/split-group-name "foo"))) diff --git a/common/test/common_tests/render_wasm/modifiers_test.cljs b/common/test/common_tests/render_wasm/modifiers_test.cljs new file mode 100644 index 0000000000..c5677373c9 --- /dev/null +++ b/common/test/common_tests/render_wasm/modifiers_test.cljs @@ -0,0 +1,111 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.render-wasm.modifiers-test + (:require + [app.common.geom.matrix :as gmt] + [app.common.render-wasm.api.modifiers :as modifiers] + [app.common.render-wasm.wasm :as wasm] + [app.common.test-helpers.render-wasm :as thw] + [clojure.test :as t])) + +(def id-a #uuid "f1234567-89ab-cdef-fedc-ba9876543210") +(def id-b #uuid "12345678-abcd-ef01-2345-6789abcdef01") +(def matrix-a (gmt/matrix 0.1 -0.2 3.75 4.125 120.5 -80.25)) +(def matrix-b (gmt/matrix -1 0.5 0.25 2 32 -64)) +(defn- f32-matrix + [matrix] + (gmt/matrix (js/Math.fround (:a matrix)) + (js/Math.fround (:b matrix)) + (js/Math.fround (:c matrix)) + (js/Math.fround (:d matrix)) + (js/Math.fround (:e matrix)) + (js/Math.fround (:f matrix)))) + +(def requested [[id-a {:transform matrix-a :kind :child}] + [id-b {:transform matrix-b :kind :parent}]]) + +(t/deftest propagation-results-can-be-applied-after-memory-growth + (let [output [[id-b matrix-a] [id-a matrix-b]] + {:keys [module state]} (thw/create-module {:output output + :grow-on-propagate true + :grow-on-alloc true})] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers] + (let [result (modifiers/propagate-modifiers requested false nil) + expected (mapv (fn [[id matrix]] [id (f32-matrix matrix)]) output) + inputs (mapv (fn [[id data]] [id (update data :transform f32-matrix)]) + requested)] + (t/is (= expected result)) + (t/is (= {:precision 0 :entries inputs} (:propagated @state))) + (modifiers/set-modifiers result) + (t/is (= expected (:preview @state))))))) + +(t/deftest pixel-snapping-respects-the-ignored-axis + (doseq [[snap ignore-axis precision] [[false nil 0] + [false :x 0] + [true nil 1] + [true :x 3] + [true :y 2]]] + (let [{:keys [module state]} (thw/create-module {})] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers] + (modifiers/propagate-modifiers requested snap ignore-axis) + (t/is (= precision (get-in @state [:propagated :precision]))))))) + +(t/deftest empty-propagation-result-releases-the-payload + (let [{:keys [module state]} (thw/create-module {})] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers] + (t/is (= [] (modifiers/propagate-modifiers requested false nil))) + (modifiers/set-modifiers [[id-a matrix-a]]) + (t/is (= [[id-a (f32-matrix matrix-a)]] (:preview @state)))))) + +(t/deftest structure-entries-preserve-order-and-default-index + (let [entries [{:type :remove-children :parent id-a :id id-b :value 0} + {:type :add-children :index 7 :parent id-b :id id-a :value 2.5} + {:type :scale-content :index 2 :parent id-a :id id-b :value 0.1}] + {:keys [module state]} (thw/create-module {:grow-on-alloc true})] + (with-redefs [wasm/internal-module module] + (modifiers/set-structure-modifiers entries) + (t/is (= (-> entries + (assoc-in [0 :index] 0) + (update-in [2 :value] js/Math.fround)) + (:structure @state)))))) + +(t/deftest empty-inputs-preserve-an-existing-preview + (let [{:keys [module state]} (thw/create-module {})] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers] + (modifiers/set-modifiers [[id-a matrix-a]]) + (modifiers/set-structure-modifiers + [{:type :add-children :index 1 :parent id-a :id id-b :value 0}]) + (let [before @state] + (t/is (= [] (modifiers/propagate-modifiers [] true nil))) + (modifiers/set-modifiers []) + (modifiers/set-structure-modifiers []) + (t/is (= before @state)))))) + +(t/deftest native-propagation-errors-preserve-diagnostics-and-release-input + (let [{:keys [module state]} (thw/create-module {:propagate-error 1})] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers] + (let [error (try + (modifiers/propagate-modifiers requested false nil) + (catch :default cause cause))] + (t/is (= {:type :wasm-error :code :non-blocking :fn "_propagate_modifiers"} + (select-keys (ex-data error) [:type :code :fn])))) + (modifiers/set-modifiers [[id-b matrix-b]]) + (t/is (= [[id-b matrix-b]] (:preview @state)))))) + +(t/deftest undecodable-output-releases-the-payload + (let [{:keys [module state]} (thw/create-module {:output [[id-a matrix-a]] + :invalid-output-pointer true})] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers] + (t/is (thrown? js/Error (modifiers/propagate-modifiers requested false nil))) + (modifiers/set-modifiers [[id-b matrix-b]]) + (t/is (= [[id-b matrix-b]] (:preview @state)))))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index c293cffe1b..5243c00c61 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -11,6 +11,7 @@ #?(:cljs [clojure.string :as str]) #?(:cljs [clojure.tools.cli :refer [parse-opts]]) #?(:cljs [common-tests.fonts-test]) + #?(:cljs [common-tests.render-wasm.modifiers-test]) #?(:cljs [goog.object :as gobj]) [clojure.test :as t] [common-tests.attrs-test] @@ -23,8 +24,10 @@ [common-tests.files-changes-test] [common-tests.files-migrations-0025-test] [common-tests.files-migrations-0026-test] + [common-tests.files-migrations-0033-test] [common-tests.files-migrations-0034-test] [common-tests.files-migrations-0035-test] + [common-tests.files-migrations-0036-test] [common-tests.files-migrations-test] [common-tests.files-ref-paths-test] [common-tests.files.comp-processors-test] @@ -35,6 +38,7 @@ [common-tests.files.shapes-builder-test] [common-tests.files.tokens-test] [common-tests.files.validate-test] + [common-tests.files.variant-test] [common-tests.geom-align-test] [common-tests.geom-bounds-layout-nil-test] [common-tests.geom-bounds-map-test] @@ -84,6 +88,7 @@ [common-tests.logic.text-touched-test] [common-tests.logic.token-apply-test] [common-tests.logic.token-test] + [common-tests.logic.variants-paste-test] [common-tests.logic.variants-reparent-test] [common-tests.logic.variants-switch-test] [common-tests.logic.variants-test] @@ -105,6 +110,7 @@ [common-tests.types.color-test] [common-tests.types.components-test] [common-tests.types.container-test] + [common-tests.types.file-load-component-objects-test] [common-tests.types.fill-test] [common-tests.types.font-test] [common-tests.types.modifiers-test] @@ -137,11 +143,14 @@ 'common-tests.files-branch-merge-test 'common-tests.files-builder-test #?(:cljs 'common-tests.fonts-test) + #?(:cljs 'common-tests.render-wasm.modifiers-test) 'common-tests.files-changes-test 'common-tests.files-migrations-0025-test 'common-tests.files-migrations-0026-test + 'common-tests.files-migrations-0033-test 'common-tests.files-migrations-0034-test 'common-tests.files-migrations-0035-test + 'common-tests.files-migrations-0036-test 'common-tests.files-migrations-test 'common-tests.files-ref-paths-test 'common-tests.files.comp-processors-test @@ -152,6 +161,7 @@ 'common-tests.files.shapes-builder-test 'common-tests.files.tokens-test 'common-tests.files.validate-test + 'common-tests.files.variant-test 'common-tests.geom-align-test 'common-tests.geom-bounds-layout-nil-test 'common-tests.geom-bounds-map-test @@ -201,6 +211,7 @@ 'common-tests.logic.text-touched-test 'common-tests.logic.token-apply-test 'common-tests.logic.token-test + 'common-tests.logic.variants-paste-test 'common-tests.logic.variants-reparent-test 'common-tests.logic.variants-switch-test 'common-tests.logic.variants-test @@ -222,6 +233,7 @@ 'common-tests.types.color-test 'common-tests.types.components-test 'common-tests.types.container-test + 'common-tests.types.file-load-component-objects-test 'common-tests.types.fill-test 'common-tests.types.font-test 'common-tests.types.modifiers-test diff --git a/common/test/common_tests/types/container_test.cljc b/common/test/common_tests/types/container_test.cljc index 9f7a308c8a..f545705d42 100644 --- a/common/test/common_tests/types/container_test.cljc +++ b/common/test/common_tests/types/container_test.cljc @@ -291,3 +291,42 @@ (t/testing "two ordinary outside shapes are both kept" (t/is (= [outside-id outside-2-id] (vec (ctc/ids-for-structure-op objects [outside-id outside-2-id]))))))) + +(t/deftest get-nesting-level-delta + (let [page-id uuid/zero + head-id (uuid/next) + frame-id (uuid/next) + sub-id (uuid/next) + head-2-id (uuid/next) + plain-id (uuid/next) + ;; copy head with a plain frame and a nested copy head inside + head (make-shape head-id + :parent-id page-id + :type :frame + :component-id (uuid/next) + :component-root true + :shape-ref (uuid/next)) + frame (make-shape frame-id :parent-id head-id :type :frame :shape-ref (uuid/next)) + sub (make-shape sub-id + :parent-id frame-id + :type :frame + :component-id (uuid/next) + :shape-ref (uuid/next)) + head-2 (make-shape head-2-id + :parent-id page-id + :type :frame + :component-id (uuid/next) + :component-root true + :shape-ref (uuid/next)) + plain (make-shape plain-id :parent-id page-id) + root (root-shape) + objects (objects-map root head frame sub head-2 plain)] + (t/testing "shape stays under the same copy head" + (t/is (= 0 (ctc/get-nesting-level-delta objects sub frame)))) + (t/testing "shape leaves its copy head to the page" + (t/is (= 1 (ctc/get-nesting-level-delta objects sub root)))) + (t/testing "shape moves to a different copy head" + (t/is (= 1 (ctc/get-nesting-level-delta objects sub head-2)))) + (t/testing "shape outside any copy head goes inside one" + (t/is (= 0 (ctc/get-nesting-level-delta objects plain frame)))))) + diff --git a/common/test/common_tests/types/file_load_component_objects_test.cljc b/common/test/common_tests/types/file_load_component_objects_test.cljc new file mode 100644 index 0000000000..a1c8bfb0d7 --- /dev/null +++ b/common/test/common_tests/types/file_load_component_objects_test.cljc @@ -0,0 +1,107 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.types.file-load-component-objects-test + (:require + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; Soft-delete snapshots the main instance into the component via +;; `load-component-objects`. When the main id is already missing from +;; the page, the old path called `gsh/move` on nil and indexed by `:id`, +;; storing `{nil {:selrect nil :points []}}` — which later breaks +;; duplicate-file schema validation (#12336). + +(defn- root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- file-data + [page-id component] + {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [])}}} + :components {(:id component) component}}) + +(t/deftest load-component-objects-skips-missing-main + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + component {:id component-id + :name "Orphan" + :path "" + :main-instance-id main-id + :main-instance-page page-id} + result (ctf/load-component-objects + (file-data page-id component) + component) + objects (:objects result)] + + (t/is (not (contains? objects nil)) + "missing main must not create a nil key in :objects") + (t/is (= {} objects) + "missing main yields an empty snapshot, not a placeholder shape"))) + +(t/deftest load-component-objects-keeps-present-main + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + main (cts/setup-shape {:id main-id + :type :frame + :name "Main" + :shapes []}) + component {:id component-id + :name "Present" + :path "" + :main-instance-id main-id + :main-instance-page page-id} + file-data {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [main-id]) + main-id main}}} + :components {component-id component}} + result (ctf/load-component-objects file-data component) + objects (:objects result)] + + (t/is (contains? objects main-id) + "present main is stored under its uuid") + (t/is (not (contains? objects nil)) + "present main never introduces a nil key") + (t/is (= main-id (get-in objects [main-id :id]))))) + +(t/deftest load-component-objects-treats-empty-objects-as-already-loaded + ;; A missing-main soft-delete stores `:objects {}`. Render/library paths + ;; call load-component-objects again on that component. The guard must + ;; treat `{}` as already loaded — otherwise it reloads from the page + ;; (and re-warns). Put the main on the page now so a mistaken re-entry + ;; would populate objects and fail this assertion. + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + main (cts/setup-shape {:id main-id + :type :frame + :name "Main" + :shapes []}) + component {:id component-id + :name "AlreadySnapshotted" + :path "" + :main-instance-id main-id + :main-instance-page page-id + :objects {}} + file-data {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [main-id]) + main-id main}}} + :components {component-id component}} + result (ctf/load-component-objects file-data component)] + + (t/is (= {} (:objects result)) + "empty :objects means already loaded; do not reload from the page"))) diff --git a/common/test/common_tests/types/path_data_test.cljc b/common/test/common_tests/types/path_data_test.cljc index 6c1b957691..de9762a2fc 100644 --- a/common/test/common_tests/types/path_data_test.cljc +++ b/common/test/common_tests/types/path_data_test.cljc @@ -2864,3 +2864,30 @@ result (path/splice-content content sub)] (t/is (path.impl/path-data? result)) (t/is (= (into (vec content) sub) (vec result))))) + +(t/deftest content-close? + (let [content [{:command :move-to :params {:x 10.3 :y 10.7}} + {:command :line-to :params {:x 24.3 :y 10.7}} + {:command :close-path}] + shifted (fn [f] (update-in content [1 :params :x] f))] + (t/testing "params within tolerance are close" + (t/is (path.segment/content-close? content (shifted #(+ % 0.0004))))) + + (t/testing "a param beyond tolerance is not close" + (t/is (not (path.segment/content-close? content (shifted #(+ % 0.01)))))) + + (t/testing "a different command is not close" + (t/is (not (path.segment/content-close? content (assoc-in content [1 :command] :move-to))))) + + (t/testing "a different segment count is not close" + (t/is (not (path.segment/content-close? content (butlast content))))) + + (t/testing "a different param count is not close" + (t/is (not (path.segment/content-close? content (assoc-in content [1 :params] {:x 24.3}))))) + + (t/testing "plain vector and PathData are comparable" + (t/is (path.segment/content-close? content (path.impl/path-data content)))) + + (t/testing "nil equals an empty content, but not a non-empty one" + (t/is (path.segment/content-close? nil [])) + (t/is (not (path.segment/content-close? nil content)))))) diff --git a/common/test/common_tests/types/text_test.cljc b/common/test/common_tests/types/text_test.cljc index ace36f0c4d..e0027b7b16 100644 --- a/common/test/common_tests/types/text_test.cljc +++ b/common/test/common_tests/types/text_test.cljc @@ -368,3 +368,8 @@ (t/is (every? #(= % "14") original-font-sizes)) (t/is (every? #(= % "32") updated-font-sizes)))) + +(t/deftest test-list-paragraph + (t/is (true? (cttx/list-paragraph? {:type "paragraph" :list-style-type "disc"}))) + (t/is (false? (cttx/list-paragraph? {:type "paragraph" :list-style-type "none"}))) + (t/is (false? (cttx/list-paragraph? {:type "paragraph"})))) diff --git a/common/test/common_tests/types/tokens_lib_test.cljc b/common/test/common_tests/types/tokens_lib_test.cljc index 0c06f7d9df..80d8119a1a 100644 --- a/common/test/common_tests/types/tokens_lib_test.cljc +++ b/common/test/common_tests/types/tokens_lib_test.cljc @@ -1893,6 +1893,226 @@ (t/is (= (:type original-ref) (:type imported-ref))) (t/is (= (:value imported-ref) (:value original-ref)))))))) +(t/deftest convert-dtcg-shadow-composite + (let [shadow {:offset-x "1" :offset-y "2" :blur "4" :spread "0" :color "#000000" :inset false}] + (t/testing "DTCG keys" + (t/is (= [shadow] + (ctob/convert-dtcg-shadow-composite + {"offsetX" "1" "offsetY" "2" "blur" "4" "spread" "0" "color" "#000000" "inset" "false"})))) + + (t/testing "legacy keys" + (t/is (= [(assoc shadow :inset true)] + (ctob/convert-dtcg-shadow-composite + [{"x" "1" "y" "2" "blur" "4" "spread" "0" "color" "#000000" "type" "innerShadow"}])))) + + (t/testing "a value already in penpot's format is kept" + (t/is (= [shadow] (ctob/convert-dtcg-shadow-composite [shadow]))) + (t/is (= [shadow] (ctob/convert-dtcg-shadow-composite shadow)))) + + (t/testing "references are kept" + (t/is (= "{shadow.sm}" (ctob/convert-dtcg-shadow-composite "{shadow.sm}")))))) + +(t/deftest import-warnings-for-dropped-data + (let [json {"core" {"color" {"$type" "color" + "$description" "Colors" + "old" {"$value" "#000" + "$deprecated" true + "$extensions" {"com.figma" {"scopes" ["ALL"]}}} + "lab" {"$value" {"colorSpace" "lab" "components" [50 20 30]}} + "oklch" {"$value" {"colorSpace" "oklch" "components" [0.628 0.2577 29.23]}}} + "motion" {"fast" {"$type" "duration" "$value" {"value" 100 "unit" "ms"}}} + "broken" {"$extends" "{missing}"}} + "$metadata" {"tokenSetOrder" ["core"]}} + {:keys [tokens-lib unknown-types unsupported]} (ctob/import-decoded-json json "")] + (t/testing "the tokens are imported without the dropped data" + (t/is (= "#000" (:value (ctob/get-token-by-name tokens-lib "core" "color.old"))))) + (t/testing "colors keep their color space" + (t/is (= "lab(50, 20, 30)" (:value (ctob/get-token-by-name tokens-lib "core" "color.lab")))) + (t/is (= "oklch(0.628, 0.2577, 29.23)" (:value (ctob/get-token-by-name tokens-lib "core" "color.oklch"))))) + (t/is (= {"core.motion.fast" "duration"} unknown-types)) + (t/is (= #{{:property "$deprecated" :path "core.color.old"} + {:property "$extensions" :path "core.color.old"} + {:property "$description" :path "core.color"} + {:property "$extends" :path "core.broken"}} + (set unsupported))))) + +(t/deftest import-warnings-for-dropped-tokens + (let [json {"core" {"$root" {"$value" "1" "$type" "number"} + "color" {"$type" "color" + "ok" {"$value" "#fff"} + "odd" {"$value" {"colorSpace" "nope" "components" [1 0 0]}} + "stray" 1} + "untyped" {"$value" "4"} + "a" {"$extends" "{b}"} + "b" {"$extends" "{a}"}} + "$metadata" {"tokenSetOrder" ["core"]}} + {:keys [tokens-lib unsupported]} (ctob/import-decoded-json json "")] + (t/testing "the tokens that can be read are imported" + (t/is (= #{"color.ok"} (set (keys (ctob/get-all-tokens-map tokens-lib)))))) + (t/is (= #{{:property "$root" :path "core.$root"} + {:property "$value" :path "core.color.odd"} + {:property "$value" :path "core.color.stray"} + {:property "$type" :path "core.untyped"} + {:property "$extends" :path "core.a"} + {:property "$extends" :path "core.b"}} + (set unsupported))))) + +(t/deftest import-warnings-for-group-extensions + (let [json {"core" {"color" {"$type" "color" + "$extensions" true + "primary" {"$value" "#fff" "$extensions" "figma"}} + "size" {"$type" "number" + "$extensions" {} + "sm" {"$value" "4"}}} + "$metadata" {"tokenSetOrder" ["core"]}} + {:keys [tokens-lib unsupported]} (ctob/import-decoded-json json "")] + (t/is (= "#fff" (:value (ctob/get-token-by-name tokens-lib "core" "color.primary")))) + (t/testing "a malformed group `$extensions` is reported, an empty one is not" + (t/is (= [{:property "$extensions" :path "core.color"}] unsupported))) + (t/testing "the reader drops a malformed token `$extensions` before penpot sees it" + (t/is (not-any? #(= "core.color.primary" (:path %)) unsupported))))) + +(t/deftest import-theme-set-statuses + (let [json {"core" {"a" {"$value" "1" "$type" "number"}} + "light" {"b" {"$value" "2" "$type" "number"}} + "dark" {"b" {"$value" "3" "$type" "number"}} + "$themes" [{"name" "light" "group" "mode" + "selectedTokenSets" {"core" "source" "light" "enabled" "dark" "disabled"}} + {"name" "dark" "group" "mode" + "selectedTokenSets" {"core" "source" "dark" "enabled"}}] + "$metadata" {"tokenSetOrder" ["core" "light" "dark"] + "activeThemes" ["mode/light" "mode/dark"]}} + lib (ctob/parse-decoded-json json "")] + (t/testing "source and enabled sets are part of the theme, disabled ones are not" + (t/is (= #{"core" "light"} (:sets (ctob/get-theme-by-name lib "mode" "light"))))) + (t/testing "activating a theme deactivates the other themes of its group" + (t/is (= #{"mode/dark"} (disj (ctob/get-legacy-active-themes lib) ctob/hidden-theme-path)))))) + +(t/deftest import-active-top-level-theme + (let [json {"core" {"a" {"$value" "1" "$type" "number"}} + "extra" {"b" {"$value" "2" "$type" "number"}} + "$themes" [{"name" "solo" "selectedTokenSets" {"core" "enabled"}}] + "$metadata" {"activeThemes" ["solo"] "activeSets" ["extra"]}} + lib (ctob/parse-decoded-json json "")] + (t/testing "a theme without group is active by its path, next to the hidden theme" + (t/is (= #{"/solo" ctob/hidden-theme-path} (ctob/get-legacy-active-themes lib))) + (t/is (= #{"extra"} (:sets (ctob/get-theme lib ctob/hidden-theme-id))))) + (t/testing "an export read back without encoding keeps what is active" + (let [lib' (ctob/parse-multi-set-dtcg-json (ctob/export-dtcg-json lib))] + (t/is (= (ctob/get-legacy-active-themes lib) (ctob/get-legacy-active-themes lib'))) + (t/is (= #{"core" "extra"} (:sets (ctob/get-theme lib' ctob/hidden-theme-id)))))))) + +(t/deftest export-default-profile-keeps-values-and-type-names + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set + :name "core" + :tokens {"radius.sm" (ctob/make-token {:name "radius.sm" :type :border-radius :value "4"}) + "size.rem" (ctob/make-token {:name "size.rem" :type :sizing :value "1.5rem"}) + "weight.bold" (ctob/make-token {:name "weight.bold" :type :font-weight :value "700"}) + "color.brand" (ctob/make-token {:name "color.brand" :type :color :value "oklch(0.628, 0.2577, 29.23)"}) + "typo.body" (ctob/make-token {:name "typo.body" :type :typography + :value {:font-weight "700" :font-size "16px" :line-height "1.5"}})}))) + tokens-status (ctos/make-tokens-status) + result (ctob/export-dtcg-json tokens-lib tokens-status)] + (t/testing "the :default profile is the export without options" + (t/is (= result (ctob/export-dtcg-json tokens-lib tokens-status {:profile :default}))) + (t/is (= (ctob/export-dtcg-multi-file tokens-lib tokens-status) + (ctob/export-dtcg-multi-file tokens-lib tokens-status {:profile :default})))) + (t/testing "values stay strings and types keep their names" + (t/is (= {"radius" {"sm" {"$value" "4" "$type" "borderRadius" "$description" ""}} + "size" {"rem" {"$value" "1.5rem" "$type" "sizing" "$description" ""}} + "weight" {"bold" {"$value" "700" "$type" "fontWeights" "$description" ""}} + "color" {"brand" {"$value" "oklch(0.628, 0.2577, 29.23)" "$type" "color" "$description" ""}} + "typo" {"body" {"$value" {"fontWeights" "700" "fontSizes" "16px" "lineHeights" "1.5"} + "$type" "typography" + "$description" ""}}} + (get result "core")))))) + +(t/deftest export-dtcg-2025-values + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set + :name "core" + :tokens {"radius.sm" (ctob/make-token {:name "radius.sm" :type :border-radius :value "4"}) + "size.rem" (ctob/make-token {:name "size.rem" :type :sizing :value "1.5rem"}) + "space.md" (ctob/make-token {:name "space.md" :type :spacing :value "{radius.sm} * 2"}) + "weight.bold" (ctob/make-token {:name "weight.bold" :type :font-weight :value "700"}) + "color.brand" (ctob/make-token {:name "color.brand" :type :color :value "oklch(0.628, 0.2577, 29.23)"}) + "shadow.sm" (ctob/make-token {:name "shadow.sm" :type :shadow + :value [{:offset-x "0" :offset-y "2px" :blur "4px" :spread "0" :color "#000000" :inset false}]})}))) + tokens-status (ctos/make-tokens-status) + result (ctob/export-dtcg-json tokens-lib tokens-status {:profile :dtcg})] + (t/testing "literals become DTCG values and types" + (t/is (= {"radius" {"sm" {"$value" {"value" 4 "unit" "px"} + "$type" "dimension" + "$description" ""}} + "size" {"rem" {"$value" {"value" 1.5 "unit" "rem"} + "$type" "dimension" + "$description" ""}} + "space" {"md" {"$value" "{radius.sm} * 2" + "$type" "dimension" + "$description" ""}} + "weight" {"bold" {"$value" 700 + "$type" "fontWeight" + "$description" ""}} + "color" {"brand" {"$value" {"colorSpace" "oklch" + "components" [0.628 0.2577 29.23] + "hex" "#ff0000"} + "$type" "color" + "$description" ""}} + "shadow" {"sm" {"$value" [{"offsetX" {"value" 0 "unit" "px"} + "offsetY" {"value" 2 "unit" "px"} + "blur" {"value" 4 "unit" "px"} + "spread" {"value" 0 "unit" "px"} + "color" {"colorSpace" "srgb" "components" [0 0 0] "hex" "#000000"} + "inset" false}] + "$type" "shadow" + "$description" ""}}} + (get result "core")))) + (t/testing "a multi-file export writes the same set" + (t/is (= (get result "core") + (get (ctob/export-dtcg-multi-file tokens-lib tokens-status {:profile :dtcg}) "core.json")))))) + +(t/deftest export-dtcg-2025-round-trip + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set + :name "core" + :tokens {"space.sm" (ctob/make-token {:name "space.sm" :type :spacing :value "4px"}) + "space.md" (ctob/make-token {:name "space.md" :type :spacing :value "{space.sm} * 2" + :description "Twice the small space"}) + "size.rem" (ctob/make-token {:name "size.rem" :type :sizing :value "1.5rem"}) + "color.brand" (ctob/make-token {:name "color.brand" :type :color :value "#ff0000"}) + "color.oklch" (ctob/make-token {:name "color.oklch" :type :color :value "oklch(0.628, 0.2577, 29.23)"}) + "color.alpha" (ctob/make-token {:name "color.alpha" :type :color :value "#0000001a"}) + "family.body" (ctob/make-token {:name "family.body" :type :font-family :value ["Inter" "sans-serif"]}) + "typo.body" (ctob/make-token {:name "typo.body" :type :typography + :value {:font-weight "bold" :font-size "16px" + :letter-spacing "0.05em" :font-family ["Inter"]}}) + "typo.ref" (ctob/make-token {:name "typo.ref" :type :typography :value "{typo.body}"}) + "shadow.sm" (ctob/make-token {:name "shadow.sm" :type :shadow + :value [{:offset-x "0px" :offset-y "2px" :blur "4px" :spread "0px" + :color "#0000001a" :inset false}]})})) + (ctob/add-set (ctob/make-token-set :name "dark")) + (ctob/add-theme (ctob/make-token-theme :id (thi/new-id! :mode-light) + :name "light" + :group "mode" + :external-id "mode-light" + :sets #{"core"}))) + tokens-status (cfo/toggle-theme-active (ctos/make-tokens-status) tokens-lib (thi/id :mode-light)) + exported (ctob/export-dtcg-json tokens-lib tokens-status {:profile :dtcg}) + imported (ctob/parse-decoded-json exported "")] + (t/testing "penpot reads its own export back, spacing and sizing as dimensions" + (doseq [token (vals (ctob/get-tokens- (ctob/get-set-by-name tokens-lib "core")))] + (let [token' (ctob/get-token-by-name imported "core" (:name token))] + (t/is (= (get {:spacing :dimensions :sizing :dimensions} (:type token) (:type token)) + (:type token')) + (:name token)) + (t/is (= (:value token) (:value token')) (:name token)) + (t/is (= (:description token) (:description token')) (:name token))))) + (t/testing "sets, themes and what is active read back" + (t/is (= ["core" "dark"] (ctob/get-set-names imported))) + (t/is (= #{"core"} (:sets (ctob/get-theme-by-name imported "mode" "light")))) + (t/is (= #{"mode/light"} (disj (ctob/get-legacy-active-themes imported) ctob/hidden-theme-path)))))) + (t/deftest token-name-path-exists?-test (t/is (true? (ctob/token-name-path-exists? "border-radius" {"border-radius" {"sm" {:name "sm"}}}))) (t/is (true? (ctob/token-name-path-exists? "border-radius" {"border-radius" {:name "sm"}}))) diff --git a/common/test/common_tests/variant_test.cljc b/common/test/common_tests/variant_test.cljc index 0dea2aa947..bcccea9f1c 100644 --- a/common/test/common_tests/variant_test.cljc +++ b/common/test/common_tests/variant_test.cljc @@ -119,6 +119,12 @@ (t/is (= (ctv/compare-properties [props-1 props-2 props-3] "&") [{:name "border" :value "&"} {:name "color" :value "&"}]))))) +(t/deftest compare-properties-preserves-order-past-eight + (let [names (mapv #(str "Property " %) (range 1 10)) + props1 (mapv (fn [n] {:name n :value "a"}) names) + props2 (mapv (fn [n] {:name n :value "b"}) names)] + (t/is (= names (mapv :name (ctv/compare-properties [props1 props2])))))) + (t/deftest check-belong-same-variant (let [components-1 [{:variant-id "a variant"} {:variant-id "a variant"}] diff --git a/docker/devenv/Dockerfile b/docker/devenv/Dockerfile index 491c2a04e9..55540a934f 100644 --- a/docker/devenv/Dockerfile +++ b/docker/devenv/Dockerfile @@ -1,4 +1,4 @@ -FROM dhi.io/debian-base:trixie-debian13-dev@sha256:94571f3dc07fbc7166f6f1cbf1fe068a7647198ee2554e6b90d59a68cc8b053c AS base +FROM dhi.io/debian-base:trixie-debian13-dev@sha256:f90fd43cdf3dd94840b4dc3a2a39fd2a08578e40e7895535e2e20a1275ba6013 AS base ENV LANG='C.UTF-8' \ LC_ALL='C.UTF-8' \ @@ -66,8 +66,8 @@ RUN set -ex; \ FROM base AS setup-node -ENV NODE_VERSION=v24.22.2 \ - PNPM_VERSION=12.18.0 \ +ENV NODE_VERSION=v24.22.3 \ + PNPM_VERSION=12.21.0 \ PATH=/opt/node/bin:$PATH RUN set -eux; \ @@ -76,16 +76,16 @@ RUN set -eux; \ aarch64|arm64) \ OPENSSL_ARCH='linux-aarch64'; \ BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-arm64.tar.gz"; \ - NODE_SHA256='95c264150aceb7f3437a29fd6984924f5f8d1511ea3cbd6c508b83f6c555dae2'; \ + NODE_SHA256='261af2af250dfd7e9f25cf151d1b2713a0adf8f10a9ad116c4947f68627affe8'; \ PNPM_ARCH='arm64'; \ - PNPM_SHA256='700bd1aa4028465b80e62b7c5bec8455be60bbafaf53a3262a4ab8e2057712ac'; \ + PNPM_SHA256='6c5913a48c93b3a556b71c3bd9bf12c6bf9fbfbbbd2dcbd1d52a85c539751b9e'; \ ;; \ amd64|x86_64) \ OPENSSL_ARCH='linux-x86_64'; \ BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.gz"; \ - NODE_SHA256='361718c444e49af2ca5a110c83ba6a8901143f23ab285ebbef0341f391f482a9'; \ + NODE_SHA256='061ec3e3062115705b56b51fa9cc330ab50d34b6969387d4e4b1a50c6809acc2'; \ PNPM_ARCH='x64'; \ - PNPM_SHA256='3726060e236c0db3e7124c0a5af123c840ba0cb76682149b63ea0baf290017c2'; \ + PNPM_SHA256='84c4773bcf4a217f4083cd7b2a91e4cee6103cf42426c1282ef37d308f55a410'; \ ;; \ *) \ echo "Unsupported arch: ${ARCH}"; \ diff --git a/docker/devenv/docker-compose.main.yml b/docker/devenv/docker-compose.main.yml index c55330a80a..6836861354 100644 --- a/docker/devenv/docker-compose.main.yml +++ b/docker/devenv/docker-compose.main.yml @@ -101,5 +101,11 @@ services: - SERENA_UPDATE_VERSION=2.1.0 - SHADOW_SERVER_URL=${SHADOW_SERVER_URL} + # Frontend dev devenv switch: makes the frontend compile with + # PENPOT_DEVENV=true so the admin-console is disabled by default + # (app.config/devenv-mode). `enable-admin-console` in config.js + # re-enables it. + - PENPOT_DEVENV=true + networks: - default diff --git a/docker/imagemagick/Dockerfile b/docker/imagemagick/Dockerfile index bc28f7570c..3f7637bc73 100644 --- a/docker/imagemagick/Dockerfile +++ b/docker/imagemagick/Dockerfile @@ -1,4 +1,4 @@ -FROM dhi.io/debian-base:trixie-debian13-dev@sha256:94571f3dc07fbc7166f6f1cbf1fe068a7647198ee2554e6b90d59a68cc8b053c AS build +FROM dhi.io/debian-base:trixie-debian13-dev@sha256:f90fd43cdf3dd94840b4dc3a2a39fd2a08578e40e7895535e2e20a1275ba6013 AS build LABEL maintainer="Penpot " ENV LANG='C.UTF-8' \ @@ -74,7 +74,7 @@ RUN set -ex; \ rm -rf /var/lib/apt/lists/* -FROM dhi.io/debian-base:trixie-debian13@sha256:5c9d8c2c90a0a4672f44a42a485c99e1b95f4bfad5c8ef49934f34b7ffb6e23b AS image +FROM dhi.io/debian-base:trixie-debian13@sha256:e3e8512202bb7d8a7093905a3615408784beba909b3cd61d3bb49367f5cfa448 AS image LABEL maintainer="Penpot " ENV LANG='C.UTF-8' \ diff --git a/docker/images/Dockerfile.backend b/docker/images/Dockerfile.backend index 201378751d..1e287b7080 100644 --- a/docker/images/Dockerfile.backend +++ b/docker/images/Dockerfile.backend @@ -1,4 +1,4 @@ -FROM dhi.io/debian-base:trixie-debian13-dev@sha256:94571f3dc07fbc7166f6f1cbf1fe068a7647198ee2554e6b90d59a68cc8b053c AS build +FROM dhi.io/debian-base:trixie-debian13-dev@sha256:f90fd43cdf3dd94840b4dc3a2a39fd2a08578e40e7895535e2e20a1275ba6013 AS build LABEL maintainer="Penpot " ENV LANG='C.UTF-8' \ @@ -53,7 +53,7 @@ RUN set -eux; \ --output /opt/jre; -FROM dhi.io/debian-base:trixie-debian13-dev@sha256:94571f3dc07fbc7166f6f1cbf1fe068a7647198ee2554e6b90d59a68cc8b053c AS image +FROM dhi.io/debian-base:trixie-debian13-dev@sha256:f90fd43cdf3dd94840b4dc3a2a39fd2a08578e40e7895535e2e20a1275ba6013 AS image LABEL maintainer="Penpot " ENV LANG='C.UTF-8' \ diff --git a/docker/images/Dockerfile.exporter b/docker/images/Dockerfile.exporter index 07c1c470c7..3d3b97b3de 100644 --- a/docker/images/Dockerfile.exporter +++ b/docker/images/Dockerfile.exporter @@ -1,4 +1,4 @@ -FROM dhi.io/node:24.22.2-debian13-dev@sha256:f6ba5e2f381fd46ef88e15ea0827025bbcc95128120211efcffa7860c0a4599c +FROM dhi.io/node:24.22.3-debian13-dev@sha256:dee4bb11f4cb32ad31000287af8795f772ac801338a166a0e9fbbadd9674a2f8 LABEL maintainer="Penpot " ENV LANG=en_US.UTF-8 \ @@ -17,6 +17,7 @@ RUN set -ex; \ locales \ ca-certificates \ gzip \ + tini \ ; \ apt-get clean; \ rm -rf /var/lib/apt/lists/*; \ @@ -95,17 +96,17 @@ WORKDIR /opt/penpot/exporter # pnpm ships as a system binary (same tarball + SHA pin as # docker/devenv/Dockerfile); the generated ./setup expects it on PATH. # Corepack is gone from Node 25+, so nothing here may use it. -ARG PNPM_VERSION=12.18.0 +ARG PNPM_VERSION=12.21.0 RUN set -eux; \ ARCH="$(dpkg --print-architecture)"; \ case "${ARCH}" in \ aarch64|arm64) \ PNPM_ARCH='arm64'; \ - PNPM_SHA256='700bd1aa4028465b80e62b7c5bec8455be60bbafaf53a3262a4ab8e2057712ac'; \ + PNPM_SHA256='6c5913a48c93b3a556b71c3bd9bf12c6bf9fbfbbbd2dcbd1d52a85c539751b9e'; \ ;; \ amd64|x86_64) \ PNPM_ARCH='x64'; \ - PNPM_SHA256='3726060e236c0db3e7124c0a5af123c840ba0cb76682149b63ea0baf290017c2'; \ + PNPM_SHA256='84c4773bcf4a217f4083cd7b2a91e4cee6103cf42426c1282ef37d308f55a410'; \ ;; \ *) \ echo "Unsupported arch: ${ARCH}"; \ @@ -126,4 +127,5 @@ RUN ./setup && chown -R penpot:penpot /opt/penpot/exporter USER penpot:penpot +ENTRYPOINT ["/usr/bin/tini", "-s", "--"] CMD ["node", "app.js"] diff --git a/docker/images/Dockerfile.frontend b/docker/images/Dockerfile.frontend index dec914780f..191f0ccda6 100644 --- a/docker/images/Dockerfile.frontend +++ b/docker/images/Dockerfile.frontend @@ -1,4 +1,4 @@ -FROM dhi.io/nginx:1.31.8-debian13-dev@sha256:e8db45540bf5851e14bd6c1f4a624bf850dafea958a06b8c84186003007ae7ca +FROM dhi.io/nginx:1.31.8-debian13-dev@sha256:2168f2f90d8dd61baf5ac0359d92f601f988ba7d7ef669dc778cd7a887130295 LABEL maintainer="Penpot " ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/images/Dockerfile.mcp b/docker/images/Dockerfile.mcp index e17cd311e5..7d2d5987cf 100644 --- a/docker/images/Dockerfile.mcp +++ b/docker/images/Dockerfile.mcp @@ -1,4 +1,4 @@ -FROM dhi.io/node:24.22.2-debian13-dev@sha256:f6ba5e2f381fd46ef88e15ea0827025bbcc95128120211efcffa7860c0a4599c AS build +FROM dhi.io/node:24.22.3-debian13-dev@sha256:dee4bb11f4cb32ad31000287af8795f772ac801338a166a0e9fbbadd9674a2f8 AS build LABEL maintainer="Penpot " ENV DEBIAN_FRONTEND=noninteractive @@ -11,7 +11,7 @@ WORKDIR /opt/penpot/mcp # pnpm ships as a system binary (same tarball + SHA pin as # docker/devenv/Dockerfile); ./setup expects it on PATH. # Corepack is gone from Node 25+, so nothing here may use it. -ARG PNPM_VERSION=12.18.0 +ARG PNPM_VERSION=12.21.0 RUN set -eux; \ apt-get -qq update; \ apt-get -qqy --no-install-recommends install curl ca-certificates gzip; \ @@ -20,11 +20,11 @@ RUN set -eux; \ case "${ARCH}" in \ aarch64|arm64) \ PNPM_ARCH='arm64'; \ - PNPM_SHA256='700bd1aa4028465b80e62b7c5bec8455be60bbafaf53a3262a4ab8e2057712ac'; \ + PNPM_SHA256='6c5913a48c93b3a556b71c3bd9bf12c6bf9fbfbbbd2dcbd1d52a85c539751b9e'; \ ;; \ amd64|x86_64) \ PNPM_ARCH='x64'; \ - PNPM_SHA256='3726060e236c0db3e7124c0a5af123c840ba0cb76682149b63ea0baf290017c2'; \ + PNPM_SHA256='84c4773bcf4a217f4083cd7b2a91e4cee6103cf42426c1282ef37d308f55a410'; \ ;; \ *) \ echo "Unsupported arch: ${ARCH}"; \ @@ -42,7 +42,7 @@ RUN set -eux; \ RUN ./setup -FROM dhi.io/node:24.22.2-debian13@sha256:7f8db65bead5b4ad1bdb62e689e6ccd3c5ad9c9f03fd56db530ade0e3d3ea982 AS image +FROM dhi.io/node:24.22.3-debian13@sha256:84adaf3bdd9fa350e753078db2928a1584e481e11225d14f96014b17ffc96c5a AS image LABEL maintainer="Penpot " ENV LANG=en_US.UTF-8 \ diff --git a/docker/images/Dockerfile.media-processor b/docker/images/Dockerfile.media-processor index 7e6cddaa86..f81a983399 100644 --- a/docker/images/Dockerfile.media-processor +++ b/docker/images/Dockerfile.media-processor @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM dhi.io/node:24.22.2-debian13-dev@sha256:f6ba5e2f381fd46ef88e15ea0827025bbcc95128120211efcffa7860c0a4599c +FROM dhi.io/node:24.22.3-debian13-dev@sha256:dee4bb11f4cb32ad31000287af8795f772ac801338a166a0e9fbbadd9674a2f8 LABEL maintainer="Penpot " ENV LANG=en_US.UTF-8 \ @@ -58,7 +58,7 @@ WORKDIR /opt/penpot/media-processor # pnpm ships as a system binary (same tarball + SHA pin as # docker/devenv/Dockerfile); ./setup expects it on PATH. # Corepack is gone from Node 25+, so nothing here may use it. -ARG PNPM_VERSION=12.18.0 +ARG PNPM_VERSION=12.21.0 RUN set -eux; \ apt-get -qq update; \ apt-get -qqy --no-install-recommends install gzip; \ @@ -67,11 +67,11 @@ RUN set -eux; \ case "${ARCH}" in \ aarch64|arm64) \ PNPM_ARCH='arm64'; \ - PNPM_SHA256='700bd1aa4028465b80e62b7c5bec8455be60bbafaf53a3262a4ab8e2057712ac'; \ + PNPM_SHA256='6c5913a48c93b3a556b71c3bd9bf12c6bf9fbfbbbd2dcbd1d52a85c539751b9e'; \ ;; \ amd64|x86_64) \ PNPM_ARCH='x64'; \ - PNPM_SHA256='3726060e236c0db3e7124c0a5af123c840ba0cb76682149b63ea0baf290017c2'; \ + PNPM_SHA256='84c4773bcf4a217f4083cd7b2a91e4cee6103cf42426c1282ef37d308f55a410'; \ ;; \ *) \ echo "Unsupported arch: ${ARCH}"; \ diff --git a/docker/images/Dockerfile.storybook b/docker/images/Dockerfile.storybook index de670de369..25a007a82a 100644 --- a/docker/images/Dockerfile.storybook +++ b/docker/images/Dockerfile.storybook @@ -1,4 +1,4 @@ -FROM dhi.io/nginx:1.31.8-alpine3.24@sha256:34c5f1f63c57c0fb1b52505aecbb213520e15918dc822eb16371edff5183cbd1 +FROM dhi.io/nginx:1.31.8-alpine3.24@sha256:ad079e1004e4ec4ec14dfc29612d542161a80bc4bf9735b8f6303dc45e588ce0 LABEL maintainer="Penpot " ARG BUNDLE_PATH="./bundle-storybook/" diff --git a/docs/.vscode/settings.json b/docs/.vscode/settings.json deleted file mode 100644 index 1e915ede10..0000000000 --- a/docs/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "editor.rulers": [ - 80 - ] -} diff --git a/docs/contributing-guide/code-contributions/index.njk b/docs/contributing-guide/code-contributions/index.njk index 2c9ef6b517..4c62007edd 100644 --- a/docs/contributing-guide/code-contributions/index.njk +++ b/docs/contributing-guide/code-contributions/index.njk @@ -18,10 +18,11 @@ Thinking of contributing to Penpot core but not sure where to start? We’ve mad

Pull requests

If you want propose a change or bug fix with the Pull-Request system firstly you should carefully read the DCO section and format your commits accordingly.

-

If you intend to fix a bug it's fine to submit a pull request right away but we still recommend to file an issue detailing what you're fixing. This is helpful in case we don't accept that specific fix but want to keep track of the issue.

+

If you intend to fix a bug please make sure it's attached to an existing issue. Pull requests that have no issue attached will be automatically rejected. Pull requests attached to issues labeled as needs triage or internal will be autorejected as well.

If you want to implement or start working in a new feature, please open a question / discussion issue for it. No pull-request will be accepted without previous chat about the changes, independently if it is a new feature, already planned feature or small quick win.

+

Please do not submit a pull request for an issue that has already been assigned to someone else. It there is no recent activity, feel free to ask for it be assigned to you.

If is going to be your first pull request, You can learn how from this free video series.

-

We will use the easy fix mark for tag for indicate issues that are easy for beginners.

+

We will use the good first issue label to mark issues that are easy to fix for beginners.

Commit message guidelines

We have very precise rules over how our git commit messages can be formatted.

@@ -35,7 +36,7 @@ Thinking of contributing to Penpot core but not sure where to start? We’ve mad [footer] -

Where type is:

+

Where type is:

  • bug :bug: a commit that fixes a bug
  • sparkles :sparkles: a commit that adds an improvement
  • diff --git a/docs/mcp/index.md b/docs/mcp/index.md index 6b55a3f816..d85bccc5bb 100644 --- a/docs/mcp/index.md +++ b/docs/mcp/index.md @@ -65,7 +65,7 @@ There are three key pieces: * **MCP server**: a service that exposes tools to your AI client. It receives requests from the client and forwards them to Penpot. * **MCP plugin in Penpot**: a plugin that runs inside Penpot and connects your open file to the MCP server. It allows the server to access that file and its pages. -* **MCP client**: the tool where you write prompts (Cursor, Claude Code, Copilot-style tools, etc.). It connects to the MCP server using a server URL and an MCP key (or your active Penpot session in the current local setup). +* **MCP client**: the tool where you write prompts (Cursor, Claude Code, Copilot-style tools, etc.). It connects to the MCP server using a server URL and an MCP key. In remote MCP, the client can get the key by asking you to sign in to Penpot, or you can add it to the URL yourself. The local setup uses your active Penpot session instead. ![How the AI client, Penpot MCP server, plugin, and design file connect](/img/mcp/mcp-flow.webp) @@ -73,7 +73,7 @@ There are three key pieces: Some important concepts for users: * **Integrations page**: MCP is configured under **Your account → Integrations → MCP Server**. Here you enable or disable MCP, get the server URL and manage the MCP key. -* **MCP key**: a personal, non-recoverable token that authenticates your AI client with the MCP server. Only one key can exist per user at a time. This is used by the remote MCP setup. +* **MCP key**: a personal, non-recoverable token that authenticates your AI client with the MCP server. Only one key can exist per user at a time. This is used by the remote MCP setup. Clients connected through Penpot sign-in use this same key. * **MCP session**: a connection from a Penpot tab to the MCP server, identified by a short session ID derived from the tab's user session. Several tabs can connect independently, including tabs showing the same file. * **Current page**: the page open in the selected session. Switching browser focus to another tab does not change which session the agent uses. @@ -130,7 +130,29 @@ Because **remote MCP** does not expose local file-system access: If you just want to try Penpot AI workflows quickly through the MCP, follow this path for the **hosted (remote) MCP server**. -### Remote MCP in 5 steps +### Remote MCP in 3 steps + +Use this path if your MCP client can sign in to remote MCP servers (OAuth), for example Claude Code, Codex, or Claude.ai custom connectors. It works in the official SaaS and in self-hosted instances that [enable it](/technical-guide/configuration/#mcp-sign-in). + +1. #### Add the server to your MCP client + Use the server URL without a key: + + ``` + https:///mcp/stream + ``` + + In the official SaaS the domain is `design.penpot.app`. See [Connect your MCP client](#connect-your-mcp-client) for client-specific commands. + +2. #### Sign in and allow access + Your client opens Penpot in the browser. Sign in if needed, check the client name and your account, and choose **Allow access**. Penpot enables MCP and creates your MCP key if you do not have one yet. If you already have a key, Penpot keeps it. Your client stores the key; you never need to copy it. + +3. #### Open a Penpot file and connect MCP + In Penpot, open a design file and choose **MCP → Connect** in the toolbar. Allowing access does not connect a file automatically. + + +### Remote MCP with an MCP key + +Use this path if your client cannot sign in to remote MCP servers, or if your Penpot instance does not offer sign-in. 1. #### Enable MCP in Penpot Go to **Your account → Integrations → MCP Server** and enable the feature. @@ -226,7 +248,7 @@ You can use Penpot MCP server in two main ways: * Hosted for you (no need to run anything on your machine). * Best option for most users, simpler installation and fewer moving parts. * Does **not** have privileged access to your local file system, it can only work with what Penpot exposes (design files, libraries, tokens, etc.). - * The **server URL** is provided in **Your account → Integrations → MCP Server** and looks like: + * The **server URL** is `https:///mcp/stream`. Clients that sign in to Penpot use it as it is. Other clients use the URL shown in **Your account → Integrations → MCP Server**, which adds your key: * `https:///mcp/stream?userToken=YOUR_MCP_KEY` * The domain depends on the Penpot installation. In the official SaaS it will be `design.penpot.app`. * **Local MCP server** @@ -239,10 +261,17 @@ You can use Penpot MCP server in two main ways: ## Connect your MCP client -* **Remote MCP** +* **Remote MCP, signing in to Penpot** + * URL: `https:///mcp/stream`, with no key. + * Add the server to your client, then start the sign-in. Penpot opens in the browser and asks you to allow access. For example: + * Claude Code: run `claude mcp add --transport http penpot https:///mcp/stream`, then run `/mcp`, select `penpot` and choose **Authenticate**. + * Codex: run `codex mcp add penpot --url https:///mcp/stream`, then `codex mcp login penpot`. + * Claude.ai and Claude Desktop: add a custom [connector](http://claude.ai/customize/connectors) with the URL above and choose **Connect**. + * Other clients: follow your client's instructions for remote MCP servers that use OAuth. +* **Remote MCP, with the MCP key in the URL** * URL (copy from your Penpot account overview): `https:///mcp/stream?userToken=YOUR_MCP_KEY` * Configure your MCP client with `npx -y add-mcp -g -n penpot ` or by following your client's instructions for connecting to a remote MCP server. - For Claude Desktop, the server can be added as a custom [connector](http://claude.ai/customize/connectors). + * Use this option for clients that cannot sign in, or if your Penpot instance does not offer sign-in. * **Local MCP** * Configure your MCP client with `npx -y add-mcp -g -n penpot http://localhost:4401/mcp` (adjust the port if you changed `PENPOT_MCP_SERVER_PORT`) or by following your client's instructions for connecting to an HTTP MCP server. @@ -273,6 +302,8 @@ Remote MCP is the easiest way to start using AI agents with Penpot. It's hosted ### Install and activate +If your client signs in to Penpot, you can skip this section: allowing access from the client enables MCP and creates your key. See [Remote MCP in 3 steps](#remote-mcp-in-3-steps). + 1. Open **Your account → Integrations**. 2. In the **MCP Server** section, read the short description to confirm that feature is available for your account. 3. Use the **Status** toggle to enable MCP Server. Penpot remembers this state per user across sessions. You still need to connect each tab explicitly. @@ -286,8 +317,16 @@ Remote MCP is the easiest way to start using AI agents with Penpot. It's hosted ### Connect -For remote mode, use the URL shown in **Your account → Integrations → MCP Server**, which includes your `userToken`. -See the section **Connect your MCP client** for details on how to connect. +There are two ways to connect a client in remote mode: + +* **Sign in to Penpot.** Add `https:///mcp/stream` to your client without a key. When the client starts the sign-in, Penpot opens in the browser. If you are not logged in, Penpot asks you to log in first. The approval page shows the client that is asking for access and the account you are signed in with. Choose **Allow access** to connect, or **Cancel** to stop without changing anything. +* **Use the MCP key.** Use the URL shown in **Your account → Integrations → MCP Server**, which includes your `userToken`. + +Allowing access enables MCP for your account and gives the client your MCP key. Penpot creates a key only if you do not have a valid one; an existing key stays the same, so clients that already use it keep working. The client receives the key directly from Penpot. It never appears in the browser address bar. + +If the approval page says the request expired or was already used, start the connection again from your MCP client. + +See the section [Connect your MCP client](#connect-your-mcp-client) for details on how to connect. ### Setup videos @@ -355,16 +394,16 @@ If no key exists and you enable MCP, a modal guides you through creating one. Th **Regenerate MCP key** -The **Regenerate MCP key** action immediately revokes the current key. A warning explains that any client using the old key will stop working until you update its configuration. After regenerating, Penpot shows the new key and an updated configuration snippet with the new `userToken` that you need to copy into your MCP client configuration. +The **Regenerate MCP key** action immediately revokes the current key. A warning explains that any client using the old key will stop working until you update its configuration. This includes clients connected by signing in to Penpot: sign in again from those clients to give them the new key. After regenerating, Penpot shows the new key and an updated configuration snippet with the new `userToken` that you need to copy into your MCP client configuration. **Expired key state** -If your key is expired, an error block explains that the connection cannot be established until you regenerate the key. Regenerating the key clears the error state. +If your key is expired, an error block explains that the connection cannot be established until you regenerate the key. Regenerating the key clears the error state. Signing in again from a client also replaces an expired key with a new one. Security recommendations to highlight in the Help Center: * Treat your MCP key like a password or access token, do not share it in screenshots or code samples. -* Regenerate the key if you suspect it may have leaked. +* Regenerate the key if you suspect it may have leaked. Clients connected by signing in to Penpot hold the same key, so regenerating it disconnects them too. * Disabling MCP Server disconnects your integrated sessions. Disconnecting one tab stops access through that connection; other connected sessions remain available. *** diff --git a/docs/package.json b/docs/package.json index 4fc709e695..563fed2b8e 100644 --- a/docs/package.json +++ b/docs/package.json @@ -39,5 +39,5 @@ "markdown-it-anchor": "^10.0.0", "markdown-it-plantuml": "^1.4.1" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml index fda4bd4ad5..a509b16520 100644 --- a/docs/pnpm-lock.yaml +++ b/docs/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 4759855e14..4cda6efef4 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -56,7 +56,7 @@ Penpot uses anonymous telemetries from the self-hosted instances to improve the Consider sharing these anonymous telemetries enabling the corresponding flag: ```bash -PENPOT_FLAGS: [...] enable-telemetries +PENPOT_FLAGS: [...] enable-telemetry ``` ## Registration and authentication @@ -270,7 +270,11 @@ PENPOT_OIDC_USER_INFO_SOURCE: __Since version 2.1.2__ Allows users to register and login with oidc without having to previously -register with another method. +register with another method. It also works with password login disabled +(`disable-login-with-password`): OIDC registration only needs this flag, +not password login. An invitation link also works as a substitute for +this flag: an invited user can register with OIDC even with both +registration flags disabled. ```bash PENPOT_FLAGS: [...] enable-oidc-registration @@ -757,6 +761,25 @@ POSTGRES_USER: penpot POSTGRES_PASSWORD: penpot ``` +#### Connection pool + +The backend keeps a pool of database connections. Its size can be tuned with: + +```bash +# Backend +PENPOT_DATABASE_MIN_POOL_SIZE: 0 +PENPOT_DATABASE_MAX_POOL_SIZE: 60 +``` + +The values above are the defaults. The pool keeps at least `MIN_POOL_SIZE` idle +connections open and never opens more than `MAX_POOL_SIZE`, growing only under load +and closing idle connections after two minutes. A minimum larger than the maximum is +capped at the maximum. + +If the PostgreSQL server is shared with other applications, make sure +`PENPOT_DATABASE_MAX_POOL_SIZE` plus the connections used by everything else stays below +the server's `max_connections` (100 by default). + ### Storage Storage refers to storing the user uploaded different objects in Penpot (assets, file data,...). @@ -814,12 +837,13 @@ These settings are equally useful if you have a Minio storage system. __Since version 2.11.0__ -You can change the default file data storage backend with `PENPOT_FILE_DATA_BACKEND` environment variable. Possible values are: +You can change the file data storage backend with `PENPOT_FILE_DATA_BACKEND` environment variable. Possible values are: -- `legacy-db`: the current default backend, continues storing the file data of files and snapshots in the same location as previous versions of Penpot (< 2.11.0), this is a conservative default behaviour and will be changed to `db` in next versions. -- `db`: stores the file data on an specific table (the future default backend). +- `db`: the default. Stores the file data on an specific table. - `storage`: stores the file data using the objects storage system (S3 or FS, depending on which one is configured) +The `legacy-db` value is still accepted as a **read-only** alias for the payloads written by versions before 2.11.0: the backend reads them, but it no longer writes file data there. **If you upgrade a deployment with `PENPOT_FILE_DATA_BACKEND=legacy-db`, remove the variable (or set it to `db`) before the upgrade**: the backend reads `legacy-db` as `db` and logs a warning, so saves keep working, but remove the variable anyway so the deployment does not claim a read-only backend. + This also comes with an additional feature that allows offload the "inactive" files on file storage backend and leaves the database only for the active files. To enable it, you should use the `enable-tiered-file-data-storage` flag and `db` as file data storage backend. ```bash @@ -983,6 +1007,33 @@ set, and sends it through the [email configuration](#email-configuration). Each email carries the name and email address of the user who sent it, and uses that address as reply-to. +#### MCP sign-in + +MCP clients with OAuth support can connect with the URL +`https:///mcp/stream`, without a key. The client opens Penpot +in the browser, the user allows access, and Penpot gives the client the user's MCP +key. This is off by default. Turn it on by setting the public URL of the +authorization service on the MCP container: + +```bash +# MCP +PENPOT_MCP_AUTH_URI: https://penpot.example.com/api/mcp/oauth +``` + +- `PENPOT_MCP_AUTH_URI`: Your Penpot public URI followed by `/api/mcp/oauth`. It + must use HTTPS and have no credentials, query or fragment. When unset, clients + must add the key to the URL as `userToken`. + +The MCP server must run in multi-user mode, which the official `penpotapp/mcp` +image does by default. The server does not start if the value is invalid, and it +logs at startup whether sign-in is on. + +The frontend container forwards the sign-in requests to the backend. If your proxy +sends `/api/` straight to the backend instead, send +`/.well-known/oauth-authorization-server/api/mcp/oauth` to the backend too. + +Clients that already add the key to the URL keep working with sign-in on. + ### Internal resolver The frontend container resolves the backend, exporter and MCP service names with the diff --git a/docs/technical-guide/developer/architecture/backend.md b/docs/technical-guide/developer/architecture/backend.md index eb0517e9eb..5a26d9d8b0 100644 --- a/docs/technical-guide/developer/architecture/backend.md +++ b/docs/technical-guide/developer/architecture/backend.md @@ -63,13 +63,16 @@ Rel(backend_app, redis, "Notifies", "pub/sub") ▸ tasks/ main.clj config.clj + health.clj http.clj + jobs.clj metrics.clj migrations.clj - notifications.clj + nitrate.clj rpc.clj setup.clj srepl.clj + system.clj worker.clj ... ``` @@ -82,9 +85,11 @@ Rel(backend_app, redis, "Notifies", "pub/sub") * migrations contains the SQL scripts that define the database schema, in the form of a sequence of migrations. * rpc is the main module to handle the RPC API calls. -* notifications.clj is the main module that manages the websocket. It allows - clients to subscribe to open files, intercepts update RPC calls and notify - them to all subscribers of the file. +* http/websocket.clj manages the websocket endpoint: clients subscribe + to open files or teams and the server keeps the subscriptions; revoked access + is re-checked and dropped. rpc/notifications.clj publishes + access-change events on the bus so the websocket re-checks the subscriptions + it owns. * setup initializes the environment (loads config variables, sets up the database, executes migrations, loads initial data, etc). * srepl sets up an interactive REPL shell, with some useful commands to be diff --git a/docs/technical-guide/developer/backend.md b/docs/technical-guide/developer/backend.md index f2e7bc4d15..d1a40b50fa 100644 --- a/docs/technical-guide/developer/backend.md +++ b/docs/technical-guide/developer/backend.md @@ -95,6 +95,52 @@ Examples: **NOTE**: if table name has more than one word, we still use - as a separator. +### Large tables + +A migration runs inside a single transaction, so it cannot use `CONCURRENTLY`, +and a `CREATE INDEX` in a migration locks the table against writes until the +whole run commits. + +Write the migration as plain, idempotent SQL: `ADD COLUMN IF NOT EXISTS`, +`CREATE INDEX IF NOT EXISTS`, and `DROP CONSTRAINT IF EXISTS` followed by +`ADD CONSTRAINT`. Do not encode shape checks as `pg_catalog` introspection. +They are easy to get wrong (forgetting `NOT condeferred` accepts an +`INITIALLY DEFERRED` foreign key, which moves the integrity check to commit) and +they duplicate what the runbook can verify by hand. + +On a large table, build the column and the index out of band before deploying, +with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block). +A partitioned table cannot take `CREATE INDEX CONCURRENTLY` on the parent: build +the index on each partition, create the parent index with +`CREATE INDEX ... ON ONLY`, and attach every partition with +`ALTER INDEX ... ATTACH PARTITION`. An interrupted concurrent build leaves the +index `INVALID`: drop it and retry. `CREATE INDEX IF NOT EXISTS` matches the +name, not the definition, so the runbook verifies `pg_index.indisvalid`, that +the index has no predicate or expression, and that every partition is attached. + +A validated foreign key is the other expensive DDL: `ADD FOREIGN KEY` runs as an +`ALTER TABLE`, so it takes `ACCESS EXCLUSIVE` on the referencing table and +`SHARE ROW EXCLUSIVE` on the referenced one, and it scans the whole referencing +table. `NOT VALID` on a partitioned table is only available since PostgreSQL 18. +A reference that a GC task resolves must be `DEFERRABLE INITIALLY IMMEDIATE`, +never `INITIALLY DEFERRED`: the immediate check fires inside the transaction +that catches the violation and retries; the deferred one fires at commit, +outside the catch. + +The runbook ends by verifying the end state (column, pending rows, index and +constraint shape) and marking the migration step as applied +(`INSERT INTO migrations (module, step) ...`), so the body does not run at +startup. The marker is the only thing that skips the migration. + +A backfill that removes a key an old reader uses (for example a JSONB pointer +that the previous GC scans) must run with the old code stopped. Otherwise the +old reader loses the reference and can delete a live object, and the rows it +writes in the meantime miss the backfill. + +`ADD COLUMN` takes a brief `ACCESS EXCLUSIVE` even when it changes nothing. A +migration accepts that lock; a deployment that cannot take it runs the runbook +and marks the step as applied. + If you need to have a global overview of the all schema of the database you can extract it using postgresql: diff --git a/docs/technical-guide/developer/devenv.md b/docs/technical-guide/developer/devenv.md index 697d119b1c..20cd2d8a3c 100644 --- a/docs/technical-guide/developer/devenv.md +++ b/docs/technical-guide/developer/devenv.md @@ -508,6 +508,14 @@ Some examples of commonly used flags: The full list of available flags can be found in `common/src/app/common/flags.cljc`. +One flag is off by default in the devenv: the frontend compiles with +`PENPOT_DEVENV=true` in the development containers, which hides all +admin-console UI (links, routes, team menu entries) whether or not you created +a `config.js`. To test the admin-console against a local Nitrate instance, +reactivate it from your `config.js`: `var penpotFlags = "enable-admin-console";`. +Your flags are parsed after the injected default, so your entry wins. Production +builds are unaffected: `PENPOT_DEVENV` only exists in the devenv containers. + After creating or modifying this file, **reload the browser** (no need to restart anything). ### Backend flags via PENPOT_FLAGS diff --git a/docs/user-guide/design-systems/variants.njk b/docs/user-guide/design-systems/variants.njk index 5b41f0fade..5a6a1ff2e1 100644 --- a/docs/user-guide/design-systems/variants.njk +++ b/docs/user-guide/design-systems/variants.njk @@ -86,6 +86,13 @@ desc: Streamline your design workflow with Penpot's Components guide! Learn to c Edit variant property +
    Order properties and values
    +
      +
    • Properties: In the Design tab, drag a property by its handle to move it up or down.
    • +
    • Values: The values dropdown follows the order of the variants in the Layers panel, from top to bottom. Each value appears where the first variant that uses it sits. To change the order, reorder the variants in the Layers panel. Empty values always go last.
    • +
    +

    The variant order applies to all properties at once, so you may not be able to set the value order you want for two properties at the same time.

    +
    Delete properties
    • From the Design tab: Select the main component (not an individual variant) and press the minus button next to the property.
    • diff --git a/docs/user-guide/export-import/exporting-layers.njk b/docs/user-guide/export-import/exporting-layers.njk index 6f1f5869c5..56e679e48f 100644 --- a/docs/user-guide/export-import/exporting-layers.njk +++ b/docs/user-guide/export-import/exporting-layers.njk @@ -38,7 +38,7 @@ desc: Learn how to export layers in Penpot, the free, open-source design collabo
      Export main menu
      -

      You can also launch the export for a page from the main menu or using the shortcut Ctrl + Shift + E. This option will include in the export all the export pressets set among a selection or in the page if nothing is selected.

      +

      You can also launch the export for a page from the main menu or using the shortcut Ctrl + Shift + E. This option will include in the export all the export presets set among a selection or in the page if nothing is selected.

      Export main menu
      diff --git a/exporter/package.json b/exporter/package.json index d5932a8aaf..dc41cadfcb 100644 --- a/exporter/package.json +++ b/exporter/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en España SL", "private": true, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "repository": { "type": "git", "url": "https://github.com/penpot/penpot" diff --git a/exporter/pnpm-lock.yaml b/exporter/pnpm-lock.yaml index 88059a630d..922cc11059 100644 --- a/exporter/pnpm-lock.yaml +++ b/exporter/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/frontend/deps.edn b/frontend/deps.edn index ab7197c661..9ddd3104c5 100644 --- a/frontend/deps.edn +++ b/frontend/deps.edn @@ -1,4 +1,4 @@ -{:paths ["src" "vendor" "resources" "test"] +{:paths ["src" "vendor" "resources" "test" "performance"] :deps {;; Carries `app.common.render-wasm.*`, shared with the headless exporter. penpot/common @@ -30,15 +30,9 @@ :git/sha "d726717" :git/url "https://github.com/funcool/rumext.git"} - penpot/tokenscript-interpreter - {:git/tag "v0.2.0" - :git/sha "20db9b1" - :git/url "https://github.com/penpot/tokenscript-interpreter.git"} - instaparse/instaparse {:mvn/version "1.5.0"} garden/garden {:git/url "https://github.com/noprompt/garden" - :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"} - } + :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"}} :aliases {:outdated @@ -67,4 +61,7 @@ "-Dpenpot.wasm.profile-marks=true" "-XX:+UnlockExperimentalVMOptions" "-XX:CompileCommand=blackhole,criterium.blackhole.Blackhole::consume"]} - }} + + :renderer-bench + {:main-opts ["-m" "shadow.cljs.devtools.cli"] + :jvm-opts ["-Dpenpot.wasm.profile-marks=false"]}}} diff --git a/frontend/package.json b/frontend/package.json index d5b917acd0..41faeb34a3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en España SL", "private": true, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "browserslist": [ "defaults" ], @@ -20,22 +20,27 @@ "build:storybook:assets": "node ./scripts/build-storybook-assets.js", "build:storybook:cljs": "clojure -M:dev:shadow-cljs compile storybook", "build:wasm": "../render-wasm/build frontend", + "build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-commands bench-render-wasm-runner", + "render-bench": "sh -c 'w=; [ \"${PENPOT_GPU:-0}\" = 1 ] && w=\"xvfb-run -a\"; exec $w node target/performance/render-wasm/runner.cjs \"$@\"' render-bench", "build:app:libs": "node ./scripts/build-libs.js", "build:app:main": "clojure -M:dev:shadow-cljs release main worker", "build:app:worker": "clojure -M:dev:shadow-cljs release worker", "build:app": "pnpm run clear:shadow-cache && pnpm run build:app:main && pnpm run build:app:libs", - "check-fmt:clj": "cljfmt check --parallel=true src/ test/", + "check-fmt:clj": "cljfmt check --parallel=true src/ performance/ test/", "check-fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js", "check-fmt:scss": "prettier -c resources/styles -c src/**/*.scss", - "fmt:clj": "cljfmt fix --parallel=true src/ test/", + "fmt:clj": "cljfmt fix --parallel=true src/ performance/ test/", "fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js -w", "fmt:scss": "prettier -c resources/styles -c src/**/*.scss -w", - "lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/", + "lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/ performance/ test/performance_tests/render_wasm/", "lint:js": "exit 0", + "lint:renderer-benchmarks": "clj-kondo --config-dir ../.clj-kondo --lint performance/ test/performance_tests/render_wasm/", "lint:scss": "pnpm exec stylelint '{src,resources}/**/*.scss'", "build:test": "pnpm run build:wasm && clojure -M:dev:shadow-cljs compile test", + "build:renderer-benchmarks:tests": "clojure -M:dev:shadow-cljs compile performance-tests", "test": "pnpm run build:test && node target/tests/test.js", "test:quiet": "node ./scripts/test-quiet.js", + "test:renderer-benchmarks": "pnpm run build:renderer-benchmarks:tests && node target/performance-tests/test.js", "test:storybook": "vitest run --project=storybook", "watch:test": "mkdir -p target/tests && concurrently \"clojure -M:dev:shadow-cljs watch test\" \"nodemon -C -d 2 -w target/tests --exec 'node target/tests/test.js'\"", "test:e2e": "playwright test --project default", diff --git a/frontend/packages/draft-js/package.json b/frontend/packages/draft-js/package.json index 19fc00e59f..27899cebfd 100644 --- a/frontend/packages/draft-js/package.json +++ b/frontend/packages/draft-js/package.json @@ -4,7 +4,7 @@ "description": "Penpot Draft-JS Wrapper", "main": "index.js", "type": "module", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "author": "Andrey Antukh", "license": "MPL-2.0", "dependencies": { diff --git a/frontend/packages/mousetrap/package.json b/frontend/packages/mousetrap/package.json index 5b22f4faac..d2cd042a43 100644 --- a/frontend/packages/mousetrap/package.json +++ b/frontend/packages/mousetrap/package.json @@ -7,7 +7,7 @@ "scripts": { "test": "node --test \"test/*.test.js\"" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "author": "Craig Campbell", "license": "Apache-2.0 WITH LLVM-exception" } diff --git a/frontend/packages/ui/package.json b/frontend/packages/ui/package.json index b65dfe3405..1726ff6d1f 100644 --- a/frontend/packages/ui/package.json +++ b/frontend/packages/ui/package.json @@ -3,7 +3,7 @@ "version": "0.0.1", "types": "./dist/index.d.ts", "type": "module", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "exports": { ".": { "import": "./dist/index.js" diff --git a/frontend/performance/benches/render_wasm/README.md b/frontend/performance/benches/render_wasm/README.md new file mode 100644 index 0000000000..2aca3fc7a1 --- /dev/null +++ b/frontend/performance/benches/render_wasm/README.md @@ -0,0 +1,1514 @@ +# WASM renderer benchmarks + +The renderer benchmark suite measures the Penpot WebAssembly (WASM) renderer in +Chromium. Scenes describe shapes; cases describe the work to perform on them. +The suite records setup, rendering, pan and zoom durations and compares runs +with different renderer builds. [How to run](#how-to-run) gives the commands. + + +## Contents + +- [Main concepts](#main-concepts) +- [Architecture](#architecture) +- [Workloads](#workloads) +- [Renderer commands](#renderer-commands) +- [Measurement](#measurement) +- [Results and reports](#results-and-reports) +- [A/B feature experiments](#ab-feature-experiments) +- [How to run](#how-to-run) +- [Unit tests](#unit-tests) +- [Scope and limitations](#scope-and-limitations) +- [Repository map](#repository-map) +- [Further reading](#further-reading) + +## Main concepts + +### Workload + +- **Renderer**: the Rust Skia renderer compiled to WebAssembly. It owns shape + storage, tiles, caches, and the WebGL2 graphics state for one canvas. +- **Module**: one instantiated copy of the renderer. The browser creates a + module per prepared page and releases it with the page. +- **Graphics context**: the WebGL2 context and renderer state bound to one + canvas. A benchmark _case_ can use either a fresh or a reused context. +- **Scene**: a recipe that produces a starting set of shapes. A scene has an + id, a closed parameter schema, and a builder function. This function is given + a random seed and returns a _snapshot_. Scenes can be **procedural**, or + **captured**. The former build shapes from code, parameters, and a random + seed, whereas the latter render a page exported from Penpot. +- **Scene snapshot**: the concrete shapes a run renders. This is only static + data, camera movement and other interactions belong to a **case**. +- **Master seed**: the initial seed for the run, supplied on the command line. + It defaults to 42 and accepts a 32-bit unsigned integer. +- **Seed**: a 32-bit integer derived from the master seed, scene id, and + canonical parameters. The seed makes a scene reproducible. Load, pan, and + zoom cases for the same scene and parameters share one seed. +- **Scene builder**: the construction API used inside a scene. It provides basic + shapes (`rect`, `path`, `circle`, `frame`, `group`, and `bool`) constructors, + and a (hopefully) self-explanatory API. +- **Shape reference**: a label that a scene attaches to a shape during + construction. A _case_ body uses references to find shapes it needs. +- **Case**: a scene plus its execution settings: parameters, base view, context + policy, optional preparation callback, optional reset function, and an + executable body. A case is the unit that is measured. +- **Case body**: the function declared by `defcase` that performs a case's work + during one attempt. It receives a command context with the case data, scene + snapshot, current view, and hooks. It can call `render-sync`, `load`, `pan`, + `zoom`, or `restore` and their measured variants. +- **View**: the camera's zoom `:scale` and offsets `:x` and `:y`. + Optional `:viewport` settings give the canvas width, height, and device pixel + ratio, the number of drawing-buffer pixels per CSS pixel along each dimension. +- **Hook**: a function supplied through `:hooks` for clock readings, + scheduling, view updates, rendering, or checking whether work may continue. +- **Case descriptor**: What gets sent to the browser. It contains the case id, + scene, scene description, derived seed, parameters, view, context policy, and + preparation settings. Functions stay in the local registry and are removed + from the descriptor; the browser resolves them by case id. +- **Catalogue**: the compiled list of case descriptors and scene declarations + for one build. Node reads the catalogue to select cases. The browser executor + contains the same declarations and looks up the case body by case id. The + current catalogue declares 14 cases. +- **Fingerprint**: SHA-256 digest of the validated starting snapshot. A + comparison checks it to establish equal starting scene content. + +### Execution + +- **Build**: one compiled renderer plus the matching catalogue and browser + executor. A build id ties the three outputs together, and the runner rejects + a mismatch. +- **Retained build**: a ready build that the suite stores under the managed + build directory for reuse in later runs. +- **Fresh case**: a case with `:context :fresh`. Each attempt gets a new page, + module, graphics context, and upload. +- **Warm case**: a case with `:context :reuse`. One page, module, graphics + context, snapshot, and upload serve all attempts of the case. +- **Preparation**: setup for one graphics context. It covers module + instantiation, WebGL2 context creation, scene build, snapshot validation, + fingerprinting, upload-order derivation, and upload. It then runs the optional + preparation callback and drains to `Full` unless the case declares + `:preparation {:render :none}`. One page owns one preparation. +- **Attempt**: one execution of a case against a preparation. It records an + `:attempt-id`, one outcome, main and optional reset phase blocks, and failure + or partial evidence. A failed attempt still consumes its allocated slot. +- **Warmup**: an attempt that runs the case before measured attempts to warm + caches and stabilize the page. The run records warmups but excludes them from + statistics. +- **Repetition**: a measured attempt. A case has a fixed number of repetitions + chosen before the run. +- **Run**: a set of attempts over selected cases. A run owns the browser + processes it starts, fixes warmup and repetition counts, records failures in + chronological order, and performs no hidden retries. +- **Run record**: the saved evidence for one run: plan, build and environment + provenance, quantity descriptions, preparations, attempts with their phase + histories, failures, terminal case outcomes, remaining slots, and termination. + The runner stores it as a data file validated by a schema. +- **Bridge**: the object that runs in the browser, receives plain requests from + Node, owns the page's renderer context, runs preparations and attempts, and + returns plain results. +- **Runner**: the Node command line interface that builds or reuses a build, + selects cases, serves the build, launches Chromium, runs attempts, saves + records, and prints reports. +- **Browser executor**: the compiled page and JavaScript that execute + preparations and case bodies. Its bridge exposes the requests that Node can + send. +- **Measurement**: a numeric result for a quantity such as elapsed duration. + The suite measures module start, graphics start, upload, first render, and + camera movement separately. +- **Metric**: the name of a measured quantity. A command event records its raw + numeric values in a vector. A saved run supplies a `:unit` and an optional + `:description` for each metric type. It stores no statistic or reduction. +- **Metric key**: `[phase scope metric-type]`. Scope is the command that recorded + the metric or `:global` for every command with that type in the phase. +- **Observation**: one eligible phase execution's sum for one metric key. Inner + calls, slices, and loop repetitions stay within that execution and do not add + observations. Warmups and failed attempts supply none. +- **Slice**: a trace entry for one renderer call: timestamp, flags, frame type, + and duration. Slices describe the work within an attempt. +- **Trace**: details of the work, including individual renderer calls, + cached previews, requested waits, and observed durations. +- **Clock reading**: a time in milliseconds from the clock's origin. Subtracting + two readings from that clock gives an elapsed duration. +- **Endpoint**: the clock reading at the end of the measured work. The renderer + endpoint is the reading immediately after a renderer call returns. +- **Render drain**: repeated renderer calls until one returns `Full`. Partial + completion schedules another call on the next animation frame. +- **Cached preview**: one render from cached tiles after a view update. +- **Camera session**: camera start, view updates with cached previews, a settling + wait, camera end, and the final render drain. +- **Active interval**: the part of a camera session from camera start through + the last cached preview and its guard check, or an explicit pause that follows + the preview. +- **Settling wait**: the requested pause after the active interval and before + ending the interaction. The observed wait includes scheduling delays. +- **Camera finalization**: the interval from immediately before ending the + interaction through the renderer call that returns `Full`. +- **Frame type**: the completion signal returned by one renderer call. `None` + means no frame; `Partial` means visible work remains; `Full` means the + renderer finished the current request; `ViewportReady` means the visible + viewport was submitted while work outside the viewport may remain. +- **Tile**: a unit of renderer work with a fixed size. A logical tile covers + 512 document units. The renderer caches rendered tiles in a texture atlas. +- **Atlas**: a GPU texture that stores many tiles or a document-space preview. + Atlases let warm pan and zoom present cached pixels before new tiles finish. + +### Command execution + +- **Renderer command**: a function that accepts a command context or its Promise + and returns a Promise of the next context. The base function uses the declared + name, such as `pan`. The measured function adds `*`, such as `pan*`, and records + an event for its work. +- **Command context**: the map with keys `:case`, `:scene`, `:view`, and `:hooks`. + It carries the case descriptor, prepared scene snapshot, current view, and + hooks. +- **Command body**: the work declared by `defcommand`. A synchronous body + returns the next context. An asynchronous body completes through callbacks + named by its declaration. +- **Composite command**: a command that combines calls to other commands. + A declaration with `:measured-variant false` defines its base function and + collects events from the measured commands it calls. +- **Command execution**: one call to `commands/execute`, which runs a function + from one command context to the next and records measured calls in completion + order. +- **Event**: the record for one measured command call. It contains the command + name, vectors of raw metric values, and an optional trace. +- **Guard**: the `:check` hook. A return authorizes work; an error rejects it. + The renderer owner is the execution authorized to use the renderer. The guard + can report cancellation, deadline expiry, context loss, or replacement of + that owner. +- **Completion callback**: `complete` or `fail` in an asynchronous command body. + The first call determines the outcome: `complete` supplies the next context + and optional measurements; `fail` supplies a cause. + +### Results + +- **Report**: the statistics and tables computed from one or more run records. + Reports run in Node or the browser and do not load the renderer. +- **Comparison**: an offline analysis of two saved run records. It pairs cases, + checks compatibility, and reports changes for each metric key with intervals. +- **A/B comparison**: a comparison of two runs that differ by one renderer + feature. The feature is off in one run and on in the other. +- **Permission**: In principle, only run records with equal builds and equal + scenes are comparable. Because one sometimes needs to override this strict + requirement, permissions allow build environment or scene differences in a + comparison. +- **Termination**: the final state of a run: completed, failed, incomplete, or + interrupted. + +### Feature experiments + +- **Feature**: a switch at compile time in the renderer's Cargo manifest. +- **Feature configuration**: the set of Cargo features enabled for one renderer + build. +- **Feature comparison**: all on and off combinations of the requested features. + $N$ requested features produce $2^N$ configurations. +- **Edge**: two configurations that differ in exactly one feature, compared from + the configuration with the feature off to the configuration with the feature + on. A comparison with $N$ features has $N 2^{N-1}$ edges. +- **Edge comparison**: the comparison record for one edge, stored with the + feature comparison. +- **Feature-comparison record**: the saved plan, configurations, runs, edges, + failures, and termination for one A/B experiment. + +## Architecture + +The suite has three components: runner, browser, and reports. They are all driven via the same CLI. + +### Node runner + +The Node runner owns everything outside the page and +controls the experiment. + +- it prepares or loads a build; +- it reads the compiled catalogue and selects cases; +- it serves the selected build from a server on `127.0.0.1`; +- it launches Chromium and creates pages; +- it schedules warmups and measured attempts; +- it records preparations, attempts, failures, and termination; +- it writes the run record as work progresses; +- it prints a final report. + +### Browser bridge + +The bridge owns everything inside the page that runs the benchmark: + +- it validates a request before assigning the page to that work; +- it imports the renderer module and creates the canvas and WebGL2 context; +- it builds, validates, fingerprints, orders, and uploads the scene; +- it carries each phase's returned context into the next phase or attempt; +- it runs the case body and records the measurements its commands select; +- it returns plain data and disposes the context when asked. + +### Reporting + +This code performs the analysis offline: + +- it normalizes run records into summaries; +- it computes statistics and intervals for each metric; +- it pairs saved cases and checks compatibility; +- it formats run summaries, comparisons, and feature-comparison edge reports; +- it imports no browser, server, or renderer code. + +### End-to-end path + +```mermaid +sequenceDiagram + participant N as Node runner + participant S as Owned server + participant B as Browser bridge + participant W as WASM renderer + + N->>N: select cases from the catalogue + N->>S: serve the selected build + N->>N: launch Chromium + N->>B: open a page + B->>W: instantiate the module + B->>W: create the WebGL2 context + B->>B: build, validate, fingerprint the snapshot + B->>W: upload shapes in tree order + B->>B: run preparation callback + B->>W: render Full unless disabled + B-->>N: preparation evidence and raw phase block + N->>N: persist the run record + N->>B: run main and reset + B-->>N: raw main and reset phase blocks + N->>N: persist the run record +``` + +The Node runner saves the initial plan before it starts a browser, and updates +the run record after each preparation and attempt, and at termination. Reports +use these saved records. + +### Builds and catalogues + +A build has three matching outputs: + +- the renderer module; +- the case catalogue compiled for Node; +- the browser executor and its page. + +Build preparation compiles the renderer in release mode with Cargo, then compiles +the catalogue and browser executor with the same build id. The runner checks +the id before case collection and when a page starts, and rejects a mismatch. + +An ordinary measurement build uses the renderer's default Cargo feature set. A +feature experiment may enable selected functional features. Both kinds of +measurement build should exclude diagnostic features such as `stats`, `profile`, +`profile-macros`, and `profile-raf`. The build checks that we are in release +mode, the optimization level, and disabled profiling marks. + +The build record captures a fixed list of environment values supplied to Cargo, +including compiler, target, and Emscripten settings. These values provide +provenance for interpreting a run. Reproducing a build also requires its other +environment settings and dependencies. + +Builds are saved under `target/performance/render-wasm/builds//`. The +manifest records the build id, status, functional settings, Git provenance, and +build facts. An experiment run can select a retained build with `--build `, +and `run` prepares a new build when it omits `--build`. + +`build` and a run that prepares its build accept `--features`, +`--[no-]default-features`, and `--build-timeout-ms`. A run with `--build` reuses +the retained build and rejects those preparation options. + +Run one build preparation at a time in a checkout and keep renderer watchers +stopped until it finishes. The renderer build lock covers Cargo compilation +and copying the renderer files. The build script releases that lock before +ClojureScript compilation. + +### Browser host + +The runner serves the selected build directory from `127.0.0.1` on an allocated +port. + +The runner launches headless Chromium with Playwright and owns the browser +process it starts. Cases run serially. A fresh case gets a new page and context +for each preparation. A warm case keeps one page and context for all of its +attempts. + +Before cases run, `--startup-timeout-ms` bounds Git provenance, server startup, +and the initial browser launch. Each case then gets one absolute `--timeout-ms` +deadline. The deadline covers the case's page setup, preparation, warmups, +repetitions, reset, and screenshot diagnostics. It does not restart for a phase, +attempt, or browser replacement. Work on a reused preparation keeps its original +case deadline; a request timeout can only tighten that deadline. A browser +launched after a failed case uses the next case's remaining deadline. + +When a page stalls past its deadline, Node stops the browser process and records +the failure. The browser bridge also checks for context loss, expired deadlines, +and work associated with a superseded renderer owner before and after +asynchronous work. A browser thread occupied by synchronous work relies on the +Node deadline to stop it. The runner persists each delivered preparation and +attempt. It does not stream browser-local records and cannot recover in-flight +data when a blocked browser cannot return; records delivered before the block +remain saved. + +Clean-up closes the page, then the browser, then the server. The runner deletes +a build selected with `--remove` only after the final run record is saved. + +## Workloads + +### Declaring a scene + +A scene declaration names an id, a description, a closed parameter schema, and a +builder function. The builder runs inside a seeded scope. Random values come +from one generator owned by that scope, so the same recipe, parameters, and seed +produce the same snapshot. + +```clojure +(defscene :rects + {:description "Seeded rectangles with translucent fills and centered strokes" + :params-schema schema:params} + [params] + (scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)} + :defaults {:rect {:x (gen-int 0 (:width params)) + :y (gen-int 0 (:height params)) + :width (gen-int 20 100) + :height (gen-int 20 100) + :fills (gen-vector (gen-fill))}}} + (doseq [_ (range (:count params))] + (rect)))) +``` + +Each scene has one construction scope. Starting another scene inside it raises +an error. Ordinary `let`, `doseq`, and functions defined outside the scope work +inside the body. Each shape constructor accepts an optional label +or a label plus attributes. Attributes override generated defaults by key. +Labels must be keywords or vectors and must be unique within the snapshot. + +Containers follow the same pattern: + +- `frame` keeps the supplied `:x`, `:y`, `:width`, and `:height`. +- `group` derives bounds from its children. A masked group uses its first child + as the mask. +- `bool` derives content and bounds from its children with the standard path + engine. It requires a boolean operation type. Frame children raise a + construction error. + +### Declaring a case + +For a scene named `:rects`, case ids start with `:rects/`. The `defcase` form +derives the scene from that keyword namespace and attaches the case body. + +```clojure +(defcase :rects/pan + {:params default-params + :view {:scale 1 :x 0 :y 0} + :context :reuse + :reset (fn [rtx] + (camera/restore rtx (get-in rtx [:case :view])))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 200 :dy 0 :settle-ms 100})) +``` + +The case declares: + +- `:params`: scene parameters. Collection validates them against the scene's + closed schema. +- `:view`: the starting camera, with an optional viewport override. The default + viewport is 1920 by 1080 at device pixel ratio 2. +- `:context`: `:fresh` creates a page, module, and context per attempt. + `:reuse` keeps one page, module, context, snapshot, and upload for all + attempts of the case. +- `:preparation`: an optional `:callback` function runs after upload and returns + the next context or its Promise. Preparation then renders Full by default. + `:render :none` completes preparation after the callback for cold-render cases. +- `:reset`: an optional function that receives the context after successful + main execution and returns the next context or its Promise. Reset runs after + every successful main, including warmups and the final attempt. + +The case body owns the command sequence, arguments, and any deliberate +repetition. The declaration supplies the starting settings; the body and the +commands it calls own the work. + +Commands receive workload arguments from the case body. The callback, main body, +and reset share the command context and can each call measured commands. + +### Case collection and seeds + +Node collects cases from the compiled catalogue. Collection loads every +declaration, validates every scene and case, derives the scene seed, selects +the case's data fields, and checks that the descriptor survives a serialization +round trip. Shape construction, browser startup, and renderer initialization +take place during execution. + +The local registry keeps the main, reset, and preparation callback functions. +The descriptor carries the declared preparation data. The browser resolves the +functions from its registry using the case id. + +The scene seed depends only on the master seed, the scene id, and the canonical +parameters. These inputs give load, pan, and zoom cases the same starting +content across case bodies, collection orders, attempt counts, and feature +configurations. They also give A/B configurations matching scene content. + +A filter selects cases by a substring of the case id. An empty selection is an +error. Declarations outside the filter are still validated. + +### Snapshots and upload order + +The snapshot contract requires the canonical root frame, shape ids that match +their object keys, consistent child and parent links, a tree that reaches every +shape from the root, and references that point at present shapes. Validation +runs before any timer starts. + +The browser derives upload order from the snapshot. The root comes first, then +descendants in the order of each shape's +child list. The browser computes the order and the fingerprint before the upload +clock starts. The upload uses the same batch serializer as the editor, with +structure, fill, and stroke data batched and path and SVG attributes handled per +shape. + +### Some example scenes + +| Scene | Content | Cases | +| ------- | ----------------------------------------------------------------------- | -------------------------------------- | +| `rects` | 1000 rectangles with random size, translucent fills, strokes, and radii | load, pan, zoom, blocking-full, mixed-pans | +| `paths` | 1000 curved paths with fills and strokes | load, pan, zoom | +| `plus` | 1000 plus signs made from two open line subpaths | load, pan, zoom | +| `masks` | 24 masked groups, each with a circle mask and 8 striped rectangles | load, pan, zoom | + +The current catalogue has 14 cases: load, pan, and zoom for each scene, plus +`rects/blocking-full` and `rects/mixed-pans`. Every load case is cold, with +`:context :fresh` and `:preparation {:render :none}`, and its body calls +`camera/load*`. The pan and zoom cases for each scene reuse one prepared context +and keep their declared offsets, steps, factor, and settling time; their reset +calls the bare `camera/restore` to return to the case view without recording an +event. `rects/blocking-full` is another fresh cold case and calls +`camera/render-sync*`. `rects/mixed-pans` runs two measured pans around one bare +pan and resets with measured `camera/restore*`. + +Scene recipes supply canonical geometry for rendering and camera workloads. +Camera sessions submit view updates and wait for settling before their final +render. Each case declares its sequence, arguments, and reset behavior in code. + +## Renderer commands + +A renderer command accepts a command context or its Promise and returns a +Promise of the next context. The command context has four fields: + +| Field | Meaning | +| --- | --- | +| `:case` | Case descriptor, including the case id. | +| `:scene` | Prepared starting snapshot with concrete shapes and references. | +| `:view` | Current zoom scale, camera offsets, and optional viewport settings. | +| `:hooks` | Functions for clocks, scheduling, rendering, view changes, and the execution guard. | + +The `:scene` field describes the starting snapshot. Commands return view changes +through `:view` and apply renderer work through hooks. Every command context +supplies `:now`, `:check`, `:frame`, `:sleep`, `:render`, `:render-sync`, +`:render-from-cache`, `:set-view`, `:set-view-start`, and `:set-view-end` as +functions. The `:now` hook returns a clock reading in milliseconds. +The `:check` guard authorizes work by returning and reports cancellation, +deadline expiry, context loss, or a superseded owner +by throwing an error. The owner is the execution authorized to use the renderer. +Returned contexts retain the execution's `:check` guard; replacing it rejects +completion. + +### Public commands + +The runtime provides five public commands. Each has a bare function and a +measured variant with `*`. Both forms accept a context or its Promise and return +a Promise of the next context. + +| Command | Work | Recorded metric types | +| --- | --- | --- | +| `render-sync` | Runs the blocking renderer path once. | `:command/elapsed` | +| `load` | Waits for an animation frame, then renders until `Full`. | `:command/elapsed`, `:renderer/call-elapsed`, `:renderer/prepared-scene-to-full` | +| `pan` | Moves by `:dx` and `:dy` through `:steps` cached previews, waits `:settle-ms`, ends the interaction, and renders until `Full`. | `:command/elapsed`, `:renderer/cache-call-elapsed`, `:renderer/call-elapsed`, `:camera/finalization-to-full` | +| `zoom` | Multiplies scale by `:factor` with the same camera session. | The same types as `pan`. | +| `restore` | Applies a view and renders immediately with `SyncTiles` until `Full`. | `:command/elapsed`, `:renderer/call-elapsed` | + +Bare commands update the context but record no event. A measured command records +one event. `load`, `pan`, `zoom`, and `restore` carry the final renderer return +captured by the protocol into `:end-ms`; their command elapsed ends at that +endpoint, not at a later Promise callback. `:renderer/call-elapsed` holds one +duration per renderer call in the final drain, in call order. `pan` and `zoom` +also record one `:renderer/cache-call-elapsed` value per cached preview and a +single `:camera/finalization-to-full` value. `load` adds +`:renderer/prepared-scene-to-full`, the drain's `:full-ms`. `render-sync` is +synchronous and records only its command elapsed. + +`restore` requires the target view's resolved viewport to equal the current +context's resolved viewport. A different width, height, or device pixel ratio +fails before the command changes the renderer state. + +### Declaring a command + +`defcommand` defines two functions from one body. The base function runs the +work; the function with a `*` suffix also records a measurement event. Measured +functions require an active command execution. Both functions resolve their +incoming context before running the body. A synchronous body +returns the next command context. + +```clojure +(ns example.renderer-commands + (:require + [benches.render-wasm.runtime.camera :as camera] + [benches.render-wasm.runtime.commands :as commands :refer [defcommand]] + [benches.render-wasm.runtime.protocol :as protocol])) + +(defcommand render-sync + "Renders the prepared scene synchronously." + [rtx] + ((get-in rtx [:hooks :render-sync])) + rtx) +``` + +`:pre` conditions validate arguments before work and timing in every build. +Input resolution, context validation, and argument validation precede the start +clock reading. Context validation checks the complete set of hooks. Output +validation and event recording follow the endpoint. + +The body duration is + +\[ +\Delta t = t_{\mathrm{end}} - t_{\mathrm{start}}, +\] + +where both clock readings use milliseconds from the same origin. A synchronous +command captures the endpoint immediately after its body returns. An +asynchronous command supplies an endpoint or captures it when `complete` runs. + +### Asynchronous completion + +The option `:async [complete fail]` supplies two completion callbacks to the +body. Their names belong to the declaration; the public command arguments stay +the same. The first callback call determines the outcome: + +- `(complete next-rtx)` succeeds with the next command context and captures the + endpoint at that call. +- `(complete next-rtx evidence)` succeeds with an explicit endpoint and raw + measurements when `evidence` supplies them. +- `(fail cause)` rejects the command with its cause and available partial trace. + +Successful completion requires a call to `complete`. A returned Promise forwards +its rejection to `fail`. The execution records diagnostics for repeated +completion calls while it is active. Its guard checks authorization at +completion, and the measured command records its event after output validation. + +The optional completion map has these fields: + +| Field | Meaning | +| --- | --- | +| `:end-ms` | Clock reading at the end of the work, using the same clock origin as the start. | +| `:metrics` | Namespaced metric keywords mapped to nonempty vectors of raw numbers. | +| `:trace` | Details of the work, such as traces of render calls and requested waits. | + +The command supplies `:command/elapsed`, its body duration in milliseconds. +`:metrics` supplies additional quantities. Command events preserve zero, +negative, `NaN`, and infinite numeric values as raw measurements. + +Camera completion illustrates the callback interface. The current helpers copy +the protocol's captured endpoint and renderer call durations into the completion +map: + +```clojure +(defn- render-evidence + [evidence] + {:end-ms (:end-ms evidence) + :metrics {:renderer/call-elapsed (mapv :duration-ms (:slices evidence))} + :trace evidence}) + +(defn- camera-evidence + [evidence] + (-> (render-evidence evidence) + (assoc-in [:metrics :renderer/cache-call-elapsed] + (mapv :duration-ms (:cached-slices evidence))) + (assoc-in [:metrics :camera/finalization-to-full] + [(:time-to-full-ms evidence)]))) + +(defcommand pan + "Moves the view by `:dx` and `:dy` through `:steps` cached previews. + Waits `:settle-ms` milliseconds, ends the interaction and renders until Full." + {:async [complete fail]} + [rtx {:keys [dx dy] :as options}] + {:pre [(valid-pan? options) (valid-view? (pan-view (:view rtx) dx dy))]} + (let [target (pan-view (:view rtx) dx dy)] + (-> (run-camera rtx target options) + (.then (fn [evidence] + (complete (assoc rtx :view target) + (camera-evidence evidence)))) + (.catch fail)))) +``` + +The final renderer call can return before the Promise callback runs. The +`:end-ms` field carries that renderer endpoint through the scheduling delay. +For example, renderer work lasting 7 ms followed by a 500 ms callback delay +still supplies a renderer duration of 7 ms. A clock reading in the later +callback would measure 507 ms. The public commands table lists the metric types +each measured command records. + +### Composition and execution + +A composite combines commands. `:measured-variant false` defines its base +function and selects measurements through the calls in its body: + +```clojure +(defcommand selected-pans + {:measured-variant false} + [rtx] + (-> rtx + (camera/pan* {:dx 10 :dy 0 :steps 2 :settle-ms 5}) + (camera/pan {:dx 0 :dy 5 :steps 2 :settle-ms 5}) + (camera/pan* {:dx -10 :dy 0 :steps 2 :settle-ms 5}))) +``` + +All three calls update the view and renderer. The first and third record +events. Command functions resolve incoming Promises, so threading passes each +updated context to the next command. `rects/mixed-pans` declares this pattern: +two measured pans and one bare pan in main, then a measured `camera/restore*` +in reset. + +A command execution runs a function from one command context to the next, +authorizes its work, and collects events in completion order. Given a prepared +command context `rtx`: + +```clojure +(commands/execute rtx selected-pans) +``` + +The returned Promise resolves to an outcome. A successful outcome contains +`:outcome :completed`, the next `:context`, and ordered `:events`. A failed +outcome contains `:outcome :failed`, `:error`, and the events recorded before +failure. Existing Errors retain their identity and partial trace. Other thrown +values become Errors with the original value in `:thrown`. An active execution +can also record repeated completion calls in `:diagnostics`. + +An event has this shape: + +```clojure +{:command :benches.render-wasm.runtime.camera/pan + :metrics {:command/elapsed [74] + :renderer/call-elapsed [7 7] + :renderer/cache-call-elapsed [2 2] + :camera/finalization-to-full [33]} + :trace {:end-ms 74}} +``` + +One event represents one measured command call. Its metric vectors describe +the individual values that call produced. The command execution owns this +history; its guard authorizes work for that execution. Pass the resulting +context to `commands/execute` to start another command execution. + +### Case phases + +The browser executes preparation, main, and reset through the command executor. +Preparation builds and uploads the scene, runs its optional callback once, and +renders Full unless the case declares `:preparation {:render :none}`. A fresh +context prepares once for its attempt; a reused context prepares once for its +case. Each phase passes its returned context to the next phase. + +A successful preparation response contains its raw block in +`[:preparation :phases]`. A failed preparation returns its available blocks in +top-level `:phases`. An attempt response contains main and any declared reset +in `:phases`: + +```clojure +{:phase :main + :preparation-id "preparation-1" + :outcome :completed + :events [{:command :benches.render-wasm.runtime.camera/render-sync + :metrics {:command/elapsed [7]}}]} +``` + +Each block retains ordered command events, raw numeric vectors, and any +completion diagnostics. A phase can complete with an empty event vector. +Initialization and upload durations stay in preparation `:diagnostics`. +Automatic initial rendering retains its raw trace under +`[:diagnostics :setup-render]`. Measured commands in the callback supply the +preparation block's selected events. + +Reset runs after successful main. Its returned context supplies the next reused +attempt. A phase failure releases its owned context and returns the failed +block with earlier events and available traces. Completed preparation retains +its own evidence when main or reset fails. Any preparation, main, or reset +failure stops the case; the runner attempts no later slot. It disposes the +case's browser and starts the next case in a new browser when the run continues. + +### Runtime code + +[`runtime/commands.clj`](runtime/commands.clj) declares commands; +[`runtime/commands.cljs`](runtime/commands.cljs) executes them. +[`runtime/camera.cljs`](runtime/camera.cljs) defines rendering and camera +commands. [`runtime/protocol.cljs`](runtime/protocol.cljs) defines render +drains, camera sessions, hook contracts, and trace fields. + +Case declarations such as [`scenes/rects.cljs`](scenes/rects.cljs) attach case +bodies through [`declarations.cljc`](declarations.cljc). Node loads these +declarations and functions during catalogue collection. Renderer commands run +through the supplied hooks when the functions execute in the browser. +[`browser/bridge.cljs`](browser/bridge.cljs) constructs the runtime context for +case bodies and prepares graphics contexts, snapshots, and uploads. Reused +cases carry their returned context into the next attempt; their reset function +controls restoration. + +## Measurement + +### Preparation and measurement boundaries + +Each duration has a defined start and endpoint. Setup and diagnostics run +around the interval whose work they prepare or describe. + +Work around those intervals includes: + +- scene build, snapshot validation, and fingerprinting; +- upload order derivation; +- request and result serialization; +- validation, logging, and screenshots; +- module script import; +- the automatic initial Full render during preparation. + +Preparation diagnostics time: + +- module instantiation; +- graphics context creation and initial size synchronization; +- shape batch upload, including the loading lifecycle and first view setup; +- the automatic render drain to `Full`. + +Measured commands record their own intervals during preparation, main, or reset. +For `pan` and `zoom`, this includes view updates, cached previews, settling, +ending the interaction, and the final drain. + +### Lifecycle of a fresh load + +```mermaid +flowchart TD + A[Import module script - untimed] --> B[Time module start] + B --> C[Time graphics start] + C --> D[Build and validate snapshot - untimed] + D --> E[Fingerprint and derive upload order - untimed] + E --> F[Time upload and first view setup] + F --> G[Run preparation callback] + G --> H[Render initial Full unless render is none] + H --> I[Run main] + I --> J[Run reset after successful main] + J --> K[Return raw phase records] + K --> L[Dispose page and context] +``` + +The module script import is untimed because the browser loads it once per page. +Module instantiation starts a new renderer state. Graphics start creates the +WebGL2 context and synchronizes the drawing buffer. Upload covers the loading +lifecycle, batch serialization, and tile preparation. Each measured command +starts its clock immediately before its work. + +### Warm interactions + +A warm case owns one prepared page. Its main body starts from the context +returned by preparation or the preceding attempt's reset. A `pan` or `zoom` +command runs this session: + +1. start the camera interaction; +2. for each view update, wait for the next animation frame, apply the view, and + present cached pixels; +3. wait for the requested settling time; +4. end the camera interaction; +5. render until `Full`. + +A pure pan can reuse cached textures while it updates which tiles are visible. +A zoom rebuilds and invalidates tile textures. The standard pan and zoom cases +request a 100 ms settling wait; `rects/mixed-pans` uses 50 ms for its three +pans. They record the requested and observed durations separately from renderer +calls. Camera finalization includes the call that ends the interaction; the run +also records that call's duration. + +### Slices and metrics + +The browser returns ordered events within each phase. Preparation diagnostics +retain `module-init-ms`, `graphics-init-ms`, and `upload-ms` separately. + +A render drain retains these trace fields: + +- `slices`: one record per renderer call with `timestamp`, `flags`, + `frame-type`, and `duration-ms`; +- `viewport-ready-ms`: the first `ViewportReady` or `Full` completion from the + drain origin; +- `full-ms`: the `Full` completion from the drain origin. + +A `pan` or `zoom` command's trace contains the camera session and final drain: + +- `cached-slices`: one record per gesture frame with `timestamp`, + `duration-ms` for the cached preview call, and `view-and-preview-ms` for the + view update plus preview call; +- `slices`: the final drain, with the same shape as a fresh drain; +- `interact-ms`: the whole camera session, from before the interaction starts + through the final `Full`; +- `active-ms`: from before the interaction starts to after the last cached + preview and its guard check, or after an explicit pause that follows it; +- `settling-requested-ms` and `settling-actual-ms`: the declared and observed + settling waits; +- `set-view-end-ms`: elapsed time during the call that ends the interaction; +- `time-to-viewport-ready-ms` and `time-to-full-ms`: durations from camera + finalization through the first `ViewportReady` or `Full`, and through `Full`, + respectively; +- `last-input-to-full-ms`: from before the last view update to the endpoint of + the renderer call that returns `Full`. + +Both drains and camera sessions return `end-ms`, the clock reading immediately +after the final renderer call returns `Full`. Completion durations end at that +reading. Promise callbacks can run later as the browser schedules queued work. + +A `load` or `restore` command uses the final drain trace directly. `load` also +records `:renderer/prepared-scene-to-full` from `:full-ms`. `restore` starts its +drain immediately with `SyncTiles` and records the same per-call durations as the +final drain. + +Measured command events keep each metric's raw values in a vector. Repeated +commands produce separate events, including when they name the same metric. +Raw values preserve zero, negative, and non-finite measurements in the saved +record. Reports sum one metric type within one phase execution before computing +statistics; saved records contain no aggregates. + +### Renderer completion + +The renderer finishes in tile budgets. One renderer call returns a frame type, +and the caller continues while the renderer reports `Partial` or +`ViewportReady`. The drain completes when a call returns `Full`. `None` or an +unknown value fails the attempt. + +| Value | Frame type | Meaning | +| ----: | --------------- | -------------------------------------------------------------- | +| 0 | `None` | no frame | +| 1 | `Partial` | visible work remains | +| 2 | `Full` | the renderer finished the current request | +| 3 | `ViewportReady` | the visible viewport was submitted; work outside it may remain | + +`Full` marks renderer submission of the completed request. Skia and WebGL queue +GPU work, and GPU execution and display follow their own schedules. A call +duration measures synchronous work on the calling thread, including any driver +stall. These clock readings describe work through the renderer return. + +For scheduled calls, the browser passes the timestamp of the animation frame. +For immediate calls, it reads the clock. Each slice records its timestamp to +describe the call sequence. + +### Graphics environment + +The benchmark uses WebGL2 with `antialias=false`, `depth=true`, `stencil=true`, +`alpha=true`, and `preserveDrawingBuffer=true`. The default viewport is 1920 by +1080 at device pixel ratio 2. The preparation records the effective CSS size, +drawing-buffer size, device pixel ratio, context attributes, renderer and vendor +strings, and a software rendering warning. + +Benchmarks run inside the Linux development container. Start it with +`./manage.sh run-devenv --gpu` to enable hardware access. This sets +`PENPOT_GPU=1`; the runner then selects ANGLE Vulkan, and `pnpm run render-bench` +starts Xvfb to supply the display that hardware rendering needs in the container. + +`--require-gpu` makes `run` and `ab` require hardware WebGL2. The command first +checks for `PENPOT_GPU=1`, before builds, catalogue checks, or output allocation. +After each preparation returns, the runner checks its existing +`:effective-graphics` metadata before executing attempts. It requires WebGL2, +usable vendor and renderer identities, and `:software false`. A rejected +preparation aborts the run with exit 1 and stops later A/B configurations. The +record retains the preparation and graphics failure evidence, and leaves pending +attempt slots unattempted. The failure names the observed renderer and the +commands needed to enable GPU access. `ab --dry-run` skips these GPU checks. + +Each preparation records the graphics identity of its own context. The run stores +the GPU policy and environment signal under diagnostics. Default runs allow +SwiftShader fallback and warn when they record software +rendering. Comparisons check effective graphics settings and resolved launch +arguments. Comparing hardware and software runs requires +`--allow-all-differences`. + +## Results and reports + +### Run record + +A run record contains: + +- the plan: selected case descriptors, warmup count, repetition count, and + timeout policy (`:case-ms`); +- metadata: Git revision and dirty flag, build facts, environment, and diagnostics; +- the resolved analysis settings; +- saved quantity descriptions; +- preparations in chronological order; +- attempts in chronological order, each with its case, preparation, + `:attempt-id`, warmup flag, outcome, main/reset phase blocks, and failure or + partial evidence; +- terminal outcomes in `:case-outcomes`; +- remaining slots for each case; +- the termination reason. + +A preparation records its case, warmup flag, setup diagnostics, phase history, +and any failure. A completed preparation also has its fingerprint and effective +graphics. An attempt records one outcome and its main and optional reset phase +blocks. A phase block retains ordered command events, raw metric vectors, and +optional completion diagnostics. Errors retain available phase evidence and +traces after a failure. + +Each case has a terminal entry in `:case-outcomes` with `:outcome` set to +`:completed`, `:failed`, `:timed-out`, or `:interrupted` and an optional +`:failure`. Recording a terminal outcome closes the case: later preparation or +attempt evidence for that case is rejected. + +### Failures and termination + +A failed attempt still consumes its allocated slot. A fresh preparation failure +records a failed attempt and consumes the current slot. A reuse preparation +failure records no main or reset execution and consumes no main slot. Any +preparation, main, or reset failure stops the case for fresh and reuse alike, +so later slots stay unattempted. The runner disposes the failed case's browser +and starts the next case in a new browser when the run continues. A browser or +page failure follows the same case-stop rule. A failure to clean up the browser +for a failed case aborts later cases. The run fixes its attempt count before +execution and uses each allocated slot at most once. + +A malformed delivered response follows the same slot rules. An attempt response +consumes the slot it invoked, keeps the valid phase prefix, and saves the raw +response as partial evidence. A malformed preparation response fails its +preparation; a fresh case still consumes the current slot. + +A screenshot failure is diagnostic when it is an ordinary error. A deadline +expiry or interruption during the screenshot stops the case, while earlier +completed attempts remain saved. + +A case that finishes all its slots without a measured command gets a failed +case outcome whose failure has `:reason :no-measurements`. The run then +terminates as failed or incomplete according to the remaining slots. + +A run terminates in one of four states: + +- **completed**: every allocated slot was attempted, every case outcome is + `:completed`, and every case has at least one measurement. +- **failed**: every allocated slot was attempted and at least one attempt failed + or one case outcome is not `:completed`. +- **incomplete**: slots remain because a case stopped early or a startup, + browser, or storage failure prevented execution. +- **interrupted**: `SIGINT` or `SIGTERM` stopped the run. + +The runner saves partial records before it exits. Use exit status to determine +whether the command completed successfully; saved records also describe work +that ended in failure. + +### Summaries + +A summary reports one entry per case and metric key +`[phase scope metric-type]`. The scope is the command that recorded the metric, +or `:global` for every command with that type in the phase. For each key, the +report sums the raw values of one metric type within each eligible phase +execution and computes statistics across those execution sums. It labels the +reduction `:statistic :sum`. + +For each metric key, the report includes: + +- the observed mean, median, minimum, maximum, sample standard deviation, and + median absolute deviation; +- a bootstrap confidence interval for the median. The interval comes from + repeated resampling of the observed execution sums; +- observed p95 and p99 values when the sample is large enough; +- counts for the valid, invalid, absent, failed, unattempted, and warmup + categories, plus command invocation and raw-value counts; +- the saved quantity unit and optional description, and any validity + diagnostics. + +Statistical `n` is the valid count. Invocations count command events and +raw-values count all recorded numbers across eligible executions; neither is the +number of observations. Warmups and failed attempts are excluded. A failed +attempt excludes both its main and reset blocks. A completed preparation remains +eligible after a later main or reset failure. A fresh warmup preparation is +excluded; one reuse preparation supplies one observation, not one per attempt. + +A finite zero is valid. Negative and non-finite raw values remain in the saved +record but invalidate the dependent execution sum and aggregate; independent +metric keys remain usable. Unknown or malformed quantity metadata appears as a +diagnostic and leaves that key without usable statistics. A phase can complete +with no events. Preparation-only and reset-only measured cases are valid. A +case that finishes all its slots without any measured command gets a failed +case outcome with `:reason :no-measurements`; the run is failed when every slot +was attempted and incomplete when slots remain. + +The case report also counts actual preparation, main, and reset executions. +Completed and failed counts use each execution's own outcome, and a separate +count records warmup executions. These phase counts do not filter by attempt +eligibility. A completed main block from an attempt whose reset failed still +appears in the main phase counts but supplies no observation. + +The default analysis settings are: + +| Setting | Default | +| ----------- | ------------ | +| Draws | 2000 | +| Seed | `0x70656e70` | +| Confidence | 0.95 | +| Minimum CI | 10 | +| Minimum p95 | 100 | +| Minimum p99 | 500 | + +The run records draw count, seed, confidence, and minimum counts, and a report +can override them at analysis time. An estimate that cannot be computed keeps a +reason and, when applicable, the minimum count it needs. + +The intervals are exploratory and conditional on available executions. +Incomplete and timed-out cases can select which observations survive. Attempts +within one run share evolving caches and can have dependent durations. Repeated +whole runs provide evidence about reproducibility across those cache conditions. + +### Comparisons + +A comparison pairs saved cases by identical case id unless `--cases A,B` +selects one explicit baseline, candidate pair. The CLI checks the selector +syntax before it loads either record, then resolves each selector by saved case +id or exact name. + +By default, a comparison requires equal: + +- scene identity, scene seed, and scene parameters; +- starting-snapshot fingerprints; +- view, context policy, and preparation settings; +- effective graphics settings and runtime environment; +- build mode, effective features, instrumentation settings, and recorded build + environment; +- warmup count, timeout policy (`:case-ms`), and saved quantity units. + +Comparison pairs metric keys `[phase scope metric-type]`. Quantity description +text does not affect compatibility; changed prose is not a difference. Unknown +or malformed quantity metadata appears in the per-side report diagnostics and +leaves that key without usable statistics. + +Git revision, dirty state, run ids, timestamps, and saved analysis defaults +provide provenance for interpreting the paired cases. + +The comparison reports absolute and percentage median changes between execution +sums, with a bootstrap interval for each change. A zero baseline suppresses a +percentage change. The comparison leaves a metric key with differing quantity +units or incomplete quantity evidence out unless the all-differences +permission allows its valid raw values. + +### Comparison permissions + +The comparison records every compatibility difference. A permission acknowledges +a class of differences and labels each acknowledged +mismatch with the rule that allowed it. + +- `--allow-build-differences` acknowledges recorded build settings, effective + features, instrumentation, and the recorded build environment. Runtime + environment, graphics, and saved quantity units stay strict. +- `--allow-scene-differences` acknowledges scene identity, scene seed, scene + parameters, and differences in fingerprint content. Execution settings stay + strict. Missing or varying fingerprint evidence still needs the + all-differences permission. +- `--allow-all-differences` acknowledges every compatibility difference, + including mixed quantity units. The report compares valid raw execution sums + in their recorded units. Record validation and measurement validity still + govern the analysis. + +Build and scene permissions combine by union. All-differences includes both. +A report shows the selected permissions, every recorded difference, and the +rule that acknowledged it. Exit status reports execution and compatibility +checks; the estimates describe the observed timing differences. + +## A/B feature experiments + +An A/B experiment answers one question: how do the selected cases behave when a +renderer feature is off versus on, with everything else held constant? + +The `ab` command takes the requested features, plans the configurations, builds +every configuration, runs every configuration, and compares every edge: a pair +of configurations that differ by one feature. + +### Planning + +A feature configuration is a set of enabled Cargo features. The planner: + +1. normalizes the requested names by trimming, deduplicating, and sorting them; +2. validates them against the renderer manifest with Cargo metadata; +3. requires an empty Cargo default feature set, so each configuration states + its features explicitly; +4. rejects diagnostic features used for profiling or statistics; +5. orders the configurations in reflected Gray order, so adjacent + configurations differ by one feature, with the first requested feature as the + most significant bit; +6. emits every pair of configurations that differ in exactly one feature, from + the configuration with that feature off to the configuration with it on. + +The command requires at least one requested feature. For $N$ features, the +comparison has $2^N$ configurations and $N 2^{N-1}$ edges. The planner +compares each edge once, in the direction from off to on. For features +`[a, b]`, the configurations are `00, 01, 11, 10` and the edges are: + +```text +00 -> 10 : a +00 -> 01 : b +01 -> 11 : a +10 -> 11 : b +``` + +`--max-features` accepts 1 through 6 and defaults to 3. When `--dry-run` +is absent, the command requires `--confirm` for more than 8 configurations +or a filter that is absent, empty, or contains only whitespace. `--dry-run` +checks the requested features using Cargo metadata, selects cases from their +declarations, and prints the features, configurations, edges, selected case +count, and time limits. It creates no comparison directory, compiles no build, +and launches no browser. + +### Running a feature comparison + +Every configuration gets its own build, catalogue, browser executor, server, +browser, and run. Each configuration runs and cleans up before the next one +starts. The configurations share the selected cases and their seeds, parameters, +views, context policies, preparation settings, and case deadlines. Collection +checks each configuration's catalogue identity and requires the same selected +case descriptors in the same order. + +```bash +pnpm run build:renderer-benchmarks:runner + +pnpm run render-bench ab \ + --features , \ + --filter rects/load \ + --warmups 3 --repetitions 10 +``` + +The feature names must exist in the renderer manifest. Placeholders appear here +because the manifest decides which features are available. + +After a configuration records attempts, scheduling continues with its recorded +failures and remaining slots. A build failure, a catalogue +identity failure, or a run with zero attempts stops scheduling later +configurations; those configurations are recorded as unattempted with a reason. +`SIGINT` and `SIGTERM` save the available state and return 130 or 143. + +### Feature-comparison output + +The default output directory is +`target/performance/render-wasm/feature-comparisons//`. It contains: + +```text +feature-comparison.transit plan, progress, edges, termination +feature-comparison.log comparison progress and diagnostics +configs//run.transit the configuration's run record +configs//run.log the configuration's log +configs//*.png optional screenshots +comparisons/__.transit one edge comparison +``` + +The feature-comparison record stores the normalized options, comparison +permissions, selected cases, configuration order, requested and effective +features, build and run references, edges, failures, and interpretation +warnings. The runner validates the record with a schema and writes it +atomically. It writes the initial plan before the first build and updates the +record after each configuration, after each edge, and at termination. The +command prints `Comparison `, `Output `, and `Status ` +on stdout at termination. When a failed outcome has an empty error list, the +command appends a `Failures` section with saved reasons and artefact references. +The section shows up to eight failed configurations and comparisons, then names +the saved record and log with the complete evidence. Progress and runtime +failures go to stderr. Each configuration has its own build and run log. + +### Edge comparisons + +An edge comparison uses the ordinary comparison path. It automatically +acknowledges one difference: the effective feature change that matches the +planner's evidence. The evidence records the requested feature, both effective +feature sets, and the added and removed features. It also requires the two +configurations' requested feature selections to differ only by the requested +toggle and to record the same default-feature policy. If the saved sets do not +match that evidence, the difference remains unacknowledged. + +The comparison record also carries the edge context: the source and target +configuration slugs and the expected feature facts. An edge is analysed when +both endpoint records exist, even if one run recorded failed attempts. A missing +endpoint or an analysis error produces an explicit edge outcome. An edge that +retains an unacknowledged difference keeps its saved report, is marked failed, +and makes the invocation exit 1. + +Other build differences require `--allow-build-differences`. Scene differences +require `--allow-scene-differences`. View, context, preparation, graphics, +runtime environment, and saved quantity-unit differences stay strict unless +`--allow-all-differences` is selected. If two configurations report the same effective feature set, the +comparison includes their edge and labels it with that fact. Effective feature +sets describe the code settings that accompany the observed timings. + +The command returns 0 when every configuration completes, every edge analysis +finishes, and every difference is acknowledged. It returns 1 for execution, +persistence, or analysis failures, including unacknowledged differences. Edge +reports show raw times and statistics for the observed differences. Establishing +a causal effect requires experimental control and repeated measurements. + +## How to run + +Run these commands from `frontend/` inside the development container with +dependencies and Playwright Chromium installed. Keep renderer watchers stopped +until preparation finishes. + +List the commands, or show one command's options and defaults: + +```bash +pnpm run render-benchhelp +pnpm run render-bench run --help +``` + +Command options select declared cases and control the build source, sampling, +output and hardware policy. Case declarations own workload bodies, scene +parameters, view, context policy, preparation, reset, and case lifecycle. +`--seed` is the explicit exception: it changes generated scene content +reproducibly, but it does not replace a case body, its command arguments, or its +lifecycle settings. + +Build the runner and its command module: + +```bash +pnpm run build:renderer-benchmarks:runner +``` + +Run one case with one measured attempt: + +```bash +pnpm run render-bench run \ + --filter paths/pan --seed 42 --warmups 0 --repetitions 1 --screenshots +``` + +Run the blocking renderer and mixed gesture proof cases: + +```bash +pnpm run render-bench run \ + --filter rects/blocking-full --seed 42 \ + --warmups 0 --repetitions 1 --screenshots + +pnpm run render-bench run \ + --filter rects/mixed-pans --seed 42 \ + --warmups 0 --repetitions 1 --screenshots +``` + +Prepare a build once and reuse it: + +```bash +pnpm run render-bench build +pnpm run render-bench run --build +``` + +Run a selection with the default sampling: + +```bash +pnpm run render-bench run --filter rects +``` + +Require hardware rendering for rectangle load, pan, and zoom in a GPU container: + +```bash +pnpm run render-bench run --filter rects --require-gpu +``` + +For a functional check, add `--warmups 0 --repetitions 1`. A successful +run renders the scenes through `Full` and saves the vendor and renderer strings +with `:software false` in each preparation's `:effective-graphics`. + +Choose a new path for `--output-dir`. The runner rejects an existing output +directory. A/B feature comparisons apply the same rule. + +Compare two saved runs offline: + +```bash +pnpm run render-bench compare \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit +``` + +Allow a difference in build configuration, or compare two given scenes, +irrespective of their differences: + +```bash +pnpm run render-bench compare \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit \ + --allow-build-differences + +pnpm run render-bench compare \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit \ + --cases rects/load,paths/load --allow-scene-differences +``` + +Merge an additional saved run into a base run: + +```bash +pnpm run render-bench merge \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit \ + --output-dir target/performance/render-wasm/runs/ +``` + +The merged record gets a new run id and a new run directory. Both input records +stay unchanged. Both inputs must be completed and compatible: same cases, +timeout policy, analysis, build facts, runtime environment, and Git provenance. +The additional run must use the same warmup count. `merge` appends only the +additional measured attempts and uses the base run's warmups. The additional +warmup attempts stay in the additional record. The command starts no build, +server, or browser. + +Inspect and clean managed outputs: + +```bash +pnpm run render-bench purge builds +pnpm run render-bench purge runs +pnpm run render-bench purge comparisons +pnpm run render-bench purge all +``` + +`purge` requires exactly one target: `builds`, `runs`, `comparisons`, or `all`. +It removes known build, standalone-run, or feature-comparison outputs and keeps +directories and unrelated files. It follows directory symlinks and can remove +matching output files from their targets, including targets outside the managed +directories. `all` removes every category. A missing, unknown or multiple target +is an argument error. + +The commands write results to stdout: + +- `build`: `Build ` and `Output ` after build preparation + ends. +- `run`: `Run ` and `Output ` after the run ends. When a + report is available, a `Report` heading and the formatted summary follow. +- `ab`: `Comparison `, `Output `, and final + `Status `. A failed outcome with an empty error list also prints a + bounded `Failures` section from the saved comparison record. +- `compare`: the comparison report. +- `merge`: `Run ` and `Output `. +- `--help` and `COMMAND --help`: usage text with that command's options and + defaults. `purge` produces no output on success. + +Each CLI command produces an outcome with an exit code, errors, log path, and +optional report text. The runner prints the final output once and returns the +numeric status. + +Progress and diagnostics go to stderr. The default shows build stages, case +headings, and attempt progress. `-q` / `--quiet` shows results, warnings, and +errors. `-v` / `--verbose` also shows debug details and raw build output. Quiet +and verbose are mutually exclusive. These options apply to every command. + +`--color` forces terminal colours; `--no-color` disables them. Automatic colour +uses a terminal stream when `NO_COLOR` is empty or absent and `TERM` is not +`dumb`. Colour choices apply to terminal diagnostics. Result text and file logs +use plain text. The runner reads child output as lines and normalizes their +line endings. Verbose terminal output prefixes and dims these lines; file logs +store the lines as text. Terminal errors include a limited extract from the end +of the log when available. + +Files receive every log level, including debug output hidden from the console. +Each build writes `build.log`; each A/B comparison writes +`feature-comparison.log`. The default run directory is +`target/performance/render-wasm/runs//`, which contains `run.log`, +`run.transit`, and optional screenshots. + +Exit 0 means the command completed successfully: a ready build for `build`, +every allocated attempt for `run`, a valid comparison for `compare`, a +validated merged record for `merge`, every configuration and edge for `ab`, and +a clean purge for `purge`. Exit 1 means a failed preparation, a failed attempt, +an incomplete run, an invalid selection, incompatible merge inputs, an +unacknowledged comparison difference, or a harness failure. Exits 130 and 143 +mean interruption by `SIGINT` and `SIGTERM` for `build`, `run` and `ab`. +Interruption uses those statuses to report aborted work. Errors writing the +command's record or log also produce diagnostics. + +`--warmups` defaults to 3, `--repetitions` to 10, `--timeout-ms` (the case +deadline) to 30000, and `--startup-timeout-ms` (host and initial browser +readiness) to 30000. `build`, a run that prepares its build, and an A/B +experiment use a default `--build-timeout-ms` of 300000; a run with `--build` +uses the retained build's recorded provenance. The master seed defaults to 42 +and accepts a 32-bit unsigned integer. A filter matches a substring of the case +id. + +## Unit tests + +Run these commands from `frontend/` inside the development container with +dependencies and Playwright Chromium installed. The benchmark tests use the +`performance-tests` build target and the runner in +`test/performance_tests/render_wasm/runner.cljs`. + +Build and run the full suite: + +```bash +pnpm run test:renderer-benchmarks +``` + +Build once, then run the compiled tests: + +```bash +pnpm run build:renderer-benchmarks:tests +node target/performance-tests/test.js +``` + +Focus on one namespace or one test var: + +```bash +node target/performance-tests/test.js \ + --focus performance-tests.render-wasm.contracts-test + +node target/performance-tests/test.js \ + --focus performance-tests.render-wasm.contracts-test/defscene-registers-metadata-and-build +``` + +Rebuild the test bundle after changing source or tests. `--focus` accepts one +namespace or one `namespace/test-var`. The runner exits 0 when the selected tests +pass and 1 when they fail. + +## Scope and limitations + +The suite records durations for setup, render calls, and camera movement inside +Chromium. +Rendering durations end when the renderer returns after submitting work. +Memory, power, frame rate, frame drops, visual correctness, network latency, +and editor input-to-paint each require measurements for those quantities. + +A fresh case starts a new page, module, and WebGL2 graphics context. The browser +process persists across successful cases, and module script import precedes +module timing. These startup conditions define measurements for fresh cases. + +Attempts share evolving caches, so bootstrap intervals describe the observations +within a run. Repeat whole runs to assess reproducibility, and judge whether the +observed change matters for the workload. + + +## Repository map + +```text +frontend/performance/benches/render_wasm/ +├── browser/ page entry and browser bridge +├── features/ feature selection, configurations and comparison edges +├── node/ +│ ├── catalog/ compiled catalog entry and retained catalog loader +│ ├── cli/ CLI entry, resolution, requests and execution lifecycle +│ │ └── commands/ per-command schemas, help and resolution +│ ├── report/ saved-record comparison and merge +│ ├── runner/ build preparation, server, browser host and run orchestration +│ └── storage/ artifacts and saved runs +├── report/ summaries, comparisons, statistics, formatting +├── runtime/ renderer commands, render drains, and camera sessions +├── scenes/ scene recipes and their cases +├── builder.cljc scene construction +├── declarations.cljc scene and case registry, schemas, seed derivation +├── cases.cljs case discovery and collection +├── codec.cljs Transit round-trip validation +├── errors.cljc failure values and diagnostics +├── snapshot.cljs snapshot validation and upload order +├── fingerprint.cljs digest of the starting snapshot +├── identity.cljc build identity and bundle compatibility checks +├── measurement.cljs projection of phase evidence into attempts +├── random.cljs seeded random generators +├── result.cljs raw run, comparison and feature-comparison records +├── README.md this document +└── REPORTING.md raw records, metrics, and statistics + +frontend/test/performance_tests/render_wasm/ +├── runner.cljs test entry point, namespace registration, focus selection +├── *_test.cljs tests for benchmark modules +├── test_helpers.cljs shared scene-test helpers +└── cli_integration.cjs tests for the CLI through Node + +frontend/package.json build and test commands +frontend/shadow-cljs.edn benchmark and performance-tests build targets +frontend/src/app/render_wasm/ production renderer wrapper +render-wasm/ Rust renderer and build +``` + +## Further reading + +- [`REPORTING.md`](REPORTING.md) describes the record schemas, raw command + histories, quantity metadata, and statistical calculations in detail. This + guide is the source for the command line and the A/B workflow. +- [`render-wasm/README.md`](../../../../render-wasm/README.md) and the renderer + source describe the Rust side. +- [`frontend/src/app/render_wasm/`](../../../src/app/render_wasm/) contains the + production renderer orchestration whose renderer bindings and upload + serializers the benchmark shares. diff --git a/frontend/performance/benches/render_wasm/browser/bridge.cljs b/frontend/performance/benches/render_wasm/browser/bridge.cljs new file mode 100644 index 0000000000..64f2857a47 --- /dev/null +++ b/frontend/performance/benches/render_wasm/browser/bridge.cljs @@ -0,0 +1,1255 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.browser.bridge + "Prepares renderer scenes and executes benchmark case phases in the browser. + + A scene is a recipe that builds a snapshot. A snapshot contains the concrete + shapes in `:objects` and label references in `:refs`. A case pairs a scene + with parameters, a camera view and execution settings. A collected case + descriptor reaches the browser through Transit; this namespace resolves the + matching local registration, which supplies the `run!` function and optional + preparation callback and reset function. + + A preparation owns one renderer lifetime: the instantiated module, canvas, + WebGL2 context, uploaded snapshot and command context. It loads the module, + creates graphics, builds and uploads the scene, runs the preparation callback + once, and then renders an initial Full frame unless the case declares + `:preparation {:render :none}`. Fresh cases prepare once per attempt. Reused + cases prepare once and carry the returned view and command context into later + attempts. + + Execution has up to three phases: preparation, main and reset. Main runs the + case's `run!` function. Reset follows each successful main execution. A + phase's raw block contains `:phase`, `:preparation-id`, `:outcome`, + selected command `:events` and, when present, `:diagnostics`. A successful + preparation exposes its blocks at `[:preparation :phases]`. A failed + preparation places its available blocks at the top level. An attempt places + main and optional reset blocks at the top level and keeps the preparation + evidence captured for that attempt. + + A command context has exactly four fields: `:case`, the collected case + descriptor; `:scene`, the snapshot; `:view`, the camera scale and offsets; + and `:hooks`, the functions for time, scheduling, checks and renderer + operations. Every command receives the full hook map with its context. The + context, module, graphics handles, canvas and snapshot stay in the browser. + Only projected evidence, such as preparation data, phase blocks, identity + data and summary or comparison data, crosses Transit. + + An owner is the current page lifetime. Lifecycle guards compare a captured + owner epoch with the current epoch; a mismatch throws a top-level + `::errors/stale` marker. A newer load, disposal or WebGL context loss + increments the epoch and supersedes the old owner. Only a top-level + `::errors/stale` marker yields a stale result. A superseded operation that + rejects for another reason returns failed evidence for that operation and + leaves the newer owner in place; a failed attempt keeps its own preparation + and phase blocks. An ordinary failure in the current owner ends that owner + and returns its phase, cause and available partial evidence. + + Renderer-call measurements end when each call returns. Setup diagnostics + retain separate intervals for module initialization, graphics setup, upload + and initial rendering. GPU execution and display proceed asynchronously + after renderer submission. + + The exported `bridge` object maps the names used by the runner to this public + interface; the runner calls it through `page.evaluate`. Report formatting + entries return text; the other entries return Transit text or, for `ping` and + `dispose`, status objects." + (:require-macros + [app.common.exceptions :as ex]) + (:require + [app.common.render-wasm.helpers :as h] + [app.common.render-wasm.serialize-shape :as serialize-shape] + [app.common.render-wasm.serializers :as sr] + [app.common.render-wasm.wasm :as wasm] + [app.common.schema :as sm] + [app.common.transit :as t] + ;; bind the enum table from the prepared renderer build's `shared.js` so + ;; a missing `shared.js` fails the compile. A stale or mismatched one throws + ;; at load + [app.render-wasm.api.enums] + [app.render-wasm.api.webgl :as webgl] + [benches.render-wasm.cases] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.fingerprint :as fingerprint] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [benches.render-wasm.runtime.commands :as commands] + [benches.render-wasm.runtime.protocol :as protocol] + [benches.render-wasm.snapshot :as snapshot])) + +;; Forward decls +(declare now guard-current! read-graphics) + +(def schema:load-scene-args + "Decoded form of the Transit request accepted by `load-scene` and + `prepare-scene-request`. + + `:case` is a collected case descriptor. `:preparation-id` labels the + preparation and defaults to the owner epoch. `:timeout-ms` bounds setup and, + through the stored preparation, later attempts that omit their own timeout. + `:module-url` and `:wasm-url` override the renderer artifacts." + [:map {:closed true} + [:case decl/schema:collected-case] + [:preparation-id {:optional true} some?] + [:timeout-ms {:optional true} [:int {:min 1}]] + [:module-url {:optional true} string?] + [:wasm-url {:optional true} string?]]) + +(def ^:const ^:private case-ms-budget + "Default elapsed-time deadline for setup or one attempt, checked by + `browser-hooks`. Requests can supply a timeout. The outer process timeout + covers blocked pages." + 30000) + +(defn- browser-hooks + "Returns the command hooks for one owner epoch and deadline. + + Args: + epoch: owner epoch that `:check` compares with the current epoch. + deadline-ms: deadline in milliseconds on the `now` clock. + + Returns: + A map with `:now`, `:frame`, `:sleep`, `:check`, `:render`, + `:render-sync`, `:render-from-cache`, `:set-view`, `:set-view-start` and + `:set-view-end`. `:check` rejects work after the owner is superseded, the + WebGL context is lost or the deadline passes." + [epoch deadline-ms] + {:now now + ;; rAF scheduling + :frame #(js/Promise. + (fn [resolve] (.requestAnimationFrame js/window resolve))) + :sleep #(js/Promise. (fn [resolve] (js/setTimeout resolve %))) + ;; checks that this execution still owns the page, WebGL context hasn't been + ;; lost, and the deadline hasn't passed + :check (fn [] + (when @wasm/context-lost? + (throw (ex-info "WebGL context lost" + {:phase "context-lost"}))) + (guard-current! epoch "protocol") + (when (>= (now) deadline-ms) + (throw (ex-info "case deadline exceeded" + {:phase "deadline"})))) + ;; WASM calls + :render (fn [timestamp flags] + (h/call wasm/internal-module "_render" timestamp flags)) + :render-sync #(h/call wasm/internal-module "_render_sync") + :render-from-cache #(h/call wasm/internal-module "_render_from_cache" 0) + :set-view #(h/call wasm/internal-module "_set_view" + (:scale %) (- (:x %)) (- (:y %))) + :set-view-start #(h/call wasm/internal-module "_set_view_start") + :set-view-end #(h/call wasm/internal-module "_set_view_end")}) + + +;; Monotonic owner generation. Async continuations capture the current epoch +;; and compare it before acting; a mismatch marks their work stale. +(defonce ^:private owner-epoch* + (atom 0)) + +;; Cached Emscripten import. Import precedes module-init timing (untimed). +(defonce ^:private module-promise* + (atom nil)) + +;; Page-scoped canvas owned by the current load. :fresh cases load once per +;; page. :reuse cases keep one preparation across all attempts. +(defonce ^:private canvas* + (atom nil)) + +;; Current completed preparation for `run-prepared`: owner, case, reusable +;; command context and browser-local graphics. +(defonce ^:private prepared* + (atom nil)) + + +(defn- mark-live! + "Marks the current renderer context as initialized and available." + [] + (reset! wasm/context-lost? false) + (set! wasm/context-initialized? true)) + +(defn- now + "Returns the browser's monotonic clock in milliseconds." + [] + (.now js/performance)) + +(defn- ensure-module! + "Imports the Emscripten glue and instantiates it with the prepared `.wasm` + bytes. + + The import is cached per page and untimed: script import precedes + module-init timing. `locateFile` points at `wasm-url`. + + Args: + module-url: URL of the Emscripten glue module. + wasm-url: URL of the `.wasm` file passed to `locateFile`. + + Returns: + A Promise of `{:module instance :factory-ms ms}`, where `:factory-ms` is + the instantiation duration in milliseconds. A failed import or factory + call rejects the Promise." + [module-url wasm-url] + (when (nil? @module-promise*) + (reset! module-promise* + (-> (js/dynamicImport module-url) + (.then (fn [module] + (let [t0 (now) + default-fn (unchecked-get module "default")] + (-> (default-fn #js {:locateFile (constantly wasm-url)}) + (.then (fn [instance] + {:module instance + :factory-ms (- (now) t0)}))))))))) + @module-promise*) + +(defn- graphics-string + "Reads a WebGL string parameter. + + Args: + context: WebGL context to query. + parameter: property name on `WEBGL_debug_renderer_info`, such as + `\"UNMASKED_RENDERER_WEBGL\"`. + + Returns: + The parameter's string value, or `\"unknown\"` when the extension is + unavailable or the read throws." + [context parameter] + (or (ex/ignoring + (when-let [ext (.getExtension ^js context "WEBGL_debug_renderer_info")] + (str (.getParameter ^js context (unchecked-get ext parameter))))) + "unknown")) + +(defn- renderer-string + "Reads the unmasked WebGL renderer identity. + + Args: + context: WebGL context to query. + + Returns: + The unmasked renderer text, or `\"unknown\"` when the debug extension is + unavailable or the read throws." + [context] + (graphics-string context "UNMASKED_RENDERER_WEBGL")) + +(defn- software-renderer? + "Recognizes common software graphics implementations from their renderer text. + + Args: + s: renderer text to inspect; nil is treated as empty. + + Returns: + `true` when the text names SwiftShader, llvmpipe, generic software or a + basic render, ignoring case; `false` otherwise." + [s] + (boolean (re-find #"(?i)swiftshader|llvmpipe|software|basic render" (or s "")))) + +(defn- teardown-canvas! + "Releases the context after successful initialization, then removes its + canvas. + + The initializer owns cleanup until it returns a context. + + Args: + canvas: canvas to remove; nil skips both steps. + + Returns: + nil after the cleanup." + [canvas] + (when (some? canvas) + (when-let [context wasm/gl-context] + (webgl/release-context! wasm/internal-module context wasm/gl-context-handle)) + (ex/ignoring (.remove ^js canvas)))) + +(defn- release-owner-resources! + "Releases the current owner's canvas, GL handles, module binding and + stored preparation. + + Returns: + nil after releasing the resources." + [] + (teardown-canvas! @canvas*) + (wasm/reset-context-state!) + ;; reset-context-state! deliberately preserves loss state for the editor + ;; reload path; bench disposal owns it explicitly. + (reset! wasm/context-lost? false) + (set! wasm/internal-module nil) + (reset! canvas* nil) + (reset! prepared* nil)) + +(defn dispose! + "Closes the current owner and returns a JavaScript status object. + + It bumps the epoch, so running continuations go stale, releases renderer + resources and drops the cached module promise so a later load re-imports + cleanly. Idempotent. Stale callers must not clean up a newer owner; use + `dispose-if-current!` from async continuations. + + Returns: + A JavaScript object with status `\"disposed\"`." + [] + (swap! owner-epoch* inc) + (release-owner-resources!) + (reset! module-promise* nil) + #js {"status" "disposed"}) + +(defn- dispose-if-current! + "Disposes only when `epoch` is still current. + + Args: + epoch: owner epoch captured by the caller. + + Returns: + `true` after disposing, or nil for stale work. Stale work leaves the newer + owner's canvas, GL context and module cache unchanged." + [epoch] + (when (= epoch @owner-epoch*) + (dispose!) + true)) + + +(defn- read-graphics + "Records the effective graphics settings for a live WebGL context. + + The result separates the requested CSS size and device pixel ratio from the + observed drawing-buffer size and the context's actual attributes. Requested + defaults live in `webgl/default-context-options`; the drawing-buffer size is + the context's actual `drawingBufferWidth` and `drawingBufferHeight`, not the + canvas attributes. Granted attributes come from `getContextAttributes`. + + Args: + ctx: live WebGL context to inspect. + width: requested CSS width in pixels. + height: requested CSS height in pixels. + dpr: requested device pixel ratio. + renderer: unmasked renderer text to record. + + Returns: + A map with the requested `:css-width`, `:css-height` and `:dpr`; the + observed `:drawing-buffer-width` and `:drawing-buffer-height`; `:webgl2` + set to `true`; the granted `:antialias`, `:depth`, `:stencil`, `:alpha` + and `:preserve-drawing-buffer` attributes; and the `:renderer`, `:vendor` + and `:software` identity fields." + [ctx width height dpr renderer] + (let [attrs (ex/ignoring (.getContextAttributes ^js ctx)) + attr #(when (some? attrs) + (unchecked-get attrs %)) + vendor (graphics-string ctx "UNMASKED_VENDOR_WEBGL")] + {:css-width width + :css-height height + :drawing-buffer-width (.-drawingBufferWidth ^js ctx) + :drawing-buffer-height (.-drawingBufferHeight ^js ctx) + :dpr dpr + :webgl2 true + :antialias (attr "antialias") + :depth (attr "depth") + :stencil (attr "stencil") + :alpha (attr "alpha") + :preserve-drawing-buffer (attr "preserveDrawingBuffer") + :renderer renderer + :vendor vendor + :software (software-renderer? renderer)})) + +(defn- install-listeners! + "Wires context lost/restored handling for `canvas`, guarded by the epoch. + + A stale canvas leaves the live owner untouched. On loss the current owner + releases its canvas and GL resources, clears the module promise, marks the + context lost and supersedes its epoch immediately. A restore event + supersedes the epoch when its owner still matches. + + Args: + canvas: canvas that receives the listeners. + epoch: owner epoch captured when the listeners are installed. + + Returns: + nil after installing both listeners." + [canvas epoch] + (.addEventListener + canvas "webglcontextlost" + (let [installed epoch] + (fn [e] + (.preventDefault ^js e) + (when (= installed @owner-epoch*) + (release-owner-resources!) + (reset! module-promise* nil) + (reset! wasm/context-lost? true) + (swap! owner-epoch* inc))))) + (.addEventListener + canvas "webglcontextrestored" + (let [installed epoch] + (fn [_] + (when (= installed @owner-epoch*) + (swap! owner-epoch* inc)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Public interface +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- terminal-failure + "Returns the bridge result for a rejected asynchronous step. + + Only a top-level `::errors/stale` marker decides staleness. A stale cause + returns `{:status \"stale\"}` and leaves the newer owner in place. Any other + cause disposes the owner when `epoch` is still current and returns failed + evidence. The evidence uses the phase from the cause, or `fallback-phase` + when absent, and keeps the original failure as plain data under `:cause`. + The function attaches stored preparation evidence when that preparation + belongs to `epoch`. + + Args: + epoch: owner epoch that the failed step belongs to. + cause: rejected value from the step. + fallback-phase: phase name used when the cause has no `:phase`. + + Returns: + `{:status \"stale\"}` for a top-level stale cause; otherwise failed + evidence with status `\"failed\"`, `:phase`, `:message`, `:cause`, optional + `:partial` data and optional `:preparation` evidence." + [epoch cause fallback-phase] + (if (errors/stale? cause) + {:status "stale"} + (let [phase (or (ex/ignoring (:phase (ex-data cause))) fallback-phase "aborted") + prepared @prepared* + preparation (when (= epoch (get-in prepared [:owner :epoch])) + (get-in prepared [:graphics :preparation]))] + (dispose-if-current! epoch) + (cond-> (errors/fail-data (errors/evidence phase cause)) + preparation (assoc :preparation preparation))))) + +(defn- guard-current! + "Throws the stale marker when `epoch` no longer owns the page. + + Callers pass their phase so the terminal handler can tell superseded work + from genuine failures. The marker is `::errors/stale` in `ex-data`; message + text is for humans only. + + Args: + epoch: owner epoch captured by the caller. + phase: phase name recorded in the stale marker. + + Returns: + nil when `epoch` is current. + + Throws: + An exception when `epoch` is stale, with `::errors/stale true` and + `:phase` in its `ex-data`." + [epoch phase] + (when-not (= epoch @owner-epoch*) + (throw (ex-info "superseded" {::errors/stale true :phase phase})))) + +(defn- build-canvas! + "Creates the bench canvas with a `width*dpr` backing buffer and an explicit + CSS size, so the drawn scene fills exactly the reported viewport. + + Args: + width: viewport width in CSS pixels. + height: viewport height in CSS pixels. + dpr: device pixel ratio used for the backing buffer. + + Returns: + An unattached canvas element whose backing buffer is `width*dpr` by + `height*dpr` and whose CSS size is `width` by `height` pixels." + [width height dpr] + (let [canvas (.createElement js/document "canvas")] + (aset canvas "width" (* width dpr)) + (aset canvas "height" (* height dpr)) + (aset (.-style ^js canvas) "width" (str width "px")) + (aset (.-style ^js canvas) "height" (str height "px")) + canvas)) + +(defn- ^:async run-phase + "Executes one lifecycle phase through the command executor. + + `run` receives a command context and returns the next context or a Promise + of it. The raw block keeps `:phase`, `:preparation-id`, `:outcome`, + selected command `:events` and optional `:diagnostics`, and is appended to + `phases`. + + Args: + phase: lifecycle phase name, such as `:preparation`, `:main` or `:reset`. + preparation-id: identifier recorded in the raw block. + rtx: command context supplied to the command executor. + run: function from a command context to the next context or a Promise. + phases: volatile that receives each raw phase block. + + Returns: + A Promise of the next command context. A failed command outcome rejects + the Promise with a phase-labeled error after the block is recorded." + [phase preparation-id rtx run phases] + (let [outcome (await (commands/execute rtx run)) + block (assoc (select-keys outcome [:outcome :events :diagnostics]) + :phase phase :preparation-id preparation-id)] + (vswap! phases conj block) + (when (= :failed (:outcome outcome)) + (let [cause (:error outcome)] + (throw (errors/with-context cause (errors/message-of cause "phase failed") + {:phase (or (:phase (ex-data cause)) (name phase))})))) + (:context outcome))) + +(defn- ^:async run-case + "Runs main and the optional reset against one prepared command context. + + Main executes the local case's `run!` function. Reset executes only after a + successful main. The context returned by main, or by reset when present, is + stored for the next reused attempt, so camera changes carry over. + + Args: + owner: map with `:epoch`, the owner epoch, and `:hooks`, the hooks used by + main and reset. + case-desc: collected case descriptor. + local-case: local case registration with `run!` and optional `:reset`. + graphics: browser-local graphics map containing the preparation evidence + and snapshot. + context: prepared command context for main. + + Returns: + A Promise of status `\"ok\"` with `:preparation` evidence, `:phases` for + main and any reset, `:scene` with the shape count in `:shapes` and the + seed in `:seed`, and `:effective-graphics`. A non-stale failure resolves + to failed evidence with this attempt's preparation and blocks, even when + a newer owner exists. A top-level `::errors/stale` marker resolves to the + stale result." + [{:keys [epoch hooks]} case-desc local-case graphics context] + (let [phases (volatile! []) + preparation (:preparation graphics) + preparation-id (:preparation-id preparation)] + (try + (let [rtx (assoc context :hooks hooks) + rtx (await (run-phase :main preparation-id rtx (:run! local-case) phases)) + rtx (if-some [reset (:reset local-case)] + (await (run-phase :reset preparation-id rtx reset phases)) + rtx)] + (guard-current! epoch "main") + (when-not (wasm/live?) + (throw (ex-info "disposed or context lost during case" {:phase "main"}))) + (swap! prepared* assoc :context rtx) + {:status "ok" + :preparation preparation + :phases @phases + :scene {:shapes (count (:objects (:snapshot graphics))) + :seed (:scene-seed case-desc)} + :effective-graphics (:effective-graphics preparation)}) + (catch :default cause + (cond-> (terminal-failure epoch cause "main") + (not (errors/stale? cause)) (assoc :preparation preparation :phases @phases)))))) + +(defn- parse-load-request + "Decodes and validates a Transit scene request before claiming the page. + + Args: + request: Transit string containing the fields in + `schema:load-scene-args`. + + Returns: + A map with `:params`, `:case-desc`, `:local-case` and `:scene`, or the + validation Error. Failure leaves the current owner in place." + [request] + (attempt + (let [params (t/decode-str request)] + (when-not (sm/validate schema:load-scene-args params) + (throw (ex-info "load-scene needs a Transit request with case" + {:phase "invalid-args"}))) + (let [case-desc (:case params) + scene (decl/registered-scene (:scene case-desc)) + local (decl/registered-case (:id case-desc))] + (when (nil? scene) + (throw (ex-info (str "unknown scene: " (:scene case-desc)) + {:phase "invalid-args"}))) + (when-not (and local + (fn? (:run! local)) + (= (:scene local) (:scene case-desc))) + (throw (ex-info (str "missing case body for " (:id case-desc)) + {:phase "invalid-args"}))) + (decl/check-registered-case! local) + {:params params + :case-desc case-desc + :local-case local + :scene scene})))) + +(defn- begin-load! + "Claims the page, releases the prior owner and resolves the viewport. + + It increments the owner epoch, releases the prior owner's canvas, graphics + resources and preparation, and clears the module promise so the next load + imports fresh. The caller supplies hooks and a deadline. + + Args: + case-desc: collected case descriptor whose `:view` supplies viewport + overrides. + + Returns: + A map with `:epoch`, the new owner epoch, and `:dims`, the resolved + viewport with `:width`, `:height` and `:dpr`." + [case-desc] + (let [epoch (swap! owner-epoch* inc)] + (release-owner-resources!) + (reset! module-promise* nil) + (let [viewport (decl/resolve-viewport (:view case-desc))] + {:epoch epoch + :dims viewport}))) + +(defn- phase-error + [phase message cause] + (ex-info message {:phase phase} cause)) + +(defn- load-module + "Loads and instantiates the renderer. + + Args: + epoch: owner epoch checked after the module resolves. + params: decoded request map; `:module-url` and `:wasm-url` override the + renderer artifact URLs. + + Returns: + A Promise of the module map with `:module` and `:factory-ms`. A failed + import or factory call rejects with the module-init phase and its cause + for bridge accounting; a superseded owner rejects with the stale marker." + [epoch params] + (-> (ensure-module! (or (:module-url params) "./render-wasm.js") + (or (:wasm-url params) "./render-wasm.wasm")) + (.catch #(throw (phase-error "module-init" + (str "module init failed: " (or (ex-message %) %)) + %))) + (.then (fn [installed] + (guard-current! epoch "module-init") + installed)))) + +(defn- initialize-graphics! + "Creates the canvas and WebGL context and binds them to the renderer. + + Args: + epoch: owner epoch used by the context-loss listeners. + viewport: map with `:width`, `:height` and `:dpr`. + installed: module map with `:module` and `:factory-ms`. + + Returns: + A map with `:module-ms`, the factory instantiation duration in + milliseconds; `:graphics-ms`, the context creation, renderer binding, + viewbox resize and listener installation duration in milliseconds; + `:renderer`, the unmasked renderer text; and `:ctx`, the graphics + resources kept in the browser. + + Throws: + The stale marker when `epoch` is no longer current. Context creation or + viewbox resize failures throw an exception with phase `graphics-init`." + [epoch {:keys [width height dpr]} {:keys [module factory-ms]}] + (guard-current! epoch "graphics-init") + (set! wasm/internal-module module) + (let [canvas (build-canvas! width height dpr)] + (.appendChild (.-body js/document) canvas) + (reset! canvas* canvas) + (let [g0 (now) + ctx (try + (webgl/init-context + canvas + {:module module + :context-id "webgl2" + :css-width width + :css-height height + :dpr dpr + :flags 0 + :browser (sr/translate-browser :chrome) + :params {}}) + (catch :default cause + (throw (phase-error "graphics-init" "graphics init failed" cause))))] + (when (nil? ctx) + (throw (phase-error "graphics-init" "WebGL2 context unavailable" nil))) + (set! wasm/gl-context-handle (:handle ctx)) + (set! wasm/gl-context (:context ctx)) + (set! wasm/canvas canvas) + (mark-live!) + (try + (h/call wasm/internal-module "_resize_viewbox" width height) + (catch :default cause + (throw (phase-error "graphics-init" "resize viewbox failed" cause)))) + (install-listeners! canvas epoch) + {:module-ms factory-ms + :graphics-ms (- (now) g0) + :renderer (renderer-string (:context ctx)) + :ctx ctx}))) + +(defn- prepare-scene + "Builds the registered scene, then fingerprints and orders its shapes. + + The scene builder returns a validated snapshot. Fingerprinting and upload + order derivation finish before the upload timer begins. + + Args: + epoch: owner epoch checked before the scene is built. + scene: registered scene declaration with a `:build` function. + case-desc: collected case descriptor whose `:params` are passed to the + builder. + seed: scene seed supplied as `:seed` to the builder. + graphics: graphics map to extend. + + Returns: + `graphics` with `:snapshot`, `:fingerprint` and `:ordered`. + + Throws: + The stale marker when `epoch` is no longer current. Other scene + preparation failures throw an exception with phase `scene-build`." + [epoch scene case-desc seed graphics] + (guard-current! epoch "scene-build") + (try + (let [snapshot ((:build scene) (assoc (:params case-desc) :seed seed))] + (assoc graphics + :snapshot snapshot + :fingerprint (fingerprint/fingerprint snapshot) + :ordered (snapshot/upload-order snapshot))) + (catch :default cause + (if (errors/stale? cause) + (throw cause) + (throw (phase-error "scene-build" "scene build failed" cause)))))) + +(defn- upload-scene! + "Sets the initial view and uploads prepared shapes inside the upload timer. + + The timer covers view setup, pool initialization, the loading lifecycle and + batch serialization. The loading lifecycle ends even when serialization + fails. + + Args: + epoch: owner epoch checked before uploading. + case-desc: collected case descriptor whose `:view` sets the camera. + graphics: prepared graphics map with `:snapshot` and `:ordered`. + + Returns: + `graphics` with `:upload-ms`, the upload duration in milliseconds, and + `:view`, the camera view that was set. + + Throws: + The stale marker when `epoch` is no longer current. Other upload failures + throw an exception with phase `upload`." + [epoch case-desc graphics] + (guard-current! epoch "upload") + (try + (let [u0 (now) + view (:view case-desc)] + (h/call wasm/internal-module "_set_view" + (:scale view) (- (:x view)) (- (:y view))) + (h/call wasm/internal-module "_init_shapes_pool" + (count (:objects (:snapshot graphics)))) + (h/call wasm/internal-module "_begin_loading") + (let [uploaded (attempt :catch-fatal + (serialize-shape/serialize-shapes-batch + (:ordered graphics) + {:include-layout? false + :include-fills-strokes? true}) + nil) + ended (attempt :catch-fatal (h/call wasm/internal-module "_end_loading") nil)] + (when (or uploaded ended) + (throw (errors/combine + [{:phase "upload" :operation :serialize-shapes :error uploaded} + {:phase "upload" :operation :end-loading :error ended}])))) + (h/call wasm/internal-module "_set_view_end") + (assoc graphics :upload-ms (- (now) u0) :view view)) + (catch :default cause + (if (errors/stale? cause) + (throw cause) + (throw (errors/with-context cause "upload failed" {:phase "upload"})))))) + +(defn- preparation-data + "Projects completed setup into preparation evidence. + + Args: + id: preparation identifier. + case-desc: collected case descriptor. + effective-graphics: graphics facts read from the live context. + graphics: browser-local graphics map with timings, snapshot, fingerprint + and optional `:setup-render`. + phases: raw preparation phase blocks. + + Returns: + Portable identity data, graphics facts, raw phase blocks and setup + diagnostics." + [id case-desc effective-graphics graphics phases] + (let [{:keys [module-ms graphics-ms upload-ms snapshot fingerprint setup-render]} graphics] + {:preparation-id id + :case-id (:id case-desc) + :phases phases + :fingerprint fingerprint + :effective-graphics effective-graphics + :diagnostics (cond-> {:module-init-ms module-ms :graphics-init-ms graphics-ms + :upload-ms upload-ms + :scene {:shapes (count (:objects snapshot)) :seed (:scene-seed case-desc)}} + setup-render (assoc :setup-render setup-render))})) + +(defn- command-context + "Returns the four-field command context for one preparation. + + Args: + case-desc: collected case descriptor. + hooks: browser hooks for the current owner and deadline. + graphics: prepared graphics map containing `:snapshot` and `:view`. + + Returns: + The command context with the uploaded snapshot in `:scene` and the + current camera view in `:view`." + [case-desc hooks graphics] + {:case case-desc + :scene (:snapshot graphics) + :view (:view graphics) + :hooks hooks}) + +(defn- ^:async prepare-initial-view + "Runs the preparation callback, then renders the initial Full frame. + + Args: + callback: function from a command context to the next context or a + Promise. + render-mode: `:none` skips the initial render; every other value renders. + setup-render: volatile that receives the drain trace when rendering runs. + rtx: command context passed to the callback. + + Returns: + A Promise of the callback's resulting context. The drain uses that + context's hooks, starts with flags 0 and records its traces and durations + in `setup-render`. A callback or drain failure rejects the Promise." + [callback render-mode setup-render rtx] + (let [rtx (await (callback rtx))] + (when-not (= :none render-mode) + (let [hooks (:hooks rtx)] + (vreset! setup-render + (await (protocol/drain hooks {:flags 0 + :origin ((:now hooks)) + :immediate false}))))) + rtx)) + +(defn- ^:async prepare-context + "Runs the preparation phase and attaches its context to `graphics`. + + The local case supplies the preparation callback; `identity` is used when + none is registered. + + Args: + preparation: map with `:owner`, `:case-desc`, `:local-case` and + `:preparation-id`. + phases: volatile that receives the raw phase block. + graphics: prepared graphics map with the uploaded snapshot and view. + + Returns: + A Promise of `graphics` with `:context`, the context returned by the + callback and initial render, and `:setup-render`, the automatic render + trace or nil. A failed preparation phase rejects the Promise." + [{:keys [owner case-desc local-case preparation-id]} phases graphics] + (let [setup-render (volatile! nil) + callback (get-in local-case [:preparation :callback] identity) + render-mode (get-in case-desc [:preparation :render]) + context (command-context case-desc (:hooks owner) graphics) + context (await (run-phase :preparation preparation-id context + (partial prepare-initial-view callback render-mode setup-render) + phases))] + (assoc graphics :context context :setup-render @setup-render))) + +(defn- prepared-state + "Collects the reusable context, graphics resources and preparation evidence. + + Args: + preparation: case definitions, owner and timeout settings. + graphics: prepared graphics and the completed command context. + evidence: portable preparation evidence. + + Returns: + State ready for its first execution, with the context stored separately + from graphics." + [preparation graphics evidence] + (assoc (select-keys preparation [:owner :case-desc :local-case :timeout-ms]) + :busy? false + :used? false + :context (:context graphics) + :graphics (assoc (dissoc graphics :context) :preparation evidence))) + +(defn- complete-preparation + "Reads effective graphics, records the completed preparation and returns it. + + Args: + preparation: map with `:owner`, `:case-desc` and `:preparation-id`. + phases: volatile of raw preparation blocks. + graphics: prepared graphics map with the live context and setup facts. + + Returns: + `{:status \"ok\" :preparation evidence}` after storing the preparation for + later attempts. + + Throws: + A top-level stale marker when the owner changed during preparation." + [{:keys [owner case-desc preparation-id] :as preparation} phases graphics] + (guard-current! (:epoch owner) "prepare") + (let [{:keys [width height dpr]} (:dims owner) + {:keys [ctx renderer]} graphics + effective-graphics (read-graphics (:context ctx) width height dpr renderer) + evidence (preparation-data preparation-id case-desc effective-graphics graphics @phases)] + (reset! prepared* (prepared-state preparation graphics evidence)) + {:status "ok" :preparation evidence})) + +(defn- record-setup + "Records setup facts for partial failure evidence and returns `graphics`. + + The stored keys are `:module-ms`, `:graphics-ms`, `:fingerprint` and + `:upload-ms` when present. + + Args: + available: volatile map that receives the available setup facts. + graphics: graphics map to read the facts from. + + Returns: + `graphics`, unchanged." + [available graphics] + (vreset! available (select-keys graphics [:module-ms :graphics-ms :fingerprint :upload-ms])) + graphics) + +(defn- preparation-failure + "Returns failure evidence for a preparation that did not complete. + + A top-level stale cause returns the stale result. + + Args: + epoch: owner epoch that the preparation belongs to. + available: volatile of setup facts recorded before the failure. + phases: volatile of raw preparation blocks. + cause: failure that ended preparation. + + Returns: + `{:status \"stale\"}` for a top-level stale cause; otherwise failed + evidence that keeps the recorded setup facts under `:partial :setup`, any + partial render data from the cause, and the raw preparation blocks at the + top level under `:phases`. It releases the owner when `epoch` is still + current." + [epoch available phases cause] + (if (errors/stale? cause) + (terminal-failure epoch cause "aborted") + (assoc (terminal-failure epoch + (errors/with-context cause (errors/message-of cause "setup failed") + {:partial {:setup @available}}) + "aborted") + :phases @phases))) + +(defn- ^:async prepare-owned-scene + "Prepares a claimed scene from a supplied module installation Promise. + + Args: + preparation: map with `:owner`, `:case-desc`, `:scene`, `:local-case` and + `:preparation-id`. + module-promise: Promise of the installed module map; the function awaits it + before graphics initialization and scene preparation. + + Returns: + A Promise of `{:status \"ok\" :preparation evidence}` on success. + Ordinary failure returns failed evidence with the failed phase, any + recorded setup facts under `:partial :setup`, and the raw preparation + blocks at the top level under `:phases`. A top-level stale marker produces + the stale result." + [{:keys [owner case-desc scene] :as preparation} module-promise] + (let [{:keys [epoch dims]} owner + available (volatile! {}) + phases (volatile! [])] + (try + (let [module (await module-promise) + graphics (->> module + (initialize-graphics! epoch dims) + (record-setup available)) + graphics (->> graphics + (prepare-scene epoch scene case-desc (:scene-seed case-desc)) + (record-setup available)) + graphics (->> graphics + (upload-scene! epoch case-desc) + (record-setup available)) + graphics (await (prepare-context preparation phases graphics))] + (complete-preparation preparation phases graphics)) + (catch :default cause + (preparation-failure epoch available phases cause))))) + +(defn- prepare-request + "Validates a Transit scene request, claims the page and prepares the scene. + + Validation runs before the owner changes. A valid request releases the prior + owner, starts a new owner epoch, builds browser hooks with a deadline and + prepares the scene. `:preparation-id` defaults to the new epoch and + `:timeout-ms` defaults to `case-ms-budget`. + + Args: + request: Transit string matching `schema:load-scene-args`. + + Returns: + A Promise of `{:status \"ok\" :preparation evidence}`, failed evidence with + a phase, message, cause and optional partial data, or the stale result. A + preparation failure also carries its available blocks under top-level + `:phases`. Invalid arguments resolve as failed `invalid-args` evidence and + leave the current owner untouched." + [request] + (try + (let [parsed (parse-load-request request)] + (if (failure? parsed) + (js/Promise.resolve (errors/fail-data (errors/evidence "invalid-args" parsed))) + (let [{:keys [params case-desc]} parsed + {:keys [epoch] :as owner} (begin-load! case-desc) + timeout-ms (or (:timeout-ms params) case-ms-budget) + deadline-ms (+ (now) timeout-ms) + owner (assoc owner :deadline-ms deadline-ms + :hooks (browser-hooks epoch deadline-ms))] + (prepare-owned-scene (assoc parsed + :owner owner + :timeout-ms timeout-ms + :preparation-id (or (:preparation-id params) epoch)) + (load-module epoch params))))) + (catch :default cause + (js/Promise.resolve (errors/fail-data (errors/evidence "setup" cause)))))) + +(defn- execute-prepared + "Runs main and optional reset against the stored preparation. + + Args: + preparation-id: identifier of the prepared scene to use. + timeout-ms: optional remaining budget in milliseconds, bounded by the + preparation's case deadline. + + Returns: + A Promise of status ok with `:preparation` evidence, top-level `:phases` + for main and any reset, `:scene` shape count and seed, and + `:effective-graphics`; or failed evidence. A top-level stale marker + produces the stale result. + + A missing, busy or mismatched preparation, or a fresh preparation already + used, resolves as failed `invalid-args` evidence, leaving the owner + unchanged. A non-stale phase failure releases that preparation, so later + attempts find it unavailable." + [{:keys [preparation-id timeout-ms]}] + (let [{:keys [owner case-desc local-case graphics context busy? used?] :as prepared} @prepared* + id (get-in graphics [:preparation :preparation-id])] + (if (or (nil? prepared) (not= id preparation-id) busy? + (and used? (= :fresh (:context case-desc)))) + (js/Promise.resolve (errors/fail-data {:phase "invalid-args" + :message "preparation unavailable, busy or already used"})) + (let [epoch (:epoch owner) + deadline-ms (if timeout-ms + (min (:deadline-ms owner) (+ (now) timeout-ms)) + (:deadline-ms owner)) + owner (assoc owner :hooks (browser-hooks epoch deadline-ms))] + (swap! prepared* assoc :busy? true :used? true) + (-> (run-case owner case-desc local-case graphics context) + (.then (fn [evidence] + (when (= epoch @owner-epoch*) + (swap! prepared* assoc :busy? false)) + evidence)) + (.catch #(terminal-failure epoch % "attempt"))))))) + +(defn- encodable-phase-prefix + [phases] + (reduce (fn [kept phase] + (try + (codec/encode-str phase) + (conj kept phase) + (catch :default _ (reduced kept)))) + [] phases)) + +(defn- encode-response + [epoch evidence] + (try + (codec/encode-str evidence) + (catch :default cause + (dispose-if-current! epoch) + (let [encoding (assoc (errors/evidence :bridge cause) :operation :encode-response) + original (when (= "failed" (:status evidence)) (errors/project-diagnostic evidence)) + recovered (if original + (assoc original :errors (conj (or (:errors original) + [(dissoc original :partial)]) encoding)) + encoding) + recovered (cond-> recovered + (seq (get-in evidence [:preparation :phases])) + (assoc :phases (encodable-phase-prefix (get-in evidence [:preparation :phases])))) + recovered (reduce (fn [out key] + (if (contains? evidence key) + (try + (codec/encode-str (get evidence key)) + (assoc out key (get evidence key)) + (catch :default _ + (if (= :phases key) + (assoc out :phases (encodable-phase-prefix (:phases evidence))) + out))) + out)) + recovered + [:preparation :phases :scene :effective-graphics])] + (t/encode-str (errors/fail-data recovered)))))) + +(defn- encode-evidence + "Encodes bridge evidence after timing and resolves failures as bridge + failures. + + Encoding runs after all measured work. If `promise` rejects or encoding + fails, the owner is disposed when the captured epoch is still current. + + Args: + promise: Promise of bridge evidence to encode. + + Returns: + A Promise of Transit text for the supplied evidence, or for failed bridge + evidence when a failure occurs." + [promise] + (let [epoch @owner-epoch*] + (-> promise + (.then (partial encode-response epoch)) + (.catch (fn [cause] + (dispose-if-current! epoch) + (t/encode-str (errors/fail-data (errors/evidence :bridge cause)))))))) + +(defn prepare-scene-request + "Prepares a scene from a Transit request and returns Transit evidence. + + Args: + request: Transit string accepted by `schema:load-scene-args`. + + Returns: + A Promise of Transit text. Success contains status ok and `:preparation` + evidence. Preparation failures contain status failed, phase, message and + available cause, partial evidence and blocks under top-level `:phases`. + Encoding failures report the bridge phase with status, phase and message. + A failure with a top-level stale marker returns status stale." + [request] + (encode-evidence (prepare-request request))) + +(defn run-prepared + "Runs one attempt against a prepared scene. + + Args: + request: Transit `{:preparation-id id :timeout-ms optional-positive-ms}`. + + Returns: + A Promise of Transit evidence. Success contains status ok, the preparation + evidence, top-level main and optional reset phase blocks, scene shape count + and seed, and effective graphics. Failures contain status failed, phase and + message. Phase failures also include their cause, preparation evidence, + recorded blocks and available partial data. Invalid requests report the + invalid-args phase; encoding failures report the bridge phase. A failure + with a top-level stale marker returns the stale result." + [request] + (encode-evidence + (try + (let [params (t/decode-str request)] + (if (sm/validate [:map {:closed true} + [:preparation-id some?] + [:timeout-ms {:optional true} [:int {:min 1}]]] params) + (execute-prepared params) + (js/Promise.resolve (errors/fail-data {:phase "invalid-args" + :message "runPrepared needs a preparation ID"})))) + (catch :default cause + (js/Promise.resolve (errors/fail-data {:phase "invalid-args" + :message (errors/message-of cause "invalid Transit request")})))))) + +(defn load-scene + "Prepares a collected case and runs one attempt. + + Args: + request: Transit string accepted by `prepare-scene-request`. + + Returns: + A Promise of Transit evidence. On success, `:preparation` holds the + preparation evidence and top-level `:phases` holds main and any reset. A + failed preparation places its available blocks at the top level. An + attempt failure retains the preparation evidence and recorded blocks + captured for that attempt. A failure with a top-level stale marker returns + the stale result." + [request] + (encode-evidence + (-> (prepare-request request) + (.then (fn [{:keys [status preparation] :as evidence}] + (if (= status "ok") + (execute-prepared {:preparation-id (:preparation-id preparation)}) + evidence)))))) + +(defn ping + "Reports that the bridge is callable. + + Returns: + A JavaScript object with status `\"ok\"`." + [] + #js {"status" "ok"}) + +(defn summarize-record + "Returns a Transit summary of a saved run record. + + Args: + record-text: Transit text for a supported run record. + analysis-text: optional Transit map of analysis settings. + + Returns: + Transit text for the summary. + + Throws: + When the run or settings cannot be decoded, validated, summarized or + encoded." + ([record-text] (summarize-record record-text nil)) + ([record-text analysis-text] + (codec/encode-str (summary/summarize (result/decode record-text) + (when analysis-text (t/decode-str analysis-text)))))) + +(defn compare-records + "Compares two saved runs and returns Transit comparison data. + + Args: + baseline-text: Transit text for the baseline run. + candidate-text: Transit text for the candidate run. + options-text: Transit map of comparison options. + + Returns: + Transit text for the comparison. + + Throws: + When a run or the options cannot be decoded, validated, compared or + encoded." + [baseline-text candidate-text options-text] + (result/encode (compare/compare-runs (result/decode baseline-text) + (result/decode candidate-text) + (t/decode-str options-text)))) + +(defn format-run + "Returns human-readable text for a saved run record. + + Args: + record-text: Transit text for a supported run record. + + Returns: + Text for the run summary. + + Throws: + When the run cannot be decoded, validated, summarized or formatted." + [record-text] + (format/format-run (summary/summarize (result/decode record-text)))) + +(defn format-comparison + "Returns human-readable text for a saved comparison record. + + Args: + record-text: Transit text for a supported comparison record. + + Returns: + Text for the comparison report. + + Throws: + When the comparison cannot be decoded, validated or formatted." + [record-text] + (format/format-comparison (result/decode record-text))) + +(def bridge + "Browser-side functions exported from the compiled module. + + Keys use the camelCase names used by the runner: `ping`, + `buildIdentity`, `loadScene`, `prepareScene`, `runPrepared`, `summarize`, + `compare`, `formatRun`, `formatComparison` and `dispose`. Each entry returns + Transit text, report text, or, for `ping` and `dispose`, a JavaScript status + object." + #js {"ping" ping + "buildIdentity" #(t/encode-str (identity/data)) + "loadScene" load-scene + "prepareScene" prepare-scene-request + "runPrepared" run-prepared + "summarize" summarize-record + "compare" compare-records + "formatRun" format-run + "formatComparison" format-comparison + "dispose" dispose!}) diff --git a/frontend/performance/benches/render_wasm/browser/index.html b/frontend/performance/benches/render_wasm/browser/index.html new file mode 100644 index 0000000000..dcf0d38dd9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/browser/index.html @@ -0,0 +1,33 @@ + + + + + Renderer benchmarks + + + + + + + + diff --git a/frontend/performance/benches/render_wasm/builder.cljc b/frontend/performance/benches/render_wasm/builder.cljc new file mode 100644 index 0000000000..3f7c35431b --- /dev/null +++ b/frontend/performance/benches/render_wasm/builder.cljc @@ -0,0 +1,512 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.builder + "Seeded construction of benchmark scenes. + + A `scene` consists of one seeded number generator, one object map, and one + label map. + + Shape and container calls attach objects to the current parent in evaluation + order. Coordinates are page-relative. Frames use authored bounds; groups and + booleans derive geometry from their children after the body completes. + + `scene` accepts a required :seed, optional :root attributes and :defaults. + It returns a validated snapshot with :objects and :refs. Labels name objects + within that snapshot. A container whose body or finalizer throws becomes + unfinished, and `finish!` rejects the snapshot. + + Default attributes follow the same theme: `*defaults*` maps a shape type to + an attribute generator map, bound by the `scene` macro from the scope + params. Per-shape attrs are merged over the generated values, so a recipe + can override values locally and rebind `*defaults*` around a section. + + Ordinary :require loads the macros and runtime helpers. Scene bodies support + let, doseq and calls to functions defined outside the scope." + #?(:cljs (:require-macros [benches.render-wasm.builder])) + #?(:cljs + (:require + [app.common.data :as d] + [app.common.files.helpers :as cfh] + [app.common.geom.shapes :as gsh] + [app.common.types.path :as path] + [app.common.types.shape :as cts] + [app.common.types.shape-tree :as ctst] + [app.common.uuid :as uuid] + [benches.render-wasm.random :as rnd :refer + [rng-float rng-hex-color rng-int rng-opacity rng-uuid]] + [benches.render-wasm.snapshot :as snapshot]))) + +#?(:clj + (do + (defn- emit-shape + [constructor label attrs] + `(~constructor + benches.render-wasm.builder/*state* + benches.render-wasm.builder/*defaults* + ~label + ~attrs)) + + (defn- emit-container + [type label-or-attrs body] + (let [labeled? (or (keyword? label-or-attrs) (vector? label-or-attrs)) + attrs (if labeled? (first body) label-or-attrs) + label (when labeled? label-or-attrs) + body (if labeled? (rest body) body)] + `(benches.render-wasm.builder/with-container + ~type ~attrs ~label (fn [] ~@body)))) + + (defmacro scene + "Runs `body` in a scene scope and returns the validated snapshot." + [params & body] + `(let [params# ~params + state# (benches.render-wasm.builder/start params#)] + (binding [benches.render-wasm.builder/*state* state# + benches.render-wasm.builder/*defaults* (:defaults params#)] + ~@body + (benches.render-wasm.builder/finish! state#)))) + + (defmacro rect + "Adds a rectangle and returns its uuid. + + Forms: (rect), (rect attrs), (rect label), (rect label attrs). + A single map argument supplies attributes; keywords and vectors label + the object in :refs. Attributes override the generated defaults." + ([] + (emit-shape 'benches.render-wasm.builder/rect! nil {})) + ([label-or-attrs] + (emit-shape 'benches.render-wasm.builder/rect! label-or-attrs {})) + ([label attrs] + (emit-shape 'benches.render-wasm.builder/rect! label attrs))) + + (defmacro path + "Adds a path and returns its uuid. + + Forms: (path), (path attrs), (path label), (path label attrs). + A single map argument supplies attributes; keywords and vectors label + the object in :refs. Attributes override the generated defaults." + ([] + (emit-shape 'benches.render-wasm.builder/path! nil {})) + ([label-or-attrs] + (emit-shape 'benches.render-wasm.builder/path! label-or-attrs {})) + ([label attrs] + (emit-shape 'benches.render-wasm.builder/path! label attrs))) + + (defmacro circle + "Adds an ellipse and returns its uuid. + + Forms: (circle), (circle attrs), (circle label), (circle label attrs). + A single map argument supplies attributes; keywords and vectors label + the object in :refs. Attributes override the generated defaults." + ([] + (emit-shape 'benches.render-wasm.builder/circle! nil {})) + ([label-or-attrs] + (emit-shape 'benches.render-wasm.builder/circle! label-or-attrs {})) + ([label attrs] + (emit-shape 'benches.render-wasm.builder/circle! label attrs))) + + (defmacro frame + "Adds a frame, runs `body` with the frame as parent, and returns its uuid. + + Forms: (frame attrs & body), (frame label attrs & body). + A keyword or vector form supplies a label in :refs. Attributes must + carry numeric :x :y :width :height. Children use page-relative + coordinates. Frames allow an empty body." + [label-or-attrs & body] + (emit-container :frame label-or-attrs body)) + + (defmacro group + "Adds a group, runs `body` with the group as parent, and returns its uuid. + + Forms: (group attrs & body), (group label attrs & body). + A keyword or vector form supplies a label in :refs. Groups derive their + geometry from children and require at least one child. With + :masked-group true, the first child supplies the mask geometry." + [label-or-attrs & body] + (emit-container :group label-or-attrs body)) + + (defmacro bool + "Adds a boolean shape, runs `body` with it as parent, and returns its uuid. + + Forms: (bool attrs & body), (bool label attrs & body). + A keyword or vector form supplies a label in :refs. Attributes require + :bool-type :union, :difference, :intersection or :exclude. + + Content and geometry derive from the children. Styles inherit from the + first child for :difference and the last child otherwise; supplied + non-nil styles take precedence. A bool requires at least one child and + accepts paths, rectangles, ellipses, groups and other booleans." + [label-or-attrs & body] + (emit-container :bool label-or-attrs body)))) + +#?(:cljs + (do + + + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Attribute generators + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + (defn gen-int + "Integer generator over `[min, max)`." + [min max] + (fn [rng] (rng-int rng min max))) + + (defn gen-float + "Float generator rounded to three decimals so snapshots stay readable." + [min max] + (fn [rng] (rnd/round3 (rng-float rng min max)))) + + (defn gen-hex-color + "Generator of a random `#rrggbb` string." + [] + (fn [rng] (rng-hex-color rng))) + + (defn gen-one-of + "Generator that picks one value per draw. Rejects an empty collection." + [values] + (when (empty? values) + (throw (ex-info "gen-one-of requires at least one value" + {:type ::no-values}))) + (fn [rng] (nth values (rng-int rng 0 (count values))))) + + (defn gen-fill + "Generator of one translucent solid fill." + [] + (fn [rng] + {:fill-color (rng-hex-color rng) + :fill-opacity (rng-opacity rng)})) + + (defn gen-stroke + "Centered stroke like the interim rectangle workload." + [width] + (fn [rng] + {:stroke-width width + :stroke-alignment :center + :stroke-color (rng-hex-color rng) + :stroke-opacity (rng-opacity rng)})) + + (defn gen-vector + "Composes generators into a generator of a vector, one value per generator." + [& generators] + (fn [rng] (mapv #(% rng) generators))) + + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Scope state + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + (def ^:dynamic *state* + "Current scene scope; bound by the `scene` macro." + nil) + + (def ^:dynamic *defaults* + "Per-shape-type attribute generator maps bound by the `scene` macro, + e.g. `{:rect {:x (gen-int 0 1920) ...}}`. Values are literals or one-arg + functions of the scope random source." + {}) + + (def ^:dynamic *parent* + "Current insertion point as `{:id parent-id :frame-id containing-frame}`; + bound by the container scopes, `uuid/zero` at the top." + {:id uuid/zero :frame-id uuid/zero}) + + (defn- check-seed! + "Requires a scene seed from 0 through 4294967295, retaining the builder + error type." + [seed] + (when-not (rnd/seed? seed) + (throw (ex-info "scene scope requires an integer :seed in [0, 2^32)" + {:type ::invalid-seed + :seed seed})))) + + (defn start + "Starts a scene scope and returns its state atom with objects, refs and :rng. + The seeded generator belongs to this scope; other scopes and report draws + cannot advance its state." + [params] + (when (some? *state*) + (throw (ex-info "scene scopes cannot nest" + {:type ::nested-scope}))) + (check-seed! (:seed params)) + (let [seed (:seed params) + root (cts/setup-shape (merge (:root params) + {:id uuid/zero + :type :frame + :name "Root Frame" + :parent-id uuid/zero + :frame-id uuid/zero + :shapes []}))] + (atom {:seed seed + :rng (rnd/create seed) + :objects {uuid/zero root} + :refs {} + :unfinished {}}))) + + (defn- ensure-state + [state] + (or state + (throw (ex-info "shape used outside a scene scope" + {:type ::outside-scope})))) + + (defn- resolve-attr + [rng spec] + (if (fn? spec) (spec rng) spec)) + + (defn- resolve-attrs + [rng defaults attrs] + (merge (reduce-kv (fn [result key spec] + (assoc result key (resolve-attr rng spec))) + {} + defaults) + attrs)) + + (defn- register-ref! + [state label id] + (when (some? label) + (when (contains? (:refs @state) label) + (throw (ex-info (str "duplicate scene label: " label) + {:type ::duplicate-label + :label label}))) + (swap! state assoc-in [:refs label] id))) + + (defn- add-object! + "Inserts `shape` under the current parent and registers `label`." + [state label shape] + (let [{:keys [id frame-id]} *parent*] + (register-ref! state label (:id shape)) + (swap! state + #(ctst/add-shape (:id shape) shape % frame-id id nil true)))) + + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Shape constructors + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + (defn- normalize-label-and-attrs + "Resolves map attributes and keyword/vector labels for shape construction." + [label attrs] + (cond + (and (map? label) (empty? attrs)) [nil label] + (nil? label) [nil attrs] + (or (keyword? label) (vector? label)) [label attrs] + :else (throw (ex-info "rect label must be a keyword or vector" + {:type ::invalid-label + :label label})))) + + (defn- construct-shape + [state defaults type label attrs] + (let [state (ensure-state state) + [label attrs] (normalize-label-and-attrs label attrs) + rng (:rng @state) + attrs (resolve-attrs rng (get defaults type) attrs) + id (rng-uuid rng) + shape (cts/setup-shape (merge attrs + {:id id + :type type}))] + (add-object! state label shape) + id)) + + (defn rect! + "Adds a rectangle with the selected defaults and attributes, returning its uuid." + [state defaults label attrs] + (construct-shape state defaults :rect label attrs)) + + (defn path! + "Adds a path with the selected defaults and attributes, returning its uuid." + [state defaults label attrs] + (construct-shape state defaults :path label attrs)) + + (defn circle! + "Adds an ellipse with the selected defaults and attributes, returning its uuid." + [state defaults label attrs] + (construct-shape state defaults :circle label attrs)) + + (def ^:private frame-bounds-keys + "Frames do not derive their bounds from children." + [:x :y :width :height]) + + (def ^:private group-geometry-keys + "Group geometry derives from its children." + [:x :y :width :height :selrect :points]) + + (def ^:private bool-geometry-keys + "Bool geometry, content and placement derive from its children." + [:x :y :width :height :selrect :points :content]) + + (def ^:private bool-transform-keys + "Construction does not accept transforms: `path/update-geometry` would + pivot them on the empty pre-finalization selrect." + [:transform :transform-inverse :rotation :flip-x :flip-y]) + + (defn- reject-attrs! + "Throws when `attrs` carries any of `attr-keys`. Shared by containers + whose geometry or content derives from their children." + [attrs id error-type message attr-keys] + (doseq [key attr-keys] + (when (some? (get attrs key)) + (throw (ex-info message + {:type error-type + :id id + :key key}))))) + + (defn- setup-frame + [id attrs] + (doseq [key frame-bounds-keys] + (let [value (get attrs key)] + (cond + (nil? value) + (throw (ex-info "frame requires :x :y :width :height" + {:type ::frame-bounds + :id id + :missing key})) + + (not (d/num? value)) + (throw (ex-info "frame bounds must be numbers" + {:type ::frame-bounds + :id id + :invalid key}))))) + (cts/setup-shape (assoc attrs :id id :type :frame))) + + (defn- setup-group + [id attrs] + (reject-attrs! attrs id ::group-geometry + "group derives its geometry from children" + group-geometry-keys) + (cts/setup-shape (assoc attrs :id id :type :group))) + + (defn- setup-bool + [id attrs] + (reject-attrs! attrs id ::bool-geometry + "bool derives its geometry and content from children" + bool-geometry-keys) + (reject-attrs! attrs id ::bool-transform + "bool construction does not accept transforms" + bool-transform-keys) + (when-not (contains? cts/bool-types (:bool-type attrs)) + (throw (ex-info "bool requires :bool-type :union :difference :intersection or :exclude" + {:type ::bool-type + :id id + :bool-type (:bool-type attrs) + :supported cts/bool-types}))) + (cts/setup-shape (assoc attrs :id id :type :bool))) + + (defn- bool-head + "Canonical style head: the first child is the base for `:difference`, the + last child otherwise." + [bool-shape children] + (if (= :difference (:bool-type bool-shape)) + (first children) + (last children))) + + (defn- finalize-bool! + [state id] + (let [scope @state + bool (get-in scope [:objects id]) + children (mapv (:objects scope) (:shapes bool))] + (when (empty? children) + (throw (ex-info "bool requires at least one child" + {:type ::empty-bool + :id id}))) + (when-let [frame (first (filter cfh/frame-shape? children))] + (throw (ex-info "bool children cannot be frames" + {:type ::frame-in-bool + :id id + :child (:id frame)}))) + (let [inherited (d/without-nils + (select-keys (bool-head bool children) + path/bool-style-properties)) + supplied (d/without-nils + (select-keys (get-in scope [:container-attrs id]) + path/bool-style-properties)) + bool (-> bool (merge inherited) (merge supplied)) + + ;; `path/update-bool-shape` dispatches to `path/wasm:calc-bool-content`, + ;; the renderer override, so scene content would depend on renderer + ;; initialization and A/B configuration. Compose the pure helpers: + ;; scene generation is untimed and must not vary with the renderer + ;; being compared. + content (path/calc-bool-content bool (:objects scope)) + bool (path/update-geometry bool content)] + (swap! state assoc-in [:objects id] bool)))) + + (defn- finalize-group! + [state id] + (let [scope @state + group (get-in scope [:objects id]) + children (mapv (:objects scope) (:shapes group))] + (when (empty? children) + (throw (ex-info "group requires at least one child" + {:type ::empty-group + :id id}))) + (let [group (if (:masked-group group) + (gsh/update-mask-selrect group children) + (gsh/update-group-selrect group children))] + (swap! state assoc-in [:objects id] group)))) + + (def ^:private container-specs + "Per-container behavior: how to build it, which parent scope it opens, + and how to finalize it after the body. `:keep-attrs?` records the raw + attrs in the scope so the finalizer can let supplied styles win over + inherited ones." + {:frame {:setup setup-frame + :parent (fn [id _parent] {:id id :frame-id id}) + :finalize (fn [_state _id] nil)} + :group {:setup setup-group + :parent (fn [id parent] {:id id :frame-id (:frame-id parent)}) + :finalize finalize-group!} + :bool {:setup setup-bool + :parent (fn [id parent] {:id id :frame-id (:frame-id parent)}) + :finalize finalize-bool! + :keep-attrs? true}}) + + (defn- check-container-attrs + [type id attrs] + (when-not (map? attrs) + (throw (ex-info (str (name type) " attrs must be a map; the label form takes the attrs map next") + {:type ::invalid-attrs :id id}))) + attrs) + + (defn with-container + "Creates a frame, group or bool, runs `thunk` with the new parent in + scope, finalizes the containers that derive state from children, and + returns the container uuid. + + Any exception from the body or the finalizer marks the container + unfinished and propagates; `finish!` then rejects the instance even if the + caller catches the exception." + [type attrs label thunk] + (let [{:keys [setup parent finalize keep-attrs?]} + (or (get container-specs type) + (throw (ex-info "unsupported container type" + {:type ::invalid-container + :container-type type}))) + state (ensure-state *state*) + id (rng-uuid (:rng @state)) + attrs (check-container-attrs type id attrs) + shape (setup id attrs)] + (when keep-attrs? + (swap! state assoc-in [:container-attrs id] attrs)) + (add-object! state label shape) + (binding [*parent* (parent id *parent*)] + (try + (thunk) + (finalize state id) + (catch :default cause + (swap! state assoc-in [:unfinished id] true) + (throw cause)))) + id)) + + (defn finish! + "Validates the scope snapshot and returns it." + [state] + (let [{:keys [objects refs unfinished]} @state] + (when (seq unfinished) + (throw (ex-info "scene contains an unfinished container" + {:type ::unfinished-container + :ids (vec (keys unfinished))}))) + (let [instance {:objects objects + :refs refs}] + (snapshot/validate! instance) + instance))))) diff --git a/frontend/performance/benches/render_wasm/cases.cljs b/frontend/performance/benches/render_wasm/cases.cljs new file mode 100644 index 0000000000..f67962538e --- /dev/null +++ b/frontend/performance/benches/render_wasm/cases.cljs @@ -0,0 +1,128 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +;; Discovery loads each scene and its transitive requires, including any +;; colocated operation functions. +;; Keep this entire import subgraph safe for Node. + +(ns benches.render-wasm.cases + "Case collection for the renderer benchmarks. + + Loading this namespace registers the scene namespaces listed here. The require + list is the discovery manifest for case collection. + + NOTE: Nothing here should import browser or renderer code, so the Node runner + and offline reports load scenes without requiring a browser. + + Usage: + + (cases/collect-cases {:master-seed 42}) + ;; => + [{:id :rects/load + :scene :rects + :scene-description \"Seeded rectangles with translucent fills and centered strokes\" + :scene-seed 2602546180 + :params {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {:render :none}} + {:id :rects/pan ...} + {:id :rects/zoom ...}]" + (:require + [app.common.schema :as sm] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :as random] + benches.render-wasm.scenes.masks + benches.render-wasm.scenes.paths + benches.render-wasm.scenes.plus + benches.render-wasm.scenes.rects + [clojure.string :as str])) + +(defn- check-master-seed! + "Collection requires an integer master seed in [0, 2^32)." + [seed] + (when-not (random/seed? seed) + (throw (ex-info "case collection requires an integer master seed in [0, 2^32)" + {:type ::invalid-master-seed + :seed seed})))) + +(defn- as-invalid-case! + "Runs `thunk`, translating core case-validation failures into this + namespace's ::invalid-case so `collect-cases` has a single error + contract. Registry mechanics (::duplicate-*, ::unknown-scene) and scene + declarations (::invalid-scene) keep their own types." + [thunk] + (try + (thunk) + (catch :default cause + (let [data (ex-data cause)] + (if (contains? #{::decl/invalid-case ::decl/non-serializable-case} (:type data)) + (throw (ex-info (ex-message cause) + (assoc data :type ::invalid-case) + cause)) + (throw cause)))))) + +(defn- check-params! + "Validates case params against the scene's closed schema." + [case-id scene-id schema params] + (when-not (sm/validate schema params) + (throw (ex-info (str "invalid params for case " case-id) + {:type ::invalid-case + :id case-id + :scene scene-id + ::sm/explain (sm/explain schema params)})))) + +(defn- validate-case! + "Validates a registered case entry against its scene: unknown scene, + case-namespace mismatch, declaration schema and closed params." + [{:keys [id scene params] :as registered} scene-by-id] + (let [entry (get scene-by-id scene)] + (when (nil? entry) + (throw (ex-info (str "case " id " names an unknown scene: " scene) + {:type ::unknown-scene + :id id + :scene scene}))) + (when-not (= (name scene) (namespace id)) + (throw (ex-info (str "case id " id " is not namespaced by its scene: " scene) + {:type ::invalid-case + :id id + :scene scene}))) + (as-invalid-case! #(decl/check-registered-case! registered)) + (check-params! id scene (:params-schema entry) params) + registered)) + +(defn- describe-case + "Projects a validated case to its collected descriptor: scene description + and seed attached, internal keys stripped, transit wire checked." + [master-seed registered scene-entry] + (let [projected (-> (decl/project-case registered) + (assoc :scene-description (:description scene-entry) + :scene-seed (decl/derive-seed master-seed + (:scene registered) + (:params registered))))] + (as-invalid-case! #(decl/check-collected-case projected)))) + +(defn collect-cases + "Selected benchmark cases as plain data. + + Validates scenes, cases and parameters before any browser work. `:filter` + is a substring match on the case id; an empty selection throws ::no-cases." + [{:keys [master-seed] :as options}] + (check-master-seed! master-seed) + (let [pattern (:filter options) + scene-by-id (into {} (map (juxt :id identity)) (decl/scenes))] + (doseq [scene (decl/scenes)] + (decl/check-registered-scene! scene)) + (let [selected (->> (decl/cases) + (map #(validate-case! % scene-by-id)) + (filter #(or (str/blank? pattern) + (str/includes? (str (:id %)) pattern))) + (mapv #(describe-case master-seed % (get scene-by-id (:scene %)))))] + (when (empty? selected) + (throw (ex-info "no benchmark cases match the selection" + {:type ::no-cases + :filter pattern}))) + selected))) diff --git a/frontend/performance/benches/render_wasm/codec.cljs b/frontend/performance/benches/render_wasm/codec.cljs new file mode 100644 index 0000000000..868514d21f --- /dev/null +++ b/frontend/performance/benches/render_wasm/codec.cljs @@ -0,0 +1,84 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.codec + "Verifies that measurements survive transit encoding. + + This ns exists mainly to preserve NaNs in the transit encoding of results. + We treat two NaN values as the same measurement. + + Any data we cannot represent raises an error." + (:require + [app.common.transit :as transit])) + +(declare equivalent?) + +(defn check-value + "Checks optional transport data within depth and size limits. + + Args: + value: Trace or diagnostic partial data to validate. + + Returns: + The supplied value when traversal stays within depth 32, 100000 visited + values and 1048576 UTF-16 text units. Sequential collections have a finite + count. Numbers retain NaN and infinities. + + Throws: + An error for unsupported values or data that exceeds either limit." + [value] + (loop [pending (list [value 0]) + budget 100000 + text-budget 1048576] + (when (neg? text-budget) + (throw (ex-info "Transport data exceeds text limits" {:type ::data-limit}))) + (if-let [[item depth] (first pending)] + (do + (when (or (zero? budget) (> depth 32)) + (throw (ex-info "Transport data exceeds traversal limits" {:type ::data-limit}))) + (let [children (cond + (map? item) (mapcat (fn [[key value]] [[key (inc depth)] [value (inc depth)]]) item) + (or (and (sequential? item) (counted? item)) (set? item)) (map #(vector % (inc depth)) item) + (or (nil? item) (boolean? item) (number? item) (string? item) + (keyword? item) (symbol? item) (uuid? item) (inst? item)) nil + :else (throw (ex-info "Unsupported transport value" {:type ::unsupported-value})))] + (recur (concat children (rest pending)) (dec budget) + (- text-budget (cond + (string? item) (count item) + (or (keyword? item) (symbol? item)) (+ (count (name item)) (count (namespace item))) + :else 0))))) + value))) + +(defn- equivalent-entry? + "Finds a map key and compares its value, including keys that contain NaN." + [other [k v]] + (if-let [entry (find other k)] + (equivalent? v (val entry)) + (boolean (some (fn [[other-k other-v]] + (and (equivalent? k other-k) (equivalent? v other-v))) other)))) + +(defn equivalent? + "Checks Transit value equality while retaining NaN as a valid raw value." + [a b] + (or (= a b) + (cond + (and (number? a) (number? b)) (and (js/Number.isNaN a) (js/Number.isNaN b)) + (and (map? a) (map? b)) (and (= (count a) (count b)) + (every? #(equivalent-entry? b %) a)) + (and (sequential? a) (sequential? b)) (and (= (count a) (count b)) + (every? true? (map equivalent? a b))) + (and (set? a) (set? b)) (and (= (count a) (count b)) + (every? (fn [item] (some #(equivalent? item %) b)) a)) + :else false))) + +(defn encode-str + "Encodes through app.common.transit and rejects any loss of supplied evidence." + ([value] (encode-str value nil)) + ([value options] + (let [text (transit/encode-str value options)] + (when-not (equivalent? value (transit/decode-str text)) + (throw (ex-info "Transit cannot preserve this value" {:type ::unrepresentable-value}))) + text))) diff --git a/frontend/performance/benches/render_wasm/declarations.cljc b/frontend/performance/benches/render_wasm/declarations.cljc new file mode 100644 index 0000000000..5a131e825e --- /dev/null +++ b/frontend/performance/benches/render_wasm/declarations.cljc @@ -0,0 +1,373 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.declarations + "Declarations, registry and contracts for renderer benchmark scenes and cases. + + A scene declares an id, description, parameters schema and build function. + A case declares an id namespaced by its scene, parameters, view, context + policy and a run function. Optional preparation and reset functions transform + the context before and after its main execution. + + Ordinary :require loads the declaration macros and runtime helpers. + Collection validates declarations and produces plain descriptors." + #?(:cljs (:require-macros + [app.common.exceptions :as ex] + [benches.render-wasm.declarations])) + #?(:clj (:require [app.common.schema :as-alias sm])) + #?(:cljs + (:require + [app.common.schema :as sm] + [app.common.transit :as t] + [clojure.string :as str]))) + +#?(:clj + (do + (defn- single-arg-vector? + "True when `form` is a vector holding one plain symbol." + [form] + (and (vector? form) + (= 1 (count form)) + (symbol? (first form)) + (not= '& (first form)))) + + (defn- check-arg-vector! + [macro id argv] + (when-not (single-arg-vector? argv) + (throw (ex-info (str macro " " id " requires one argument vector with one symbol") + {:type ::invalid-run-function + :id id + :form argv})))) + + (defmacro defscene + "Registers a scene and returns its id. + + `opts` supplies :description and :params-schema. `argv` names one scene + parameter symbol, and `body` builds the snapshot. + + Example: + + (defscene :rects opts [params] (scene params))" + [id opts argv & body] + (check-arg-vector! "defscene" id argv) + `(benches.render-wasm.declarations/register-scene! + (assoc ~opts :id ~id :ns ~(str *ns*) :build (fn ~argv ~@body)))) + + (defn- case-registration-form + [id scene opts run-fn] + (let [options (gensym "options")] + `(let [~options ~opts] + (when (contains? ~options :run!) + (throw (ex-info (str "defcase " ~id " requires its run function as the fourth argument") + {:type :benches.render-wasm.declarations/invalid-run-function + :id ~id}))) + (when (nil? ~scene) + (throw (ex-info (str "defcase " ~id " requires a :scene/case id") + {:type :benches.render-wasm.declarations/invalid-run-function + :id ~id + :form ~id}))) + (benches.render-wasm.declarations/register-case! + (assoc ~options :id ~id :scene ~scene :ns ~(str *ns*) + ~@(when run-fn [:run! run-fn])))))) + + (defn- scene-from-id + "Returns the scene keyword from a qualified case id, or nil." + [id] + (when-let [scene-ns (namespace id)] + (keyword scene-ns))) + + (defmacro defcase + "Registers a case and returns its id. + + The scene comes from the namespace part of the :scene/case id. An + optional argument vector and body supply the run function, which accepts + one command context and returns its next context or a Promise of it. + `[:preparation :callback]` runs once after upload. Preparation renders + Full by default; `:render :none` selects a cold first render. `:reset` + runs after each successful main execution. Collection produces plain + case descriptors. + + Example: + + (defcase :rects/pan options [context] (pan* context pan-options))" + ([id opts] + (case-registration-form id (scene-from-id id) opts nil)) + ([id opts argv & body] + (check-arg-vector! "defcase" id argv) + (case-registration-form id (scene-from-id id) opts `(fn ~argv ~@body)))))) + +#?(:cljs + (do + ;; Case execution context: + ;; :case - collected descriptor + ;; :scene - scene snapshot {:objects ... :refs ...} + ;; :view - current camera view + ;; :hooks - clock, scheduling, guard and renderer operations + ;; Case bodies return the next context or a Promise of it. + + ;; Declarations and their transitive imports load in Node and the browser. + ;; Browser capabilities enter through the execution context; resource + ;; acquisition belongs to untimed preparation. Scenes keep their imports + ;; safe for Node discovery. + + (def default-viewport + "Default viewport dimensions and device pixel ratio for a case." + {:width 1920 :height 1080 :dpr 2}) + + (def schema:viewport + [:map {:closed true} + [:width {:optional true} ::sm/positive-safe-number] + [:height {:optional true} ::sm/positive-safe-number] + [:dpr {:optional true} ::sm/positive-safe-number]]) + + (def schema:view + "Camera placement with finite coordinates, positive scale and optional + viewport overrides." + [:map {:closed true} + [:scale ::sm/positive-safe-number] + [:x ::sm/safe-number] + [:y ::sm/safe-number] + [:viewport {:optional true} schema:viewport]]) + + (defn resolve-viewport + "Resolves a view's viewport overrides against the default viewport." + [view] + (merge default-viewport (:viewport view))) + + (def schema:scene + [:map {:closed true} + [:id simple-keyword?] + [:ns string?] + [:description string?] + [:params-schema vector?] + [:build fn?]]) + + (def schema:registered-case + [:map {:closed true} + [:id qualified-keyword?] + [:scene simple-keyword?] + [:ns string?] + [:params [:map]] + [:view schema:view] + [:context [:enum :fresh :reuse]] + [:preparation {:optional true} + [:map + [:render {:optional true} [:enum :full :none]] + [:callback {:optional true} fn?]]] + [:reset {:optional true} fn?] + [:run! {:optional true} fn?]]) + + (def schema:collected-case + [:map {:closed true} + [:id qualified-keyword?] + [:scene simple-keyword?] + [:scene-description string?] + [:scene-seed [:int {:min 0 :max 4294967295}]] + [:params [:map]] + [:view schema:view] + [:context [:enum :fresh :reuse]] + [:preparation [:map [:render {:optional true} [:enum :full :none]]]]]) + + (defonce registry + (atom {:scenes {} + :scene-order [] + :cases {} + :case-order []})) + + (defn- register-entry + [entries-key order-key kind duplicate-type {:keys [id ns] :as entry}] + (let [existing (get-in @registry [entries-key id])] + (when (and (some? existing) (not= ns (:ns existing))) + (throw (ex-info (str "duplicate " kind " id: " id) + {:type duplicate-type + :id id + :ns ns + :existing-ns (:ns existing)}))) + (swap! registry + #(cond-> (assoc-in % [entries-key id] entry) + (not (contains? (get % entries-key) id)) + (update order-key conj id)))) + id) + + (defn register-scene! + "Registers a scene. The same id from the same namespace replaces the + entry in declaration order. Another namespace raises ::duplicate-scene." + [scene] + (register-entry :scenes :scene-order "scene" ::duplicate-scene scene)) + + (defn register-case! + "Registers a case. The same id from the same namespace replaces the + entry in declaration order. Another namespace raises ::duplicate-case." + [bench-case] + (register-entry :cases :case-order "case" ::duplicate-case bench-case)) + + (defn- unregister-entry + [entries-key order-key id] + (swap! registry + (fn [reg] + (-> reg + (update entries-key dissoc id) + (update order-key #(vec (remove #{id} %)))))) + id) + + (defn unregister-scene! + "Removes a scene registration and returns its id." + [id] + (unregister-entry :scenes :scene-order id)) + + (defn unregister-case! + "Removes a case registration and returns its id." + [id] + (unregister-entry :cases :case-order id)) + + (defn scenes + "Returns registered scenes in declaration order." + [] + (mapv (:scenes @registry) (:scene-order @registry))) + + (defn cases + "Returns registered cases in declaration order." + [] + (mapv (:cases @registry) (:case-order @registry))) + + (defn registered-scene + "Returns the full registration for a scene id." + [id] + (get-in @registry [:scenes id])) + + (defn registered-case + "Returns the full registration for a case id, including its run function." + [id] + (get-in @registry [:cases id])) + + (defn- check-params-schema! + [{:keys [id params-schema]}] + (try + (sm/schema params-schema) + (catch :default cause + (throw (ex-info (str "invalid params schema for scene " id) + {:type ::invalid-scene :id id} + cause))))) + + (defn- check-declaration + [schema declaration message error-type] + (when-not (sm/validate schema declaration) + (throw (ex-info (str message (:id declaration)) + {:type error-type + :id (:id declaration) + ::sm/explain (sm/explain schema declaration)}))) + declaration) + + (defn check-registered-scene! + "Validates a scene declaration and returns it unchanged." + [scene] + (check-params-schema! scene) + (check-declaration schema:scene scene "invalid scene declaration: " ::invalid-scene)) + + (defn check-registered-case! + "Validates a registered case declaration and returns it unchanged." + [bench-case] + (check-declaration schema:registered-case bench-case "invalid case declaration: " ::invalid-case)) + + (defn- check-param-keys! + "Rejects params keys that `canonical-params` cannot tell apart: only + simple keywords render distinctly by name." + [scene-id params] + (letfn [(check [value path] + (cond + (map? value) + (doseq [[k v] value] + (when-not (simple-keyword? k) + (throw (ex-info (str "seed identity requires simple keyword keys at " path) + {:type ::invalid-params + :scene scene-id + :path path + :key k}))) + (check v (str path "/" (name k)))) + + (vector? value) + (dorun (map-indexed (fn [index v] (check v (str path "[" index "]"))) value))))] + (check params ""))) + + (defn- canonical-params + "Stable textual identity of a params map: keys render sorted by name, + vectors keep their order, nested values render the same way. Map key + order is an accident of construction, so sorting keeps the identity on + the parameters themselves: equal parameters give the same seed whatever + their literal order. Keys render by name, so params maps use simple + keyword keys; derive-seed rejects anything else." + [params] + (letfn [(render [value] + (cond + (map? value) + (str "{" + (str/join "," + (map (fn [[k v]] (str (name k) ":" (render v))) + (sort-by (comp name key) value))) + "}") + + (vector? value) + (str "[" (str/join "," (map render value)) "]") + + :else + (pr-str value)))] + (render params))) + + (defn derive-seed + "FNV-1a 32-bit keyed hash of a case's parameter identity. + + Fowler-Noll-Vo, 32-bit, offset basis 2166136261, prime 16777619, hashed + over `\\0/`. A keyed hash, not + the scene PRNG: the scene's mulberry32 generator is seeded with the + result, and a sequential draw cannot substitute because it would couple + cases through collection order. Keyed by scene and params only, so the + seed does not depend on case name, collection order, operation body, + attempts or A/B feature selection. Params maps use simple keyword keys; + namespaced keywords and string keys cannot be told apart by name and make + derive-seed throw ::invalid-params." + [master-seed scene-id params] + (check-param-keys! scene-id params) + (let [text (str master-seed "\u0000" (name scene-id) "/" (canonical-params params))] + (loop [index 0 + seed 2166136261] + (if (< index (count text)) + (recur (inc index) + (unsigned-bit-shift-right + (js/Math.imul (bit-xor seed (.charCodeAt text index)) 16777619) + 0)) + seed)))) + + (defn transit-round-trips? + "True when `value` survives Transit encoding and decoding with the same + wire representation, including NaN and infinities." + [value] + (true? (ex/ignoring + (let [encoded (t/encode-str value)] + (= encoded (-> encoded t/decode-str t/encode-str)))))) + + (defn project-case + "Returns a data descriptor with the declared preparation settings. + + Args: + bench-case: registered case with local executable functions. + + Returns: + Case data with an explicit preparation map." + [bench-case] + (-> bench-case + (select-keys [:id :scene :params :view :context :preparation]) + (update :preparation #(dissoc (or % {}) :callback)))) + + (defn check-collected-case + "Validates a collected case descriptor and its Transit representation, + returning the descriptor unchanged." + [projected] + (check-declaration schema:collected-case projected "invalid collected case: " ::invalid-case) + (when-not (transit-round-trips? projected) + (throw (ex-info (str "collected case does not survive the transit wire: " (:id projected)) + {:type ::non-serializable-case + :id (:id projected)}))) + projected))) diff --git a/frontend/performance/benches/render_wasm/errors.cljc b/frontend/performance/benches/render_wasm/errors.cljc new file mode 100644 index 0000000000..c30d05be68 --- /dev/null +++ b/frontend/performance/benches/render_wasm/errors.cljc @@ -0,0 +1,500 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.errors + "Failure values and plain diagnostics for renderer benchmarks. + + A step that can fail returns its value or an Error. `attempt` catches thrown + values and `failure?` identifies the returned Error. Fatal failures mark + writes to the owner's record or log and propagate to a boundary that uses + `:catch-fatal`. + + `evidence` and `describe-cause` produce diagnostics for saved records and + Transit responses." + #?(:cljs (:require + [benches.render-wasm.codec :as codec] + [clojure.string :as str])) + #?(:cljs (:require-macros + [app.common.exceptions :as ex] + [benches.render-wasm.errors]))) + +#?(:clj + (defmacro attempt + "Evaluates `body` and returns its value, or the failure it throws. + + A thrown value becomes an Error through `as-failure`. Fatal failures + propagate unless a leading `:catch-fatal` requests them as values. + Inside an `^:async` function, `body` may use `await`. + + Examples: + + (attempt (parse text)) + (attempt (await (open-page owner))) + (attempt :catch-fatal (await (run-session state)))" + [& forms] + (let [catch-fatal? (= :catch-fatal (first forms)) + body (if catch-fatal? (rest forms) forms) + thrown (gensym "thrown") + failure (gensym "failure")] + `(try + ~@body + (catch :default ~thrown + (let [~failure (benches.render-wasm.errors/as-failure ~thrown)] + ~(if catch-fatal? + failure + `(if (benches.render-wasm.errors/fatal? ~failure) + (throw ~failure) + ~failure)))))))) + +#?(:cljs + (do + (def ^:private schema:cause-value + [:or nil? boolean? keyword? number? string?]) + + (def schema:cause-detail + "One level of a portable exception description." + [:map {:closed true} + [:message string?] + [:fn {:optional true} schema:cause-value] + [:code {:optional true} schema:cause-value] + [:type {:optional true} schema:cause-value] + [:hint {:optional true} schema:cause-value]]) + + (def ^:private diagnostic-fields + [[:phase string?] + [:message string?] + [:operation {:optional true} keyword?] + [:path {:optional true} [:string {:max 1024}]] + [:status {:optional true} [:enum "failed" "stale"]] + [:cause {:optional true} [:vector schema:cause-detail]] + [:partial {:optional true} map?] + [:reason {:optional true} :any] + [:deadline-exceeded {:optional true} boolean?] + [:expected-id {:optional true} [:string {:max 1024}]] + [:actual-id {:optional true} [:string {:max 1024}]] + [:command {:optional true} [:string {:max 1024}]] + [:exit-code {:optional true} :int] + [:pid {:optional true} [:int {:min 1}]] + [:pgid {:optional true} [:int {:min 1}]] + [:signals {:optional true} [:vector [:string {:max 1024}]]] + [:profile-path {:optional true} [:string {:max 1024}]] + [:fatal {:optional true} boolean?] + [:unresolved-process {:optional true} boolean?] + [:signal {:optional true} [:string {:max 1024}]] + [:endpoint {:optional true} [:string {:max 1024}]]]) + + (def schema:error-entry + "One failure occurrence with its operation and portable diagnostics." + (into [:map {:closed true}] diagnostic-fields)) + + (def schema:diagnostic + "Failure location, message and an optional ordered collection of errors." + (into [:map] + (conj diagnostic-fields + [:errors {:optional true} [:vector {:min 1} schema:error-entry]]))) + + (def ^:private evidence-cause-depth 3) + (def ^:private max-cause-chars 500) + (def ^:private unrenderable-value "unrenderable value") + (def ^:private unrenderable-cause-text "unrenderable cause") + + (defn failure? + "True when `x` is an Error." + [x] + (instance? js/Error x)) + + (defn message-of + "Returns the error message or string form of `cause`, with a fallback + when formatting fails. The default fallback is unknown failure." + ([cause] + (message-of cause "unknown failure")) + ([cause fallback] + (or (ex/ignoring (let [message (ex-message cause)] + (when (string? message) message))) + (ex/ignoring (str cause)) + fallback))) + + (defn fatal-error + "Returns an exception that `fatal?` recognizes, with `data` and `cause`." + [message data cause] + (ex-info message (assoc data :fatal true) cause)) + + (defn fatal? + "True when `cause` or one of its causes has the ex-data `:fatal` marker." + [cause] + (loop [cause cause + seen #{}] + (cond + (nil? cause) false + (:fatal (ex-data cause)) true + (contains? seen cause) false + :else (recur (ex-cause cause) (conj seen cause))))) + + (defn as-failure + "Returns an Error for `thrown`. Errors pass through; other values supply + the message and the `:thrown` ex-data." + [thrown] + (if (failure? thrown) + thrown + (ex-info (if (string? thrown) + thrown + (str "non-error value thrown: " (or (ex/ignoring (pr-str thrown)) "unprintable value"))) + {:thrown thrown}))) + + (defn with-data + "Adds default ex-data to an Error, keeping its message and using it as + the cause. Existing keys win. Other values pass through." + [x data] + (if (failure? x) + (ex-info (message-of x) (merge data (ex-data x)) x) + x)) + + (declare combine) + + (defn with-context + "Returns an exception with `message` and added diagnostic context. + New fields override existing fields; existing partial measurements win + over earlier measurements in `data`. An exception with ex-data keeps + its underlying cause. Other values become the cause." + [cause message data] + (let [existing (ex-data cause) + malformed (and (contains? existing :partial) (not (map? (:partial existing))) + (map? (:partial data))) + source (if malformed (combine [{:error cause :partial (:partial existing)}]) cause) + context (cond-> (merge (ex-data source) data) + (contains? data :partial) + (assoc :partial (if (map? (:partial data)) + (merge (:partial data) (when (map? (:partial existing)) (:partial existing))) + (:partial data))))] + (ex-info message context (if (and (some? existing) (not malformed)) (ex-cause cause) source)))) + + (defn- merge-error-context + [outer inner] + (cond-> (merge outer inner) + (and (map? (:partial outer)) (map? (:partial inner))) + (assoc :partial (merge (:partial outer) (:partial inner))))) + + (defn- get-error-collection + [cause] + (loop [current cause + seen #{} + context {}] + (let [data (ex-data current) + context (merge-error-context context (dissoc data ::errors))] + (if-let [entries (::errors data)] + (update entries 0 #(merge-error-context context %)) + (when (and (ex-cause current) (not (contains? seen current))) + (recur (ex-cause current) (conj seen current) context)))))) + + (defn get-errors + "Returns failure occurrences in order, retaining their Error objects. + + Args: + cause: An Error, including one wrapped with additional context. + + Returns: + A flat vector of maps with an Error under :error and available phase, + operation and path context." + [cause] + (or (get-error-collection cause) + [(assoc (select-keys (ex-data cause) [:phase :operation :path]) :error cause)])) + + (defn combine + "Combines failure occurrences into an Error, or returns nil on success. + + Args: + entries: Ordered maps with :error and optional :phase, :operation and + :path. Nil errors represent successful operations. An Error may + contain a collection from an earlier operation. + + Returns: + An Error retaining a flat collection and the first error's message + and data. Retained Error objects expose their causes. Any fatal + occurrence makes the combined error fatal." + [entries] + (let [sources (filterv #(failure? (:error %)) entries) + error (:error (first sources)) + entries (reduce (fn [collected {:keys [error] :as entry}] + (let [context (dissoc entry :error) + group (get-errors error) + group (if (seq context) + (mapv #(merge-error-context context %) group) + group)] + (if (empty? collected) group (into collected group)))) + [] sources)] + (when error + (ex-info (message-of error) + (cond-> (merge (select-keys (first entries) [:phase :operation :path]) + (ex-data error) + {::errors entries}) + (some #(fatal? (:error %)) sources) (assoc :fatal true)) + (if (::errors (ex-data error)) (ex-cause error) error))))) + + (defn stale? + "True when the exception's own data marks superseded work." + [cause] + (true? (::stale (ex-data cause)))) + + (defn fail-data + "Marks diagnostic data with the failed status." + [data] + (assoc data :status "failed")) + + (defn bounded-text + "Returns text within a size limit, preserving UTF-16 boundaries. + + Converts `value` to a string and limits it to max-chars UTF-16 code + units. If conversion fails, it returns 'unrenderable value'. When + truncating, we check whether the cut would leave a high surrogate + without its matching low surrogate. If so, we remove that last code unit + to keep the pair intact. + + Args: + value: Value to convert to text. + max-chars: Positive limit in UTF-16 units. + + Returns: + Bounded text, or a fallback string when conversion fails." + [value max-chars] + (or (ex/ignoring + (let [text (str value)] + (if (<= (count text) max-chars) + text + (let [cut (subs text 0 max-chars) + last (.charCodeAt cut (dec max-chars))] + (if (and (>= last 0xD800) + (<= last 0xDBFF) + (let [nxt (.charCodeAt text max-chars)] + (and (>= nxt 0xDC00) (<= nxt 0xDFFF)))) + (subs cut 0 (dec max-chars)) + cut))))) + (subs unrenderable-value 0 (min max-chars (count unrenderable-value))))) + + (defn terminal-text + "Returns bounded text with terminal controls escaped. + + Args: + value: Value to render as text. + max-chars: Positive limit in UTF-16 units after escaping. + + Returns: + Text with C0, DEL and C1 controls represented as Unicode escapes." + [value max-chars] + (bounded-text + (or (ex/ignoring + (str/replace (bounded-text value max-chars) #"[\x00-\x1f\x7f-\x9f]" + (fn [character] + (str "\\u" (.padStart (.toString (.charCodeAt character 0) 16) 4 "0"))))) + unrenderable-value) + max-chars)) + + (defn format-value + "Returns a bounded, readable value with terminal controls escaped. + + Args: + value: Value to print. Strings use JSON quoting. + max-chars: Positive limit in UTF-16 units after escaping. + + Returns: + Readable text, or fallback text when printing fails." + [value max-chars] + (terminal-text + (or (ex/ignoring + (if (string? value) + (js/JSON.stringify (bounded-text value max-chars)) + (binding [*print-level* 8 *print-length* 32] + (pr-str value)))) + unrenderable-value) + max-chars)) + + (defn project-context + "Returns bounded identity, process and ownership details. + + Args: + data: Diagnostic data with available identity or process fields. + + Returns: + Bounded text, integer process IDs and exit code, attempted signals, + and available fatal or unresolved-ownership markers." + [data] + (cond-> {} + (string? (:expected-id data)) (assoc :expected-id (bounded-text (:expected-id data) 1024)) + (string? (:actual-id data)) (assoc :actual-id (bounded-text (:actual-id data) 1024)) + (string? (:command data)) (assoc :command (bounded-text (:command data) 1024)) + (integer? (:exit-code data)) (assoc :exit-code (:exit-code data)) + (and (integer? (:pid data)) (pos? (:pid data))) (assoc :pid (:pid data)) + (and (integer? (:pgid data)) (pos? (:pgid data))) (assoc :pgid (:pgid data)) + (and (vector? (:signals data)) (every? string? (:signals data))) + (assoc :signals (mapv #(bounded-text % 1024) (:signals data))) + (string? (:profile-path data)) (assoc :profile-path (bounded-text (:profile-path data) 1024)) + (true? (:fatal data)) (assoc :fatal true) + (true? (:unresolved-process data)) (assoc :unresolved-process true) + (string? (:signal data)) (assoc :signal (bounded-text (:signal data) 1024)) + (string? (:endpoint data)) (assoc :endpoint (bounded-text (:endpoint data) 1024)))) + + (declare project-diagnostic) + + (defn- sanitize-cause-value + [value] + (cond + (or (nil? value) (boolean? value) (keyword? value) (number? value)) + value + + (string? value) + (bounded-text value max-cause-chars) + + :else + (bounded-text (or (ex/ignoring + (binding [*print-level* 8 *print-length* 32] + (pr-str value))) + unrenderable-value) + max-cause-chars))) + + (defn- describe-cause-level + [cause] + (or (ex/ignoring + (let [message (ex/ignoring (ex-message cause)) + data (ex/ignoring (ex-data cause)) + text (cond + (string? message) message + (string? cause) cause + (nil? cause) "unknown failure" + :else (binding [*print-level* 8 *print-length* 32] (pr-str cause))) + out {:message (sanitize-cause-value text)}] + (if (map? data) + (reduce (fn [m k] + (if (contains? data k) + (assoc m k (sanitize-cause-value (get data k))) + m)) + out + [:fn :code :type :hint]) + out))) + {:message unrenderable-cause-text})) + + (defn describe-cause + "Returns up to three cause levels as plain data. Each level keeps a + bounded message and available :fn, :code, :type and :hint details. + Unprintable values produce fallback strings." + [cause] + (or (ex/ignoring + (loop [current cause + depth 0 + acc [] + seen #{}] + (if (or (nil? current) (contains? seen current) (>= depth evidence-cause-depth)) + (if (seq acc) + acc + [{:message "unknown failure"}]) + (let [level (describe-cause-level current) + next (ex/ignoring (ex-cause current))] + (recur next (inc depth) (conj acc level) (conj seen current)))))) + [{:message unrenderable-cause-text}])) + + (defn- diagnostic-data + [phase cause] + (let [partial (ex/ignoring (:partial (ex-data cause)))] + (cond-> (merge (project-context (ex-data cause)) + (select-keys (ex-data cause) [:operation]) + {:phase (if (keyword? phase) (name phase) (str phase)) + :message (bounded-text (message-of cause) 1024) + :cause (describe-cause cause)}) + (contains? (ex-data cause) :partial) (assoc :partial partial) + (string? (:path (ex-data cause))) + (assoc :path (bounded-text (:path (ex-data cause)) 1024))))) + + (defn evidence + "Returns plain diagnostics, including ordered failure occurrences. + + Args: + phase: Failure location for the enclosing operation. + cause: Error with available operation context and collected failures. + + Returns: + Portable diagnostics with bounded cause descriptions. Collected + errors retain occurrence order and messages within 1024 UTF-16 units." + [phase cause] + (let [entries (get-error-collection cause) + projected (when entries + (mapv (fn [{:keys [error operation path partial] entry-phase :phase :as entry}] + (cond-> (merge (diagnostic-data (or entry-phase phase) error) + (project-context entry) + (select-keys entry [:reason :deadline-exceeded]) + {:message (bounded-text (message-of error) 1024)}) + operation (assoc :operation operation) + partial (assoc :partial partial) + (string? path) (assoc :path (bounded-text path 1024)))) + entries))] + (project-diagnostic + (cond-> (diagnostic-data phase cause) + projected (assoc :errors projected) + (map? (:partial (first projected))) (assoc :partial (:partial (first projected))) + (some :unresolved-process projected) (assoc :unresolved-process true))))) + + (defn- partial-encoding-error + [path cause] + {:phase "diagnostic-encoding" :operation :encode-diagnostic + :path (format-value path 1024) + :message "Diagnostic partial evidence cannot cross Transit" + :cause (describe-cause cause)}) + + (defn- project-partial + [partial] + (if (map? partial) + (reduce-kv (fn [[kept rejected] key value] + (try + (codec/check-value {key value}) + (codec/encode-str {key value}) + [(assoc kept key value) rejected] + (catch :default cause + [kept (conj rejected (partial-encoding-error [:partial key] cause))]))) + [{} []] + partial) + [nil [(partial-encoding-error [:partial] + (ex-info "Diagnostic partial evidence must be a map" {}))]])) + + (defn- project-entry + [entry] + (let [[partial rejected] (when (contains? entry :partial) (project-partial (:partial entry))) + projected (cond-> (project-context entry) + (contains? #{"failed" "stale"} (:status entry)) (assoc :status (:status entry)) + (boolean? (:deadline-exceeded entry)) (assoc :deadline-exceeded (:deadline-exceeded entry)) + (keyword? (:operation entry)) (assoc :operation (:operation entry)) + (contains? entry :phase) (assoc :phase (bounded-text (:phase entry) 1024)) + (contains? entry :message) (assoc :message (bounded-text (:message entry) 1024)) + (contains? entry :reason) (assoc :reason (sanitize-cause-value (:reason entry))) + (contains? entry :cause) + (assoc :cause (mapv (fn [level] + (reduce-kv (fn [out key value] + (assoc out key (sanitize-cause-value value))) + {} (select-keys level [:message :fn :code :type :hint]))) + (take evidence-cause-depth (:cause entry)))) + (some? partial) (assoc :partial partial) + (string? (:path entry)) (assoc :path (bounded-text (:path entry) 1024)))] + [projected rejected])) + + (defn project-diagnostic + "Returns bounded diagnostic fields and representable partial evidence. + + Args: + evidence: Execution evidence with optional failure details. + + Returns: + Diagnostic fields with bounded text. Partial entries that cannot + cross Transit produce ordered encoding diagnostics. Representable + partial values retain their raw numeric values." + [evidence] + (let [alias? (and (seq (:errors evidence)) (contains? evidence :partial) + (identical? (:partial evidence) (:partial (first (:errors evidence))))) + [projected rejected] (project-entry (if alias? (dissoc evidence :partial) evidence)) + entries (mapv project-entry (:errors evidence)) + rejected (into (vec rejected) (mapcat second) entries) + errors (mapv first entries) + projected (cond-> projected + (and alias? (contains? (first errors) :partial)) + (assoc :partial (:partial (first errors))))] + (cond-> projected + (seq errors) (assoc :errors errors) + (seq rejected) (assoc :errors (into (if (seq errors) errors [(dissoc projected :partial)]) rejected))))))) diff --git a/frontend/performance/benches/render_wasm/features/plan.cljs b/frontend/performance/benches/render_wasm/features/plan.cljs new file mode 100644 index 0000000000..e2b49d49d8 --- /dev/null +++ b/frontend/performance/benches/render_wasm/features/plan.cljs @@ -0,0 +1,76 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.features.plan + "Requested-feature configurations and comparison edges." + (:require + [benches.render-wasm.features.selection :as selection] + [clojure.string :as str])) + +(def ^:const default-max-features 3) + +(defn check-manifest + "Checks names and empty declared defaults against Cargo metadata's feature + definitions. Cargo owns activation; these definitions are not resolved here." + [features definitions] + (let [features (:features (selection/feature-selection {:features features}))] + (when-not (and (map? definitions) + (every? string? (keys definitions)) + (every? #(and (vector? %) (every? string? %)) (vals definitions))) + (throw (ex-info "invalid Cargo feature definitions" {:type ::invalid-manifest}))) + (when (seq (get definitions "default")) + (throw (ex-info "ab requires empty declared Cargo defaults" {:type ::nonempty-defaults}))) + (when-let [unknown (seq (remove #(contains? definitions %) features))] + (throw (ex-info "unknown requested Cargo features" + {:type ::unknown-features :features (vec unknown)}))) + features)) + +(defn plan + "Plans sorted, MSB-first reflected Gray configurations and all off-to-on + edges. Each selection disables Cargo defaults. Execution needs confirmation + above eight configurations or with an empty filter; dry-run needs none." + [features {:keys [max-features confirm dry-run filter] + :or {max-features default-max-features confirm false dry-run false}}] + (let [features (:features (selection/feature-selection {:features features})) + n (count features)] + (when-not (and (int? max-features) (<= 1 max-features 6)) + (throw (ex-info "max-features must be an integer from 1 to 6" {:type ::invalid-limit}))) + (when-not (and (boolean? confirm) (boolean? dry-run) + (or (nil? filter) (string? filter))) + (throw (ex-info "invalid feature planning options" {:type ::invalid-options}))) + (when (zero? n) + (throw (ex-info "ab requires at least one requested feature" {:type ::missing-features}))) + (when (> n max-features) + (throw (ex-info "requested features exceed max-features" + {:type ::too-many-features :count n :max-features max-features}))) + (when (and (not dry-run) (not confirm) + (or (> (bit-shift-left 1 n) 8) (str/blank? filter))) + (throw (ex-info "ab requires --confirm above eight configurations or with an empty filter" + {:type ::confirmation-required}))) + (let [slug (fn [bits] + (str/join (map #(if (bit-test bits %) "1" "0") + (reverse (range n))))) + gray-values (mapv #(bit-xor % (bit-shift-right % 1)) + (range (bit-shift-left 1 n))) + configurations (mapv (fn [bits] + {:slug (slug bits) + :selection {:default-features? false + :features (into [] + (keep-indexed + (fn [i feature] + (when (bit-test bits (- n i 1)) + feature))) + features)}}) + gray-values) + edges (vec + (for [bits gray-values + [i feature] (map-indexed vector features) + :let [bit (- n i 1)] + :when (not (bit-test bits bit))] + {:from (slug bits) + :to (slug (bit-set bits bit)) + :feature feature}))] + {:features features :configurations configurations :edges edges}))) diff --git a/frontend/performance/benches/render_wasm/features/selection.cljs b/frontend/performance/benches/render_wasm/features/selection.cljs new file mode 100644 index 0000000000..338c1e713e --- /dev/null +++ b/frontend/performance/benches/render_wasm/features/selection.cljs @@ -0,0 +1,29 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.features.selection + "Feature validation.") + +(def ^:private diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"}) + +(defn feature-name? + "True for a Cargo feature name made of letters, digits, underscores or hyphens." + [value] + (and (string? value) + (boolean (re-matches #"[A-Za-z0-9_-]+" value)))) + +(defn feature-selection + "Validates measurement feature selection before allocating an attempt. + Cargo validates functional feature names during the retained build." + [{:keys [default-features? features] + :or {default-features? true features []}}] + (when-not (and (boolean? default-features?) (vector? features) + (every? feature-name? features)) + (throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features}))) + (when-let [selected (seq (filter diagnostic-features features))] + (throw (ex-info "measurement builds cannot enable diagnostic Cargo features" + {:type ::diagnostic-features :features (vec selected)}))) + {:default-features? default-features? :features (vec (sort (distinct features)))}) diff --git a/frontend/performance/benches/render_wasm/fingerprint.cljs b/frontend/performance/benches/render_wasm/fingerprint.cljs new file mode 100644 index 0000000000..1d8be614cc --- /dev/null +++ b/frontend/performance/benches/render_wasm/fingerprint.cljs @@ -0,0 +1,77 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.fingerprint + "Scene fingerprints to uniquely identify snapshots by their content." + (:require + [benches.render-wasm.codec :as codec] + [goog.crypt :as crypt] + [goog.crypt.Sha256])) + +(declare canonicalize) + +(defn- canonical-text + "Encodes a Transit representation as text." + [value] + (js/JSON.stringify value)) + +(defn- canonical-array + "Preserves array order while canonicalizing each element." + [value] + (into-array (map canonicalize (array-seq value)))) + +(defn- canonical-pairs + "Orders a compound map by canonical key text, preserving key/value pairs." + [value] + (->> (partition 2 (array-seq value)) + (map (fn [[k v]] [(canonicalize k) (canonicalize v)])) + (sort-by (comp canonical-text first)) + (mapcat identity) + (into-array))) + +(defn- canonicalize + "Orders verbose Transit objects, compound maps and unordered sets. + Tagged ordered-map and ordered-set representations retain their array order." + [value] + (cond + (array? value) + (canonical-array value) + + (and (some? value) (= "object" (goog/typeOf value))) + (let [keys (sort (array-seq (js/Object.keys value))) + out (js/Object.create nil)] + (doseq [k keys] + (let [v (unchecked-get value k)] + (aset out k + (cond + (= k "~#cmap") (canonical-pairs v) + (= k "~#set") (->> (array-seq v) + (map canonicalize) + (sort-by canonical-text) + (into-array)) + :else (canonicalize v))))) + out) + + :else value)) + +(defn snapshot-text + "Returns canonical Transit text for a validated snapshot. + Realizing :objects removes ObjectsMap's serialized shape strings while keeping + production shape, geometry and path tags in the values." + [snapshot] + (-> (assoc snapshot :objects (into {} (:objects snapshot))) + (codec/encode-str {:type :json-verbose}) + (js/JSON.parse) + (canonicalize) + (canonical-text))) + +(defn fingerprint + "Returns the SHA-256 and digest for a validated snapshot." + [snapshot] + (let [hash (goog.crypt.Sha256.)] + (.update hash (crypt/stringToUtf8ByteArray (snapshot-text snapshot))) + {:algorithm :sha-256 + :digest (crypt/byteArrayToHex (.digest hash))})) diff --git a/frontend/performance/benches/render_wasm/identity.cljc b/frontend/performance/benches/render_wasm/identity.cljc new file mode 100644 index 0000000000..79e4c3ef4a --- /dev/null +++ b/frontend/performance/benches/render_wasm/identity.cljc @@ -0,0 +1,36 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.identity + "Build identity shared by the isolated Node catalog and browser bundle." + #?(:cljs (:require-macros [benches.render-wasm.identity :refer [compiled-profile-marks]])) + #?(:clj (:require [app.render-wasm.performance :as performance]))) + +#?(:clj + (defmacro compiled-profile-marks + "Captures the profiling switch used by renderer macros in this compiler." + [] + (performance/enabled?))) + +#?(:cljs + (do + (goog-define build-id "unprepared") + + (defn data + [] + {:build-id build-id :cljs-profile-marks (compiled-profile-marks)}) + + (defn check + "Checks a loaded bundle's identity against the selected build." + [expected-id actual] + (when-not (and (map? actual) + (string? (:build-id actual)) + (= expected-id (:build-id actual))) + (throw (ex-info "renderer build identity mismatch" + {:type ::mismatch + :expected-id expected-id + :actual-id (:build-id actual)}))) + actual))) diff --git a/frontend/performance/benches/render_wasm/measurement.cljs b/frontend/performance/benches/render_wasm/measurement.cljs new file mode 100644 index 0000000000..a605ac2aa9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/measurement.cljs @@ -0,0 +1,85 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.measurement + "Projects browser phase evidence into saved attempts." + (:require + [app.common.schema :as sm] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.result :as result])) + +(def ^:private schema:response + [:multi {:dispatch :status} + ["ok" [:map [:status [:= "ok"]]]] + ["failed" (sm/merge errors/schema:diagnostic [:map [:status [:= "failed"]]])] + ["stale" [:map {:closed true} [:status [:= "stale"]]]]]) + +(defn partial-phases + "Returns the valid prefix of delivered phase history. + + Args: + phases: raw blocks from a browser response. + preparation-id: expected owner of the blocks. + order: allowed phase names in execution order. + + Returns: + Valid blocks up to the first malformed block or failed phase." + [phases preparation-id order] + (loop [phases (when (sequential? phases) (seq phases)) order order out []] + (if-let [phase (first phases)] + (if (and (sm/validate result/schema:phase phase) + (= preparation-id (:preparation-id phase)) + (= (first order) (:phase phase)) + (every? #(= 1 (count (get-in % [:metrics :command/elapsed]))) (:events phase))) + (let [out (conj out phase)] + (if (= :failed (:outcome phase)) out + (recur (next phases) (next order) out))) + out) + out))) + +(defn check-response + "Returns checked bridge evidence or an invalid-output failure. + + Stale responses retain their status and gain an ownership diagnostic. + Malformed responses retain their valid phase prefix and raw response. + + Args: + phase: The preparation or attempt receiving the response. + preparation-id: Expected owner of delivered phase blocks. + response: Decoded bridge envelope. + + Returns: + Execution evidence with a known status and diagnostic fields on failure." + [phase preparation-id response] + (if (sm/validate schema:response response) + (if (= "stale" (:status response)) + (assoc response :phase (name phase) :message "browser context was superseded") + response) + (assoc (errors/fail-data + (errors/evidence :invalid-output + (ex-info "invalid bridge response" + {:type ::invalid-response :partial {:response response}}))) + :phases (partial-phases (:phases response) preparation-id + (if (= :preparation phase) [:preparation] [:main :reset]))))) + +(defn attempt + "Returns an attempt with raw phase histories and available failure evidence. + + Args: + case-id: saved case identifier. + preparation-id: preparation identifier, or nil for an early failure. + warmup?: whether this execution occupies a warmup slot. + evidence: browser response with status, phases and optional failure details. + + Returns: + An attempt ready for chronological recording." + [case-id preparation-id warmup? evidence] + (cond-> {:case-id case-id :preparation-id preparation-id :warmup? warmup? + :outcome (if (= "ok" (:status evidence)) :completed :failed) + :phases (or (:phases evidence) [])} + (not= "ok" (:status evidence)) + (assoc :failure (dissoc (errors/project-diagnostic evidence) :partial)) + (:partial evidence) (assoc :partial (:partial evidence)))) diff --git a/frontend/performance/benches/render_wasm/node/catalog/entry.cljs b/frontend/performance/benches/render_wasm/node/catalog/entry.cljs new file mode 100644 index 0000000000..3d3e61194b --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/catalog/entry.cljs @@ -0,0 +1,59 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.catalog.entry + "Node entry points of a compiled renderer benchmark build. + + The `:bench-render-wasm-catalog` build in `frontend/shadow-cljs.edn` names + this namespace as its compilation entry through `:exports`, which exposes + `collect-cases` to Node as `collectCases` and `build-identity` as + `buildIdentity`. Both functions use Transit-encoded strings at the Node + boundary. + + Requiring this namespace loads the benchmark scene declarations. Each build's + `cases.cjs` artifact has its own scene registry and compiled identity; load + the artifact of the build being queried." + (:require + [app.common.transit :as transit] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.identity :as identity])) + +(defn collect-cases + "Collects the benchmark cases selected from this build. + + Args: + encoded-selection: A Transit-encoded map. The required :master-seed is an + integer from 0 through 4294967295 used to derive the seed of each case. + The optional :filter is a substring; a blank value selects all cases, + otherwise only case ids that contain it are collected. + + Returns: + A Transit-encoded map with :build-id, the identity compiled into this build, + and :cases, a non-empty vector of validated case descriptors. + Each descriptor has the case's :id and :scene, a :scene-description, its + :params and :scene-seed, and the execution settings used by the runner. + + Throws: + When the selection cannot be decoded, :master-seed is missing or not in + 0 through 4294967295, no cases are selected, or a registered scene or case + fails validation. Case collection failures carry a :type in their + exception data." + [encoded-selection] + (codec/encode-str + {:build-id identity/build-id + :cases (cases/collect-cases (transit/decode-str encoded-selection))})) + +(defn build-identity + "Returns the identity compiled into this build. + + Returns: + A Transit-encoded string. Decoding it yields a map with :build-id, the + identifier compiled into this artifact, defaulting to `unprepared`, and + :cljs-profile-marks, a boolean that records whether renderer profiling + marks were enabled at compile time." + [] + (codec/encode-str (identity/data))) diff --git a/frontend/performance/benches/render_wasm/node/catalog/loader.cljs b/frontend/performance/benches/render_wasm/node/catalog/loader.cljs new file mode 100644 index 0000000000..0d30924d96 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/catalog/loader.cljs @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.catalog.loader + "Loads case descriptors from a selected build, without the runner's registry." + (:require + ["module" :as node-module] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.declarations :as declarations] + [benches.render-wasm.identity :as identity])) + +(defn- load-catalog + [build] + (let [cases-path (:cases-path build)] + (if (exists? js/require) + (js/require cases-path) + ((node-module/createRequire cases-path) cases-path)))) + +(defn build-identity + "Reads and validates the selected catalog's compiled build facts." + [build] + (let [loaded (transit/decode-str (.buildIdentity ^js (load-catalog build)))] + (identity/check (:id build) loaded) + loaded)) + +(defn collect + "Requires the selected cases.cjs and checks both identity responses. + + build comes from artifacts/load-build or a newly completed build. The + createRequire branch also supports the Node ESM test entry points." + [build selection] + (let [catalog (load-catalog build) + loaded (transit/decode-str (.buildIdentity ^js catalog))] + (identity/check (:id build) loaded) + (let [response (transit/decode-str + (.collectCases ^js catalog (codec/encode-str selection))) + cases (:cases response)] + (identity/check (:id build) response) + (when-not (and (vector? cases) (seq cases)) + (throw (ex-info "renderer build returned an invalid case catalog" + {:type ::invalid-catalog + :build-id (:id build)}))) + (mapv declarations/check-collected-case cases)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs new file mode 100644 index 0000000000..279bde3100 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs @@ -0,0 +1,55 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.ab + "Options for comparing configurations of Cargo features." + (:require + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.settings :as settings])) + +(def schema:max-features + [:int {:min 1 :max 6}]) + +(def schema:options + [:merge shared/schema:measurement shared/schema:comparison-permissions + [:map {:closed true} + [:features [:vector {:min 1} shared/schema:feature]] + [:max-features {:default feature-plan/default-max-features} schema:max-features] + [:confirm {:default false} :boolean] + [:dry-run {:default false} :boolean] + [:build-timeout-ms {:default settings/default-build-timeout-ms} shared/schema:timeout-ms]]]) + +(def ^:const usage "runner ab --features A,B [options]") + +(def ^:const description + "Compare configurations of Cargo features. + Managed output: frontend/target/performance/render-wasm/feature-comparisons//feature-comparison.transit") + +(def cli-options + (into [[nil "--features FEATURES" "Comma-separated functional Cargo features" + :parse-fn shared/parse-feature-list] + (shared/number-option "--max-features N" "Maximum number of requested features" + schema:max-features "must be an integer from 1 to 6") + [nil "--confirm" "Confirm above eight configurations or with a blank filter"] + [nil "--dry-run" "Validate and print the plan"] + (shared/build-timeout-option "Deadline for metadata and each configuration build")] + (concat shared/measurement-options shared/comparison-options))) + +(defn resolve-options + "Resolves AB arguments and requires a feature selection." + [options arguments] + (shared/check-arguments arguments 0) + (when-not (contains? options :features) + (throw (ex-info "ab requires at least one requested feature (--features)" + {:type ::shared/invalid-args}))) + options) + +(defn check-options + "Checks feature limits and confirmation for resolved AB options." + [options] + (feature-plan/plan (:features options) options) + options) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/build.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/build.cljs new file mode 100644 index 0000000000..ae5b376236 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/build.cljs @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.build + "Options for preparing a benchmark build." + (:require + [benches.render-wasm.features.selection :as selection] + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.settings :as settings])) + +(def schema:options + [:map {:closed true} + [:default-features {:default true} :boolean] + [:features {:default []} [:vector shared/schema:feature]] + [:build-timeout-ms {:default settings/default-build-timeout-ms} shared/schema:timeout-ms]]) + +(def ^:const usage "runner build [options]") + +(def ^:const description "Prepare a renderer build with the selected Cargo features.") + +(def cli-options + [[nil "--[no-]default-features" "Enable Cargo defaults"] + [nil "--features FEATURES" "Comma-separated functional Cargo features" + :parse-fn shared/parse-feature-list] + (shared/build-timeout-option "Complete preparation deadline")]) + +(defn resolve-options + "Resolves build arguments and supplied flags." + [options arguments] + (shared/check-arguments arguments 0) + options) + +(defn check-options + "Checks measurement feature policy and returns the resolved build options." + [resolved] + (selection/feature-selection {:default-features? (:default-features resolved) + :features (:features resolved)}) + resolved) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs new file mode 100644 index 0000000000..eefc8fe0e2 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs @@ -0,0 +1,50 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.compare + "Options for comparing saved benchmark records." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.shared :as shared] + [clojure.string :as str])) + +(def schema:case-pair + [:tuple ::sm/text ::sm/text]) + +(def schema:options + [:merge shared/schema:comparison-permissions + [:map {:closed true} + [:baseline-path ::sm/text] + [:candidate-path ::sm/text] + [:case-pair {:default nil} [:maybe schema:case-pair]]]]) + +(def ^:const usage "runner compare BASELINE CANDIDATE [options]") + +(def ^:const description + "Compare two saved records. Paths resolve from the caller's working directory. +--cases selects one exact baseline,candidate pair by saved case ID or name.") + +(def ^:private valid-case-pair? + (sm/validator schema:case-pair)) + +(defn- parse-case-pair + [text] + (let [pair (str/split text #"," -1)] + (when-not (valid-case-pair? pair) + (throw (ex-info "--cases requires one exact nonblank A,B pair" + {:type ::shared/invalid-args}))) + pair)) + +(def cli-options + (into [[nil "--cases A,B" "Select one baseline,candidate pair" :id :case-pair + :parse-fn parse-case-pair]] + shared/comparison-options)) + +(defn resolve-options + "Resolves two literal paths and supplied comparison flags." + [options arguments] + (shared/check-arguments arguments 2) + (assoc options :baseline-path (first arguments) :candidate-path (second arguments))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs new file mode 100644 index 0000000000..070e8c6c79 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs @@ -0,0 +1,31 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.merge + "Options for merging saved benchmark records." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.shared :as shared])) + +(def schema:options + [:map {:closed true} + [:base-path ::sm/text] + [:additional-path ::sm/text] + [:output-dir {:default nil} [:maybe ::sm/text]]]) + +(def ^:const usage "runner merge BASE ADDITIONAL [options]") + +(def ^:const description + "Merge two saved records. Paths resolve from the caller's working directory.") + +(def cli-options + [shared/output-option]) + +(defn resolve-options + "Resolves two literal paths and the supplied output directory." + [options arguments] + (shared/check-arguments arguments 2) + (assoc options :base-path (first arguments) :additional-path (second arguments))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs new file mode 100644 index 0000000000..b00c7a6105 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs @@ -0,0 +1,32 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.purge + "Options for purging managed benchmark artifacts." + (:require + [benches.render-wasm.node.cli.shared :as shared])) + +(def targets [:builds :runs :comparisons :all]) + +(def schema:options + [:map {:closed true} + [:target (into [:enum] targets)]]) + +(def ^:const usage "runner purge builds|runs|comparisons|all [options]") + +(def ^:const description "Purge files in one explicit category of managed benchmark artifacts.") + +(def cli-options []) + +(defn resolve-options + "Resolves the required artifact category." + [options arguments] + (shared/check-arguments arguments 1) + (let [target (some #(when (= (name %) (first arguments)) %) targets)] + (when-not target + (throw (ex-info "purge requires builds, runs, comparisons or all" + {:type ::shared/invalid-args}))) + (assoc options :target target))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/run.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/run.cljs new file mode 100644 index 0000000000..a6280e1541 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/run.cljs @@ -0,0 +1,69 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.run + "Options for a benchmark run and its selected build source." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.commands.build :as build] + [benches.render-wasm.node.cli.shared :as shared])) + +(def schema:build + [:multi {:dispatch :kind} + [:prepare + [:merge build/schema:options + [:map {:closed true} + [:kind [:= :prepare]]]]] + [:existing + [:map {:closed true} + [:kind [:= :existing]] + [:id shared/schema:build-id]]]]) + +(def schema:options + [:merge shared/schema:measurement + [:map {:closed true} + [:build schema:build] + [:remove {:default false} :boolean]]]) + +(def ^:const usage "runner run [options]") + +(def ^:const description + "Run cases using existing or new builds. +Managed output: frontend/target/performance/render-wasm/runs//run.transit") + +(def cli-options + (into [[nil "--build ID" "Reuse a ready build" :id :build-id + :validate [(sm/validator shared/schema:build-id) "must be a benchmark artifact ID"]]] + (concat build/cli-options shared/measurement-options + [[nil "--remove" "Remove the used build after shutdown"]]))) + +(defn resolve-options + "Selects an existing build or a preparation branch from supplied flags." + [options arguments] + (shared/check-arguments arguments 0) + (let [preparation (select-keys options [:default-features :features :build-timeout-ms])] + (when (and (contains? options :build-id) (seq preparation)) + (throw (ex-info "--build cannot be combined with --features, --[no-]default-features or --build-timeout-ms" + {:type ::shared/invalid-args}))) + (assoc (dissoc options :build-id :default-features :features :build-timeout-ms) + :build (if (contains? options :build-id) + {:kind :existing :id (:build-id options)} + (assoc preparation :kind :prepare))))) + +(defn check-options + "Checks measurement feature policy for the selected build source." + [options] + (when (= :prepare (get-in options [:build :kind])) + (build/check-options (:build options))) + options) + +(defn get-help-defaults + "Returns CLI defaults for run options with a preparation source." + [options] + (assoc (dissoc options :build) + :default-features (get-in options [:build :default-features]) + :features (get-in options [:build :features]) + :build-timeout-ms (get-in options [:build :build-timeout-ms]))) diff --git a/frontend/performance/benches/render_wasm/node/cli/console.cljs b/frontend/performance/benches/render_wasm/node/cli/console.cljs new file mode 100644 index 0000000000..16c9720347 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/console.cljs @@ -0,0 +1,40 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.console + "Console settings carried by resolved benchmark requests." + (:require + [benches.render-wasm.node.cli.shared :as shared])) + +(def schema:console + [:map {:closed true} + [:level {:default :info} [:enum :info :warn :debug]] + [:color {:default :auto} [:enum :auto :always :never]]]) + +(def cli-options + [["-q" "--quiet" "Only warnings, errors and results"] + ["-v" "--verbose" "Show debug details and raw build output"] + [nil "--[no-]color" "Colorize terminal output"]]) + +(defn console-level + "Returns the console level selected by quiet or verbose options." + [opts] + (cond (:quiet opts) :warn (:verbose opts) :debug :else :info)) + +(defn color-mode + "Returns automatic, forced or disabled console coloring." + [opts] + (case (:color opts) true :always false :never :auto)) + +(defn resolve-options + "Resolves supplied console flags into console settings." + [options] + (when (and (:quiet options) (:verbose options)) + (throw (ex-info "--quiet and --verbose are mutually exclusive" + {:type ::shared/invalid-args}))) + (cond-> {} + (or (:quiet options) (:verbose options)) (assoc :level (console-level options)) + (contains? options :color) (assoc :color (color-mode options)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/deferred.cljs b/frontend/performance/benches/render_wasm/node/cli/deferred.cljs new file mode 100644 index 0000000000..3cda492436 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/deferred.cljs @@ -0,0 +1,74 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.deferred + "Deferred execution entry for build, run and feature comparison commands." + (:require + [app.common.transit :as transit] + [benches.render-wasm.errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.feature-comparison :as feature-comparison] + [benches.render-wasm.node.runner.run :as run] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.report.format :as format])) + +(defn- ^:async build-command + [{:keys [get-root signal]} options] + (let [created (atom nil) + result (attempt :catch-fatal + (await (build/prepare + {:root (get-root) + :selection {:default-features? (:default-features options) + :features (:features options)} + :build-timeout-ms (:build-timeout-ms options) + :signal signal :on-created #(reset! created %)}))) + result (if (failure? result) + (outcome/failed result {:build @created}) + (outcome/succeeded {:build result :log-path (:log-path result)}))] + (cond-> result + @created (assoc :report (str "Build " (:id @created) "\nOutput " (:dir @created)))))) + +(defn- ^:async run-command + [{:keys [get-root signal services]} options] + (let [{:keys [home report] :as result} (await (run/run (get-root) options signal services))] + (assoc result :report (str "Run " (:run-id home) "\nOutput " (:dir home) + (when report (str "\nReport\n" report)))))) + +(defn- ^:async ab-command + [{:keys [get-root signal services]} options] + (let [root (get-root) + prepared (await (feature-comparison/prepare root options signal))] + (if (:dry-run options) + (outcome/succeeded {:report (feature-comparison/format-plan prepared)}) + (let [{:keys [home exit-code] :as result} + (await (feature-comparison/execute root prepared signal {:services services})) + interrupted (.-aborted signal) + failures (when (and home (= 1 exit-code) (not interrupted) + (= :failed (get-in result [:comparison :status])) + (empty? (:errors result))) + (format/format-feature-comparison-failures (:comparison result) home))] + (cond-> result + home (assoc :report (str "Comparison " (:comparison-id home) + "\nOutput " (:dir home) + "\nStatus " (cond + interrupted "interrupted" + (zero? exit-code) "completed" + :else "failed") + (when failures (str "\n" failures))))))))) + +(def ^:private handlers + {"build" build-command "run" run-command "ab" ab-command}) + +(defn ^:async execute + "Accepts a Transit string containing a complete request and returns its + numeric exit status after final publication." + [text] + (let [request (attempt :catch-fatal (transit/decode-str text))] + (if (failure? request) + (execution/fail request) + (await (execution/execute request handlers {:get-root artifacts/frontend-root}))))) diff --git a/frontend/performance/benches/render_wasm/node/cli/execution.cljs b/frontend/performance/benches/render_wasm/node/cli/execution.cljs new file mode 100644 index 0000000000..db3d45aa4b --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/execution.cljs @@ -0,0 +1,258 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.execution + "Validate benchmark command requests, run their handlers and print results. + + Both the CLI entry and the separately compiled renderer command entry use + this lifecycle. Callers supply a complete request and the handlers available + at that entry. Handlers own the command's files, processes and cleanup; this + module supplies cancellation, prints final reports and errors, and returns + a numeric status for the entry to use as its exit code. + + `fail` reports parsing, loading or decoding failures that occur before a + request reaches `execute`. Signal helpers also support work outside the + checked command lifecycle. Execution and failure reporting share console + settings; do not overlap calls that require different settings or streams." + (:require + [app.common.schema :as sm] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.registry :as registry] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.report.format :as format])) + +(defn on-signals + "Listen for process interrupts so the caller can stop work and finish cleanup. + + Args: + callback: Function called with the string `SIGINT` or `SIGTERM` each time the + Node process receives that signal while the listeners are installed. + + Returns: + An idempotent function that removes only the listeners installed by this + call. Call it when the work settles. This helper does not abort work, + wait for cleanup or choose an exit status." + [callback] + (let [interrupt (fn [] (callback "SIGINT")) + terminate (fn [] (callback "SIGTERM"))] + (.on js/process "SIGINT" interrupt) + (.on js/process "SIGTERM" terminate) + (fn [] + (.removeListener js/process "SIGINT" interrupt) + (.removeListener js/process "SIGTERM" terminate)))) + +(defn ^:async with-interruption + "Run work with cooperative cancellation and preserve its result on interruption. + + The first SIGINT or SIGTERM aborts the supplied signal. Its reason is an + exception with `:signal` set to that signal's name. Later signals do not + replace the reason or force termination. The work must observe cancellation + and settle after its own cleanup; this wrapper imposes no timeout and cannot + preempt synchronous code. Listeners are removed after the work settles. + + Args: + f: Function called once with an AbortSignal. Return an outcome map, or a + promise of one, containing `:exit-code`, an `:errors` vector and + `:log-path` (nil when there is no log). Additional fields may retain partial + results, file paths or a final `:report` string. + + Returns: + A promise of the outcome. Thrown or rejected failures become outcomes with + exit code 1. After interruption, set the exit code to 130 for SIGINT or 143 + for SIGTERM and retain only errors recognized by `errors/fatal?`. Other + outcome fields are preserved. This function does not print the outcome." + [f] + (let [controller (js/AbortController.) + signal (.-signal controller) + interrupted (atom nil) + remove-signals (on-signals + (fn [signal-name] + (when-not (.-aborted signal) + (reset! interrupted signal-name) + (.abort controller + (ex-info "interrupted" + {:type ::interrupted :signal signal-name})))))] + (try + (let [result (attempt :catch-fatal (await (f signal))) + finished (if (failure? result) (outcome/failed result {}) result)] + (cond-> finished + @interrupted (-> (update :errors #(filterv errors/fatal? %)) + (assoc :exit-code (outcome/interruption-code @interrupted))))) + (finally + (remove-signals))))) + +(defn- configure-console + [console local] + (log/configure (merge (select-keys local [:stdout :stderr]) console))) + +(defn- check-gpu + [command options] + (when (and (:require-gpu options) + (or (= command "run") + (and (= command "ab") (not (:dry-run options)))) + (not= "1" (.-PENPOT_GPU (.-env js/process)))) + (throw (ex-info (str "--require-gpu needs PENPOT_GPU=1. " settings/gpu-guidance) + {:type ::gpu-required})))) + +(defn- log-failures + [finished] + (doseq [[index cause] (map-indexed vector (:errors finished))] + (let [data (ex-data cause) + context (mapv (fn [[key value]] [key (errors/format-value value 1024)]) + (errors/project-context data)) + path (when-let [path (:path data)] + (errors/format-value path 1024)) + validation (some-> (::sm/explain data) sm/simplify) + entries (:errors (errors/evidence (or (:phase data) :execution) cause)) + alias? (and (seq entries) + (= (errors/bounded-text (errors/message-of cause) 1024) (:message (first entries)))) + entry-context (when (and alias? + (not= (select-keys data [:phase :operation :path]) + (select-keys (first entries) [:phase :operation :path]))) + (format/format-error-entry + (select-keys (first entries) [:phase :operation :path]))) + entries (if alias? + (rest entries) + entries)] + (log/error {:message (errors/message-of cause) + :fields (into (mapv (fn [[key value]] + [key (if (string? value) + (errors/terminal-text value 1024) + (errors/format-value value 1024))]) + (filterv (comp some? second) + [[:type (:type data)] [:cause-type (:cause-type data)] + [:path path] + [:role (:role data)] [:operation (:operation data)] + [:phase (:phase data)] [:build (:build-id data)] + [:reason (:reason data)] [:manifest-error (:manifest-error data)] + [:baseline (:baseline data)] [:candidate (:candidate data)] + [:features (:features data)] + [:count (:count data)] [:max-features (:max-features data)] + [:validation validation]])) + (concat context + (map #(vector :difference (errors/format-value (:field %) 1024)) (:mismatches data)) + (when entry-context [[:error-context entry-context]]) + (map #(vector :error (format/format-error-entry %)) (take 8 entries)) + (when (> (count entries) 8) + [[:errors-omitted (- (count entries) 8)]]))) + :tail-log (:log-path data) + :tail? (zero? index)})))) + +(defn- report-fallback + [cause local] + (attempt :catch-fatal + (let [message (errors/message-of cause) + stream (or (:stderr local) (.-stderr js/process))] + (.write ^js stream + (str "Could not report benchmark failure: " + (errors/terminal-text message 1024) "\n"))))) + +(defn- finish + "Print the optional report and errors without retrying failed report writes." + [result local] + (binding [log/*log-path* (:log-path result)] + (let [finished (if-some [report (:report result)] + (outcome/publish result #(log/result report)) + result) + reported (attempt :catch-fatal (log-failures finished))] + (if (failure? reported) + (do + (report-fallback reported local) + (:exit-code (outcome/add-error finished reported))) + (:exit-code finished))))) + +(defn fail + "Report a failure that occurred before checked command execution could begin. + + Use this for argument parsing, module loading or request decoding failures. + Do not report a failure again after `execute` has returned its numeric status. + + Args: + cause: Error or thrown value to report. Error data may supply a log path, + usage text, validation details or comparison differences for diagnosis. + console: Trusted console settings with `:level` (`:info`, `:warn`, `:debug`) + and `:color` (`:auto`, `:always`, `:never`). An empty map uses defaults. + Do not take these settings from a request that failed validation. + local: Optional `:stdout` and `:stderr` writable streams with a Node-style + `write` method. Omitted streams use the process streams. + + Returns: + The number 1 after attempting to write the error to stderr and any log + named by its error data. Reporting failures get one bounded stderr fallback + attempt. The one-argument form uses default console settings and streams." + ([cause] + (fail cause {} {})) + ([cause console local] + (configure-console console local) + (finish (outcome/failed (errors/as-failure cause) {}) local))) + +(defn ^:async execute + "Validate a command request, await its handler and print its final outcome. + + Check the request without adding defaults or coercing values. Invalid + requests use default console settings; valid requests choose their console + settings before handler lookup and command policy checks. Unsupported + commands and policy failures are reported without invoking a handler or + looking up the frontend root. + + For `run` and non-dry-run `ab` commands, `:require-gpu` requires the environment + variable `PENPOT_GPU` to equal the string `1` before invoking the handler. + This checks an environment prerequisite, not actual GPU availability. + + Args: + received: Complete request map with a string `:command`, resolved + `:console` settings and command-specific `:options`. Obtain it from + `cli/parse-resolve` or decode it from the CLI's Transit request. Missing + required fields and unknown fields fail validation. + handlers: Map of supported command-name strings to functions called as + `(handler context options)`. Return an outcome or a promise of one, with + `:exit-code`, an `:errors` vector and `:log-path` (nil when there is no log). + An optional `:report` string supplies all final stdout text. Other + result fields do not generate output. Handlers own progress logging + and must await their work and cleanup before returning. + local: Optional map containing `:stdout` and `:stderr` writable streams, + a zero-argument `:get-root` function returning the frontend directory, + and a `:services` map of handler-specific overrides. The context contains + an AbortSignal at `:signal`, plus `:get-root` and `:services` when supplied. + Root lookup and service calls remain the handler's responsibility. These + functions, streams and services are local, not part of the request. + + Returns: + A promise of a numeric exit status after the handler settles and final + output writes have been attempted. Print `:report` once, when present, + followed by errors on stderr, using the outcome's log path for file output. + No report means no final stdout text. Output writes do not wait for streams + to drain. The two-argument form uses process streams and supplies no root + function or services. + + Validation, policy and handler failures produce status 1. Interruption + follows `with-interruption`: await cleanup, preserve partial result fields + and fatal errors, and return 130 for SIGINT or 143 for SIGTERM. A report or + error-reporting failure changes other statuses to 1, with one bounded + stderr fallback if error reporting fails. Interrupted statuses take + precedence. Failed report writes are not retried." + ([received handlers] + (execute received handlers {})) + ([received handlers local] + (configure-console {} local) + (let [result (attempt :catch-fatal + (let [{:keys [command console options]} (request/check received)] + (configure-console console local) + (let [handler (or (get handlers command) + (throw (ex-info "command is unavailable at this execution entry" + {:type ::unsupported-command :command command}))) + options (registry/check-options command options)] + (check-gpu command options) + (await (with-interruption + (fn [signal] + (handler (assoc (select-keys local [:get-root :services]) + :signal signal) + options))))))) + result (if (failure? result) (outcome/failed result {}) result)] + (finish result local)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/help.cljs b/frontend/performance/benches/render_wasm/node/cli/help.cljs new file mode 100644 index 0000000000..7ab3f06c11 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/help.cljs @@ -0,0 +1,25 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.help + "Options for benchmark CLI help." + (:require + [benches.render-wasm.node.cli.registry :as registry])) + +(def schema:options + [:map {:closed true} + [:topic {:default nil} + [:maybe (into [:enum] registry/command-names)]]]) + +(def ^:const usage "runner COMMAND [options]") + +(def ^:const description + "Renderer benchmarks. Use COMMAND --help for command options. +Numeric options accept JavaScript number syntax, including exponents, radix +prefixes and surrounding whitespace, within each option's integer bounds.") + +(def cli-options + [["-h" "--help" "Print help"]]) diff --git a/frontend/performance/benches/render_wasm/node/cli/main.cljs b/frontend/performance/benches/render_wasm/node/cli/main.cljs new file mode 100644 index 0000000000..8f9a5f0414 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/main.cljs @@ -0,0 +1,60 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.main + "Command-line entry for renderer benchmarks and saved results." + (:require + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.cli.resolve :as cli] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.report.compare :as compare] + [benches.render-wasm.node.report.merge :as merge] + [benches.render-wasm.node.storage.artifacts :as artifacts])) + +(defn- help-command + [_ {:keys [topic]}] + (outcome/succeeded {:report (cli/get-help topic)})) + +(defn- compare-command + [_ {:keys [baseline-path candidate-path] :as options}] + (let [result (compare/execute baseline-path candidate-path options)] + (outcome/succeeded {:comparison (:comparison result) :report (:text result)}))) + +(defn- merge-command + [{:keys [get-root]} {:keys [base-path additional-path] :as options}] + (let [{:keys [run home]} (merge/execute (get-root) base-path additional-path options)] + (outcome/succeeded {:run run :home home :log-path (:log-path home) + :report (str "Run " (:run-id run) "\nOutput " (:dir home))}))) + +(defn- purge-command + [{:keys [get-root]} {:keys [target]}] + (artifacts/purge (get-root) target) + (outcome/succeeded {})) + +(def ^:private handlers + {"help" help-command "compare" compare-command "merge" merge-command "purge" purge-command}) + +(defn- ^:async run-cli + [argv] + (let [request (attempt :catch-fatal (cli/parse-resolve argv))] + (if (failure? request) + (execution/fail request) + (if (contains? handlers (:command request)) + (await (execution/execute request handlers {:get-root artifacts/frontend-root})) + (let [status (attempt :catch-fatal + (let [commands (js/require (path/join js/__dirname "commands.cjs"))] + (await (.execute commands (transit/encode-str request)))))] + (if (failure? status) + (execution/fail status (:console request) {}) + status)))))) + +(defn ^:async -main + "Runs argv and sets the process exit status after publication and cleanup." + [& argv] + (set! (.-exitCode js/process) (await (run-cli argv)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/registry.cljs b/frontend/performance/benches/render_wasm/node/cli/registry.cljs new file mode 100644 index 0000000000..a9541448ec --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/registry.cljs @@ -0,0 +1,68 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.registry + "Ordered command definitions for parsing, request schemas and help." + (:require + [benches.render-wasm.node.cli.commands.ab :as ab] + [benches.render-wasm.node.cli.commands.build :as build] + [benches.render-wasm.node.cli.commands.compare :as compare] + [benches.render-wasm.node.cli.commands.merge :as merge] + [benches.render-wasm.node.cli.commands.purge :as purge] + [benches.render-wasm.node.cli.commands.run :as run])) + +(def commands + [["build" {:usage build/usage + :description build/description + :options build/cli-options + :resolve build/resolve-options + :schema build/schema:options + :check build/check-options}] + ["run" {:usage run/usage + :description run/description + :options run/cli-options + :resolve run/resolve-options + :schema run/schema:options + :help-options {:build {:kind :prepare}} + :help-defaults run/get-help-defaults + :check run/check-options}] + ["ab" {:usage ab/usage + :description ab/description + :options ab/cli-options + :resolve ab/resolve-options + :schema ab/schema:options + :check ab/check-options}] + ["compare" {:usage compare/usage + :description compare/description + :options compare/cli-options + :resolve compare/resolve-options + :schema compare/schema:options}] + ["merge" {:usage merge/usage + :description merge/description + :options merge/cli-options + :resolve merge/resolve-options + :schema merge/schema:options}] + ["purge" {:usage purge/usage + :description purge/description + :options purge/cli-options + :resolve purge/resolve-options + :schema purge/schema:options}]]) + +(def command-names (mapv first commands)) + +(def ^:private definitions (into {} commands)) + +(defn get-definition + "Returns a command definition, or nil for an unknown command." + [command] + (get definitions command)) + +(defn check-options + "Checks command policy and returns the resolved options." + [command options] + (if-let [check (:check (get-definition command))] + (check options) + options)) diff --git a/frontend/performance/benches/render_wasm/node/cli/request.cljs b/frontend/performance/benches/render_wasm/node/cli/request.cljs new file mode 100644 index 0000000000..0aedff2c2c --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/request.cljs @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.request + "Resolved benchmark requests, sender defaults and receiver validation." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.console :as console] + [benches.render-wasm.node.cli.help :as help] + [benches.render-wasm.node.cli.registry :as registry] + [malli.transform :as mt])) + +(defn- command-schema + [command options] + [:map {:closed true} + [:command [:= command]] + [:console {:default {}} console/schema:console] + [:options {:default {}} options]]) + +(def schema:request + (into [:multi {:dispatch :command}] + (map (fn [[command definition]] + [command (command-schema command (:schema definition))])) + (conj registry/commands ["help" {:schema help/schema:options}]))) + +(def ^:private default-decoder + (delay (sm/decoder schema:request (mt/default-value-transformer)))) + +(defn apply-defaults + "Fills contextual defaults for the selected command and build source." + [partial-request] + (@default-decoder partial-request)) + +(def check + "Returns a complete valid request, or throws with schema explain data." + (sm/check-fn schema:request + :hint "invalid renderer benchmark request" + :type ::invalid-request)) diff --git a/frontend/performance/benches/render_wasm/node/cli/resolve.cljs b/frontend/performance/benches/render_wasm/node/cli/resolve.cljs new file mode 100644 index 0000000000..3f7508f84c --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/resolve.cljs @@ -0,0 +1,106 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.resolve + "Command-specific argument resolution and help for renderer benchmarks." + (:require + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.cli.console :as console] + [benches.render-wasm.node.cli.help :as help] + [benches.render-wasm.node.cli.registry :as registry] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.cli.shared :as shared] + [clojure.string :as str] + [clojure.tools.cli :as tools])) + +(defn- get-definition + [topic] + (if (nil? topic) + {:usage help/usage :description help/description :options []} + (registry/get-definition topic))) + +(defn- get-declarations + [definition] + (into help/cli-options (concat console/cli-options (:options definition)))) + +(defn- argument-error + [topic message cause] + (let [usage (or (:usage (get-definition topic)) help/usage)] + (ex-info (str message "\nUsage: " usage) + (assoc (ex-data cause) + :type ::invalid-args :cause-type (:type (ex-data cause)) + :command topic :usage usage) + cause))) + +(defn parse-resolve + "Returns a complete request for command-first argv, or throws an argument error. + A leading pnpm forwarding delimiter is accepted." + [argv] + (let [argv (if (= "--" (first argv)) (rest argv) argv) + first-arg (first argv) + topic (when (and first-arg (not (str/starts-with? first-arg "-"))) first-arg) + args (if topic (rest argv) argv)] + (when (and topic (nil? (registry/get-definition topic))) + (throw (argument-error topic (str "unknown command: " topic) nil))) + (let [definition (get-definition topic)] + (try + (let [{:keys [options arguments errors]} + (tools/parse-opts args (get-declarations definition) :no-defaults true) + _ (when (seq errors) + (throw (ex-info (str/join "\n" errors) {:type ::shared/invalid-args}))) + console (console/resolve-options options) + partial (if (:help options) + (do + (when (and (nil? topic) (seq arguments)) + (throw (ex-info "a command must precede its options" + {:type ::shared/invalid-args}))) + {:command "help" :console console :options {:topic topic}}) + (do + (when-not topic + (throw (ex-info "a command is required" {:type ::shared/invalid-args}))) + {:command topic :console console + :options ((:resolve definition) + (dissoc options :help :quiet :verbose :color) arguments)})) + resolved (request/check (request/apply-defaults partial))] + (update resolved :options #(registry/check-options (:command resolved) %))) + (catch :default cause + (if (contains? #{::shared/invalid-args ::request/invalid-request + ::feature-plan/too-many-features ::feature-plan/confirmation-required} + (:type (ex-data cause))) + (throw (argument-error topic (ex-message cause) cause)) + (throw cause))))))) + +(defn- get-defaults + [topic] + (let [definition (get-definition topic) + partial {:command (or topic "help") :options (or (:help-options definition) {})} + resolved (request/apply-defaults partial) + options ((or (:help-defaults definition) identity) (:options resolved))] + (merge (:console resolved) options))) + +(defn get-help + "Returns root help for nil or help for a known command name." + [topic] + (let [definition (get-definition topic) + defaults (get-defaults topic) + summary (:summary + (tools/parse-opts + [] (get-declarations definition) + :no-defaults true + :summary-fn + (fn [specs] + (tools/summarize + (mapv (fn [{:keys [id] :as spec}] + (if (contains? defaults id) + (assoc spec :default (get defaults id) + :default-desc (pr-str (get defaults id))) + spec)) + specs)))))] + (str "Usage: " (:usage definition) "\n\n" (:description definition) "\n\n" + (when (nil? topic) + (str (str/join "\n" (map (fn [[_ definition]] (str " " (:usage definition))) registry/commands)) + "\n\n")) + "Options:\n" summary))) diff --git a/frontend/performance/benches/render_wasm/node/cli/shared.cljs b/frontend/performance/benches/render_wasm/node/cli/shared.cljs new file mode 100644 index 0000000000..671d9036e9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/shared.cljs @@ -0,0 +1,109 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.shared + "Value constraints and option groups shared by benchmark commands." + (:require + [app.common.schema :as sm] + [benches.render-wasm.features.selection :as selection] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.random :as random] + [clojure.string :as str])) + +(def schema:nonnegative-count + [:int {:min 0 :max js/Number.MAX_SAFE_INTEGER}]) + +(def schema:positive-count + [:int {:min 1 :max js/Number.MAX_SAFE_INTEGER}]) + +(def schema:timeout-ms + "Positive integer milliseconds within Node's timer range." + [:int {:min 1 :max 2147483647}]) + +(def schema:seed + [:fn {:error/message "expected a uint32 seed"} random/seed?]) + +(def schema:build-id + [:fn {:error/message "expected a benchmark artifact ID"} artifacts/valid-id?]) + +(def schema:feature + [:fn {:error/message "expected a Cargo feature name"} selection/feature-name?]) + +(def schema:measurement + [:map {:closed true} + [:seed {:default 42} schema:seed] + [:filter {:default nil} [:maybe :string]] + [:warmups {:default 3} schema:nonnegative-count] + [:repetitions {:default 10} schema:positive-count] + [:timeout-ms {:default 30000} schema:timeout-ms] + [:startup-timeout-ms {:default settings/default-startup-timeout-ms} schema:timeout-ms] + [:output-dir {:default nil} [:maybe ::sm/text]] + [:screenshots {:default false} :boolean] + [:require-gpu {:default false} :boolean]]) + +(def schema:comparison-permissions + [:map {:closed true} + [:allow-build-differences {:default false} :boolean] + [:allow-scene-differences {:default false} :boolean] + [:allow-all-differences {:default false} :boolean]]) + +(defn parse-feature-list + [text] + (:features (selection/feature-selection {:features (mapv str/trim (str/split text #"," -1))}))) + +(defn- parse-number + [text] + (if (str/blank? text) js/NaN (js/Number text))) + +(defn number-option + "Declares a numeric option with the given value constraint. + Values accept JavaScript number syntax, including exponents, radix prefixes + and surrounding whitespace." + [flag description schema message] + [nil flag description + :parse-fn parse-number + :validate [(sm/validator schema) message]]) + +(def output-option + [nil "--output-dir DIR" "Explicit output home; must not exist" + :validate [(sm/validator ::sm/text) "must be a nonblank path"]]) + +(def measurement-options + [(number-option "--seed N" "Master scene seed" schema:seed + "must be a uint32 integer") + [nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"] + (number-option "--warmups N" "Warmup attempts per case" schema:nonnegative-count + "must be a nonnegative JavaScript-safe integer") + (number-option "--repetitions N" "Measured attempts per case" schema:positive-count + "must be a positive JavaScript-safe integer") + (number-option "--timeout-ms MS" "Case deadline including setup, attempts and diagnostics" schema:timeout-ms + "must be a positive integer within Node's timer range") + (number-option "--startup-timeout-ms MS" "Host/browser readiness deadline" schema:timeout-ms + "must be a positive integer within Node's timer range") + output-option + [nil "--screenshots" "Capture one screenshot per case"] + [nil "--require-gpu" "Fail when GPU rendering is unavailable"]]) + +(def comparison-options + [[nil "--allow-build-differences" "Allow recorded build differences"] + [nil "--allow-scene-differences" "Allow scene inputs and fingerprint content differences"] + [nil "--allow-all-differences" "Allow all comparison compatibility differences"]]) + +(defn build-timeout-option + "Declares a preparation deadline with the given description." + [description] + (number-option "--build-timeout-ms MS" description schema:timeout-ms + "must be a positive integer within Node's timer range")) + +(defn check-arguments + "Checks the number of positional arguments." + [arguments expected] + (when-not (= expected (count arguments)) + (throw (ex-info (str "expected " expected " positional " + (if (= expected 1) "argument" "arguments") + ", received " (count arguments)) + {:type ::invalid-args :arguments arguments})))) diff --git a/frontend/performance/benches/render_wasm/node/files.cljs b/frontend/performance/benches/render_wasm/node/files.cljs new file mode 100644 index 0000000000..e902a5ca87 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/files.cljs @@ -0,0 +1,51 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.files + "Filesystem paths and storage for benchmark artifacts." + (:require + ["fs" :as fs] + ["path" :as path] + [benches.render-wasm.errors :as errors :refer-macros [attempt]] + [clojure.string :as str])) + +(defn contained? + "True when `file` is `root` or a path below it." + [root file] + (let [relative (path/relative root file)] + (and (not (path/isAbsolute relative)) + (not= relative "..") + (not (str/starts-with? relative (str ".." path/sep)))))) + +(defn get-real-path + "Resolves symlink ancestors even when the final path does not exist yet." + [file] + (let [file (path/resolve file)] + (if (fs/existsSync file) + (fs/realpathSync file) + (path/join (get-real-path (path/dirname file)) (path/basename file))))) + +(defn atomic-write + "Writes (UTF-8) content to a temporary file and renames it over `file`, + so readers see either the previous complete file or the new one. The parent + directory must exist. + + A failed write or rename leaves the previous file untouched and attempts to + remove the temporary file. Failures retain operation context in order." + [file content] + (let [temporary (str file "." (random-uuid) ".tmp") + written (attempt :catch-fatal + (fs/writeFileSync temporary content #js {:encoding "utf8" :flag "wx"}) + nil) + renamed (when-not written + (attempt :catch-fatal (fs/renameSync temporary file) nil)) + removed (when (or written renamed) + (attempt :catch-fatal (fs/rmSync temporary #js {:force true}) nil))] + (when-let [failure (errors/combine + [{:phase "storage" :operation :write-temporary :path file :error written} + {:phase "storage" :operation :rename-temporary :path file :error renamed} + {:phase "storage" :operation :remove-temporary :path temporary :error removed}])] + (throw failure)))) diff --git a/frontend/performance/benches/render_wasm/node/log.cljs b/frontend/performance/benches/render_wasm/node/log.cljs new file mode 100644 index 0000000000..1db99a5f80 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/log.cljs @@ -0,0 +1,181 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.log + "Console output and scoped file logs for the benchmark CLI. + + Public API: + + (log/configure {:level :info + :color :auto + :stdout stream + :stderr stream}) + (log/result text) + (log/info text) + (log/debug text) + (log/warn text) + (log/error {:message ... + :fields [...] + :tail-log ...}) + + Every output function uses the same private one: + + ... + └── log/write <-- this guy + ├── append text to the bound log file + └── filter and render console output + ├── result → stdout + └── diagnostics → stderr + + The CLI selects the logging threshold for the console: + - Default: :info + - --quiet: :warn + - --verbose: :debug + + But console filtering and file logging are independent: + + | Function | Console destination | Console visibility | File log | + |----------|---------------------|---------------------|----------| + | result | stdout | Always | Always | + | info | stderr | Default and verbose | Always | + | debug | stderr | Verbose | Always | + | warn | stderr | Always | Always | + | error | stderr | Always | Always | + + So: quiet mode still records detailed diagnostics in files. + Also, the console renderer adds headings, prefixes, and colors." + (:require-macros + [app.common.exceptions :as ex]) + (:require + ["fs" :as fs] + [benches.render-wasm.errors :as errors] + [clojure.string :as str])) + +;; Each high-level task binds its own path and awaits its work inside that scope: +;; (binding [log/*log-path* (:log-path home)] +;; (await (execute-task ...))) +;; Returning or throwing from a child restores the parent's destination +(def ^:dynamic *log-path* nil) + +(defonce ^:private config (atom nil)) + +(defn configure + "Configures console level, color mode and output streams." + [options] + (reset! config (merge {:level :info :color :auto + :stdout (.-stdout js/process) + :stderr (.-stderr js/process)} + options))) + +(defn- color? + [stream mode] + (case mode + :always true + :never false + (and (.-isTTY ^js stream) + (empty? (.-NO_COLOR (.-env js/process))) + (not= "dumb" (.-TERM (.-env js/process)))))) + +(defn- paint + [enabled code text] + (if enabled (str "\u001b[" code "m" text "\u001b[0m") text)) + +(defn- render + [level text color] + (str/join "\n" + (map (fn [line] + (case level + :info (if (re-find #"^\S" line) (paint color "1;36" line) line) + :debug (paint color "2" (str "· " line)) + :warn (paint color "33" (str "! " line)) + :error (paint color "31" line) + line)) + (str/split-lines text)))) + +(defn- write + [level text] + (when-not @config (configure {})) + (let [{:keys [stdout stderr color] console-level :level} @config + text (str/replace (str text) #"\n$" "") + failure (when *log-path* + (try + (fs/appendFileSync *log-path* (str text "\n") "utf8") + nil + (catch :default cause + (errors/fatal-error "cannot append benchmark log" + {:type ::storage-error :log-path *log-path*} + cause)))) + enabled (case level + :debug (= :debug console-level) + :info (contains? #{:info :debug} console-level) + true) + stream (if (= :result level) stdout stderr) + display (if (contains? #{:debug :info :warn} level) + (->> (str/split-lines (errors/bounded-text text 1024)) + (map #(errors/terminal-text % 1024)) + (str/join "\n") + (#(errors/bounded-text % 1024))) + text)] + (when enabled + (.write ^js stream (str (if (= :result level) + display + (render level display (color? stream color))) "\n"))) + (when failure + (if (= :error level) + (.write ^js stderr (str "Could not append log: " (errors/terminal-text *log-path* 1024) "\n")) + (throw failure))))) + +(defn result [text] (write :result text)) +(defn info [text] (write :info text)) +(defn debug [text] (write :debug text)) +(defn warn [text] (write :warn text)) + +(defn- get-tail + [path] + (when path + (ex/ignoring + (let [fd (fs/openSync path "r")] + (try + (let [size (.-size (fs/fstatSync fd)) + length (min size 8192) + buffer (js/Buffer.alloc length) + read (fs/readSync fd buffer 0 length (- size length)) + text (->> (str/split-lines (.toString buffer "utf8" 0 read)) + (remove str/blank?) + (take-last 20) + (map #(errors/terminal-text % (* 6 8192))) + (str/join "\n")) + start (max 0 (- (count text) 8192)) + first (.charCodeAt text start) + start (if (and (pos? start) (<= 0xDC00 first 0xDFFF) + (<= 0xD800 (.charCodeAt text (dec start)) 0xDBFF)) + (inc start) + start)] + (str/split-lines (subs text start))) + (finally (fs/closeSync fd))))))) + +(defn error + "Prints a terminal failure with ordered fields and an optional bounded log tail." + [{:keys [message fields tail-log tail?] :or {tail? true}}] + (let [tail-path (or tail-log *log-path*) + fields (cond-> (vec fields) + (and *log-path* (not (some #(= :log (first %)) fields))) + (conj [:log (errors/terminal-text *log-path* 1024)]) + (and tail-log (not= tail-log *log-path*)) + (conj [:tail-log (errors/terminal-text tail-log 1024)])) + width (reduce max 0 (map #(count (name (first %))) fields)) + tail (when tail? (get-tail tail-path))] + (write :error + (str "✖ " (errors/terminal-text message 1024) + (apply str + (map (fn [[key value]] + (let [label (name key)] + (str "\n " label + (apply str (repeat (+ 2 (- width (count label))) " ")) + value))) + fields)) + (when (seq tail) + (str "\n tail\n" (str/join "\n" (map #(str " " %) tail)))))))) diff --git a/frontend/performance/benches/render_wasm/node/outcome.cljs b/frontend/performance/benches/render_wasm/node/outcome.cljs new file mode 100644 index 0000000000..0f630f8668 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/outcome.cljs @@ -0,0 +1,57 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.outcome + "Result of a benchmark command. + + An outcome is a map with `:exit-code`. A command that failed adds `:errors` + (a vector, primary failure first) and `:log-path`. Exit code 130 and 143 + mark an interruption and no failure replaces them." + (:require + [benches.render-wasm.errors :as errors :refer [attempt failure?]])) + +(defn interruption-code + "Returns the exit code for the interrupting signal name: 143 for SIGTERM, + 130 otherwise." + [signal-name] + (case signal-name "SIGTERM" 143 130)) + +(defn succeeded + "Returns a successful outcome with command result data." + [data] + (merge {:exit-code 0 :errors [] :log-path nil} data)) + +(defn failed + "Returns an outcome for `cause` with defaults for errors, exit code and log. + + Args: + cause: Failure supplying the default error and log path. + data: Outcome fields that override defaults, including :errors, + :exit-code and :log-path. + + Returns: + An outcome with exit code 1 and the failure's log path as defaults." + [cause data] + (merge {:errors [cause] + :exit-code 1 + :log-path (:log-path (ex-data cause))} + data)) + +(defn add-error + "Appends `cause` to the errors of `outcome`. An exit code that is not an + interruption becomes 1." + [outcome cause] + (-> outcome + (update :errors (fnil conj []) cause) + (update :exit-code #(if (contains? #{130 143} %) % 1)))) + +(defn publish + "Calls `f` to print a result of `outcome`. A failure of `f` is added to the + errors of the returned outcome." + [outcome f] + (let [published (attempt :catch-fatal (f))] + (cond-> outcome + (failure? published) (add-error published)))) diff --git a/frontend/performance/benches/render_wasm/node/process.cljs b/frontend/performance/benches/render_wasm/node/process.cljs new file mode 100644 index 0000000000..a01af9077e --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/process.cljs @@ -0,0 +1,604 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.process + "Deadline-bound Node commands with owned POSIX process groups." + (:require + ["child_process" :as child-process] + ["fs" :as fs] + ["node:readline" :as readline] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [clojure.string :as str])) + +(def default-termination-grace-ms 5000) +(def ^:private final-termination-wait-ms 5000) + +(def ^:private max-timer-delay-ms + "Largest delay that Node timers accept; longer delays overflow a signed 32-bit integer." + 2147483647) + +(def ^:private default-options + {:termination-grace-ms default-termination-grace-ms + :capture-output? true}) + +(defn signal-name + "Returns the signal name for an aborted AbortSignal, defaulting to SIGINT." + [signal] + (when (and signal (.-aborted ^js signal)) + (let [reason (.-reason ^js signal)] + (cond + (string? reason) reason + (keyword? reason) (name reason) + :else (or (:signal (ex-data reason)) "SIGINT"))))) + +(defn deadline + "Returns an absolute Date.now() timestamp, in milliseconds. All commands + in a preparation can share it, including commands waiting for a lock. + This is a wall clock deadline, so clock changes affect the time left." + [timeout-ms] + (+ (js/Date.now) timeout-ms)) + +(defn remaining + "Returns the nonnegative milliseconds left on a Date.now() deadline." + [deadline] + (max 0 (- deadline (js/Date.now)))) + + +(defn- unresolved-cause? + [cause seen] + (cond + (nil? cause) false + (:unresolved-process (ex-data cause)) true + (contains? seen cause) false + :else (recur (ex-cause cause) (conj seen cause)))) + +(defn unresolved? + "True when a command's process group may still be running. + + Args: + cause: An Error, including collected errors and wrapped causes, or nil. + + Returns: + True if any error reports :unresolved-process true after termination." + [cause] + (boolean + (some (fn [{:keys [error] :as entry}] + (or (:unresolved-process entry) (unresolved-cause? error #{}))) + (errors/get-errors cause)))) + +(defn- signal-group + [pid signal] + (try + (.kill js/process (- pid) signal) + true + (catch :default cause + (if (= "ESRCH" (.-code cause)) + false + (throw cause))))) + +(defn- linux-group-member? + [pid entry] + (when (re-matches #"\d+" entry) + (try + ;; comm may contain spaces or ')'; fields after its final ')' start + ;; with state, ppid and pgrp. Zombies retain a group but no resources. + (let [stat (fs/readFileSync (str "/proc/" entry "/stat") "utf8") + fields (str/split (str/trim (subs stat (inc (.lastIndexOf stat ")")))) #"\s+")] + (and (= pid (js/Number (nth fields 2))) + (not (contains? #{"Z" "X" "x"} (first fields))))) + (catch :default cause + (if (contains? #{"ENOENT" "ESRCH"} (.-code cause)) + false + (throw cause)))))) + +(defn- group-running? + [pid] + (and (signal-group pid 0) + (if (= "linux" (.-platform js/process)) + ;; Container init need not reap orphan zombies promptly. Waiting + ;; for kill(-pgid, 0) to return ESRCH can therefore hang forever. + (some #(linux-group-member? pid %) (array-seq (fs/readdirSync "/proc"))) + true))) + +(defn- pause + [ms] + (js/Promise. (fn [resolve _] (js/setTimeout resolve ms)))) + + +(defn- get-group-observation + "Checks whether a process group has a live member. + + Args: + pid: The group's leader PID. + + Returns: + A map with :status set to :running, :stopped or :unknown, and :error + when the operating system rejects the probe." + [pid] + (let [result (attempt :catch-fatal (group-running? pid))] + (if (failure? result) + {:status :unknown :error result} + {:status (if result :running :stopped)}))) + +(defn- wait-again? + "Decides whether to take another process-group observation. + + Args: + observation: A map with :status :running, :stopped or :unknown. + context: The absolute :deadline and :phase, either :grace before SIGKILL + or :confirmation after SIGKILL. + now: Current time in milliseconds since the Unix epoch. + + Returns: + True while time remains and the group may be running. Grace ends on a + failed probe; confirmation continues through probe failures." + [observation {:keys [phase deadline]} now] + (and (not= :stopped (:status observation)) + (< now deadline) + (or (= :confirmation phase) (= :running (:status observation))))) + +(defn- group-failed + "Returns the failure of an operation on a command group. + + Args: + kind: The failure type. + action: Text that describes the failed operation. The message appends the + bounded text of the cause. + cause: The Error that the operation raised. + data: Diagnostic data of the failure." + [kind action cause data] + {:type kind + :message (str action ": " (errors/bounded-text (errors/message-of cause) 500)) + :cause cause + :data data}) + +(defn- ^:async wait-for-group + [{:keys [pid deadline signals] :as context} record] + (let [observation (get-group-observation pid) + now (js/Date.now)] + (when-let [cause (:error observation)] + (record :probe-group (group-failed :process-probe "Could not inspect command group" cause {:signals signals}))) + (if (wait-again? observation context now) + (do + (await (pause (min 10 (- deadline now)))) + (await (wait-for-group context record))) + observation))) + +(defn- send-group-signal + [pid signal record] + (let [result (attempt :catch-fatal (signal-group pid signal))] + (when (failure? result) + (record :signal-group + (group-failed :process-signal (str "Could not send " signal " to command group") result {:signal signal}))) + result)) + +(defn- ^:async terminate-group + [pid grace-deadline record] + (let [signals ["SIGTERM"] + term (send-group-signal pid "SIGTERM" record) + grace {:pid pid :phase :grace :signals signals + :deadline (if (failure? term) (js/Date.now) grace-deadline)} + observation (await (wait-for-group grace record))] + (if (= :stopped (:status observation)) + {:signals signals :unresolved-process false} + (let [signals (conj signals "SIGKILL")] + (send-group-signal pid "SIGKILL" record) + (let [confirmation (assoc grace :phase :confirmation :signals signals + :deadline (deadline final-termination-wait-ms)) + observation (await (wait-for-group confirmation record)) + unresolved (not= :stopped (:status observation))] + (when unresolved + (record :terminate-group + {:type :process-unresolved + :message "Command group ownership remains unresolved after SIGKILL" + :data {:fatal true :unresolved-process true :signals signals}})) + {:signals signals :unresolved-process unresolved}))))) + +(defn- process-error + [{:keys [command args cwd]} result {:keys [type message cause data]}] + (ex-info message + (merge {:type type :command command :args args :cwd cwd} result data) + cause)) + +(defn- cancelled + [^js signal] + (let [reason (.-reason signal)] + {:type :process-cancelled + :message "Command cancelled" + :cause (when (failure? reason) reason) + :data {:reason (signal-name signal)}})) + +(defn- timed-out + [deadline] + {:type :process-timeout + :message "Command deadline expired" + :data {:deadline deadline}}) + +(defn- spawn-failed + [^js cause] + {:type :process-spawn + :message "Could not spawn command" + :cause cause + :data {:code (.-code cause)}}) + +(defn- stream-done + [^js stream] + (js/Promise. + (fn [resolve _] + (.once stream "end" resolve) + (.once stream "close" resolve) + (.once stream "error" resolve)))) + +(defn- drain-output + "Allow queued pipe data to drain, but never wait indefinitely for an EOF." + [done grace-deadline] + (js/Promise. + (fn [resolve _] + (let [timer (js/setTimeout resolve (max 10 (remaining grace-deadline)))] + (.then done + (fn [_] + (js/clearTimeout timer) + (resolve))))))) + +(def ^:private empty-result {:exit-code nil :signal nil :stdout "" :stderr ""}) +(def ^:private read-failed-message "Could not read command output") +(def ^:private callback-failed-message "Command output callback failed") + +(defn- group-ids + "Returns {:pid pid :pgid pid} for the leader of a detached process group." + [pid] + {:pid pid :pgid pid}) + +(defn- launch-error + [options failure] + (process-error options empty-result failure)) + +(defn- launch-failure + "Returns the failure that forbids starting a command, or nil. + + Args: + options: Command settings with the absolute :deadline and the optional + AbortSignal :signal." + [{:keys [signal deadline]}] + (cond + (and signal (.-aborted ^js signal)) (cancelled signal) + (zero? (remaining deadline)) (timed-out deadline))) + +(defn- output-failed + [stream cause message] + {:type :process-output :message message :cause cause :data {:stream stream}}) + +(defn- completion-failure + "Returns the failure to record for the reason a command completes. + + Args: + reason: The exit status {:exit-code :signal} or a failure map with :type. + + Returns: + The failure, or nil for a zero exit code. Output failures are recorded + when they occur." + [reason] + (let [kind (:type reason)] + (cond + (= :process-output kind) nil + kind reason + (not= 0 (:exit-code reason)) {:type :process-exit :message "Command exited unsuccessfully"}))) + +(defn- spawn-child + "Spawns the command in a new process group. + + Args: + options: Command settings with :command and literal :args, and optional + :cwd and :env. A map or JS object under :env merges over process.env. + + Returns: + The ChildProcess. + + Throws: + A launch error when the operating system rejects the spawn." + [{:keys [command args cwd env] :as options}] + (let [environment (js/Object.assign #js {} (.-env js/process) (if (map? env) (clj->js env) env))] + (try + (child-process/spawn command (clj->js args) + #js {:cwd cwd :env environment :detached true + :stdio #js ["pipe" "pipe" "pipe"]}) + (catch :default cause + (throw (launch-error options (spawn-failed cause))))))) + +(defn- watch-exit + "Observes the exit and the process errors of a child. + + Returns: + A map with :promise, which resolves with the first outcome, and :state, + a volatile holding the latest outcome. An outcome is {:exit-code :signal} + or the failure of a process error." + [^js child] + (let [state (volatile! nil) + observe (fn [resolve outcome] + (vreset! state outcome) + (resolve outcome))] + {:state state + :promise (js/Promise. + (fn [resolve _] + (.once child "exit" (fn [code signal] (observe resolve {:exit-code code :signal signal}))) + (.once child "error" #(observe resolve (spawn-failed %)))))})) + +(defn- destroy-stream + [^js stream] + (when stream (.destroy stream))) + +(defn- failure-recorder + "Returns (record operation failure), which appends a failure occurrence. + + Args: + options: Command settings. Occurrences carry its :command, :args and :cwd. + child: The ChildProcess. + exit-state: Volatile with the latest exit outcome. Occurrences carry the + exit status known when they occur. + failures: Volatile vector that receives the occurrences in order." + [options ^js child exit-state failures] + (fn [operation failure] + (let [context (merge empty-result + (group-ids (.-pid child)) + (select-keys @exit-state [:exit-code :signal]))] + (vswap! failures conj {:phase "process" :operation operation + :error (process-error options context failure)})))) + +(defn- new-completion + "Returns the settlement of the completion of a command. + + Args: + record: Function from `failure-recorder`. + + Returns: + A map with :promise and :settle. `settle` takes a completion reason. The + first reason records its failure, if any, and resolves :promise. Later + reasons do nothing." + [record] + (let [^js completion (js/Promise.withResolvers) + settled (volatile! false)] + {:promise (.-promise completion) + :settle (fn [reason] + (when-not @settled + (vreset! settled true) + (when-let [failure (completion-failure reason)] + (record :run-command failure)) + ((.-resolve completion) reason)))})) + +(defn- watch-deadline + "Calls `settle` with a timeout failure once `deadline` passes. + + Args: + deadline: Absolute Date.now() timestamp in milliseconds. + settle: Function of a completion reason. + + Returns: + A function that cancels the pending check." + [deadline settle] + (let [timer (volatile! nil) + check (fn check [] + (let [ms (remaining deadline)] + (if (zero? ms) + (settle (timed-out deadline)) + (vreset! timer (js/setTimeout check (min ms max-timer-delay-ms))))))] + (check) + #(js/clearTimeout @timer))) + +(defn- watch-abort + "Calls `settle` with a cancellation failure when `signal` aborts, or at once + when it already aborted. + + Args: + signal: An AbortSignal, or nil. + settle: Function of a completion reason. + + Returns: + A function that removes the listener." + [^js signal settle] + (if signal + (let [abort (fn [_] (settle (cancelled signal)))] + (.addEventListener signal "abort" abort #js {:once true}) + (when (.-aborted signal) (abort nil)) + #(.removeEventListener signal "abort" abort)) + (constantly nil))) + +(defn- watch-completion + "Calls `settle` with the first completion reason among the exit outcome, the + deadline and the abort. + + Args: + options: Command settings with :deadline and the optional :signal. + exited: Promise of the exit outcome. + settle: Function of a completion reason. + + Returns: + A function that stops watching the deadline and the signal." + [{:keys [deadline signal]} ^js exited settle] + (.then exited settle) + (let [stop-deadline (watch-deadline deadline settle) + stop-abort (watch-abort signal settle)] + (fn [] + (stop-deadline) + (stop-abort)))) + +(defn- open-line-reader + "Returns a readline interface that passes the complete lines of `input` to + `on-line` with `stream`. Callback failures and stream errors go to `fail`. + Stream errors keep going to `fail` after the reader closes." + [^js input stream on-line fail] + (let [reader (readline/createInterface #js {:input input :terminal false :crlfDelay js/Infinity}) + unreadable #(fail stream % read-failed-message)] + (.on reader "line" + (fn [line] + (try + (on-line line stream) + (catch :default cause + (fail stream cause callback-failed-message))))) + (.on reader "error" unreadable) + (.once reader "close" #(.on input "error" unreadable)) + reader)) + +(defn- observe-output + "Decodes stdout and stderr of a child as UTF-8 and routes their text. + + Appends each chunk to `captured` when :capture-output? is set, and calls + :on-output with each chunk and :on-line with each complete line. + + Args: + options: Command settings with :capture-output?, :on-output and :on-line. + child: The ChildProcess. + captured: Volatile map from :stdout and :stderr to vectors of text. + fail: Function of a stream, a cause and a message that receives the + failures of streams and callbacks. + + Returns: + A vector with the line readers it opened." + [{:keys [on-output on-line capture-output?]} ^js child captured fail] + (let [route (fn [stream text] + (when capture-output? (vswap! captured update stream conj text)) + (when on-output + (try + (on-output text stream) + (catch :default cause + (fail stream cause callback-failed-message))))) + listen (fn [^js input stream] + (.setEncoding input "utf8") + (.on input "data" #(route stream %)) + (when-not on-line + (.on input "error" #(fail stream % read-failed-message))))] + (listen (.-stdout child) :stdout) + (listen (.-stderr child) :stderr) + (if on-line + [(open-line-reader (.-stdout child) :stdout on-line fail) + (open-line-reader (.-stderr child) :stderr on-line fail)] + []))) + +(defn- close-stdin + "Closes the stdin of a child. Write errors go to `fail`, except EPIPE, which + a command that exits without reading stdin causes." + [^js child fail] + (.on (.-stdin child) "error" + (fn [cause] + (when-not (= "EPIPE" (.-code cause)) + (fail :stdin cause "Could not close command stdin")))) + (.end (.-stdin child))) + +(defn- command-result + "Returns the exit status and the captured text of a finished command." + [status captured] + {:exit-code (:exit-code status) + :signal (:signal status) + :stdout (str/join "" (:stdout captured)) + :stderr (str/join "" (:stderr captured))}) + +(defn- ^:async terminate-and-drain + "Terminates the process group of a child and drains its queued output. + + Args: + options: Command settings with :termination-grace-ms. + child: The ChildProcess. + exit: Map from `watch-exit`. + output-done: Promise that resolves when stdout and stderr end. + record: Function from `failure-recorder`. + + Returns: + A promise of {:status exit-status :termination termination}. The status + comes from the exit of the child. When the group stays unresolved, the + status is the one already known, and the child no longer keeps this + process alive." + [{:keys [termination-grace-ms]} ^js child {exit-state :state exited :promise} output-done record] + (let [grace-deadline (deadline termination-grace-ms) + termination (await (terminate-group (.-pid child) grace-deadline record)) + unresolved? (:unresolved-process termination)] + (when unresolved? (.unref child)) + (let [status (if unresolved? @exit-state (await exited))] + (await (drain-output output-done grace-deadline)) + {:status status :termination termination}))) + +(defn- ^:async supervise + "Watches a spawned command until it completes, then terminates its group. + + Args: + options: The options of `run` with defaults applied. + child: The spawned ChildProcess. + exit: Map from `watch-exit`. + + Returns: + A promise of {:exit-code 0 :stdout string :stderr string}. + + Throws: + The Error that combines the recorded failures with the result, the + termination and the process identity." + [options ^js child exit] + (let [pid (.-pid child) + failures (volatile! []) + captured (volatile! {:stdout [] :stderr []}) + record (failure-recorder options child (:state exit) failures) + {completed :promise settle :settle} (new-completion record) + fail (fn [stream cause message] + (let [failure (output-failed stream cause message)] + (record :read-output failure) + (settle failure))) + output-done (js/Promise.all #js [(stream-done (.-stdout child)) + (stream-done (.-stderr child))]) + readers (observe-output options child captured fail) + stop-waiting (watch-completion options (:promise exit) settle)] + (try + (close-stdin child fail) + (await completed) + (stop-waiting) + (let [{:keys [status termination]} (await (terminate-and-drain options child exit output-done record)) + result (command-result status @captured) + failure (errors/combine @failures)] + (if failure + (throw (errors/with-context failure (errors/message-of failure) + (merge result termination (group-ids pid)))) + (dissoc result :signal))) + (finally + (run! #(.close ^js %) readers) + (stop-waiting))))) + +(defn ^:async run + "Runs a command and owns its detached POSIX process group. + + An expired deadline or aborted signal starts no process. Output callbacks + receive UTF-8 chunks or complete lines, including a final unterminated + line. Completion follows the exit of the root process, not the close of its + output pipes, because descendants can keep the pipes open. Cleanup sends + SIGTERM and waits the termination grace. If the group still runs, it sends + SIGKILL once and allows five more seconds for the group to stop. Unresolved + ownership produces a fatal failure. + + Args: + options: Command settings with :command, literal :args, optional :cwd and + :env, absolute :deadline and optional AbortSignal under :signal. + :termination-grace-ms defaults to 5000. :on-output receives text and + :stdout/:stderr; :on-line receives a line and its stream. Output capture + defaults to true and accepts :capture-output? false for streamed output. + + Returns: + A promise resolving to {:exit-code 0 :stdout string :stderr string}. + + Throws: + Failures reject with command, output and process identity. Collected + occurrences retain execution, output, signalling and probe failures in + order. :process-unresolved marks unresolved group ownership as fatal. + Cancellation records the signal name as :reason and retains an Error + supplied as the abort reason as a cause." + [options] + (let [options (merge default-options options)] + (when-let [failure (launch-failure options)] + (throw (launch-error options failure))) + (let [^js child (spawn-child options) + exit (watch-exit child)] + (try + (if (.-pid child) + (await (supervise options child exit)) + (throw (launch-error options (await (:promise exit))))) + (finally + (destroy-stream (.-stdin child)) + (destroy-stream (.-stdout child)) + (destroy-stream (.-stderr child))))))) diff --git a/frontend/performance/benches/render_wasm/node/report/compare.cljs b/frontend/performance/benches/render_wasm/node/report/compare.cljs new file mode 100644 index 0000000000..98261f82e4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/report/compare.cljs @@ -0,0 +1,46 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.report.compare + "Offline comparison of two saved schema:run records. + + Inputs are user file paths resolved from the caller's cwd." + (:require + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.result :as result])) + +(defn execute + "Reads and compares two saved run files. + + Runs are only compared if the build options and scene configuration match. + These can be overriden via flags. + + It is also possible to restrict comparison to specific scene ids. + + Options: + - `:case-pair` contains the baseline and candidate selectors, or nil. + - The three `:allow-*-differences` options select build, scene or all + compatibility permissions. + + Returns the validated `comparison` with its selected permissions, retained + differences and formatted text." + [baseline-path candidate-path {:keys [case-pair allow-build-differences + allow-scene-differences allow-all-differences]}] + (let [baseline (runs/get-record baseline-path {:role :baseline}) + candidate (runs/get-record candidate-path {:role :candidate}) + comparison (result/check-record + (compare/compare-runs + baseline candidate + (cond-> {:allow-build-differences (boolean allow-build-differences) + :allow-scene-differences (boolean allow-scene-differences) + :allow-all-differences (boolean allow-all-differences)} + case-pair + (assoc :pairs (compare/selector-pair baseline candidate + case-pair)))))] + {:comparison comparison + :text (format/format-comparison comparison)})) diff --git a/frontend/performance/benches/render_wasm/node/report/merge.cljs b/frontend/performance/benches/render_wasm/node/report/merge.cljs new file mode 100644 index 0000000000..d719265544 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/report/merge.cljs @@ -0,0 +1,28 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.report.merge + "Offline merge of two saved schema:run records." + (:require + ["fs" :as fs] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.result :as result])) + +(defn execute + "Reads and merges two saved run files from the caller's cwd, then writes a + new run home. Inputs remain unchanged. Returns the merged record and home." + [root base-path additional-path {:keys [output-dir]}] + (let [base (runs/get-record base-path {:role :base}) + additional (runs/get-record additional-path {:role :additional}) + run-id (str (random-uuid)) + merged (result/merge-runs base additional {:run-id run-id}) + home (runs/create-home {:root root + :run-id run-id + :output-dir output-dir})] + (runs/write-record! home merged) + (fs/appendFileSync (:log-path home) + (str "merge " (:run-id base) " + " (:run-id additional) "\n")) + {:run merged :home home})) diff --git a/frontend/performance/benches/render_wasm/node/runner/browser.cljs b/frontend/performance/benches/render_wasm/node/runner/browser.cljs new file mode 100644 index 0000000000..6d1a237745 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/browser.cljs @@ -0,0 +1,385 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.browser + "Owns Chromium through process/run from spawn, then attaches Playwright by CDP. + Shutdown stops the owned process group before disconnecting the client." + (:require-macros + [app.common.exceptions :as ex]) + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + ["playwright" :as playwright] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors :refer [attempt]] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.settings :as settings])) + +(defn gpu-active? + "Returns whether the development container enables GPU access." + [] + (= "1" (.-PENPOT_GPU (.-env js/process)))) + +(defn launch-settings + "Returns the resolved Chromium settings for this environment." + [] + {:headless true + :transport :cdp + :args (cond-> ["--headless" + "--no-sandbox" + "--disable-dev-shm-usage" + "--enable-unsafe-swiftshader" + "--no-first-run" + "--no-default-browser-check" + "--disable-background-networking" + "--disable-background-timer-throttling" + "--disable-renderer-backgrounding" + "--hide-scrollbars" + "--mute-audio"] + (gpu-active?) (into ["--enable-gpu" + "--use-gl=angle" + "--use-angle=vulkan"]))}) + +(defn within + "Calls work while its absolute deadline and optional AbortSignal allow it. + Clears the timer/listener on every outcome and handles late rejections. + The caller owns shutdown of the browser running the work." + [work {:keys [deadline signal phase]}] + (js/Promise. + (fn [resolve reject] + (let [settled (atom false) + timer (atom nil)] + (letfn [(cleanup [] + (js/clearTimeout @timer) + (when signal + (.removeEventListener ^js signal "abort" aborted))) + (succeeded [value] + (when (compare-and-set! settled false true) + (cleanup) + (resolve value))) + (failed [cause] + (when (compare-and-set! settled false true) + (cleanup) + (reject cause))) + (aborted [] + (failed (or (.-reason ^js signal) + (ex-info "browser work aborted" {:type ::aborted :phase phase})))) + (expired [] + (failed (ex-info "browser work timed out" {:type ::timeout :phase phase})))] + (when signal + (.addEventListener ^js signal "abort" aborted #js {:once true})) + (cond + (and signal (.-aborted ^js signal)) + (aborted) + + (zero? (process/remaining deadline)) + (expired) + + :else + (do + (reset! timer (js/setTimeout expired (process/remaining deadline))) + (try + (.then (js/Promise.resolve (work)) succeeded failed) + (catch :default cause + (failed cause)))))))))) + +(defn- deadline-failure + [cause deadline] + (if (contains? (ex-data cause) ::deadline-exceeded) + cause + (errors/with-context cause (errors/message-of cause) + {::deadline-exceeded (or (= ::timeout (:type (ex-data cause))) + (zero? (process/remaining deadline)))}))) + +(defn evaluate + "Calls a fixed bridge method, optionally with a Transit string. + JSON quoting embeds the argument; descriptors and results remain Transit." + ([^js page method] + (.evaluate page (str "window.__benchBridge." method "()"))) + ([^js page method text] + (.evaluate page (str "window.__benchBridge." method "(" (.stringify js/JSON text) ")")))) + +(defn- endpoint-waiter + "Keeps the last 64 KiB of startup stderr until a complete endpoint line arrives." + [] + (let [callbacks (atom nil) + settled (atom false) + output (atom "") + address (atom nil) + promise (js/Promise. (fn [resolve reject] + (reset! callbacks {:resolve resolve :reject reject})))] + ;; Cancellation may precede the first within call. Keep the endpoint's + ;; rejection handled even when that call never subscribes to it. + (.catch promise (fn [_] nil)) + {:promise promise + :address address + :on-output (fn [text stream] + (when (and (= :stderr stream) (not @settled)) + (let [buffer (str @output text)] + (if-let [[_ endpoint] (re-find #"DevTools listening on (ws://[^\s]+)\r?\n" buffer)] + (do + (reset! settled true) + (reset! address endpoint) + (reset! output "") + ((:resolve @callbacks) endpoint)) + (reset! output (subs buffer (max 0 (- (count buffer) 65536)))))))) + :failed (fn [cause] + (when (compare-and-set! settled false true) + ((:reject @callbacks) + (ex-info "Chromium exited before its CDP endpoint became ready" + (merge (errors/project-context (ex-data cause)) + {:type ::endpoint-exit :stderr @output}) + cause))))})) + +(defn- ^:async stop-process + [{:keys [controller process]}] + (.abort controller) + (try + (await process) + nil + (catch :default cause + (let [entries (remove (fn [{:keys [error]}] + (and (= :process-cancelled (:type (ex-data error))) + (not (errors/fatal? error)))) + (errors/get-errors cause))] + (when-let [failure (errors/combine entries)] + (throw (errors/with-data failure (errors/project-context (ex-data cause))))))))) + +(defn- disconnect + "Disconnects only after the group has stopped. A late CDP attachment is also + closed by this continuation; the caller waits at most one termination grace." + [{:keys [connection]}] + (within (fn [] + (if-let [pending @connection] + (.then pending + (fn [^js browser] + (when (.isConnected browser) + (.close browser))) + (fn [_] nil)) + nil)) + {:deadline (process/deadline process/default-termination-grace-ms) + :phase :browser-disconnect})) + +(defn ^:async close-page + "Closes an owned page and its context within one termination grace. + Repeated or already-closed pages resolve without work." + [{:keys [page context]}] + (when (or page context) + (let [deadline (process/deadline process/default-termination-grace-ms) + page-failure + (attempt :catch-fatal + (when (and page (not (.isClosed ^js page))) + (await (within #(.close ^js page) + {:deadline deadline :phase :browser-page-close}))) + nil) + context-failure + (attempt :catch-fatal + (when context + (await (within #(.close ^js context) + {:deadline deadline :phase :browser-context-close}))) + nil)] + (when-let [cause (errors/combine + [{:phase "cleanup" :operation :close-page :error page-failure} + {:phase "cleanup" :operation :close-context :error context-failure}])] + (throw cause))))) + +(defn- ^:async shutdown-owner + "Disposes the owned process group, CDP connection and profile. + Successful process shutdown releases page/context resources. Other shutdown + failures retain the available page/context failure evidence." + [{:keys [user-data-dir] :as owner}] + (let [page-failure (attempt :catch-fatal (await (close-page owner)) nil) + process-failure (attempt :catch-fatal (await (stop-process owner)) nil) + disconnect-failure (attempt :catch-fatal (await (disconnect owner)) nil) + profile-failure (when-not (process/unresolved? process-failure) + (attempt :catch-fatal + (fs/rmSync user-data-dir #js {:recursive true :force true}) + nil))] + (when (or process-failure disconnect-failure profile-failure) + (throw (errors/with-data + (errors/combine + [{:phase "cleanup" :operation :close-page :error page-failure} + {:phase "cleanup" :operation :stop-process :profile-path user-data-dir :error process-failure} + {:phase "cleanup" :operation :disconnect-browser :error disconnect-failure} + {:phase "cleanup" :operation :remove-profile :path user-data-dir :error profile-failure}]) + {:profile-path user-data-dir}))))) + +(defn close + "Returns one shutdown Promise for this owner. Aborting the internal process + lifecycle invokes process/run's SIGTERM/5s/SIGKILL group cleanup, even during + startup or blocked page JS. A settled process exit or cancellation releases + the owned process resources." + [{:keys [shutdown signal abort-listener] :as owner}] + (or @shutdown + (do + (when signal + (.removeEventListener ^js signal "abort" @abort-listener)) + (let [closing (shutdown-owner owner)] + (reset! shutdown closing) + closing)))) + +(defn ^:async launch + "Spawns a Chromium instance and attaches via Chrome DevTools Protocol (CDP). + + Options include an optional signal, startup-timeout-ms (30000 by default) + and an optional absolute deadline shared with server startup. The result + owns the process group, user-data directory and CDP connection. It holds no + page or context; use new-page for those. + + Startup failures include available process details, endpoint and profile." + [{:keys [signal startup-timeout-ms deadline] + :or {startup-timeout-ms settings/default-startup-timeout-ms}}] + (let [deadline (or deadline (process/deadline startup-timeout-ms)) + budget (process/remaining deadline)] + (when (and signal (.-aborted ^js signal)) + (throw (or (.-reason ^js signal) (ex-info "browser startup aborted" {:type ::aborted})))) + (when (zero? budget) + (throw (ex-info "browser startup timed out" {:type ::timeout :phase :browser-startup}))) + (let [^js chromium (.-chromium playwright) + command (.executablePath chromium) + user-data-dir (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-browser-")) + controller (js/AbortController.) + endpoint (endpoint-waiter) + running (process/run {:command command + :args (into (:args (launch-settings)) + ["--remote-debugging-port=0" + "--remote-debugging-address=127.0.0.1" + (str "--user-data-dir=" user-data-dir) + "about:blank"]) + :deadline js/Number.MAX_SAFE_INTEGER + :signal (.-signal controller) + :capture-output? false + :on-output (:on-output endpoint)}) + owner {:controller controller :process running :shutdown (atom nil) + :connection (atom nil) :user-data-dir user-data-dir + :signal signal :abort-listener (atom nil)} + bounds {:deadline deadline :signal signal :phase :browser-startup} + abort (fn [] + ;; Startup/caller cleanup also awaits this same promise. + (.catch (close owner) (fn [_] nil)))] + (.then running + (fn [result] + ((:failed endpoint) (ex-info "Chromium exited during startup" + {:type ::process-exit :command command + :exit-code (:exit-code result)}))) + (:failed endpoint)) + (reset! (:abort-listener owner) abort) + (when signal + (.addEventListener ^js signal "abort" abort #js {:once true})) + (try + (let [address (await (within (fn [] (:promise endpoint)) bounds)) + ^js browser (await (within (fn [] + (let [connecting (.connectOverCDP chromium address + #js {:timeout (max 1 (process/remaining deadline))})] + (reset! (:connection owner) connecting) + connecting)) + bounds))] + (assoc owner :browser browser :version (.version browser) :endpoint address)) + (catch :default cause + (let [observed cause + cause (deadline-failure cause deadline) + cleanup-cause (attempt :catch-fatal (await (close owner)) nil) + cleanup (when (errors/failure? cleanup-cause) + (let [reported (loop [current observed seen #{}] + (if (or (nil? current) (contains? seen current)) + (into seen (map :error (errors/get-errors observed))) + (recur (ex-cause current) (conj seen current))))] + (errors/combine + (remove #(contains? reported (:error %)) + (errors/get-errors cleanup-cause)))))] + (if cleanup + (throw (errors/with-context + (errors/combine [{:phase "browser-startup" :operation :start-browser :error cause} + {:phase "cleanup" :operation :close-browser :error cleanup}]) + "browser shutdown failed after startup error" + (merge (errors/project-context (ex-data cause)) + (errors/project-context (ex-data cleanup-cause)) + {:type ::startup-cleanup + :endpoint @(:address endpoint) :user-data-dir user-data-dir + :startup-failure (errors/describe-cause cause)}))) + (throw (ex-info (errors/message-of cause) + (assoc (ex-data cause) :endpoint @(:address endpoint) :user-data-dir user-data-dir) + cause))))))))) + +(defn ^:async new-page + "Creates a context and page in a browser process we own, navigates to host, + waits for the bridge and checks its compiled build identity. + + Returns {:page :context}. + + Failures close the started page/context before rejecting." + [owner host {:keys [viewport signal startup-timeout-ms deadline] + :or {startup-timeout-ms settings/default-startup-timeout-ms}}] + (let [deadline (or deadline (process/deadline startup-timeout-ms)) + bounds {:deadline deadline :signal signal :phase :browser-page-startup} + ^js browser (:browser owner) + ^js context (await (within #(.newContext browser + #js {:viewport #js {:width (:width viewport) + :height (:height viewport)} + :deviceScaleFactor (:dpr viewport)}) + bounds))] + (try + (let [page (await (within #(.newPage context) bounds)) + page-own {:page page :context context}] + (try + (await (within #(.goto ^js page (str (:base-url host) "/") + #js {:timeout (max 1 (process/remaining deadline))}) + bounds)) + (await (within #(.waitForFunction ^js page "window.__benchReady === true" nil + #js {:timeout (max 1 (process/remaining deadline))}) + bounds)) + (identity/check (get-in host [:build :id]) + (transit/decode-str (await (within #(evaluate page "buildIdentity") bounds)))) + page-own + (catch :default cause + (let [cause (deadline-failure cause deadline)] + (ex/ignoring (await (close-page page-own))) + (throw cause))))) + (catch :default cause + (let [cause (deadline-failure cause deadline)] + (ex/ignoring (await (close-page {:context context}))) + (throw cause)))))) + +(defn connected? + "True while the owned CDP connection is live." + [{:keys [browser]}] + (boolean (and browser (.isConnected ^js browser)))) + +(defn prepare-scene + "Calls prepareScene and decodes its Transit response under bounds." + [page request bounds] + (.then (within #(evaluate page "prepareScene" request) bounds) + transit/decode-str)) + +(defn run-prepared + "Calls runPrepared and decodes its Transit response under bounds." + [page request bounds] + (.then (within #(evaluate page "runPrepared" request) bounds) + transit/decode-str)) + +(defn screenshot + "Writes one PNG under bounds. The timeout belongs to Playwright's call." + [page file {:keys [deadline] :as bounds}] + (.then (within #(.screenshot ^js page + #js {:path file + :timeout (max 1 (process/remaining deadline))}) + bounds) + (fn [_] nil))) + +(def service + "Interface to the browser used by the serial runner. + + Tests can replace this map with mocks." + {:launch launch + :new-page new-page + :close-page close-page + :close close + :connected? connected? + :prepare prepare-scene + :run run-prepared + :screenshot screenshot}) diff --git a/frontend/performance/benches/render_wasm/node/runner/build.cljs b/frontend/performance/benches/render_wasm/node/runner/build.cljs new file mode 100644 index 0000000000..ca7697354e --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/build.cljs @@ -0,0 +1,216 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.build + "These functions build the renderer and package it with the code that runs + benchmarks in Node and the browser. + + Each build gets its own directory under + frontend/target/performance/render-wasm/builds/. It contains the renderer + compiled to WebAssembly, a list of benchmarks that Node can load, and the + page and JavaScript that run them in a browser. + + Run `prepare` from Node in a checkout with the project's build tools and + dependencies installed. Stop other builds and any processes that rebuild the + renderer on file changes until it finishes." + (:require + ["fs" :as fs] + ["path" :as path] + ["url" :as url] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.features.selection :as selection] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [clojure.string :as str])) + +(defn ^:async git-provenance + "Read the Git commit ID and whether the checkout containing `root` has local + changes, including staged changes and untracked files. + + `bounds` supplies :deadline, an absolute time in milliseconds since the Unix + epoch, and an optional :signal AbortSignal to cancel the work. + + Return a promise of a map containing :sha, the commit ID, and :dirty, which + is true when the checkout has local changes. If Git information is + unavailable, log a warning and return nil for both values. Fatal process + failures, timeouts and cancellation reject the promise." + [root bounds] + (try + (let [options (merge bounds {:command "git" :cwd root}) + sha (await (process/run (assoc options :args ["rev-parse" "HEAD"]))) + status (await (process/run (assoc options :args ["status" "--porcelain"])))] + {:sha (str/trim (:stdout sha)) :dirty (not (str/blank? (:stdout status)))}) + (catch :default cause + (if (or (errors/fatal? cause) + (contains? #{:process-timeout :process-cancelled} (:type (ex-data cause)))) + (throw cause) + (do + (log/warn (str "Git provenance unavailable: " (errors/message-of cause))) + {:sha nil :dirty nil}))))) + +(defn- check-profile-marks + "Require :cljs-profile-marks to be false in the map describing the compiled + code. This flag indicates whether extra calls record timings for profiling; + those calls can affect benchmark measurements. Return the map or throw." + [compiled] + (when-not (false? (:cljs-profile-marks compiled)) + (throw (ex-info "compiled benchmark enables profiling marks or omits its setting" + {:type ::profile-marks :compiled compiled}))) + compiled) + +(defn- ^:async validate-browser + "Checks that the JavaScript compiled for the browser belongs to the requested + build and has profiling disabled. + + build: supplies the expected :id and :browser-dir, the directory containing + browser.js. + root: the frontend directory. + bounds: supplies :deadline, an absolute time in milliseconds since the Unix + epoch, and an optional :signal AbortSignal for cancellation. + + Returns a promise of the code's build information, including :build-id and + :cljs-profile-marks. Rejects the promise if loading or checking the code fails." + [build root bounds] + (let [module-url (.-href (url/pathToFileURL (path/join (:browser-dir build) "browser.js"))) + script "const {default: bridge} = await import(process.env.RENDER_BENCH_MODULE_URL); process.stdout.write(bridge.buildIdentity());" + result (await (process/run (merge bounds + {:command (.-execPath js/process) :cwd root + :args ["--input-type=module" "-e" script] + :env {"RENDER_BENCH_MODULE_URL" module-url}})))] + (check-profile-marks (identity/check (:id build) (transit/decode-str (:stdout result)))))) + +(defn ^:async prepare + "Build the renderer and the programs that run benchmarks, and save the files + for later benchmark runs. + + Pass an options map with :root set to the frontend directory. If omitted, + the function locates that directory from the Node program's location. + + Cargo features choose which optional parts of the renderer to compile. Use + :selection {:default-features? true :features []} to enable Cargo's defaults. + Put additional feature names from render-wasm/Cargo.toml in the vector, and + set :default-features? to false to disable the declared defaults. The + :features option accepts the same map when :selection is absent. Omit both + options to use Cargo's defaults. + + :build-timeout-ms sets the time allowed for building, in milliseconds + (300000 by default). An optional `:signal AbortSignal` cancels the work. + An optional :on-created function receives a map with the build's ID and + paths after its directory, log and initial metadata file exist. This callback + runs before compilation. Await the promise from `prepare` before using the + compiled files. + + The promise resolves to a map with :id, the build's unique identifier; + :dir, its output directory; :log-path, its compiler log; and :manifest, + metadata describing the build and its settings. :cases-path points to the + JavaScript module that lists the benchmarks for Node. :browser-dir and + :renderer-dir contain the compiled programs, and :index-path points to the + HTML page. + + A failure rejects the promise and keeps any files already written for + inspection. Error data includes :build-id and :log-path when available." + [{:keys [root features selection build-timeout-ms signal on-created] + :or {build-timeout-ms settings/default-build-timeout-ms}}] + (when-not (and (js/Number.isSafeInteger build-timeout-ms) (pos? build-timeout-ms)) + (throw (ex-info "build timeout must be a positive integer" {:type ::invalid-timeout}))) + (let [root (or root (artifacts/frontend-root)) + selection (selection/feature-selection (or selection features)) + deadline (process/deadline build-timeout-ms) + bounds {:deadline deadline :signal signal} + initial (artifacts/create-build root selection {:git {:sha nil :dirty nil} + :feature-selection selection}) + build-ref (atom initial) + phase (atom :storage)] + (binding [log/*log-path* (:log-path initial)] + (try + (when on-created (on-created initial)) + (reset! phase :provenance) + (let [git (await (git-provenance root bounds))] + (swap! build-ref #(artifacts/write-manifest % (assoc-in (:manifest %) [:provenance :git] git)))) + (reset! phase :renderer) + (log/info "Renderer compiling") + (let [build @build-ref + facts-path (path/join (:renderer-dir build) "cargo-build.json") + args (cond-> ["frontend"] + (not (:default-features? selection)) (conj "--no-default-features") + (seq (:features selection)) (into ["--features" (str/join "," (:features selection))]))] + (await (process/run (merge bounds + {:command (path/resolve root ".." "render-wasm" "build") + :args args :cwd (path/resolve root ".." "render-wasm") + :capture-output? false :on-line (fn [line _] (log/debug line)) + :env {"PENPOT_WASM_PREPARED" "1" "NODE_ENV" "production" + "BUILD_MODE" "release" + "VERSION_TAG" (:id build) + "FRONTEND_DEST" (:renderer-dir build) + "FRONTEND_SHARED_DEST" (path/join root "src" "app" "render_wasm" "api" "shared.js") + "RENDER_WASM_SHARED_ARTIFACT" (path/join (:renderer-dir build) "shared.js") + "RENDER_WASM_BUILD_INFO" facts-path}}))) + (let [facts (js/JSON.parse (fs/readFileSync facts-path "utf8")) + profile (js->clj (.-profile facts) :keywordize-keys true) + resolved (:features (selection/feature-selection {:features (js->clj (.-features facts)) + :default-features? (:default-features? selection)}))] + (when-not (and (= "3" (:opt_level profile)) + (false? (:debug_assertions profile))) + (throw (ex-info "Cargo did not produce an optimized renderer" + {:type ::renderer-profile :profile profile}))) + (swap! build-ref #(artifacts/write-manifest + % (cond-> (-> (:manifest %) + (assoc-in [:functional :features] resolved) + (assoc-in [:functional :default-features?] (boolean (some #{"default"} resolved))) + (assoc-in [:provenance :cargo-profile] profile)) + (js/Object.hasOwn facts "environment") + (assoc :environment (js->clj (.-environment facts)))))))) + (reset! phase :cljs) + (log/info "Modules compiling") + (let [build @build-ref] + (await (process/run (merge bounds + {:command "clojure" :cwd root + :args ["-M:dev:renderer-bench" "release" + "bench-render-wasm-catalog" "bench-render-wasm-browser"] + :capture-output? false :on-line (fn [line _] (log/debug line)) + :env {"RENDER_BENCH_BUILD_ID" (:id build) + "RENDER_BENCH_CASES_PATH" (:cases-path build) + "RENDER_BENCH_BROWSER_DIR" (:browser-dir build)}}))) + (fs/copyFileSync (path/join root "performance" "benches" "render_wasm" "browser" "index.html") + (:index-path build)) + (reset! phase :identity) + (log/info "Validating") + (check-profile-marks (catalog/build-identity build)) + (catalog/collect build {:master-seed 42}) + (await (validate-browser build root bounds)) + (artifacts/check-files build) + (when (zero? (process/remaining deadline)) + (throw (ex-info "preparation deadline expired" {:type :process-timeout}))) + (log/info " ready") + (let [ready (artifacts/write-manifest build (assoc (:manifest build) :status :ready))] + (reset! build-ref ready) + (artifacts/load-build root (:id ready)))) + (catch :default cause + (let [build @build-ref + context (errors/project-context (ex-data cause)) + failure (errors/evidence @phase cause) + saved (when (= :building (get-in build [:manifest :status])) + (attempt :catch-fatal + (artifacts/write-manifest + build (assoc (:manifest build) :status :failed :failure failure)))) + manifest-error (when (failure? saved) (errors/message-of saved)) + data (cond-> (merge context + {:type ::preparation-failed :build-id (:id build) + :phase @phase :log-path (or (:log-path (ex-data cause)) (:log-path build)) + :reason (str/join " -> " (keep :message (:cause failure)))}) + manifest-error (assoc :manifest-error manifest-error))] + (throw (if manifest-error + (errors/fatal-error + "renderer build preparation failed" data + (errors/combine [{:error cause} + {:phase "storage" :operation :write-manifest + :path (:manifest-path build) :error saved}])) + (ex-info "renderer build preparation failed" data cause))))))))) diff --git a/frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs b/frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs new file mode 100644 index 0000000000..4e04f071b4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs @@ -0,0 +1,880 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.feature-comparison + "Plan and run benchmarks for combinations of optional Cargo renderer features. + + `prepare` reads Cargo metadata and collects the selected benchmark cases. + Call `format-plan` to inspect that data, or await `execute` to build and run + each configuration in sequence. Each configuration disables Cargo defaults. + An edge is a comparison between two configurations that differ by one + requested feature, from disabled to enabled. + + Execution saves progress, run records, logs and edge reports for inspection + after completion, failure or cancellation. Builds use the checkout's renderer + and benchmark sources. Keep other renderer builds and watchers stopped until + execution finishes." + (:require + ["fs" :as fs] + ["node:timers/promises" :as timers] + ["path" :as path] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.run :as run] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.node.time :as time] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.result :as result] + [clojure.set :as set] + [clojure.string :as str])) + +(def ^:private warnings + ["Timing values do not determine success." + "Intervals are exploratory; the comparison records no p-value verdicts."]) + +(def default-services + {:prepare-build build/prepare + :collect-catalog catalog/collect + :create-run-home runs/create-home + :execute-run run/execute}) + +(defn create-home + "Creates a directory and an empty log for a new feature comparison. + + Args: + root: The frontend directory. + comparison-id: Contains 1 to 128 ASCII letters, digits, underscores or + hyphens, starting with a letter or digit. + output-dir: With nil `output-dir`, uses + `target/performance/render-wasm/feature-comparisons/` under + root. An explicit path resolves from the Node process's working directory. + The chosen directory must not exist and must not be inside the managed + builds tree, including through symlink ancestors. + + Returns: + A map containing `:comparison-id`, `:dir`, `:log-path` and `:record-path`. + The log is `feature-comparison.log`; `:record-path` names the future + `feature-comparison.transit` file, which this function does not write. + + Throws: + Invalid identifiers, rejected paths and filesystem failures throw an error; + path and filesystem errors include `:path` when available." + [root comparison-id output-dir] + (artifacts/check-home-id comparison-id "invalid feature comparison id") + (let [default-dir (path/join (artifacts/managed-root root) + "feature-comparisons" comparison-id) + dir (artifacts/check-output-dir (or output-dir default-dir))] + (when (artifacts/inside-builds? root dir) + (artifacts/invalid-home "feature comparison cannot use a directory inside builds" {:path dir})) + (assoc (artifacts/create-home dir "feature-comparison" + "cannot create feature comparison home") + :comparison-id comparison-id))) + +(defn write-record + "Validates and saves a feature-comparison record. + + Encodes `record` as Transit after schema validation, then atomically replaces + that file so readers see either the previous complete record or the new one. + + Args: + home: Supplies `:record-path` in an existing directory. + record: The feature-comparison record. + + Returns: + The record. + + Throws: + Validation, write and rename failures throw without replacing the previous + record." + [home record] + (files/atomic-write (:record-path home) (result/encode record)) + record) + +(defn- permissions + [options] + {:allow-build-differences (boolean (:allow-build-differences options)) + :allow-scene-differences (boolean (:allow-scene-differences options)) + :allow-all-differences (boolean (:allow-all-differences options))}) + +(defn- storage-error + [cause log-path] + (errors/fatal-error "cannot persist feature comparison" + {:type ::storage-error :cause (errors/message-of cause) :log-path log-path} + cause)) + +(defn- change-config + [record slug f] + (update record :configurations + #(mapv (fn [config] + (if (= slug (:slug config)) + (f config) + config)) + %))) + +(defn- change-edge + [record from to f] + (update record :edges + #(mapv (fn [edge] + (if (and (= from (:from edge)) (= to (:to edge))) + (f edge) + edge)) + %))) + +(defn- config-by-slug + [record slug] + (some #(when (= slug (:slug %)) %) (:configurations record))) + +(defn- persist + "Save updated progress before exposing it as the current and last saved record." + [state update] + (let [next (assoc (update @(:record-ref state)) :updated-at (time/now)) + saved (try + (write-record (:home state) next) + (catch :default cause + (throw (storage-error cause (get-in state [:home :log-path])))))] + (reset! (:record-ref state) saved) + (reset! (:saved-ref state) saved) + saved)) + +(defn- initial-record + [prepared comparison-id started-at] + (let [plan (:plan prepared) + options (:options prepared)] + {:type :feature-comparison + :comparison-id comparison-id + :started-at started-at + :updated-at started-at + :status :running + :options options + :permissions (permissions options) + :cases (:cases prepared) + :features (:features plan) + :configurations (mapv #(assoc % :status :planned) (:configurations plan)) + :edges (mapv #(assoc % :status :planned) (:edges plan)) + :warnings warnings})) + +(defn- fail-config + [state slug phase cause] + (persist state + (fn [record] + (change-config record slug + #(-> % (assoc :status :failed + :failure (errors/evidence phase cause)) + (dissoc :reason)))))) + +(defn- interrupt-config + [state slug signal failure] + (persist state + (fn [record] + (change-config record slug + #(cond-> (assoc % :status :failed :reason signal) + (some? failure) (assoc :failure failure)))))) + +(defn- mark-configs-unattempted + [state configurations reason] + (when-let [[configuration & pending] (seq configurations)] + (let [slug (:slug configuration)] + (when (= :planned (:status (config-by-slug @(:record-ref state) slug))) + (persist state #(change-config % slug (fn [entry] + (assoc entry :status :unattempted :reason reason))))) + (mark-configs-unattempted state pending reason)))) + +(defn- mark-edge + [state edge status data] + (persist state + (fn [record] + (change-edge record (:from edge) (:to edge) + #(merge % (assoc data :status status)))))) + +(defn- mark-edges-unattempted + ([state reason] + (mark-edges-unattempted state (:edges @(:record-ref state)) reason)) + ([state edges reason] + (when-let [[edge & pending] (seq edges)] + (when (= :planned (:status edge)) + (mark-edge state edge :unattempted {:reason reason})) + (mark-edges-unattempted state pending reason)))) + +(defn- storage-failure + "Combines earlier failures with a failed comparison-storage operation. + + Args: + operation: Operation that failed while saving comparison evidence. + causes: Earlier failures in occurrence order. + cause: Failure of the storage operation. + + Returns: + An Error retaining earlier failures followed by the storage failure." + [operation causes cause] + (errors/combine (conj (mapv #(hash-map :error %) causes) + {:phase "comparison-storage" :operation operation :error cause}))) + +(defn- finish + [state status exit-code causes] + (let [saved (attempt :catch-fatal (persist state #(assoc % :status status :ended-at (time/now)))) + failure (when (failure? saved) + (storage-failure :finish-comparison causes saved))] + {:comparison (if failure @(:saved-ref state) saved) + :home (:home state) + :log-path (get-in state [:home :log-path]) + :errors (if failure [failure] (vec (remove nil? causes))) + :exit-code (if (and failure (not (contains? #{130 143} exit-code))) 1 exit-code)})) + +(defn- finish-root + [state error] + (let [record @(:record-ref state) + completed (and (every? #(= :completed (:status %)) (:configurations record)) + (every? #(= :completed (:status %)) (:edges record)))] + (if completed + (finish state :completed 0 [error]) + (finish state :failed 1 [error])))) + +(defn- failure-outcome + [state signal cause] + (assoc (outcome/failed cause {:comparison @(:saved-ref state) + :home (:home state) + :log-path (get-in state [:home :log-path])}) + :exit-code (if signal (outcome/interruption-code signal) 1))) + +(defn- interrupt-comparison + [state signal error] + (let [reason (str "interrupted by " signal) + recorded (attempt :catch-fatal + (mark-configs-unattempted state (:configurations @(:record-ref state)) reason) + (mark-edges-unattempted state reason) + nil)] + (if (failure? recorded) + (failure-outcome state signal + (storage-failure :record-interruption [error] recorded)) + (finish state :interrupted (outcome/interruption-code signal) [error])))) + +(defn- record-created-build + [state slug build] + (persist state + #(change-config % slug + (fn [configuration] + (-> configuration + (assoc :status :building + :build (select-keys build [:id :manifest-path :log-path])) + (dissoc :failure :reason)))))) + +(defn- stop-outcome + [slug reason cause] + {:stop? true + :reason (str "not attempted after " reason " in configuration " slug) + :error cause}) + +(defn- fail-configuration + [state slug phase reason cause] + (fail-config state slug phase cause) + (stop-outcome slug reason cause)) + +(defn- interrupt-configuration + [state slug signal phase cause] + (interrupt-config state slug signal (when cause (errors/evidence phase cause))) + (cond-> {:interrupted? true :signal signal} + cause (assoc :error cause))) + +(defn- fail-or-interrupt + [state slug phase cause] + (try + (if-let [signal (process/signal-name (:signal state))] + (interrupt-configuration state slug signal phase cause) + (fail-configuration state slug phase (str (name phase) " failure") cause)) + (catch :default recorded + (throw (storage-failure :record-configuration-failure [cause] recorded))))) + +(defn- begin-configuration + [state configuration] + (let [slug (:slug configuration)] + (if-let [signal (process/signal-name (:signal state))] + {:configuration configuration + :outcome (interrupt-configuration state slug signal nil nil)} + (do + (persist state #(change-config % slug (fn [entry] + (-> entry (assoc :status :building) + (dissoc :failure :reason))))) + {:configuration configuration})))) + +(defn- accept-build + [state flow selected] + (let [{:keys [slug selection]} (:configuration flow) + effective (get-in selected [:manifest :functional :features]) + saved-selection (get-in selected [:manifest :provenance :feature-selection]) + valid-features? (and (vector? effective) (every? string? effective))] + (if (and (= selection saved-selection) valid-features?) + (do + (persist state #(change-config % slug (fn [entry] + (assoc entry :status :ready :effective-features effective)))) + (log/info (str "Configuration " slug " built")) + (assoc flow :build selected)) + (let [cause (ex-info "renderer build manifest disagrees with the planned selection" + {:type ::invalid-build :log-path (:log-path selected) + :selection selection :saved-selection saved-selection})] + (assoc flow :outcome (fail-configuration state slug :build "build failure" cause)))))) + +(defn- ^:async build-configuration + [state flow] + (if (:outcome flow) + flow + (let [{:keys [slug selection]} (:configuration flow) + built (attempt :catch-fatal + (await ((:prepare-build (:services state)) + {:root (:root state) + :selection selection + :build-timeout-ms (get-in state [:options :build-timeout-ms]) + :signal (:signal state) + :on-created (partial record-created-build state slug)}))) + signal (process/signal-name (:signal state))] + (cond + (and (failure? built) (errors/fatal? built) (not (process/unresolved? built))) + (throw built) + + (failure? built) + (assoc flow :outcome (fail-or-interrupt state slug :build built)) + + signal + (assoc flow :outcome (interrupt-configuration state slug signal nil nil)) + + :else (accept-build state flow built))))) + +(defn- check-catalog + [state flow] + (if (:outcome flow) + flow + (let [slug (get-in flow [:configuration :slug]) + selected (:build flow) + mismatch? (-> (attempt + (not= ((:collect-catalog (:services state)) + selected + {:master-seed (get-in state [:options :seed]) + :filter (get-in state [:options :filter])}) + (:cases state))) + (errors/with-data {:log-path (:log-path selected)})) + signal (process/signal-name (:signal state)) + step (cond + (failure? mismatch?) (fail-or-interrupt state slug :catalog mismatch?) + signal (interrupt-configuration state slug signal nil nil) + mismatch? + (fail-configuration state slug :catalog "catalog mismatch" + (ex-info "build catalog disagrees with the prepared cases" + {:type ::catalog-mismatch + :log-path (:log-path selected) + :configuration slug})))] + (cond-> flow step (assoc :outcome step))))) + +(defn- allocate-run + [state flow] + (if (:outcome flow) + flow + (let [slug (get-in flow [:configuration :slug]) + home (attempt + ((:create-run-home (:services state)) + {:root (:root state) :run-id (str (random-uuid)) + :output-dir (path/join (get-in state [:home :dir]) "configs" slug)}))] + (if (failure? home) + (assoc flow :outcome (fail-configuration state slug :run-home "run-home failure" home)) + (assoc flow :run-home home))))) + +(defn- ^:async execute-run + [state flow] + (attempt + (await ((:execute-run (:services state)) + (:root state) (:run-home flow) (:build flow) (:cases state) + (run/get-execution-options (:options state)) + (:signal state))))) + +(defn- get-evidence-error + [slug home response] + (or (some #(when (errors/fatal? %) %) (:errors response)) + (attempt + (when-not (= (get-in response [:run :run-id]) (:run-id (runs/get-record (:record-path home)))) + (throw (ex-info "saved run identity disagrees with execution result" + {:type ::invalid-run :configuration slug})))))) + +(defn- get-run-evidence + [slug home response] + (let [run (:run response) + home (or (:home response) home) + evidence-error (get-evidence-error slug home response) + error (errors/with-data (or evidence-error (first (:errors response))) + {:log-path (:log-path home)})] + {:run run + :home home + :evidence-error evidence-error + :error error + :gpu-failure (some #(when (= :gpu (:stage %)) %) (get-in run [:metadata :diagnostics :failures])) + :run-ref {:run-id (:run-id run) :record-path (:record-path home) + :log-path (:log-path home) :exit-code (:exit-code response)}})) + +(defn- get-run-failure + [{:keys [run evidence-error gpu-failure error]}] + (cond + evidence-error (errors/evidence :run-evidence evidence-error) + gpu-failure gpu-failure + (map? (get-in run [:termination :failure])) (get-in run [:termination :failure]) + error (errors/evidence :run error))) + +(defn- classify-run + [signal evidence] + (let [termination (get-in evidence [:run :termination :reason]) + completed (and (nil? (:evidence-error evidence)) (= :completed termination) + (zero? (get-in evidence [:run-ref :exit-code]))) + interrupted (or signal (when (= :interrupted termination) + (or (get-in evidence [:run :termination :signal]) "SIGINT"))) + failure (get-run-failure evidence) + reason (or interrupted (when (and (not completed) (nil? failure)) "run did not complete"))] + (assoc evidence :completed completed :signal interrupted :failure failure :reason reason))) + +(defn- record-run-result + [state slug {:keys [completed run-ref reason failure]}] + (persist state + #(change-config % slug + (fn [entry] + (if completed + (-> entry (assoc :status :completed :run run-ref) (dissoc :failure :reason)) + (cond-> (assoc entry :status :failed :run run-ref) + (some? reason) (assoc :reason reason) + (some? failure) (assoc :failure failure))))))) + +(defn- zero-attempt-outcome + [slug home] + (stop-outcome slug "a run with no attempts" + (ex-info "configuration run made no attempts" + {:type ::zero-attempts :log-path (:log-path home) :configuration slug}))) + +(defn- config-run-outcome + [slug {:keys [run home completed signal reason error evidence-error gpu-failure]}] + (cond + completed + (if (empty? (:attempts run)) + (zero-attempt-outcome slug home) + (do (log/info (str "Configuration " slug " completed")) {:stop? false})) + + signal {:interrupted? true :signal reason :error error} + evidence-error (stop-outcome slug "run evidence failure" error) + gpu-failure (stop-outcome slug "GPU failure" + (ex-info (:message gpu-failure) + {:type ::gpu-required :log-path (:log-path home) :configuration slug})) + (empty? (:attempts run)) (zero-attempt-outcome slug home) + :else (do (log/warn (str "Configuration " slug " failed")) {:stop? false :error error}))) + +(defn- finish-config-run + [state flow response] + (let [slug (get-in flow [:configuration :slug]) + facts (get-run-evidence slug (:run-home flow) response) + evidence (classify-run (process/signal-name (:signal state)) facts)] + (try + (record-run-result state slug evidence) + (catch :default cause + (throw (storage-failure :record-run-result [(:error evidence)] cause)))) + (config-run-outcome slug evidence))) + +(defn- get-missing-run-cause + "Returns why execution produced no run record, or nil when it produced one." + [response home] + (-> (cond + (failure? response) response + (nil? (:run response)) (or (first (:errors response)) + (ex-info "run executor returned no record" {:type ::missing-run}))) + (errors/with-data {:log-path (:log-path home)}))) + +(defn- ^:async run-configuration + [state flow] + (if (:outcome flow) + flow + (let [slug (get-in flow [:configuration :slug]) + home (:run-home flow)] + (persist state #(change-config % slug (fn [entry] + (assoc entry :status :running + :run {:run-id (:run-id home) + :record-path (:record-path home) + :log-path (:log-path home)})))) + (log/info (str "Configuration " slug " running")) + (let [response (await (execute-run state flow)) + cause (get-missing-run-cause response home)] + (assoc flow :outcome (if cause + (fail-configuration state slug :run "run failure" cause) + (finish-config-run state flow response))))))) + +(defn- ^:async execute-config + [state configuration] + (let [started (begin-configuration state configuration) + built (await (build-configuration state started)) + catalogued (check-catalog state built) + allocated (allocate-run state catalogued) + executed (await (run-configuration state allocated))] + (:outcome executed))) + +(defn- sorted-features + [features] + (when-not (and (vector? features) (every? string? features)) + (throw (ex-info "run metadata lacks an effective feature vector" + {:type ::missing-features}))) + (vec (sort (distinct features)))) + +(defn- edge-facts + [baseline candidate edge] + (let [baseline-features (sorted-features + (get-in baseline [:metadata :build :functional :features])) + candidate-features (sorted-features + (get-in candidate [:metadata :build :functional :features])) + baseline-set (set baseline-features) + candidate-set (set candidate-features)] + {:from (:from edge) + :to (:to edge) + :feature (:feature edge) + :baseline-features baseline-features + :candidate-features candidate-features + :added (vec (sort (set/difference candidate-set baseline-set))) + :removed (vec (sort (set/difference baseline-set candidate-set)))})) + +(defn- save-comparison + [path comparison log-path] + (let [text (result/encode comparison)] + (try + (files/atomic-write path text) + (catch :default cause + (throw (storage-error cause log-path)))))) + +(defn- unacknowledged-count + [comparison] + (reduce + 0 (map (fn [pair] (count (remove :waived? (:mismatches pair)))) + (:pairs comparison)))) + +(defn- get-comparison-options + [record facts] + (assoc (:permissions record) + :expected-feature-change (select-keys facts [:feature :baseline-features + :candidate-features :added :removed]) + :edge facts + :on-unallowed-difference :record)) + +(defn- analyze-edge + "Returns the path of the saved comparison and its count of unacknowledged + differences, or the failure, fatal failures included." + [state edge] + (attempt + :catch-fatal + (let [record @(:record-ref state) + baseline-config (config-by-slug record (:from edge)) + candidate-config (config-by-slug record (:to edge)) + baseline (runs/get-record (get-in baseline-config [:run :record-path])) + candidate (runs/get-record (get-in candidate-config [:run :record-path])) + facts (edge-facts baseline candidate edge) + comparison (compare/compare-runs baseline candidate (get-comparison-options record facts)) + path (path/join (get-in state [:home :dir]) + "comparisons" + (str (:from edge) "__" (:to edge) ".transit"))] + (save-comparison path comparison (get-in state [:home :log-path])) + {:path path + :unacknowledged (unacknowledged-count comparison)}))) + +(defn- record-edge-outcome + "Records the result of analyzing an edge. + + Returns: + An Error retaining accumulated analysis failures, or nil on success. + + Throws: + Fatal analysis or persistence failures with available earlier failures." + [state edge result error] + (let [fatal? (and (failure? result) (errors/fatal? result)) + analyzed (if (failure? result) + (errors/combine [{:error error} + {:phase "comparison" :operation :analyze-edge :error result}]) + error) + recorded (attempt + :catch-fatal + (cond + fatal? + (do + (mark-edge state edge :failed {:failure (errors/evidence :comparison-storage result)}) + (mark-edges-unattempted state "not attempted after comparison storage failure")) + + (failure? result) + (mark-edge state edge :failed {:failure (errors/evidence :comparison result)}) + + (pos? (:unacknowledged result)) + (do + (log/warn (str "Comparison " (:from edge) " -> " (:to edge) + " has " (:unacknowledged result) " unacknowledged differences")) + (mark-edge state edge :failed {:path (:path result) + :failure {:unacknowledged-differences (:unacknowledged result)}})) + + :else + (mark-edge state edge :completed {:path (:path result)})) + nil)] + (cond + (failure? recorded) + (throw (storage-failure :record-edge [analyzed] recorded)) + + fatal? (throw analyzed) + :else analyzed))) + +(defn- ^:async analyze-edges + [state edges error] + (await (timers/setImmediate)) + (if-let [signal (process/signal-name (:signal state))] + (do + (try + (mark-edges-unattempted state (str "interrupted by " signal)) + (catch :default cause + (throw (storage-failure :record-interruption [error] cause)))) + {:interrupted signal :error error}) + (if-let [[edge & pending] (seq edges)] + (analyze-edges state pending (record-edge-outcome state edge (analyze-edge state edge) error)) + {:done true :error error}))) + +(defn- create-comparison-directory + [state] + (try + (fs/mkdirSync (path/join (get-in state [:home :dir]) "comparisons") #js {:recursive true}) + (catch :default cause + (throw (storage-error cause (get-in state [:home :log-path])))))) + +(defn- stop-unresolved-comparison + [state error] + (let [recorded (attempt :catch-fatal + (mark-edges-unattempted state "process group may still be running") + nil)] + (if (failure? recorded) + (failure-outcome state (process/signal-name (:signal state)) + (storage-failure :skip-comparisons [error] recorded)) + (finish state :failed 1 [error])))) + +(defn- ^:async run-edges + [state error] + (let [signal (process/signal-name (:signal state))] + (cond + signal (interrupt-comparison state signal error) + (process/unresolved? error) (stop-unresolved-comparison state error) + :else + (do + (try + (create-comparison-directory state) + (catch :default cause + (throw (storage-failure :create-comparison-directory [error] cause)))) + (let [analyzed (attempt :catch-fatal + (await (analyze-edges state (:edges @(:record-ref state)) error)))] + (cond + (failure? analyzed) + (if-let [signal (process/signal-name (:signal state))] + (failure-outcome state signal analyzed) + (finish state :failed 1 [analyzed])) + (:interrupted analyzed) (interrupt-comparison state (:interrupted analyzed) (:error analyzed)) + :else (finish-root state (:error analyzed)))))))) + +(defn- ^:async run-comparisons + [state configurations] + (if-let [signal (process/signal-name (:signal state))] + (interrupt-comparison state signal nil) + (if-let [[configuration & pending] (seq configurations)] + (let [step (await (execute-config state configuration))] + (cond + (:interrupted? step) + (interrupt-comparison state (:signal step) (:error step)) + + (:stop? step) + (do + (try + (mark-configs-unattempted state pending (:reason step)) + (catch :default cause + (throw (storage-failure :stop-configurations [(:error step)] cause)))) + (run-edges state (:error step))) + + :else (run-comparisons state pending))) + (run-edges state nil)))) + +(defn- create-comparison-state + [root prepared signal services home comparison-id] + {:root root + :home home + :options (:options prepared) + :cases (:cases prepared) + :signal signal + :services services + :record-ref (atom (initial-record prepared comparison-id (time/now))) + :saved-ref (atom nil)}) + +(defn- initialize-comparison + [state] + (let [initial @(:record-ref state) + home (:home state) + saved (write-record home initial)] + (reset! (:record-ref state) saved) + (reset! (:saved-ref state) saved))) + +(defn- ^:async execute-comparison + [state] + (let [home (:home state)] + (binding [log/*log-path* (:log-path home)] + (let [result (attempt :catch-fatal + (initialize-comparison state) + (await (run-comparisons state (:configurations @(:record-ref state)))))] + (if (failure? result) + (failure-outcome state (process/signal-name (:signal state)) result) + result))))) + +(defn ^:async execute + "Build and benchmark every prepared feature configuration, then compare runs. + + The caller must handle `:dry-run` by calling `format-plan` instead; this function + always executes. + + Create a fresh comparison directory and save its initial record before the + first build. Build and run configurations serially, checking each compiled + catalog against the prepared cases. Runs live in `configs/` under the + comparison directory; edge reports live in `comparisons/__.transit`. + Build artifacts remain in the managed builds directory. + + Timing differences alone do not cause failure. Unacknowledged differences + in build, scene or other comparison metadata do cause an edge to fail. + The prepared options `:allow-build-differences` and `:allow-scene-differences` + acknowledge their respective categories; `:allow-all-differences` acknowledges + every compatibility mismatch. Reports retain acknowledged differences. + + Build or catalog failures stop later configurations. A failed run may allow + later configurations when it retains attempts and usable evidence; missing + evidence, GPU failure or zero attempts stop them. Cancellation skips remaining + work. Files already written remain for diagnosis. + + Args: + root: The frontend directory. + prepared: The result of `prepare`. + signal: An optional AbortSignal that cancels execution. + settings: May contain `:services` overrides with the same contracts as the + defaults: `:prepare-build` uses `node.runner.build/prepare`, `:collect-catalog` + uses `node.catalog.loader/collect`, `:create-run-home` uses `node.storage.runs/create-home`, + and `:execute-run` uses `node.runner.run/execute`. Omitted services use + `default-services`. + + Returns: + Await the returned promise for a map with `:comparison`, the last + successfully saved progress record; `:home`, the comparison ID and paths; + `:log-path`; `:errors`, retained error objects; and `:exit-code`. The exit + code is 0 only when all configurations and edges complete, 1 on failure, + or 130/143 on interruption. + + If storage fails, the returned `:comparison` reflects the last successful + atomic write, or nil if no record was saved; `:home` is nil if directory + creation failed. Inspect the saved statuses and `:errors` together, since + `:errors` is not an exhaustive list of failed configurations or edges." + [root prepared signal settings] + (let [services (merge default-services (:services settings)) + comparison-id (str (random-uuid)) + home (attempt (create-home root comparison-id (:output-dir prepared)))] + (if (failure? home) + (outcome/failed home {:comparison nil :home nil}) + (await + (execute-comparison + (create-comparison-state root prepared signal services home comparison-id)))))) + + + +(defn- get-root-features + "Read feature definitions for cwd/Cargo.toml from Cargo metadata JSON." + [cwd text] + (let [metadata (js->clj (js/JSON.parse text)) + manifest (fs/realpathSync (path/join cwd "Cargo.toml")) + package (first (filter #(= manifest (fs/realpathSync + (path/resolve cwd (get % "manifest_path")))) + (get metadata "packages")))] + (when-not package + (throw (ex-info "Cargo metadata does not contain the renderer manifest" + {:type ::missing-package :manifest-path manifest}))) + (get package "features"))) + +(defn ^:async prepare + "Collect and validate a feature comparison plan without building or running it. + + Run `cargo metadata --offline --locked --no-deps` in `render-wasm` with the + inherited environment. Cargo and its offline metadata must be available. + `:build-timeout-ms` bounds that subprocess in milliseconds, not the entire + planning operation. + Require empty declared Cargo defaults and known requested feature names; + Cargo remains responsible for resolving feature activation during builds. + + Planning creates no comparison output and starts no build, server or browser, + including for a dry-run. + + Args: + root: The frontend directory, with render-wasm as its sibling. + opts: Resolved CLI options, including defaults. `:features` supplies the + requested Cargo feature names, `:seed` supplies the master scene seed, + and `:filter` selects case IDs by substring. `:max-features` limits the + requested feature count. Execution requires `:confirm` for more than + eight configurations or a blank filter; `:dry-run` bypasses that + confirmation requirement only. + + An optional `:output-dir` resolves from the Node process's working + directory and must not exist. Check it without creating or reserving it. + `execute` checks it again and also rejects paths inside the managed + builds tree. + signal: An optional AbortSignal that cancels the process. + + Returns: + Await the promise for a map with `:plan`, `:cases`, `:output-dir` and + `:options`. The `:plan` map contains `:features` in sorted order, + `:configurations` with a `:slug` and Cargo `:selection`, and `:edges` with + `:from` and `:to` slugs and a `:feature`. Slugs are bit strings in sorted + feature order: `0` means disabled, `1` enabled. The `:cases` vector holds + case descriptions collected once. `:output-dir` is the absolute explicit + path or nil. `:options` contains the supplied options with normalized + features and output path. Pass this map to `format-plan` or `execute`. + + Throws: + Invalid options or paths, Cargo failures or cancellation, unsupported + manifest features, and case collection failures reject the promise. + An empty case selection is an error." + [root opts signal] + (let [plan (feature-plan/plan (:features opts) opts) + output-dir (artifacts/check-output-dir (:output-dir opts)) + cwd (path/resolve root ".." "render-wasm") + metadata (await (process/run + {:command "cargo" + :args ["metadata" "--offline" "--locked" "--no-deps" "--format-version" "1"] + :cwd cwd + :deadline (process/deadline (:build-timeout-ms opts settings/default-build-timeout-ms)) + :signal signal}))] + (feature-plan/check-manifest (:features plan) (get-root-features cwd (:stdout metadata))) + {:plan plan + :cases (cases/collect-cases {:master-seed (:seed opts) :filter (:filter opts)}) + :output-dir output-dir + :options (assoc opts :features (:features plan) :output-dir output-dir)})) + +(defn format-plan + "Return a text summary of the map produced by `prepare`. + + This function neither collects cases again nor reads the filesystem, + starts processes or executes benchmarks. + + Returns: + Feature names, configurations, comparison edges, case count, sampling + counts and configured timeouts." + [{:keys [plan cases options]}] + (str "Feature comparison plan\n" + "Features: " (str/join ", " (:features plan)) "\n" + "Configurations (Cargo defaults disabled):\n" + (str/join "\n" (map #(str " " (:slug %) ": " + (str/join ", " (get-in % [:selection :features]))) + (:configurations plan))) "\n" + "Edges:\n" + (str/join "\n" (map #(str " " (:from %) " -> " (:to %) ": " (:feature %)) + (:edges plan))) "\n" + "Cases: " (count cases) "\n" + "Attempts per case: " (:warmups options) " warmups, " (:repetitions options) " measured\n" + "Timeouts (ms): case " (:timeout-ms options) + ", build " (:build-timeout-ms options) ", startup " (:startup-timeout-ms options) "\n")) diff --git a/frontend/performance/benches/render_wasm/node/runner/run.cljs b/frontend/performance/benches/render_wasm/node/runner/run.cljs new file mode 100644 index 0000000000..47bcc0fc0b --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/run.cljs @@ -0,0 +1,916 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.run + "Renderer benchmark runs over fixed builds. + + Runs are stored in a `home`, and execute via a loopback host and at most one + Chromium process. + + `:fresh` cases prepare once per slot on a new page and context. + `:reuse` cases prepare once and run every slot on the same page and context. + + A failed attempt consumes its slot. + + Browser failures leave the current case's remaining slots unattempted and + continue with the next case in a new browser." + (:require-macros + [app.common.exceptions :as ex]) + (:require + ["module" :as node-module] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.declarations :as declarations] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.measurement :as measurement] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.browser :as browser] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.server :as server] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.node.time :as time] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [clojure.string :as str])) + +(defn get-execution-options + "Returns resolved settings for executing a prepared run." + [options] + (select-keys options [:seed :filter :warmups :repetitions :timeout-ms + :startup-timeout-ms :screenshots :require-gpu :remove])) + +(defn- cause-evidence + "Plain failure evidence for a rejected browser, server or persistence action." + [cause fallback-phase] + (cond-> (errors/fail-data + (errors/evidence (or (:phase (ex-data cause)) fallback-phase) cause)) + (contains? (ex-data cause) ::browser/deadline-exceeded) + (assoc :deadline-exceeded (::browser/deadline-exceeded (ex-data cause))))) + +(defn- interruption + [signal] + (when-let [name (process/signal-name signal)] + {:signal name :cause (.-reason ^js signal)})) + +(defn- interrupted? + [state] + (some? (interruption (:signal state)))) + +(defn- check-deadline + [deadline] + (when (zero? (process/remaining deadline)) + (throw (ex-info "case deadline exceeded" {:type ::browser/timeout :phase :deadline})))) + +(defn- failure-outcome + [state deadline evidence] + (cond + (interrupted? state) :interrupted + (if (contains? evidence :deadline-exceeded) + (:deadline-exceeded evidence) + (or (zero? (process/remaining deadline)) (= "deadline" (:phase evidence)))) :timed-out + :else :failed)) + +(defn- keep-sink-failure + "Remembers the first failed write to the run record or log. The failure stays + in the outcome even when a later write succeeds." + [state cause] + (swap! (:sink-failure-ref state) #(or % cause)) + cause) + +(defn- write-log + [state output text] + (try + (output text) + (catch :default cause + (throw (keep-sink-failure + state + (if (errors/fatal? cause) + cause + (errors/fatal-error "cannot write the run log" {} cause))))))) + +(defn- warn-safely + "Appends runner diagnostics without replacing an error already under handling." + [state text] + (ex/ignoring (write-log state log/warn text)) + nil) + +(defn- persist + [state] + (try + (runs/write-record! (:home state) @(:run-ref state)) + (catch :default cause + (throw (keep-sink-failure + state + (errors/fatal-error "cannot save the run record" {:log-path (get-in state [:home :log-path])} cause)))))) + +(defn- record-preparation + [state preparation] + (swap! (:run-ref state) result/record-preparation preparation) + (persist state) + (write-log state log/debug (str "Prepared " (result/case-name (:case-id preparation))))) + +(defn- record-attempt + [state measured] + (swap! (:run-ref state) result/record-attempt measured) + (persist state) + (let [remaining (get-in @(:run-ref state) [:remaining (:case-id measured)]) + total (+ (:warmups (:opts state)) (:repetitions (:opts state))) + index (- total (:warmups remaining) (:repetitions remaining))] + (write-log state (if (= :failed (:outcome measured)) log/warn log/info) + (str " attempt " index "/" total " " (name (:outcome measured)) + (when (:warmup? measured) " (warmup)"))))) + +(defn- record-failure + [state case-id stage evidence] + (let [failure (cond-> (assoc (errors/project-diagnostic evidence) + :stage stage) + (some? case-id) (assoc :case-id case-id))] + (reset! (:harness-failed? state) true) + (swap! (:run-ref state) update-in [:metadata :diagnostics :failures] + (fnil conj []) failure) + (persist state) + (write-log state log/warn (str (name stage) " failed" + (when (some? case-id) (str " for " (result/case-name case-id))) + ": " (:message failure) "\n")) + failure)) + +(defn- record-screenshot-status + [state case-id status] + (swap! (:run-ref state) assoc-in + [:metadata :diagnostics :screenshots (result/case-name case-id)] status) + (persist state)) + +(defn- screenshot-status + [state case-id] + (get-in @(:run-ref state) + [:metadata :diagnostics :screenshots (result/case-name case-id)])) + +(defn- record-screenshot-skip + [state case-id reason] + (when (:screenshots (:opts state)) + (when-not (screenshot-status state case-id) + (record-screenshot-status state case-id {:status :skipped :reason reason})))) + +(defn- ensure-screenshot-status + [state case reason] + (when (:screenshots (:opts state)) + (when-not (screenshot-status state (:id case)) + (record-screenshot-skip state (:id case) reason)))) + +(defn- ^:async screenshot-case + [state case page deadline] + (if (:screenshots (:opts state)) + (let [{:keys [file path]} (runs/screenshot-path (:home state) (:id case)) + bounds {:deadline deadline + :signal (:signal state) :phase :screenshot} + shot (attempt (await ((get-in state [:services :screenshot]) page path bounds)))] + (if (failure? shot) + (do + (record-screenshot-status state (:id case) + {:file file :status :failed + :message (errors/message-of shot) + :cause (errors/describe-cause shot)}) + (write-log state log/warn (str "Screenshot failed for " + (result/case-name (:id case)) ": " + (errors/message-of shot) "\n")) + (let [outcome (failure-outcome state deadline (cause-evidence shot :screenshot))] + (when (= :timed-out outcome) + (record-failure state (:id case) :case-deadline (cause-evidence shot :screenshot))) + (if (= :failed outcome) :ok outcome))) + (do + (record-screenshot-status state (:id case) {:file file :status :ok}) + (write-log state log/debug (str "Screenshot " file)) + :ok))) + :ok)) + +(defn- case-slots + "Returns each scheduled slot in order: all warmups, then measured slots. + True marks a warmup." + [opts] + (vec (concat (repeat (:warmups opts) true) + (repeat (:repetitions opts) false)))) + +(defn- renderer-url + [build-id file] + (str "/renderer/" file "?build=" (js/encodeURIComponent build-id))) + +(defn- ^:async prepare-case + [state case page deadline warmup?] + (check-deadline deadline) + (let [build-id (:id (:selected-build state)) + preparation-id (str (random-uuid)) + response (attempt + (measurement/check-response + :preparation preparation-id + (await ((get-in state [:services :prepare]) + page + (transit/encode-str + {:case case + :preparation-id preparation-id + :timeout-ms (max 1 (process/remaining deadline)) + :module-url (renderer-url build-id "render-wasm.js") + :wasm-url (renderer-url build-id "render-wasm.wasm")}) + {:deadline deadline :signal (:signal state) :phase :preparation})))) + accepted (when (and (not (failure? response)) (= "ok" (:status response))) + (attempt + (let [preparation (assoc (:preparation response) :warmup? warmup?)] + (when-not (and (= preparation-id (:preparation-id preparation)) + (= (:id case) (:case-id preparation)) + (not (:failure preparation))) + (throw (ex-info "preparation success response is inconsistent" {}))) + (record-preparation state preparation) + preparation)))] + (if (and accepted (not (failure? accepted))) + {:preparation accepted} + (let [failure (cond + (failure? accepted) (assoc (cause-evidence accepted :invalid-output) + :partial {:response response}) + (failure? response) (cause-evidence response :preparation) + :else response) + phases (if (failure? accepted) (get-in response [:preparation :phases]) (:phases failure))] + (record-preparation state {:preparation-id preparation-id :case-id (:id case) + :warmup? warmup? + :phases (measurement/partial-phases phases preparation-id [:preparation]) + :diagnostics (or (get-in failure [:partial :setup]) {}) + :failure (errors/project-diagnostic failure)}) + {:failure failure :preparation-id preparation-id})))) + +(defn- ^:async run-attempt + "Returns the bridge evidence of one attempt, or the failure." + [state page preparation-id deadline] + (attempt + (measurement/check-response + :attempt preparation-id + (await ((get-in state [:services :run]) + page + (transit/encode-str + {:preparation-id preparation-id + :timeout-ms (max 1 (process/remaining deadline))}) + {:deadline deadline :signal (:signal state) :phase :attempt}))))) + +(defn- ^:async recycle-browser + [state] + (when-let [owner @(:owner-ref state)] + (let [closed (attempt (await ((get-in state [:services :close]) owner)))] + (if (failure? closed) + (do + (record-failure state nil :browser-close (cause-evidence closed :browser-close)) + (reset! (:abort-run? state) true)) + (do + (reset! (:owner-ref state) nil) + (reset! (:page-ref state) nil)))))) + +(defn- ^:async close-current-page + [state case] + (let [owner @(:owner-ref state) + connected? (get-in state [:services :connected?])] + (if (and owner connected? (not (connected? owner))) + (do + (record-failure state (:id case) :browser-disconnected + {:phase "browser-disconnected" :message "browser disconnected"}) + (reset! (:page-ref state) nil) + (await (recycle-browser state)) + false) + (if-let [page @(:page-ref state)] + (do + (reset! (:page-ref state) nil) + (let [closed (attempt (await ((get-in state [:services :close-page]) page)))] + (if (failure? closed) + (do + (record-failure state (:id case) :browser-close (cause-evidence closed :browser-close)) + (await (recycle-browser state)) + false) + true))) + true)))) + +(defn- ^:async ensure-browser + [state deadline] + (or @(:owner-ref state) + (let [opts (:opts state) + options {:startup-timeout-ms (:startup-timeout-ms opts) + :signal (:signal state) :deadline deadline} + owner (await ((get-in state [:services :launch]) options))] + (reset! (:owner-ref state) owner) + (swap! (:run-ref state) assoc-in [:metadata :environment :browser] (:version owner)) + (persist state) + (write-log state log/debug (str "Browser " (:version owner))) + owner))) + +(defn- usable-graphics-identity? + [value] + (and (string? value) + (not (contains? #{"" "unknown" "unavailable" "null" "undefined"} + (str/lower-case (str/trim value)))))) + +(defn- check-gpu + [state preparation] + (let [graphics (:effective-graphics preparation) + accepted (and (true? (:webgl2 graphics)) + (false? (:software graphics)) + (usable-graphics-identity? (:vendor graphics)) + (usable-graphics-identity? (:renderer graphics)))] + (if (and (:require-gpu (:opts state)) (not accepted)) + (do + (record-failure state (:case-id preparation) :gpu + {:status "failed" :phase "gpu" + :message (str "GPU check failed; renderer: " (pr-str (:renderer graphics)) + ". Hardware WebGL2 with a usable vendor and renderer is required." + " " settings/gpu-guidance) + :partial {:graphics graphics}}) + (reset! (:abort-run? state) true) + :abort) + :ok))) + +(defn- ^:async open-case-page + [state case owner deadline] + (let [opened (attempt + (await ((get-in state [:services :new-page]) + owner @(:host-ref state) + {:viewport (declarations/resolve-viewport (:view case)) + :deadline deadline + :startup-timeout-ms (:startup-timeout-ms (:opts state)) + :signal (:signal state)})))] + (cond + (not (failure? opened)) + (do + (reset! (:page-ref state) opened) + {:page (:page opened)}) + + (interrupted? state) + {:outcome :interrupted} + + :else + (let [evidence (cause-evidence opened :browser-page) + outcome (failure-outcome state deadline evidence)] + (record-failure state (:id case) :browser-page evidence) + (await (recycle-browser state)) + {:outcome outcome})))) + +(defn- ^:async open-page + [state case deadline] + (check-deadline deadline) + (if (interrupted? state) + {:outcome :interrupted} + (let [owner (attempt (await (ensure-browser state deadline)))] + (cond + (not (failure? owner)) + (await (open-case-page state case owner deadline)) + + (interrupted? state) + {:outcome :interrupted} + + :else + (let [evidence (cause-evidence owner :browser-launch) + outcome (failure-outcome state deadline evidence)] + (record-failure state (:id case) :browser-launch evidence) + (if (and (= :timed-out outcome) + (not= ::browser/startup-cleanup (:type (ex-data owner)))) + {:outcome outcome} + (do + (reset! (:abort-run? state) true) + {:outcome :abort}))))))) + +(defn- ^:async execute-slot + [state case page preparation-id warmup? deadline] + (let [response (await (run-attempt state page preparation-id deadline)) + evidence (if (failure? response) (cause-evidence response :attempt) response) + accepted (attempt + (let [measured (measurement/attempt (:id case) preparation-id warmup? evidence)] + (record-attempt state measured) + measured)) + measured (if (failure? accepted) + (let [invalid (assoc (cause-evidence accepted :invalid-output) + :phases (measurement/partial-phases (:phases evidence) preparation-id [:main :reset]) + :partial {:response evidence}) + failed (measurement/attempt (:id case) preparation-id warmup? invalid)] + (record-attempt state failed) + failed) + accepted)] + (cond + (and (failure? response) (interrupted? state)) :interrupted + (= :completed (:outcome measured)) :ok + :else (failure-outcome state deadline (if (failure? response) + (cause-evidence response :attempt) + response))))) + +(defn- ^:async finish-slot + [state case page outcome final? deadline] + (if (and final? (= :ok outcome)) + (await (screenshot-case state case page deadline)) + (do + (when (not= :ok outcome) + (record-screenshot-skip state (:id case) :no-completed-measured-attempt)) + outcome))) + +(defn- ^:async run-fresh-slot + [state case page warmup? final? deadline] + (let [prepared (await (prepare-case state case page deadline warmup?))] + (if-let [evidence (:failure prepared)] + (do + (record-attempt state (measurement/attempt (:id case) (:preparation-id prepared) warmup? + (dissoc evidence :phases))) + (await (finish-slot state case page + (failure-outcome state deadline evidence) final? deadline))) + (let [preparation (:preparation prepared)] + (if (= :abort (check-gpu state preparation)) + :abort + (let [outcome (await (execute-slot state case page (:preparation-id preparation) + warmup? deadline))] + (await (finish-slot state case page outcome final? deadline)))))))) + +(defn- ^:async run-fresh-page + [state case warmup? final? deadline] + (let [{:keys [page outcome]} (await (open-page state case deadline))] + (if page + (let [outcome (await (run-fresh-slot state case page warmup? final? deadline))] + (if (= :ok outcome) + (if (await (close-current-page state case)) outcome :failed) + outcome)) + (or outcome :abandon)))) + +(defn- ^:async run-fresh-slots + [state case slots deadline] + (if-let [[warmup? & pending] (seq slots)] + (let [outcome (await (run-fresh-page state case warmup? (empty? pending) deadline))] + (if (= :ok outcome) + (run-fresh-slots state case pending deadline) + outcome)) + :done)) + +(defn- ^:async run-fresh-case + [state case deadline] + (let [outcome (await (run-fresh-slots state case (case-slots (:opts state)) deadline))] + (ensure-screenshot-status state case :case-abandoned) + outcome)) + +(defn- ^:async run-reuse-slots + [state case page preparation-id slots deadline] + (if-let [[warmup? & pending] (seq slots)] + (let [_ (check-deadline deadline) + executed (await (execute-slot state case page preparation-id warmup? deadline)) + outcome (await (finish-slot state case page executed (empty? pending) deadline))] + (if (= :ok outcome) + (run-reuse-slots state case page preparation-id pending deadline) + outcome)) + :done)) + +(defn- ^:async abandon-preparation + [state case evidence deadline] + (record-failure state (:id case) :preparation evidence) + (ensure-screenshot-status state case :no-completed-measured-attempt) + (failure-outcome state deadline evidence)) + +(defn- ^:async run-reuse-prepared + [state case page preparation deadline] + (let [accepted (check-gpu state preparation) + outcome (if (= :abort accepted) + :abort + (await (run-reuse-slots state case page (:preparation-id preparation) + (case-slots (:opts state)) deadline)))] + (ensure-screenshot-status state case :case-abandoned) + (if (= :done outcome) + (if (await (close-current-page state case)) outcome :failed) + outcome))) + +(defn- ^:async run-reuse-page + [state case page deadline] + (let [prepared (await (prepare-case state case page deadline false))] + (if-let [evidence (:failure prepared)] + (await (abandon-preparation state case evidence deadline)) + (await (run-reuse-prepared state case page (:preparation prepared) deadline))))) + +(defn- ^:async run-reuse-case + [state case deadline] + (let [{:keys [page outcome]} (await (open-page state case deadline))] + (cond + page (await (run-reuse-page state case page deadline)) + (= :abort outcome) :abort + + :else + (do + (ensure-screenshot-status state case :case-abandoned) + (or outcome :abandon))))) + + +(defn- fail-startup + "Records the startup failure and aborts the run. An interrupted run records + nothing. Returns false." + [state stage cause] + (when-not (interrupted? state) + (record-failure state nil stage (cause-evidence cause stage)) + (reset! (:abort-run? state) true)) + false) + +(defn- ^:async launch-initial + [state deadline] + (let [owner (attempt (await (ensure-browser state deadline)))] + (if (failure? owner) + (fail-startup state :browser-launch owner) + true))) + +(defn- ^:async record-provenance + [state deadline] + (let [git (attempt (await (build/git-provenance (:root state) + {:deadline deadline :signal (:signal state)})))] + (if (failure? git) + (fail-startup state :runner-git git) + (do + (swap! (:run-ref state) assoc-in [:metadata :diagnostics :runner-git] git) + (persist state) + true)))) + +(defn- ^:async start-server + [state deadline] + (let [remaining (process/remaining deadline) + host (attempt + (when (zero? remaining) + (throw (ex-info "server startup timed out" + {:type ::startup-timeout :phase :server-startup}))) + (await (server/open (:selected-build state) + {:startup-timeout-ms remaining :signal (:signal state)})))] + (if (failure? host) + (fail-startup state :server-startup host) + (do + (reset! (:host-ref state) host) + true)))) + +(defn- session-active? + [state] + (and (not @(:abort-run? state)) (not (interrupted? state)))) + +(defn- get-case-failure + [run case-id] + (or (some :failure (reverse (filter #(= case-id (:case-id %)) (:attempts run)))) + (some :failure (reverse (filter #(= case-id (:case-id %)) (:preparations run)))) + (some #(when (= case-id (:case-id %)) %) + (reverse (get-in run [:metadata :diagnostics :failures]))))) + +(defn- ^:async run-case + [state case deadline] + (let [executed (attempt + (if (= :fresh (:context case)) + (await (run-fresh-case state case deadline)) + (await (run-reuse-case state case deadline)))) + outcome (if (failure? executed) + (let [evidence (cause-evidence executed :case)] + (record-failure state (:id case) :case evidence) + (failure-outcome state deadline evidence)) + executed) + unmeasured? (and (= :done outcome) + (not (result/has-measurements? @(:run-ref state) (:id case)))) + outcome (if unmeasured? + (do + (record-failure state (:id case) :case + {:phase "case" :reason :no-measurements + :message "case has no measurements"}) + :failed) + outcome) + saved-outcome (cond + (= :done outcome) :completed + (= :interrupted outcome) :interrupted + (= :timed-out outcome) :timed-out + :else :failed) + failure (when (not= :completed saved-outcome) + (get-case-failure @(:run-ref state) (:id case)))] + (swap! (:run-ref state) result/record-case-outcome (:id case) + (cond-> {:outcome saved-outcome} + failure (assoc :failure failure))) + (ensure-screenshot-status state case :case-abandoned) + (persist state) + (when (not= :done outcome) + (await (recycle-browser state))) + outcome)) + +(defn- ^:async run-cases + [state cases] + (when (and (seq cases) (session-active? state)) + (let [case (first cases) + deadline (process/deadline (:timeout-ms (:opts state)))] + (write-log state log/info (str "▸ Case " (result/case-name (:id case)) + " (" (name (:context case)) ")")) + (let [outcome (await (run-case state case deadline))] + (if (#{:abort :interrupted} outcome) + (do + (reset! (:abort-run? state) true) + outcome) + (run-cases state (rest cases))))))) + +(defn- ^:async run-session + [state] + (let [deadline (process/deadline (:startup-timeout-ms (:opts state))) + provenance-ready? (await (record-provenance state deadline)) + server-ready? (and provenance-ready? (await (start-server state deadline))) + browser-ready? (and server-ready? (session-active? state) (await (launch-initial state deadline)))] + (when (and browser-ready? (session-active? state)) + (await (run-cases state (:cases state)))))) + +(defn- get-termination + [run interrupted harness-failed?] + (let [all-attempted? (every? (fn [[_ slots]] + (and (zero? (:warmups slots)) + (zero? (:repetitions slots)))) + (:remaining run)) + failed? (or (some #(= :failed (:outcome %)) (:attempts run)) + (some #(not= :completed (:outcome %)) (vals (:case-outcomes run)))) + reason (cond + interrupted :interrupted + all-attempted? (if failed? :failed :completed) + :else :incomplete) + failures (get-in run [:metadata :diagnostics :failures]) + attempt-failure (some #(when (:failure %) (:failure %)) + (reverse (:attempts run))) + termination (cond-> {:reason reason} + interrupted + (assoc :signal (:signal interrupted)) + + (and (seq failures) + (or (not= :completed reason) harness-failed?)) + (assoc :failure (last failures)) + + (and (not= :completed reason) + (empty? failures) + (some? attempt-failure)) + (assoc :failure attempt-failure))] + termination)) + +(defn- complete-screenshots + [screenshots cases] + (let [defaults (into {} (map (fn [case] + [(result/case-name (:id case)) {:status :skipped :reason :not-run}])) + cases)] + (merge defaults screenshots))) + +(defn- finish-run + [state] + (let [termination (get-termination @(:run-ref state) (interruption (:signal state)) + @(:harness-failed? state))] + (when (:screenshots (:opts state)) + (swap! (:run-ref state) update-in [:metadata :diagnostics :screenshots] + complete-screenshots (:cases state))) + (swap! (:run-ref state) result/finish-run (time/now) termination) + (persist state) + @(:run-ref state))) + +(defn- ^:async release-resource + "Closes the resource in `reference`. Returns nil or the failure." + [reference close] + (attempt :catch-fatal + (when-let [resource @reference] + (reset! reference nil) + (await (close resource))) + nil)) + +(defn- ^:async cleanup + [state] + (let [page-failure (await (release-resource (:page-ref state) (get-in state [:services :close-page]))) + browser-failure (await (release-resource (:owner-ref state) (get-in state [:services :close]))) + server-failure (await (release-resource (:host-ref state) server/close)) + failure (errors/combine + [{:phase "cleanup" :operation :close-page :error page-failure} + {:phase "cleanup" :operation :close-browser :error browser-failure} + {:phase "cleanup" :operation :close-server :error server-failure}])] + (when failure + (warn-safely state (str "run: cleanup failed: " (errors/message-of failure) "\n"))) + failure)) + +(defn- remove-build + [root selected-build] + (artifacts/remove-builds root [(:id selected-build)])) + +(defn- git-facts + [selected-build] + (merge {:sha nil :dirty nil} + (get-in selected-build [:manifest :provenance :git]))) + +(defn- playwright-version + [root] + (ex/ignoring + (let [require-package (if (exists? js/require) + js/require + (node-module/createRequire (path/join root "package.json")))] + (.-version (require-package "playwright/package.json"))))) + +(defn- environment + [root _selected-build] + (let [cpus (array-seq (os/cpus))] + {:node (.-version js/process) + :playwright (playwright-version root) + :browser nil + :platform (os/platform) + :release (os/release) + :arch (os/arch) + :cpu {:models (vec (distinct (map #(.-model %) cpus))) + :logical-count (count cpus)} + :launch (browser/launch-settings)})) + +(defn- metadata + [root selected-build opts] + {:git (git-facts selected-build) + :build (cond-> {:functional (or (get-in selected-build [:manifest :functional]) {}) + :provenance (assoc (get-in selected-build [:manifest :provenance]) + :build-id (:id selected-build))} + (contains? (:manifest selected-build) :environment) + (assoc :environment (get-in selected-build [:manifest :environment]))) + :environment (environment root selected-build) + :scored? true + :diagnostics (cond-> {:seed (:seed opts) + :filter (:filter opts) + :warmups (:warmups opts) + :repetitions (:repetitions opts) + :timeout-ms (:timeout-ms opts) + :startup-timeout-ms (:startup-timeout-ms opts) + :screenshots? (boolean (:screenshots opts)) + :require-gpu? (boolean (:require-gpu opts)) + :penpot-gpu (or (.-PENPOT_GPU (.-env js/process)) nil) + :runner-git {:sha nil :dirty nil} + :runtime-helper-identity :unverified + :screenshots {} + :failures []} + (:build-timeout-ms opts) (assoc :build-timeout-ms (:build-timeout-ms opts)))}) + +(defn- create-run-record + [root run-id selected-build cases opts] + (result/create-run + {:run-id run-id + :started-at (time/now) + :plan {:cases cases + :warmups (:warmups opts) + :repetitions (:repetitions opts) + :timeout-policy {:case-ms (:timeout-ms opts)}} + :metadata (metadata root selected-build opts)})) + +(defn- create-run-state + [root home selected-build cases opts signal] + {:root root + :selected-build selected-build + :cases cases + :opts opts + :signal signal + :home home + :run-ref (atom (create-run-record root (:run-id home) selected-build cases opts)) + :sink-failure-ref (atom nil) + :owner-ref (atom nil) + :page-ref (atom nil) + :host-ref (atom nil) + :abort-run? (atom false) + :harness-failed? (atom false) + :services (merge browser/service (:browser opts))}) + +(defn- record-session-failure + [state stage cause] + (let [recorded (attempt :catch-fatal + (record-failure state nil stage (cause-evidence cause stage)))] + (when (failure? recorded) + (warn-safely state (str "Could not record " (when (= :cleanup stage) "cleanup ") + "failure: " (errors/message-of recorded) "\n"))))) + +(defn- ^:async execute-session + "Runs the session and retains failures that require final reporting. + + Args: + state: The session's execution state. + + Returns: + The failure that ended execution, or nil. Interrupted sessions retain + fatal failures." + [state] + (let [ended (attempt :catch-fatal (await (run-session state)))] + (when (and (failure? ended) (or (not (interrupted? state)) (errors/fatal? ended))) + (record-session-failure state :runner ended) + ended))) + +(defn- ^:async cleanup-session + [state] + (when-let [cause (await (cleanup state))] + (record-session-failure state :cleanup cause) + cause)) + +(defn- remove-run-build + [state saved? failures] + (when (and saved? (:remove (:opts state)) (not-any? process/unresolved? failures)) + (attempt :catch-fatal + (remove-build (:root state) (:selected-build state)) + nil))) + +(defn- format-run-report + "Returns the report text or the failure." + [state run] + (let [report (attempt (format/format-run (summary/summarize run)))] + (when (failure? report) + (warn-safely state (str "Report failed: " (errors/message-of report) "\n"))) + report)) + +(defn- log-run-completion + [state run] + (attempt :catch-fatal + (write-log state log/debug (str "Run " (name (get-in run [:termination :reason])))) + nil)) + +(defn- get-exit-code + [run causes harness-failed?] + (case (get-in run [:termination :reason]) + :interrupted (outcome/interruption-code (get-in run [:termination :signal])) + :completed (if (or (seq causes) harness-failed?) 1 0) + 1)) + +(defn- finish-execution + [state session-error cleanup-error] + (let [saved (attempt :catch-fatal (finish-run state)) + run (if (failure? saved) @(:run-ref state) saved) + removal (remove-run-build state (not (failure? saved)) + (filter failure? [session-error cleanup-error saved @(:sink-failure-ref state)])) + report (format-run-report state run) + logged (log-run-completion state run) + causes (->> [cleanup-error session-error saved removal report logged + @(:sink-failure-ref state)] + (filter failure?) + distinct + vec) + home (:home state)] + {:run run + :home home + :log-path (:log-path home) + :report (when-not (failure? report) report) + :errors causes + :exit-code (get-exit-code run causes @(:harness-failed? state))})) + +(defn ^:async execute + "Executes a prepared selection. Owns host and browser. + + - `root`: the frontend checkout root. + - `home`: the reserved output home. + - `selected-build`: the prepared build. + - `cases`: the selected cases. + - `options`: resolved execution settings, including optional local :browser services. + - `signal`: an optional abort signal. + + A browser startup failure leaves a valid partial record. + + Returns the run, home, report, log path, exit code and the failures in + :errors, a cleanup failure first. :errors includes any failed record or log + write, including a failure followed by a successful write. An interruption + is not a failure." + [root home selected-build cases options signal] + (binding [log/*log-path* (:log-path home)] + (let [state (create-run-state root home selected-build cases options signal)] + (persist state) + (let [session-error (await (execute-session state)) + cleanup-error (await (cleanup-session state))] + (finish-execution state session-error cleanup-error))))) + +(defn- ^:async get-selected-build + [root opts signal] + (let [{:keys [kind id default-features features build-timeout-ms]} (:build opts)] + (case kind + :existing (artifacts/load-build root id) + :prepare (await (build/prepare + {:root root + :selection {:default-features? default-features :features features} + :build-timeout-ms build-timeout-ms :signal signal}))))) + +(defn- get-cases + [selected-build opts] + (try + (catalog/collect selected-build {:master-seed (:seed opts) :filter (:filter opts)}) + (catch :default cause + (throw (ex-info (errors/message-of cause) + (assoc (ex-data cause) :build-id (:id selected-build) + :log-path (:log-path selected-build)) + cause))))) + +(defn ^:async run + "Prepares or loads a build, collects matching cases and executes one run. + + - `root`: the frontend checkout root. + - `options`: resolved run options with a tagged :build source. + - `signal`: an optional abort signal. + - `services`: local service overrides, with browser functions under :browser. + + Returns an outcome with the reserved home, available evidence and failures." + [root options signal services] + (let [home (runs/create-home {:root root :run-id (str (random-uuid)) + :output-dir (:output-dir options) :remove (:remove options)}) + opts (cond-> (assoc (get-execution-options options) :browser (:browser services)) + (= :prepare (get-in options [:build :kind])) + (assoc :build-timeout-ms (get-in options [:build :build-timeout-ms])))] + (binding [log/*log-path* (:log-path home)] + (try + (let [selected-build (await (get-selected-build root options signal)) + cases (get-cases selected-build opts)] + (log/info (str " build " (:id selected-build))) + (log/info (str " selected " (count cases) " cases")) + (await (execute root home selected-build cases opts signal))) + (catch :default cause + (outcome/failed (-> cause errors/as-failure + (errors/with-data {:log-path (:log-path home)})) + {:home home :run nil :report nil})))))) diff --git a/frontend/performance/benches/render_wasm/node/runner/server.cljs b/frontend/performance/benches/render_wasm/node/runner/server.cljs new file mode 100644 index 0000000000..24457ebb36 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/server.cljs @@ -0,0 +1,309 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.server + "An owned loopback HTTP server for one immutable renderer build." + (:require + ["fs" :as fs] + ["http" :as http] + ["path" :as path] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.settings :as settings] + [clojure.string :as str])) + +(def ^:private mime + {".html" "text/html; charset=utf-8" + ".js" "application/javascript" + ".cjs" "application/javascript" + ".mjs" "application/javascript" + ".wasm" "application/wasm" + ".css" "text/css; charset=utf-8" + ".json" "application/json" + ".svg" "image/svg+xml" + ".png" "image/png"}) + +(def ^:private status-text + {400 "bad request" + 403 "forbidden" + 404 "not found" + 405 "method not allowed"}) + +(def ^:private fs-error-status + {"ENOENT" 404 + "ENOTDIR" 404 + "EACCES" 403}) + +(def ^:private owners + "Maps each owned server to {:sockets Set :shutdown atom}." + (js/WeakMap.)) + +;; Request URL to file + +(defn- rejection + "Returns an Error that `error-status` maps to `status`." + [status message] + (ex-info message {:status status})) + +(defn- unsafe-pathname? + [pathname] + (or (str/includes? pathname "\u0000") + (str/includes? pathname "\\") + (some #{".."} (str/split pathname #"/")))) + +(defn- parse-pathname + "Returns the decoded path of a request URL without its query and fragment. + + Throws: + 400 for a path without a leading slash and for a malformed escape. + 403 for a path with a NUL, a backslash or a `..` segment." + [url] + (let [pathname (js/decodeURIComponent (first (str/split url #"[?#]" 2)))] + (cond + (not (str/starts-with? pathname "/")) (throw (rejection 400 "invalid request path")) + (unsafe-pathname? pathname) (throw (rejection 403 "forbidden request path")) + :else pathname))) + +(defn- pathname->relative + [pathname] + (if (= pathname "/") "index.html" (subs pathname 1))) + +(defn- contained + "Returns `file` when it lies within `root`, otherwise throws 403 with `message`." + [root message file] + (if (files/contained? root file) + file + (throw (rejection 403 message)))) + +(defn- request-file + "Returns the real path of the file that `url` selects below `root`. + + Throws: + 400 or 403 for a bad path. + 403 for a path or a symlink that leaves `root`. + The filesystem error when the file cannot be resolved." + [root url] + (->> (parse-pathname url) + (pathname->relative) + (path/resolve root) + (contained root "forbidden request path") + (fs/realpathSync) + (contained root "forbidden artifact symlink"))) + +;; File to response + +(defn- error-status + "Returns the HTTP status for a failure while serving a request." + [error] + (or (:status (ex-data error)) + (when (instance? js/URIError error) 400) + (get fs-error-status (.-code ^js error) 500))) + +(defn- error-response + [method status] + {:status status + :headers {"Content-Type" "text/plain; charset=utf-8"} + :body (when-not (= "HEAD" method) + (get status-text status "server error"))}) + +(defn- method-not-allowed + [method] + (assoc-in (error-response method 405) [:headers "Allow"] "GET, HEAD")) + +(defn- file-response + [method file ^js stat] + {:status 200 + :headers {"Content-Type" (get mime (path/extname file) "application/octet-stream") + "Content-Length" (.-size stat)} + :body (when (= "GET" method) + (fs/readFileSync file))}) + +(defn- get-file-response + [root method url] + (try + (let [file (request-file root url) + ^js stat (fs/statSync file)] + (if (.isFile stat) + (file-response method file stat) + (error-response method 404))) + (catch :default error + (error-response method (error-status error))))) + +(defn- respond + "Returns {:status :headers :body} for a request with `method` and `url`." + [root method url] + (if (contains? #{"GET" "HEAD"} method) + (get-file-response root method url) + (method-not-allowed method))) + +(defn- write-response + [^js response {:keys [status headers body]}] + (.writeHead response status (clj->js headers)) + (.end response body)) + +(defn- serve + [root ^js request ^js response] + (write-response response (respond root (.-method request) (.-url request)))) + +;; Shutdown + +(defn- close-failure + "Returns `error` unless it is nil or reports a server that was not running." + [error] + (when-not (or (nil? error) + (= "ERR_SERVER_NOT_RUNNING" (.-code ^js error))) + error)) + +(defn- close-server + "Closes `server` and destroys its `sockets`. + + Returns a promise that resolves once the server closed. A server that is + not running counts as closed." + [^js server ^js sockets] + (js/Promise. + (fn [resolve reject] + (let [settle (fn [error] + (if-let [failure (close-failure error)] + (reject failure) + (resolve nil)))] + (try + (.close server settle) + (run! #(.destroy ^js %) (array-seq (js/Array.from sockets))) + (catch :default error + (settle error))))))) + +(defn close + "Closes an owned host once and destroys only that server's connections. + + Args: + host: Map returned by `open`. + + Returns: + Promise that resolves once the server closed. Repeated calls return the + same promise, including during startup." + [{:keys [server]}] + (let [{:keys [sockets shutdown]} (.get owners server)] + (or @shutdown + (reset! shutdown (close-server server sockets))))) + +;; Startup + +(defn- track-connection + [^js sockets shutdown ^js socket] + (.add sockets socket) + (.once socket "close" #(.delete sockets socket)) + (when @shutdown + (.destroy socket))) + +(defn- create-host + "Returns {:server :build} for a server of `build`'s files that does not + listen yet." + [build] + (let [root (fs/realpathSync (:dir build)) + ^js server (http/createServer (partial serve root)) + sockets (js/Set.) + shutdown (atom nil)] + (.set owners server {:sockets sockets :shutdown shutdown}) + (.on server "connection" (partial track-connection sockets shutdown)) + {:server server :build build})) + +(defn- aborted? + [^js signal] + (and signal (.-aborted signal))) + +(defn- aborted-error + [] + (ex-info "renderer server startup aborted" {:type ::aborted})) + +(defn- timeout-error + [timeout-ms] + (ex-info "renderer server startup timed out" + {:type ::startup-timeout :timeout-ms timeout-ms})) + +(defn- subscribe-startup + "Routes the end of `server` startup to `succeed` or `fail`. + + `succeed` receives the listening event. `fail` receives the first error + event of `server`, an `::aborted` error when `signal` (optional) aborts and + a `::startup-timeout` error after `timeout-ms`. + + Returns a function that removes the subscriptions." + [^js server signal timeout-ms succeed fail] + (let [on-abort #(fail (aborted-error)) + timer (js/setTimeout #(fail (timeout-error timeout-ms)) timeout-ms)] + (.once server "listening" succeed) + (.once server "error" fail) + (when signal + (.addEventListener ^js signal "abort" on-abort #js {:once true})) + (fn [] + (js/clearTimeout timer) + (.removeListener server "listening" succeed) + (.removeListener server "error" fail) + (when signal + (.removeEventListener ^js signal "abort" on-abort))))) + +(defn- ^:async listen + "Waits until `server` listens on 127.0.0.1 and an ephemeral port. + + Aborting `close-signal` closes the server. Rejects as `subscribe-startup` + describes, and when `signal` is already aborted, without listening." + [^js server close-signal signal timeout-ms] + (when (aborted? signal) + (throw (aborted-error))) + ;; Nothing awaits before the subscription, so an abort that follows the + ;; call reaches it. + (let [^js startup (js/Promise.withResolvers) + release (subscribe-startup server signal timeout-ms + (.-resolve startup) (.-reject startup))] + (try + (.listen server #js {:host "127.0.0.1" :port 0 :signal close-signal}) + (await (.-promise startup)) + (finally + (release))))) + +(defn- ^:async startup-failure + "Closes `host` after a failed startup. + + Returns `error`, or an Error that also holds the failure of the close." + [host ^js controller error] + (.abort controller) + (try + (await (close host)) + error + (catch :default close-error + (errors/combine + [{:phase "server-startup" :operation :listen :error error} + {:phase "cleanup" :operation :close-server :error close-error}])))) + +(defn- base-url + [^js server] + (str "http://127.0.0.1:" (.-port (.address server)))) + +(defn ^:async open + "Starts serving build's directory on 127.0.0.1 and an ephemeral port. + + Args: + build: Map with the directory to serve at `:dir`. + options: Optional map with `:startup-timeout-ms` (30000 by default) and + `:signal`, an AbortSignal. + + Returns: + Promise of {:server NodeHttpServer :base-url string :build build}. + + Throws: + Rejects on abort, timeout and listen errors, after closing the owned + server. A failed close joins the startup error." + ([build] (open build {})) + ([build {:keys [startup-timeout-ms signal] + :or {startup-timeout-ms settings/default-startup-timeout-ms}}] + (let [host (create-host build) + controller (js/AbortController.)] + (try + (await (listen (:server host) (.-signal controller) signal startup-timeout-ms)) + (assoc host :base-url (base-url (:server host))) + (catch :default error + (let [failure (await (startup-failure host controller error))] + (throw failure))))))) diff --git a/frontend/performance/benches/render_wasm/node/settings.cljs b/frontend/performance/benches/render_wasm/node/settings.cljs new file mode 100644 index 0000000000..100d8ac791 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/settings.cljs @@ -0,0 +1,14 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.settings + "Shared operational settings and guidance.") + +(def default-build-timeout-ms 300000) +(def default-startup-timeout-ms 30000) + +(def gpu-guidance + "Start the container with ./manage.sh run-devenv --gpu and use pnpm run render-bench.") diff --git a/frontend/performance/benches/render_wasm/node/storage/artifacts.cljs b/frontend/performance/benches/render_wasm/node/storage/artifacts.cljs new file mode 100644 index 0000000000..61e53643d2 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/storage/artifacts.cljs @@ -0,0 +1,403 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.storage.artifacts + "Managed renderer builds and output homes for benchmark runs and comparisons. + + Here, 'managed' means that the runner owns the build directory and its + contents, which can be deleted by means of the `purge` command." + (:require + ["fs" :as fs] + ["path" :as path] + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.time :as time] + [clojure.string :as str])) + +(def schema:build + "Stored renderer build identity, lifecycle and provenance." + [:map + [:type [:= :renderer-build]] + [:id [:string {:min 1}]] + [:status [:enum :building :ready :failed]] + [:created-at :string] + [:updated-at :string] + [:environment {:optional true} [:map-of :string :string]] + [:failure {:optional true} errors/schema:diagnostic] + [:functional :map] + [:provenance :map]]) + +(def ^:private executable-dir + ;; CJS boots use __dirname. Node's ESM test boots have an absolute argv[1] + ;; instead; capture it before a caller changes cwd. Neither uses a cwd root. + (if (exists? js/__dirname) + js/__dirname + (let [script (aget js/process.argv 1)] + (when (and (string? script) (path/isAbsolute script)) + (path/dirname script))))) + +(defn frontend-root + "Finds the frontend checkout by walking up from the Node executable. + + Supports target/performance/render-wasm/runner.cjs and the ESM + target/{performance-tests,tests}/test.js boots, independent of caller cwd." + [] + (when-not executable-dir + (throw (ex-info "cannot locate the Node executable directory" + {:type ::missing-executable-dir}))) + (loop [dir (path/resolve executable-dir)] + (let [deps (path/join dir "deps.edn") + scenes (path/join dir "performance" "benches" "render_wasm") + parent (path/dirname dir)] + (cond + (and (fs/existsSync deps) (.isFile ^js (fs/statSync deps)) + (fs/existsSync scenes) (.isDirectory ^js (fs/statSync scenes))) + dir + + (= parent dir) + (throw (ex-info "cannot locate the frontend checkout" + {:type ::missing-frontend-root + :executable-dir executable-dir})) + + :else + (recur parent))))) + +(defn valid-id? + "True for IDs of up to 128 letters, digits, underscores or hyphens, + starting with a letter or digit." + [id] + (and (string? id) + (boolean (re-matches #"[A-Za-z0-9][A-Za-z0-9_-]{0,127}" id)))) + +(defn- check-id + [id] + (when-not (valid-id? id) + (throw (ex-info "invalid renderer build id" {:type ::invalid-id :id id}))) + id) + +(defn invalid-home + "Throws an ::invalid-home exception with `data` and an optional cause." + ([message data] + (throw (ex-info message (assoc data :type ::invalid-home)))) + ([message data cause] + (throw (ex-info message (assoc data :type ::invalid-home) cause)))) + +(defn check-home-id + "Returns `id` when it is valid, or throws ::invalid-home with `message`." + [id message] + (when-not (valid-id? id) + (invalid-home message {:id id})) + id) + +(defn check-output-dir + "Resolves an explicit user directory from the caller's cwd and checks that it + does not exist. Returns nil for nil. Does not create anything." + [dir] + (when (some? dir) + (when (str/blank? dir) + (invalid-home "output directory requires a nonempty path" {})) + (let [resolved (path/resolve dir) + exists? (try + (fs/lstatSync resolved #js {:throwIfNoEntry false}) + (catch :default cause + (let [detail (errors/message-of cause)] + (invalid-home (str "cannot inspect output directory: " detail) + {:path resolved :cause detail} + cause))))] + (when exists? + (invalid-home "output directory already exists" {:path resolved})) + resolved))) + +(defn- create-home* + [dir stem] + (let [log-path (path/join dir (str stem ".log"))] + (fs/mkdirSync dir #js {:recursive true}) + (fs/writeFileSync log-path "" #js {:encoding "utf8" :flag "wx"}) + {:dir dir + :log-path log-path + :record-path (path/join dir (str stem ".transit"))})) + +(defn create-home + "Creates `dir` and its parents, reserves an empty log. + + A filesystem failure throws ::invalid-home with `message` and `:path`, and + keeps the original cause. + + Args: + dir: path to create. + stem: names the .log and .transit files. + message: used for errors + + Returns: + Directory created, log path and record path." + [dir stem message] + (try + (create-home* dir stem) + (catch :default cause + (invalid-home message + {:path dir :cause (errors/message-of cause)} + cause)))) + +(defn- checked-path + [root file] + (let [resolved (fs/realpathSync file)] + (when-not (files/contained? root resolved) + (throw (ex-info "renderer artifact path escapes its managed directory" + {:type ::unsafe-path :path file}))) + resolved)) + +(defn managed-root + "Returns the managed path under an existing frontend checkout. + + Checks its nearest existing ancestor before creating or removing paths, + so a managed-root or ancestor symlink cannot escape the frontend root." + [root] + (let [frontend (fs/realpathSync root) + managed (path/resolve root "target" "performance" "render-wasm")] + (loop [existing managed] + (if (fs/existsSync existing) + (checked-path frontend existing) + (recur (path/dirname existing)))) + managed)) + +(defn inside-builds? + "Checks the whole builds tree, resolving symlink ancestors of both paths." + [frontend-root output-path] + (files/contained? (files/get-real-path (path/join (managed-root frontend-root) "builds")) + (files/get-real-path output-path))) + +(defn- build-reference + [root id manifest] + (let [dir (path/join (managed-root root) "builds" id)] + {:id id + :dir dir + :manifest-path (path/join dir "build.transit") + :log-path (path/join dir "build.log") + :cases-path (path/join dir "cases.cjs") + :browser-dir (path/join dir "browser") + :renderer-dir (path/join dir "renderer") + :index-path (path/join dir "index.html") + :manifest manifest})) + +(defn create-build + "Allocates a UUID build and saves its :building build.transit and empty log. + + - root: the frontend checkout. + - selection: {:default-features? boolean :features vector} + - provenance: {:git {:sha string-or-nil :dirty boolean-or-nil}}. + + Returns a build reference. + " + [root selection provenance] + (let [id (str (random-uuid)) + now (time/now) + manifest {:type :renderer-build + :id id + :status :building + :created-at now + :updated-at now + :functional {:mode :release + :default-features? (:default-features? selection) + :features (:features selection) + :cljs-profile-marks false} + :provenance provenance} + build (build-reference root id manifest)] + (fs/mkdirSync (path/dirname (:dir build)) #js {:recursive true}) + (checked-path (fs/realpathSync (managed-root root)) (path/dirname (:dir build))) + (fs/mkdirSync (:dir build)) + (fs/mkdirSync (:browser-dir build)) + (fs/mkdirSync (:renderer-dir build)) + (fs/writeFileSync (:log-path build) "" "utf8") + (files/atomic-write (:manifest-path build) (codec/encode-str manifest)) + build)) + +(defn write-manifest + "Atomically replaces a manifest and returns the updated build reference. + + Only :building builds can change. An unchanged terminal manifest is a + no-op; :ready and :failed builds cannot otherwise change. + Read the saved status so an old build reference cannot reopen a build." + [build manifest] + (let [current (transit/decode-str (fs/readFileSync (:manifest-path build) "utf8"))] + (when-not (and (= (:id build) (:id current) (:id manifest)) + (sm/validate schema:build current) + (sm/validate schema:build manifest)) + (throw (ex-info "invalid schema:build" + {:type ::invalid-manifest :id (:id build)}))) + (if (contains? #{:ready :failed} (:status current)) + (if (= current manifest) + (assoc build :manifest current) + (throw (ex-info "renderer build is already terminal" + {:type ::terminal-build + :id (:id build) + :status (:status current)}))) + (let [manifest (assoc manifest :updated-at (time/now))] + (files/atomic-write (:manifest-path build) (codec/encode-str manifest)) + (assoc build :manifest manifest))))) + +(defn- require-file + [root file] + (let [resolved (checked-path root file)] + (when-not (.isFile ^js (fs/statSync resolved)) + (throw (ex-info "renderer artifact is not a file" + {:type ::missing-artifact :path file}))) + resolved)) + +(defn check-files + "Checks required files and selected directory containment; returns build. + + build is a reference from create-build or load-build. This check neither + reads nor changes manifest status. The build helper must also check catalog + and browser identities before writing its terminal :ready manifest." + [build] + ;; Build directories use /target/performance/render-wasm/builds/. + (let [root (path/resolve (:dir build) ".." ".." ".." ".." "..") + managed (fs/realpathSync (managed-root root)) + dir (checked-path managed (:dir build))] + (require-file dir (:cases-path build)) + (require-file dir (path/join (:browser-dir build) "browser.js")) + (require-file dir (path/join (:renderer-dir build) "render-wasm.js")) + (require-file dir (path/join (:renderer-dir build) "render-wasm.wasm")) + (require-file dir (path/join (:renderer-dir build) "shared.js")) + (require-file dir (:index-path build)) + build)) + +(defn load-build + "Loads a ready schema:build and checks every required artifact. + + IDs cannot contain path segments. The build, manifest and artifact paths + must remain inside the managed root and the selected build after resolving + symlinks. Building, failed, incomplete and unsupported builds are rejected." + [root id] + (check-id id) + (let [managed (fs/realpathSync (managed-root root)) + build (build-reference root id nil) + dir (checked-path managed (:dir build)) + file (require-file dir (:manifest-path build)) + manifest (transit/decode-str (fs/readFileSync file "utf8"))] + (when-not (and (sm/validate schema:build manifest) + (= id (:id manifest))) + (throw (ex-info "invalid schema:build" + {:type ::unsupported-manifest :id id}))) + (when-not (= :ready (:status manifest)) + (throw (ex-info "renderer build is not ready" + {:type ::not-ready :id id :status (:status manifest)}))) + (check-files (assoc build :manifest manifest)))) + +(def ^:private build-outputs + ["build.transit" "build.log" "cases.cjs" "index.html" + "browser/browser.js" "renderer/render-wasm.js" "renderer/render-wasm.wasm" + "renderer/render-wasm.wasm.map" "renderer/shared.js" "renderer/cargo-build.json" + "renderer/worker/render.js"]) + +(defn- directory? + [file] + (and (fs/existsSync file) (.isDirectory (fs/statSync file)))) + +(defn- unlink-if-present + [file] + (when-not (directory? file) + (try + (fs/unlinkSync file) + (catch :default error + (when-not (contains? #{"ENOENT" "ENOTDIR"} (.-code error)) + (throw error)))))) + +(defn- clean-build + [dir] + (when (directory? dir) + (doseq [file build-outputs] + (unlink-if-present (path/join dir file))))) + +(defn- purge-builds + [root] + (when (directory? root) + (doseq [id (array-seq (fs/readdirSync root))] + (clean-build (path/join root id))))) + +(defn- purge-runs + [root] + (loop [pending [root] + visited #{}] + (when-let [dir (peek pending)] + (let [pending (pop pending) + real (when (directory? dir) (fs/realpathSync dir))] + (if (or (nil? real) (contains? visited real)) + (recur pending visited) + (do + (unlink-if-present (path/join dir "run.log")) + (let [names (array-seq (fs/readdirSync dir))] + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".png")] + (unlink-if-present (path/join dir file))) + (unlink-if-present (path/join dir "run.transit")) + (recur (into pending (map #(path/join dir %) names)) + (conj visited real))))))))) + +(defn- purge-feature-comparisons + "Unlinks feature-comparison records and logs, configuration run evidence and + edge reports from the output directory. + + Keeps directories, builds and unrelated files." + [root] + (loop [pending [root] + visited #{}] + (when-let [dir (peek pending)] + (let [pending (pop pending) + real (when (directory? dir) (fs/realpathSync dir))] + (if (or (nil? real) (contains? visited real)) + (recur pending visited) + (let [parent (path/basename (path/dirname dir)) + base (path/basename dir) + names (array-seq (fs/readdirSync dir))] + (when (= "feature-comparisons" parent) + (unlink-if-present (path/join dir "feature-comparison.log")) + (unlink-if-present (path/join dir "feature-comparison.transit"))) + (when (= "configs" parent) + (unlink-if-present (path/join dir "run.log")) + (unlink-if-present (path/join dir "run.transit"))) + (when (= "comparisons" base) + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".transit")] + (unlink-if-present (path/join dir file)))) + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".png")] + (unlink-if-present (path/join dir file))) + (recur (into pending (map #(path/join dir %) names)) + (conj visited real)))))))) + +(defn purge + "Unlinks known outputs from :builds, :runs, :comparisons or all of them (:all). + + The caller must be idle: no build, server, run or comparison may still own + these paths. Follows directory symlinks; keeps directories and unrelated + files." + [root category] + (let [managed (managed-root root)] + (case category + :builds (purge-builds (path/join managed "builds")) + :runs (purge-runs (path/join managed "runs")) + :comparisons (purge-feature-comparisons + (path/join managed "feature-comparisons")) + :all (do + (purge-builds (path/join managed "builds")) + (purge-runs (path/join managed "runs")) + (purge-feature-comparisons + (path/join managed "feature-comparisons"))) + (throw (ex-info "invalid renderer artifact category" + {:type ::invalid-category :category category}))))) + +(defn remove-builds + "Unlinks known outputs of selected build IDs. The caller must be idle." + [root ids] + (let [ids (mapv check-id ids) + managed (managed-root root) + builds (path/join managed "builds")] + (doseq [id ids] + (clean-build (path/join builds id))))) diff --git a/frontend/performance/benches/render_wasm/node/storage/runs.cljs b/frontend/performance/benches/render_wasm/node/storage/runs.cljs new file mode 100644 index 0000000000..1dbe608070 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/storage/runs.cljs @@ -0,0 +1,78 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.storage.runs + "Manages saved run destinations, schema:run persistence and diagnostic logs. + + Runs are saved under /target/performance/render-wasm/runs. + + --output-dir selects another destination. Purge cleans files reachable from + the default runs tree, including explicit destinations and directory links." + (:require + ["fs" :as fs] + ["path" :as path] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.result :as result] + [clojure.string :as str])) + +(defn create-home + "Creates one run home and its empty log. The record is written separately + after the plan exists. An explicit directory must not exist; a managed home + uses the frontend root regardless of cwd." + [{:keys [root run-id output-dir remove]}] + (artifacts/check-home-id run-id "invalid run id") + (let [dir (artifacts/check-output-dir + (or output-dir (path/join (artifacts/managed-root root) "runs" run-id)))] + (when (and remove (artifacts/inside-builds? root dir)) + (artifacts/invalid-home "--remove cannot use an output directory inside builds" {:path dir})) + (assoc (artifacts/create-home dir "run" "cannot create run home") :run-id run-id))) + +(defn get-record + "Reads and validates a saved run record. + + Args: + file: Input path, resolved relative to cwd. + options: Optional :role identifying the input within an operation. + + Returns: + The validated run record. + + Throws: + An error retaining its message, type and cause, with :path, :operation + and the supplied :role in ex-data." + ([file] + (get-record file {})) + ([file {:keys [role]}] + (try + (when-not (and (string? file) (fs/existsSync file)) + (throw (ex-info "run record is missing" {:type ::missing-run}))) + (let [record (result/decode (fs/readFileSync file "utf8"))] + (when-not (= :run-record (:type record)) + (throw (ex-info "expected a saved run record" {:type ::invalid-run}))) + record) + (catch :default cause + (throw (errors/with-data + (errors/as-failure cause) + (cond-> {:path file :operation :read-record} + role (assoc :role role)))))))) + +(defn write-record! + "Validates schema:run and atomically replaces the saved file." + [home run] + (files/atomic-write (:record-path home) (result/encode run))) + +(defn screenshot-file + "Returns the relative PNG filename for a case, replacing path separators." + [case-id] + (str (str/replace (result/case-name case-id) #"[/\\]" "-") ".png")) + +(defn screenshot-path + "Returns the relative filename and absolute path under the run home." + [home case-id] + (let [file (screenshot-file case-id)] + {:file file :path (path/join (:dir home) file)})) diff --git a/frontend/performance/benches/render_wasm/node/time.cljs b/frontend/performance/benches/render_wasm/node/time.cljs new file mode 100644 index 0000000000..db0c9aff34 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/time.cljs @@ -0,0 +1,12 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.time) + +(defn now + "Returns the current UTC time as an ISO 8601 string." + [] + (.toISOString (js/Date.))) diff --git a/frontend/performance/benches/render_wasm/quantities.cljs b/frontend/performance/benches/render_wasm/quantities.cljs new file mode 100644 index 0000000000..ea114106ed --- /dev/null +++ b/frontend/performance/benches/render_wasm/quantities.cljs @@ -0,0 +1,25 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.quantities + "Portable descriptions of raw measured quantities.") + +(def schema:description + [:map {:closed true} + [:unit keyword?] + [:description {:optional true} string?]]) + +(def descriptions + {:command/elapsed + {:unit :ms :description "Command core start through its completion endpoint."} + :renderer/call-elapsed + {:unit :ms :description "Elapsed time of each synchronous renderer call."} + :renderer/cache-call-elapsed + {:unit :ms :description "Elapsed time of each synchronous cached renderer call."} + :renderer/prepared-scene-to-full + {:unit :ms :description "Prepared scene render request through the renderer return at Full."} + :camera/finalization-to-full + {:unit :ms :description "Camera finalization through the renderer return at Full."}}) diff --git a/frontend/performance/benches/render_wasm/random.cljs b/frontend/performance/benches/render_wasm/random.cljs new file mode 100644 index 0000000000..2ad63e7a80 --- /dev/null +++ b/frontend/performance/benches/render_wasm/random.cljs @@ -0,0 +1,73 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.random + "Seeded numbers for reproducible scene construction and report resampling. + + A seed is an integer from 0 through 4294967295. Each call to `create` owns a + separate Mulberry32 state. Calling the returned function advances that state + and returns a number in [0,1). Equal seeds and equal numbers of calls produce + equal sequences, including seed zero." + (:require + [app.common.types.color :as clr] + [app.common.uuid :as uuid])) + + +(defn seed? + "Returns true for integers from 0 through 4294967295, inclusive." + [value] + (and (integer? value) (<= 0 value 4294967295))) + +(defn create + "Returns a function that draws seeded numbers from [0, 1) using Mulberry32. + Each function owns its state. A seed outside the uint32 range throws + ::invalid-seed; uint32 means an integer from 0 through 4294967295." + [seed] + (when-not (seed? seed) + (throw (ex-info "random seed must be an integer in [0, 2^32)" + {:type ::invalid-seed :seed seed}))) + (let [state (volatile! (bit-or seed 0))] + (fn [] + (let [s (bit-or (+ @state 0x6d2b79f5) 0) + _ (vreset! state s) + v (js/Math.imul (bit-xor s (unsigned-bit-shift-right s 15)) + (bit-or s 1)) + v (bit-xor v (+ v (js/Math.imul (bit-xor v (unsigned-bit-shift-right v 7)) + (bit-or v 61)))) + v (bit-xor v (unsigned-bit-shift-right v 14))] + (/ (unsigned-bit-shift-right v 0) 4294967296))))) + +(defn round3 + "Rounds to 3 decimal places" + [value] + (/ (js/Math.round (* 1000 value)) 1000)) + + +(defn rng-float + "Draws a float in `[min, max)` from the random source." + [rng min max] + (+ min (* (rng) (- max min)))) + +(defn rng-int + "Draws an integer in `[min, max)` from the random source." + [rng min max] + (js/Math.floor (rng-float rng min max))) + +(defn rng-uuid + "Draws a deterministic uuid from two 32-bit values of the random source." + [rng] + (uuid/custom (rng-int rng 0 4294967296) + (rng-int rng 0 4294967296))) + +(defn rng-hex-color + "Draws a hex color from three RGB channels in [0,256)." + [rng] + (clr/rgb->hex [(rng-int rng 0 256) (rng-int rng 0 256) (rng-int rng 0 256)])) + +(defn rng-opacity + "Draws opacity in [0.1,0.999], rounded to three decimal places." + [rng] + (min 0.999 (round3 (rng-float rng 0.1 1)))) diff --git a/frontend/performance/benches/render_wasm/report/analysis.cljs b/frontend/performance/benches/render_wasm/report/analysis.cljs new file mode 100644 index 0000000000..ef2e11c958 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/analysis.cljs @@ -0,0 +1,241 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.analysis + "Calculates statistics and median changes for benchmark measurements. + + summarize/summarize-metric passes one vector to `statistics`. An attempt is + one execution of a benchmark case. Each value represents one attempt, including + a batch average: the attempt's total divided by its number of inner calls. + compare/compare-runs passes baseline and candidate vectors to changes. + Those callers exclude warmups, failed attempts and invalid measurements, + and supply already validated finite values and resolved settings. + + Report fields use maps: {:value number} contains a calculated number; + {:lower number :upper number} contains interval bounds; {:reason keyword} + explains why a valid input cannot supply that result. For example, + :insufficient-observations can include :minimum, the required sample size. + This contract preserves unavailable fields through Transit and lets + report.format display the reason. quantile and resolve-settings reject invalid + arguments at their boundaries. + + Resampling draws a sample of the original size, allowing repeated values. + Repeating this process gives medians whose central :confidence fraction + defines a percentile confidence interval. These intervals describe the + supplied measurements; evolving caches limit their interpretation." + (:require + [app.common.schema :as sm] + [benches.render-wasm.random :as random])) + +(def defaults + "Default settings for report calculations. + :draws is the number of replacement samples; :seed controls their generator. + :confidence is the fraction between interval bounds. :min-ci, :min-p95 and + :min-p99 set the required observation counts for those results." + {:draws 2000 + :seed 0x70656e70 + :confidence 0.95 + :min-ci 10 + :min-p95 100 + :min-p99 500}) + +(def schema:settings + "Valid settings for sample draws, seed and report limits. + The seed is an integer in [0, 2^32); confidence lies strictly between 0 and 1. + Draw and minimum observation counts are positive integers." + [:map {:closed true} + [:draws [:int {:min 1 :max 2147483647}]] + [:seed [:fn random/seed?]] + [:confidence [:fn #(and (number? %) (js/Number.isFinite %) (< 0 % 1))]] + [:min-ci [:int {:min 1}]] + [:min-p95 [:int {:min 1}]] + [:min-p99 [:int {:min 1}]]]) + +(defn resolve-settings + "Returns defaults merged with supplied overrides, or defaults for nil. + Accepts a complete settings map too. Invalid values or unknown keys throw + ::invalid-settings. Callers store this resolved map with their results." + [overrides] + (let [settings (merge defaults overrides)] + (when-not (sm/validate schema:settings settings) + (throw (ex-info "invalid analysis settings" {:type ::invalid-settings + :settings settings}))) + settings)) + +(defn- finite-number? + "Returns true for numbers other than NaN and positive or negative infinity." + [value] + (and (number? value) (js/Number.isFinite value))) + +(defn- observation-vector + "Returns a vector of supplied finite measurements, or an empty vector for nil. + A non-sequence or any non-finite/non-numeric entry throws ::invalid-observations. + Report callers exclude unusable raw entries before requesting calculations." + [values] + (when-not (and (or (nil? values) (sequential? values)) + (every? finite-number? values)) + (throw (ex-info "observations must be a sequence of finite numbers" + {:type ::invalid-observations :values values}))) + (vec values)) + +(defn- sorted-quantile + "Returns an interpolated value from a nonempty vector sorted in ascending order. + Probability 0 selects the minimum, 1 the maximum and 0.5 the median. + At position (count - 1) * probability, a fractional index gives the weighted + average of its neighboring values. Callers validate the probability." + [sorted probability] + (let [position (* (dec (count sorted)) probability) + lower (js/Math.floor position) + upper (js/Math.ceil position) + weight (- position lower)] + (+ (* (- 1 weight) (nth sorted lower)) + (* weight (nth sorted upper))))) + +(defn quantile + "Returns an interpolated quantile as a number, or nil for an empty sequence. + Probability 0 selects the minimum, 1 the maximum and 0.5 the median. + For example, [1 2 3 4] at probability 0.25 gives 1.75. + Values must form a sequence of finite numbers. Probability must be a finite + number in [0, 1]. Invalid arguments throw ::invalid-observations or + ::invalid-probability; report callers handle unavailable results separately." + [values probability] + (when-not (and (finite-number? probability) (<= 0 probability 1)) + (throw (ex-info "quantile probability must be a finite number in [0, 1]" + {:type ::invalid-probability :probability probability}))) + (let [values (observation-vector values)] + (when (seq values) + (sorted-quantile (vec (sort values)) probability)))) + +(defn- number-result + "Returns {:value value} for a finite calculated number. + Non-finite arithmetic yields {:reason :undefined-estimate}, so a report can + retain the field and explain why it has no number." + [value] + (if (finite-number? value) + {:value value} + {:reason :undefined-estimate})) + +(defn- resampled-median + "Returns the median of one replacement sample drawn from a nonempty vector. + The sample has the same size as values; repeated indices are allowed. + rng is a function that draws a number from [0, 1) on each call." + [values rng] + (let [n (count values) + sample (vec (sort (repeatedly n #(nth values (js/Math.floor (* n (rng)))))))] + (sorted-quantile sample 0.5))) + +(defn- interval + "Returns bounds containing the central confidence fraction of calculated draws. + For confidence 0.95, the bounds are the 0.025 and 0.975 quantiles. + Empty or non-finite draws yield {:reason :undefined-bootstrap-estimate}; + callers retain that reason rather than discarding an undefined draw." + [values confidence] + (if (and (seq values) (every? finite-number? values)) + (let [sorted (vec (sort values)) + tail (/ (- 1 confidence) 2) + lower (sorted-quantile sorted tail) + upper (sorted-quantile sorted (- 1 tail))] + (if (and (finite-number? lower) (finite-number? upper)) + {:lower lower :upper upper} + {:reason :undefined-bootstrap-estimate})) + {:reason :undefined-bootstrap-estimate})) + +(defn- quantile-result + "Returns a quantile in {:value number} when the sorted sample reaches minimum. + Otherwise returns {:reason :insufficient-observations :minimum minimum}. + Probability selects the quantile, such as 0.95 for p95." + [sorted probability minimum] + (if (< (count sorted) minimum) + {:reason :insufficient-observations :minimum minimum} + (number-result (sorted-quantile sorted probability)))) + +(defn statistics + "Returns statistics for one vector of finite measurement values. + + `values` must be an already validated vector of finite measurements, and + `settings` must be the complete map returned by `resolve-settings`. + Callers enforce these preconditions. + `:count` is the number of original measurements, including each batch average + once. Numeric fields contain {:value number}; unavailable fields contain + {:reason keyword}, with :minimum for an observation requirement. + + :sample-sd is sqrt(sum((x - mean)^2) / (count - 1)). + :mad is the median absolute distance from the median, without scaling. + :p95 and :p99 are observed 0.95 and 0.99 quantiles. :median-ci contains + bounds from repeatedly drawing replacement samples and taking their medians. + Empty input retains every field with :no-valid-observations. + Insufficient counts and undefined arithmetic retain reasons." + [values settings] + (let [{:keys [draws seed confidence min-ci min-p95 min-p99]} settings + n (count values)] + (if (zero? n) + (let [absent {:reason :no-valid-observations}] + {:count 0 :mean absent :median absent :min absent :max absent + :sample-sd absent :mad absent :median-ci absent :p95 absent :p99 absent}) + (let [sorted (vec (sort values)) + mean (reduce (fn [total x] (+ total (/ x n))) 0 values) + median (sorted-quantile sorted 0.5) + rng (random/create seed)] + {:count n + :mean (number-result mean) + :median (number-result median) + :min (number-result (first sorted)) + :max (number-result (peek sorted)) + :sample-sd (if (= n 1) + {:reason :insufficient-observations :minimum 2} + (number-result (js/Math.sqrt + (reduce (fn [total x] + (+ total (/ (* (- x mean) (- x mean)) (dec n)))) + 0 values)))) + :mad (number-result (sorted-quantile (vec (sort (map #(js/Math.abs (- % median)) values))) 0.5)) + :median-ci (if (< n min-ci) + {:reason :insufficient-observations :minimum min-ci} + (interval (vec (repeatedly draws #(resampled-median values rng))) + confidence)) + :p95 (quantile-result sorted 0.95 min-p95) + :p99 (quantile-result sorted 0.99 min-p99)})))) + +(defn changes + "Returns candidate-minus-baseline changes between the two observed medians. + Inputs must be already validated vectors of finite measurements; settings + must be the complete map returned by resolve-settings. Callers enforce these + preconditions. :absolute and :percentage contain + {:value number}; :absolute-ci and :percentage-ci contain {:lower number :upper number}. + Any unavailable result carries {:reason keyword} instead. + + Each interval draw samples both inputs independently, allowing repeated + measurements, then calculates the change between those two sample medians. + Both original counts must reach :min-ci. A zero observed baseline suppresses + percentages; any zero resampled baseline suppresses the whole percentage + interval. Empty inputs or undefined arithmetic retain reasons. + compare/compare-runs calls this once per shared metric." + [baseline candidate settings] + (let [{:keys [draws seed confidence min-ci]} settings] + (if (or (empty? baseline) (empty? candidate)) + {:absolute {:reason :no-valid-observations} + :percentage {:reason :no-valid-observations} + :absolute-ci {:reason :no-valid-observations} + :percentage-ci {:reason :no-valid-observations}} + (let [b (sorted-quantile (vec (sort baseline)) 0.5) + c (sorted-quantile (vec (sort candidate)) 0.5) + supported (and (>= (count baseline) min-ci) (>= (count candidate) min-ci)) + rng (random/create seed) + samples (when supported + (vec (repeatedly draws + (fn [] [(resampled-median baseline rng) + (resampled-median candidate rng)])))) + low-count {:reason :insufficient-observations :minimum min-ci}] + {:absolute (number-result (- c b)) + :percentage (if (zero? b) {:reason :zero-baseline} (number-result (* 100 (/ (- c b) b)))) + :absolute-ci (if supported + (interval (mapv (fn [[b c]] (- c b)) samples) confidence) + low-count) + :percentage-ci (cond + (zero? b) {:reason :zero-baseline} + (not supported) low-count + (some (fn [[b _]] (zero? b)) samples) {:reason :zero-bootstrap-baseline} + :else (interval (mapv (fn [[b c]] (* 100 (/ (- c b) b))) samples) + confidence))})))) diff --git a/frontend/performance/benches/render_wasm/report/compare.cljs b/frontend/performance/benches/render_wasm/report/compare.cljs new file mode 100644 index 0000000000..904ebb48b3 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/compare.cljs @@ -0,0 +1,281 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.compare + "Offline comparisons of paired cases from two saved schema:run records. + + Matching case identifiers select default pairs; explicit pairs only select + cases. Permissions acknowledge recorded differences without hiding them." + (:require + [benches.render-wasm.report.analysis :as analysis] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [clojure.set :as set] + [clojure.string :as str])) + +(defn resolve-case + "Resolves an identifier or exact name only from the saved plan. + Unknown or ambiguous selectors report the available names for that file." + [run selector] + (let [ids (mapv :id (get-in run [:plan :cases])) + matches (if (some #(= selector %) ids) + [selector] + (filterv #(= selector (result/case-name %)) ids))] + (if (= 1 (count matches)) + (first matches) + (throw (ex-info (str "cannot resolve saved case " (pr-str selector) + "; available cases: " (str/join ", " (map result/case-name ids))) + {:type ::unknown-case :selector selector :available (mapv result/case-name ids)}))))) + +(defn selector-pair + "Resolves baseline and candidate selectors into an extensible pair list." + [baseline candidate [baseline-selector candidate-selector]] + [[(resolve-case baseline baseline-selector) (resolve-case candidate candidate-selector)]]) + +(defn- difference + "Returns one compatibility difference when recorded values disagree." + [field baseline candidate] + (when (not= baseline candidate) + {:field field :baseline baseline :candidate candidate :waived? false})) + +(defn- preparations + "Returns chronological preparation evidence for an exact saved case." + [run id] + (filterv #(= id (:case-id %)) (:preparations run))) + +(defn- preparation-facts + "Collects distinct recorded fingerprints or graphics, independent of prep IDs." + [run id field] + (set (map field (preparations run id)))) + +(defn- scene-settings + "Selects scene identity and generation inputs, separate from execution." + [definition] + (select-keys definition [:scene :scene-seed :params])) + +(defn- execution-settings + "Selects view, context and preparation settings for execution compatibility." + [definition] + (select-keys definition [:view :context :preparation])) + +(defn- evidence-differences + "Checks cross-run equality plus missing or varying preparation evidence." + [field baseline candidate] + (concat + (when-let [d (difference field baseline candidate)] [d]) + (when (or (not= 1 (count baseline)) (not= 1 (count candidate)) + (contains? baseline nil) (contains? candidate nil)) + [{:field [field :incomplete-or-varying-evidence] + :baseline baseline :candidate candidate :waived? false}]))) + +(defn- feature-names? + "Checks the vector-of-strings shape of recorded or expected feature names." + [features] + (and (vector? features) (every? string? features))) + +(defn- get-effective-feature-evidence + "Returns comparison values and eligibility for automatic feature acknowledgment. + Missing, nil and explicitly empty features remain distinct. Collections compare + as sets, but only saved vectors of strings qualify for automatic acknowledgment." + [run] + (let [features (get-in run [:metadata :build :functional :features] ::missing-features)] + {:value (if (coll? features) (set features) features) + :valid (feature-names? features)})) + +(defn- pair-differences + "Checks saved inputs, graphics, environment, build, sampling and metric definitions." + [baseline candidate b c shared] + (let [bdef (some #(when (= b (:id %)) %) (get-in baseline [:plan :cases])) + cdef (some #(when (= c (:id %)) %) (get-in candidate [:plan :cases]))] + (vec + (concat + (keep identity + [(difference :scene (scene-settings bdef) (scene-settings cdef)) + (difference :execution (execution-settings bdef) (execution-settings cdef)) + (difference :environment (get-in baseline [:metadata :environment]) + (get-in candidate [:metadata :environment])) + (difference :effective-features (:value (get-effective-feature-evidence baseline)) + (:value (get-effective-feature-evidence candidate))) + (difference :functional-build (dissoc (get-in baseline [:metadata :build :functional]) :features) + (dissoc (get-in candidate [:metadata :build :functional]) :features)) + (difference :build-environment (get-in baseline [:metadata :build :environment]) + (get-in candidate [:metadata :build :environment])) + (difference :warmups (get-in baseline [:plan :warmups]) + (get-in candidate [:plan :warmups])) + (difference :timeout-policy (get-in baseline [:plan :timeout-policy]) + (get-in candidate [:plan :timeout-policy])) + (difference :scored? (get-in baseline [:metadata :scored?]) + (get-in candidate [:metadata :scored?]))]) + (evidence-differences :fingerprint (preparation-facts baseline b :fingerprint) + (preparation-facts candidate c :fingerprint)) + (evidence-differences :effective-graphics (preparation-facts baseline b :effective-graphics) + (preparation-facts candidate c :effective-graphics)) + (mapcat (fn [metric-id] + (let [bdefs (map #(dissoc % :description) (summary/definitions baseline b metric-id)) + cdefs (map #(dissoc % :description) (summary/definitions candidate c metric-id))] + (evidence-differences [:metric metric-id] (set bdefs) (set cdefs)))) + shared))))) + +(defn- feature-change-evidence? + "Checks feature facts against saved effective features and requested selections. + Both selections must record one off-to-on requested toggle with equal default + policy. Effective transitive additions/removals remain unrestricted, but must + match the supplied facts." + [baseline candidate {:keys [feature baseline-features candidate-features added removed]}] + (let [b (get-effective-feature-evidence baseline) + c (get-effective-feature-evidence candidate) + b-request (get-in baseline [:metadata :build :provenance :feature-selection]) + c-request (get-in candidate [:metadata :build :provenance :feature-selection])] + (and (:valid b) (:valid c) + (string? feature) + (every? feature-names? + [baseline-features candidate-features added removed]) + (map? b-request) (map? c-request) + (boolean? (:default-features? b-request)) + (boolean? (:default-features? c-request)) + (= (:default-features? b-request) (:default-features? c-request)) + (feature-names? (:features b-request)) + (feature-names? (:features c-request)) + (not (contains? (set (:features b-request)) feature)) + (= (conj (set (:features b-request)) feature) (set (:features c-request))) + (= (:value b) (set baseline-features)) + (= (:value c) (set candidate-features)) + (= (set/difference (:value c) (:value b)) (set added)) + (= (set/difference (:value b) (:value c)) (set removed))))) + +(defn- expected-feature-change? + [baseline candidate expected] + (and (feature-change-evidence? baseline candidate expected) + (contains? (set (:added expected)) (:feature expected)))) + +(defn- acknowledge-difference + "Retains a mismatch and records its applicable rule, if any. + Automatic feature evidence and scoped permissions take precedence over all. + Fingerprint content is scene data; incomplete/varying evidence is not." + [{:keys [field] :as mismatch} permissions expected-feature-change?] + (let [rule (cond + (and (= :effective-features field) expected-feature-change?) :feature-toggle + (and (:allow-build-differences permissions) + (contains? #{:effective-features :functional-build :build-environment} field)) :build + (and (:allow-scene-differences permissions) + (contains? #{:scene :fingerprint} field)) :scene + (:allow-all-differences permissions) :all)] + (cond-> mismatch + rule (assoc :waived? true :rule rule)))) + +(defn- side-report + "Reports one side with saved definitions, setup evidence and sample accounting." + [run id shared analysis] + (summary/case-report run id shared analysis)) + +(defn- metric-changes + "Keeps diagnostic changes unavailable for differing or varying definitions. + Only all-differences permits comparing their valid raw values without conversion." + [baseline candidate b c id mismatches allow-all-differences analysis] + (if (and (not allow-all-differences) + (some #(contains? #{[:metric id] [[:metric id] :incomplete-or-varying-evidence]} + (:field %)) mismatches)) + {:absolute {:reason :inconsistent-metric-definitions} + :percentage {:reason :inconsistent-metric-definitions} + :absolute-ci {:reason :inconsistent-metric-definitions} + :percentage-ci {:reason :inconsistent-metric-definitions}} + (analysis/changes (:values (summary/observations baseline b id)) + (:values (summary/observations candidate c id)) analysis))) + +(defn- compare-pair + "Compares shared numeric observations after resolving the compatibility policy." + [baseline candidate [b c] explicit? permissions expected-feature-change? on-unallowed-difference analysis] + (let [b (resolve-case baseline b) + c (resolve-case candidate c) + candidate-ids (set (summary/metric-ids candidate c)) + shared (filterv #(contains? candidate-ids %) (summary/metric-ids baseline b)) + mismatches (mapv #(acknowledge-difference % permissions expected-feature-change?) + (pair-differences baseline candidate b c shared)) + allow-all-differences (:allow-all-differences permissions)] + (when (empty? shared) + (throw (ex-info "selected cases have no shared metric identifiers" + {:type ::no-shared-metrics :baseline b :candidate c}))) + (when (and (not= :record on-unallowed-difference) (some #(not (:waived? %)) mismatches)) + (throw (ex-info "incompatible saved runs" + {:type ::incompatible :baseline b :candidate c :mismatches mismatches}))) + {:explicit? explicit? + :mismatches mismatches + :baseline (side-report baseline b shared analysis) + :candidate (side-report candidate c shared analysis) + :metrics (into {} + (map (fn [id] + [id (metric-changes baseline candidate b c id mismatches + allow-all-differences analysis)])) + shared)})) + +(defn- provenance + "Retains run identity and facts without using Git/run/time as compatibility gates." + [run] + (select-keys run [:run-id :started-at :ended-at :metadata :termination])) + +(defn compare-runs + "Creates a schema:comparison from constructed or decoded schema:run records. + + Inputs must already satisfy schema:run. Encoding validates the returned + comparison at the boundary. + + Default pairs use identical identifiers. Explicit pairs select cases without + changing compatibility checks. + + - `:pairs` accepts an extensible vector of explicit [baseline candidate] pairs. + - `:allow-build-differences` acknowledges build settings and effective features. + - `:allow-scene-differences` acknowledges scene inputs and fingerprint content. + It never acknowledges incomplete or varying evidence or execution settings. + - `:allow-all-differences` acknowledges any compatibility mismatch and permits + mixed-definition statistics on valid raw observations, without conversions. + Scoped permissions combine by union. Every mismatch retains its values, + legacy `:waived?` field and, only when acknowledged, its `:rule`. + - `:expected-feature-change` is planner evidence with string `:feature` and + vectors `:baseline-features`, `:candidate-features`, `:added`, `:removed`. + Automatic acknowledgment requires saved effective vectors of strings, exact + sets/deltas with that feature added, and saved requested selections differing + only by that toggle, with the same recorded default-feature policy. + - `:edge` attaches checked feature-comparison context: `:from`, `:to`, and the five expected + feature fields. When both options are supplied, their feature facts must agree. + The ordinary encoding boundary validates the saved edge's shape. + - Internal `:on-unallowed-difference :record` returns diagnostic reports instead + of throwing for unacknowledged differences. It keeps strict statistics. + - `:analysis` resolves from method defaults, independent of each run's defaults." + ([baseline candidate] (compare-runs baseline candidate {})) + ([baseline candidate {:keys [pairs allow-build-differences allow-scene-differences + allow-all-differences expected-feature-change edge + on-unallowed-difference analysis]}] + (let [explicit? (some? pairs) + candidate-ids (set (map :id (get-in candidate [:plan :cases]))) + pairs (or pairs (->> (get-in baseline [:plan :cases]) + (map :id) + (filter #(contains? candidate-ids %)) + (mapv #(vector % %)))) + analysis (analysis/resolve-settings analysis) + permissions {:allow-build-differences (boolean allow-build-differences) + :allow-scene-differences (boolean allow-scene-differences) + :allow-all-differences (boolean allow-all-differences)} + expected? (expected-feature-change? baseline candidate expected-feature-change)] + (when (and (some? edge) (not (feature-change-evidence? baseline candidate edge))) + (throw (ex-info "edge context disagrees with saved run feature evidence" + {:type ::invalid-edge :edge edge}))) + (when (and (some? edge) (some? expected-feature-change) + (not= expected-feature-change + (select-keys edge [:feature :baseline-features :candidate-features :added :removed]))) + (throw (ex-info "edge context disagrees with expected feature facts" + {:type ::invalid-edge :edge edge :expected-feature-change expected-feature-change}))) + (when (or (empty? pairs) (not (every? #(and (vector? %) (= 2 (count %))) pairs))) + (throw (ex-info "comparison needs saved case pairs" + {:type ::invalid-pairs + :baseline (mapv (comp result/case-name :id) (get-in baseline [:plan :cases])) + :candidate (mapv (comp result/case-name :id) (get-in candidate [:plan :cases]))}))) + (cond-> {:type :comparison :analysis analysis + :baseline (provenance baseline) :candidate (provenance candidate) + :permissions permissions + :source-coverage summary/source-coverage :interpretation summary/interpretation + :pairs (mapv #(compare-pair baseline candidate % explicit? permissions expected? + on-unallowed-difference analysis) pairs)} + (some? edge) (assoc :edge edge))))) diff --git a/frontend/performance/benches/render_wasm/report/format.cljs b/frontend/performance/benches/render_wasm/report/format.cljs new file mode 100644 index 0000000000..d586167e30 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/format.cljs @@ -0,0 +1,357 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.format + "Text formatting for run summaries and saved comparisons. + + Text leads with observed median, bootstrap interval and distinct counts. + Full statistics and acknowledged differences include units and phase identity." + (:require + [benches.render-wasm.errors :as errors] + [benches.render-wasm.result :as result] + [clojure.string :as str])) + +(defn- number->text + "Formats safe integers in decimal form and other estimates with six significant digits." + [value] + (if (js/Number.isSafeInteger value) + (str value) + (.toPrecision value 6))) + +(defn- estimate->text + "Formats an estimate, interval or suppression reason." + [estimate] + (cond + (contains? estimate :value) (number->text (:value estimate)) + (contains? estimate :lower) (str "[" (number->text (:lower estimate)) ", " + (number->text (:upper estimate)) "]") + :else (str "unavailable (" (some-> (:reason estimate) name) + (when-some [n (:minimum estimate)] (str "; need " n)) ")"))) + +(defn- counts->text + "Prints valid, invalid, absent, failed, unattempted and excluded warmup counts." + [{:keys [valid invalid absent failed unattempted warmup invocations raw-values]}] + (str "valid=" valid " invalid=" invalid " absent=" absent " failed=" failed + " unattempted=" unattempted " warmup=" warmup + " invocations=" invocations " raw-values=" raw-values " n=" valid)) + +(defn metric->text + "Formats statistics across phase execution sums with their units and counts. + + Args: + id: vector of phase, command or global scope, and metric type. + report: quantity descriptions, accounting counts, statistics and diagnostics. + + Returns: + Text with observed estimates, support limits and raw-value counts." + [id {:keys [definitions counts statistics diagnostics]}] + (str (pr-str id) " sum per execution; median " (estimate->text (or (:median statistics) statistics)) + " CI " (estimate->text (or (:median-ci statistics) statistics)) " | " (counts->text counts) + "\n definitions " (pr-str definitions) + (when (seq diagnostics) (str "\n diagnostics " (pr-str diagnostics))) + (if-let [reason (:reason statistics)] + (str "\n unavailable (" (name reason) ")") + (str "\n mean " (estimate->text (:mean statistics)) + " min " (estimate->text (:min statistics)) " max " (estimate->text (:max statistics)) + " sample SD " (estimate->text (:sample-sd statistics)) + " MAD " (estimate->text (:mad statistics)) + " p95 " (estimate->text (:p95 statistics)) " p99 " (estimate->text (:p99 statistics)))))) + +(defn- quoted-detail + [value] + (errors/format-value value 1024)) + +(defn- identifier->text + [value] + (errors/format-value value 1024)) + +(def ^:private process-detail-keys + [:command :pid :pgid :signals :profile-path :fatal :unresolved-process]) + +(defn format-error-entry + "Formats an error with its location and available process details. + + Args: + entry: A projected error occurrence. + + Returns: + Bounded error text and process details when a group may still be running." + [{:keys [phase operation message path] :as entry}] + (str (when operation (str (errors/format-value operation 1024) " ")) + (quoted-detail phase) + (when (some? message) (str ": " (quoted-detail message))) + (when path (str "\n Path " (quoted-detail path))) + (when (:unresolved-process entry) + (apply str + (map (fn [[key value]] (str "\n " (name key) " " (identifier->text value))) + (select-keys entry process-detail-keys)))))) + +(defn- failure->text + [{:keys [failure reason]}] + (if failure + (if-let [n (:unacknowledged-differences failure)] + (str n " unacknowledged " (if (= n 1) "difference" "differences")) + (if-let [entries (seq (:errors failure))] + (let [header (apply dissoc failure process-detail-keys) + omitted (drop 8 entries) + unresolved (filter :unresolved-process omitted)] + (str (if (not= (errors/bounded-text (:message failure) 1024) (:message (first entries))) + (str (format-error-entry header) "\n ") + (when (not= (select-keys failure [:phase :operation :path]) + (select-keys (first entries) [:phase :operation :path])) + (str "Context " (format-error-entry (dissoc header :message)) "\n "))) + (str/join "\n " (map format-error-entry (take 8 entries))) + (when (seq omitted) + (str "\n " (count omitted) " more errors recorded")) + (when-let [entry (first unresolved)] + (str "\n " (count unresolved) " unresolved process errors in the saved record; first: " + (format-error-entry entry))))) + (format-error-entry failure))) + (quoted-detail reason))) + +(defn- same-diagnostic? + [left right] + (= (dissoc (errors/project-diagnostic left) :status :partial) + (dissoc (errors/project-diagnostic right) :status :partial))) + +(defn- case-failures + [{:keys [case-id preparations failed-attempts]}] + (let [preparations (filter :failure preparations) + by-id (into {} (map (juxt :preparation-id identity)) preparations)] + (concat + (map (fn [preparation] + (assoc (select-keys preparation [:preparation-id :warmup? :failure]) + :kind :preparation :case-id case-id)) + preparations) + (keep (fn [{:keys [preparation-id failure phases] :as attempt}] + (let [preparation (get by-id preparation-id) + copied? (and preparation + (every? #(= :preparation (:phase %)) phases) + (same-diagnostic? failure (:failure preparation)))] + (when (and failure (not copied?)) + (assoc (select-keys attempt [:attempt-id :preparation-id :warmup? :failure :partial]) + :kind :attempt :case-id case-id)))) + failed-attempts)))) + +(defn- run-failures + [cases metadata termination] + (let [case-entries (vec (mapcat case-failures cases)) + by-case (group-by :case-id case-entries) + harness (keep-indexed + (fn [index failure] + (let [copied? (and (= :preparation (:stage failure)) + (some #(and (= :preparation (:kind %)) + (same-diagnostic? (:failure %) failure)) + (get by-case (:case-id failure))))] + (when-not copied? + {:kind :harness :index index :stage (:stage failure) + :case-id (:case-id failure) :failure failure}))) + (get-in metadata [:diagnostics :failures])) + recorded (into case-entries harness) + by-owner (group-by :case-id recorded) + outcomes (keep (fn [{:keys [case-id outcome]}] + (when-let [failure (:failure outcome)] + (when-not (some #(same-diagnostic? failure (:failure %)) (get by-owner case-id)) + {:kind :case :case-id case-id :failure failure}))) + cases) + entries (into recorded outcomes)] + (cond-> entries + (and (:failure termination) + (not-any? #(same-diagnostic? (:failure termination) (:failure %)) entries)) + (conj {:kind :termination :failure (:failure termination)})))) + +(defn- diagnostic->text + [failure] + (let [context (errors/project-context failure) + context (if (:unresolved-process failure) (apply dissoc context process-detail-keys) context) + details (merge (select-keys (first (:cause failure)) [:type :code :fn]) + context + (select-keys failure [:reason :deadline-exceeded]))] + (str (failure->text {:failure failure}) + (apply str (map (fn [[key value]] + (str "\n " (name key) " " (identifier->text value))) + (sort-by (comp name key) (filter (comp some? val) details))))))) + +(defn- failure-entry->text + [{:keys [kind case-id preparation-id attempt-id index stage warmup? failure partial]}] + (str " " (case kind + :preparation (str "Preparation " (identifier->text preparation-id)) + :attempt (str "Attempt " attempt-id " (preparation " (identifier->text preparation-id) ")") + :harness (str "Harness " index " " (identifier->text stage)) + :case "Case outcome" + :termination "Termination") + (when (some? case-id) (str " (case " (identifier->text case-id) ")")) + (when warmup? " warmup") + ": " (diagnostic->text failure) + (when (or partial (:partial failure)) "\n Partial evidence recorded"))) + +(defn- failures->text + [heading entries] + (when (seq entries) + (str "\n\n" heading "\n" (str/join "\n" (map failure-entry->text entries))))) + +(defn- screenshots->text + [heading metadata failures cases] + (let [screenshots (get-in metadata [:diagnostics :screenshots]) + names (frequencies (map result/case-name + (distinct (concat (map :case-id cases) (keep :case-id failures))))) + deadlines (group-by #(result/case-name (:case-id %)) + (filter #(and (= :harness (:kind %)) + (= :case-deadline (:stage %)) + (some? (:case-id %))) failures))] + (when (seq screenshots) + (str "\n\n" heading "\n" + (str/join + "\n" + (map (fn [[case-name {:keys [status message reason file cause]}]] + (let [matches (when (and (= 1 (get names case-name)) (string? message) (seq cause)) + (filter #(and (= message (get-in % [:failure :message])) + (= cause (get-in % [:failure :cause]))) + (get deadlines case-name))) + related (when (= 1 (count matches)) (first matches))] + (str " " (identifier->text case-name) ": " + (if related + (str "failed; see Harness " (:index related)) + (case status + :failed "warning" + :ok "ok" + :skipped "skipped" + (identifier->text status))) + (when (and message (not related)) (str "; " (quoted-detail message))) + (when reason (str "; reason " (identifier->text reason))) + (when file (str "\n File " (quoted-detail file)))))) + (sort-by (comp str key) screenshots))))))) + +(defn- case-status->text + "Displays pending slots, phase counts and failed execution identities." + [{:keys [remaining failed-attempts phases outcome]}] + (str " remaining " (pr-str remaining) + (when outcome (str "\n outcome " (pr-str (:outcome outcome)))) + "\n phases " (pr-str phases) + " failed-warmups=" (count (filter :warmup? failed-attempts)) + (when (seq failed-attempts) + (str "\n failed attempts " + (pr-str (mapv #(select-keys % [:attempt-id :preparation-id :warmup? :outcome]) failed-attempts)))))) + +(defn format-run + "Formats run statistics, recorded failures and screenshot diagnostics." + [{:keys [run-id metadata analysis termination source-coverage interpretation cases]}] + (let [failures (run-failures cases metadata termination)] + (str "Run " (pr-str run-id) " " (pr-str (dissoc termination :failure)) + "\nGit " (pr-str (:git metadata)) + "\nAnalysis " (pr-str analysis) + "\n" source-coverage "\n" interpretation + (when (some #(get-in % [:effective-graphics :software]) + (mapcat :preparations cases)) "\nSOFTWARE RENDERER recorded") + (apply str (map (fn [{:keys [case-id metrics] :as case-report}] + (str "\n\nCase " (result/case-name case-id) + (case-status->text case-report) + (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) + cases)) + (failures->text "Failures" failures) + (screenshots->text "Screenshots" metadata failures cases)))) + +(defn- side->text + "Displays one comparison side's definitions, preparation facts and observations." + [label {:keys [case-id definition preparations metrics] :as side}] + (str "\n" label " " (result/case-name case-id) + (case-status->text side) + "\n case " (pr-str definition) + "\n preparations " (pr-str (mapv #(select-keys % [:preparation-id :warmup? :fingerprint :effective-graphics]) + preparations)) + (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) + +(defn- configuration-failure->text + [{:keys [slug run build] :as configuration}] + (str " Configuration " (quoted-detail slug) ": " (failure->text configuration) + (cond + run (str "\n Run record " (quoted-detail (:record-path run)) + "\n Log " (quoted-detail (:log-path run))) + build (str "\n Build manifest " (quoted-detail (:manifest-path build)) + "\n Log " (quoted-detail (:log-path build)))))) + +(defn- edge-failure->text + [{:keys [from to feature path] :as edge}] + (str " Comparison " (quoted-detail from) " -> " (quoted-detail to) + " (feature " (quoted-detail feature) "): " (failure->text edge) + (when path (str "\n Report " (quoted-detail path))))) + +(defn format-feature-comparison-failures + "Formats failed configurations and comparisons with artifact references. + + Args: + record: Validated feature-comparison record. + home: Comparison :record-path and :log-path. + + Returns: + Text for up to eight failures in configuration and comparison order, + with an omitted count and references to the complete evidence. String + fields escape C0 controls (U+0000 through U+001F) and use at most + 1024 UTF-16 units. + Returns nil when the record contains no failed entries." + [record home] + (let [configurations (filter #(= :failed (:status %)) (:configurations record)) + edges (filter #(= :failed (:status %)) (:edges record)) + total (+ (count configurations) (count edges)) + entries (concat (map configuration-failure->text configurations) + (map edge-failure->text edges))] + (when (pos? total) + (str "Failures\n" (str/join "\n" (take 8 entries)) + (when (> total 8) (str "\n " (- total 8) " more failures in the saved record")) + "\n Record " (quoted-detail (:record-path home)) + "\n Log " (quoted-detail (:log-path home)))))) + +(defn format-comparison + "Formats selected permissions, acknowledged/unacknowledged differences and changes. + Both saved definitions remain visible. Edge labels describe actual feature + evidence, including equal effective sets, without predicting a timing effect." + [{:keys [baseline candidate analysis permissions edge source-coverage interpretation pairs]}] + (let [side-cases (fn [side] + (let [cases (map side pairs) + by-id (group-by :case-id cases)] + (map #(first (get by-id %)) (distinct (map :case-id cases))))) + baseline-cases (side-cases :baseline) + candidate-cases (side-cases :candidate) + baseline-failures (run-failures baseline-cases (:metadata baseline) (:termination baseline)) + candidate-failures (run-failures candidate-cases (:metadata candidate) (:termination candidate))] + (str "Permissions " (pr-str permissions) "\n" + (when edge + (str "Edge " (:from edge) " -> " (:to edge) " feature " (:feature edge) + (when (= (set (:baseline-features edge)) (set (:candidate-features edge))) + " (effective feature sets identical)") + "\n effective features " (pr-str (select-keys edge [:baseline-features :candidate-features :added :removed])) + "\n")) + (when (some #(get-in % [:effective-graphics :software]) + (mapcat (fn [pair] (concat (get-in pair [:baseline :preparations]) + (get-in pair [:candidate :preparations]))) pairs)) + "SOFTWARE RENDERER recorded\n") + "Baseline " (pr-str (:run-id baseline)) " Git " (pr-str (get-in baseline [:metadata :git])) + "\nBaseline termination " (pr-str (dissoc (:termination baseline) :failure)) + "\nCandidate " (pr-str (:run-id candidate)) " Git " (pr-str (get-in candidate [:metadata :git])) + "\nCandidate termination " (pr-str (dissoc (:termination candidate) :failure)) + "\nAnalysis " (pr-str analysis) "\n" source-coverage "\n" interpretation + (failures->text "Baseline failures" baseline-failures) + (screenshots->text "Baseline screenshots" (:metadata baseline) baseline-failures baseline-cases) + (failures->text "Candidate failures" candidate-failures) + (screenshots->text "Candidate screenshots" (:metadata candidate) candidate-failures candidate-cases) + (apply str + (map (fn [{:keys [baseline candidate mismatches metrics]}] + (str "\n\nPair " (result/case-name (:case-id baseline)) "," + (result/case-name (:case-id candidate)) + (apply str + (map (fn [mismatch] + (str "\n" (if (:waived? mismatch) "ACKNOWLEDGED" "UNACKNOWLEDGED") + " DIFFERENCE " (pr-str mismatch))) mismatches)) + (side->text "Baseline" baseline) + (side->text "Candidate" candidate) + (str/join "\n" + (map (fn [[id change]] + (str "\n" (pr-str id) + " median change " (estimate->text (:absolute change)) + " CI " (estimate->text (:absolute-ci change)) + "; percent " (estimate->text (:percentage change)) + " CI " (estimate->text (:percentage-ci change)))) metrics)))) + pairs))))) diff --git a/frontend/performance/benches/render_wasm/report/summarize.cljs b/frontend/performance/benches/render_wasm/report/summarize.cljs new file mode 100644 index 0000000000..e6c1e8e0c9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/summarize.cljs @@ -0,0 +1,238 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.summarize + "Offline analysis of raw command histories, grouped within phase executions." + (:require + [app.common.schema :as sm] + [benches.render-wasm.quantities :as quantities] + [benches.render-wasm.report.analysis :as analysis])) + +(def source-coverage + "Recorded inputs and settings checked; runtime-helper identity is unverified. Git SHA and dirty state record provenance.") + +(def interpretation + "Intervals are exploratory and conditional on available executions. Incomplete and timed-out cases can select which observations survive. Caches can correlate attempts; repeat whole runs to investigate findings. Full denotes renderer submission completion. Sums combine only one metric type within one phase execution. Finite clock resolution can produce valid zero durations.") + +(defn case-attempts + "Returns chronological attempts for an exact saved case identifier." + [run case-id] + (filterv #(= case-id (:case-id %)) (:attempts run))) + +(defn get-failed-attempts + "Returns failed warmup and measured attempts with their saved evidence." + [run case-id] + (filterv #(= :failed (:outcome %)) (case-attempts run case-id))) + +(defn executions + "Returns actual phase executions with their eligibility and saved identities. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + phase: preparation, main or reset. + + Returns: + Chronological phase histories with phase and preparation/attempt outcomes." + [run case-id phase] + (let [attempts (case-attempts run case-id)] + (if (= :preparation phase) + (let [definition (some #(when (= case-id (:id %)) %) (get-in run [:plan :cases]))] + (mapv (fn [prep] + (let [block (first (:phases prep))] + {:preparation-id (:preparation-id prep) + :events (:events block) + :warmup? (and (not= :reuse (:context definition)) (:warmup? prep)) + :outcome (:outcome block) + :preparation-outcome (if (:failure prep) :failed :completed)})) + (filter #(and (= case-id (:case-id %)) (seq (:phases %))) (:preparations run)))) + (into [] + (keep (fn [attempt] + (when-let [block (some #(when (= phase (:phase %)) %) (:phases attempt))] + (merge (select-keys attempt [:attempt-id :preparation-id :warmup?]) + {:outcome (:outcome block) :attempt-outcome (:outcome attempt) + :events (:events block)})))) + attempts)))) + +(defn phase-counts + "Counts actual phase executions, including empty histories and excluded work. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + + Returns: + Execution, completion, failure and warmup counts for each phase." + [run case-id] + (into {} + (map (fn [phase] + (let [blocks (executions run case-id phase)] + [phase {:executions (count blocks) + :completed (count (filter #(= :completed (:outcome %)) blocks)) + :failed (count (filter #(= :failed (:outcome %)) blocks)) + :warmup (count (filter :warmup? blocks))}]))) + [:preparation :main :reset])) + +(defn metric-ids + "Returns [phase command-or-global metric-type] keys in first appearance order. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + + Returns: + Distinct report keys, including quantities recorded during failed work." + [run case-id] + (vec (distinct + (for [phase [:preparation :main :reset] + block (executions run case-id phase) + event (:events block) + metric (keys (:metrics event)) + scope [(:command event) :global]] + [phase scope metric])))) + +(defn definitions + "Returns valid saved quantity descriptions for a report key. + + Args: + run: validated saved run. + _case-id: exact saved case identifier. + id: vector of phase, command or global scope, and metric type. + + Returns: + A vector containing the description, or an empty vector for invalid metadata." + [run _case-id [_phase _scope metric]] + (let [description (get-in run [:quantities metric])] + (if (sm/validate quantities/schema:description description) [description] []))) + +(defn- invalid-reason + [value] + (cond + (not (js/Number.isFinite value)) :non-finite + (neg? value) :negative + :else nil)) + +(defn- execution-value + [block scope metric] + (let [events (->> (:events block) + (map-indexed vector) + (filter (fn [[_ event]] + (and (or (= :global scope) (= scope (:command event))) + (contains? (:metrics event) metric))))) + values (vec (mapcat #(get-in (second %) [:metrics metric]) events)) + invalid (vec (for [[event-index event] events + [value-index value] (map-indexed vector (get-in event [:metrics metric])) + :let [reason (invalid-reason value)] + :when reason] + (merge (select-keys block [:attempt-id :preparation-id]) + {:event-index event-index :value-index value-index + :value value :reason reason}))) + total (when (and (seq values) (empty? invalid)) (reduce + 0 values))] + {:invocations (count events) :raw-values (count values) + :diagnostics (cond-> invalid + (and (some? total) (not (js/Number.isFinite total))) + (conj {:reason :non-finite-sum})) + :value total})) + +(defn observations + "Sums one metric type per eligible phase execution. + Invocation and raw-value counts cover eligible executions, including invalid + values. Statistical n counts valid execution sums. Failed attempts exclude + both main and reset. Successful preparation survives later attempt failures. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + id: vector of phase, command or global scope, and metric type. + + Returns: + Valid execution sums, accounting counts and invalidity diagnostics." + [run case-id [phase scope metric :as id]] + (let [description (get-in run [:quantities metric]) + diagnostic (when (empty? (definitions run case-id id)) + {:reason (if (contains? (:quantities run) metric) + :malformed-quantity :unknown-quantity) + :metric metric :description description})] + (reduce + (fn [out {:keys [warmup? outcome attempt-outcome preparation-outcome] :as block}] + (let [{:keys [value invocations raw-values diagnostics]} (execution-value block scope metric) + kind (cond + warmup? :warmup + (or (= :failed outcome) (= :failed attempt-outcome) + (= :failed preparation-outcome)) :failed + (zero? invocations) :absent + (or diagnostic (seq diagnostics)) :invalid + :else :valid) + eligible? (not (#{:warmup :failed} kind))] + (cond-> (update-in out [:counts kind] inc) + eligible? (update-in [:counts :invocations] + invocations) + eligible? (update-in [:counts :raw-values] + raw-values) + eligible? (update :diagnostics into diagnostics) + (= kind :valid) (update :values conj value)))) + {:values [] :diagnostics (if diagnostic [diagnostic] []) + :counts {:valid 0 :invalid 0 :absent 0 :failed 0 :warmup 0 + :invocations 0 :raw-values 0 + :unattempted (if (= :preparation phase) 0 + (get-in run [:remaining case-id :repetitions]))}} + (executions run case-id phase)))) + +(defn summarize-metric + "Computes statistics across execution sums for one phase, scope and metric type. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + id: vector of phase, command or global scope, and metric type. + analysis: resolved statistical settings. + + Returns: + Quantity descriptions, counts, diagnostics and statistics across executions." + [run case-id id analysis] + (let [{:keys [values counts diagnostics]} (observations run case-id id)] + {:definitions (definitions run case-id id) + :statistic :sum :counts counts :diagnostics diagnostics + :statistics (analysis/statistics values analysis)})) + +(defn case-report + "Reports saved case facts and phase quantities using one analysis method. + + Args: + run: validated saved run. + id: exact saved case identifier. + ids: selected report keys. + analysis: resolved statistical settings. + + Returns: + Case definition, raw preparation/failure evidence, counts and metric reports." + [run id ids analysis] + {:case-id id + :definition (some #(when (= id (:id %)) %) (get-in run [:plan :cases])) + :outcome (get-in run [:case-outcomes id]) + :preparations (filterv #(= id (:case-id %)) (:preparations run)) + :phases (phase-counts run id) + :remaining (get-in run [:remaining id]) + :failed-attempts (get-failed-attempts run id) + :metrics (into {} (map (fn [metric-id] + [metric-id (summarize-metric run id metric-id analysis)])) ids)}) + +(defn summarize + "Reports a validated saved run using saved cases and quantity descriptions. + + Args: + run: validated saved run. + overrides: optional statistical settings overriding the saved defaults. + + Returns: + A report with provenance, termination and separate analyses for each case." + ([run] (summarize run nil)) + ([run overrides] + (let [analysis (analysis/resolve-settings (merge (:analysis run) overrides))] + {:run-id (:run-id run) :metadata (:metadata run) :analysis analysis + :termination (:termination run) + :source-coverage source-coverage :interpretation interpretation + :cases (mapv (fn [{:keys [id]}] + (case-report run id (metric-ids run id) analysis)) + (get-in run [:plan :cases]))}))) diff --git a/frontend/performance/benches/render_wasm/result.cljs b/frontend/performance/benches/render_wasm/result.cljs new file mode 100644 index 0000000000..5fd87b64d4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/result.cljs @@ -0,0 +1,696 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.result + "Schemas for experiments and reports." + (:require + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.quantities :as quantities] + [benches.render-wasm.report.analysis :as analysis])) + +(def schema:event + "One measured command invocation with its ordered raw values." + [:map {:closed true} + [:command qualified-keyword?] + [:metrics [:map-of qualified-keyword? [:vector {:min 1} number?]]] + [:trace {:optional true} :any]]) + +(def schema:phase + "Raw command history for one phase execution, including partial failures." + [:map {:closed true} + [:phase [:enum :preparation :main :reset]] + [:preparation-id some?] + [:outcome [:enum :completed :failed]] + [:events [:vector schema:event]] + [:diagnostics {:optional true} [:vector :any]]]) + +(def schema:fingerprint + "The content digest of the actual validated starting snapshot." + [:map {:closed true} + [:algorithm [:= :sha-256]] + [:digest [:re #"^[0-9a-f]{64}$"]]]) + +(def schema:preparation + "Setup diagnostics and callback measurements for one owned scene." + [:map {:closed true} + [:preparation-id some?] + [:case-id some?] + [:warmup? boolean?] + [:fingerprint {:optional true} schema:fingerprint] + [:effective-graphics {:optional true} map?] + [:diagnostics map?] + [:phases [:vector schema:phase]] + [:failure {:optional true} errors/schema:diagnostic]]) + +(def schema:attempt + "One chronological execution; failure consumes a slot and may retain evidence." + [:map {:closed true} + [:attempt-id [:int {:min 0}]] + [:case-id some?] + [:preparation-id [:maybe some?]] + [:warmup? boolean?] + [:outcome [:enum :completed :failed]] + [:phases [:vector schema:phase]] + [:failure {:optional true} errors/schema:diagnostic] + [:partial {:optional true} map?]]) + +(def schema:metadata + "Git, functional build settings, environment and diagnostic provenance." + [:map {:closed true} + [:git [:map [:sha [:maybe string?]] [:dirty [:maybe boolean?]]]] + [:build [:map + [:functional map?] + [:provenance {:optional true} map?] + [:environment {:optional true} [:map-of :string :string]]]] + [:environment map?] + [:scored? boolean?] + [:diagnostics {:optional true} + [:map [:failures {:optional true} [:vector errors/schema:diagnostic]]]]]) + +(def schema:termination + "A run's current state or final reason, with optional diagnostic details." + [:map + [:reason keyword?] + [:failure {:optional true} errors/schema:diagnostic]]) + +(def schema:case-outcome + [:map {:closed true} + [:outcome [:enum :completed :failed :timed-out :interrupted]] + [:failure {:optional true} errors/schema:diagnostic]]) + +(def schema:run + "schema:run stores saved case definitions, independent of any registry. + Build :functional holds compatibility settings; :provenance holds identities." + [:map {:closed true} + [:type [:= :run-record]] + [:run-id some?] + [:started-at string?] + [:ended-at [:maybe string?]] + [:plan [:map {:closed true} + [:cases [:vector [:map [:id some?]]]] + [:warmups [:int {:min 0}]] + [:repetitions [:int {:min 0}]] + [:timeout-policy [:map {:closed true} [:case-ms [:int {:min 1}]]]]]] + [:metadata schema:metadata] + [:analysis analysis/schema:settings] + [:quantities [:map-of qualified-keyword? :any]] + [:preparations [:vector schema:preparation]] + [:attempts [:vector schema:attempt]] + [:case-outcomes [:map-of :any schema:case-outcome]] + [:remaining [:map-of :any [:map {:closed true} + [:warmups [:int {:min 0}]] + [:repetitions [:int {:min 0}]]]]] + [:termination schema:termination]]) + +(def schema:estimate + "A finite computed estimate or an explicit suppression reason." + [:or + [:map {:closed true} [:value [:fn #(and (number? %) (js/Number.isFinite %))]]] + [:map {:closed true} [:reason keyword?] [:minimum {:optional true} [:int {:min 1}]]]]) + +(def schema:interval + "Finite ordered percentile bounds or an explicit suppression reason." + [:or + [:and + [:map {:closed true} + [:lower [:fn #(and (number? %) (js/Number.isFinite %))]] + [:upper [:fn #(and (number? %) (js/Number.isFinite %))]]] + [:fn #(<= (:lower %) (:upper %))]] + [:map {:closed true} [:reason keyword?] [:minimum {:optional true} [:int {:min 1}]]]]) + +(def schema:statistics + "Observed statistics and supported intervals for one metric." + [:or + [:map {:closed true} + [:count [:int {:min 0}]] + [:mean schema:estimate] [:median schema:estimate] [:min schema:estimate] [:max schema:estimate] + [:sample-sd schema:estimate] [:mad schema:estimate] [:p95 schema:estimate] [:p99 schema:estimate] + [:median-ci schema:interval]] + [:map {:closed true} [:reason keyword?] [:count [:int {:min 0}]]]]) + +(def schema:metric-report + "A sum within each phase execution and statistics across eligible executions." + [:map {:closed true} + [:definitions [:vector quantities/schema:description]] + [:statistic [:= :sum]] + [:counts [:map {:closed true} + [:valid [:int {:min 0}]] [:invalid [:int {:min 0}]] [:absent [:int {:min 0}]] + [:failed [:int {:min 0}]] [:unattempted [:int {:min 0}]] [:warmup [:int {:min 0}]] + [:invocations [:int {:min 0}]] [:raw-values [:int {:min 0}]]]] + [:diagnostics [:vector map?]] + [:statistics schema:statistics]]) + +(def schema:comparison-side + "One saved case's definition, preparations and shared metric reports." + [:map {:closed true} + [:case-id some?] [:definition [:map [:id some?]]] + [:outcome [:maybe schema:case-outcome]] + [:preparations [:vector schema:preparation]] + [:phases [:map-of :keyword [:map {:closed true} + [:executions nat-int?] [:completed nat-int?] + [:failed nat-int?] [:warmup nat-int?]]]] + [:remaining [:map {:closed true} [:warmups [:int {:min 0}]] [:repetitions [:int {:min 0}]]]] + [:failed-attempts [:vector schema:attempt]] + [:metrics [:map-of :any schema:metric-report]]]) + +(def schema:provenance + "Saved run identity, timestamps, metadata and termination for comparison." + [:map {:closed true} + [:run-id some?] [:started-at string?] [:ended-at [:maybe string?]] + [:metadata schema:metadata] [:termination schema:termination]]) + +(def schema:permissions + [:map {:closed true} + [:allow-build-differences boolean?] + [:allow-scene-differences boolean?] + [:allow-all-differences boolean?]]) + +(def schema:feature-comparison + "Saved feature comparison progress. Case data and options remain ordinary + extensible maps; configuration/edge states and retained references are typed." + [:map {:closed true} + [:type [:= :feature-comparison]] + [:comparison-id string?] + [:started-at string?] + [:updated-at string?] + [:ended-at {:optional true} string?] + [:status [:enum :planned :running :completed :failed :interrupted]] + [:options :map] + [:permissions schema:permissions] + [:cases [:vector [:map [:id some?]]]] + [:features [:vector string?]] + [:configurations + [:vector + [:map + [:slug string?] + [:selection [:map {:closed true} + [:default-features? boolean?] + [:features [:vector string?]]]] + [:status [:enum :planned :building :ready :running :completed :failed :unattempted]] + [:build {:optional true} [:map [:id string?] [:manifest-path string?] [:log-path string?]]] + [:run {:optional true} [:map [:run-id string?] [:record-path string?] [:log-path string?] + [:exit-code {:optional true} :int]]] + [:effective-features {:optional true} [:vector string?]] + [:failure {:optional true} errors/schema:diagnostic] + [:reason {:optional true} string?]]]] + [:edges + [:vector + [:map + [:from string?] [:to string?] [:feature string?] + [:status [:enum :planned :completed :failed :unattempted]] + [:path {:optional true} string?] + [:failure {:optional true} + [:or errors/schema:diagnostic + [:map {:closed true} [:unacknowledged-differences [:int {:min 1}]]]]] + [:reason {:optional true} string?]]]] + [:warnings [:vector string?]]]) + +(def schema:comparison + "Resolved analysis, selected permissions, every mismatch and per-side reports. + Every acknowledged difference has a rule; unacknowledged differences have none. + Optional edge context records requested toggles and Cargo's actual effective + sets/deltas, including aliases." + [:map {:closed true} + [:type [:= :comparison]] + [:analysis analysis/schema:settings] + [:baseline schema:provenance] + [:candidate schema:provenance] + [:permissions schema:permissions] + [:edge {:optional true} + [:map {:closed true} + [:from string?] [:to string?] [:feature string?] + [:baseline-features [:vector string?]] [:candidate-features [:vector string?]] + [:added [:vector string?]] [:removed [:vector string?]]]] + [:source-coverage string?] + [:interpretation string?] + [:pairs [:vector + [:map {:closed true} + [:explicit? boolean?] + [:mismatches [:vector + [:and + [:map {:closed true} + [:field :any] [:baseline :any] [:candidate :any] [:waived? boolean?] + [:rule {:optional true} [:enum :feature-toggle :build :scene :all]]] + [:fn #(= (:waived? %) (contains? % :rule))]]]] + [:baseline schema:comparison-side] [:candidate schema:comparison-side] + [:metrics [:map-of :any [:map {:closed true} + [:absolute schema:estimate] [:percentage schema:estimate] + [:absolute-ci schema:interval] [:percentage-ci schema:interval]]]]]]]]) + +(defn case-name + "Returns the saved case identifier as shown in commands and reports." + [id] + (if (keyword? id) (subs (str id) 1) (str id))) + +(defn- require-valid! + "Rejects malformed external data with enough context for the caller." + [valid? message data] + (when-not valid? + (throw (ex-info message (assoc data :type ::invalid-record))))) + +(defn- incompatible! + "Rejects runs whose recorded evidence cannot be merged." + [message data] + (throw (ex-info message (assoc data :type ::incompatible-runs)))) + +(defn- require-compatible! + "Requires equal values at `path`, naming the disagreeing field." + [base additional path label] + (let [base-value (get-in base path) + additional-value (get-in additional path)] + (when-not (= base-value additional-value) + (incompatible! (str "cannot merge runs with different " label) + {:field label + :path path + :base base-value + :additional additional-value})))) + +(defn- initial-remaining + "Creates each saved case's fixed budget of unattempted slots." + [{:keys [cases warmups repetitions]}] + (into {} (map (fn [{:keys [id]}] [id {:warmups warmups :repetitions repetitions}])) cases)) + +(defn- consume-slot + "Consumes a warmup or measured slot; completed and failed attempts count alike." + [remaining {:keys [case-id warmup?]}] + (let [kind (if warmup? :warmups :repetitions) + path [case-id kind] + left (get-in remaining path)] + (require-valid! (and (some? left) (pos? left)) "no planned slot for attempt" + {:case-id case-id :slot kind}) + (require-valid! (or warmup? (zero? (get-in remaining [case-id :warmups]))) + "measured attempt precedes planned warmups" {:case-id case-id}) + (update-in remaining path dec))) + +(defn has-measurements? + "Reports whether a case has measured commands in its saved phase histories. + + Args: + run: constructed or decoded run record. + case-id: exact saved case identifier. + + Returns: + True when any preparation, main or reset block contains a command event." + [run case-id] + (boolean + (some (comp seq :events) + (mapcat :phases (filter #(= case-id (:case-id %)) + (concat (:preparations run) (:attempts run))))))) + +(defn get-unmeasured-cases + "Returns saved case ids whose phase histories contain no measured commands. + + Args: + run: constructed or decoded run record. + + Returns: + A vector of case identifiers in plan order." + [run] + (into [] + (keep (fn [{:keys [id]}] + (when-not (has-measurements? run id) + id))) + (get-in run [:plan :cases]))) + +(defn- check-completion + "Requires all planned attempts to succeed before a run can be completed." + [run termination] + (when (= :completed (:reason termination)) + (require-valid! (every? #(every? zero? (vals %)) (vals (:remaining run))) + "completed run still has unattempted slots" {}) + (require-valid! (not-any? #(= :failed (:outcome %)) (:attempts run)) + "completed run has failed attempts" {}) + (require-valid! (not-any? :failure (:preparations run)) + "completed run has failed preparations" {}) + (require-valid! (every? #(= :completed (:outcome %)) (vals (:case-outcomes run))) + "completed run has failed cases" {}) + (require-valid! (= (set (map :id (get-in run [:plan :cases]))) (set (keys (:case-outcomes run)))) + "completed run needs every case outcome" {}) + (require-valid! (empty? (get-unmeasured-cases run)) + "completed case has no measurements" {:case-ids (get-unmeasured-cases run)}))) + +(defn- check-phases + [record allowed] + (let [phases (:phases record)] + (require-valid! (= (mapv :phase phases) + (filterv (set (map :phase phases)) allowed)) + "phase order or identity is invalid" {}) + (doseq [phase phases] + (require-valid! (= (:preparation-id record) (:preparation-id phase)) + "phase references a different preparation" {}) + (doseq [event (:events phase)] + (require-valid! (= 1 (count (get-in event [:metrics :command/elapsed]))) + "command needs one elapsed value" {:command (:command event)}))) + (when-not (or (:failure record) (= :failed (:outcome record))) + (require-valid! (every? #(= :completed (:outcome %)) phases) + "successful execution has a failed phase" {})))) + +(defn- check-preparation + [preparation] + (require-valid! (sm/validate schema:preparation preparation) "invalid preparation" {}) + (check-phases preparation [:preparation]) + (when-not (:failure preparation) + (require-valid! (and (:fingerprint preparation) (:effective-graphics preparation) + (= [:preparation] (mapv :phase (:phases preparation)))) + "completed preparation needs scene, graphics and phase evidence" {}))) + +(defn- check-attempt + [attempt preparation] + (require-valid! (sm/validate schema:attempt attempt) "invalid attempt" {}) + (check-phases attempt [:main :reset]) + (when (:preparation-id attempt) + (require-valid! (= (:case-id attempt) (:case-id preparation)) + "attempt references an unknown or different preparation" {})) + (when (= :completed (:outcome attempt)) + (require-valid! (and preparation (not (:failure preparation)) + (not (:failure attempt)) + (= :main (get-in attempt [:phases 0 :phase]))) + "completed attempt needs successful preparation and main" {})) + (when (seq (:phases attempt)) + (require-valid! (= :main (get-in attempt [:phases 0 :phase])) + "attempt phases must start with main" {}) + (require-valid! (not (:failure preparation)) "attempt phases need successful preparation" {})) + (when (= :failed (get-in attempt [:phases 0 :outcome])) + (require-valid! (= 1 (count (:phases attempt))) "reset follows failed main" {}))) + +(defn- check-case-outcome + [run case-id outcome] + (require-valid! (contains? (:remaining run) case-id) "unknown case outcome" {}) + (require-valid! (sm/validate schema:case-outcome outcome) "invalid case outcome" {}) + (when (= :completed (:outcome outcome)) + (require-valid! (and (not (:failure outcome)) + (every? zero? (vals (get-in run [:remaining case-id]))) + (not-any? #(and (= case-id (:case-id %)) (= :failed (:outcome %))) (:attempts run)) + (not-any? #(and (= case-id (:case-id %)) (:failure %)) (:preparations run)) + (has-measurements? run case-id)) + "completed case has pending, failed or unmeasured work" {:case-id case-id}))) + +(defn check-run + "Validates schema:run and chronological accounting at a saved-data boundary." + [run] + (require-valid! (sm/validate schema:run run) "invalid schema:run" {}) + (let [case-ids (mapv :id (get-in run [:plan :cases])) + preps (:preparations run) + prep-ids (mapv :preparation-id preps) + prep-map (into {} (map (juxt :preparation-id identity)) preps) + left (reduce consume-slot (initial-remaining (:plan run)) (:attempts run))] + (require-valid! (= (count case-ids) (count (set case-ids))) "duplicate saved case" {}) + (require-valid! (= (count prep-ids) (count (set prep-ids))) "duplicate preparation" {}) + (require-valid! (every? #(contains? (set case-ids) %) (map :case-id preps)) "unknown preparation case" {}) + (require-valid! (= left (:remaining run)) "remaining slots disagree with attempts" {}) + (doseq [[case-id outcome] (:case-outcomes run)] + (check-case-outcome run case-id outcome)) + (doseq [prep preps] + (check-preparation prep)) + (doseq [[attempt-id attempt] (map-indexed vector (:attempts run))] + (require-valid! (= attempt-id (:attempt-id attempt)) "attempts are not chronological" {}) + (check-attempt attempt (get prep-map (:preparation-id attempt)))) + (check-completion run (:termination run))) + run) + +(defn create-run + "Creates an empty raw run from externally supplied identities and facts. + The plan defaults to 3 warmups, 10 repetitions and a 30000 ms case timeout. + For each case, all warmup slots precede measured slots. Failed attempts consume + their slot too; attempts retain chronological order across cases." + [{:keys [run-id started-at plan metadata analysis quantities]}] + (let [plan (merge {:warmups 3 :repetitions 10 :timeout-policy {:case-ms 30000}} plan)] + (check-run {:type :run-record + :run-id run-id :started-at started-at :ended-at nil + :plan plan :metadata metadata :analysis (analysis/resolve-settings analysis) + :quantities (merge quantities/descriptions quantities) + :preparations [] :attempts [] :case-outcomes {} :remaining (initial-remaining plan) + :termination {:reason :running}}))) + +(defn- check-open! + "Checks that a constructed or decoded run still accepts evidence." + [run] + (require-valid! (= :running (get-in run [:termination :reason])) "run already terminated" {})) + +(defn record-preparation + "Checks and appends new setup evidence without consuming an attempt slot. + The run must come from create-run or decode; earlier evidence is trusted." + [run preparation] + (check-open! run) + (require-valid! (not (contains? (:case-outcomes run) (:case-id preparation))) "case already terminated" {}) + (check-preparation preparation) + (require-valid! (contains? (:remaining run) (:case-id preparation)) "unknown preparation case" {}) + (require-valid! (not-any? #(= (:preparation-id preparation) (:preparation-id %)) + (:preparations run)) + "duplicate preparation" {}) + (update run :preparations conj preparation)) + +(defn record-attempt + "Checks and appends one new execution, consuming its scheduled slot. + The run must come from create-run or decode. + Missing preparation is allowed for failures before scene setup completes. + Earlier attempts are trusted; encode and decode check the complete record." + [run attempt] + (check-open! run) + (require-valid! (not (contains? (:case-outcomes run) (:case-id attempt))) "case already terminated" {}) + (let [attempt (-> (merge {:preparation-id nil :phases []} attempt) + (assoc :attempt-id (count (:attempts run))))] + (check-attempt attempt (some #(when (= (:preparation-id attempt) (:preparation-id %)) %) + (:preparations run))) + (-> run + (update :attempts conj attempt) + (update :remaining consume-slot attempt)))) + +(defn finish-run + "Records externally supplied end time and termination details, retaining gaps." + [run ended-at termination] + (check-open! run) + (require-valid! (sm/validate schema:termination termination) "invalid termination" {}) + (require-valid! (not= :running (:reason termination)) "termination needs an end reason" {}) + (check-completion run termination) + (assoc run :ended-at ended-at :termination termination)) + +(defn record-case-outcome + "Records the terminal outcome of one case. + + Args: + run: constructed or decoded run that accepts new evidence. + case-id: exact saved case identifier. + outcome: completion, failure, timeout or interruption with available cause. + + Returns: + The run with its case outcome. + + Throws: + ExceptionInfo for unknown cases, duplicate outcomes or malformed evidence." + [run case-id outcome] + (check-open! run) + (require-valid! (not (contains? (:case-outcomes run) case-id)) "duplicate case outcome" {}) + (check-case-outcome run case-id outcome) + (assoc-in run [:case-outcomes case-id] outcome)) + +(defn merge-runs + "Returns a new validated run that appends `additional`'s measured attempts to + `base`. + + Both runs must be completed and compatible. The merged plan keeps `base`'s + warmups and adds `additional`'s measured repetitions. Warmup attempts + and preparations used only by them are excluded from `additional`. + + The merged record uses the base run's quantity descriptions. + The caller supplies the merged run id and may supply its end time." + [base additional {:keys [run-id ended-at]}] + (let [base (check-run base) + additional (check-run additional)] + (require-valid! (and (string? run-id) (seq run-id)) + "merged run needs a run id" {}) + (doseq [[side run] [[:base base] [:additional additional]]] + (when-not (= :completed (get-in run [:termination :reason])) + (incompatible! "only completed runs can be merged" + {:side side + :run-id (:run-id run) + :reason (get-in run [:termination :reason])}))) + (when-not (and (string? (:ended-at base)) + (string? (:ended-at additional))) + (incompatible! "completed runs need end times" + {:base (:ended-at base) + :additional (:ended-at additional)})) + (doseq [[path label] [[[:plan :cases] "case definitions"] + [[:plan :timeout-policy] "timeout policy"] + [[:plan :warmups] "warmup count"] + [[:analysis] "analysis settings"] + [[:metadata :git] "git provenance"] + [[:metadata :build] "build facts"] + [[:metadata :environment] "runtime environment"] + [[:metadata :scored?] "scored flag"]]] + (require-compatible! base additional path label)) + (doseq [run [base additional]] + (require-valid! (every? #(sm/validate quantities/schema:description %) + (vals (:quantities run))) + "cannot merge malformed quantity descriptions" {:run-id (:run-id run)})) + (when-not (= (update-vals (:quantities base) #(dissoc % :description)) + (update-vals (:quantities additional) #(dissoc % :description))) + (incompatible! "cannot merge runs with different quantity units" {})) + (let [base-attempts (:attempts base) + additional-measured (filterv #(not (:warmup? %)) (:attempts additional)) + base-preparations (:preparations base) + base-preparation-ids (into #{} (map :preparation-id) base-preparations) + used-preparation-ids (into #{} (keep :preparation-id) additional-measured) + additional-preparations (filterv #(contains? used-preparation-ids + (:preparation-id %)) + (:preparations additional))] + (when-let [duplicate (some #(contains? base-preparation-ids (:preparation-id %)) + additional-preparations)] + (incompatible! "cannot merge runs with duplicate preparation ids" + {:preparation-id (:preparation-id duplicate)})) + (let [offset (count base-attempts) + attempts (into (vec base-attempts) + (map-indexed (fn [index attempt] + (assoc attempt :attempt-id (+ offset index))) + additional-measured)) + plan (assoc (:plan base) + :repetitions (+ (get-in base [:plan :repetitions]) + (get-in additional [:plan :repetitions]))) + remaining (reduce consume-slot (initial-remaining plan) attempts) + ended-at (or ended-at + (let [base-end (:ended-at base) + additional-end (:ended-at additional)] + (if (neg? (compare base-end additional-end)) + additional-end + base-end))) + metadata (update (:metadata base) :diagnostics + #(assoc (or % {}) + :merge + {:base-run-id (:run-id base) + :additional-run-id (:run-id additional) + :additional-warmups-excluded + (count (filter :warmup? (:attempts additional))) + :merged-at ended-at}))] + (check-run {:type :run-record + :run-id run-id + :started-at (:started-at base) + :ended-at ended-at + :plan plan + :metadata metadata + :analysis (:analysis base) + :quantities (:quantities base) + :preparations (into (vec base-preparations) additional-preparations) + :attempts attempts + :case-outcomes (merge (:case-outcomes base) (:case-outcomes additional)) + :remaining remaining + :termination {:reason :completed}}))))) + +(defn- check-feature-configuration + "Requires evidence that matches each saved configuration's progress." + [{:keys [status build run effective-features failure reason]}] + (when (#{:ready :running :completed} status) + (require-valid! (some? build) "feature comparison configuration needs a build" {:status status})) + (when (#{:ready :completed} status) + (require-valid! (some? effective-features) + "feature comparison configuration needs effective features" {:status status})) + (when (#{:running :completed} status) + (require-valid! (some? run) "feature comparison configuration needs a run" {:status status})) + (when (= :completed status) + (require-valid! (some? (:exit-code run)) + "completed feature comparison configuration needs a run exit code" {}) + (require-valid! (zero? (:exit-code run)) + "completed feature comparison configuration needs a successful run" {})) + (when (= :failed status) + (require-valid! (or (some? failure) (some? reason)) + "failed feature comparison configuration needs a failure or reason" {})) + (when (= :unattempted status) + (require-valid! (some? reason) + "unattempted feature comparison configuration needs a reason" {}))) + +(defn- check-feature-edge + "Requires a path for completed analysis and a reason for unattempted work." + [{:keys [status path failure reason]}] + (case status + :completed + (require-valid! (some? path) "completed feature comparison edge needs a report path" {}) + + :failed + (require-valid! (or (some? failure) (some? reason)) + "failed feature comparison edge needs a failure or reason" {}) + + :unattempted + (require-valid! (some? reason) + "unattempted feature comparison edge needs a reason" {}) + + nil)) + +(defn check-feature-comparison + "Validates saved feature-comparison progress against its planned experiment. + Configurations and edges must match the pure planner after status evidence is + attached. Terminal records cannot retain active work, and completed work must + retain its build, run, effective-feature and report evidence." + [record] + (require-valid! (sm/validate schema:feature-comparison record) + "invalid schema:feature-comparison" {}) + (let [features (:features record) + configurations (:configurations record) + edges (:edges record) + expected (feature-plan/plan features + {:max-features (count features) + :dry-run true + :confirm true}) + terminal? (contains? #{:completed :failed :interrupted} (:status record)) + active? (comp #{:planned :building :ready :running} :status)] + (require-valid! (= features (:features expected)) + "feature comparison features disagree with normalized selection" {}) + (require-valid! (= (mapv #(select-keys % [:slug :selection]) configurations) + (:configurations expected)) + "feature comparison configurations disagree with planned features" {}) + (require-valid! (= (mapv #(select-keys % [:from :to :feature]) edges) + (:edges expected)) + "feature comparison edges disagree with planned configurations" {}) + (require-valid! (= (count (:cases record)) (count (set (map :id (:cases record))))) + "duplicate feature comparison case" {}) + (when terminal? + (require-valid! (some? (:ended-at record)) + "terminal feature comparison needs an end time" {}) + (require-valid! (not-any? active? configurations) + "terminal feature comparison still has active configurations" {}) + (require-valid! (not-any? active? edges) + "terminal feature comparison still has active edges" {})) + (when-not terminal? + (require-valid! (nil? (:ended-at record)) + "unfinished feature comparison has an end time" {})) + (when (= :planned (:status record)) + (require-valid! (every? (comp #{:planned} :status) configurations) + "planned feature comparison has started configurations" {}) + (require-valid! (every? (comp #{:planned} :status) edges) + "planned feature comparison has started edges" {})) + (when (= :completed (:status record)) + (require-valid! (every? (comp #{:completed} :status) configurations) + "completed feature comparison has unfinished configurations" {}) + (require-valid! (every? (comp #{:completed} :status) edges) + "completed feature comparison has unfinished edges" {})) + (doseq [configuration configurations] + (check-feature-configuration configuration)) + (doseq [edge edges] + (check-feature-edge edge))) + record) + +(defn check-record + "Validates a run, comparison or feature-comparison at a Transit boundary." + [record] + (case (:type record) + :run-record (check-run record) + :comparison (do + (require-valid! (sm/validate schema:comparison record) "invalid schema:comparison" {}) + record) + :feature-comparison (check-feature-comparison record) + (throw (ex-info "unsupported benchmark format" {:type ::unsupported-format})))) + +(defn encode + "Encodes a supported record with transit." + [record] + (codec/encode-str (check-record record))) + +(defn decode + "Decodes from transit and validates a supported record." + [text] + (check-record (transit/decode-str text))) diff --git a/frontend/performance/benches/render_wasm/runtime/camera.cljs b/frontend/performance/benches/render_wasm/runtime/camera.cljs new file mode 100644 index 0000000000..cc334fe604 --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/camera.cljs @@ -0,0 +1,151 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.camera + "Rendering and camera commands over a prepared snapshot. + + Commands accept a context or its Promise and return a Promise of the next + context. Pan, zoom and restore update its view. Starred variants record raw + elapsed time and renderer evidence in the current execution phase. + + Load waits for an animation frame and renders through Full. Pan and zoom + submit paced cached previews, settle, then render through Full. Restore + applies a view and renders immediately with SyncTiles. Render-sync uses the + blocking renderer path. Full denotes renderer submission completion. + + Measured async commands finish at the final renderer return captured in + `:end-ms`. Input resolution and argument validation precede timing; output + validation and event recording follow it. Traces retain renderer calls, + cached previews, waits and protocol endpoints." + (:require + [app.common.schema :as sm] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.runtime.commands :refer [defcommand]] + [benches.render-wasm.runtime.protocol :as protocol])) + +(defcommand render-sync + "Renders the prepared scene synchronously. + + Args: + rtx: context or Promise of it. + + Returns: + A Promise of the context after the blocking renderer call." + [rtx] + ((get-in rtx [:hooks :render-sync])) + rtx) + +(def ^:private valid-pan? + (sm/validator + [:map {:closed true} + [:dx ::sm/safe-number] + [:dy ::sm/safe-number] + [:steps [:int {:min 1}]] + [:settle-ms [:and ::sm/safe-number [:>= 0]]]])) + +(def ^:private valid-zoom? + (sm/validator + [:map {:closed true} + [:factor [:and ::sm/safe-number [:> 0]]] + [:steps [:int {:min 1}]] + [:settle-ms [:and ::sm/safe-number [:>= 0]]]])) + +(def ^:private valid-view? (sm/validator decl/schema:view)) + +(defn- pan-view + [view dx dy] + (-> view (update :x + dx) (update :y + dy))) + +(defn- zoom-view + [view factor] + (update view :scale * factor)) + +(defn- render-evidence + [evidence] + {:end-ms (:end-ms evidence) + :metrics {:renderer/call-elapsed (mapv :duration-ms (:slices evidence))} + :trace evidence}) + +(defn- camera-evidence + [evidence] + (-> (render-evidence evidence) + (assoc-in [:metrics :renderer/cache-call-elapsed] (mapv :duration-ms (:cached-slices evidence))) + (assoc-in [:metrics :camera/finalization-to-full] [(:time-to-full-ms evidence)]))) + +(defn- run-camera + [rtx target {:keys [steps settle-ms]}] + (-> (protocol/start-camera! rtx {:settle-ms settle-ms}) + (protocol/animate-view! {:to target :steps steps}) + (protocol/finish-camera!))) + +(defcommand pan + "Moves the view by `:dx` and `:dy` through `:steps` cached previews. + Waits `:settle-ms` milliseconds, ends the interaction and renders until Full. + + Args: + rtx: context or Promise of it. + options: finite offsets, positive step count and nonnegative settling wait. + + Returns: + A Promise of the context with its updated view." + {:async [complete fail]} + [rtx {:keys [dx dy] :as options}] + {:pre [(valid-pan? options) (valid-view? (pan-view (:view rtx) dx dy))]} + (let [target (pan-view (:view rtx) dx dy)] + (-> (run-camera rtx target options) + (.then #(complete (assoc rtx :view target) (camera-evidence %))) + (.catch fail)))) + +(defcommand load + "Renders the uploaded snapshot until the renderer returns Full. + Waits for an animation frame before the first renderer call. + + Args: + rtx: context or Promise of it. + + Returns: + A Promise of the context after renderer completion." + {:async [complete fail]} + [rtx] + (let [hooks (:hooks rtx)] + (-> (protocol/drain hooks {:flags 0 :origin ((:now hooks)) :immediate false}) + (.then #(complete rtx (assoc-in (render-evidence %) + [:metrics :renderer/prepared-scene-to-full] [(:full-ms %)]))) + (.catch fail)))) + +(defcommand zoom + "Multiplies view scale through paced cached previews, then renders through Full. + + Args: + rtx: context or Promise of it. + options: positive finite factor, positive steps and nonnegative settle-ms. + + Returns: + A Promise of the context with its updated scale." + {:async [complete fail]} + [rtx {:keys [factor] :as options}] + {:pre [(valid-zoom? options) (valid-view? (zoom-view (:view rtx) factor))]} + (let [target (zoom-view (:view rtx) factor)] + (-> (run-camera rtx target options) + (.then #(complete (assoc rtx :view target) (camera-evidence %))) + (.catch fail)))) + +(defcommand restore + "Restores a view and renders immediately through Full with SyncTiles. + + Args: + rtx: context or Promise of it. + view: scale and offsets to restore. Its resolved viewport must match rtx. + + Returns: + A Promise of the context with the restored view." + {:async [complete fail]} + [rtx view] + {:pre [(valid-view? view) + (= (decl/resolve-viewport view) (decl/resolve-viewport (:view rtx)))]} + (-> (protocol/restore (:hooks rtx) view) + (.then #(complete (assoc rtx :view view) (render-evidence %))) + (.catch fail))) diff --git a/frontend/performance/benches/render_wasm/runtime/commands.clj b/frontend/performance/benches/render_wasm/runtime/commands.clj new file mode 100644 index 0000000000..438e71eb85 --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/commands.clj @@ -0,0 +1,90 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.commands + "Declarations of renderer commands. + A command transforms a command context, a map of case data, a scene snapshot, + the current camera view and hooks. A scene is a recipe for shapes; a case + pairs a scene with its work. A snapshot contains concrete shapes and their + references. The view stores zoom scale and camera offsets. Hooks are functions + for clocks, scheduling and renderer operations. A measured command records + an event containing raw numeric values for named quantities, called metrics.") + +(defmacro defcommand + "Defines a base command and its measured function with a `*` suffix. + + Both accept a command context or its Promise and return a Promise of the next + context. A synchronous body returns that context. The measured function + requires an active command execution and records one event with + `:command/elapsed`, the body duration in milliseconds. + + Options: + - `:measured-variant false` defines a composite's base function. A composite + combines commands and records events from the measured commands it calls. + - `:async [complete fail]` lets the body complete asynchronously. Successful + completion requires `complete`; `fail` reports a cause. The first call to + either callback determines the outcome. A returned Promise forwards its + rejection to `fail`. + + `complete` accepts the next context and an optional map. `:end-ms` supplies + the clock reading at the end of the work. When the map omits `:end-ms`, + `complete` captures a reading on entry. `:metrics` maps namespaced metric + keywords to nonempty vectors of raw numbers. + `:trace` contains renderer details such as individual render calls and waits. + + Argument destructuring and `:pre` conditions precede work and timing in every + build. A failed condition rejects the returned Promise. The condition map + accepts `:pre`. Output validation and event recording follow the endpoint." + [command & forms] + (let [[doc forms] (if (string? (first forms)) + [(first forms) (next forms)] + [nil forms]) + [options forms] (if (map? (first forms)) + [(first forms) (next forms)] + [{} forms]) + [argv & body] forms + [conditions body] (if (and (map? (first body)) (next body)) + [(first body) (next body)] + [nil body]) + measured (get options :measured-variant true) + handlers (:async options) + command-id (keyword (str (get-in &env [:ns :name])) (name command)) + starred (symbol (str command "*")) + prepare (gensym "prepare")] + (when-not (and (simple-symbol? command) + (vector? argv) + (seq argv) + (boolean? measured) + (every? #{:measured-variant :async} (keys options)) + (or (not (contains? options :async)) + (and (vector? handlers) + (= 2 (count handlers)) + (every? simple-symbol? handlers) + (apply distinct? handlers))) + (every? #{:pre} (keys conditions)) + (or (nil? (:pre conditions)) (vector? (:pre conditions)))) + (throw (ex-info "invalid command declaration" {:command command}))) + `(let [~prepare (fn ~argv + ~@(for [condition (:pre conditions)] + `(when-not ~condition + (throw (ex-info "command precondition failed" + {:type ::invalid-arguments + :command ~command-id + :condition '~condition})))) + (fn ~(or handlers []) ~@body))] + (defn ~command + ~@(when doc [doc]) + [rtx# & args#] + (invoke {:command ~command-id :measured false :composite ~(not measured) + :async ~(some? handlers)} + rtx# ~prepare args#)) + ~@(when measured + [`(defn ~starred + ~(str "Measured variant of `" command "`." (when doc (str " " doc))) + [rtx# & args#] + (invoke {:command ~command-id :measured true :composite false + :async ~(some? handlers)} + rtx# ~prepare args#))])))) diff --git a/frontend/performance/benches/render_wasm/runtime/commands.cljs b/frontend/performance/benches/render_wasm/runtime/commands.cljs new file mode 100644 index 0000000000..6fa8f86ecc --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/commands.cljs @@ -0,0 +1,241 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.commands + "Execution of renderer commands and collection of their measurements. + + A command transforms a command context: a map containing case data (`:case`), + the prepared scene snapshot (`:scene`), the current camera view (`:view`) and + functions for clocks, scheduling and renderer operations (`:hooks`). A scene + is a recipe for shapes; a case pairs a scene with its work. A snapshot contains + the concrete shapes and their references. A view contains zoom scale, camera + offsets and optional viewport settings for canvas dimensions and pixel ratio. + The `:check` hook is a guard: + returning authorizes work, and throwing an error rejects it. + + A command execution evaluates a function from one context to the next and + records measured command calls in completion order. Each call records an + event with the command name and its metrics. A metric names a quantity; + each event keeps its raw numeric + values in a vector. An optional trace contains details such as render calls + and waits. Each execution owns its event history and authorizes work while + active. Its outcome contains the next context or an error and earlier events." + (:require-macros [benches.render-wasm.runtime.commands]) + (:require + [app.common.schema :as sm] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors :refer [attempt failure?]])) + +(def ^:private schema:context + [:map {:closed true} + [:case [:map [:id qualified-keyword?]]] + [:scene [:map [:objects map?] [:refs map?]]] + [:view decl/schema:view] + [:hooks [:and + [:map-of :keyword fn?] + [:map + [:now fn?] + [:check fn?] + [:frame fn?] + [:sleep fn?] + [:render fn?] + [:render-sync fn?] + [:render-from-cache fn?] + [:set-view fn?] + [:set-view-start fn?] + [:set-view-end fn?]]]]]) + +(def ^:private valid-context? (sm/validator schema:context)) +(def ^:private schema:completion + [:map {:closed true} + [:end-ms {:optional true} number?] + [:metrics {:optional true} + [:map-of qualified-keyword? [:vector {:min 1} number?]]] + [:trace {:optional true} :any]]) +(def ^:private valid-completion? (sm/validator schema:completion)) +(def ^:private owners (js/WeakMap.)) + +(defn- check-context + [rtx] + (when-not (valid-context? rtx) + (throw (ex-info "invalid command context" + {:type ::invalid-context + ::sm/explain (sm/explain schema:context rtx)})))) + +(defn- check-output + [rtx output] + (check-context output) + (when-not (identical? (get-in rtx [:hooks :check]) + (get-in output [:hooks :check])) + (throw (ex-info "command changed execution ownership" + {:type ::ownership-lost}))) + ((get-in rtx [:hooks :check])) + output) + +(defn- scope-context + [rtx events diagnostics active] + (check-context rtx) + (let [previous-check (get-in rtx [:hooks :check]) + previous-owner (.get owners previous-check) + base-check (or (:base-check previous-owner) previous-check) + check (fn [] + (when-not @active + (throw (ex-info "command execution has ended" + {:type ::inactive-execution}))) + (base-check)) + scoped (assoc-in rtx [:hooks :check] check)] + (.set owners check {:base-check base-check + :record #(do (check) (vswap! events conj %)) + :diagnose #(when @active (vswap! diagnostics conj %))}) + (check) + scoped)) + +(defn ^:async ^:no-doc execute + "Runs `run` with a command context and resolves to its outcome. + + `run` returns the next context or its Promise. A successful outcome contains + `:outcome :completed`, `:context` and ordered `:events`. A failed outcome + contains `:outcome :failed`, `:error` and events recorded before failure. + Existing Errors retain their identity; other thrown values become Errors + that retain the value in `:thrown`. + + The context's guard authorizes work during this execution. Passing a returned + context to another execution establishes a new authorization. Repeated calls + to asynchronous completion callbacks can supply `:diagnostics` while the + execution is active. Returned contexts must retain this execution's `:check` + guard. A changed guard rejects completion." + [rtx run] + (let [events (volatile! []) + diagnostics (volatile! []) + active (volatile! true) + result (try + (attempt :catch-fatal + (let [scoped (scope-context (await rtx) events diagnostics active)] + (check-output scoped (await (run scoped))))) + (finally + (vreset! active false)))] + (cond-> (if (failure? result) + {:outcome :failed :error result :events @events} + {:outcome :completed :context result :events @events}) + (seq @diagnostics) (assoc :diagnostics @diagnostics)))) + +(defn- prepare-invocation + [{:keys [command measured] :as invocation} rtx prepare args] + (check-context rtx) + (let [{:keys [check now]} (:hooks rtx) + owner (.get owners check)] + (check) + (when (and measured (nil? owner)) + (throw (ex-info "measured command requires an execution" + {:type ::missing-execution :command command}))) + (assoc invocation + :input rtx + :core (apply prepare rtx args) + :now now + :record (:record owner) + :diagnose (:diagnose owner)))) + +(defn- thenable? + [output] + (and (some? output) (fn? (.-then output)))) + +(defn- run-core + [{:keys [command measured composite core now] :as invocation}] + (if composite + (assoc invocation :output (core)) + (let [start (when measured (now)) + output (core) + end (when measured (now))] + (when (thenable? output) + (.catch (js/Promise.resolve output) errors/as-failure) + (throw (ex-info "synchronous command returned asynchronous work" + {:type ::async-result :command command}))) + (cond-> (assoc invocation :output output) + measured (assoc :metrics {:command/elapsed [(- end start)]}))))) + +(defn- finish-invocation + [{:keys [measured input output record] :as invocation}] + (check-output input output) + (when (contains? invocation :trace) + (try + (codec/check-value (:trace invocation)) + (codec/encode-str (:trace invocation)) + (catch :default cause + (throw (ex-info "command trace cannot cross Transit" + {:type ::invalid-trace} cause))))) + (when measured + (record (select-keys invocation [:command :metrics :trace]))) + output) + +(defn- complete-invocation + [{:keys [command measured now] :as invocation} output evidence start] + ;; A completion callback can run after the renderer returns. A supplied + ;; endpoint preserves the renderer return time across that scheduling delay. + (let [end (when measured + (if (and (map? evidence) (contains? evidence :end-ms)) + (:end-ms evidence) + (now)))] + (when-not (and (valid-completion? evidence) + (not (contains? (:metrics evidence) :command/elapsed))) + (throw (ex-info "invalid command completion evidence" + {:type ::invalid-completion :command command + ::sm/explain (sm/explain schema:completion evidence)}))) + (cond-> (merge invocation evidence {:output output}) + measured (assoc-in [:metrics :command/elapsed] [(- end start)])))) + +(defn- claim-terminal + [terminal diagnose command signal] + (if-some [first-signal @terminal] + (do + (when diagnose + (diagnose {:type ::duplicate-terminal :command command + :first-signal first-signal :signal signal})) + false) + (do + (vreset! terminal signal) + true))) + +(defn- run-async-core + [{:keys [command measured core now diagnose] :as invocation}] + (js/Promise. + (fn [resolve reject] + (let [terminal (volatile! nil) + start (volatile! nil) + complete (fn complete + ([output] (complete output {})) + ([output evidence] + (when (claim-terminal terminal diagnose command :complete) + (let [result (attempt :catch-fatal + (-> (complete-invocation invocation output evidence @start) + (finish-invocation)))] + (if (failure? result) (reject result) (resolve result)))) + nil)) + fail (fn [cause] + (when (claim-terminal terminal diagnose command :fail) + (reject (errors/as-failure cause))) + nil) + result (attempt :catch-fatal + (vreset! start (when measured (now))) + (let [returned (core complete fail)] + (when (thenable? returned) + (.catch (js/Promise.resolve returned) fail))))] + (when (failure? result) + (fail result)))))) + +(defn ^:async invoke + "Runs a command with a context or its Promise and returns the next context. + Resolves to that context as a Promise; errors reject it. A measured command + records one event with elapsed time, additional metric values and any trace." + [command rtx prepare args] + (let [invocation (prepare-invocation command (await rtx) prepare args)] + (if (:async command) + (await (run-async-core invocation)) + (let [result (run-core invocation)] + (finish-invocation + (if (:composite command) + (assoc result :output (await (:output result))) + result)))))) diff --git a/frontend/performance/benches/render_wasm/runtime/protocol.clj b/frontend/performance/benches/render_wasm/runtime/protocol.clj new file mode 100644 index 0000000000..0b8768b259 --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/protocol.clj @@ -0,0 +1,30 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.protocol + "Forms for evaluating renderer operations as Promises. + A Promise represents a result that can complete with a value or reject with + a cause. These forms include synchronous errors in that result.") + +(defmacro resolve-or-reject + "Evaluates `body` and returns a Promise of its result. + A synchronous throw rejects the Promise." + [& body] + `(try + (js/Promise.resolve (do ~@body)) + (catch :default cause# + (js/Promise.reject cause#)))) + +(defmacro resolve-or-reject-with + "Evaluates `body` and returns a Promise of its result. + A synchronous throw rejects with `on-error`, evaluated with `cause` bound + to the thrown value." + [cause on-error & body] + `(try + (js/Promise.resolve (do ~@body)) + (catch :default cause# + (let [~cause cause#] + (js/Promise.reject ~on-error))))) diff --git a/frontend/performance/benches/render_wasm/runtime/protocol.cljs b/frontend/performance/benches/render_wasm/runtime/protocol.cljs new file mode 100644 index 0000000000..84ed08f9ba --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/protocol.cljs @@ -0,0 +1,424 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.protocol + "Rendering until completion and timing camera movement. + + A frame type describes the progress of one renderer call. A render drain is + a sequence of calls that ends when the renderer returns Full. Each call + produces a slice: its timestamp, flags, frame type and duration. Partial and + ViewportReady schedule another call on the next animation frame. Full marks + renderer submission of the completed request. GPU execution and display + proceed asynchronously. + + A view contains zoom `:scale` and offsets `:x` and `:y`. A cached preview + draws from existing tiles, the fixed regions of the scene that the renderer + caches. A camera session starts an interaction, applies views with cached + previews, waits for the requested settling time, ends the interaction and + renders until Full. Each preview waits for an animation frame. Explicit + pauses can extend the active interval between camera start and settling. + Camera finalization starts immediately before ending the interaction and + ends at the final renderer return. + + ## Hooks and errors + + Hooks are functions that supply clock, scheduling and renderer operations: + + - `:now` returns a clock reading in milliseconds. + - `:frame` resolves a Promise with the next animation frame's timestamp. + - `:sleep` accepts a requested wait in milliseconds and returns its Promise. + - `:check` is a guard. Returning authorizes work; throwing rejects it. It can + report cancellation, an expired deadline, loss of the WebGL2 graphics + context or replacement of the active renderer owner. An owner is the + execution authorized to use the renderer. + - `:render` accepts a timestamp and flags and returns a frame type. + - `:render-from-cache` draws one cached preview. + - `:set-view` accepts a view containing scale and both offsets. + - `:set-view-start` and `:set-view-end` start and end the interaction. + + Guards run before work, after awaited frames and waits, and around render + calls. Guard and renderer errors reject the returned Promises. Errors keep + available traces of render calls and pauses under `:partial`. + + Frame types come from `app.common.render-wasm.wasm`. Partial and ViewportReady + continue rendering, Full completes it, and other values reject with + `::unexpected-frame-type`. Each subsequent call receives the preceding frame + type as its flags. SyncTiles asks the renderer to update which tiles the + current view requires. + View restoration and camera finalization use that flag for their first call + and start immediately. A drain can also wait for an animation frame before + its first call. Later calls wait for animation frames. + + ## Render calls and measured durations + + A trace records individual calls and waits. Durations use milliseconds and + preserve raw numeric values, including zero. A drain origin is the clock + reading from which its completion durations start. + + - `:slices` contains one entry per renderer call: + - `:timestamp`: the animation frame's timestamp or the current clock + reading for an immediate call. + - `:flags`: the flags passed to the renderer. + - `:frame-type`: the returned frame type. + - `:duration-ms`: elapsed time during the call. + - `:cached-slices` contains one entry per cached preview: + - `:timestamp`: the animation frame's timestamp for the view update. + - `:duration-ms`: elapsed time during the cached render call. + - `:view-and-preview-ms`: elapsed time for the view update and preview. + - `:viewport-ready-ms` measures from the drain origin to the first call that + returns ViewportReady or Full. It equals `:full-ms` when the first such + call returns Full. + - `:full-ms` measures from the drain origin to the call that returns Full. + - `:end-ms` is the clock reading immediately after that final call returns. + - `:interact-ms` measures from immediately before camera start through the + final renderer return. + - `:active-ms` measures from camera start through the last cached preview and + its guard check, or through an explicit pause that follows the preview. + - `:authored-sleeps` contains requested and observed durations of explicit + pauses during the active interval. + - `:settling-requested-ms` is the requested settling wait. + - `:settling-actual-ms` measures from the end of the active interval to the + start of camera finalization. + - `:set-view-end-ms` measures the call that ends the interaction. + - `:time-to-viewport-ready-ms` measures from camera finalization to the first + ViewportReady or Full result. `:time-to-full-ms` ends at Full. Both include + the call that ends the interaction. + - `:last-input-to-full-ms` measures from immediately before the last view + update to the final renderer return. The initial input time equals camera + start; each view update replaces it." + (:require-macros + [benches.render-wasm.runtime.protocol :refer [resolve-or-reject resolve-or-reject-with]]) + (:require + [app.common.render-wasm.wasm :as wasm] + [benches.render-wasm.errors :as errors])) + + +(defn- partial-error + "Returns an error with completed traces under `:partial`. + Trace fields already present in the error take precedence over `evidence`." + [cause evidence] + (errors/with-context cause (errors/message-of cause "renderer protocol failed") + {:partial evidence})) + +(defn- next-render-timestamp + "Resolves to the current clock reading for an immediate render, or to the + next animation frame's timestamp." + [{:keys [now frame]} immediate?] + (if immediate? + (js/Promise.resolve (now)) + (frame))) + + +(defn- render-slice + "Returns the updated drain state, frame type and endpoint for one render call. + The state includes the completed call in its `:slices` trace." + [{:keys [now check render]} + {:keys [flags] :as state} + timestamp] + (check) + (let [started-ms (now) + frame-type (render timestamp flags) + ended-ms (now) + slice {:timestamp timestamp + :flags flags + :frame-type frame-type + :duration-ms (- ended-ms started-ms)}] + (try + (check) + (catch :default cause + (throw (partial-error cause {:slices (conj (:slices state) slice)})))) + {:state (update state :slices conj slice) + :frame-type frame-type + :ended-ms ended-ms})) + + +(defn- valid-frame-type? + "Recognizes the progressive and final renderer frame types." + [frame-type] + (or (= frame-type wasm/FRAME_TYPE_PARTIAL) + (= frame-type wasm/FRAME_TYPE_VIEWPORT_READY) + (= frame-type wasm/FRAME_TYPE_FULL))) + + +(defn- viewport-ready-time + "Returns the elapsed time to the first ViewportReady or Full result." + [current-ready-ms frame-type ended-ms origin-ms] + (or current-ready-ms + (when (or (= frame-type wasm/FRAME_TYPE_VIEWPORT_READY) + (= frame-type wasm/FRAME_TYPE_FULL)) + (- ended-ms origin-ms)))) + + +(defn- advance-drain-state + "Updates the continuation flags and completion time for a returned frame type. + An unexpected type throws an error containing the completed render calls." + [{:keys [viewport-ready-ms] :as state} + frame-type + ended-ms + origin-ms] + (when-not (valid-frame-type? frame-type) + (throw (ex-info (str "unexpected frame type: " (pr-str frame-type)) + {:type ::unexpected-frame-type + :frame-type frame-type + :partial (select-keys state [:slices :viewport-ready-ms])}))) + (assoc state + :flags frame-type + :viewport-ready-ms + (viewport-ready-time viewport-ready-ms + frame-type + ended-ms + origin-ms))) + +(defn- drain-complete? + "Returns whether the renderer reported Full completion." + [frame-type] + (= frame-type wasm/FRAME_TYPE_FULL)) + +(defn- drain-result + "Returns traces of render calls, the final clock reading and completion durations + measured from the drain origin." + [{:keys [slices viewport-ready-ms]} + ended-ms + origin-ms] + {:slices slices + :end-ms ended-ms + :viewport-ready-ms viewport-ready-ms + :full-ms (- ended-ms origin-ms)}) + + +(defn drain + "Renders until a call returns Full and resolves to its traces and durations. + + Options: + - `:flags` supplies the first render call's flags. + - `:origin` supplies the starting clock reading for completion durations. + - `:immediate` starts the first call immediately when true. Otherwise, the + first call waits for an animation frame. Subsequent calls wait for frames. + + The result contains `:slices`, the traces of render calls; `:end-ms`, the final + renderer clock reading; and `:viewport-ready-ms` and `:full-ms`, durations + measured from `:origin`. Partial and ViewportReady continue with the returned + frame type as the next flags. A guard failure or unexpected frame type + rejects the Promise with completed traces in the error's `:partial` field." + [{:keys [check] :as hooks} + {:keys [flags origin immediate]}] + (resolve-or-reject + (check) + (letfn [(step [state immediate?] + (-> (next-render-timestamp hooks immediate?) + (.then ;; recurse or drain + (fn [timestamp] + (let [{:keys [frame-type ended-ms] :as rendered} + (render-slice hooks state timestamp) + next-state + (advance-drain-state (:state rendered) + frame-type + ended-ms + origin)] + + (if (drain-complete? frame-type) + (drain-result next-state ended-ms origin) + (step next-state false))))) + (.catch (fn [cause] + (throw (partial-error cause (select-keys state [:slices :viewport-ready-ms])))))))] + (step {:flags flags + :slices [] + :viewport-ready-ms nil} + immediate)))) + +(defn restore + "Restores `view` and renders its tiles until Full. + The first call uses SyncTiles and starts immediately. Completion durations + begin after restoring the view and ending the interaction. Returns a Promise + with recorded render calls, completion durations and the final clock reading + `:end-ms`." + [{:keys [now check set-view set-view-start set-view-end] :as hooks} view] + (resolve-or-reject + (check) + (set-view-start) + (set-view view) + (set-view-end) + (drain hooks {:flags wasm/RENDER_FLAG_SYNC_TILES + :origin (now) + :immediate true}))) + +(defn start-camera! + "Starts a camera session from the context's current `:view` and `:hooks`. + Resolves to the session with its start time and requested settling wait. + The initial active interval covers the operation that starts the camera. + `:settle-ms` must give a finite, nonnegative duration in milliseconds." + [{:keys [hooks view]} {:keys [settle-ms]}] + (resolve-or-reject + (when-not (and (number? settle-ms) + (js/Number.isFinite settle-ms) + (<= 0 settle-ms)) + (throw (ex-info "settle-ms must be a nonnegative finite number" + {:type ::invalid-settle-ms :settle-ms settle-ms}))) + (let [{:keys [now check set-view-start]} hooks] + (check) + (let [started-ms (now)] + (set-view-start) + {:hooks hooks + :view view + :settle-ms settle-ms + :started-ms started-ms + :last-input-ms started-ms + :active-end-ms (now) + :cached-slices [] + :authored-sleeps []})))) + +(defn preview-view! + "Updates the view and draws one cached preview on the next animation frame. + Resolves to the session with the new view and measured preview durations. + Errors retain completed previews and pauses under `:partial`." + [session-p view] + (-> (js/Promise.resolve session-p) + (.then (fn [{:keys [hooks] :as session}] + (let [{:keys [now frame check render-from-cache set-view]} hooks + evidence (select-keys session [:cached-slices :authored-sleeps])] + (resolve-or-reject-with cause + (partial-error cause evidence) + (check) + (-> (frame) + (.then (fn [timestamp] + (check) + (let [input-ms (now) + call-start (now)] + (set-view view) + (let [cache-start (now)] + (render-from-cache) + (let [cache-end (now) + slices (conj (:cached-slices session) + {:timestamp timestamp + :duration-ms (- cache-end cache-start) + :view-and-preview-ms (- cache-end call-start)})] + (try + (check) + (catch :default cause + (throw (partial-error cause (assoc evidence :cached-slices slices))))) + (assoc session :view view :last-input-ms input-ms + :active-end-ms (now) :cached-slices slices)))))) + (.catch (fn [cause] (throw (partial-error cause evidence))))))))))) + +(defn- interpolate + "Interpolates $v(t) = (1-t)a + tb$ between finite endpoints. + + Args: + a: starting value. + b: target value. + t: interpolation fraction in [0, 1]. + + Returns: + The interpolated value, preserving positive endpoints near zero." + [a b t] + (if (< t 0.5) + (+ a (* t (- b a))) + (+ b (* (- 1 t) (- a b))))) + +(defn animate-view! + "Moves to `to` through `steps` linearly interpolated views in sequence. + Each update waits for an animation frame and draws one cached preview. + Resolves to the session with its final view. Steps must be a positive integer; + errors retain completed previews and pauses under `:partial`." + [session-p {:keys [to steps]}] + (-> (js/Promise.resolve session-p) + (.then (fn [{from :view :as session}] + (when-not (and (integer? steps) (pos? steps)) + (throw (ex-info "camera steps must be a positive integer" + {:type ::invalid-steps :steps steps + :partial (select-keys session [:cached-slices :authored-sleeps])}))) + (reduce (fn [session-p i] + (let [t (/ i steps) + view (if (= i steps) + to + {:scale (interpolate (:scale from) (:scale to) t) + :x (interpolate (:x from) (:x to) t) + :y (interpolate (:y from) (:y to) t)})] + (preview-view! session-p + view))) + (js/Promise.resolve session) + (range 1 (inc steps))))))) + +(defn sleep! + "Pauses the camera session and records the requested and observed wait. + The wait must be finite and nonnegative. The pause extends the active + interval. Resolves to the updated session; + errors retain completed previews and pauses under `:partial`." + [session-p requested-ms] + (-> (js/Promise.resolve session-p) + (.then (fn [{:keys [hooks] :as session}] + (let [{:keys [now sleep check]} hooks + evidence (select-keys session [:cached-slices :authored-sleeps])] + (resolve-or-reject-with cause + (partial-error cause evidence) + (when-not (and (number? requested-ms) + (js/Number.isFinite requested-ms) + (<= 0 requested-ms)) + (throw (ex-info "sleep duration must be a nonnegative finite number" + {:type ::invalid-sleep :requested-ms requested-ms}))) + (check) + (let [started-ms (now)] + (-> (sleep requested-ms) + (.then (fn [_] + (check) + (let [ended-ms (now)] + (-> session + (assoc :active-end-ms ended-ms) + (update :authored-sleeps conj + {:requested-ms requested-ms + :actual-ms (- ended-ms started-ms)}))))) + (.catch (fn [cause] (throw (partial-error cause evidence)))))))))))) + +(defn- finish-interaction + "Ends the interaction and resolves to camera durations and recorded render calls. + Errors retain completed previews and pauses under `:partial`." + [{:keys [now check set-view-end] :as hooks} + {:keys [settle-ms started-ms cached-slices last-input-ms active-end-ms authored-sleeps]}] + (check) + + (let [finalization-start (now) + t0 (now)] + (set-view-end) + + (let [set-view-end-ms (- (now) t0)] + (-> (drain hooks + {:flags wasm/RENDER_FLAG_SYNC_TILES + :origin finalization-start + :immediate true}) + (.then + (fn [{:keys [slices end-ms viewport-ready-ms full-ms]}] + {:cached-slices cached-slices + :slices slices + :end-ms end-ms + :interact-ms (- end-ms started-ms) + :active-ms (- active-end-ms started-ms) + :authored-sleeps authored-sleeps + :settling-requested-ms settle-ms + :settling-actual-ms (- finalization-start active-end-ms) + :set-view-end-ms set-view-end-ms + :time-to-viewport-ready-ms viewport-ready-ms + :time-to-full-ms full-ms + :last-input-to-full-ms (- end-ms last-input-ms)})) + (.catch (fn [cause] + (throw (partial-error cause {:cached-slices cached-slices + :authored-sleeps authored-sleeps})))))))) + + +(defn finish-camera! + "Waits for settling, ends the interaction and renders until Full. + Resolves to cached previews, traces of final render calls, pauses and measured + camera durations. `:end-ms` is the clock reading at the final renderer return. + Errors retain completed previews and pauses under `:partial`." + [session-p] + (-> (js/Promise.resolve session-p) + (.then (fn [{:keys [hooks settle-ms] :as session}] + (let [evidence (select-keys session [:cached-slices :authored-sleeps])] + (resolve-or-reject-with cause + (partial-error cause evidence) + ((:check hooks)) + (-> ((:sleep hooks) settle-ms) + (.then (fn [_] (finish-interaction hooks session))) + (.catch (fn [cause] (throw (partial-error cause evidence))))))))))) diff --git a/frontend/performance/benches/render_wasm/scenes/masks.cljs b/frontend/performance/benches/render_wasm/scenes/masks.cljs new file mode 100644 index 0000000000..8deb1c4cbb --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/masks.cljs @@ -0,0 +1,59 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.masks + "Fixed grid of masked groups with circle masks and ordered descendants." + (:require + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.runtime.camera :as camera])) + +(def ^:const ^:private default-params + {:width 1920 :height 1080}) + +(def ^:const ^:private schema:params + [:map {:closed true} + [:width [:int {:min 1920}]] + [:height [:int {:min 1080}]]]) + +(decl/defscene :masks + {:description "Twenty-four circle masks with ordered striped descendants" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)}} + (doseq [i (range 24)] + (let [x (+ 25 (* 310 (mod i 6))) + y (+ 25 (* 255 (quot i 6)))] + (sb/group [:mask i] {:masked-group true} + (sb/circle {:x x :y y :width 180 :height 180 + :fills [{:fill-color "#ffffff"}]}) + (doseq [j (range 8)] + (sb/rect {:x (+ x (* 28 j) -22) + :y (- y 12) + :width 25 + :height 205 + :fills [{:fill-color (if (even? j) "#1254b8" "#ff8b38")}]})))))))) + +(def ^:const ^:private base-view {:scale 1 :x 0 :y 0}) + +(decl/defcase :masks/load + {:params default-params :view base-view :context :fresh :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :masks/pan + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 120 :dy 40 :settle-ms 100})) + +(decl/defcase :masks/zoom + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.6 :settle-ms 100})) diff --git a/frontend/performance/benches/render_wasm/scenes/paths.cljs b/frontend/performance/benches/render_wasm/scenes/paths.cljs new file mode 100644 index 0000000000..d96382ad30 --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/paths.cljs @@ -0,0 +1,78 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.paths + "Curved paths" + (:require + [app.common.schema :as sm] + [app.common.types.path :as path] + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :refer [rng-int]] + [benches.render-wasm.runtime.camera :as camera])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Config +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:const ^:private default-params + {:count 1000 :width 1920 :height 1080}) + +(def ^:const ^:private schema:params + [:map {:closed true} + [:count [:int {:min 1 :max 100000}]] + [:width ::sm/positive-safe-number] + [:height ::sm/positive-safe-number]]) + +(def ^:const ^:private base-view {:scale 1 :x 0 :y 0}) + +(defn- curved-content + [{:keys [width height]}] + (fn [rng] + (let [x (rng-int rng 0 width) + y (rng-int rng 0 height) + w (rng-int rng 20 100) + h (rng-int rng 20 100)] + (path/from-plain + [{:command :move-to :params {:x x :y y}} + {:command :curve-to + :params {:c1x (+ x w) :c1y y + :c2x (+ x w) :c2y (+ y h) + :x (+ x w) :y (+ y h)}} + {:command :line-to :params {:x x :y (+ y h)}} + {:command :close-path}])))) + + +(decl/defscene :paths + {:description "Curved paths" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)} + :defaults {:path {:content (curved-content params) + :fills (sb/gen-vector (sb/gen-fill)) + :strokes (sb/gen-vector (sb/gen-stroke 3))}}} + (doseq [_ (range (:count params))] + (sb/path))))) + + +(decl/defcase :paths/load + {:params default-params :view base-view :context :fresh :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :paths/pan + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 160 :dy 40 :settle-ms 100})) + +(decl/defcase :paths/zoom + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.5 :settle-ms 100})) diff --git a/frontend/performance/benches/render_wasm/scenes/plus.cljs b/frontend/performance/benches/render_wasm/scenes/plus.cljs new file mode 100644 index 0000000000..d4021bcd85 --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/plus.cljs @@ -0,0 +1,68 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.plus + "Plus signs made of two open line subpaths" + (:require + [app.common.types.path :as path] + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :refer [rng-int]] + [benches.render-wasm.runtime.camera :as camera])) + +(def ^:private default-params + {:count 1000 :width 1920 :height 1080}) + +(def ^:private schema:params + [:map {:closed true} + [:count [:int {:min 1 :max 100000}]] + [:width [:int {:min 100}]] + [:height [:int {:min 100}]]]) + +(defn- plus-content + [{:keys [width height]}] + (fn [rng] + (let [x (rng-int rng 20 width) + y (rng-int rng 20 height) + r (rng-int rng 10 40)] + (path/from-plain + [{:command :move-to :params {:x (- x r) :y y}} + {:command :line-to :params {:x (+ x r) :y y}} + {:command :move-to :params {:x x :y (- y r)}} + {:command :line-to :params {:x x :y (+ y r)}}])))) + + +(decl/defscene :plus + {:description "Plus signs made of two open line subpaths" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)} + :defaults {:path {:content (plus-content params) + :fills [] + :strokes (sb/gen-vector (sb/gen-stroke 4))}}} + (doseq [_ (range (:count params))] + (sb/path))))) + +(def ^:private base-view {:scale 1 :x 0 :y 0}) + +(decl/defcase :plus/load + {:params default-params :view base-view :context :fresh :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :plus/pan + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 140 :dy 60 :settle-ms 100})) + +(decl/defcase :plus/zoom + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.4 :settle-ms 100})) diff --git a/frontend/performance/benches/render_wasm/scenes/rects.cljs b/frontend/performance/benches/render_wasm/scenes/rects.cljs new file mode 100644 index 0000000000..6e8ec5b83c --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/rects.cljs @@ -0,0 +1,118 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.rects + "Rectangle scene, the smallest benchmark workload. + + Distributes rectangles over the canvas with varied sizes, translucent + fills and centered strokes, all drawn from the scope's seeded generator. + The `defscene` body produces the scene snapshot; `defcase` declares + the scene and its standard cases. The three cases share parameters, so + they render the same scene." + (:require + [app.common.schema :as sm] + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.runtime.camera :as camera])) + +;; This namespace is also loaded by Node discovery. Keep operation helpers +;; portable: browser API/helper imports belong in browser adapters supplied +;; through rtx, not in this require list or its transitive dependencies. +;; See declarations' dependency contract before adding a capability. + +(def ^:const ^:private default-params + {:count 1000 + :width 1920 + :height 1080 + :min-size 20 + :max-size 100}) + +(def ^:private schema:params + "Workload parameters. Sizes are finite. `:min-size` > `:max-size` is rejected" + [:and + [:map {:closed true} + [:count [:int {:min 1 :max 100000}]] + [:width ::sm/positive-safe-number] + [:height ::sm/positive-safe-number] + [:min-size ::sm/positive-safe-number] + [:max-size ::sm/positive-safe-number]] + [:fn {:error/message "min-size exceeds max-size"} + (fn [{:keys [min-size max-size]}] + (<= min-size max-size))]]) + +(defn- rect-defaults + "Attribute generators for one rectangle of the workload." + [{:keys [width height min-size max-size]}] + {:x (sb/gen-int 0 width) + :y (sb/gen-int 0 height) + :width (sb/gen-int min-size max-size) + :height (sb/gen-int min-size max-size) + :fills (sb/gen-vector (sb/gen-fill)) + :strokes (sb/gen-vector (sb/gen-stroke 10)) + :r1 (sb/gen-int 0 24) + :r2 (sb/gen-int 0 24) + :r3 (sb/gen-int 0 24) + :r4 (sb/gen-int 0 24)}) + +(decl/defscene :rects + {:description "Seeded rectangles with translucent fills and centered strokes" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 + :y 0 + :width (:width params) + :height (:height params)} + :defaults {:rect (rect-defaults params)}} + (doseq [_ (range (:count params))] + (sb/rect))))) + +(def ^:private base-view + {:scale 1 :x 0 :y 0}) + +(decl/defcase :rects/load + {:params default-params + :view base-view + :context :fresh + :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :rects/pan + {:params default-params + :view base-view + :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 200 :dy 0 :settle-ms 100})) + +(decl/defcase :rects/zoom + {:params default-params + :view base-view + :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.5 :settle-ms 100})) + +(decl/defcase :rects/blocking-full + {:params default-params + :view base-view + :context :fresh + :preparation {:render :none}} + [rtx] + (camera/render-sync* rtx)) + +(decl/defcase :rects/mixed-pans + {:params default-params + :view base-view + :context :reuse + :reset #(camera/restore* % (get-in % [:case :view]))} + [rtx] + (-> rtx + (camera/pan* {:dx 120 :dy 0 :steps 12 :settle-ms 50}) + (camera/pan {:dx 0 :dy 80 :steps 8 :settle-ms 50}) + (camera/pan* {:dx -120 :dy 0 :steps 12 :settle-ms 50}))) diff --git a/frontend/performance/benches/render_wasm/snapshot.cljs b/frontend/performance/benches/render_wasm/snapshot.cljs new file mode 100644 index 0000000000..50c4f337e2 --- /dev/null +++ b/frontend/performance/benches/render_wasm/snapshot.cljs @@ -0,0 +1,203 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.snapshot + "Utilities shared by generated and saved-page scenes. + + A scene snapshot is + `{:objects {uuid shape ...} :refs {label uuid ...}}`. + + - `:objects` is a Penpot page object map. + - `:refs` is a snapshot-local lookup from custom labels to object uuids. + + A scene snapshot is just static data. It is independent of camera, operations, + WASM buffers and renderer calls, and it never mutates after upload. + + In order for benchmark scenes to follow editor semantics faithfully, + validation uses penpot's facilities. We do perform a manual check for + reachability from the root here because neither the editor validator nor file + loading guarantees it. + + Authoring and scoped construction are ticket17; containers ticket18/19; + saved-page loading ticket21. + + Which shape types may own children is container semantics owned by + ticket18/19; this contract only checks hierarchy consistency." + (:require + [app.common.files.helpers :as cfh] + [app.common.files.validate :as cfv] + [app.common.geom.shapes.tree-seq :as gts] + [app.common.schema :as sm] + [app.common.types.page :as ctp] + [app.common.uuid :as uuid])) + +(def schema:snapshot + "Envelope schema for a scene snapshot." + [:map {:title "RendererBenchmarkSceneSnapshot"} + [:objects ctp/schema:objects] + [:refs [:map-of :any ::sm/uuid]]]) + +(defn- fail! + [hint data] + (throw (ex-info (str "Invalid scene snapshot: " hint) + (merge {:type ::invalid-snapshot + :hint hint} + data)))) + +(defn- check-schema! + [instance] + (when-not (sm/validate schema:snapshot instance) + (fail! "instance does not match the scene snapshot schema" + {::sm/explain (sm/explain schema:snapshot instance)}))) + +(defn- check-root! + [instance] + (let [root (get-in instance [:objects uuid/zero])] + (when-not (and (some? root) (cfh/root? root)) + (fail! "root object must be a frame at uuid/zero" + {:id uuid/zero})) + (when-not (= uuid/zero (:parent-id root)) + (fail! "root object :parent-id must be uuid/zero" + {:id uuid/zero + :parent-id (:parent-id root)})) + (when-not (= uuid/zero (:frame-id root)) + (fail! "root object :frame-id must be uuid/zero" + {:id uuid/zero + :frame-id (:frame-id root)})))) + +(defn- check-object-ids! + [objects] + (doseq [[id shape] objects] + (when-not (= id (:id shape)) + (fail! "object key does not match the shape :id" + {:key id + :shape-id (:id shape)})))) + +(defn- check-child-links! + "Guard the reachability walk against malformed child links. + + `objects` is an object map keyed by id. A `shape` can have children + in `:shapes`. When present, `:shapes` must be a vector, and each listed + child that exists must point back to the listing shape." + [objects] + (doseq [[id shape] objects + :let [children (:shapes shape)]] + (when (some? children) + (when-not (vector? children) + (fail! "shape :shapes must be a vector" + {:id id})) + (doseq [child-id children + :let [child (get objects child-id)] + :when (some? child)] + (when-not (= id (:parent-id child)) + (fail! "shape :shapes lists a child with a different :parent-id" + {:id id + :child child-id})))))) + +(defn- check-reachability! + "Check the root graph through the cycle-safe descendant walk. + + Runs `check-child-links!` first, which keeps the walk linear: one parent + per child and vector `:shapes` mean the walk cannot fan out over shared + children or loop on a cycle. + + Detects missing child ids, objects reachable more than once (shared + children or cycles), detached subtrees and a root listed as its own + descendant. Runs before the production validator, whose walk follows + `:shapes` links and would loop on a reachable cycle." + [objects] + (check-child-links! objects) + (let [ids (cfh/get-children-ids objects uuid/zero) + seen (set ids)] + + (when (contains? seen uuid/zero) + (fail! "root must not be a descendant of itself" + {:id uuid/zero})) + + (doseq [id ids] + (when-not (contains? objects id) + (fail! "shape :shapes references a missing object" + {:child id}))) + + (when-let [dup (first (for [[id n] (frequencies ids) :when (> n 1)] id))] + (fail! "an object is reachable more than once" + {:id dup})) + + (doseq [id (keys objects) + :when (not= id uuid/zero)] + (when-not (contains? seen id) + (fail! "object is not reachable from the root" + {:id id}))))) + +;; Captured-page policy (tickets 02/26; implementation pending): extraction +;; maps canonical component/detach-shape over every selected-page shape, +;; without resolving component libraries, then rejects remaining unsupported +;; references/resources before validation. Warn and record when detachment +;; changes shapes; do not carry document context or use editor detach flows. +;; Thus the empty file/libraries below describe a standalone snapshot, not +;; support for arbitrary component-bearing pages. Move reusable checks to +;; shared .cljc code for JVM extraction and CLJS loading; do not copy them. + +(defn- check-referential-integrity! + "Run the production referential validation over the reachable graph. + + `cfv/validate-shape` checks parent/child links, frame ids and geometry + with editor semantics. The file and page only carry the uuids the + validation error schema requires; scenes contain no component shapes. + Unreachable objects never reach this walk; `check-reachability!` owns + them." + [instance] + (let [file {:id uuid/zero :data {}} + page {:id uuid/zero :objects (:objects instance)} + errors (cfv/validate-shape uuid/zero file page {})] + (when-let [{:keys [hint] :as error} (first errors)] + (fail! hint (select-keys error [:code :shape-id :args]))))) + +(defn- check-refs! + [instance] + (let [objects (:objects instance)] + (doseq [[label target] (:refs instance)] + (when-not (contains? objects target) + (fail! "ref target is not present in :objects" + {:label label + :target target}))))) + +(defn validate! + "Validates a scene snapshot and returns it unchanged. + + Throws `ex-info` with `{:type ::invalid-snapshot :hint ...}` when the + snapshot is invalid. Schema failures also carry `::sm/explain`. + + Explicit `throw`s are used instead of assertions so the checks survive + elided assertions." + [instance] + (check-schema! instance) + (let [objects (:objects instance)] + (check-root! instance) + (check-object-ids! objects) + ;; Reachability must run before the production validator, because that walk + ;; follows `:shapes` links and would not terminate on a reachable cycle + (check-reachability! objects) + (check-referential-integrity! instance) + (check-refs! instance)) + instance) + +(defn upload-order + "Returns the instance objects as a vector in parent-before-child order. + + First root, then `:shapes` in order. + + Note that upload order carries no renderer semantics because a shape's child + list is uploaded from its own `:shapes` vector. + + Assumes a validated instance (every child id exists)." + [instance] + (vec (gts/get-children-seq uuid/zero (:objects instance)))) + +(defn ref-id + "Resolves an author label to the referenced object uuid, or nil." + [instance label] + (get-in instance [:refs label])) diff --git a/frontend/playwright.config.js b/frontend/playwright.config.js index a7e5496a7a..db8b9be4fc 100644 --- a/frontend/playwright.config.js +++ b/frontend/playwright.config.js @@ -48,6 +48,9 @@ export default defineConfig({ locale: "en-US", + /* Profiles without a theme follow the system one; run the UI in dark */ + colorScheme: "dark", + permissions: ["clipboard-write", "clipboard-read"], }, @@ -73,9 +76,7 @@ export default defineConfig({ }, { name: "ds", - // Accounts without a theme follow the system one; keep the visual - // specs on the dark theme so their screenshots stay comparable. - use: { ...devices["Desktop Chrome"], colorScheme: "dark" }, + use: { ...devices["Desktop Chrome"] }, testDir: "./playwright/ui/visual-specs", expect: { timeout: expectTimeout, diff --git a/frontend/playwright/data/dashboard/check-nitrate-sso-authorized.json b/frontend/playwright/data/dashboard/check-nitrate-sso-authorized.json index 0d710478b5..4adb4d785d 100644 --- a/frontend/playwright/data/dashboard/check-nitrate-sso-authorized.json +++ b/frontend/playwright/data/dashboard/check-nitrate-sso-authorized.json @@ -1 +1,3 @@ -{ "~:authorized": true } +{ + "~:authorized": true +} diff --git a/frontend/playwright/data/dashboard/get-teams-organization-switcher.json b/frontend/playwright/data/dashboard/get-teams-organization-switcher.json new file mode 100644 index 0000000000..91b9c2a20e --- /dev/null +++ b/frontend/playwright/data/dashboard/get-teams-organization-switcher.json @@ -0,0 +1,91 @@ +[ + { + "~:features": { + "~#set": [ + "layout/grid", + "styles/v2", + "fdata/pointer-map", + "fdata/objects-map", + "components/v2", + "fdata/shape-data-type" + ] + }, + "~:permissions": { + "~:type": "~:membership", + "~:is-owner": true, + "~:is-admin": true, + "~:can-edit": true + }, + "~:name": "Brand Studio", + "~:modified-at": "~m1713533116375", + "~:id": "~uc7ce0794-0992-8105-8004-38e630f40f6d", + "~:created-at": "~m1713533116375", + "~:is-default": true, + "~:organization": { + "~:id": "~u11111111-1111-4111-8111-111111111111", + "~:name": "Brand Studio", + "~:slug": "brand-studio", + "~:owner-id": "~u22222222-2222-4222-8222-222222222222", + "~:avatar-bg-url": null, + "~:permissions": { + "~:create-teams": "any" + } + } + }, + { + "~:features": { + "~#set": [ + "layout/grid", + "styles/v2", + "fdata/pointer-map", + "fdata/objects-map", + "components/v2", + "fdata/shape-data-type" + ] + }, + "~:permissions": { + "~:type": "~:membership", + "~:is-owner": false, + "~:is-admin": false, + "~:can-edit": true + }, + "~:name": "Product", + "~:modified-at": "~m1713533116375", + "~:id": "~udd33ff88-f4e5-8033-8003-8096cc07bdf3", + "~:created-at": "~m1713533116375", + "~:is-default": true, + "~:organization": { + "~:id": "~u33333333-3333-4333-8333-333333333333", + "~:name": "Product Studio With a Long Organization Name That Must Remain Available", + "~:slug": "product-studio", + "~:owner-id": "~u44444444-4444-4444-8444-444444444444", + "~:avatar-bg-url": null, + "~:permissions": { + "~:create-teams": "any" + } + } + }, + { + "~:features": { + "~#set": [ + "layout/grid", + "styles/v2", + "fdata/pointer-map", + "fdata/objects-map", + "components/v2", + "fdata/shape-data-type" + ] + }, + "~:permissions": { + "~:type": "~:membership", + "~:is-owner": true, + "~:is-admin": true, + "~:can-edit": true + }, + "~:name": "Personal projects", + "~:modified-at": "~m1713533116375", + "~:id": "~ue3c2fd6d-2d3e-47b3-87f6-41b467dff123", + "~:created-at": "~m1713533116375", + "~:is-default": true + } +] diff --git a/frontend/playwright/data/workspace/get-file-tokens-alias.json b/frontend/playwright/data/workspace/get-file-tokens-alias.json new file mode 100644 index 0000000000..3a0609a3d0 --- /dev/null +++ b/frontend/playwright/data/workspace/get-file-tokens-alias.json @@ -0,0 +1,293 @@ +{ + "~:features": { + "~#set": [ + "fdata/path-data", + "plugins/runtime", + "design-tokens/v1", + "variants/v1", + "layout/grid", + "styles/v2", + "fdata/objects-map", + "tokens/numeric-input", + "render-wasm/v1", + "components/v2", + "fdata/shape-data-type" + ] + }, + "~:team-id": "~uc6b102e2-5aaa-809c-8007-dcd1eab2135d", + "~:permissions": { + "~:type": "~:membership", + "~:is-owner": true, + "~:is-admin": true, + "~:can-edit": true, + "~:can-read": true, + "~:is-logged": true + }, + "~:has-media-trimmed": false, + "~:comment-thread-seqn": 0, + "~:name": "New File 11", + "~:revn": 3, + "~:modified-at": "~m1779204621124", + "~:vern": 0, + "~:id": "~uc7ce0794-0992-8105-8004-38f280443849", + "~:is-shared": false, + "~:migrations": { + "~#ordered-set": [ + "legacy-2", + "legacy-3", + "legacy-5", + "legacy-6", + "legacy-7", + "legacy-8", + "legacy-9", + "legacy-10", + "legacy-11", + "legacy-12", + "legacy-13", + "legacy-14", + "legacy-16", + "legacy-17", + "legacy-18", + "legacy-19", + "legacy-25", + "legacy-26", + "legacy-27", + "legacy-28", + "legacy-29", + "legacy-31", + "legacy-32", + "legacy-33", + "legacy-34", + "legacy-36", + "legacy-37", + "legacy-38", + "legacy-39", + "legacy-40", + "legacy-41", + "legacy-42", + "legacy-43", + "legacy-44", + "legacy-45", + "legacy-46", + "legacy-47", + "legacy-48", + "legacy-49", + "legacy-50", + "legacy-51", + "legacy-52", + "legacy-53", + "legacy-54", + "legacy-55", + "legacy-56", + "legacy-57", + "legacy-59", + "legacy-62", + "legacy-65", + "legacy-66", + "legacy-67", + "0001-remove-tokens-from-groups", + "0002-normalize-bool-content-v2", + "0002-clean-shape-interactions", + "0003-fix-root-shape", + "0003-convert-path-content-v2", + "0005-deprecate-image-type", + "0006-fix-old-texts-fills", + "0008-fix-library-colors-v4", + "0009-clean-library-colors", + "0009-add-partial-text-touched-flags", + "0010-fix-swap-slots-pointing-non-existent-shapes", + "0011-fix-invalid-text-touched-flags", + "0012-fix-position-data", + "0013-fix-component-path", + "0013-clear-invalid-strokes-and-fills", + "0014-fix-tokens-lib-duplicate-ids", + "0014-clear-components-nil-objects", + "0015-fix-text-attrs-blank-strings", + "0015-clean-shadow-color", + "0016-copy-fills-from-position-data-to-text-node", + "0017-fix-layout-flex-dir", + "0018-remove-unneeded-objects-from-components", + "0019-fix-missing-swap-slots", + "0020-sync-component-id-with-near-main" + ] + }, + "~:version": 67, + "~:project-id": "~u4cdd76d8-0e6d-8168-8008-0118118e1a1a", + "~:created-at": "~m1779204571619", + "~:backend": "legacy-db", + "~:data": { + "~:pages": [ + "~u9fb430ed-e1e9-81bc-8008-0b7ae978d9c5" + ], + "~:pages-index": { + "~u9fb430ed-e1e9-81bc-8008-0b7ae978d9c5": { + "~:objects": { + "~#penpot/objects-map/v2": { + "~u00000000-0000-0000-0000-000000000000": "[\"~#shape\",[\"^ \",\"~:y\",0,\"~:hide-fill-on-export\",false,\"~:transform\",[\"~#matrix\",[\"^ \",\"~:a\",1.0,\"~:b\",0.0,\"~:c\",0.0,\"~:d\",1.0,\"~:e\",0.0,\"~:f\",0.0]],\"~:rotation\",0,\"~:name\",\"Root Frame\",\"~:width\",0.01,\"~:type\",\"~:frame\",\"~:points\",[[\"~#point\",[\"^ \",\"~:x\",0.0,\"~:y\",0.0]],[\"^:\",[\"^ \",\"~:x\",0.01,\"~:y\",0.0]],[\"^:\",[\"^ \",\"~:x\",0.01,\"~:y\",0.01]],[\"^:\",[\"^ \",\"~:x\",0.0,\"~:y\",0.01]]],\"~:r2\",0,\"~:proportion-lock\",false,\"~:transform-inverse\",[\"^3\",[\"^ \",\"~:a\",1.0,\"~:b\",0.0,\"~:c\",0.0,\"~:d\",1.0,\"~:e\",0.0,\"~:f\",0.0]],\"~:r3\",0,\"~:r1\",0,\"~:id\",\"~u00000000-0000-0000-0000-000000000000\",\"~:parent-id\",\"~u00000000-0000-0000-0000-000000000000\",\"~:frame-id\",\"~u00000000-0000-0000-0000-000000000000\",\"~:strokes\",[],\"~:x\",0,\"~:proportion\",1.0,\"~:r4\",0,\"~:selrect\",[\"~#rect\",[\"^ \",\"~:x\",0,\"~:y\",0,\"^6\",0.01,\"~:height\",0.01,\"~:x1\",0,\"~:y1\",0,\"~:x2\",0.01,\"~:y2\",0.01]],\"~:fills\",[[\"^ \",\"~:fill-color\",\"#FFFFFF\",\"~:fill-opacity\",1]],\"~:flip-x\",null,\"^H\",0.01,\"~:flip-y\",null,\"~:shapes\",[]]]" + } + }, + "~:id": "~u9fb430ed-e1e9-81bc-8008-0b7ae978d9c5", + "~:name": "Page 1" + } + }, + "~:id": "~uc7ce0794-0992-8105-8004-38f280443849", + "~:options": { + "~:components-v2": true, + "~:base-font-size": "16px" + }, + "~:tokens-status": { + "~#penpot/tokens-status": { + "~:active-theme-ids": { + "~#set": [] + }, + "~:active-set-ids": { + "~#set": [ + "~u1dc5cafc-6d57-808d-8008-0b7af8180cbd" + ] + } + } + }, + "~:tokens-lib": { + "~#penpot/tokens-lib": { + "~:sets": { + "~#ordered-map": [ + [ + "S-Global", + { + "~#penpot/token-set": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8180cbd", + "~:name": "Global", + "~:description": "", + "~:modified-at": "~m1779204621126", + "~:tokens": { + "~#ordered-map": [ + [ + "color.primary", + { + "~#penpot/token": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8181201", + "~:name": "color.primary", + "~:type": "~:color", + "~:value": "#ff0000", + "~:description": "", + "~:modified-at": "~m1779204586592" + } + } + ], + [ + "color.alias", + { + "~#penpot/token": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8181202", + "~:name": "color.alias", + "~:type": "~:color", + "~:value": "{color.primary}", + "~:description": "", + "~:modified-at": "~m1779204586592" + } + } + ], + [ + "font-size.base", + { + "~#penpot/token": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8181203", + "~:name": "font-size.base", + "~:type": "~:font-size", + "~:value": "16", + "~:description": "", + "~:modified-at": "~m1779204586592" + } + } + ], + [ + "font-size.alias", + { + "~#penpot/token": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8181204", + "~:name": "font-size.alias", + "~:type": "~:font-size", + "~:value": "{font-size.base}", + "~:description": "", + "~:modified-at": "~m1779204586592" + } + } + ], + [ + "typography.base", + { + "~#penpot/token": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8181205", + "~:name": "typography.base", + "~:type": "~:typography", + "~:value": { + "~:font-family": [ + "Roboto" + ], + "~:font-size": "18", + "~:font-weight": "bold" + }, + "~:description": "", + "~:modified-at": "~m1779204586592" + } + } + ], + [ + "typography.alias", + { + "~#penpot/token": { + "~:id": "~u1dc5cafc-6d57-808d-8008-0b7af8181206", + "~:name": "typography.alias", + "~:type": "~:typography", + "~:value": "{typography.base}", + "~:description": "", + "~:modified-at": "~m1779204586592" + } + } + ] + ] + } + } + } + ] + ] + }, + "~:themes": { + "~#ordered-map": [ + [ + "", + { + "~#ordered-map": [ + [ + "__PENPOT__HIDDEN__TOKEN__THEME__", + { + "~#penpot/token-theme": { + "~:id": "~u00000000-0000-0000-0000-000000000000", + "~:name": "__PENPOT__HIDDEN__TOKEN__THEME__", + "~:group": "", + "~:description": "", + "~:is-source": false, + "~:external-id": "", + "~:modified-at": "~m1779204586593", + "~:sets": { + "~#set": [ + "Global" + ] + } + } + } + ] + ] + } + ] + ] + }, + "~:active-themes": { + "~#set": [ + "/__PENPOT__HIDDEN__TOKEN__THEME__" + ] + } + } + } + } +} \ No newline at end of file diff --git a/frontend/playwright/ui/specs/dashboard-teams.spec.js b/frontend/playwright/ui/specs/dashboard-teams.spec.js index 31153ac4cb..860a6aad18 100644 --- a/frontend/playwright/ui/specs/dashboard-teams.spec.js +++ b/frontend/playwright/ui/specs/dashboard-teams.spec.js @@ -24,3 +24,66 @@ test("BUG 12359 - Selected invitations count is not pluralized", async ({ await page.getByText("test2@mail.com").check(); await expect(page.getByText("2 invitations selected")).toBeVisible(); }); + +async function setupOrganizationTeamSwitcher(page) { + await DashboardPage.mockConfigFlags(page, [ + "disable-onboarding", + "enable-admin-console", + ]); + await DashboardPage.mockRPC( + page, + "get-teams", + "dashboard/get-teams-organization-switcher.json", + ); + await DashboardPage.mockRPC( + page, + "check-nitrate-sso", + "dashboard/check-nitrate-sso-authorized.json", + ); + + const dashboardPage = new DashboardPage(page); + await dashboardPage.goToDashboard(); + return dashboardPage; +} + +test("User sees the destination organization when creating a team", async ({ + page, +}) => { + const organizationName = + "Product Studio With a Long Organization Name That Must Remain Available"; + const dashboardPage = await setupOrganizationTeamSwitcher(page); + + await dashboardPage.teamDropdown.click(); + await page.getByRole("menuitem", { name: organizationName }).click(); + await page.getByRole("menuitem", { name: "Create new team" }).click(); + + const title = page.getByRole("heading", { + name: `Create new team in ${organizationName}`, + exact: true, + }); + await expect(title).toBeVisible(); + await expect(title).toHaveText(`Create new team in ${organizationName}`); + + const titleFits = await title.evaluate( + (element) => element.scrollWidth <= element.clientWidth, + ); + expect(titleFits).toBe(true); + + const modal = title.locator("xpath=../../.."); + const modalBounds = await modal.boundingBox(); + expect(modalBounds.width).toBeLessThanOrEqual(512); +}); + +test("Creating a team under Other teams does not name an organization", async ({ + page, +}) => { + const dashboardPage = await setupOrganizationTeamSwitcher(page); + + await dashboardPage.teamDropdown.click(); + await page.getByRole("menuitem", { name: "Other teams" }).click(); + await page.getByRole("menuitem", { name: "Create new team" }).click(); + + await expect( + page.getByRole("heading", { name: "Create new team", exact: true }), + ).toBeVisible(); +}); diff --git a/frontend/playwright/ui/specs/tokens/apply.spec.js b/frontend/playwright/ui/specs/tokens/apply.spec.js index e13a377ae7..6be288b751 100644 --- a/frontend/playwright/ui/specs/tokens/apply.spec.js +++ b/frontend/playwright/ui/specs/tokens/apply.spec.js @@ -58,7 +58,7 @@ test.describe("Tokens: Apply token", () => { const colorPicker = page.getByTestId("colorpicker"); await expect(colorPicker).toBeVisible(); - await colorPicker.getByTestId('opt-token-color').click(); + await colorPicker.getByTestId("opt-token-color").click(); await colorPicker.getByRole("button", { name: "black" }).click(); // The applied token badge replaces the plain color swatch/input in @@ -1649,9 +1649,7 @@ test.describe("Numeric Input and Token Integration Tests", () => { ).toBeVisible(); }); - test("Invalid formula reverts to previous value in padding inputs", async ({ - page, - }) => { + test("Invalid formula reverts to the previous value", async ({ page }) => { const { workspacePage, tokensSidebar, tokenContextMenuForToken } = await setupTokensFileRender(page, { flags: ["enable-token-combobox", "enable-feature-token-input"], @@ -1690,120 +1688,14 @@ test.describe("Numeric Input and Token Integration Tests", () => { // Enter a valid value first await verticalPaddingInput.fill("23"); await verticalPaddingInput.press("Enter"); - // Wait for potential error handling - await page.waitForTimeout(500); + await expect(verticalPaddingInput).toHaveValue("23"); - expect(await verticalPaddingInput.inputValue()).toMatch("23"); - - // Enter invalid expression + // An invalid expression reverts to the previous valid value await verticalPaddingInput.fill("abc+1"); await verticalPaddingInput.press("Enter"); - // Wait for potential error handling - await page.waitForTimeout(500); - - // Value should revert to previous valid value - expect(await verticalPaddingInput.inputValue()).toMatch("23"); - - // Should NOT contain invalid characters - expect(await verticalPaddingInput.inputValue()).not.toContain("abc"); - }); - - test("Division by zero reverts to previous value", async ({ page }) => { - const { workspacePage, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - flags: ["enable-token-combobox", "enable-feature-token-input"], - }); - - await page.getByRole("tab", { name: "Layers" }).click(); - - await workspacePage.layers.getByTestId("layer-row").nth(1).click(); - - const layoutSection = - workspacePage.rightSidebar.getByTestId("inspect-layout"); - - const addLayoutButton = layoutSection - .getByRole("button", { name: "Add layout" }) - .first(); - - await addLayoutButton.click(); - - await page.getByText("Flex layout").click(); - - // Apply a spacing token to the Column gap property - const tokensTabButton = page.getByRole("tab", { name: "Tokens" }); - await tokensTabButton.click(); - await unfoldTokenType(tokensSidebar, "spacing"); - - await tokensSidebar - .getByRole("button", { name: "spacing.lg" }) - .click({ button: "right" }); - - await tokenContextMenuForToken.getByText("Column gap").click(); - - const verticalPaddingInput = layoutSection.getByRole("textbox", { - name: "Vertical padding", - }); - - // Enter a valid value first - await verticalPaddingInput.fill("23"); - await verticalPaddingInput.press("Enter"); - // Wait for potential error handling - await page.waitForTimeout(500); - - expect(await verticalPaddingInput.inputValue()).toMatch("23"); - - // Enter invalid expression - await verticalPaddingInput.fill("10/0"); - await verticalPaddingInput.press("Enter"); - - // Wait for potential error handling - await page.waitForTimeout(500); - - // Value should revert to previous valid value - expect(await verticalPaddingInput.inputValue()).toMatch("23"); - - // Should NOT contain invalid characters - expect(await verticalPaddingInput.inputValue()).not.toContain("10/0"); - - // Value should revert - expect(await verticalPaddingInput.inputValue()).toMatch(/^(\d+|--)$/); - expect(await verticalPaddingInput.inputValue()).not.toBe("Infinity"); - }); - - test("Negative expression result handled correctly", async ({ page }) => { - const { workspacePage, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - flags: ["enable-token-combobox", "enable-feature-token-input"], - }); - - await page.getByRole("tab", { name: "Layers" }).click(); - - await workspacePage.layers.getByTestId("layer-row").nth(1).click(); - const widthInput = workspacePage.rightSidebar.getByRole("textbox", { - name: "Width", - }); - await expect(widthInput).toBeVisible(); - - // Enter a valid value first - await widthInput.fill("23"); - await widthInput.press("Enter"); - - // Wait for potential error handling - await page.waitForTimeout(500); - expect(await widthInput.inputValue()).toMatch("23"); - - // Enter a negative expression - await widthInput.fill("10-50"); - await widthInput.press("Enter"); - - // Wait for potential error handling - await page.waitForTimeout(500); - - expect(await widthInput.inputValue()).toMatch("0.01"); - - // Should NOT negative values - expect(await widthInput.inputValue()).not.toContain("-40"); + await expect(verticalPaddingInput).toHaveValue("23"); + await expect(verticalPaddingInput).not.toHaveValue(/abc/); }); test("Token pill show broken reference when set is not activated", async ({ @@ -1903,371 +1795,107 @@ test("BUG: 14234, Numeric input token filtering must be case sensitive", async ( test("BUG: 10471, Correct tooltip on right sidebar tokens", async ({ page, }) => { - const { - workspacePage, - tokensSidebar, - tokenContextMenuForToken, - tokenThemesSetsSidebar, - tokenSetGroupItems, - } = await setupTokensFileRender(page, { - file: "workspace/get-file-token-tooltip.json", - }); + const { workspacePage, tokenThemesSetsSidebar } = await setupTokensFileRender( + page, + { + file: "workspace/get-file-token-tooltip.json", + }, + ); await page.getByRole("tab", { name: "Layers" }).click(); - const borderRadiusSection = page.getByRole("region", { - name: "Border radius section", - }); const fillSection = workspacePage.rightSidebar.getByRole("region", { name: "Fill section", }); - const typographySection = workspacePage.rightSidebar.getByRole("region", { - name: "Text section", - }); - // ---------------------------------- - // Select rectangle with active token - // ---------------------------------- - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Rectangle with token" }) - .click(); - await expect(borderRadiusSection).toBeVisible(); - const brPill = borderRadiusSection.getByRole("button", { name: "base-50" }); - await expect(brPill).toBeVisible(); - await brPill.hover(); - await expect(page.getByRole("tooltip", { name: "base-50" })).toBeVisible(); - await expect(fillSection).toBeVisible(); - const fillTokenPill = fillSection.getByLabel("out-ref", { - exact: true, - }); - await expect(fillTokenPill).toBeVisible(); - await fillTokenPill.hover(); - const colorTokenTooltip = page.getByRole("tooltip", { - name: "out-ref", - }); - await expect(colorTokenTooltip).toBeVisible(); - await expect(colorTokenTooltip).toHaveText( - "Name: out-refResolved value: #da1fea", - ); - // ---------------------------------- - // Select text with active token - // ---------------------------------- - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Text with token" }) - .click(); - await expect(fillSection).toBeVisible(); - const textFillTokenPill = fillSection.getByLabel("out-ref", { - exact: true, - }); - await expect(textFillTokenPill).toBeVisible(); - await textFillTokenPill.hover(); - await expect(colorTokenTooltip).toBeVisible(); - await expect(colorTokenTooltip).toHaveText( - "Name: out-refResolved value: #da1fea", - ); - await expect(typographySection).toBeVisible(); - const typographyTokenPill = typographySection.getByLabel("out-typo", { - exact: true, - }); - await expect(typographyTokenPill).toBeVisible(); - await typographyTokenPill.hover(); - const typographyTokenTooltip = page.getByRole("tooltip", { - name: "out-typo", - }); - await expect(typographyTokenTooltip).toBeVisible(); - await expect(typographyTokenTooltip).toHaveText( - 'Name: out-typoResolved value:- font-family: "Arizonia"', - ); - // ----------------------------------------- - // Select rectangle layer with deleted token - // ----------------------------------------- - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Deleted token rect" }) - .click(); + await test.step("Deleted token", async () => { + await workspacePage.layers + .getByTestId("layer-row") + .filter({ hasText: "Deleted token rect" }) + .click(); - await expect(borderRadiusSection).toBeVisible(); - const deletedBrPill = borderRadiusSection.getByRole("button", { - name: "{deleted} token does not exist or", - }); - await expect(deletedBrPill).toBeVisible(); - await deletedBrPill.hover(); - await expect( - page.getByRole("tooltip", { name: "{deleted} token does not exist or" }), - ).toBeVisible(); - await expect(fillSection).toBeVisible(); - const fillBrokenTokenPill = fillSection.getByLabel("blue", { - exact: true, - }); - await expect(fillBrokenTokenPill).toBeVisible(); - await fillBrokenTokenPill.hover(); - const fillBrokenTokenTooltip = page.getByRole("tooltip", { - name: "blue", - }); - await expect(fillBrokenTokenTooltip).toBeVisible(); - await expect(fillBrokenTokenTooltip).toHaveText( - "{blue} token does not exist or has been deleted.", - ); - // ------------------------------------ - // Select text layer with deleted token - // ------------------------------------ - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Text with deleted token" }) - .click(); - await expect(typographySection).toBeVisible(); - const brokenTypographyTokenPill = typographySection.getByLabel( - "deleted-typo", - { exact: true }, - ); - await expect(brokenTypographyTokenPill).toBeVisible(); - await brokenTypographyTokenPill.hover(); - const brokenTypographyTokenTooltip = page.getByRole("tooltip", { - name: "deleted-typo", - }); - await expect(brokenTypographyTokenTooltip).toBeVisible(); - await expect(brokenTypographyTokenTooltip).toHaveText( - "{deleted-typo} token does not exist or has been deleted.", - ); + await expect(fillSection).toBeVisible(); - // --------------------------------------------------- - // Select rectangle layer with deleted reference token - // --------------------------------------------------- - await page.getByRole("tab", { name: "Tokens" }).click(); - await tokenThemesSetsSidebar.getByRole("button", { name: "Dark" }).click(); + const brokenPill = fillSection.getByLabel("blue", { exact: true }); + await expect(brokenPill).toBeVisible(); + await brokenPill.hover(); - await tokenThemesSetsSidebar - .getByRole("button", { name: "Dark" }) - .getByRole("checkbox") - .click(); - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Rectangle with deleted reference" }) - .click(); - await expect(borderRadiusSection).toBeVisible(); - const deletedReferenceBrPill = borderRadiusSection.getByRole("button", { - name: "Reference in {ref-1} is not valid or is not in any active set.", + const tooltip = page.getByRole("tooltip", { name: "blue" }); + await expect(tooltip).toBeVisible(); + await expect(tooltip).toHaveText( + "{blue} token does not exist or has been deleted.", + ); }); - await expect(deletedReferenceBrPill).toBeVisible(); - await deletedReferenceBrPill.hover(); - await expect( - page.getByRole("tooltip", { - name: "Reference in {ref-1} is not valid or is not in any active set.", - }), - ).toBeVisible(); - await expect(fillSection).toBeVisible(); - const fillDeletedReferenceTokenPill = fillSection.getByLabel("ref-grfeen", { - exact: true, - }); - await expect(fillDeletedReferenceTokenPill).toBeVisible(); - await fillDeletedReferenceTokenPill.hover(); - const fillDeletedReferenceTokenTooltip = page.getByRole("tooltip", { - name: "Reference in {ref-grfeen} is not valid or is not in any active set.", - }); - await expect(fillDeletedReferenceTokenTooltip).toBeVisible(); - await expect(fillDeletedReferenceTokenTooltip).toHaveText( - "Reference in {ref-grfeen} is not valid or is not in any active set.", - ); - // ---------------------------------------------- - // Select text layer with deleted reference token - // ---------------------------------------------- + await test.step("Reference outside the active sets", async () => { + await page.getByRole("tab", { name: "Tokens" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Text with deleted reference" }) - .click(); + await tokenThemesSetsSidebar.getByRole("button", { name: "Dark" }).click(); + await tokenThemesSetsSidebar + .getByRole("button", { name: "Dark" }) + .getByRole("checkbox") + .click(); - await expect(typographySection).toBeVisible(); - const deletedRefTypographyTokenPill = typographySection.getByLabel( - "ref-typo", - { exact: true }, - ); - await expect(deletedRefTypographyTokenPill).toBeVisible(); - await deletedRefTypographyTokenPill.hover(); - const deletedRefTypographyTokenTooltip = page.getByRole("tooltip", { - name: "Reference in {ref-typo} is not valid or is", - }); - await expect(deletedRefTypographyTokenTooltip).toBeVisible(); - await expect(deletedRefTypographyTokenTooltip).toHaveText( - "Reference in {ref-typo} is not valid or is not in any active set.", - ); - await expect(fillSection).toBeVisible(); - const fillTextDeletedReferenceTokenPill = fillSection.getByLabel( - "ref-grfeen", - { + await page.getByRole("tab", { name: "Layers" }).click(); + + await workspacePage.layers + .getByTestId("layer-row") + .filter({ hasText: "Rectangle with deleted reference" }) + .click(); + + await expect(fillSection).toBeVisible(); + + const brokenReferencePill = fillSection.getByLabel("ref-grfeen", { exact: true, - }, - ); - await expect(fillTextDeletedReferenceTokenPill).toBeVisible(); - await fillTextDeletedReferenceTokenPill.hover(); - const fillTextDeletedReferenceTokenTooltip = page.getByRole("tooltip", { - name: "Reference in {ref-grfeen} is not valid or is not in any active set.", - }); - await expect(fillTextDeletedReferenceTokenTooltip).toBeVisible(); - await expect(fillTextDeletedReferenceTokenTooltip).toHaveText( - "Reference in {ref-grfeen} is not valid or is not in any active set.", - ); + }); + await expect(brokenReferencePill).toBeVisible(); + await brokenReferencePill.hover(); - // ----------------------------------------------------------- - // Select rectangle layer with reference token on inactive set - // ----------------------------------------------------------- - await page.getByRole("tab", { name: "Tokens" }).click(); - await tokenThemesSetsSidebar.getByRole("button", { name: "Global" }).click(); + const tooltip = page.getByRole("tooltip", { + name: "Reference in {ref-grfeen} is not valid or is not in any active set.", + }); + await expect(tooltip).toBeVisible(); + await expect(tooltip).toHaveText( + "Reference in {ref-grfeen} is not valid or is not in any active set.", + ); + }); - await tokenThemesSetsSidebar - .getByRole("button", { name: "Global" }) - .getByRole("checkbox") - .click(); - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Rectangle with not active reference" }) - .click(); - await expect(borderRadiusSection).toBeVisible(); - const nonActiveReferenceBrPill = borderRadiusSection.getByRole("button", { - name: "Reference in {in-br} is not valid or is not in any active set.", - }); - await expect(nonActiveReferenceBrPill).toBeVisible(); - await nonActiveReferenceBrPill.hover(); - await expect( - page.getByRole("tooltip", { - name: "Reference in {in-br} is not valid or is not in any active set.", - }), - ).toBeVisible(); - await expect(fillSection).toBeVisible(); - const fillNonActiveReferenceTokenPill = fillSection.getByLabel("in-color", { - exact: true, - }); - await expect(fillNonActiveReferenceTokenPill).toBeVisible(); - await fillNonActiveReferenceTokenPill.hover(); - const fillNonActiveReferenceTokenTooltip = page.getByRole("tooltip", { - name: "Reference in {in-color} is not valid or is not in any active set.", - }); - await expect(fillNonActiveReferenceTokenTooltip).toBeVisible(); - await expect(fillNonActiveReferenceTokenTooltip).toHaveText( - "Reference in {in-color} is not valid or is not in any active set.", - ); - // ------------------------------------------------------ - // Select text layer with reference token on inactive set - // ------------------------------------------------------ + await test.step("Token in an inactive set", async () => { + await page.getByRole("tab", { name: "Tokens" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Text with not active reference" }) - .click(); + await tokenThemesSetsSidebar + .getByRole("button", { name: "Global" }) + .click(); + await tokenThemesSetsSidebar + .getByRole("button", { name: "Global" }) + .getByRole("checkbox") + .click(); - await expect(typographySection).toBeVisible(); - const notActiveRefTypographyTokenPill = typographySection.getByLabel( - "in-typo", - { exact: true }, - ); - await expect(notActiveRefTypographyTokenPill).toBeVisible(); - await notActiveRefTypographyTokenPill.hover(); - const notActiveRefTypographyTokenTooltip = page.getByRole("tooltip", { - name: "Reference in {in-typo} is not valid or is", - }); - await expect(notActiveRefTypographyTokenTooltip).toBeVisible(); - await expect(notActiveRefTypographyTokenTooltip).toHaveText( - "Reference in {in-typo} is not valid or is not in any active set.", - ); - await expect(fillSection).toBeVisible(); - const fillTextNotActiveReferenceTokenPill = fillSection.getByLabel( - "in-color", - { - exact: true, - }, - ); - await expect(fillTextNotActiveReferenceTokenPill).toBeVisible(); - await fillTextNotActiveReferenceTokenPill.hover(); - const fillTextNotActiveReferenceTokenTooltip = page.getByRole("tooltip", { - name: "Reference in {in-color} is not valid or is not in any active set.", - }); - await expect(fillTextNotActiveReferenceTokenTooltip).toBeVisible(); - await expect(fillTextNotActiveReferenceTokenTooltip).toHaveText( - "Reference in {in-color} is not valid or is not in any active set.", - ); + await tokenThemesSetsSidebar.getByRole("button", { name: "Dark" }).click(); + await tokenThemesSetsSidebar + .getByRole("button", { name: "Dark" }) + .getByRole("checkbox") + .click(); - // ------------------------------------------------- - // Select rectangle layer with token on inactive set - // ------------------------------------------------- - await page.getByRole("tab", { name: "Tokens" }).click(); - await tokenThemesSetsSidebar.getByRole("button", { name: "Dark" }).click(); + await page.getByRole("tab", { name: "Layers" }).click(); - await tokenThemesSetsSidebar - .getByRole("button", { name: "Dark" }) - .getByRole("checkbox") - .click(); - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Rectangle with not active token" }) - .click(); - await expect(borderRadiusSection).toBeVisible(); - const nonActiveBrPill = borderRadiusSection.getByRole("button", { - name: "{border-radius} token is not in any active set", - }); - await expect(nonActiveBrPill).toBeVisible(); - await nonActiveBrPill.hover(); - await expect( - page.getByRole("tooltip", { - name: "{border-radius} token is not in any active set or has an invalid value", - }), - ).toBeVisible(); - await expect(fillSection).toBeVisible(); - const fillNonActiveTokenPill = fillSection.getByLabel("red", { - exact: true, - }); - await expect(fillNonActiveTokenPill).toBeVisible(); - await fillNonActiveTokenPill.hover(); - const fillNonActiveTokenTooltip = page.getByRole("tooltip", { - name: "{red} token is not in any active set or has an invalid value.", - }); - await expect(fillNonActiveTokenTooltip).toBeVisible(); - await expect(fillNonActiveTokenTooltip).toHaveText( - "{red} token is not in any active set or has an invalid value.", - ); + await workspacePage.layers + .getByTestId("layer-row") + .filter({ hasText: "Rectangle with not active token" }) + .click(); - // -------------------------------------------- - // Select text layer with token on inactive set - // -------------------------------------------- + await expect(fillSection).toBeVisible(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Text with not active token" }) - .click(); + const inactivePill = fillSection.getByLabel("red", { exact: true }); + await expect(inactivePill).toBeVisible(); + await inactivePill.hover(); - await expect(typographySection).toBeVisible(); - const notActiveTypographyTokenPill = typographySection.getByLabel( - "typo-2", - { exact: true }, - ); - await expect(notActiveTypographyTokenPill).toBeVisible(); - await notActiveTypographyTokenPill.hover(); - const notActiveTypographyTokenTooltip = page.getByRole("tooltip", { - name: "{typo-2} token is not in any active set or has an invalid value.", + const tooltip = page.getByRole("tooltip", { + name: "{red} token is not in any active set or has an invalid value.", + }); + await expect(tooltip).toBeVisible(); + await expect(tooltip).toHaveText( + "{red} token is not in any active set or has an invalid value.", + ); }); - await expect(notActiveTypographyTokenTooltip).toBeVisible(); - await expect(notActiveTypographyTokenTooltip).toHaveText( - "{typo-2} token is not in any active set or has an invalid value.", - ); - await expect(fillSection).toBeVisible(); - const fillTextNotActiveTokenPill = fillSection.getByLabel( - "red", - { - exact: true, - }, - ); - await expect(fillTextNotActiveTokenPill).toBeVisible(); - await fillTextNotActiveTokenPill.hover(); - const fillTextNotActiveTokenTooltip = page.getByRole("tooltip", { - name: "{red} token is not in any active set or has an invalid value.", - }); - await expect(fillTextNotActiveTokenTooltip).toBeVisible(); - await expect(fillTextNotActiveTokenTooltip).toHaveText( - "{red} token is not in any active set or has an invalid value.", - ); }); diff --git a/frontend/playwright/ui/specs/tokens/crud.spec.js b/frontend/playwright/ui/specs/tokens/crud.spec.js index 6fb6a0cf42..f9d4fe3284 100644 --- a/frontend/playwright/ui/specs/tokens/crud.spec.js +++ b/frontend/playwright/ui/specs/tokens/crud.spec.js @@ -5,7 +5,10 @@ import { setupEmptyTokensFileRender, setupTokensFileRender, setupTypographyTokensFileRender, - testTokenCreationFlow, + openTokenCreationForm, + assertTokenCreationLadder, + saveToken, + createTokenWithValue, unfoldTokenType, createToken, createSet, @@ -17,19 +20,64 @@ test.beforeEach(async ({ page }) => { }); test.describe("Tokens - creation", () => { - test("User creates border radius token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Border Radius", - namePlaceholder: "Enter border radius token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", + test("User creates tokens of every numeric type", async ({ page }) => { + const { tokensUpdateCreateModal } = await setupEmptyTokensFileRender(page); + + const cases = [ + ["Border Radius", "border radius", "my-token-border-radius"], + ["Dimensions", "dimensions", "my-token-dimensions"], + ["Font Size", "font size", "my-token-font-size"], + ["Letter spacing", "letter spacing", "my-token-letter-spacing"], + ["Number", "number", "my-token-number"], + ["Rotation", "rotation", "my-token-rotation"], + ["Sizing", "sizing", "my-token-sizing"], + ["Spacing", "spacing", "my-token-spacing"], + ["Stroke width", "stroke width", "my-token-stroke-width"], + ]; + + const [first, ...rest] = cases; + + await test.step(first[0], async () => { + const [tokenLabel, placeholder, name] = first; + const namePlaceholder = `Enter ${placeholder} token name`; + + await openTokenCreationForm(page, tokensUpdateCreateModal, { + tokenLabel, + namePlaceholder, + }); + + await assertTokenCreationLadder(tokensUpdateCreateModal, { name }); + + await saveToken(page, tokensUpdateCreateModal, { + tokenLabel, + name, + value: "2 + 3", + resolvedText: "Resolved value: 5", + }); + + // A second token that references the first one + await openTokenCreationForm(page, tokensUpdateCreateModal, { + tokenLabel, + namePlaceholder, + }); + + await saveToken(page, tokensUpdateCreateModal, { + tokenLabel, + name: `${name}-ref`, + value: `{${name}} - 2`, + resolvedText: "Resolved value: 3", + }); }); + + for (const [tokenLabel, placeholder, name] of rest) { + await test.step(tokenLabel, async () => { + await createTokenWithValue(page, tokensUpdateCreateModal, { + tokenLabel, + namePlaceholder: `Enter ${placeholder} token name`, + name, + }); + }); + } }); test("User creates border radius token with combobox", async ({ page }) => { @@ -187,126 +235,6 @@ test.describe("Tokens - creation", () => { ).toBeEnabled(); }); - test("User creates dimensions token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Dimensions", - namePlaceholder: "Enter dimensions token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates font size token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Font Size", - namePlaceholder: "Enter font size token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates letter spacing token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Letter spacing", - namePlaceholder: "Enter letter spacing token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates number token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Number", - namePlaceholder: "Enter number token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates rotation token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Rotation", - namePlaceholder: "Enter rotation token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates sizing token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Sizing", - namePlaceholder: "Enter sizing token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates spacing token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Spacing", - namePlaceholder: "Enter spacing token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - - test("User creates stroke width token", async ({ page }) => { - await testTokenCreationFlow(page, { - tokenLabel: "Stroke width", - namePlaceholder: "Enter stroke width token name", - valuePlaceholder: "Enter a value or alias with {alias}", - invalidValue: "red", - validValue: "2 + 3", - selfReferenceValue: "{my-token}", - missingReferenceValue: "{missing-token}", - secondValidValue: "{my-token} - 2", - resolvedValueText: "Resolved value: 5", - secondResolvedValueText: "Resolved value: 3", - }); - }); - test("User creates color token and auto created set show up in the sidebar", async ({ page, }) => { @@ -569,7 +497,9 @@ test.describe("Tokens - creation", () => { }); await selectDropdown.click(); - const fontOption = tokensUpdateCreateModal.getByRole('img', { name: 'ABeeZee' }) + const fontOption = tokensUpdateCreateModal.getByRole("img", { + name: "ABeeZee", + }); await expect(fontOption).toBeVisible(); await fontOption.click(); @@ -583,7 +513,9 @@ test.describe("Tokens - creation", () => { name: "Search font", }); await searchField.fill("alme"); - const fontOption2 = tokensUpdateCreateModal.getByRole('img', {name: "Almendra Display"}); + const fontOption2 = tokensUpdateCreateModal.getByRole("img", { + name: "Almendra Display", + }); await expect(fontOption2).toBeVisible(); await fontOption2.click(); @@ -1133,7 +1065,7 @@ test.describe("Tokens - creation", () => { // 8. Valid spread → resolved - await blurField.fill("3 - 3"); + await spreadField.fill("3 - 3"); await expect( tokensUpdateCreateModal.getByText("Resolved value: 0"), ).toBeVisible(); @@ -1175,7 +1107,7 @@ test.describe("Tokens - creation", () => { await referenceField.fill("{my-token}"); await expect( tokensUpdateCreateModal.getByText( - "Resolved value: - X: 6 - Y: 10 - Blur: 0 - Spread: 0 ", + "Resolved value: - X: 6 - Y: 10 - Blur: 4 - Spread: 0 ", ), ).toBeVisible(); @@ -1521,7 +1453,9 @@ test.describe("Tokens - creation", () => { }); await selectDropdown.click(); - const fontOption = tokensUpdateCreateModal.getByRole("img", {name: "ABeeZee"}); + const fontOption = tokensUpdateCreateModal.getByRole("img", { + name: "ABeeZee", + }); await expect(fontOption).toBeVisible(); await fontOption.click(); @@ -1536,7 +1470,9 @@ test.describe("Tokens - creation", () => { name: "Search font", }); await searchField.fill("alme"); - const fontOption2 = tokensUpdateCreateModal.getByRole("img", {name: "Almendra Display"}); + const fontOption2 = tokensUpdateCreateModal.getByRole("img", { + name: "Almendra Display", + }); await expect(fontOption2).toBeVisible(); await fontOption2.click(); await expect( @@ -1820,85 +1756,6 @@ test.describe("Tokens - creation", () => { }); }); -test("User creates grouped color token", async ({ page }) => { - const { workspacePage, tokensUpdateCreateModal, tokensSidebar } = - await setupEmptyTokensFileRender(page); - - await tokensSidebar.getByRole("button", { name: "Add Token: Color" }).click(); - - // Create grouped color token with mouse - - await expect(tokensUpdateCreateModal).toBeVisible(); - - const nameField = tokensUpdateCreateModal.getByLabel("Name"); - const valueField = tokensUpdateCreateModal.getByLabel("Value"); - - await nameField.click(); - await nameField.fill("dark.primary"); - - await valueField.click(); - await valueField.fill("red"); - - const submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - await expect(submitButton).toBeEnabled(); - await submitButton.click(); - - await unfoldTokenType(tokensSidebar, "color"); - - await expect(tokensSidebar.getByLabel("primary")).toBeEnabled(); -}); - -test("User can't create regular token with value missing", async ({ page }) => { - const { tokensUpdateCreateModal } = await setupEmptyTokensFileRender(page); - - const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); - await tokensTabPanel - .getByRole("button", { name: "Add Token: Color" }) - .click(); - - await expect(tokensUpdateCreateModal).toBeVisible(); - - const nameField = tokensUpdateCreateModal.getByLabel("Name"); - const submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - - // Initially submit button should be disabled - await expect(submitButton).toBeDisabled(); - - // Fill in name but leave value empty - await nameField.click(); - await nameField.fill("primary"); - - // Submit button should remain disabled when value is empty - await expect(submitButton).toBeDisabled(); -}); - -test("User duplicate color token", async ({ page }) => { - const { tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page); - - await expect(tokensSidebar).toBeVisible(); - - await unfoldTokenType(tokensSidebar, "color"); - - const colorToken = tokensSidebar.getByRole("button", { - name: "colors.blue.100", - }); - - await colorToken.click({ button: "right" }); - await expect(tokenContextMenuForToken).toBeVisible(); - - await tokenContextMenuForToken.getByText("Duplicate token").click(); - await expect(tokenContextMenuForToken).not.toBeVisible(); - - await expect( - tokensSidebar.getByRole("button", { name: "colors.blue.100-copy" }), - ).toBeVisible(); -}); - test("User disables the current set but token still have resolved values shown in the sidebar", async ({ page, }) => { @@ -1932,401 +1789,135 @@ test("User disables the current set but token still have resolved values shown i }); test.describe("User can't create groups that clash with token names", () => { - const createBadToken = async (page, type, name, textFieldName, value, inputType) => { + const clashError = (name) => + `Group name of ${name} conflicts with a token of the same name in another active set.`; + + const setupClashingFile = async (page) => { + const { tokensUpdateCreateModal, tokenThemesSetsSidebar, tokensSidebar } = + await setupTokensFileRender(page, { + file: "workspace/get-file-tokens-all-types.json", + flags: ["enable-token-shadow"], + }); + + await expect(tokensSidebar).toBeVisible(); + + await createSet(tokenThemesSetsSidebar, "Second set"); + + return { tokensUpdateCreateModal }; + }; + + const openTokenForm = async (page, tokensUpdateCreateModal, type) => { const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); - const { tokensUpdateCreateModal } = await setupTokensFileRender(page, { - // Same file as the outer setup: re-navigating would reload the - // app and wipe the fixture tokens this test clashes against. - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - // Add a token of the given type await tokensTabPanel .getByRole("button", { name: `Add Token: ${type}` }) .click(); + await expect(tokensUpdateCreateModal).toBeVisible(); - - // Fill the bad name - const nameField = tokensUpdateCreateModal.getByLabel("Name"); - await nameField.fill(name); - - // Fill the value - const valueField = tokensUpdateCreateModal.getByRole(inputType, { - name: textFieldName, - }); - await valueField.fill(value); - - // Check that the value has an error - const errorNode = tokensUpdateCreateModal.getByText( - `Group name of ${name} conflicts with a token of the same name in another active set.`, - ); - - await expect(errorNode).toBeVisible(); - - // Check that the form cannot be saved - const submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - await expect(submitButton).toBeDisabled(); }; - test("User can't create Border Radius token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Border Radius", "rad1.bad", "Value", "10", "combobox"); - }); - - test("User can't create Color token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Color", "col1.bad", "Value", "red", "textbox"); - }); - - test("User can't create Dimensions token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Dimensions", "dim1.bad", "Value", "100", "combobox"); - }); - - test("User can't create Font Size token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Font Size", "fsiz1.bad", "Value", "16", "combobox"); - }); - - test("User can't create Font Weight token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Font Weight", "wei1.bad", "Value", "400", "combobox"); - }); - - test("User can't create Letter Spacing token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Letter Spacing", "lspa1.bad", "Value", "1", "combobox"); - }); - - test("User can't create Number token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Number", "num1.bad", "Value", "10", "combobox"); - }); - - test("User can't create Rotation token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Rotation", "rot1.bad", "Value", "90", "combobox"); - }); - - test("User can't create Sizing token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Sizing", "siz1.bad", "Value", "100", "combobox"); - }); - - test("User can't create Spacing token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Spacing", "spa1.bad", "Value", "10", "combobox"); - }); - - test("User can't create Stroke Width token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Stroke Width", "str1.bad", "Value", "2", "combobox"); - }); - - test("User can't create Text Case token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken(page, "Text Case", "cas1.bad", "Value", "uppercase", "combobox"); - }); - - test("User can't create Text Decoration token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar, tokenContextMenuForToken } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - await createBadToken( - page, - "Text Decoration", - "dec1.bad", - "Value", - "strike-through", - "combobox" - ); - }); - - test("User can't create Typography token with group name that clashes with existing token", async ({ - page, - }) => { - const { tokenThemesSetsSidebar, tokensSidebar } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await expect(tokensSidebar).toBeVisible(); - - await createSet(tokenThemesSetsSidebar, "Second set"); - - const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); - - const { tokensUpdateCreateModal } = await setupTokensFileRender(page, { - // Same file as the outer setup: re-navigating would reload the - // app and wipe the fixture tokens this test clashes against. - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], - }); - - await tokensTabPanel - .getByRole("button", { name: `Add Token: Typography` }) + const closeTokenForm = async (tokensUpdateCreateModal) => { + await tokensUpdateCreateModal + .getByRole("button", { name: "Cancel" }) .click(); - await expect(tokensUpdateCreateModal).toBeVisible(); - const nameField = tokensUpdateCreateModal.getByLabel("Name"); - await nameField.fill("typ1.bad"); + await expect(tokensUpdateCreateModal).not.toBeVisible(); + }; - const fontFamilyField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Font family", - }); - await fontFamilyField.fill("Arial"); + const expectClashError = async (tokensUpdateCreateModal, name) => { + await expect( + tokensUpdateCreateModal.getByText(clashError(name)), + ).toHaveCount(1); - const fontSizeField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Font size", - }); - await fontSizeField.fill("16"); + await expect( + tokensUpdateCreateModal.getByRole("button", { name: "Save" }), + ).toBeDisabled(); + }; - const fontWeightField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Font weight", - }); - await fontWeightField.fill("400"); - - const lineHeightField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Line height", - }); - await lineHeightField.fill("1.5"); - - const letterSpacingField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Letter spacing", - }); - await letterSpacingField.fill("0"); - - const textCaseField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Text case", - }); - await textCaseField.fill("none"); - - const textDecorationField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Text decoration", - }); - await textDecorationField.fill("none"); - - const submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - - const errorNode = tokensUpdateCreateModal.getByText( - "Group name of typ1.bad conflicts with a token of the same name in another active set.", - ); - await expect(errorNode).toHaveCount(1); - await expect(submitButton).toBeDisabled(); - }); - - test("User can't create Shadow token with group name that clashes with existing token", async ({ + test("User can't create a simple token with group name that clashes with existing token", async ({ page, }) => { - const { tokenThemesSetsSidebar, tokensSidebar } = - await setupTokensFileRender(page, { - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], + const { tokensUpdateCreateModal } = await setupClashingFile(page); + + const cases = [ + ["Border Radius", "rad1.bad", "Value", "10", "combobox"], + ["Color", "col1.bad", "Value", "red", "textbox"], + ["Dimensions", "dim1.bad", "Value", "100", "combobox"], + ["Font Size", "fsiz1.bad", "Value", "16", "combobox"], + ["Font Weight", "wei1.bad", "Value", "400", "combobox"], + ["Letter Spacing", "lspa1.bad", "Value", "1", "combobox"], + ["Number", "num1.bad", "Value", "10", "combobox"], + ["Rotation", "rot1.bad", "Value", "90", "combobox"], + ["Sizing", "siz1.bad", "Value", "100", "combobox"], + ["Spacing", "spa1.bad", "Value", "10", "combobox"], + ["Stroke Width", "str1.bad", "Value", "2", "combobox"], + ["Text Case", "cas1.bad", "Value", "uppercase", "combobox"], + ["Text Decoration", "dec1.bad", "Value", "strike-through", "combobox"], + ]; + + for (const [type, name, field, value, role] of cases) { + await test.step(type, async () => { + await openTokenForm(page, tokensUpdateCreateModal, type); + await tokensUpdateCreateModal.getByLabel("Name").fill(name); + await tokensUpdateCreateModal + .getByRole(role, { name: field }) + .fill(value); + await expectClashError(tokensUpdateCreateModal, name); + await closeTokenForm(tokensUpdateCreateModal); }); + } + }); - await expect(tokensSidebar).toBeVisible(); + test("User can't create a composite token with group name that clashes with existing token", async ({ + page, + }) => { + const { tokensUpdateCreateModal } = await setupClashingFile(page); - await createSet(tokenThemesSetsSidebar, "Second set"); + await test.step("Typography", async () => { + await openTokenForm(page, tokensUpdateCreateModal, "Typography"); + await tokensUpdateCreateModal.getByLabel("Name").fill("typ1.bad"); - const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); + const fields = { + "Font family": "Arial", + "Font size": "16", + "Font weight": "400", + "Line height": "1.5", + "Letter spacing": "0", + "Text case": "none", + "Text decoration": "none", + }; - const { tokensUpdateCreateModal } = await setupTokensFileRender(page, { - // Same file as the outer setup: re-navigating would reload the - // app and wipe the fixture tokens this test clashes against. - file: "workspace/get-file-tokens-all-types.json", - flags: ["enable-token-shadow"], + for (const [label, value] of Object.entries(fields)) { + await tokensUpdateCreateModal + .getByRole("textbox", { name: label }) + .fill(value); + } + + await expectClashError(tokensUpdateCreateModal, "typ1.bad"); + await closeTokenForm(tokensUpdateCreateModal); }); - await tokensTabPanel - .getByRole("button", { name: `Add Token: Shadow` }) - .click(); - await expect(tokensUpdateCreateModal).toBeVisible(); + await test.step("Shadow", async () => { + await openTokenForm(page, tokensUpdateCreateModal, "Shadow"); + await tokensUpdateCreateModal.getByLabel("Name").fill("sha1.bad"); - const nameField = tokensUpdateCreateModal.getByLabel("Name"); - await nameField.fill("sha1.bad"); + const fields = { + Color: "red", + X: "7", + Y: "8", + Blur: "5", + Spread: "1", + }; - const colorField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Color", + for (const [label, value] of Object.entries(fields)) { + await tokensUpdateCreateModal + .getByRole("textbox", { name: label }) + .fill(value); + } + + await expectClashError(tokensUpdateCreateModal, "sha1.bad"); + await closeTokenForm(tokensUpdateCreateModal); }); - await colorField.fill("red"); - - const offsetXField = tokensUpdateCreateModal.getByRole("textbox", { - name: "X", - }); - await offsetXField.fill("7"); - - const offsetYField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Y", - }); - await offsetYField.fill("8"); - - const blurField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Blur", - }); - await blurField.fill("5"); - - const spreadField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Spread", - }); - await spreadField.fill("1"); - - const submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - - const errorNode = tokensUpdateCreateModal.getByText( - "Group name of sha1.bad conflicts with a token of the same name in another active set.", - ); - await expect(errorNode).toHaveCount(1); - await expect(submitButton).toBeDisabled(); }); }); @@ -2538,6 +2129,76 @@ test.describe("Tokens tab - edition", () => { await expect(valueField).toHaveValue(/^rgba(.*)$/); }); + const openEditTokenForm = async (setup, type, name) => { + const { tokensSidebar, tokenContextMenuForToken, tokensUpdateCreateModal } = + setup; + const kebabType = type.replace(/\s/g, "-"); + + await unfoldTokenType(tokensSidebar, type); + await tokensSidebar + .getByTestId(`section-${kebabType}`) + .getByRole("button", { name }) + .first() + .click({ button: "right" }); + await expect(tokenContextMenuForToken).toBeVisible(); + await tokenContextMenuForToken.getByText("Edit token").click(); + await expect(tokensUpdateCreateModal).toBeVisible(); + }; + + test("BUG: 12672 Edit form shows the resolved color of an alias token", async ({ + page, + }) => { + const setup = await setupTokensFileRender(page, { + file: "workspace/get-file-tokens-alias.json", + }); + const { tokensUpdateCreateModal } = setup; + + await openEditTokenForm(setup, "color", "alias"); + + await expect(tokensUpdateCreateModal.getByLabel("Value")).toHaveValue( + "{color.primary}", + ); + await expect( + tokensUpdateCreateModal.getByText("Resolved value: #ff0000"), + ).toBeVisible(); + + const swatchColor = tokensUpdateCreateModal + .getByTestId("token-form-color-bullet") + .locator("div[style*='background']") + .first(); + await expect(swatchColor).toHaveCSS("background-color", "rgb(255, 0, 0)"); + }); + + test("BUG: 12788 Edit form shows the resolved value of an alias token on open", async ({ + page, + }) => { + const setup = await setupTokensFileRender(page, { + file: "workspace/get-file-tokens-alias.json", + }); + const { tokensUpdateCreateModal } = setup; + const resolvedValue = tokensUpdateCreateModal.getByText("Resolved value:"); + + // Font size alias + await openEditTokenForm(setup, "font size", "alias"); + await expect( + tokensUpdateCreateModal.getByText("Resolved value: 16"), + ).toBeVisible(); + await page.keyboard.press("Escape"); + + // Font size with a plain value: no hint repeating the value + await openEditTokenForm(setup, "font size", "base"); + await expect(tokensUpdateCreateModal.getByLabel("Value")).toHaveValue("16"); + await expect(resolvedValue).not.toBeVisible(); + await page.keyboard.press("Escape"); + + // Typography alias + await openEditTokenForm(setup, "typography", "alias"); + await expect(resolvedValue).toBeVisible(); + await expect( + tokensUpdateCreateModal.getByText("Font Size: 18"), + ).toBeVisible(); + }); + test("User sees self-reference error when editing a token to create a circular reference", async ({ page, }) => { diff --git a/frontend/playwright/ui/specs/tokens/helpers.js b/frontend/playwright/ui/specs/tokens/helpers.js index b758696423..e8b0379c07 100644 --- a/frontend/playwright/ui/specs/tokens/helpers.js +++ b/frontend/playwright/ui/specs/tokens/helpers.js @@ -212,130 +212,118 @@ const setupTypographyTokensFileRender = async (page, options = {}) => { }); }; -const testTokenCreationFlow = async ( +const tokenValuePlaceholder = "Enter a value or alias with {alias}"; + +const openTokenCreationForm = async ( page, - { - tokenLabel, - namePlaceholder, - valuePlaceholder, - validValue, - invalidValue, - selfReferenceValue, - missingReferenceValue, - secondValidValue, - resolvedValueText, - secondResolvedValueText, - }, + tokensUpdateCreateModal, + { tokenLabel, namePlaceholder }, ) => { - const invalidValueError = "Invalid token value"; - const emptyNameError = "Name should be at least 1 character"; - const selfReferenceError = "Token has self reference"; - const missingReferenceError = "Missing token references"; - - const { tokensUpdateCreateModal, tokensSidebar } = - await setupEmptyTokensFileRender(page); - - // Open modal const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); - const addTokenButton = tokensTabPanel.getByRole("button", { - name: `Add Token: ${tokenLabel}`, - }); + await tokensTabPanel + .getByRole("button", { name: `Add Token: ${tokenLabel}` }) + .click(); - await addTokenButton.click(); await expect(tokensUpdateCreateModal).toBeVisible(); - // Placeholder checks await expect( tokensUpdateCreateModal.getByPlaceholder(namePlaceholder), ).toBeVisible(); await expect( - tokensUpdateCreateModal.getByPlaceholder(valuePlaceholder), + tokensUpdateCreateModal.getByPlaceholder(tokenValuePlaceholder), ).toBeVisible(); +}; +const assertTokenCreationLadder = async ( + tokensUpdateCreateModal, + { name, invalidValue = "red" }, +) => { const nameField = tokensUpdateCreateModal.getByLabel("Name"); const valueField = tokensUpdateCreateModal.getByLabel("Value"); const submitButton = tokensUpdateCreateModal.getByRole("button", { name: "Save", }); - // 1. Name filled + empty value → disabled - await nameField.fill("my-token"); + // 1. Name filled + empty value -> disabled + await nameField.fill(name); await expect(submitButton).toBeDisabled(); - // 2. Invalid value → disabled + error message + // 2. Invalid value -> disabled + error message await valueField.fill(invalidValue); - - const invalidValueErrorNode = - tokensUpdateCreateModal.getByText(invalidValueError); - - await expect(invalidValueErrorNode).toBeVisible(); + await expect( + tokensUpdateCreateModal.getByText("Invalid token value"), + ).toBeVisible(); await expect(submitButton).toBeDisabled(); - // 3. Empty name → disabled + error message + // 3. Empty name -> disabled + error message await nameField.fill(""); - - const emptyNameErrorNode = tokensUpdateCreateModal.getByText(emptyNameError); - - await expect(emptyNameErrorNode).toBeVisible(); - await expect(submitButton).toBeDisabled(); - - // 4. Self reference → disabled + error message - await nameField.fill("my-token"); - await valueField.fill(selfReferenceValue); - - const selfRefErrorNode = - tokensUpdateCreateModal.getByText(selfReferenceError); - - await expect(selfRefErrorNode).toBeVisible(); - await expect(submitButton).toBeDisabled(); - - // 5. Missing reference → disabled + error message - await valueField.fill(missingReferenceValue); - - const missingRefErrorNode = tokensUpdateCreateModal.getByText( - missingReferenceError, - ); - - await expect(missingRefErrorNode).toBeVisible(); - await expect(submitButton).toBeDisabled(); - - // - // ------- SUCCESSFUL CREATION ------- - // - - // 6. Basic valid value → enabled - await valueField.fill(validValue); await expect( - tokensUpdateCreateModal.getByText(resolvedValueText), + tokensUpdateCreateModal.getByText("Name should be at least 1 character"), ).toBeVisible(); - await expect(submitButton).toBeEnabled(); + await expect(submitButton).toBeDisabled(); + // 4. Self reference -> disabled + error message + await nameField.fill(name); + await valueField.fill(`{${name}}`); + await expect( + tokensUpdateCreateModal.getByText("Token has self reference"), + ).toBeVisible(); + await expect(submitButton).toBeDisabled(); + + // 5. Missing reference -> disabled + error message + await valueField.fill("{missing-token}"); + await expect( + tokensUpdateCreateModal.getByText("Missing token references"), + ).toBeVisible(); + await expect(submitButton).toBeDisabled(); +}; + +const saveToken = async ( + page, + tokensUpdateCreateModal, + { tokenLabel, name, value, resolvedText }, +) => { + const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); + + await tokensUpdateCreateModal.getByLabel("Name").fill(name); + await tokensUpdateCreateModal.getByLabel("Value").fill(value); + + await expect(tokensUpdateCreateModal.getByText(resolvedText)).toBeVisible(); + + const submitButton = tokensUpdateCreateModal.getByRole("button", { + name: "Save", + }); + await expect(submitButton).toBeEnabled(); await submitButton.click(); - await expect( - tokensTabPanel.getByRole("button", { name: "my-token" }), - ).toBeEnabled(); + await unfoldTokenType(tokensTabPanel, tokenLabel.toLocaleLowerCase()); - // - // ------- SECOND TOKEN WITH VALID REFERENCE ------- - // + await expect(tokensTabPanel.getByRole("button", { name })).toBeEnabled(); +}; - await addTokenButton.click(); +const createTokenWithValue = async ( + page, + tokensUpdateCreateModal, + { + tokenLabel, + namePlaceholder, + name, + value = "2 + 3", + resolvedText = "Resolved value: 5", + }, +) => { + await openTokenCreationForm(page, tokensUpdateCreateModal, { + tokenLabel, + namePlaceholder, + }); - await nameField.fill("my-token-2"); - await valueField.fill(secondValidValue); - - await expect( - tokensUpdateCreateModal.getByText(secondResolvedValueText), - ).toBeVisible(); - await expect(submitButton).toBeEnabled(); - - await submitButton.click(); - - await expect( - tokensTabPanel.getByRole("button", { name: "my-token-2" }), - ).toBeEnabled(); + await saveToken(page, tokensUpdateCreateModal, { + tokenLabel, + name, + value, + resolvedText, + }); }; const unfoldTokenType = async (tokensTabPanel, type) => { @@ -427,7 +415,10 @@ export { setupTokensFileRender, setupTypographyTokensFile, setupTypographyTokensFileRender, - testTokenCreationFlow, + openTokenCreationForm, + assertTokenCreationLadder, + saveToken, + createTokenWithValue, unfoldTokenType, createToken, createSet, diff --git a/frontend/playwright/ui/specs/tokens/remapping.spec.js b/frontend/playwright/ui/specs/tokens/remapping.spec.js index 703519e791..99cfb76d46 100644 --- a/frontend/playwright/ui/specs/tokens/remapping.spec.js +++ b/frontend/playwright/ui/specs/tokens/remapping.spec.js @@ -1,11 +1,7 @@ import { test, expect } from "@playwright/test"; import { WorkspacePage } from "../../pages/WorkspacePage"; import { WasmWorkspacePage } from "../../pages/WasmWorkspacePage"; -import { - createToken, - setupTokensFileRender, - setupTypographyTokensFileRender, -} from "./helpers"; +import { createToken, setupTokensFileRender } from "./helpers"; test.beforeEach(async ({ page }) => { await WorkspacePage.init(page); @@ -123,354 +119,60 @@ test.describe("Remapping a single token", () => { "{base-shadow}", ); - // Rename base-shadow token - await renameToken(page, "base-shadow", "foundation-shadow"); - - // Check for remapping modal const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - await expect(remappingModal).toContainText("base-shadow"); - await expect(remappingModal).toContainText("foundation-shadow"); - const confirmButton = remappingModal.getByRole("button", { - name: "remap tokens", + await test.step("Remap the references", async () => { + await renameToken(page, "base-shadow", "foundation-shadow"); + + await expect(remappingModal).toBeVisible({ timeout: 5000 }); + await expect(remappingModal).toContainText("base-shadow"); + await expect(remappingModal).toContainText("foundation-shadow"); + + await remappingModal + .getByRole("button", { name: "remap tokens" }) + .click(); + + await expect( + tokensSidebar.getByRole("button", { name: "foundation-shadow" }), + ).toBeVisible(); + await expect( + tokensSidebar.getByRole("button", { name: "derived-shadow" }), + ).toBeVisible(); }); - await confirmButton.click(); - // Verify token was renamed - await expect( - tokensSidebar.getByRole("button", { name: "foundation-shadow" }), - ).toBeVisible(); - await expect( - tokensSidebar.getByRole("button", { name: "derived-shadow" }), - ).toBeVisible(); - }); + await test.step("Close the modal and keep the original name", async () => { + await renameToken(page, "foundation-shadow", "shadow-base"); - test("User renames and updates shadow token - referenced token and applied shapes update", async ({ - page, - }) => { - const { - tokensUpdateCreateModal, - tokensSidebar, - tokenContextMenuForToken, - workspacePage, - } = await setupTokensFileRender(page, { flags: ["enable-token-shadow"] }); + await expect(remappingModal).toBeVisible({ timeout: 5000 }); + await remappingModal.getByRole("button", { name: "close" }).click(); - // Create base shadow token - await createToken( - page, - "Shadow", - "primary-shadow", - "Color", - "textbox", - "#000000", - ); - - // Create derived shadow token that references base - await createCompositeDerivedToken( - page, - "Shadow", - "card-shadow", - "{primary-shadow}", - ); - - // Apply the referenced token to a shape - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Button" }) - .click(); - - await page.getByRole("tab", { name: "Tokens" }).click(); - const cardShadowToken = tokensSidebar.getByRole("button", { - name: "card-shadow", + await expect( + tokensSidebar.getByRole("button", { name: "foundation-shadow" }), + ).toBeVisible(); + await expect( + tokensSidebar.getByRole("button", { name: "derived-shadow" }), + ).toBeVisible(); }); - await cardShadowToken.click(); - // Rename and update value of base token - const primaryToken = tokensSidebar.getByRole("button", { - name: "primary-shadow", + await test.step("Do not remap and break the reference", async () => { + await renameToken(page, "foundation-shadow", "shadow-base"); + + await expect(remappingModal).toBeVisible({ timeout: 5000 }); + await remappingModal + .getByRole("button", { name: "don't remap" }) + .click(); + + await expect( + tokensSidebar.getByRole("button", { name: "shadow-base" }), + ).toBeVisible(); + await expect( + tokensSidebar.locator('[aria-label="Missing reference"]'), + ).toBeVisible(); }); - await primaryToken.click({ button: "right" }); - await tokenContextMenuForToken.getByText("Edit token").click(); - - await expect(tokensUpdateCreateModal).toBeVisible(); - const nameField = tokensUpdateCreateModal.getByLabel("Name"); - await nameField.fill("main-shadow"); - - // Update the color value - const colorField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Color", - }); - await colorField.fill("#FF0000"); - - const submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - await submitButton.click(); - - // Confirm remapping - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const confirmButton = remappingModal.getByRole("button", { - name: "remap tokens", - }); - await confirmButton.click(); - - // Verify base token was renamed - await expect( - tokensSidebar.getByRole("button", { name: "main-shadow" }), - ).toBeVisible(); - - // Verify referenced token still exists - await expect( - tokensSidebar.getByRole("button", { name: "card-shadow" }), - ).toBeVisible(); - - // Verify the shape still has the token applied with the NEW name - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Button" }) - .click(); - - // Verify the shape still has the shadow applied with the UPDATED color value - // Expand the shadow section to access the color field - const shadowSection = - workspacePage.rightSidebar.getByTestId("shadow-section"); - await expect(shadowSection).toBeVisible(); - - // Click to expand the shadow options (the menu button) - const shadowMenuButton = shadowSection - .getByRole("button", { name: "options" }) - .first(); - await shadowMenuButton.click(); - - // Wait for the advanced options to appear - await page.waitForTimeout(500); - - // Verify the color value has updated from #000000 to #FF0000 - const colorInput = shadowSection.getByRole("textbox", { name: "Color" }); - expect(colorInput).not.toBeNull(); - const colorValue = await colorInput.inputValue(); - expect(colorValue.toUpperCase()).toBe("FF0000"); - }); - }); - - test.describe("Typography Token Remapping", () => { - test("User renames typography token with alias references", async ({ - page, - }) => { - const { - tokensUpdateCreateModal, - tokensSidebar, - tokenContextMenuForToken, - } = await setupTypographyTokensFileRender(page); - - const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); - - // Create base typography token - await createToken( - page, - "Typography", - "base-text", - "Font size", - "textbox", - "16", - ); - - // Create derived typography token - await createCompositeDerivedToken( - page, - "Typography", - "body-text", - "{base-text}", - ); - - // Rename base token - await renameToken(page, "base-text", "default-text"); - - // Check for remapping modal - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const confirmButton = remappingModal.getByRole("button", { - name: "remap tokens", - }); - await confirmButton.click(); - - // Verify token was renamed - await expect( - tokensSidebar.getByRole("button", { name: "default-text" }), - ).toBeVisible(); - await expect( - tokensSidebar.getByRole("button", { name: "body-text" }), - ).toBeVisible(); - }); - - test("User renames and updates typography token - referenced token and applied shapes update", async ({ - page, - }) => { - const { - tokensUpdateCreateModal, - tokensSidebar, - tokenContextMenuForToken, - workspacePage, - } = await setupTypographyTokensFileRender(page); - - const tokensTabPanel = page.getByRole("tabpanel", { name: "tokens" }); - - // Create base typography token - await createToken( - page, - "Typography", - "body-style", - "Font size", - "textbox", - "16", - ); - - // Create derived typography token - await tokensTabPanel - .getByRole("button", { name: "Add Token: Typography" }) - .click(); - await expect(tokensUpdateCreateModal).toBeVisible(); - - let nameField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Name", - }); - await nameField.fill("paragraph-style"); - - const referenceToggle = - tokensUpdateCreateModal.getByTestId("reference-opt"); - await referenceToggle.click(); - - const referenceField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Reference", - }); - await referenceField.fill("{body-style}"); - - let submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - await submitButton.click(); - await expect(tokensUpdateCreateModal).not.toBeVisible(); - - // Apply the referenced token to a text shape - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Some Text" }) - .click(); - - await page.getByRole("tab", { name: "Tokens" }).click(); - const paragraphToken = tokensSidebar.getByRole("button", { - name: "paragraph-style", - }); - await paragraphToken.click(); - - // Rename and update value of base token - const bodyToken = tokensSidebar.getByRole("button", { - name: "body-style", - }); - await bodyToken.click({ button: "right" }); - await tokenContextMenuForToken.getByText("Edit token").click(); - - await expect(tokensUpdateCreateModal).toBeVisible(); - nameField = tokensUpdateCreateModal.getByLabel("Name"); - await nameField.fill("text-base"); - - // Update the font size value - const fontSizeField = tokensUpdateCreateModal.getByRole("textbox", { - name: "Font size", - }); - await fontSizeField.fill("18"); - - submitButton = tokensUpdateCreateModal.getByRole("button", { - name: "Save", - }); - await submitButton.click(); - - // Confirm remapping - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const confirmButton = remappingModal.getByRole("button", { - name: "remap tokens", - }); - await confirmButton.click(); - - // Verify base token was renamed - await expect( - tokensSidebar.getByRole("button", { name: "text-base" }), - ).toBeVisible(); - - // Verify referenced token still exists - await expect( - tokensSidebar.getByRole("button", { name: "paragraph-style" }), - ).toBeVisible(); - - // Verify the text shape still has the token applied with NEW name and value - await page.getByRole("tab", { name: "Layers" }).click(); - await workspacePage.layers - .getByTestId("layer-row") - .filter({ hasText: "Some Text" }) - .click(); - - // This proves the remapping worked and the value update propagated through the reference - const tokenPillSidebar = workspacePage.rightSidebar.getByLabel('paragraph-style') - await expect(tokenPillSidebar).toBeVisible(); }); }); test.describe("Border Radius Token Remapping", () => { - test("User renames border radius token with alias references", async ({ - page, - }) => { - const { tokensSidebar } = await setupTokensFileRender(page, { - flags: ["enable-token-combobox", "enable-feature-token-input"], - }); - - // Create base border radius token - await createTokenCombobox( - page, - "Border Radius", - "base-radius", - "Value", - "4", - ); - - // Create derived border radius token - await createTokenCombobox( - page, - "Border Radius", - "card-radius", - "Value", - "{base-radius}", - ); - - // Rename base token - await renameToken(page, "base-radius", "primary-radius"); - - // Check for remapping modal - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const confirmButton = remappingModal.getByRole("button", { - name: "remap tokens", - }); - await confirmButton.click(); - - // Verify token was renamed - await expect( - tokensSidebar.getByRole("button", { name: "primary-radius" }), - ).toBeVisible(); - await expect( - tokensSidebar.getByRole("button", { name: "card-radius" }), - ).toBeVisible(); - }); - test("User renames and updates border radius token - referenced token updates", async ({ page, }) => { @@ -552,98 +254,6 @@ test.describe("Remapping a single token", () => { await expect(currentValue).toHaveValue("{radius-base}"); }); }); - - test.describe("Cancel remap", () => { - test("Only rename - breaks reference", async ({ page }) => { - const { tokensSidebar } = await setupTokensFileRender(page, { - flags: ["enable-token-shadow"], - }); - - // Create base shadow token - await createToken( - page, - "Shadow", - "base-shadow", - "Color", - "textbox", - "#000000", - ); - - // Create derived shadow token that references base-shadow - await createCompositeDerivedToken( - page, - "Shadow", - "derived-shadow", - "{base-shadow}", - ); - - // Rename base-shadow token - await renameToken(page, "base-shadow", "foundation-shadow"); - - // Check for remapping modal - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const cancelButton = remappingModal.getByRole("button", { - name: "don't remap", - }); - await cancelButton.click(); - - // Verify token was renamed - await expect( - tokensSidebar.getByRole("button", { - name: "foundation-shadow", - }), - ).toBeVisible(); - await expect( - tokensSidebar.locator('[aria-label="Missing reference"]'), - ).toBeVisible(); - }); - - test("Cancel process - no changes applied", async ({ page }) => { - const { tokensSidebar } = await setupTokensFileRender(page, { - flags: ["enable-token-shadow"], - }); - - // Create base shadow token - await createToken( - page, - "Shadow", - "base-shadow", - "Color", - "textbox", - "#000000", - ); - - // Create derived shadow token that references base-shadow - await createCompositeDerivedToken( - page, - "Shadow", - "derived-shadow", - "{base-shadow}", - ); - - // Rename base-shadow token - await renameToken(page, "base-shadow", "foundation-shadow"); - - // Check for remapping modal - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const closeButton = remappingModal.getByRole("button", { - name: "close", - }); - await closeButton.click(); - - // Verify original token name still exists - await expect( - tokensSidebar.getByRole("button", { name: "base-shadow" }), - ).toBeVisible(); - await expect( - tokensSidebar.getByRole("button", { name: "derived-shadow" }), - ).toBeVisible(); - }); - }); }); test.describe("Remapping group of tokens", () => { @@ -750,107 +360,4 @@ test.describe("Remapping group of tokens", () => { fillSection.getByLabel("brand.primary", { exact: true }), ).toBeVisible(); }); - - test("User renames a group - and remaps", async ({ page }) => { - const { tokensSidebar } = await setupTokensFileRender(page); - const workspacePage = new WasmWorkspacePage(page); - const rightSidebar = workspacePage.rightSidebar; - - // Create multiple tokens in a group - await createToken( - page, - "Color", - "light.primary", - "Value", - "textbox", - "#FFFFFF", - ); - await createToken( - page, - "Color", - "light.secondary", - "Value", - "textbox", - "#EEEEEE", - ); - - // Verify that the node and child token are visible before deletion - const lightNode = tokensSidebar.getByRole("button", { - name: "light", - exact: true, - }); - const lightNodeToken = tokensSidebar.getByRole("button", { - name: "primary", - }); - - // Select a node and right click on it to open context menu - await expect(lightNode).toBeVisible(); - await expect(lightNodeToken).toBeVisible(); - - // Apply token to a shape to ensure remapping modal appears with applied token reference - await page.getByRole("tab", { name: "Layers" }).click(); - await page - .getByTestId("layer-row") - .filter({ hasText: "Rectangle" }) - .first() - .click(); - - await page.getByRole("tab", { name: "Tokens" }).click(); - const lightPrimaryToken = tokensSidebar.getByRole("button", { - name: "primary", - }); - await lightPrimaryToken.click(); - - // Right click on the node to rename - - await lightNode.click({ button: "right" }); - const renameNodeButton = page.getByRole("button", { - name: "Rename", - exact: true, - }); - await expect(renameNodeButton).toBeVisible(); - await renameNodeButton.click(); - - // Expect the rename modal to be visible, fill in the new name and submit - const tokenRenameNodeModal = page.getByTestId("token-rename-node-modal"); - await expect(tokenRenameNodeModal).toBeVisible(); - - const nameField = tokenRenameNodeModal.getByRole("textbox", { - name: "Name", - }); - await nameField.fill("lighter"); - - const submitButton = tokenRenameNodeModal.getByRole("button", { - name: "Rename", - }); - await submitButton.click(); - - // Ensure that the remapping modal appears and confirm remap - const remappingModal = page.getByTestId("token-remapping-modal"); - await expect(remappingModal).toBeVisible({ timeout: 5000 }); - - const confirmButton = remappingModal.getByRole("button", { - name: "remap tokens", - }); - await confirmButton.click(); - - // Verify that the node has been renamed and tokens are still visible - const lighterNode = tokensSidebar.getByRole("button", { - name: "lighter", - exact: true, - }); - - await expect(lighterNode).toBeVisible(); - - // Verify that the applied token reference has been updated in the right sidebar for the selected shape - const fillSection = rightSidebar.getByRole("region", { - name: "Fill section", - }); - await expect(fillSection).toBeVisible(); - - const tokenReference = fillSection.getByLabel("lighter.primary", { - exact: true, - }); - await expect(tokenReference).toBeVisible(); - }); }); diff --git a/frontend/playwright/ui/specs/tokens/sets.spec.js b/frontend/playwright/ui/specs/tokens/sets.spec.js index 9f1f3ed3f4..7aba083163 100644 --- a/frontend/playwright/ui/specs/tokens/sets.spec.js +++ b/frontend/playwright/ui/specs/tokens/sets.spec.js @@ -186,6 +186,61 @@ test.describe("Tokens: Sets Tab", () => { await expect(darkSet).toHaveCount(0); }); + test("Folding a group after reordering collapses only that group", async ({ + page, + }) => { + const { tokenThemesSetsSidebar, tokenSetGroupItems } = + await setupTokensFileRender(page); + + // The reorder swaps two groups, so every tree index changes while the DOM + // nodes keep their positions. + await createSet(tokenThemesSetsSidebar, "Colors/primary"); + + const lightDarkGroup = tokenSetGroupItems.filter({ hasText: "LightDark" }); + const colorsGroup = tokenSetGroupItems.filter({ hasText: "Colors" }); + + await expect(tokenSetGroupItems).toHaveCount(2); + await expect(tokenSetGroupItems.first()).toContainText("LightDark"); + + // Drag the Colors group above the LightDark group. The sets list is a small + // scroll container, so both groups cannot be under the pointer at once and + // the HTML5 drag events are dispatched directly. + const target = await lightDarkGroup.boundingBox(); + const dataTransfer = await page.evaluateHandle(() => new DataTransfer()); + + await colorsGroup.dispatchEvent("dragstart", { dataTransfer }); + await lightDarkGroup.dispatchEvent("dragenter", { dataTransfer }); + await lightDarkGroup.dispatchEvent("dragover", { + dataTransfer, + clientX: target.x + 100, + clientY: target.y + 2, + }); + await lightDarkGroup.dispatchEvent("drop", { + dataTransfer, + clientX: target.x + 100, + clientY: target.y + 2, + }); + await colorsGroup.dispatchEvent("dragend", { dataTransfer }); + + await expect(tokenSetGroupItems.first()).toContainText("Colors"); + + // Folding the first group hides only its own children. + await tokenSetGroupItems + .first() + .getByTestId("tokens-set-group-collapse") + .click(); + + await expect( + tokenThemesSetsSidebar.getByRole("button", { + name: "primary", + exact: true, + }), + ).toHaveCount(0); + await expect( + tokenThemesSetsSidebar.getByRole("button", { name: "dark", exact: true }), + ).toBeVisible(); + }); + test("Change current theme", async ({ page }) => { const { tokenThemesSetsSidebar, tokenSetItems } = await setupTokensFileRender(page); diff --git a/frontend/playwright/ui/specs/tokens/themes.spec.js b/frontend/playwright/ui/specs/tokens/themes.spec.js index 855c0716ca..dc8d9a90ed 100644 --- a/frontend/playwright/ui/specs/tokens/themes.spec.js +++ b/frontend/playwright/ui/specs/tokens/themes.spec.js @@ -291,4 +291,30 @@ test.describe("Tokens: Themes modal", () => { tokenThemeUpdateCreateModal.getByText("Changed Group name"), ).toBeVisible(); }); + + test("Clicking a set name ticks and unticks it", async ({ page }) => { + const { tokenThemeUpdateCreateModal, workspacePage } = + await setupTokensFileRender(page); + + await workspacePage.openTokenThemesModal(); + + await tokenThemeUpdateCreateModal + .getByText("3 active sets") + .first() + .click(); + + const darkSet = tokenThemeUpdateCreateModal.getByRole("button", { + name: "dark", + exact: true, + }); + const darkCheckbox = darkSet.getByRole("checkbox"); + + await expect(darkCheckbox).not.toBeChecked(); + + await darkSet.click(); + await expect(darkCheckbox).toBeChecked(); + + await darkSet.click(); + await expect(darkCheckbox).not.toBeChecked(); + }); }); diff --git a/frontend/playwright/ui/specs/workspace-shared-library.spec.js b/frontend/playwright/ui/specs/workspace-shared-library.spec.js index b9bab493f1..4f65635f30 100644 --- a/frontend/playwright/ui/specs/workspace-shared-library.spec.js +++ b/frontend/playwright/ui/specs/workspace-shared-library.spec.js @@ -162,3 +162,62 @@ test("BUG 14214 - Updates tab refreshes after syncing a freshly linked library", updatesPanel.getByText("There are no Shared Libraries that need update"), ).toBeVisible(); }); + +test("Swap component menu keeps the search when changing library", async ({ + page, +}) => { + const workspace = new WasmWorkspacePage(page); + await workspace.setupEmptyFile(page); + + await workspace.mockRPC( + /get\-file\?/, + "workspace/get-file-9066-swap-groups.json", + ); + await workspace.mockRPC( + new RegExp(`get\\-file\\?id=${sharedFileId}`), + "workspace/get-file-9056_shared.json", + ); + await workspace.mockRPC( + "get-file-libraries?file-id=*", + "workspace/get-file-libraries-9056.json", + ); + await workspace.mockRPC( + `get-file-fragment?file-id=${sharedFileId}&fragment-id=${sharedFileFragmentId1}`, + "workspace/get-file-fragment-9056_shared-1.json", + ); + await workspace.mockRPC( + `get-file-fragment?file-id=${sharedFileId}&fragment-id=${sharedFileFragmentId2}`, + "workspace/get-file-fragment-9056_shared-2.json", + ); + + await workspace.goToWorkspace({ + fileId: "e179d9df-de35-80bf-8005-283bbd5516b0", + pageId: "e179d9df-de35-80bf-8005-283bbd5516b1", + }); + + await workspace.clickToggableLayer("Board"); + await workspace.clickToggableLayer("Group"); + await workspace.layers.getByText("A", { exact: true }).click(); + await workspace.rightSidebar.getByRole("button", { name: /^A / }).click(); + + const search = workspace.rightSidebar.locator( + "#swap-component-search-filter", + ); + await search.fill("rect"); + await expect( + workspace.rightSidebar.getByText("There are no assets in this library yet"), + ).toBeVisible(); + + await workspace.rightSidebar + .getByRole("combobox") + .filter({ hasText: "New File 10" }) + .click(); + await workspace.rightSidebar + .getByRole("option", { name: "published component" }) + .click(); + + await expect(search).toHaveValue("rect"); + await expect( + workspace.rightSidebar.getByText("Rectangle", { exact: true }), + ).toBeVisible(); +}); diff --git a/frontend/playwright/ui/specs/workspace.spec.js b/frontend/playwright/ui/specs/workspace.spec.js index b59266f5eb..051530b877 100644 --- a/frontend/playwright/ui/specs/workspace.spec.js +++ b/frontend/playwright/ui/specs/workspace.spec.js @@ -107,8 +107,6 @@ test("Selection size badge appears on selection and hides on deselect", async ({ test("Selection size badge uses component color for component selection", async ({ page, }) => { - // The expected component color below is the dark palette value. - await page.emulateMedia({ colorScheme: "dark" }); const workspacePage = new WasmWorkspacePage(page); await workspacePage.setupEmptyFile(); await workspacePage.mockGetFile("components/get-file-13267.json"); diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 9f290ceb3f..e054fa7f2a 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/frontend/resources/images/assets/brand-github.svg b/frontend/resources/images/assets/brand-github.svg index cfb34953e0..c950a1d608 100644 --- a/frontend/resources/images/assets/brand-github.svg +++ b/frontend/resources/images/assets/brand-github.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/frontend/resources/images/icons/pencil.svg b/frontend/resources/images/icons/pencil.svg index 1295c5c7e6..50dfe16628 100644 --- a/frontend/resources/images/icons/pencil.svg +++ b/frontend/resources/images/icons/pencil.svg @@ -1,4 +1,3 @@ - - - + + diff --git a/frontend/resources/images/icons/scale.svg b/frontend/resources/images/icons/scale.svg new file mode 100644 index 0000000000..93b7010809 --- /dev/null +++ b/frontend/resources/images/icons/scale.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index ef226a62b0..c18338fbf4 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -70,7 +70,8 @@ :compiler-options {:output-feature-set :es2020 :output-wrapper false - :warnings {:fn-deprecated false}} + :warnings {:fn-deprecated false} + :closure-defines {app.config/devenv-mode #shadow/env ["PENPOT_DEVENV" :as :bool :default false]}} :release {:closure-defines {goog.DEBUG false @@ -175,4 +176,92 @@ {:compiler-options {:fn-invoke-direct true :elide-asserts true - :anon-fn-naming-policy :off}}}}} + :anon-fn-naming-policy :off}}} + + :performance-tests + {:target :esm + :output-dir "target/performance-tests" + :runtime :custom + :js-options {:js-provider :import} + + + :modules + {:test {:init-fn performance-tests.render-wasm.runner/init + :prepend-js ";if (typeof globalThis.navigator?.userAgent === 'undefined') { globalThis.navigator = {userAgent: ''}; };"}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-runner + {:target :node-script + :output-to "target/performance/render-wasm/runner.cjs" + :output-dir "target/performance/render-wasm/runner/" + :main benches.render-wasm.node.cli.main/-main + :devtools {:autoload false} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-commands + {:target :node-library + :output-to "target/performance/render-wasm/commands.cjs" + :output-dir "target/performance/render-wasm/commands/" + :exports {:execute benches.render-wasm.node.cli.deferred/execute} + :devtools {:autoload false} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-browser + {:target :esm + :output-dir #shadow/env ["RENDER_BENCH_BROWSER_DIR" :default "target/performance/render-wasm/browser/"] + :closure-defines {benches.render-wasm.identity/build-id #shadow/env ["RENDER_BENCH_BUILD_ID" :default "unprepared"]} + ;; Bundle npm dependencies for the dedicated static host. + :js-options {:js-provider :shadow} + + :modules + {:browser + {:exports {default benches.render-wasm.browser.bridge/bridge}}} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-catalog + {:target :node-library + :output-to #shadow/env ["RENDER_BENCH_CASES_PATH" :default "target/performance/render-wasm/cases.cjs"] + :output-dir "target/performance/render-wasm/catalog/" + :exports {:collectCases benches.render-wasm.node.catalog.entry/collect-cases + :buildIdentity benches.render-wasm.node.catalog.entry/build-identity} + :closure-defines {benches.render-wasm.identity/build-id #shadow/env ["RENDER_BENCH_BUILD_ID" :default "unprepared"]} + :compiler-options {:output-feature-set :es2020 + :warnings {:fn-deprecated false}} + :release {:compiler-options {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}}}} diff --git a/frontend/src/app/config.cljs b/frontend/src/app/config.cljs index 6bdfa17381..370159ca82 100644 --- a/frontend/src/app/config.cljs +++ b/frontend/src/app/config.cljs @@ -73,10 +73,26 @@ :browser :webworker)) +;; --- Compile-time devenv switch +;; +;; Set to true only by the devenv containers (PENPOT_DEVENV=true in +;; docker-compose.main.yml); production builds leave the default. When true, +;; the frontend disables the admin-console by default; `enable-admin-console` +;; in the user's `config.js` re-enables it (user flags parse after the +;; injected default and the last flag wins). +(goog-define devenv-mode false) + +(defn- apply-devenv-defaults + "Prepends the devenv default `disable-admin-console` before the user flags. + User flags are parsed after it, so `enable-admin-console` wins." + [devenv? flags] + (if devenv? (concat [:disable-admin-console] flags) flags)) + (defn- parse-flags [global] (let [flags (obj/get global "penpotFlags" "") - flags (sequence (map keyword) (str/words flags))] + flags (sequence (map keyword) (str/words flags)) + flags (apply-devenv-defaults devenv-mode flags)] (flags/parse flags/default flags))) (defn- parse-version diff --git a/frontend/src/app/main/data/common.cljs b/frontend/src/app/main/data/common.cljs index c2a7de99af..229f5852cc 100644 --- a/frontend/src/app/main/data/common.cljs +++ b/frontend/src/app/main/data/common.cljs @@ -13,6 +13,7 @@ [app.common.time :as ct] [app.common.types.organization :as cto] [app.common.types.team :as ctt] + [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.helpers :as dsh] [app.main.data.modal :as modal] @@ -237,8 +238,8 @@ (defn- check-team-sso [team-id] - (let [url (rt/get-current-href)] - (->> (rp/cmd! :check-nitrate-sso {:team-id team-id :url url}) + (let [query-string (rt/get-current-query-string)] + (->> (rp/cmd! :check-nitrate-sso {:team-id team-id :query-string query-string}) (rx/mapcat (fn [{:keys [authorized redirect-uri]}] (if authorized (rx/empty) @@ -405,6 +406,26 @@ ;; NAVEGATION EVENTS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defn- resolve-team-id + "The team a team scoped navigation must carry: the UUID the caller + passes, the team initialized in the store, the team of the route + already on screen, or the profile default. A navigation carries a nil + id only when none of the four exists. + + The route goes before the profile default on purpose: the emitters that + run from a team screen know the team the user is looking at, and the + personal team would open their file under another team. + + `team-container*` raises `:not-found :invalid-team-id` for a dashboard + or workspace route without a valid id, and `:not-found` is not reported + to the error pipeline, so a navigation that loses the id ends on the + not-found page without leaving a trace." + [state team-id] + (or (when (uuid? team-id) team-id) + (:current-team-id state) + (some-> (rt/get-query-param (rt/get-params state) :team-id) uuid/parse*) + (dm/get-in state [:profile :default-team-id]))) + (defn go-to-feedback [] (ptk/reify ::go-to-feedback @@ -420,7 +441,7 @@ ptk/WatchEvent (watch [_ state _] (let [profile (get state :profile) - team-id (or team-id (:current-team-id state)) + team-id (resolve-team-id state team-id) project-id (if (= project-id :default) (:default-project-id profile) project-id) @@ -434,7 +455,7 @@ (ptk/reify ::go-to-dashboard-search ptk/WatchEvent (watch [_ state stream] - (let [team-id (:current-team-id state)] + (let [team-id (resolve-team-id state nil)] (rx/merge (->> (rx/of (rt/nav :dashboard-search {:team-id team-id @@ -454,7 +475,7 @@ (ptk/reify ::go-to-dashboard-libraries ptk/WatchEvent (watch [_ state _] - (let [team-id (or team-id (:current-team-id state))] + (let [team-id (resolve-team-id state team-id)] (rx/of (rt/nav :dashboard-libraries {:team-id team-id})))))) @@ -463,7 +484,7 @@ (ptk/reify ::go-to-dashboard-fonts ptk/WatchEvent (watch [_ state _] - (let [team-id (or team-id (:current-team-id state))] + (let [team-id (resolve-team-id state team-id)] (rx/of (rt/nav :dashboard-fonts {:team-id team-id})))))) (defn go-to-dashboard-recent @@ -472,15 +493,9 @@ ptk/WatchEvent (watch [_ state _] (let [profile (get state :profile) - team-id (cond - (= :default team-id) + team-id (if (= :default team-id) (:default-team-id profile) - - (uuid? team-id) - team-id - - :else - (:current-team-id state)) + (resolve-team-id state team-id)) params {:team-id team-id}] (rx/of (modal/hide) (rt/nav :dashboard-recent params options)))))) @@ -490,7 +505,7 @@ (ptk/reify ::go-to-dashboard-members ptk/WatchEvent (watch [_ state _] - (let [team-id (:current-team-id state)] + (let [team-id (resolve-team-id state nil)] (rx/of (rt/nav :dashboard-members {:team-id team-id})))))) (defn go-to-dashboard-invitations @@ -498,7 +513,7 @@ (ptk/reify ::go-to-dashboard-invitations ptk/WatchEvent (watch [_ state _] - (let [team-id (:current-team-id state)] + (let [team-id (resolve-team-id state nil)] (rx/of (rt/nav :dashboard-invitations {:team-id team-id})))))) (defn go-to-dashboard-webhooks @@ -506,7 +521,7 @@ (ptk/reify ::go-to-dashboard-webhooks ptk/WatchEvent (watch [_ state _] - (let [team-id (:current-team-id state)] + (let [team-id (resolve-team-id state nil)] (rx/of (rt/nav :dashboard-webhooks {:team-id team-id})))))) (defn go-to-dashboard-settings @@ -514,7 +529,7 @@ (ptk/reify ::go-to-dashboard-settings ptk/WatchEvent (watch [_ state _] - (let [team-id (:current-team-id state)] + (let [team-id (resolve-team-id state nil)] (rx/of (rt/nav :dashboard-settings {:team-id team-id})))))) (defn go-to-workspace @@ -522,7 +537,7 @@ (ptk/reify ::go-to-workspace ptk/WatchEvent (watch [_ state _] - (let [team-id (or team-id (:current-team-id state)) + (let [team-id (resolve-team-id state team-id) file-id (or file-id (:current-file-id state)) page-id (or page-id (:current-page-id state) (-> (dsh/lookup-file-data state file-id) @@ -575,15 +590,9 @@ ptk/WatchEvent (watch [_ state _] (let [profile (get state :profile) - team-id (cond - (= :default team-id) + team-id (if (= :default team-id) (:default-team-id profile) - - (uuid? team-id) - team-id - - :else - (:current-team-id state)) + (resolve-team-id state team-id)) params {:team-id team-id}] (rx/of (modal/hide) (rt/nav :dashboard-deleted params options)))))) diff --git a/frontend/src/app/main/data/dashboard.cljs b/frontend/src/app/main/data/dashboard.cljs index 939ef5ec48..1c31ef4976 100644 --- a/frontend/src/app/main/data/dashboard.cljs +++ b/frontend/src/app/main/data/dashboard.cljs @@ -416,7 +416,7 @@ ptk/WatchEvent (watch [_ _ _] - (let [params {:id id :name name}] + (let [params (with-meta {:id id :name name} (meta params))] (->> (rp/cmd! :rename-project params) (rx/ignore)))))) @@ -478,7 +478,7 @@ ptk/WatchEvent (watch [_ _ _] - (let [params (select-keys params [:id :name])] + (let [params (with-meta (select-keys params [:id :name]) (meta params))] (->> (rp/cmd! :rename-file params) (rx/ignore)))))) @@ -503,15 +503,18 @@ ptk/WatchEvent (watch [_ _ _] - (let [params {:id id :is-shared is-shared}] + (let [mdata (meta params) + origin (::ev/origin mdata "dashboard")] (rx/concat - (->> (rp/cmd! :set-file-shared params) + (->> (rp/cmd! :set-file-shared (with-meta {:id id :is-shared is-shared} mdata)) (rx/ignore)) (when is-shared (->> (rp/cmd! :get-file-summary {:id id}) (rx/map (fn [summary] (when (-> summary :variants :count pos?) - (ev/event {::ev/name "set-file-variants-shared" ::ev/origin "dashboard"}))))))))))) + (ev/event (merge {::ev/name "set-file-variants-shared" + ::ev/origin origin} + (select-keys mdata [::ev/context]))))))))))))) (defn set-file-thumbnail [file-id thumbnail-id] diff --git a/frontend/src/app/main/data/exports/assets.cljs b/frontend/src/app/main/data/exports/assets.cljs index f041557921..b10250e833 100644 --- a/frontend/src/app/main/data/exports/assets.cljs +++ b/frontend/src/app/main/data/exports/assets.cljs @@ -28,6 +28,8 @@ [cuerdas.core :as str] [potok.v2.core :as ptk])) +(def raster-export-types #{:png :jpeg :webp}) + (def default-timeout 5000) ;; Longest wait for the pending changes of the exported files to be saved. diff --git a/frontend/src/app/main/data/exports/wasm.cljs b/frontend/src/app/main/data/exports/wasm.cljs index b23556474a..f6e6b4aab8 100644 --- a/frontend/src/app/main/data/exports/wasm.cljs +++ b/frontend/src/app/main/data/exports/wasm.cljs @@ -55,7 +55,7 @@ :pdf (wasm.api/render-shape-pdf object-id) :svg (wasm.api/render-shape-svg object-id scale) (if (= :client (wasm.api/export-target object-id scale type)) - (wasm.api/render-shape-pixels object-id scale type) + (wasm.api/render-shape-pixels object-id scale type true) (wasm.api/render-shape-raster object-id scale type))))) (defn export-image-uri diff --git a/frontend/src/app/main/data/fonts.cljs b/frontend/src/app/main/data/fonts.cljs index dca3c54263..252e95ebc1 100644 --- a/frontend/src/app/main/data/fonts.cljs +++ b/frontend/src/app/main/data/fonts.cljs @@ -18,6 +18,7 @@ [app.main.fonts :as fonts] [app.main.repo :as rp] [app.main.store :as st] + [app.main.worker :as mw] [app.util.i18n :refer [tr]] [app.util.storage :as storage] [app.util.webapi :as wa] @@ -85,6 +86,17 @@ (->> (rp/cmd! :get-font-variants {:team-id team-id}) (rx/map fonts-fetched))))) +(defn- sync-worker-fonts + "Store watcher that sends the worker the team's custom fonts each + time they change. The worker's render looks them up in its own + store, which nothing else fills." + [_ _ old-state new-state] + (let [fonts (:fonts new-state)] + (when-not (identical? fonts (:fonts old-state)) + (mw/emit! {:cmd :fonts/sync :fonts fonts})))) + +(add-watch st/state ::sync-worker-fonts sync-worker-fonts) + (defn upload-font-variant "Uploads a single font variant item using the chunked upload API. @@ -94,7 +106,7 @@ can assemble the chunks and materialise the final font-variant record. Returns an observable that emits the created font-variant." - [{:keys [data team-id font-id font-family font-weight font-style] :as _item}] + [{:keys [data team-id font-id font-family font-weight font-style variant-name] :as _item}] ;; Upload each mtype as a separate chunked session in parallel, collect ;; all [mtype session-id] pairs, then call create-font-variant with :uploads. (->> (rx/from (seq data)) @@ -108,12 +120,100 @@ {}) (rx/mapcat (fn [uploads] (rp/cmd! :create-font-variant - {:team-id team-id - :font-id font-id - :font-family font-family - :font-weight font-weight - :font-style font-style - :uploads uploads}))))) + {:team-id team-id + :font-id font-id + :font-family font-family + :font-weight font-weight + :font-style font-style + :variant-name variant-name + :uploads uploads}))))) + +(defn resolve-font-metadata + "Resolve the upload metadata for a font file. + + `names` is a map carrying the `:family` and `:variant` values read by + opentype.js; each may be nil when the font has no usable name records. + `filename` is the original file name. The opentype.js values win per + field; any missing value is derived from the filename instead." + [names filename] + (let [{:keys [family variant]} names + base-name (str/replace filename #"\.[^.]+$" "") + variant (not-empty (str/trim (or variant ""))) + family (not-empty (str/trim (or family ""))) + variant-source (or variant base-name)] + {:font-family (or family (cm/parse-font-family base-name)) + :font-weight (cm/parse-font-weight variant-source) + :font-style (cm/parse-font-style variant-source) + :variant-name variant})) + +(defn prepare-font-entry + "Prepare a single read font entry for upload. + + `:font` is the object parsed by opentype.js (nil for woff2, which cannot + be parsed); `:name`, `:type` and `:data` come from reading the file. + Returns the entry metadata, the `:content` to upload and, for parsed + fonts, the `:height-warning?` flag. A font with missing vertical-metric + tables uploads with `:height-warning? false` instead of failing." + [{:keys [font type name data] :as _params}] + (if font + ;; Font was parsed with opentype.js (ttf, otf, woff) + (let [;; Select the first nonblank name candidate: a blank + ;; preferredFamily/preferredSubfamily must not shadow a valid + ;; fontFamily/fontSubfamily. + family (cm/first-nonblank-string + (.getEnglishName ^js font "preferredFamily") + (.getEnglishName ^js font "fontFamily")) + variant (cm/first-nonblank-string + (.getEnglishName ^js font "preferredSubfamily") + (.getEnglishName ^js font "fontSubfamily")) + + ;; Vertical metrics determine the baseline in a text and the space between lines of + ;; text. For historical reasons, there are three pairs of ascender/descender + ;; values, known as hhea, OS/2 and uSWin metrics. Depending on the font, operating + ;; system and application a different set will be used to render text on the + ;; screen. On Mac, Safari and Chrome use the hhea values to render text. Firefox + ;; will respect the useTypoMetrics setting and will use the OS/2 if it is set. If + ;; the useTypoMetrics is not set, Firefox will also use metrics from the hhea + ;; table. On Windows, all browsers use the usWin metrics, but respect the + ;; useTypoMetrics setting and if set will use the OS/2 values. + + hhea (some-> ^js font .-tables .-hhea) + os2 (some-> ^js font .-tables .-os2) + + hhea-ascender (some-> ^js hhea .-ascender abs) + hhea-descender (some-> ^js hhea .-descender abs) + + win-ascent (some-> ^js os2 .-usWinAscent abs) + win-descent (some-> ^js os2 .-usWinDescent abs) + + os2-ascent (some-> ^js os2 .-sTypoAscender abs) + os2-descent (some-> ^js os2 .-sTypoDescender abs) + + ;; useTypoMetrics can be read from the 7th bit + f-selection (some-> ^js os2 .-fsSelection (bit-test 7)) + + height-warning? (boolean + (and hhea-ascender hhea-descender win-ascent win-descent + (or (not= hhea-ascender win-ascent) + (not= hhea-descender win-descent) + (and f-selection os2-ascent os2-descent + (or (not= hhea-ascender os2-ascent) + (not= hhea-descender os2-descent)))))) + data (js/Uint8Array. data) + metadata (resolve-font-metadata {:family family :variant variant} name)] + (assoc metadata + :content {:data data + :name name + :type type} + :height-warning? height-warning?)) + ;; Font could not be parsed (woff2), extract metadata from filename + (let [data (js/Uint8Array. data) + metadata (resolve-font-metadata {} name)] + (assoc metadata + :content {:data data + :name name + :type type} + :height-warning? false)))) (defn- parse-font-mtype "Detects a font mime-type from the first four bytes of an ArrayBuffer. @@ -159,84 +259,6 @@ (log/warn :msg (str/fmt "skipping file %s, unsupported format" name)) {:error name})))) -(defn- prepare-font - "Builds a ready-to-upload font entry from a parsed blob. - - Never throws for missing font metadata: a nil subfamily (opentype.js - returns nil when the font carries no English preferred/subfamily - names, as seen with some Fontshare families) defaults to regular - weight/style, and missing vertical-metric tables default to no - height warning." - [{:keys [font type name data] :as _params}] - (let [name (or name "")] - (if (some? font) - ;; Font was parsed with opentype.js (ttf, otf, woff) - (let [family (or (.getEnglishName ^js font "preferredFamily") - (.getEnglishName ^js font "fontFamily") - "") - variant (or (.getEnglishName ^js font "preferredSubfamily") - (.getEnglishName ^js font "fontSubfamily") - "") - - ;; Vertical metrics determine the baseline in a text and the space between lines of - ;; text. For historical reasons, there are three pairs of ascender/descender - ;; values, known as hhea, OS/2 and uSWin metrics. Depending on the font, operating - ;; system and application a different set will be used to render text on the - ;; screen. On Mac, Safari and Chrome use the hhea values to render text. Firefox - ;; will respect the useTypoMetrics setting and will use the OS/2 if it is set. If - ;; the useTypoMetrics is not set, Firefox will also use metrics from the hhea - ;; table. On Windows, all browsers use the usWin metrics, but respect the - ;; useTypoMetrics setting and if set will use the OS/2 values. - - hhea (some-> ^js font .-tables .-hhea) - os2 (some-> ^js font .-tables .-os2) - - hhea-ascender (some-> ^js hhea .-ascender abs) - hhea-descender (some-> ^js hhea .-descender abs) - - win-ascent (some-> ^js os2 .-usWinAscent abs) - win-descent (some-> ^js os2 .-usWinDescent abs) - - os2-ascent (some-> ^js os2 .-sTypoAscender abs) - os2-descent (some-> ^js os2 .-sTypoDescender abs) - - ;; useTypoMetrics can be read from the 7th bit - f-selection (some-> ^js os2 .-fsSelection (bit-test 7)) - - height-warning? (boolean - (and hhea-ascender hhea-descender win-ascent win-descent - (or (not= hhea-ascender win-ascent) - (not= hhea-descender win-descent) - (and f-selection os2-ascent os2-descent - (or (not= hhea-ascender os2-ascent) - (not= hhea-descender os2-descent)))))) - data (js/Uint8Array. data)] - {:content {:data data - :name name - :type type} - :font-family (or family "") - :font-weight (cm/parse-font-weight variant) - :font-style (cm/parse-font-style variant) - :variant-name variant - :height-warning? height-warning?}) - ;; Font could not be parsed (woff2), extract metadata from filename - (let [base-name (str/replace name #"\.[^.]+$" "") - ;; Strip known weight/style tokens and separators to derive family name - ;; Use word boundaries to avoid matching substrings (e.g. "Boldini" should not match "bold") - raw-family-name (-> base-name - (str/replace #"(?i)(^|[-_\s])(extra\s*black|ultra\s*black|extra\s*bold|ultra\s*bold|semi\s*bold|demi\s*bold|extra\s*light|ultra\s*light|hairline|thin|light|normal|regular|medium|bold|black|heavy|solid|italic)([-_\s]|$)" "$1$3") - (str/replace #"[-_\s]+" " ") - (str/trim)) - family-name (if (str/blank? raw-family-name) base-name raw-family-name) - data (js/Uint8Array. data)] - {:content {:data data - :name name - :type type} - :font-family family-name - :font-weight (cm/parse-font-weight base-name) - :font-style (cm/parse-font-style base-name) - :height-warning? false})))) - (defn process-upload "Given a seq of blobs and the team id, creates a ready-to-use fonts map with temporal ID's associated to each font entry. @@ -296,7 +318,7 @@ (if (:error item) (update acc :errors conj (str "'" (:error item) "'")) (try - (update acc :fonts #(join % (prepare-font item))) + (update acc :fonts #(join % (prepare-font-entry item))) (catch :default cause (log/warn :msg "skipping font file" :cause cause :name (:name item)) (update acc :errors conj (str "'" (:name item) "'")))))) diff --git a/frontend/src/app/main/data/imports.cljs b/frontend/src/app/main/data/imports.cljs index 6fd079b015..415b204b0b 100644 --- a/frontend/src/app/main/data/imports.cljs +++ b/frontend/src/app/main/data/imports.cljs @@ -15,6 +15,7 @@ (:require [app.common.exceptions :as ex] [app.common.json :as json] + [app.common.logging :as log] [app.common.schema :as sm] [app.common.uuid :as uuid] [app.main.data.jobs :as dj] @@ -25,8 +26,7 @@ [app.util.i18n :as i18n :refer [tr]] [app.util.zip :as uz] [beicon.v2.core :as rx] - [cuerdas.core :as str] - [promesa.core :as p])) + [cuerdas.core :as str])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; MESSAGES OF A FAILURE @@ -157,25 +157,43 @@ {:files {} :objects {} :references []} entries)) +(defn- read-reference + "The metadata of one of the media or the thumbnails of a file, decoded, + paired with the file it belongs to. A reference that cannot be read is + unknown, never a failure of the package: it only leaves the size of its + file unknown." + [file-id entry] + (->> (rx/from (uz/read-as-text entry)) + (rx/map (fn [text] [file-id (json/decode text)])) + (rx/catch (fn [cause] + (log/warn :msg "cannot read a reference of the package" + :file-id file-id + :entry (.-filename ^js entry) + :cause cause) + (rx/of [file-id ::unreadable]))))) + (defn- read-objects "The objects every file of a container references, read from the - metadata of its media and its thumbnails." + metadata of its media and its thumbnails, one entry at a time. + + Every read opens its own decompression stream and the container of a + big file can hold thousands of references: reading them all at once + opens thousands of streams and is what breaks the analysis on the + browsers that cannot keep up with them." [references] - (->> (map (fn [[file-id entry]] - (->> (uz/read-as-text entry) - (p/fmap (fn [text] [file-id (json/decode text)])))) - references) - (p/all) - (p/fmap - (fn [decoded] - (reduce (fn [result [file-id data]] - (->> [:media-id :thumbnail-id] - (keep #(get data %)) - (reduce (fn [result object-id] - (update result file-id (fnil conj #{}) object-id)) - result))) - {} - decoded))))) + (->> (rx/from references) + (rx/mapcat (fn [[file-id entry]] + (read-reference file-id entry))) + (rx/reduce + (fn [[referenced unreadable] [file-id data]] + (if (= ::unreadable data) + [referenced (conj unreadable file-id)] + [(reduce (fn [referenced object-id] + (update referenced file-id (fnil conj #{}) object-id)) + referenced + (keep #(get data %) [:media-id :thumbnail-id])) + unreadable])) + [{} #{}]))) (defn- file-sizes "How much every file of a container occupies inside it: its own entries @@ -186,13 +204,17 @@ [entries] (let [{:keys [files objects references]} (index-container entries)] (->> (read-objects references) - (p/fmap - (fn [referenced] - (reduce-kv (fn [sizes file-id object-ids] - (update sizes file-id (fnil + 0) - (reduce + 0 (keep objects object-ids)))) - files - referenced)))))) + (rx/map + (fn [[referenced unreadable]] + (let [sizes (reduce-kv (fn [sizes file-id object-ids] + (update sizes file-id (fnil + 0) + (reduce + 0 (keep objects object-ids)))) + files + referenced)] + ;; a file with a reference that could not be read has no + ;; size at all: a number that misses one of its objects is + ;; worse than no number + (reduce dissoc sizes unreadable))))))) (defn- analyze-file "The entries of one file the user dropped: one per file of a version 3 @@ -215,9 +237,8 @@ (rx/merge-map (fn [manifest] (if (= (:type manifest) "penpot/export-files") - (->> (uz/get-entries zip-reader) - (p/mcat file-sizes) - (rx/from) + (->> (rx/from (uz/get-entries zip-reader)) + (rx/mapcat file-sizes) (rx/map (fn [sizes] (-> file (assoc :type :binfile-v3) diff --git a/frontend/src/app/main/data/nitrate.cljs b/frontend/src/app/main/data/nitrate.cljs index 1fbbce503c..7d1889e779 100644 --- a/frontend/src/app/main/data/nitrate.cljs +++ b/frontend/src/app/main/data/nitrate.cljs @@ -412,8 +412,11 @@ :add-method "move-existing-team-to-organization" :subscription-status subscription-status}) pending-id (str (uuid/next)) - callback-url (dom/append-query-param (rt/get-current-href) - :pending-action-id pending-id) + ;; The callback travels as a query string; the token is kept in + ;; its own shape because `append-query-param` works on URLs. + callback-query-string (-> (rt/get-current-token) + (dom/append-query-param :pending-action-id pending-id) + rt/token->query-string) ;; The permissions may have changed in Nitrate since the modal ;; was opened: refresh them and explain why the action failed. on-error @@ -428,7 +431,8 @@ (rx/concat (when-not skip-audit? (rx/of audit-event)) - (->> (rp/cmd! :check-nitrate-sso {:organization-id organization-id :url callback-url}) + (->> (rp/cmd! :check-nitrate-sso {:organization-id organization-id + :query-string callback-query-string}) (rx/mapcat (fn [{:keys [authorized redirect-uri]}] (if authorized @@ -446,32 +450,35 @@ (defn check-organization-sso "Asks the backend whether the organization SSO gate can be satisfied for - `dest-url`, returning an observable of the raw `:check-nitrate-sso` - result: `:authorized` with a `:reason` of `:sso-satisfied` or - `:no-team-access`, or `:authorized false` with a `:redirect-uri` (nil - when SSO is required but the provider is unusable). Failures are not - caught, so a network blip stays a network error for the caller to - handle instead of masquerading as an answer." - [{:keys [team-id organization-id dest-url]}] + `query-string` (the route params of the client, `screen=…¶ms`, without + the separator), returning an observable of the raw `:check-nitrate-sso` + result: `:authorized` with a `:reason` of `:sso-satisfied`, + `:no-team-access` or `:no-organization-access`, or `:authorized false` with + a `:redirect-uri` (nil when SSO is required but the provider is unusable). + Failures are not caught, so a network blip stays a network error for the + caller to handle instead of masquerading as an answer." + [{:keys [team-id organization-id query-string]}] (rp/cmd! :check-nitrate-sso (d/without-nils {:team-id team-id :organization-id organization-id - :url dest-url}))) + :query-string query-string}))) (defn retry-organization-sso "Retries the organization SSO login flow after a failed attempt, reusing the same check-nitrate-sso RPC used elsewhere to move the user through the organization's identity provider. Passing `team-id` enables the backend's non-member short-circuit. Falls back to navigating straight - to `dest-url` when no fresh SSO redirect is needed or available." - [{:keys [dest-url] :as params}] - (ptk/reify ::retry-organization-sso - ptk/WatchEvent - (watch [_ _ _] - (->> (check-organization-sso params) - (rx/map (fn [{:keys [redirect-uri]}] - (rt/nav-raw :uri (or redirect-uri dest-url)))) - (rx/catch (fn [_] - (rx/of (rt/nav-raw :uri dest-url)))))))) + to the route of `query-string` when no fresh SSO redirect is needed or + available." + [{:keys [query-string] :as params}] + (let [dest-url (rt/token->url query-string)] + (ptk/reify ::retry-organization-sso + ptk/WatchEvent + (watch [_ _ _] + (->> (check-organization-sso params) + (rx/map (fn [{:keys [redirect-uri]}] + (rt/nav-raw :uri (or redirect-uri dest-url)))) + (rx/catch (fn [_] + (rx/of (rt/nav-raw :uri dest-url))))))))) (defn- fetch-organizations-allowed "Returns an rx observable of an `organizations-allowed` map (organization-id -> boolean). diff --git a/frontend/src/app/main/data/plugins.cljs b/frontend/src/app/main/data/plugins.cljs index e9b15e0849..4d3f8b5ef5 100644 --- a/frontend/src/app/main/data/plugins.cljs +++ b/frontend/src/app/main/data/plugins.cljs @@ -57,61 +57,76 @@ (update [_ state] (update-in state [:workspace-local :open-plugins] (fnil disj #{}) id)))) +(defn- global-blocked? + "True, after warning the user, when the plugin asks for global scope + or management permissions it is not allowed to have." + [plugin] + (when (and (preg/global-requested? plugin) + (not (preg/global-allowed? plugin))) + (log/warn :hint "Global plugins can only be loaded from localhost" + :plugin-id (:plugin-id plugin) + :host (:host plugin)) + (st/emit! (ntf/warn (tr "workspace.plugins.error.global-not-allowed"))) + true)) + (defn start-plugin! - [{:keys [plugin-id name version description host code permissions allow-background scope]} ^js extensions] - (let [load-plugin (unchecked-get ug/global "ɵloadPlugin")] - (if (fn? load-plugin) - (-> (load-plugin - #js {:pluginId plugin-id - :name name - :version version - :description description - :host host - :code code - :allowBackground (boolean allow-background) - :scope scope - :permissions (apply array permissions)} - nil - extensions) + [{:keys [plugin-id name version description host code permissions allow-background scope] :as plugin} ^js extensions] + (when-not (global-blocked? plugin) + (let [load-plugin (unchecked-get ug/global "ɵloadPlugin")] + (if (fn? load-plugin) + (-> (load-plugin + #js {:pluginId plugin-id + :name name + :version version + :description description + :host host + :code code + :allowBackground (boolean allow-background) + :scope scope + :permissions (apply array permissions)} + nil + extensions) - (p/catch (fn [cause] - (ex/print-throwable cause :prefix "Plugin Error") - (errors/flash :cause cause :type :handled)))) + (p/catch (fn [cause] + (ex/print-throwable cause :prefix "Plugin Error") + (errors/flash :cause cause :type :handled)))) - (log/warn :hint "Plugin runtime not initialized yet" - :plugin-id plugin-id - :action "start-plugin!")))) - -(defn- load-plugin! - [{:keys [plugin-id name version description host code icon permissions scope]}] - (st/emit! (pflag/initialize plugin-id version) - (save-current-plugin plugin-id)) - - (let [load-plugin (unchecked-get ug/global "ɵloadPlugin")] - (if (fn? load-plugin) - (-> (load-plugin - #js {:pluginId plugin-id - :name name - :description description - :version version - :host host - :code code - :icon icon - :scope scope - :permissions (apply array permissions)} - (fn [] - (st/emit! (remove-current-plugin plugin-id)))) - - (p/catch (fn [cause] - (st/emit! (remove-current-plugin plugin-id)) - (ex/print-throwable cause :prefix "Plugin Error") - (errors/flash :cause cause :type :handled)))) - - (do (log/warn :hint "Plugin runtime not initialized yet" :plugin-id plugin-id - :action "load-plugin!") - (st/emit! (remove-current-plugin plugin-id)))))) + :action "start-plugin!"))))) + +(defn- load-plugin! + [{:keys [plugin-id name version description host code icon permissions scope] :as plugin}] + (when-not (global-blocked? plugin) + (st/emit! (pflag/initialize plugin-id version) + (save-current-plugin plugin-id)) + + (let [load-plugin (unchecked-get ug/global "ɵloadPlugin")] + (if (fn? load-plugin) + (-> (load-plugin + #js {:pluginId plugin-id + :name name + :description description + :version version + :host host + :code code + :icon icon + :scope scope + :permissions (apply array permissions)} + (fn [] + (st/emit! (remove-current-plugin plugin-id)))) + + (p/catch (fn [cause] + (st/emit! (remove-current-plugin plugin-id)) + (ex/print-throwable cause :prefix "Plugin Error") + (errors/flash :cause cause :type :handled)))) + + (do + (log/warn :hint "Plugin runtime not initialized yet" + :plugin-id plugin-id + :action "load-plugin!") + (st/emit! (remove-current-plugin plugin-id) + (ntf/warn (tr "plugins.runtime-unavailable")))))))) (defn open-plugin! [{:keys [url] :as manifest} user-can-edit?] @@ -154,10 +169,12 @@ ;; Manifests match exactly - safe to load :else (load-plugin! manifest)))) - (fn [_err] + (fn [err] ;; Error fetching the manifest - can't verify integrity ;; Show error instead of loading potentially tampered code - (st/emit! (ntf/warn (tr "workspace.plugins.error.unreachable")))))) + (st/emit! (ntf/warn (if (= ::preg/global-not-allowed (:type (ex-data err))) + (tr "workspace.plugins.error.global-not-allowed") + (tr "workspace.plugins.error.unreachable"))))))) ;; Bundled plugins (no URL) - trusted, load directly (load-plugin! manifest))) diff --git a/frontend/src/app/main/data/team.cljs b/frontend/src/app/main/data/team.cljs index fe2b796ec8..6659fa8aae 100644 --- a/frontend/src/app/main/data/team.cljs +++ b/frontend/src/app/main/data/team.cljs @@ -13,6 +13,7 @@ [app.common.types.organization :as cto] [app.common.types.team :as ctt] [app.common.uri :as u] + [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.event :as ev] [app.main.data.helpers :as dsh] @@ -24,7 +25,9 @@ [app.main.repo :as rp] [app.main.router :as rt] [app.util.clipboard :as clipboard] + [app.util.dom :as dom] [app.util.i18n :refer [tr]] + [app.util.session-state :as ss] [app.util.storage :as storage] [beicon.v2.core :as rx] [clojure.string :as str] @@ -45,7 +48,7 @@ (if (or (not (contains? cf/flags :admin-console)) (= team-id default-team-id)) (rx/of team-id) - (->> (rp/cmd! :check-nitrate-sso {:team-id team-id :url (rt/get-current-href)}) + (->> (rp/cmd! :check-nitrate-sso {:team-id team-id :query-string (rt/get-current-query-string)}) (rx/map (fn [{:keys [authorized]}] (if authorized team-id default-team-id)))))) @@ -111,8 +114,8 @@ (f team))))))) (defn check-and-create-team - "Fetches fresh team data from the server to ensure up-to-date organization - permissions, then shows the team-form modal or a no-permission modal." + "Fetches fresh team data, checks organization permissions and SSO, then + shows the team form or the appropriate access error." [team-id] (ptk/reify ::check-and-create-team ptk/WatchEvent @@ -127,13 +130,44 @@ {:organization-perms organization :profile-id profile-id :team-perms (:permissions team)}) - true)] - (rx/of (if can-create? - (modal/show :team-form (if in-organization? - {:organization-id (:id organization) - :organization-name (:name organization)} - {})) - (modal/show :no-permission-modal {:type :create-team})))))))))) + true) + team-form-modal (modal/show :team-form (if in-organization? + {:organization-id (:id organization) + :organization-name (:name organization)} + {}))] + (cond + (not can-create?) + (rx/of (modal/show :no-permission-modal {:type :create-team})) + + (not in-organization?) + (rx/of team-form-modal) + + :else + (let [pending-id (str (uuid/next)) + callback-url (dom/append-query-param (rt/get-current-href) + :pending-action-id pending-id)] + (->> (rp/cmd! :check-nitrate-sso {:team-id team-id + :url callback-url}) + (rx/mapcat + (fn [{:keys [authorized redirect-uri]}] + (cond + authorized + (rx/of team-form-modal) + + redirect-uri + (do + (ss/save-pending-action! pending-id + {:type :create-team-in-organization + :team-id team-id}) + (rx/of (rt/nav-raw :uri (str redirect-uri)))) + + :else + (rx/of (rt/assign-exception + {:type :sso-error + :organization-id (:id organization) + :organization-name (:name organization) + :team-id team-id + :is-dashboard true}))))))))))))))) (defn check-and-delete-team "Fetches fresh team data from the server to ensure up-to-date organization diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 659a46be9b..2a6fa5a9cd 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -190,15 +190,20 @@ (rx/merge (->> (rx/from libraries) (rx/merge-map - (fn [{:keys [id synced-at is-indirect]}] + (fn [{:keys [id synced-at is-indirect] :as library}] (->> (rp/cmd! :get-file {:id id :features features}) (rx/map (fn [file] (-> file (assoc :synced-at synced-at) (assoc :library-of file-id) - (assoc :is-indirect is-indirect))))))) - (rx/mapcat resolve-file) - (rx/map library-resolved)) + (assoc :is-indirect is-indirect)))) + (rx/mapcat resolve-file) + (rx/map library-resolved) + ;; A broken library must not break the whole file + (rx/catch (fn [cause] + (if (dwl/broken-library-error? cause) + (rx/of (dwl/library-load-failed file-id library cause)) + (rx/throw cause)))))))) (->> (rx/from libraries) (rx/map :id) (rx/mapcat (fn [file-id] @@ -314,7 +319,8 @@ ::ev/name "open-workspace-file" ::ev/origin "workspace" :file-id file-id - :team-id team-id))))))) + :team-id team-id + :renderer (features/get-active-renderer state)))))))) (defn- restore-pending-changes "Applies the edits still waiting to be saved to a freshly loaded `file`. @@ -1716,6 +1722,7 @@ (dm/export dwt/update-positions) (dm/export dwt/flip-horizontal-selected) (dm/export dwt/flip-vertical-selected) +(dm/export dwt/scale-selected) (dm/export dwly/set-opacity) ;; Common diff --git a/frontend/src/app/main/data/workspace/clipboard.cljs b/frontend/src/app/main/data/workspace/clipboard.cljs index 877e535f75..4569501922 100644 --- a/frontend/src/app/main/data/workspace/clipboard.cljs +++ b/frontend/src/app/main/data/workspace/clipboard.cljs @@ -160,10 +160,8 @@ (let [page-id (:current-page-id state) data (dsh/lookup-file-data state) objects (-> (dsh/get-page data page-id) - (get :objects)) - - components (cfv/find-variant-components data objects (:id shape))] - (into {} (map (juxt :id :variant-properties) components)))) + (get :objects))] + (cfv/collect-variant-properties data objects shape))) ;; Collects all the items together and split images into a @@ -179,7 +177,8 @@ (some? images) (update :images into images) - (ctk/is-variant-container? item) + (or (ctk/is-variant-container? item) + (ctk/is-variant-main? item)) (update :variant-properties merge (collect-variants state item)))) (maybe-translate [shape objects parent-frame-id] diff --git a/frontend/src/app/main/data/workspace/colors.cljs b/frontend/src/app/main/data/workspace/colors.cljs index 9594da29e8..6bd0ca3961 100644 --- a/frontend/src/app/main/data/workspace/colors.cljs +++ b/frontend/src/app/main/data/workspace/colors.cljs @@ -830,13 +830,19 @@ (assoc :opacity alpha) (d/without-nils))) +(defn merge-color-components + [color changes] + (cond-> (merge color changes) + (and (:hex changes) (not (:hsl changes))) + (dissoc :hsl))) + (defn clear-color-components [data] - (dissoc data :hex :alpha :r :g :b :h :s :v :image)) + (dissoc data :hex :alpha :r :g :b :h :s :v :hsl :image)) (defn clear-image-components [data] - (dissoc data :hex :alpha :r :g :b :h :s :v :color)) + (dissoc data :hex :alpha :r :g :b :h :s :v :hsl :color)) (defn- create-gradient [type] @@ -928,7 +934,9 @@ new-gradient (-> gradient (dissoc :stops) (assoc :shape-id shape-id))] - (if (and (= (:stops state) new-stops) (= (:gradient state) new-gradient)) + (if (and (= (mapv #(dissoc % :hsl) (:stops state)) + new-stops) + (= (:gradient state) new-gradient)) state (-> state (assoc :opacity (:opacity data)) @@ -1100,7 +1108,7 @@ (fn [state] (let [type (:type state) state (-> state - (update :current-color merge changes) + (update :current-color merge-color-components changes) (update :current-color materialize-color-components) (update :current-color #(if (not= type :image) (dissoc % :image) %)) ;; current color can be a library one @@ -1108,9 +1116,8 @@ (update :current-color dissoc :ref-id :ref-file))] (if-let [stop (:editing-stop state)] - (update-in state [:stops stop] (fn [data] (->> changes - (merge data) - (materialize-color-components)))) + (update-in state [:stops stop] (fn [data] (-> (merge-color-components data changes) + (materialize-color-components)))) (-> state (dissoc :gradient :stops :editing-stop) diff --git a/frontend/src/app/main/data/workspace/libraries.cljs b/frontend/src/app/main/data/workspace/libraries.cljs index 97ff3bd1a3..30421af407 100644 --- a/frontend/src/app/main/data/workspace/libraries.cljs +++ b/frontend/src/app/main/data/workspace/libraries.cljs @@ -423,8 +423,7 @@ ;; When activated the wasm rendering we need to recreate its thumbnail on creation (if (features/active-feature? state "render-wasm/v1") - (rx/of (dwt.wasm/render-thumbnail file-id page-id (:id root)) - (dwt.wasm/persist-thumbnail file-id page-id (:id root))) + (rx/of (dwt.wasm/render-thumbnail file-id page-id (:id root) :persist? true)) (rx/empty))))))))))) (defn add-component @@ -659,7 +658,8 @@ (merge (meta it)))) (dwu/start-undo-transaction undo-id) (dch/commit-changes changes) - (dwtp/propagate-workspace-tokens) ;; Make the new instance get the token values from the current file, not from the component's library + ;; Make the new instance get the token values from the current file, not from the component's library + (dwtp/propagate-workspace-tokens {:page-id (:id page) :root-ids [(:id new-shape)]}) (ptk/data-event :layout/update {:ids [(:id new-shape)]}) (dws/select-shapes (d/ordered-set (:id new-shape))) (when start-move? @@ -970,8 +970,8 @@ page-id (:main-instance-page component) root-id (:main-instance-id component)] (if (features/active-feature? state "render-wasm/v1") - ;; WASM: render immediately, UI only — server persist happens - ;; on the debounced path (update-component-thumbnail). Frame + ;; WASM: ask the worker to render it for the UI, without uploading; + ;; the debounced path (update-component-thumbnail) uploads it. Frame ;; thumbnails are only shown by the SVG viewport. (when (= tag "component") (dwt.wasm/render-thumbnail file-id page-id root-id)) @@ -1172,7 +1172,7 @@ (dch/commit-changes changes) ;; Make the new instance get the token values from the current file, ;; not from the component's library - (dwtp/propagate-workspace-tokens) + (dwtp/propagate-workspace-tokens {:page-id (:id page) :root-ids [(:id new-shape)]}) (when (and (features/active-feature? state "render-wasm/v1") (seq new-text-ids)) (dwwt/resize-wasm-text-all new-text-ids {:skip-component-sync? true}))) @@ -1700,14 +1700,18 @@ component-events-s ;; WASM only: render the thumbnail on every component - ;; change so single edits (fill, etc.) update instantly. + ;; change so single edits (fill, etc.) update without waiting + ;; for the 5 s idle upload. ;; Non-WASM persists on every render, so it stays on the ;; debounced path below to avoid per-edit backend posts. (if (features/active-feature? state "render-wasm/v1") - (->> component-events-s - (rx/filter (ptk/type? ::component-changed)) - (rx/map deref) - (rx/map render-component-thumbnail-event)) + (rx/merge + (->> component-events-s + (rx/filter (ptk/type? ::component-changed)) + (rx/map deref) + (rx/map render-component-thumbnail-event)) + ;; Shows the thumbnails the worker renders + (dwt.wasm/component-thumbnail-events)) (rx/empty)) ;; Persist to the server in batches, 5s after the user @@ -1849,6 +1853,7 @@ (let [libraries (:shared-files state) library (get libraries library-id) variants-count (-> library :library-summary :variants count) + tokens-count (or (-> library :library-summary :tokens-count) 0) loaded-libraries (->> (dsh/lookup-libraries state) (remove (fn [[_ lib]] @@ -1869,14 +1874,25 @@ (rx/of (ntf/error (tr "errors.circular-library-reference"))) (rx/throw cause)))))) (rx/of (ptk/reify ::link-file-to-library-finished)) - (when (pos? variants-count) + (when (or (pos? variants-count) (pos? tokens-count)) (->> (rp/cmd! :get-library-usage {:file-id library-id}) - (rx/map (fn [library-usage] - (ev/event {::ev/name "link-file-to-library-variants" - :file-id file-id - :library-id library-id - :variants-count variants-count - :library-used-in (:used-in library-usage)})))))))))) + (rx/mapcat (fn [library-usage] + (let [used-in (:used-in library-usage)] + (rx/from + (cond-> [] + (pos? variants-count) + (conj (ev/event {::ev/name "link-file-to-library-variants" + :file-id file-id + :library-id library-id + :variants-count variants-count + :library-used-in used-in})) + + (pos? tokens-count) + (conj (ev/event {::ev/name "link-file-to-library-tokens" + :file-id file-id + :library-id library-id + :tokens-count tokens-count + :library-used-in used-in})))))))))))))) (defn cleanup-unlinked-libraries "Remove libraries from state that are no longer linked to the given file. @@ -1920,3 +1936,31 @@ (rx/mapcat (fn [_] (rp/cmd! :get-file-libraries {:file-id file-id}))) (rx/map (partial cleanup-unlinked-libraries file-id)))))) + +(defn broken-library-error? + "True when `cause` comes from the backend failing to load a library + file. Transport and transient errors (network, gateway, offline, + rate limit) do not count, so they never disconnect a library." + [cause] + (= :server-error (:type (ex-data cause)))) + +(defn library-load-failed + "Handles a library of `file-id` that fails to load, so a single + broken library does not break the whole file: a directly connected + library is disconnected and the user is told; an indirect one is only + dropped from the state." + [file-id {:keys [id name is-indirect]} cause] + (ptk/reify ::library-load-failed + ptk/WatchEvent + (watch [_ _ _] + (log/error :hint "unable to load library" + :file-id (str file-id) + :library-id (str id) + :cause cause) + (if is-indirect + (rx/of (ptk/reify ::drop-broken-library + ptk/UpdateEvent + (update [_ state] + (update state :files dissoc id)))) + (rx/of (unlink-file-from-library file-id id) + (ntf/warn (tr "workspace.libraries.broken-library-disconnected" name))))))) diff --git a/frontend/src/app/main/data/workspace/media.cljs b/frontend/src/app/main/data/workspace/media.cljs index 71ac10d86a..6c202e321f 100644 --- a/frontend/src/app/main/data/workspace/media.cljs +++ b/frontend/src/app/main/data/workspace/media.cljs @@ -14,6 +14,7 @@ [app.common.files.shapes-builder :as sb] [app.common.math :as mth] [app.common.media :as media] + [app.common.path-names :as cpn] [app.common.schema :as sm] [app.common.types.container :as ctn] [app.common.types.fills :as types.fills] @@ -435,10 +436,12 @@ fdata (dsh/lookup-file-data state file-id) page (dsh/get-page fdata page-id) - pos (dsh/get-viewport-center state)] + pos (dsh/get-viewport-center state) + + name (cpn/remove-extension (:name svg-data))] (->> (create-shapes-svg file-id (:objects page) pos svg-data) - (rx/map (partial add-shapes-and-component it fdata page (:name svg-data)))))))) + (rx/map (partial add-shapes-and-component it fdata page name))))))) (defn upload-media-components [params] diff --git a/frontend/src/app/main/data/workspace/path/shortcuts.cljs b/frontend/src/app/main/data/workspace/path/shortcuts.cljs index fbe1483de6..e4f149f607 100644 --- a/frontend/src/app/main/data/workspace/path/shortcuts.cljs +++ b/frontend/src/app/main/data/workspace/path/shortcuts.cljs @@ -269,17 +269,17 @@ :section [:workspace] :fn #(st/emit! (drp/move-selected :down false))} - :move-unit-left {:tooltip ds/right-arrow + :move-unit-left {:tooltip ds/left-arrow :label (fn [] (tr "shortcuts.move-unit-left")) - :command "right" - :section [:workspace] - :fn #(st/emit! (drp/move-selected :right false))} - - :move-unit-right {:tooltip ds/left-arrow - :label (fn [] (tr "shortcuts.move-unit-right")) :command "left" :section [:workspace] - :fn #(st/emit! (drp/move-selected :left false))}}) + :fn #(st/emit! (drp/move-selected :left false))} + + :move-unit-right {:tooltip ds/right-arrow + :label (fn [] (tr "shortcuts.move-unit-right")) + :command "right" + :section [:workspace] + :fn #(st/emit! (drp/move-selected :right false))}}) (defn get-tooltip [shortcut] (assert (contains? shortcuts shortcut) (str shortcut)) diff --git a/frontend/src/app/main/data/workspace/shape_layout.cljs b/frontend/src/app/main/data/workspace/shape_layout.cljs index 68a34dad95..0a5c7b4cfe 100644 --- a/frontend/src/app/main/data/workspace/shape_layout.cljs +++ b/frontend/src/app/main/data/workspace/shape_layout.cljs @@ -436,6 +436,14 @@ (ptk/data-event :layout/update {:ids ids}) (dwu/commit-undo-transaction undo-id)))))) +(defn grid-in-copy? + "Check if the grid is a component copy or lives inside one. Copies + must keep the structure of their main, so tracks that add shapes are + not allowed there." + [objects grid] + (or (ctc/in-component-copy? grid) + (ctn/has-any-copy-parent? objects grid))) + (defn duplicate-layout-track [ids type index] (assert (#{:row :column} type)) @@ -443,53 +451,53 @@ (ptk/reify ::duplicate-layout-track ptk/WatchEvent (watch [it state _] - (let [file-id (:current-file-id state) - page (dsh/lookup-page state) - objects (:objects page) - libraries (dsh/lookup-libraries state) - library-data (dsh/lookup-file state file-id) - shape-id (first ids) - base-shape (get objects shape-id) + (let [objects (dsh/lookup-page-objects state) + base-shape (get objects (first ids))] + (when-not (grid-in-copy? objects base-shape) + (let [file-id (:current-file-id state) + page (dsh/lookup-page state) + libraries (dsh/lookup-libraries state) + library-data (dsh/lookup-file state file-id) - shapes-by-track - (if (= type :column) - (ctl/shapes-by-column base-shape index false) - (ctl/shapes-by-row base-shape index false)) + shapes-by-track + (if (= type :column) + (ctl/shapes-by-column base-shape index false) + (ctl/shapes-by-row base-shape index false)) - ;; Change to set in order to use auxiliary functions - selected (set shapes-by-track) + ;; Change to set in order to use auxiliary functions + selected (set shapes-by-track) - changes - (-> (pcb/empty-changes it) - (cll/generate-duplicate-changes objects page selected (gpt/point 0 0) libraries library-data file-id) - (cll/generate-duplicate-changes-update-indices objects selected)) + changes + (-> (pcb/empty-changes it) + (cll/generate-duplicate-changes objects page selected (gpt/point 0 0) libraries library-data file-id) + (cll/generate-duplicate-changes-update-indices objects selected)) - ;; Creates a map with shape-id => duplicated-shape-id - ids-map - (->> changes - :redo-changes - (filter #(= (:type %) :add-obj)) - (filter #(selected (:old-id %))) - (map #(vector (:old-id %) (get-in % [:obj :id]))) - (into {})) + ;; Creates a map with shape-id => duplicated-shape-id + ids-map + (->> changes + :redo-changes + (filter #(= (:type %) :add-obj)) + (filter #(selected (:old-id %))) + (map #(vector (:old-id %) (get-in % [:obj :id]))) + (into {})) - changes - (-> changes - (pcb/update-shapes - ids - (fn [shape objects] - ;; The duplication could have altered the grid so we restore the values, we'll calculate the good ones now - (let [shape (merge shape (select-keys base-shape [:layout-grid-cells :layout-grid-columns :layout-grid-rows]))] - (case type - :row (ctl/duplicate-row shape objects index ids-map) - :column (ctl/duplicate-column shape objects index ids-map)))) - {:with-objects? true})) + changes + (-> changes + (pcb/update-shapes + ids + (fn [shape objects] + ;; The duplication could have altered the grid so we restore the values, we'll calculate the good ones now + (let [shape (merge shape (select-keys base-shape [:layout-grid-cells :layout-grid-columns :layout-grid-rows]))] + (case type + :row (ctl/duplicate-row shape objects index ids-map) + :column (ctl/duplicate-column shape objects index ids-map)))) + {:with-objects? true})) - undo-id (js/Symbol)] - (rx/of (dwu/start-undo-transaction undo-id) - (dch/commit-changes changes) - (ptk/data-event :layout/update {:ids ids}) - (dwu/commit-undo-transaction undo-id)))))) + undo-id (js/Symbol)] + (rx/of (dwu/start-undo-transaction undo-id) + (dch/commit-changes changes) + (ptk/data-event :layout/update {:ids ids}) + (dwu/commit-undo-transaction undo-id)))))))) (defn reorder-layout-track [ids type from-index to-index move-content?] @@ -918,7 +926,9 @@ type (:type copied) undo-id (js/Symbol)] - (when (and (seq track-indices) (some? type)) + (when (and (seq track-indices) + (some? type) + (not (grid-in-copy? objects grid))) (let [shapes-by-track-fn (if (= type :row) ctl/shapes-by-row diff --git a/frontend/src/app/main/data/workspace/shortcuts.cljs b/frontend/src/app/main/data/workspace/shortcuts.cljs index b322ae01a0..e18ec9c6dd 100644 --- a/frontend/src/app/main/data/workspace/shortcuts.cljs +++ b/frontend/src/app/main/data/workspace/shortcuts.cljs @@ -59,6 +59,14 @@ (.preventDefault event) (st/emit! (toggle-layout-flag :display-guides))))) +(defn- escape + "Leave the scale content mode when it is active, keeping the + selection; otherwise clear the current drawing and selection." + [] + (if (contains? (deref refs/workspace-layout) :scale-text) + (st/emit! :interrupt (dw/remove-layout-flag :scale-text)) + (st/emit! :interrupt (dwdc/clear-drawing) (dw/deselect-all true)))) + (defn- emit-when-no-readonly [& events] (let [can-edit? (:can-edit (deref refs/permissions)) @@ -182,7 +190,7 @@ :subsections [:edit] :section [:workspace] :customizable false - :fn #(st/emit! :interrupt (dwdc/clear-drawing) (dw/deselect-all true))} + :fn escape} :find {:tooltip (ds/meta "F") :command (ds/c-mod "f") :subsections [:edit] :label (fn [] (tr "shortcuts.find")) @@ -327,19 +335,19 @@ :subsections [:modify-layers] :fn #(emit-when-no-readonly (dwt/move-selected :down false))} - :move-unit-left {:tooltip ds/right-arrow + :move-unit-left {:tooltip ds/left-arrow :label (fn [] (tr "shortcuts.move-unit-left")) - :command ["right" "alt+right"] - :subsections [:modify-layers] - :section [:workspace] - :fn #(emit-when-no-readonly (dwt/move-selected :right false))} - - :move-unit-right {:tooltip ds/left-arrow - :label (fn [] (tr "shortcuts.move-unit-right")) :command ["left" "alt+left"] + :subsections [:modify-layers] + :section [:workspace] + :fn #(emit-when-no-readonly (dwt/move-selected :left false))} + + :move-unit-right {:tooltip ds/right-arrow + :label (fn [] (tr "shortcuts.move-unit-right")) + :command ["right" "alt+right"] :section [:workspace] :subsections [:modify-layers] - :fn #(emit-when-no-readonly (dwt/move-selected :left false))} + :fn #(emit-when-no-readonly (dwt/move-selected :right false))} :artboard-selection {:tooltip (ds/meta (ds/alt "G")) :label (fn [] (tr "shortcuts.artboard-selection")) diff --git a/frontend/src/app/main/data/workspace/texts.cljs b/frontend/src/app/main/data/workspace/texts.cljs index 0464cee98b..ace273017c 100644 --- a/frontend/src/app/main/data/workspace/texts.cljs +++ b/frontend/src/app/main/data/workspace/texts.cljs @@ -437,7 +437,9 @@ (-> (merge default-text-attrs node) (assoc :fills fills))) - node))))] + (cond->> node + (txt/is-paragraph-node? node) + (merge txt/default-list-attrs))))))] (attrs/get-attrs-multi nodes attrs))) (defn current-root-values diff --git a/frontend/src/app/main/data/workspace/thumbnails_wasm.cljs b/frontend/src/app/main/data/workspace/thumbnails_wasm.cljs index b5b78b92c5..0a880c0ec7 100644 --- a/frontend/src/app/main/data/workspace/thumbnails_wasm.cljs +++ b/frontend/src/app/main/data/workspace/thumbnails_wasm.cljs @@ -5,190 +5,42 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.main.data.workspace.thumbnails-wasm - "WASM-based component thumbnail rendering. - Renders component previews using the existing workspace WASM context - via render-shape-pixels (SurfaceId::Export), avoiding a separate - WASM module in the worker. - - Two-phase design: - - render-thumbnail: immediate WASM render → UI update (no server) - - persist-thumbnail: pushes current data-uri to the server (debounced)" + "Component thumbnails with render-wasm/v1. The worker renders them + from the shapes of their mains, which the main thread sends, and + uploads the ones asked for." (:require - [app.common.data.macros :as dm] [app.common.files.helpers :as cfh] - [app.common.logging :as l] - [app.common.math :as mth] - [app.common.thumbnails :as thc] - [app.common.time :as ct] [app.main.data.helpers :as dsh] [app.main.data.workspace.thumbnails :as dwt] - [app.main.repo :as rp] - [app.main.store :as st] - [app.render-wasm.api :as wasm.api] - [app.util.timers :as timers] + [app.main.worker :as mw] [app.util.webapi :as wapi] [beicon.v2.core :as rx] - [cuerdas.core :as str] [potok.v2.core :as ptk])) -(l/set-level! :warn) - -(defn- png-bytes->data-uri - "Converts a Uint8Array of PNG bytes to a data:image/png;base64 URI." - [png-bytes] - (let [blob (wapi/create-blob png-bytes "image/png") - reader (js/FileReader.)] - (js/Promise. - (fn [resolve reject] - (set! (.-onload reader) - (fn [] (resolve (.-result reader)))) - (set! (.-onerror reader) - (fn [e] (reject e))) - (.readAsDataURL reader blob))))) - -;; This constant stores the target thumbnail minimum max-size so -;; the images doesn't lose quality when rendered -(def ^:private ^:const target-size 200) - -(defn- render-component-pixels - "Renders a component frame using the workspace WASM context. - Returns an observable that emits a data-uri string. - Deferred by one animation frame so that process-shape-changes! - has time to sync all child shapes to WASM memory first." - [file-id page-id frame-id] - (rx/create - (fn [subs] - (let [req-id - (timers/raf - (fn [_] - (try - (let [objects (dsh/lookup-page-objects @st/state file-id page-id)] - (if-let [frame (get objects frame-id)] - (let [{ext-w :width ext-h :height} (wasm.api/get-shape-extrect frame-id) - {sel-w :width sel-h :height} (:selrect frame) - max-size (mth/max (or ext-w sel-w) (or ext-h sel-h)) - scale (mth/max 1 (/ target-size max-size)) - ;; Thumbnails are always PNG: they need the alpha - ;; channel and are consumed as a `image/png` data uri. - png-bytes (wasm.api/render-shape-pixels frame-id scale :png)] - (if (or (nil? png-bytes) (zero? (.-length png-bytes))) - (do - (l/error :hint "render-shape-pixels returned empty" :frame-id (str frame-id)) - (rx/end! subs)) - (.then - (png-bytes->data-uri png-bytes) - (fn [data-uri] - (rx/push! subs data-uri) - (rx/end! subs)) - (fn [err] - (rx/error! subs err))))) - - ;; A missing frame. Nothing to render, so end quietly. - (rx/end! subs))) - (catch :default err - (rx/error! subs err)))))] - #(timers/cancel-af! req-id))))) - (defn render-thumbnail - "Renders a component thumbnail via WASM and updates the UI immediately. - When `persist?` is true, also persists the rendered thumbnail to the - server in the same observable chain (guaranteeing correct ordering)." - [file-id page-id frame-id & {:keys [persist?] :or {persist? false}}] + "Sends the worker the shapes of the main whose root is `root-id`, to + render its component thumbnail, and to upload it when `persist?` is + true. Nothing comes back: `component-thumbnail-events` shows the + render." + [file-id page-id root-id & {:keys [persist?]}] + (ptk/reify ::render-thumbnail + ptk/EffectEvent + (effect [_ state _] + (let [objects (dsh/lookup-page-objects state file-id page-id)] + ;; A removed component can bring a nil or deleted root + (when (contains? objects root-id) + (let [shapes (cfh/get-children-with-self objects root-id)] + (mw/emit! {:cmd :thumbnails/render-component + :file-id file-id + :page-id page-id + :root-id root-id + :persist? persist? + :objects (into {} (map (juxt :id identity)) shapes)}))))))) - (let [object-id (thc/fmt-object-id file-id page-id frame-id "component")] - (ptk/reify ::render-thumbnail - cljs.core/IDeref - (-deref [_] object-id) - - ptk/WatchEvent - (watch [_ state stream] - ;; When the component is removed it can arrived a render - ;; request with frame-id=null - (when (some? frame-id) - (letfn [(load-objects-stream - [] - (rx/create - (fn [subs] - (let [objects (dsh/lookup-page-objects state file-id page-id) - - ;; retrieves a subtree with only the id and its children - ;; to be loaded before rendering the thumbnail - subtree - (into {} - (map #(vector (:id %) %)) - (cfh/get-children-with-self objects frame-id))] - (try - (wasm.api/set-objects subtree #(rx/push! subs %)) - (catch :default err - (rx/error! subs err))))))) - - (get-subtree - [state] - (-> (dsh/lookup-page-objects state file-id page-id) - (cfh/get-children-with-self frame-id))) - - (persist-to-server - [data-uri] - ;; A change during the render (a rollback, an edit) makes - ;; it stale; that change brings its own render - (if (= (get-subtree state) (get-subtree @st/state)) - (let [blob (wapi/data-uri->blob data-uri)] - (->> (rp/cmd! :create-file-object-thumbnail - {:file-id file-id - :object-id object-id - :media blob - :tag "component"}) - (rx/catch rx/empty) - (rx/ignore))) - (rx/empty))) - - (do-render-thumbnail - [] - (let [tp (ct/tpoint-ms)] - (->> (render-component-pixels file-id page-id frame-id) - (rx/mapcat - (fn [data-uri] - (l/dbg :hint "component thumbnail rendered (wasm)" - :elapsed (dm/str (tp) "ms")) - (if persist? - (rx/merge - (rx/of (dwt/assoc-thumbnail object-id data-uri)) - (persist-to-server data-uri)) - (rx/of (dwt/assoc-thumbnail object-id data-uri))))) - - (rx/catch (fn [err] - (js/console.error err) - (l/error :hint "error rendering component thumbnail" :frame-id (str frame-id)) - (rx/empty))) - - (rx/take-until - (->> stream - (rx/filter (ptk/type? ::dwt/clear-thumbnail)) - (rx/filter #(= (deref %) object-id)))))))] - - (if (not= page-id (:current-page-id state)) - (->> (load-objects-stream) - (rx/mapcat do-render-thumbnail)) - (do-render-thumbnail)))))))) - -(defn persist-thumbnail - "Persists the current component thumbnail data-uri to the server. - Expects that `render-thumbnail` has already been called so the - data-uri is present in app state. If not, this is a no-op." - [file-id page-id frame-id] - (let [object-id (thc/fmt-object-id file-id page-id frame-id "component")] - (ptk/reify ::persist-thumbnail - ptk/WatchEvent - (watch [_ state _] - (let [data-uri (dm/get-in state [:thumbnails object-id :uri])] - (if (and (some? data-uri) - (str/starts-with? data-uri "data:")) - (let [blob (wapi/data-uri->blob data-uri)] - (->> (rp/cmd! :create-file-object-thumbnail - {:file-id file-id - :object-id object-id - :media blob - :tag "component"}) - (rx/catch rx/empty) - (rx/ignore))) - (rx/empty))))))) +(defn component-thumbnail-events + "Shows each component thumbnail the worker renders for + `render-thumbnail`." + [] + (->> (mw/ask-many! {:cmd :thumbnails/watch-components}) + (rx/map (fn [{:keys [object-id blob]}] + (dwt/assoc-thumbnail object-id (wapi/create-uri blob)))))) diff --git a/frontend/src/app/main/data/workspace/tokens/application.cljs b/frontend/src/app/main/data/workspace/tokens/application.cljs index 3ab4427ce6..cc76192a18 100644 --- a/frontend/src/app/main/data/workspace/tokens/application.cljs +++ b/frontend/src/app/main/data/workspace/tokens/application.cljs @@ -733,6 +733,8 @@ (all-attrs-appliable-for-token? attributes (:type token))))))) shape-ids (d/nilv (keys shapes) []) any-variant? (->> shapes vals (some ctk/is-variant?) boolean) + file-data (dsh/lookup-file-data state) + from-library (not (cfo/effective-tokens-source? file-data (:id file-data))) resolved-value (get-in resolved-tokens [(:name token) :resolved-value]) tokenized-attributes (cfo/attributes-map attributes token) @@ -743,7 +745,9 @@ (-> {::ev/name "apply-tokens" :type type :applied-to attributes - :applied-to-variant any-variant?} + :applied-to-variant any-variant? + :shape-count (count shape-ids) + :from-library from-library} (merge (meta it))))) (dwu/start-undo-transaction undo-id) (dwsh/update-shapes shape-ids (fn [shape] @@ -771,7 +775,7 @@ Splits out `shape-ids` into separate default actions: - Layouts take the `default` update function - Shapes inside layout will only take margin" - [{:keys [token shapes attr]}] + [{:keys [token shapes]}] (ptk/reify ::apply-spacing-token-separated ptk/WatchEvent (watch [it state _] @@ -784,7 +788,7 @@ (group-by #(if (ctsl/any-layout-immediate-child? objects %) :frame-children :other) shapes)] (rx/of - (-> (apply-token {:attributes (or attr attributes) + (-> (apply-token {:attributes attributes :token token :shape-ids (map :id other) :on-update-shape on-update-shape}) @@ -825,113 +829,87 @@ (update shape :applied-tokens #(when % (apply dissoc % attributes))))))))) +(defn- target-shapes + "The selected shapes a token goes on. A selected group can stand for + its children, since groups take no tokens." + [objects shape-ids expand-with-children] + (cond->> (into [] (keep (d/getf objects)) shape-ids) + expand-with-children + (into [] (mapcat (fn [shape] + (if (= (:type shape) :group) + (keep objects (:shapes shape)) + [shape])))))) + +(defn- apply-action + "Works out which attributes the token goes on: the chosen ones, or its + type's defaults. A spacing token without chosen attributes goes on + layouts, and only on the margins of the shapes inside them." + [token attrs] + (let [{:keys [attributes on-update-shape]} (get token-properties (:type token))] + (if (and (= (:type token) :spacing) (nil? attrs)) + {:action :apply-spacing} + {:action :apply + :attributes (if (empty? attrs) attributes attrs) + :on-update-shape (or (get attr->shape-update (first attrs)) on-update-shape)}))) + +(defn- toggle-action + "Works out whether toggling the token removes it from the selection + or applies it, and on which attributes. From the token pill it comes + off when any shape has it; on chosen attributes (a design tab input, + a plugin) only when every shape has it on all of them, so a partial + application is completed instead." + [token attrs shapes] + (let [{:keys [attributes all-attributes]} (get token-properties (:type token)) + target-attrs (or attrs all-attributes attributes) + unapply? (if (seq attrs) + (cfo/shapes-applied-all? + (cfo/shapes-ids-by-applied-attributes token shapes target-attrs) + (into #{} (map :id) shapes) + target-attrs) + (cfo/shapes-token-applied? token shapes target-attrs))] + (if unapply? + {:action :unapply :attributes target-attrs} + (apply-action token attrs)))) + +(defn- action->event + [{:keys [action attributes on-update-shape]} token shapes shape-ids] + (case action + :unapply + (unapply-token {:attributes attributes + :token-name (:name token) + :shape-ids shape-ids}) + + :apply-spacing + (apply-spacing-token-separated {:token token + :shapes shapes}) + + :apply + (apply-token {:attributes attributes + :token token + :shape-ids shape-ids + :on-update-shape on-update-shape}))) + (defn toggle-token [{:keys [token attrs shape-ids expand-with-children]}] (ptk/reify ::on-toggle-token ptk/WatchEvent (watch [it state _] - (let [objects (dsh/lookup-page-objects state) - shapes (into [] (keep (d/getf objects)) shape-ids) - - shapes - (if expand-with-children - (into [] - (mapcat (fn [shape] - (if (= (:type shape) :group) - (keep objects (:shapes shape)) - [shape]))) - shapes) - shapes) - - {:keys [attributes all-attributes on-update-shape]} - (get token-properties (:type token)) - - on-update-shape - (if (seq attrs) - (or (get attr->shape-update (first attrs)) on-update-shape) - on-update-shape) - - target-attrs - (or attrs all-attributes attributes) - - unapply-tokens? - (if (seq attrs) - ;; Explicit attributes come from an input or a plugin apply - ;; call. Only toggle off when the token already covers every - ;; target attribute on every selected shape; a partial - ;; application is completed instead of removed. - (and (seq target-attrs) - (cfo/shapes-applied-all? - (cfo/shapes-ids-by-applied-attributes token shapes target-attrs) - (into #{} (map :id) shapes) - target-attrs)) - ;; No explicit attributes (token pill): toggle off when any - ;; selected shape has the token on any attribute. - (cfo/shapes-token-applied? token shapes target-attrs)) - - shape-ids - (map :id shapes)] - - (if unapply-tokens? - (rx/of - (unapply-token {:attributes target-attrs - :token-name (:name token) - :shape-ids shape-ids})) - (rx/of - (cond - (and (= (:type token) :spacing) - (nil? attrs)) - (-> (apply-spacing-token-separated {:token token - :attr attrs - :shapes shapes}) - (with-meta (meta it))) - - :else - (-> (apply-token {:attributes (if (empty? attrs) attributes attrs) - :token token - :shape-ids shape-ids - :on-update-shape on-update-shape}) - (with-meta (meta it)))))))))) + (let [shapes (target-shapes (dsh/lookup-page-objects state) shape-ids expand-with-children) + action (toggle-action token attrs shapes) + event (action->event action token shapes (map :id shapes))] + (rx/of + (if (= :unapply (:action action)) + event + (with-meta event (meta it)))))))) (defn apply-token-from-input [{:keys [token attrs shape-ids expand-with-children]}] (ptk/reify ::apply-token-from-input ptk/WatchEvent (watch [_ state _] - (let [objects (dsh/lookup-page-objects state) - shapes (into [] (keep (d/getf objects)) shape-ids) - - shapes - (if expand-with-children - (into [] - (mapcat (fn [shape] - (if (= (:type shape) :group) - (keep objects (:shapes shape)) - [shape]))) - shapes) - shapes) - - {:keys [attributes _ on-update-shape]} - (get token-properties (:type token)) - - on-update-shape - (if (seq attrs) - (or (get attr->shape-update (first attrs)) on-update-shape) - on-update-shape)] - + (let [shapes (target-shapes (dsh/lookup-page-objects state) shape-ids expand-with-children)] (rx/of - (cond - (and (= (:type token) :spacing) - (nil? attrs)) - (apply-spacing-token-separated {:token token - :attr attrs - :shapes shapes}) - - :else - (apply-token {:attributes (if (empty? attrs) attributes attrs) - :token token - :shape-ids shape-ids - :on-update-shape on-update-shape}))))))) + (action->event (apply-action token attrs) token shapes shape-ids)))))) (defn apply-token-on-color-selected diff --git a/frontend/src/app/main/data/workspace/tokens/import_export.cljs b/frontend/src/app/main/data/workspace/tokens/import_export.cljs index 05b221f1c7..6b8e64e430 100644 --- a/frontend/src/app/main/data/workspace/tokens/import_export.cljs +++ b/frontend/src/app/main/data/workspace/tokens/import_export.cljs @@ -50,25 +50,44 @@ (reduce (fn [acc [k v]] (update acc v conj k)) {} m)) -(defn- show-unknown-types-warning [unknown-tokens] - (let [type->tokens (group-by-value unknown-tokens)] - (l/wrn :hint "unsupported token types found during import" - :tokens (str/join ", " (map (fn [[path type]] (str path " (" type ")")) unknown-tokens))) - (ntf/show {:content (tr "workspace.tokens.unknown-token-type-message") - :is-html true - :detail (->> (for [[token-type token-paths] type->tokens] - (str (tr "workspace.tokens.unknown-token-type-section" - (dom/escape-html token-type) - (tr "labels.warning-count" (i18n/c (count token-paths)))) - "
        " - (->> token-paths - (sort) - (map #(str "
      • " (dom/escape-html %) "
      • ")) - (str/join "")) - "
      ")) - (str/join "")) - :type :toast - :level :info}))) +(defn- skipped-detail + "HTML lists of the skipped token paths, one section per label, with + `section-fn` called with the label and the count text." + [section-fn label->paths] + (->> (for [[label paths] (sort-by key label->paths)] + (str (section-fn (dom/escape-html label) + (tr "labels.warning-count" (i18n/c (count paths)))) + "
        " + (->> paths + (sort) + (map #(str "
      • " (dom/escape-html %) "
      • ")) + (str/join "")) + "
      ")) + (str/join ""))) + +(defn- import-warnings-notification + "One toast for what the import skipped: tokens of types penpot does not + have, and token data it does not store (`$extensions`, `$deprecated`, + group descriptions, values it cannot read). Its details list both, + grouped by type and by property." + [unknown-types unsupported] + {:content (if (seq unsupported) + (tr "workspace.tokens.unsupported-token-data-message") + (tr "workspace.tokens.unknown-token-type-message")) + :is-html true + :detail (str (skipped-detail #(tr "workspace.tokens.unknown-token-type-section" %1 %2) + (group-by-value unknown-types)) + (skipped-detail #(tr "workspace.tokens.unsupported-token-data-section" %1 %2) + (update-vals (group-by :property unsupported) #(mapv :path %)))) + :type :toast + :level :info}) + +(defn- show-import-warnings + [unknown-types unsupported] + (l/wrn :hint "token data skipped during import" + :types (str/join ", " (map (fn [[path type]] (str path " (" type ")")) unknown-types)) + :data (str/join ", " (map (fn [{:keys [path property]}] (str path " (" property ")")) unsupported))) + (ntf/show (import-warnings-notification unknown-types unsupported))) (defn- decode-json [json-string] @@ -80,8 +99,7 @@ (defn- parse-decoded-json [decoded-json file-name] (try - {:tokens-lib (ctob/parse-decoded-json decoded-json file-name) - :unknown-tokens (ctob/get-tokens-of-unknown-type decoded-json {})} + (ctob/import-decoded-json decoded-json file-name) (catch js/Error e (let [err (or (extract-error-with-code e) (extract-name-error e) @@ -91,9 +109,9 @@ (defn- validate-library "Resolve tokens in the library and search for errors. Reference errors are ignored, since it can be resolved by the user in the UI. All the other errors are thrown as exceptions." - [{:keys [tokens-lib unknown-tokens]}] - (when unknown-tokens - (st/emit! (show-unknown-types-warning unknown-tokens))) + [{:keys [tokens-lib unknown-types unsupported]}] + (when (or unknown-types unsupported) + (st/emit! (show-import-warnings unknown-types unsupported))) (try (let [tokens-tree (ctob/get-all-tokens-map tokens-lib) resolved-tokens (if (contains? cf/flags :tokenscript) diff --git a/frontend/src/app/main/data/workspace/tokens/propagation.cljs b/frontend/src/app/main/data/workspace/tokens/propagation.cljs index 2027e5be42..aae7f17945 100644 --- a/frontend/src/app/main/data/workspace/tokens/propagation.cljs +++ b/frontend/src/app/main/data/workspace/tokens/propagation.cljs @@ -97,8 +97,11 @@ (->> (map (fn [[value attrs]] [attrs {value #{object-id}}]) attrs-values-map) (into {}))) -(defn- collect-shapes-update-info [resolved-tokens objects] - (loop [items (seq objects) +(defn- collect-shapes-update-info + "Collect the token updates of `shapes`, a seq of [id shape] entries of + `objects`." + [resolved-tokens objects shapes] + (loop [items (seq shapes) frame-ids #{} text-ids [] tokens {}] @@ -150,27 +153,49 @@ (pcb/with-page page) (pcb/mod-page {:background color}))))))))) +(defn- scoped-shapes + "Entries of `objects` for the shapes in `root-ids` and all their descendants." + [objects root-ids] + (into [] + (comp (mapcat #(cfh/get-children-ids-with-self objects %)) + (distinct) + (keep (fn [id] + (when-let [shape (get objects id)] + [id shape])))) + root-ids)) + (defn propagate-tokens "Propagate tokens values to all shapes and page backgrounds where they - are applied" - [state resolved-tokens] + are applied. A non-nil `scope` of `:page-id` and `:root-ids` limits it + to those shapes and their descendants in that page." + [state resolved-tokens {scope-page-id :page-id root-ids :root-ids :as scope}] (let [file-id (get state :current-file-id) current-page-id (get state :current-page-id) fdata (dsh/lookup-file-data state file-id) + scoped? (some? scope) tpoint (ct/tpoint-ms)] - (l/inf :status "START" :hint "propagate-tokens") - (->> (rx/concat - (rx/of current-page-id) - (->> (rx/from (:pages fdata)) - (rx/filter (fn [id] (not= id current-page-id))))) + (l/inf :status "START" :hint "propagate-tokens" :scoped scoped?) + (->> (if scoped? + (rx/of scope-page-id) + (rx/concat + (rx/of current-page-id) + (->> (rx/from (:pages fdata)) + (rx/filter (fn [id] (not= id current-page-id)))))) (rx/mapcat (fn [page-id] (let [page (dsh/get-page fdata page-id) + objects + (:objects page) + [attrs frame-ids text-ids] - (collect-shapes-update-info resolved-tokens (:objects page)) + (collect-shapes-update-info resolved-tokens + objects + (if scoped? + (scoped-shapes objects root-ids) + objects)) actions (actionize-shapes-update-info page-id attrs) @@ -188,7 +213,7 @@ (when (seq observable) (apply rx/merge observable)) (when (seq normal) (rx/concat-all (rx/of normal))) - (when (:background-token page) + (when (and (not scoped?) (:background-token page)) (rx/of (update-canvas-background page-id resolved-tokens))) (->> (rx/from frame-ids) @@ -207,31 +232,37 @@ (l/inf :status "END" :hint "propagate-tokens" :elapsed elapsed))))))) (defn propagate-workspace-tokens - [] - (ptk/reify ::propagate-workspace-tokens - ptk/WatchEvent - (watch [_ state _] - (let [tokens-lib (dsh/lookup-tokens-lib state) - tokens-status (dsh/lookup-tokens-status state)] - (when (and tokens-lib tokens-status) - (when-let [tokens-tree (cfo/get-tokens-in-active-sets tokens-status tokens-lib)] - (->> (if (contains? cf/flags :tokenscript) - (rx/of (ts/resolve-tokens tokens-tree)) - (sd/resolve-tokens tokens-tree)) - (rx/mapcat - (fn [sd-tokens] - (let [undo-id (js/Symbol)] - (rx/concat - (rx/of (dwu/start-undo-transaction undo-id :timeout false)) + "Apply the resolved values of the active tokens to the shapes that use + them. Without `scope` it covers every page of the file. A `scope` of + `:page-id` and `:root-ids` limits it to those shapes and their + descendants." + ([] + (propagate-workspace-tokens nil)) + ([scope] + (ptk/reify ::propagate-workspace-tokens + ptk/WatchEvent + (watch [_ state _] + (let [tokens-lib (dsh/lookup-tokens-lib state) + tokens-status (dsh/lookup-tokens-status state)] + (when (and tokens-lib tokens-status) + (when-let [tokens-tree (cfo/get-tokens-in-active-sets tokens-status tokens-lib)] + (->> (if (contains? cf/flags :tokenscript) + (rx/of (ts/resolve-tokens tokens-tree)) + (sd/resolve-tokens tokens-tree)) + (rx/mapcat + (fn [sd-tokens] + (let [undo-id (js/Symbol)] + (rx/concat + (rx/of (dwu/start-undo-transaction undo-id :timeout false)) - ;; FIXME: now the tokens propagations is done by accumulating the update-shapes - ;; into a single commit-changes. This is not really the best way, the token application - ;; should be done with a changes_builder and sending only one `commit-changes` instead - ;; of creating lots of `update-shapes`. - (rx/of (dwsh/update-shapes-buffer-start)) - (->> (propagate-tokens state sd-tokens) - (rx/catch #(rx/concat - (rx/of (dwsh/update-shapes-buffer-stop)) - (rx/throw %)))) - (rx/of (dwsh/update-shapes-buffer-stop)) - (rx/of (dwu/commit-undo-transaction undo-id))))))))))))) + ;; FIXME: now the tokens propagations is done by accumulating the update-shapes + ;; into a single commit-changes. This is not really the best way, the token application + ;; should be done with a changes_builder and sending only one `commit-changes` instead + ;; of creating lots of `update-shapes`. + (rx/of (dwsh/update-shapes-buffer-start)) + (->> (propagate-tokens state sd-tokens scope) + (rx/catch #(rx/concat + (rx/of (dwsh/update-shapes-buffer-stop)) + (rx/throw %)))) + (rx/of (dwsh/update-shapes-buffer-stop)) + (rx/of (dwu/commit-undo-transaction undo-id)))))))))))))) diff --git a/frontend/src/app/main/data/workspace/transforms.cljs b/frontend/src/app/main/data/workspace/transforms.cljs index 3e5f8ced3a..b2372250f0 100644 --- a/frontend/src/app/main/data/workspace/transforms.cljs +++ b/frontend/src/app/main/data/workspace/transforms.cljs @@ -1561,8 +1561,14 @@ ;; Shapes that already belong to the frame leave nothing to move (when (and (some? frame-id) (seq ids)) - (rx/of (dch/commit-changes changes) - (dwc/expand-collapse frame-id))) + (let [layout-ids (into #{} + (filter (partial cfh/has-layout? objects)) + (cons frame-id (map #(dm/get-in objects [% :parent-id]) ids)))] + (rx/concat + (rx/of (dch/commit-changes changes) + (dwc/expand-collapse frame-id)) + (when (seq layout-ids) + (rx/of (ptk/data-event :layout/update {:ids layout-ids})))))) (when add-component-to-variant? (rx/of (ev/event {::ev/name "add-component-to-variant"}))) (when add-new-variant? @@ -1624,6 +1630,68 @@ (rx/of (dwm/apply-wasm-modifiers modifiers {:ignore-snap-pixel true})) (rx/of (dwm/apply-modifiers {:modifiers modifiers :ignore-snap-pixel true})))))))) +;; -- Scale --------------------------------------------------------- + +(defn- anchor->point + "Returns the point of `rect` selected by `anchor`, a + `[horizontal vertical]` pair. Horizontal is one of `:left`, `:center` + or `:right`; vertical is one of `:top`, `:center` or `:bottom`." + [rect [h v]] + (let [{:keys [x y width height]} rect] + (gpt/point (case h + :left x + :right (+ x width) + (+ x (/ width 2))) + (case v + :top y + :bottom (+ y height) + (+ y (/ height 2)))))) + +(defn scale-selected + "Scales the selected shapes and their content (strokes, texts, + shadows...) by `factor`, keeping the `anchor` point of the selection + bounds in place. Behaves like a resize in the scale content mode: + constraints are ignored." + [factor anchor] + (dm/assert! + "expected a positive scale factor" + (and (number? factor) (pos? factor))) + + (ptk/reify ::scale-selected + ptk/WatchEvent + (watch [_ state _] + (let [objects (dsh/lookup-page-objects state) + shapes (->> (dsh/lookup-selected state {:omit-blocked? true}) + (map (d/getf objects)) + (remove ctk/is-variant-container?))] + (if (or (empty? shapes) (mth/close? factor 1)) + (rx/empty) + (let [ids (map :id shapes) + origin (anchor->point (gsh/shapes->rect shapes) anchor) + scalev (gpt/point factor factor) + + modifier + (fn [shape] + (let [grow-type (get shape :grow-type) + new-grow-type (when (cfh/text-shape? shape) + (dwm/next-grow-type grow-type scalev))] + (cond-> (ctm/resize-modifiers scalev origin) + :always + (ctm/scale-content factor) + + (and (some? new-grow-type) (not= new-grow-type grow-type)) + (ctm/change-property :grow-type new-grow-type)))) + + modif-tree (dwm/build-modif-tree ids objects modifier)] + + (if (features/active-feature? state "render-wasm/v1") + (rx/of (dwm/apply-wasm-modifiers modif-tree + :ignore-constraints true + :ignore-snap-pixel true)) + (rx/of (dwm/apply-modifiers {:modifiers modif-tree + :ignore-constraints true + :ignore-snap-pixel true}))))))))) + (defn fit-layout-modifiers [objects frame] ;; Set temporaly the auto flag and calculate a reflow to resize and position diff --git a/frontend/src/app/main/data/workspace/variants.cljs b/frontend/src/app/main/data/workspace/variants.cljs index 49eb435f34..8c7cdbe34c 100644 --- a/frontend/src/app/main/data/workspace/variants.cljs +++ b/frontend/src/app/main/data/workspace/variants.cljs @@ -235,14 +235,10 @@ (let [data (dsh/lookup-file-data state)] (with-main-page data {:variant-id variant-id} (fn [page-id objects] - (let [variant-components (cfv/find-variant-components data objects variant-id) - properties-empty-pos (->> variant-components - (mapcat :variant-properties) - (group-by :name) + (let [properties-empty-pos (->> (cfv/extract-properties-values data objects variant-id) (map-indexed - (fn [i [_ v]] - [i (->> v - (map :value) + (fn [i {:keys [value]}] + [i (->> value (remove empty?) empty?)])) (reverse)) diff --git a/frontend/src/app/main/data/workspace/versions.cljs b/frontend/src/app/main/data/workspace/versions.cljs index 30fab0de28..f14ccd36e2 100644 --- a/frontend/src/app/main/data/workspace/versions.cljs +++ b/frontend/src/app/main/data/workspace/versions.cljs @@ -106,7 +106,7 @@ (rx/of (update-versions-state {:editing nil}) (ev/event {::ev/name "rename-version" :file-id file-id})) - (->> (rp/cmd! :update-file-snapshot {:id id :label label}) + (->> (rp/cmd! :update-file-snapshot {:id id :file-id file-id :label label}) (rx/map fetch-versions))))))) (defn- clear-preview-state @@ -149,9 +149,10 @@ (ptk/reify ::delete-version ptk/WatchEvent - (watch [_ _ _] - (->> (rp/cmd! :delete-file-snapshot {:id id}) - (rx/map fetch-versions))))) + (watch [_ state _] + (let [file-id (:current-file-id state)] + (->> (rp/cmd! :delete-file-snapshot {:id id :file-id file-id}) + (rx/map fetch-versions)))))) (defn pin-version [id] @@ -161,8 +162,10 @@ (watch [_ state _] (when-let [version (->> (dm/get-in state [:workspace-versions :data]) (d/seek #(= (:id %) id)))] - (let [params {:id id - :label (ct/format-inst (:created-at version) :localized-date)}] + (let [file-id (:current-file-id state) + params {:id id + :file-id file-id + :label (ct/format-inst (:created-at version) :localized-date)}] (->> (rp/cmd! :update-file-snapshot params) (rx/mapcat (fn [_] @@ -175,18 +178,20 @@ (assert (uuid? id) "expected valid uuid for `id`") (ptk/reify ::lock-version ptk/WatchEvent - (watch [_ _ _] - (->> (rp/cmd! :lock-file-snapshot {:id id}) - (rx/map fetch-versions))))) + (watch [_ state _] + (let [file-id (:current-file-id state)] + (->> (rp/cmd! :lock-file-snapshot {:id id :file-id file-id}) + (rx/map fetch-versions)))))) (defn unlock-version [id] (assert (uuid? id) "expected valid uuid for `id`") (ptk/reify ::unlock-version ptk/WatchEvent - (watch [_ _ _] - (->> (rp/cmd! :unlock-file-snapshot {:id id}) - (rx/map fetch-versions))))) + (watch [_ state _] + (let [file-id (:current-file-id state)] + (->> (rp/cmd! :unlock-file-snapshot {:id id :file-id file-id}) + (rx/map fetch-versions)))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; RESTORE VERSION EVENTS diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index bdad62f68a..753467a9a3 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -16,6 +16,7 @@ [app.main.data.nitrate :as dnt] [app.main.data.notifications :as ntf] [app.main.data.workspace :as-alias dw] + [app.main.features :as features] [app.main.repo :as rp] [app.main.router :as rt] [app.main.store :as st] @@ -410,7 +411,8 @@ :hint hint :href (rt/get-current-href) :report report - :occurrences occurrences}))) + :occurrences occurrences + :renderer (features/get-active-renderer @st/state)}))) (defn submit-report "Report the error report to the audit log subsystem, subject to the @@ -812,11 +814,11 @@ [{:keys [organization-id team-id] :as error}] (when-not @sso-renewal-pending? (vreset! sso-renewal-pending? true) - (let [dest-url (rt/get-current-href)] + (let [query-string (rt/get-current-query-string)] (->> (dnt/check-organization-sso {:organization-id organization-id :team-id team-id - :dest-url dest-url}) + :query-string query-string}) ;; Release the guard however the check ends, including an ;; unsubscription or a completion without a result: a stuck guard ;; would silently drop every later rejection. diff --git a/frontend/src/app/main/features.cljs b/frontend/src/app/main/features.cljs index b5fcfdc528..7ced696941 100644 --- a/frontend/src/app/main/features.cljs +++ b/frontend/src/app/main/features.cljs @@ -85,6 +85,13 @@ "feature not supported") (contains? (:features state) feature)) +(defn get-active-renderer + "Name of the renderer active in `state`, for telemetry: `wasm` (WebGL) or + `svg`, or nil before the features are initialized." + [state] + (when (contains? state :features) + (if (active-feature? state "render-wasm/v1") "wasm" "svg"))) + (defn active-features? "Given a state and a set of features, check if the features are all enabled." ([state a] diff --git a/frontend/src/app/main/render.cljs b/frontend/src/app/main/render.cljs index 4a45630454..c2c95c468c 100644 --- a/frontend/src/app/main/render.cljs +++ b/frontend/src/app/main/render.cljs @@ -460,7 +460,7 @@ (mf/defc object-svg {::mf/wrap [mf/memo]} - [{:keys [objects object-id embed skip-children] + [{:keys [objects object-id embed skip-children class] :or {embed false} :as props}] (let [object (get objects object-id) @@ -482,6 +482,7 @@ [:& (mf/provider export/include-metadata-ctx) {:value false} [:& (mf/provider embed/context) {:value embed} [:svg {:id (dm/str "screenshot-" object-id) + :class class :view-box vbox :width (ust/format-precision width viewbox-decimal-precision) :height (ust/format-precision height viewbox-decimal-precision) diff --git a/frontend/src/app/main/router.cljs b/frontend/src/app/main/router.cljs index a466f9908b..070c2a764c 100644 --- a/frontend/src/app/main/router.cljs +++ b/frontend/src/app/main/router.cljs @@ -192,8 +192,15 @@ (ts/asap (partial dom/reload-current-window force?))))) (defn nav-raw - [& {:keys [href uri]}] + "Navigate to a raw URL, out of the router. + + The destination is exposed through `@event`, like `navigate` does, so + listeners and tests can tell where the user is being sent." + [& {:keys [href uri] :as props}] (ptk/reify ::nav-raw + IDeref + (-deref [_] props) + ptk/EffectEvent (effect [_ _ _] (cond @@ -207,6 +214,41 @@ [] (.-href globals/location)) +(defn token->query-string + "The query string of `token`, without the `?` that separates it from the + path. The history token is `location.search`, so it carries that delimiter; + a query string does not, and this is what the organization SSO RPC takes: the + server joins it under its own public URI and puts the separator back." + [token] + (let [token (or token "")] + (if (str/starts-with? token "?") + (subs token 1) + token))) + +(defn token->url + "Build the absolute URL for a route token under `cf/public-uri`. Accepts + the token in either shape: with its leading `?` or as a bare query string. + A blank token lands on the public URI, without a trailing separator." + [token] + (let [query-string (token->query-string token)] + (if (str/blank? query-string) + (str cf/public-uri) + (str cf/public-uri "?" query-string)))) + +(defn get-current-token + "Return the current history token: the query string that carries the route, + with the leading `?` that separates it from the path. The application + concatenates it over the base path, so it keeps the delimiter; use + `get-current-query-string` for the value the SSO flow sends to the backend. + The server builds the destination URL out of that one, so the client never + chooses the origin of it." + [] + (bhistory/get-token)) + +(defn get-current-query-string + "The query string of the current route (see `token->query-string`)." + [] + (token->query-string (get-current-token))) ;; --- History API diff --git a/frontend/src/app/main/ui/dashboard.cljs b/frontend/src/app/main/ui/dashboard.cljs index 952714bec8..68bf9b2f60 100644 --- a/frontend/src/app/main/ui/dashboard.cljs +++ b/frontend/src/app/main/ui/dashboard.cljs @@ -18,6 +18,7 @@ [app.main.data.plugins :as dp] [app.main.data.profile :as dprof] [app.main.data.project :as dpj] + [app.main.data.team :as dtm] [app.main.refs :as refs] [app.main.router :as rt] [app.main.store :as st] @@ -233,8 +234,10 @@ (st/emit! (ev/event {::ev/name "install-plugin" :name (:name plugin) :url plugin-url})) (open-permissions-dialog plugin)) (st/emit! (notif/error (tr "dashboard.plugins.parse-error"))))) - (fn [_] - (st/emit! (notif/error (tr "dashboard.plugins.bad-url")))))) + (fn [err] + (st/emit! (notif/error (if (= ::preg/global-not-allowed (:type (ex-data err))) + (tr "workspace.plugins.error.global-not-allowed") + (tr "dashboard.plugins.bad-url"))))))) (binding [storage/*sync* true] (swap! storage/session dissoc :plugin-url)))))) @@ -305,6 +308,8 @@ (st/emit! (dnt/add-team-to-organization {:team-id (:team-id action) :organization-id (:organization-id action) :skip-audit? true})) + :create-team-in-organization + (st/emit! (dtm/check-and-create-team (:team-id action))) nil))))) (mf/defc dashboard* diff --git a/frontend/src/app/main/ui/dashboard/grid.cljs b/frontend/src/app/main/ui/dashboard/grid.cljs index da458ac63d..ee1c6f010f 100644 --- a/frontend/src/app/main/ui/dashboard/grid.cljs +++ b/frontend/src/app/main/ui/dashboard/grid.cljs @@ -51,6 +51,10 @@ (def thumbnail-width 252) +;; Larger than the grid card: the image is also the link preview card +;; and has to stay sharp on high-DPI screens. +(def ^:private wasm-thumbnail-width 600) + ;; --- Grid Item Thumbnail (defn- persist-thumbnail @@ -59,30 +63,42 @@ (->> (rp/cmd! :create-file-thumbnail params) (rx/map :id)))) +(defn- render-thumbnail-with-wasm + "Returns a stream with the PNG blob of the file thumbnail." + [file-id revn] + (->> (mw/ask! {:cmd :thumbnails/generate-for-file-wasm + :revn revn + :file-id file-id + :width wasm-thumbnail-width}) + (rx/map :blob))) + +(defn- render-thumbnail-with-svg + "Returns a stream with the PNG blob of the file thumbnail." + [file-id revn] + (->> (mw/ask! {:cmd :thumbnails/generate-for-file + :revn revn + :file-id file-id + :width thumbnail-width}) + (rx/mapcat + (fn [{:keys [fonts] :as result}] + (->> (fonts/render-font-styles fonts) + (rx/map (fn [styles] + (-> result + (assoc :styles styles + :width thumbnail-width))))))) + (rx/mapcat thr/render))) + (defn render-thumbnail + "Returns a stream with the PNG blob of the file thumbnail." [file-id revn] (if (features/active-feature? @st/state "render-wasm/v1") - (mw/ask! {:cmd :thumbnails/generate-for-file-wasm - :revn revn - :file-id file-id - :width thumbnail-width}) - (->> (mw/ask! {:cmd :thumbnails/generate-for-file - :revn revn - :file-id file-id - :width thumbnail-width}) - (rx/mapcat - (fn [{:keys [fonts] :as result}] - (->> (fonts/render-font-styles fonts) - (rx/map (fn [styles] - (-> result - (assoc :styles styles - :width thumbnail-width)))))))))) + (render-thumbnail-with-wasm file-id revn) + (render-thumbnail-with-svg file-id revn))) (defn- ask-for-thumbnail "Creates some hooks to handle the files thumbnails cache" [file-id revn] (->> (render-thumbnail file-id revn) - (rx/mapcat thr/render) (rx/mapcat (partial persist-thumbnail file-id revn)))) (mf/defc grid-item-thumbnail* diff --git a/frontend/src/app/main/ui/dashboard/sidebar.cljs b/frontend/src/app/main/ui/dashboard/sidebar.cljs index d9514fb927..ca85e0ea26 100644 --- a/frontend/src/app/main/ui/dashboard/sidebar.cljs +++ b/frontend/src/app/main/ui/dashboard/sidebar.cljs @@ -656,7 +656,7 @@ [:* (if (contains? cf/flags :admin-console) [:* - [:> nitrate-sidebar* {:profile profile :teams teams}] + [:> nitrate-sidebar* {:profile profile :teams teams :team team}] [:> nitrate-current-plan* {:profile profile}]] (when (contains? cf/flags :subscriptions) (if (show-subscription-dashboard-banner? profile) diff --git a/frontend/src/app/main/ui/dashboard/subscription.cljs b/frontend/src/app/main/ui/dashboard/subscription.cljs index c93644d90c..d983f789ac 100644 --- a/frontend/src/app/main/ui/dashboard/subscription.cljs +++ b/frontend/src/app/main/ui/dashboard/subscription.cljs @@ -9,6 +9,7 @@ [app.main.data.event :as ev] [app.main.data.modal :as modal] [app.main.data.nitrate :as dnt] + [app.main.data.profile :as dprof] [app.main.refs :as refs] [app.main.router :as rt] [app.main.store :as st] @@ -28,6 +29,27 @@ type "professional")) +(defn get-team-plan-type + [team] + (if (some? (dm/get-in team [:organization :id])) + "enterprise" + (get-subscription-type (:subscription team)))) + +(defn get-profile-subscription-type + [profile] + (if (dnt/is-valid-license? profile) + (dm/get-in profile [:subscription :type]) + (get-subscription-type (dm/get-in profile [:props :subscription])))) + +(defn show-team-plan? + "The plan of a team outside an organization is the plan of its owner, + which the backend puts in the team `:subscription`. It is hidden from + every member when the owner has an Enterprise or Nitrate plan." + [team] + (or (some? (dm/get-in team [:organization :id])) + (not (contains? #{"enterprise" "nitrate"} + (get-subscription-type (:subscription team)))))) + (mf/defc cta-power-up* [{:keys [top-title top-description bottom-description bottom-button bottom-button-href has-dropdown is-highlighted]}] (let [show-data* (mf/use-state false) @@ -118,8 +140,48 @@ :has-dropdown false :is-highlighted false}])))) +(mf/defc nitrate-enterprise-banner* + [{:keys [on-click button-label profile]}] + (let [collapsed? (true? (get-in profile [:props :promotional-banner-collapsed])) + toggle-collapse + (mf/use-fn + (mf/deps collapsed?) + #(st/emit! (dprof/update-profile-props + {:promotional-banner-collapsed (not collapsed?)})))] + + [:div {:class (stl/css-case :nitrate-banner true + :nitrate-enterprise-banner true + :collapsed collapsed?)} + [:button {:class (stl/css :nitrate-enterprise-toggle) + :type "button" + :aria-controls "nitrate-enterprise-content" + :aria-expanded (not collapsed?) + :on-click toggle-collapse} + [:> icon* {:icon-id i/pencil + :size "m" + :class (stl/css :nitrate-enterprise-pencil) + :aria-hidden true}] + [:span {:class (stl/css :nitrate-enterprise-title)} + (tr "subscription.dashboard.banner.unlock-enterprise")] + [:> icon* {:icon-id i/arrow-down + :size "m" + :class (stl/css :nitrate-enterprise-chevron) + :aria-hidden true}]] + [:div {:id "nitrate-enterprise-content" + :class (stl/css :nitrate-enterprise-panel) + :aria-hidden collapsed?} + [:div {:class (stl/css :nitrate-enterprise-panel-inner)} + [:span {:class (stl/css :nitrate-enterprise-description)} + (tr "subscription.dashboard.banner.unlock-features-description")] + [:> button* {:variant "primary" + :type "button" + :class (stl/css :nitrate-bottom-button) + :on-click on-click + :disabled collapsed?} + button-label]]]])) + (mf/defc nitrate-sidebar* - [{:keys [profile teams]}] + [{:keys [profile teams team]}] (let [nitrate? (dnt/is-valid-license? profile) nitrate-license (:subscription profile) manual-license? (:manual nitrate-license) @@ -151,6 +213,12 @@ subscription-type (if nitrate? (:type nitrate-license) (get-subscription-type (-> profile :props :subscription))) + show-unlock-features? + (and (contains? cf/flags :admin-console) + (not nitrate?) + (contains? #{"professional" "unlimited"} subscription-type) + (nil? (dm/get-in team [:organization :id]))) + team-count (count teams) account-age-days (dnt/account-age-days profile) @@ -210,19 +278,14 @@ :class (stl/css :nitrate-bottom-button) :on-click handle-go-to-cc} (tr "nitrate.activation-success.create-organization")]]] - ;; Banner for users without nitrate license - (when (not nitrate?) - [:div {:class (stl/css :nitrate-banner :highlighted)} - [:div {:class (stl/css :nitrate-content)} - [:span {:class (stl/css :nitrate-title)} (tr "subscription.dashboard.banner.unlock-features")]] - [:div {:class (stl/css :nitrate-content)} - [:span {:class (stl/css :nitrate-info)} (tr "subscription.dashboard.banner.unlock-features-description-text")] - [:> button* {:variant "primary" - :type "button" - :class (stl/css :nitrate-bottom-button) - :on-click handle-click} (if (:subscription profile) - (tr "subscription.dashboard.banner.upgrade-nitrate") - (tr "nitrate.form.try-free"))]]])) + ;; Banner for Professional/Unlimited users outside an organization + (when show-unlock-features? + [:> nitrate-enterprise-banner* + {:profile profile + :on-click handle-click + :button-label (if (:subscription profile) + (tr "subscription.dashboard.banner.upgrade-nitrate") + (tr "nitrate.form.try-free"))}])) ;; Banner for users with nitrate license almost expired or expired (when show-subscription-warning? @@ -249,7 +312,7 @@ (let [nitrate? (dnt/is-valid-license? profile) nitrate-license (:subscription profile) subscription (-> profile :props :subscription) - subscription-type (if nitrate? (:type nitrate-license) (get-subscription-type subscription)) + subscription-type (get-profile-subscription-type profile) subscription-is-trial (= "trialing" (:status (if nitrate? nitrate-license subscription))) go-to-subscription (mf/use-fn #(st/emit! (rt/nav :settings-subscription)))] [:button {:class (stl/css :nitrate-current-plan) @@ -272,6 +335,7 @@ [{:keys [is-owner team]}] (let [subscription (:subscription team) subscription-type (get-subscription-type subscription) + team-plan-type (get-team-plan-type team) subscription-is-trial (= "trialing" (:status subscription)) go-to-manage-subscription @@ -286,22 +350,23 @@ href (str "payments/subscriptions/show?returnUrl=" href)] (st/emit! (rt/nav-raw :href href)))))] - [:div {:class (stl/css :team)} - [:div {:class (stl/css :team-label)} - (tr "subscription.dashboard.team-plan")] - [:span {:class (stl/css :team-text)} - (case subscription-type - "professional" (tr "subscription.settings.professional") - "unlimited" (if subscription-is-trial - (tr "subscription.settings.unlimited-trial") - (tr "subscription.settings.unlimited")) + (when (show-team-plan? team) + [:div {:class (stl/css :team)} + [:div {:class (stl/css :team-label)} + (tr "subscription.dashboard.team-plan")] + [:span {:class (stl/css :team-text)} + (case team-plan-type + "professional" (tr "subscription.settings.professional") + "unlimited" (if subscription-is-trial + (tr "subscription.settings.unlimited-trial") + (tr "subscription.settings.unlimited")) - "enterprise" (tr "subscription.settings.enterprise"))] - (when (and is-owner (not= subscription-type "professional")) - [:button {:class (stl/css :manage-subscription-link) - :on-click go-to-manage-subscription - :data-testid "manage-subscription-link"} - (tr "subscription.settings.manage-your-subscription")])])) + "enterprise" (tr "subscription.settings.enterprise"))] + (when (and is-owner (not= subscription-type "professional")) + [:button {:class (stl/css :manage-subscription-link) + :on-click go-to-manage-subscription + :data-testid "manage-subscription-link"} + (tr "subscription.settings.manage-your-subscription")])]))) (mf/defc menu-team-icon* [{:keys [subscription-type]}] diff --git a/frontend/src/app/main/ui/dashboard/subscription.scss b/frontend/src/app/main/ui/dashboard/subscription.scss index 4699b44d92..7d0bdcb0f0 100644 --- a/frontend/src/app/main/ui/dashboard/subscription.scss +++ b/frontend/src/app/main/ui/dashboard/subscription.scss @@ -208,25 +208,132 @@ } .nitrate-banner { + --nitrate-enterprise-gradient-start: var(--color-accent-success); + --nitrate-enterprise-gradient-end: var(--color-token-border); + + :global(.light) & { + --nitrate-enterprise-gradient-start: var(--color-background-info); + --nitrate-enterprise-gradient-end: var(--color-accent-primary-muted); + } + display: flex; border-radius: var(--sp-s); flex-direction: column; margin: var(--sp-m) var(--sp-m) 0; - background: var(--color-accent-select); + background: linear-gradient( + 110deg, + var(--nitrate-enterprise-gradient-start), + var(--nitrate-enterprise-gradient-end) + ); border: $b-1 solid var(--color-accent-primary-muted); padding: var(--sp-l); } +.nitrate-enterprise-banner { + display: grid; + grid-template-rows: auto minmax(0, 1fr); + background: linear-gradient( + 110deg, + var(--nitrate-enterprise-gradient-start), + var(--nitrate-enterprise-gradient-end) + ); + transition: grid-template-rows 240ms ease; +} + +.nitrate-enterprise-banner.collapsed { + grid-template-rows: auto minmax(0, 0fr); +} + +.nitrate-enterprise-toggle { + appearance: none; + background: transparent; + border: 0; + color: var(--color-foreground-primary); + cursor: pointer; + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: center; + gap: var(--sp-s); + inline-size: 100%; + padding: 0; + text-align: start; + font: inherit; + + &:focus-visible { + outline: var(--color-accent-primary) solid $b-2; + outline-offset: var(--sp-xxs); + border-radius: $br-4; + } +} + +.nitrate-enterprise-title { + @include use-typography("body-large"); + + color: var(--color-foreground-primary); + font-weight: 600; +} + +.nitrate-enterprise-pencil { + --icon-fill-color: currentcolor; + --icon-stroke-color: none; + + color: var(--color-foreground-primary); +} + +.nitrate-enterprise-chevron { + color: var(--color-foreground-secondary); + transition: transform 240ms ease; +} + +.collapsed .nitrate-enterprise-chevron { + transform: rotate(180deg); +} + +.nitrate-enterprise-panel { + min-block-size: 0; + overflow: hidden; + opacity: 1; + transition: opacity 180ms ease; +} + +.collapsed .nitrate-enterprise-panel { + opacity: 0; + pointer-events: none; +} + +.nitrate-enterprise-panel-inner { + display: flex; + flex-direction: column; + min-block-size: 0; + padding-block-start: var(--sp-s); +} + +.nitrate-enterprise-description { + @include use-typography("body-medium"); + + color: var(--color-foreground-primary); + margin-block-end: var(--sp-l); +} + +@media (prefers-reduced-motion: reduce) { + .nitrate-enterprise-banner, + .nitrate-enterprise-chevron, + .nitrate-enterprise-panel { + transition: none; + } +} + .nitrate-title { @include use-typography("body-large"); color: var(--color-foreground-primary); + font-weight: 600; } .nitrate-info { @include use-typography("body-medium"); - color: var(--color-foreground-secondary); + color: var(--color-foreground-primary); margin-block: var(--sp-s) var(--sp-xxl); } diff --git a/frontend/src/app/main/ui/dashboard/team.cljs b/frontend/src/app/main/ui/dashboard/team.cljs index b123c31772..b126559cac 100644 --- a/frontend/src/app/main/ui/dashboard/team.cljs +++ b/frontend/src/app/main/ui/dashboard/team.cljs @@ -665,7 +665,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (mf/defc invitation-role-selector* - [{:keys [role status on-change]}] + [{:keys [role status on-change can-invite]}] (let [show? (mf/use-state false) label (cond (= role :owner) (tr "labels.owner") @@ -686,7 +686,7 @@ (on-change role event))))] [:* - (if (= status :pending) + (if (and ^boolean can-invite (= status :pending)) [:button {:class (stl/css :rol-selector :has-priv) :type "button" :aria-haspopup "listbox" @@ -714,15 +714,17 @@ (tr "labels.viewer")]]]])) (defn invitation-link-copyable? - "The link of an owner invitation can only be copied by the team owner: an - admin cannot create nor promote an owner invitation, so the backend does - not mint its token for them." - [is-owner? role] - (or (not= :owner role) is-owner?)) + "The link of an invitation invites as much as sending it, so it needs the + organization invite policy (`can-invite?`). The link of an owner invitation + can only be copied by the team owner: an admin cannot create nor promote an + owner invitation, so the backend does not mint its token for them." + [can-invite? is-owner? role] + (and can-invite? + (or (not= :owner role) is-owner?))) (mf/defc invitation-actions* {::mf/private true} - [{:keys [invitation team-id is-owner?]}] + [{:keys [invitation team-id is-owner? can-invite]}] (let [email (:email invitation) copied* (mf/use-state false) copied? (deref copied*) @@ -768,7 +770,7 @@ (-> (dtm/copy-invitation-link params) (with-meta {::ev/origin :team}))))))] - (when (invitation-link-copyable? is-owner? (:role invitation)) + (when (invitation-link-copyable? can-invite is-owner? (:role invitation)) [:> icon-button* {:variant "ghost" :aria-label (tr "labels.copy-invitation-link") :on-click on-copy @@ -777,7 +779,7 @@ (mf/defc invitation-row* {::mf/wrap [mf/memo] ::mf/private true} - [{:keys [invitation team-id selected on-select-change is-owner?]}] + [{:keys [invitation team-id selected on-select-change is-owner? can-invite]}] (let [expired? (:expired invitation) email (:email invitation) @@ -811,24 +813,29 @@ [:div {:class (stl/css :table-row :table-row-invitations)} [:div {:class (stl/css :table-field :field-email)} - [:div {:class (stl/css :input-wrapper)} - [:label - [:span {:class (stl/css-case :input-checkbox true - :global/checked (is-selected? email))} - deprecated-icon/status-tick] + ;; Selecting an invitation only leads to resending or deleting it, and + ;; both need the organization invite policy. + (if ^boolean can-invite + [:div {:class (stl/css :input-wrapper)} + [:label + [:span {:class (stl/css-case :input-checkbox true + :global/checked (is-selected? email))} + deprecated-icon/status-tick] - [:input {:type "checkbox" - :id (dm/str "email-" email) - :data-attr email - :value email - :checked (is-selected? email) - :on-change on-change}] - email]]] + [:input {:type "checkbox" + :id (dm/str "email-" email) + :data-attr email + :value email + :checked (is-selected? email) + :on-change on-change}] + email]] + [:div email])] [:div {:class (stl/css :table-field :field-roles)} [:> invitation-role-selector* {:role role :status status + :can-invite can-invite :on-change on-change-role}]] [:div {:class (stl/css :table-field :field-status)} @@ -838,11 +845,12 @@ [:> invitation-actions* {:invitation invitation :team-id team-id - :is-owner? is-owner?}]]])) + :is-owner? is-owner? + :can-invite can-invite}]]])) (mf/defc empty-invitation-table* {::mf/private true} - [{:keys [team]}] + [{:keys [team can-invite]}] (let [route (mf/deref refs/route) invite-email (-> route :query-params :invite-email) @@ -855,11 +863,21 @@ [:div {:class (stl/css :empty-invitations)} [:div (tr "labels.no-invitations")] [:div (tr "labels.no-invitations-gather-people")] - [:> button* {:variant "primary" - :class (stl/css :btn-empty-invitations) - :on-click on-invite-member - :data-testid "invite-member"} - (tr "dashboard.invite-profile")]])) + ;; Same as the header button: disabled with its reason when the + ;; organization invite policy leaves the profile out. + (let [invite-button (mf/html + [:> button* {:variant "primary" + :class (stl/css :btn-empty-invitations) + :on-click on-invite-member + :disabled (not can-invite) + :data-testid "invite-member"} + (tr "dashboard.invite-profile")])] + (if ^boolean can-invite + invite-button + [:> tooltip* {:content (tr "dashboard.invite-profile-disabled") + :id "empty-invite-member-disabled-tooltip" + :tab-index 0} + invite-button]))])) (mf/defc invitation-modal {::mf/register modal/components @@ -1224,18 +1242,17 @@ (tr "dashboard.invitations.no-permission")]] [:* - (when (> (count @selected) 0) + ;; Managing the pending invitations (role, resend, delete, link) + ;; needs the organization invite policy; the list only needs admin. + (when (and can-invite? (> (count @selected) 0)) [:div {:class (stl/css :invitations-actions)} [:div (tr "team.invitations-selected" (i18n/c (count @selected)))] - ;; Resending goes through the invite command, so it needs the - ;; organization invite policy; deleting only needs team admin. - (when can-invite? - [:div - [:> button* {:variant "secondary" - :type "button" - :on-click on-resend} - (tr "labels.resend-invitation")]]) + [:div + [:> button* {:variant "secondary" + :type "button" + :on-click on-resend} + (tr "labels.resend-invitation")]] [:> icon-button* {:on-click on-delete :variant "destructive" :aria-label (tr "labels.delete-invitation") @@ -1261,13 +1278,14 @@ "arrow-down") :on-click on-order-by-status}]]] (if (empty? @invitations) - [:> empty-invitation-table* {:team team}] + [:> empty-invitation-table* {:team team :can-invite can-invite?}] [:div {:class (stl/css :table-rows)} (for [invitation @invitations] [:> invitation-row* {:key (:email invitation) :invitation invitation :is-owner? is-owner? + :can-invite can-invite? :team-id team-id :selected selected :on-select-change on-select-change}])])])])) @@ -1782,4 +1800,5 @@ (tr "labels.num-of-files" (i18n/c (:files stats)))]]] (when (contains? cfg/flags :subscriptions) - [:> team* {:is-owner (:is-owner permissions) :team team}])]]])) + [:> team* {:is-owner (:is-owner permissions) + :team team}])]]])) diff --git a/frontend/src/app/main/ui/dashboard/team_form.cljs b/frontend/src/app/main/ui/dashboard/team_form.cljs index 08f6dcaad6..c1b45cde76 100644 --- a/frontend/src/app/main/ui/dashboard/team_form.cljs +++ b/frontend/src/app/main/ui/dashboard/team_form.cljs @@ -83,7 +83,7 @@ (mf/defc team-form-modal {::mf/register modal/components ::mf/register-as :team-form} - [{:keys [team organization-id] :as props}] + [{:keys [team organization-id organization-name]}] (let [initial (mf/use-memo (mf/deps team organization-id) (fn [] @@ -117,7 +117,9 @@ [:h2 {:class (stl/css :modal-title)} (tr "labels.rename-team")] [:h2 {:class (stl/css :modal-title)} - (tr "labels.create-team")]) + (if organization-name + (tr "dashboard.create-team-in-organization" organization-name) + (tr "labels.create-team"))]) [:> icon-button* {:variant "ghost" :tooltip-class (stl/css :modal-close-btn) diff --git a/frontend/src/app/main/ui/dashboard/team_form.scss b/frontend/src/app/main/ui/dashboard/team_form.scss index 1047c8f3d8..2cf4583bfd 100644 --- a/frontend/src/app/main/ui/dashboard/team_form.scss +++ b/frontend/src/app/main/ui/dashboard/team_form.scss @@ -26,6 +26,9 @@ @include use-typography("headline-large"); color: var(--color-foreground-primary); + max-inline-size: 100%; + padding-inline-end: var(--sp-xxl); + overflow-wrap: anywhere; } // Tooltip wrapper of the close button: it places it in the corner. diff --git a/frontend/src/app/main/ui/ds/controls/combobox.cljs b/frontend/src/app/main/ui/ds/controls/combobox.cljs index 2174d84a78..8dfb5647ce 100644 --- a/frontend/src/app/main/ui/ds/controls/combobox.cljs +++ b/frontend/src/app/main/ui/ds/controls/combobox.cljs @@ -12,7 +12,7 @@ [app.main.constants :refer [max-input-length]] [app.main.ui.ds.controls.select :refer [handle-focus-change]] [app.main.ui.ds.controls.shared.options-dropdown :refer [options-dropdown* schema:option]] - [app.main.ui.ds.foundations.assets.icon :refer [icon*] :as i] + [app.main.ui.ds.foundations.assets.icon :refer [icon* icon-list] :as i] [app.util.dom :as dom] [app.util.globals :as globals] [app.util.keyboard :as kbd] @@ -23,6 +23,9 @@ [rumext.v2.util :as mfu]) (:import goog.events.EventType)) +(def ^:private schema:icon + [:and :string [:fn #(contains? icon-list %)]]) + (def ^:private schema:combobox [:map [:id {:optional true} :string] @@ -30,6 +33,7 @@ [:class {:optional true} :string] [:max-length {:optional true} :int] [:placeholder {:optional true} :string] + [:icon {:optional true} [:maybe schema:icon]] [:disabled {:optional true} :boolean] [:default-selected {:optional true} :string] [:select-only {:optional true} :boolean] @@ -39,7 +43,7 @@ (mf/defc combobox* {::mf/schema schema:combobox} - [{:keys [id options class placeholder disabled has-error default-selected select-only max-length empty-to-end on-change] :rest props}] + [{:keys [id options class placeholder icon disabled has-error default-selected select-only max-length empty-to-end on-change] :rest props}] (let [;; NOTE: we use mfu/bean here for transparently handle ;; options provide as clojure data structures or javascript ;; plain objects and lists. @@ -106,6 +110,7 @@ (reset! selected-id* id) (reset! is-open* false) (reset! focused-id* nil) + (mf/set-ref-val! value-ref nil) (when (fn? on-change) (on-change id)))))) @@ -139,6 +144,9 @@ (if (some? selected-option) selected-id input-value))] + ;; The value was delivered, so an unmount must not + ;; report it again. + (mf/set-ref-val! value-ref nil) (on-change value)))))))) on-input-click @@ -202,6 +210,7 @@ (reset! selected-id* focused-id) (reset! is-open* false) (reset! focused-id* nil) + (mf/set-ref-val! value-ref nil) (dom/blur! (mf/ref-val input-ref)) (when (and (fn? on-change) (some? focused-id)) @@ -239,8 +248,9 @@ (d/seek #(= selected-id (get % :id)) options))) icon - (when selected-option - (get selected-option :icon)) + (or (when selected-option + (get selected-option :icon)) + icon) avatar (when selected-option diff --git a/frontend/src/app/main/ui/ds/controls/combobox.mdx b/frontend/src/app/main/ui/ds/controls/combobox.mdx index 86b8e11119..c4b39840c1 100644 --- a/frontend/src/app/main/ui/ds/controls/combobox.mdx +++ b/frontend/src/app/main/ui/ds/controls/combobox.mdx @@ -53,6 +53,18 @@ These are available in the `app.main.ds.foundations.assets.icon` namespace. ]}] ``` +The `combobox*` itself also accepts an optional `icon`. It shows at the start +of the input when the selected option has no icon of its own, for example when +the user types a value that matches no option. + +```clj +[:> combobox* + {:icon i/expand + :default-selected "1" + :options [{ :label "1x" :id "1" } + { :label "2x" :id "2" }]}] +``` + ### Avatars diff --git a/frontend/src/app/main/ui/ds/controls/combobox.stories.jsx b/frontend/src/app/main/ui/ds/controls/combobox.stories.jsx index 63882556a9..3794d6e9f5 100644 --- a/frontend/src/app/main/ui/ds/controls/combobox.stories.jsx +++ b/frontend/src/app/main/ui/ds/controls/combobox.stories.jsx @@ -72,6 +72,12 @@ export const WithIcons = { }, }; +export const WithIcon = { + args: { + icon: "expand", + }, +}; + export const EmptyToEnd = { args: { emptyToEnd: true, diff --git a/frontend/src/app/main/ui/ds/foundations/assets/icon.cljs b/frontend/src/app/main/ui/ds/foundations/assets/icon.cljs index 4c7743a20a..d30a77c821 100644 --- a/frontend/src/app/main/ui/ds/foundations/assets/icon.cljs +++ b/frontend/src/app/main/ui/ds/foundations/assets/icon.cljs @@ -253,6 +253,7 @@ (def ^:icon-id rotation "rotation") (def ^:icon-id row "row") (def ^:icon-id row-reverse "row-reverse") +(def ^:icon-id scale "scale") (def ^:icon-id search "search") (def ^:icon-id separate-nodes "separate-nodes") (def ^:icon-id settings "settings") diff --git a/frontend/src/app/main/ui/exports/preview.cljs b/frontend/src/app/main/ui/exports/preview.cljs new file mode 100644 index 0000000000..493726c8df --- /dev/null +++ b/frontend/src/app/main/ui/exports/preview.cljs @@ -0,0 +1,178 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.main.ui.exports.preview + (:require-macros [app.main.style :as stl]) + (:require + [app.common.files.helpers :as cfh] + [app.common.geom.shapes.bounds :as gsb] + [app.common.logging :as log] + [app.main.data.exports.assets :as de] + [app.main.data.exports.wasm :as wasm.exports] + [app.main.features :as features] + [app.main.refs :as refs] + [app.main.render :as render] + [app.main.ui.components.title-bar :refer [title-bar*]] + [app.main.ui.ds.buttons.button :refer [button*]] + [app.main.ui.ds.controls.select :refer [select*]] + [app.main.ui.hooks :as hooks] + [app.render-wasm.api :as wasm.api] + [app.util.i18n :refer [tr]] + [app.util.strings :as ust] + [app.util.timers :as timers] + [app.util.webapi :as wapi] + [cuerdas.core :as str] + [okulary.core :as l] + [rumext.v2 :as mf])) + +;; Keep in sync with --export-preview-size in preview.scss. +(def ^:private preview-size 240) + +(defn preview-settings + "Keep raster previews small without changing the displayed export dimensions. + SVG and PDF use a raster approximation at their natural size." + [{:keys [width height]} {:keys [type scale]}] + (let [raster? (contains? de/raster-export-types type) + scale (if raster? (or scale 1) 1)] + {:type (if raster? type :png) + :scale (min scale (/ preview-size (max 1 width height))) + :width (js/Math.ceil (* width scale)) + :height (js/Math.ceil (* height scale))})) + +(defn preview-bounds + "Use the same bounds as the active export renderer." + [objects object wasm?] + (or (when (and wasm? (wasm.api/initialized?)) + (wasm.api/get-shape-extrect (:id object))) + (let [bounds (gsb/get-object-bounds objects object {:ignore-margin? false}) + rounded-size #(js/parseFloat (ust/format-precision % render/viewbox-decimal-precision))] + (-> bounds + (update :width rounded-size) + (update :height rounded-size))))) + +(mf/defc wasm-preview* + {::mf/private true} + [{:keys [objects object-id export bounds]}] + (let [image* (mf/use-state nil) + image @image* + retry* (mf/use-state 0) + retry @retry* + width (:width bounds) + height (:height bounds) + retry-fn (mf/use-fn #(swap! retry* inc))] + ;; Keep the current image alive while the next render is debounced. + ;; Release its URI only after replacement or when the preview unmounts. + (mf/use-effect + (mf/deps (:uri image)) + (fn [] + (when-let [uri (:uri image)] + #(wapi/revoke-uri uri)))) + (mf/use-effect + (mf/deps objects object-id export width height retry) + (fn [] + (let [task (timers/schedule + 200 + (fn [] + (try + (if (wasm.api/initialized?) + (let [settings (preview-settings {:width width :height height} export) + uri (wasm.exports/export-image-uri + (assoc settings :object-id object-id))] + (reset! image* {:uri uri})) + (reset! image* {:error true})) + (catch :default cause + (log/error :hint "export preview failed" :cause cause) + (reset! image* {:error true})))))] + #(timers/dispose! task)))) + (cond + (:uri image) + [:img {:class (stl/css :image) + :src (:uri image) + :alt (tr "workspace.options.export.preview-image")}] + + (:error image) + [:div {:class (stl/css :status) :role "status"} + [:span (tr "workspace.options.export.preview-error")] + [:> button* {:variant "secondary" :on-click retry-fn} (tr "labels.retry")]] + + :else + [:span {:class (stl/css :status) :role "status"} (tr "labels.loading")]))) + +(mf/defc preview-content* + {::mf/private true} + [{:keys [object-id exports from page-id]}] + (let [index* (mf/use-state 0) + index (min @index* (dec (count exports))) + export (nth exports index) + options (mf/with-memo [exports] + (mapv (fn [index export] + {:id (str index) + :label (str (inc index) ". " (str/upper (name (:type export))) + (when (contains? de/raster-export-types (:type export)) + (str " · " (:scale export) "x")) + (when (seq (:suffix export)) (str " · " (:suffix export))))}) + (range (count exports)) exports)) + wasm-enabled (features/use-feature "render-wasm/v1") + wasm? (and (= from :workspace) wasm-enabled) + objects-ref (mf/with-memo [object-id from page-id] + (l/derived + (fn [source] + (let [objects (if (= from :viewer) + (get-in source [:pages page-id :objects]) + source)] + (select-keys objects + (cons object-id (cfh/get-children-ids objects object-id))))) + (if (= from :viewer) refs/viewer-data refs/workspace-page-objects) + =)) + objects (mf/deref objects-ref) + object (get objects object-id) + bounds (mf/with-memo [objects object wasm?] + (when object + (preview-bounds objects object wasm?))) + on-change (mf/use-fn + (fn [value] + (reset! index* (js/parseInt value 10))))] + (when object + (let [settings (preview-settings bounds export)] + [:div {:class (stl/css :content)} + (when (> (count exports) 1) + [:> select* {:aria-label (tr "workspace.options.export.preview-preset") + :default-selected (str index) + :options options + :on-change on-change}]) + [:div {:class (stl/css-case :canvas true :opaque (= :jpeg (:type export))) + :style {"--export-preview-ratio" (/ (max 1 (:width bounds)) + (max 1 (:height bounds)))}} + (if wasm? + [:> wasm-preview* {:objects objects + :object-id object-id + :export export + :bounds bounds}] + [:div {:class (stl/css :svg) + :role "img" + :aria-label (tr "workspace.options.export.preview-image")} + [:& render/object-svg {:objects objects + :object-id object-id + :class (stl/css :svg-image)}]])] + [:div {:class (stl/css :dimensions)} + (str (:width settings) " × " (:height settings) " · " (str/upper (name (:type export))))]])))) + +(mf/defc export-preview* + [{:keys [object-id exports from page-id] :or {from :workspace}}] + (let [open* (hooks/use-persisted-state ::open false) + open @open* + panel-id (str "export-preview-" object-id) + toggle-fn (mf/use-fn #(swap! open* not))] + [:div {:class (stl/css :preview)} + [:> title-bar* {:title (tr "workspace.options.export.preview") + :collapsable true + :collapsed (not open) + :on-collapsed toggle-fn + :aria-expanded open + :aria-controls panel-id}] + (when open + [:div {:id panel-id} + [:> preview-content* {:object-id object-id :exports exports :from from :page-id page-id}]])])) diff --git a/frontend/src/app/main/ui/exports/preview.scss b/frontend/src/app/main/ui/exports/preview.scss new file mode 100644 index 0000000000..f5c27b4410 --- /dev/null +++ b/frontend/src/app/main/ui/exports/preview.scss @@ -0,0 +1,82 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) KALEIDOS SUBSIDIARY SL + +@use "ds/typography.scss" as t; +@use "ds/_borders.scss" as *; +@use "ds/_utils.scss" as *; + +.preview { + // Keep in sync with preview-size in preview.cljs. + --export-preview-size: #{px2rem(240)}; + + grid-column: 1 / -1; + min-inline-size: 0; +} + +.content { + display: flex; + flex-direction: column; + gap: var(--sp-s); + padding: var(--sp-s); +} + +.canvas { + display: flex; + justify-content: center; + align-items: center; + align-self: center; + inline-size: min(100%, calc(var(--export-preview-size) * var(--export-preview-ratio))); + aspect-ratio: var(--export-preview-ratio); + overflow: hidden; + background: repeating-conic-gradient(rgb(212 212 212) 0% 25%, rgb(255 255 255) 0% 50%); + background-size: var(--sp-s) var(--sp-s); + border: $b-1 solid var(--color-background-quaternary); + border-radius: $br-4; +} + +.opaque { + background: rgb(255 255 255); +} + +.image { + display: block; + inline-size: 100%; + block-size: 100%; + object-fit: contain; +} + +.svg { + display: flex; + justify-content: center; + inline-size: 100%; + block-size: 100%; + pointer-events: none; +} + +.svg-image { + inline-size: 100%; + block-size: 100%; +} + +.dimensions, +.status { + @include t.use-typography("body-small"); + + color: var(--color-foreground-secondary); +} + +.dimensions { + text-align: center; +} + +.status { + display: flex; + flex-direction: column; + gap: var(--sp-s); + padding: var(--sp-m); + text-align: center; + background: var(--color-background-primary); +} diff --git a/frontend/src/app/main/ui/inspect/attributes.cljs b/frontend/src/app/main/ui/inspect/attributes.cljs index 2818edf9c1..4c96828d8a 100644 --- a/frontend/src/app/main/ui/inspect/attributes.cljs +++ b/frontend/src/app/main/ui/inspect/attributes.cljs @@ -87,4 +87,5 @@ :type type :page-id page-id :file-id file-id + :from from :share-id share-id}]])) diff --git a/frontend/src/app/main/ui/inspect/attributes/text.cljs b/frontend/src/app/main/ui/inspect/attributes/text.cljs index e6434bd722..799f8e0db7 100644 --- a/frontend/src/app/main/ui/inspect/attributes/text.cljs +++ b/frontend/src/app/main/ui/inspect/attributes/text.cljs @@ -129,6 +129,14 @@ "uppercase" (tr "inspect.attributes.typography.text-transform.uppercase") "unset" (tr "inspect.attributes.typography.text-transform.unset"))]]]]) + (when (types.text/list-paragraph? style) + [:div {:class (stl/css :text-row)} + [:div {:class (stl/css :global/attr-label)} "List Style"] + [:div {:class (stl/css :global/attr-value)} + [:> copy-button* {:data (copy-style-data style :list-style-type)} + [:div {:class (stl/css :button-children)} + (tr "inspect.attributes.typography.list-style-type.disc")]]]]) + [:> copy-button* {:data (str/trim text) :class (stl/css :attributes-content-row)} [:span {:class (stl/css :content) diff --git a/frontend/src/app/main/ui/inspect/exports.cljs b/frontend/src/app/main/ui/inspect/exports.cljs index c70e37ff05..fa8a54162c 100644 --- a/frontend/src/app/main/ui/inspect/exports.cljs +++ b/frontend/src/app/main/ui/inspect/exports.cljs @@ -17,6 +17,7 @@ [app.main.ui.ds.buttons.button :refer [button*]] [app.main.ui.ds.buttons.icon-button :refer [icon-button*]] [app.main.ui.ds.foundations.assets.icon :as i] + [app.main.ui.exports.preview :refer [export-preview*]] [app.util.dom :as dom] [app.util.i18n :refer [tr c]] [app.util.keyboard :as kbd] @@ -28,7 +29,7 @@ (mf/defc exports {::mf/wrap [#(mf/memo % =)]} - [{:keys [shapes page-id file-id share-id type] :as props}] + [{:keys [shapes page-id file-id share-id type from] :or {from :viewer} :as props}] (let [exports (mf/use-state []) xstate (mf/deref refs/export) vstate (mf/deref refs/viewer-data) @@ -44,7 +45,7 @@ scale-enabled? (mf/use-callback (fn [export] - (#{:png :jpeg :webp} (:type export)))) + (contains? de/raster-export-types (:type export)))) in-progress? (:in-progress xstate) @@ -218,5 +219,10 @@ :disabled in-progress?} (if in-progress? (tr "workspace.options.exporting-object") - (tr "workspace.options.export-object" (c (count shapes))))])])) - + (tr "workspace.options.export-object" (c (count shapes))))]) + (when (and (= 1 (count shapes)) (seq @exports)) + [:> export-preview* {:key (-> shapes first :id) + :object-id (-> shapes first :id) + :exports @exports + :from from + :page-id page-id}])])) diff --git a/frontend/src/app/main/ui/inspect/styles.cljs b/frontend/src/app/main/ui/inspect/styles.cljs index 8eddf353e8..96a54a7b47 100644 --- a/frontend/src/app/main/ui/inspect/styles.cljs +++ b/frontend/src/app/main/ui/inspect/styles.cljs @@ -285,4 +285,5 @@ :type type :page-id page-id :file-id file-id + :from from :share-id share-id}]]])) diff --git a/frontend/src/app/main/ui/inspect/styles/panels/text.cljs b/frontend/src/app/main/ui/inspect/styles/panels/text.cljs index f68c764913..a2491d78f7 100644 --- a/frontend/src/app/main/ui/inspect/styles/panels/text.cljs +++ b/frontend/src/app/main/ui/inspect/styles/panels/text.cljs @@ -180,6 +180,12 @@ :property (str "text-transform: " (:text-transform style) ";") :copiable true}])) + (when (txt/list-paragraph? style) + [:> properties-row* {:term "List Style" + :detail (:list-style-type style) + :property (str "list-style-type: " (:list-style-type style) ";") + :copiable true}]) + [:pre {:class (stl/css :text-content-wrapper) :role "presentation"} [:> property-detail-copiable* {:copied copied diff --git a/frontend/src/app/main/ui/routes.cljs b/frontend/src/app/main/ui/routes.cljs index fa9926911b..618e4ed99f 100644 --- a/frontend/src/app/main/ui/routes.cljs +++ b/frontend/src/app/main/ui/routes.cljs @@ -164,7 +164,7 @@ active. If so, calls :check-nitrate-sso and either proceeds with navigation or redirects to the SSO provider URL. Successful checks are cached for five minutes per profile and team; redirect results are never cached." - [match send-event-info? url] + [match send-event-info? query-string] (let [route-name (name (get-in match [:data :name])) relevant? (and (contains? cf/flags :admin-console) (or (str/starts-with? route-name "dashboard") @@ -188,7 +188,7 @@ (navigate) :else - (->> (rp/cmd! :check-nitrate-sso {:team-id team-id :url url}) + (->> (rp/cmd! :check-nitrate-sso {:team-id team-id :query-string query-string}) (rx/subs! (fn [{:keys [authorized redirect-uri]}] (if authorized @@ -206,7 +206,7 @@ and deliberately do NOT proceed with normal navigation: emitting `rt/navigated` would clear the exception that was just assigned. Otherwise, delegate to `check-sso-and-navigate`." - [match send-event-info? url] + [match send-event-info? query-string] (let [route-name (name (get-in match [:data :name])) sso-error? (some? (get-in match [:query-params :sso-error])) organization-id (some-> (get-in match [:query-params :organization-id]) uuid/parse*) @@ -223,7 +223,7 @@ :team-id team-id :is-workspace is-workspace? :is-dashboard is-dashboard?})) - (check-sso-and-navigate match send-event-info? url)))) + (check-sso-and-navigate match send-event-info? query-string)))) (declare on-query-navigate) @@ -260,9 +260,7 @@ (defn- on-query-navigate [router token send-event-info?] - (let [token-query (if (str/starts-with? (or token "") "?") - (subs token 1) - (or token "")) + (let [token-query (rt/token->query-string token) query-params (u/query-string->map token-query) ;; Without a `screen` the token targets the app root; its params ;; (e.g. `template`) are forwarded to the dashboard or kept for @@ -270,7 +268,7 @@ root? (nil? (rt/get-query-param query-params :screen)) match (rt/match router token)] (if (some? match) - (handle-sso-error-and-navigate match send-event-info? (rt/get-current-href)) + (handle-sso-error-and-navigate match send-event-info? (rt/get-current-query-string)) ;; We just recheck with an additional profile request; this ;; avoids some race conditions that causes unexpected redirects diff --git a/frontend/src/app/main/ui/settings/options.cljs b/frontend/src/app/main/ui/settings/options.cljs index b841f3567d..00898df09c 100644 --- a/frontend/src/app/main/ui/settings/options.cljs +++ b/frontend/src/app/main/ui/settings/options.cljs @@ -20,6 +20,7 @@ [app.main.ui.ds.foundations.typography :as t] [app.main.ui.ds.foundations.typography.heading :refer [heading*]] [app.main.ui.ds.foundations.typography.text :refer [text*]] + [app.render-wasm.api.webgl :as webgl] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] [app.util.theme :as theme] @@ -101,10 +102,11 @@ handle-render-change (mf/use-fn (fn [enabled?] - (st/emit! (ev/event {::ev/name (if enabled? - "enable-webgl-rendering" - "disable-webgl-rendering") - ::ev/origin "settings"}) + (st/emit! (ev/event (assoc (webgl/get-environment) + ::ev/name (if enabled? + "enable-webgl-rendering" + "disable-webgl-rendering") + ::ev/origin "settings")) (du/update-profile-props {:renderer (if enabled? :wasm :svg)}) (ntf/success (if enabled? (tr "webgl.toast.webgl-render-enabled") diff --git a/frontend/src/app/main/ui/shapes/text/html_text.cljs b/frontend/src/app/main/ui/shapes/text/html_text.cljs index 299ee49de7..7ffd80c38c 100644 --- a/frontend/src/app/main/ui/shapes/text/html_text.cljs +++ b/frontend/src/app/main/ui/shapes/text/html_text.cljs @@ -9,6 +9,7 @@ [app.common.data :as d] [app.common.data.macros :as dm] [app.common.text :as legacy.txt] + [app.common.types.text :as txt] [app.main.ui.shapes.text.styles :as sts] [rumext.v2 :as mf])) @@ -39,17 +40,48 @@ [:div.paragraph-set {:style style :class class} children])) (mf/defc render-paragraph* - [{:keys [node children shape is-code]}] + [{:keys [node children shape is-code is-list-item]}] (let [style (when-not is-code (sts/generate-paragraph-styles shape node)) class (when is-code (:$id node)) dir (:text-direction node "auto")] - [:p.paragraph {:style style :dir dir :class class} children])) + (if is-list-item + [:li {:style style :dir dir :class class} children] + [:p.paragraph {:style style :dir dir :class class} children]))) + +(declare render-node*) + +(defn- render-children + "Child elements of `parent`. In code mode, runs of list paragraphs are + grouped into a `
        ` of `
      • `." + [props parent is-code] + (let [render-child + (fn [[index child-node] is-list-item] + (mf/html + [:> render-node* + (mf/spread-props props + {:node child-node + :parent parent + :index index + :key index + :is-list-item is-list-item})])) + + children (d/enumerate (:children parent))] + (if (and is-code (= "paragraph-set" (:type parent))) + (mapcat (fn [run] + (if (txt/list-paragraph? (second (first run))) + [(mf/html + [:ul {:key (dm/str "list-" (first (first run)))} + (for [child run] + (render-child child true))])] + (map #(render-child % false) run))) + (partition-by (comp txt/list-paragraph? second) children)) + (map #(render-child % false) children)))) ;; -- Text nodes (mf/defc render-node* {::mf/props :obj} [{:keys [node is-code] :as props}] - (let [{:keys [type text children] :as parent} node] + (let [{:keys [type text]} node] (if (string? text) [:> render-text* props] (let [component (case type @@ -59,14 +91,7 @@ nil)] (when component [:> component props - (for [[index child-node] (d/enumerate children)] - [:> render-node* - (mf/spread-props props - {:node child-node - :parent parent - :index index - :key index - :is-code is-code})])]))))) + (render-children props node is-code)]))))) (mf/defc text-shape* {::mf/forward-ref true} diff --git a/frontend/src/app/main/ui/static.cljs b/frontend/src/app/main/ui/static.cljs index 588dd0f970..2b58bb8db2 100644 --- a/frontend/src/app/main/ui/static.cljs +++ b/frontend/src/app/main/ui/static.cljs @@ -524,12 +524,14 @@ SSO exchange with the identity provider fails." {::mf/private true} [{:keys [organization-id team-id profile is-workspace is-dashboard organization-name]}] - (let [clean-url + (let [clean-token (mf/with-memo [] - (-> (rt/get-current-href) + (-> (rt/get-current-token) (dom/remove-query-param :sso-error) (dom/remove-query-param :organization-id))) + clean-url (rt/token->url clean-token) + _ (mf/with-effect [] ;; Consume the marker once: scrub it from the URL bar so a ;; browser refresh doesn't keep re-showing this dialog. @@ -545,14 +547,14 @@ on-retry (mf/use-fn - (mf/deps organization-id team-id clean-url) + (mf/deps organization-id team-id clean-token) (fn [] (st/emit! (rt/assign-exception nil)) (if (or team-id organization-id) ;; Retry with team-id and/or organization-id to trigger SSO check (st/emit! (dnt/retry-organization-sso {:team-id team-id :organization-id organization-id - :dest-url clean-url})) + :query-string (rt/token->query-string clean-token)})) ;; Fallback: just navigate to clean URL (st/emit! (rt/nav-raw :uri clean-url)))))] diff --git a/frontend/src/app/main/ui/viewer/header.cljs b/frontend/src/app/main/ui/viewer/header.cljs index a433cc5c7f..8e9c45f92c 100644 --- a/frontend/src/app/main/ui/viewer/header.cljs +++ b/frontend/src/app/main/ui/viewer/header.cljs @@ -35,6 +35,11 @@ (dm/get-in state [:viewer-local :fullscreen?])) st/state)) +(def ^:private share-modal-open-ref + (l/derived (fn [state] + (= :share-link (dm/get-in state [::modal/modal :type]))) + st/state)) + (defn open-login-dialog [] (modal/show! :login-register {})) @@ -131,6 +136,15 @@ (mf/defc header-options [{:keys [section zoom page file index permissions interactions-mode share]}] (let [fullscreen? (mf/deref fullscreen-ref) + share-open? (mf/deref share-modal-open-ref) + + share-btn-ref (mf/use-ref nil) + share-was-open* (mf/use-ref false) + + set-share-btn-ref + (mf/use-fn + (fn [node] + (mf/set-ref-val! share-btn-ref node))) toggle-fullscreen (mf/use-fn @@ -144,11 +158,19 @@ open-share-dialog (mf/use-fn - (mf/deps page) + (mf/deps page file) (fn [] (modal/show! :share-link {:page page :file file}) (modal/disallow-click-outside!))) + toggle-share-dialog + (mf/use-fn + (mf/deps open-share-dialog share-open?) + (fn [] + (if share-open? + (modal/hide!) + (open-share-dialog)))) + handle-increase (mf/use-fn #(st/emit! dv/increase-zoom)) @@ -168,6 +190,12 @@ handle-zoom-fit (mf/use-fn #(st/emit! dv/zoom-to-fit))] + (mf/with-effect [share-open?] + (let [was-open? (mf/ref-val share-was-open*)] + (when (and was-open? (not share-open?)) + (some-> (mf/ref-val share-btn-ref) dom/focus!)) + (mf/set-ref-val! share-was-open* share-open?))) + (mf/with-effect [permissions share] (when (and (:in-team permissions) @@ -211,7 +239,11 @@ (when (:in-team permissions) [:> button* {:variant "primary" :class (stl/css :share-btn) - :on-click open-share-dialog} + :on-ref set-share-btn-ref + :aria-expanded share-open? + :aria-haspopup "dialog" + :data-allow-click-modal "share-link" + :on-click toggle-share-dialog} (tr "labels.share")]) (when-not (:is-logged permissions) diff --git a/frontend/src/app/main/ui/viewer/share_link.cljs b/frontend/src/app/main/ui/viewer/share_link.cljs index 6d43e1c60b..b063ee9af8 100644 --- a/frontend/src/app/main/ui/viewer/share_link.cljs +++ b/frontend/src/app/main/ui/viewer/share_link.cljs @@ -10,7 +10,6 @@ [app.common.data :as d] [app.common.data.macros :as dm] [app.common.logging :as log] - [app.common.uuid :as uuid] [app.main.data.common :as dc] [app.main.data.event :as ev] [app.main.data.modal :as modal] @@ -18,113 +17,259 @@ [app.main.refs :as refs] [app.main.router :as rt] [app.main.store :as st] - [app.main.ui.components.select :refer [select]] [app.main.ui.ds.buttons.button :refer [button*]] [app.main.ui.ds.buttons.icon-button :refer [icon-button*]] + [app.main.ui.ds.controls.checkbox :refer [checkbox*]] + [app.main.ui.ds.controls.select :refer [select*]] + [app.main.ui.ds.controls.utilities.input-field :refer [input-field*]] [app.main.ui.ds.foundations.assets.icon :refer [icon*] :as i] - [app.main.ui.icons :as deprecated-icon] [app.util.clipboard :as clipboard] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] + [app.util.keyboard :as kbd] [rumext.v2 :as mf])) (log/set-level! :warn) -(defn- prepare-params +(defn- link-params [{:keys [pages who-comment who-inspect]}] {:pages pages :who-comment who-comment :who-inspect who-inspect}) -(mf/defc share-link-dialog - {::mf/register modal/components - ::mf/register-as :share-link - ::mf/wrap-props false} - [{:keys [file page]}] - (let [current-page page - current-page-id (:id page) - slinks (mf/deref refs/share-links) - router (mf/deref refs/router) - route (mf/deref refs/route) - zoom-type (mf/deref refs/viewer-zoom-type) - page-ids (dm/get-in file [:data :pages]) +(defn- link->url + [router route page-ids zoom-type link] + (let [page-id (d/seek #(contains? (:pages link) %) page-ids) + params (-> (:query-params route) + (assoc :share-id (:id link)) + (assoc :page-id page-id) + (assoc :index "0")) + params (if (nil? zoom-type) + (dissoc params :zoom) + (assoc params :zoom zoom-type))] + (rt/resolve-uri router :viewer params))) - perms-visible* (mf/use-state false) - perms-visible? (deref perms-visible*) +(defn- copy-url-to-clipboard + [url] + (clipboard/to-clipboard url) + (st/emit! (ntf/show {:level :info + :type :toast + :content (tr "common.share-link.link-copied-success") + :timeout 1000}) + (ev/event {::ev/name "copy-share-link" + ::ev/origin "viewer"}))) - confirm* (mf/use-state false) - confirm? (deref confirm*) +(defn- use-collapsible + [] + (let [expanded* (mf/use-state false) + expanded (deref expanded*) + content-id (mf/use-id) + toggle (mf/use-fn + #(swap! expanded* not))] + {:expanded expanded + :content-id content-id + :toggle toggle})) - options* (mf/use-state - {:pages-mode "current" - :all-pages false - :pages #{(:id page)} - :who-comment "team" - :who-inspect "team"}) - options (deref options*) +(mf/defc collapsible-panel* + {::mf/private true} + [{:keys [class title children]}] + (let [{:keys [expanded content-id toggle]} (use-collapsible)] + [:div {:class class} + [:button {:class (stl/css :collapsible-title) + :type "button" + :aria-expanded expanded + :aria-controls (when expanded content-id) + :on-click toggle} + [:> icon* {:icon-id (if expanded i/arrow-down i/arrow-right) + :class (stl/css :collapsible-title-icon)}] + title] - current-link - (mf/with-memo [slinks options page-ids] - (let [{:keys [pages who-comment who-inspect] :as params} (prepare-params options) - slink (d/seek #(and (= (:who-inspect %) who-inspect) - (= (:who-comment %) who-comment) - (= (:pages %) pages)) - slinks)] - (when slink - (let [page-id (d/seek #(contains? (:pages slink) %) page-ids) - params (-> (:query-params route) - (assoc :share-id (:id slink)) - (assoc :page-id page-id) - (assoc :index "0")) - params (if (nil? zoom-type) - (dissoc params :zoom) - (assoc params :zoom zoom-type)) + (when ^boolean expanded + [:div {:id content-id} + children])])) - href (rt/resolve-uri router :viewer params)] - href)))) +(mf/defc permissions-select-row* + {::mf/private true} + [{:keys [label value options on-change]}] + [:div {:class (stl/css :permissions-select-row)} + [:div {:class (stl/css :permissions-label)} label] + [:div {:class (stl/css :permissions-control)} + [:> select* {:default-selected (dm/str value) + :options options + :has-portal true + :aria-label label + :on-change on-change}]]]) - on-close +(mf/defc permissions-page-selection* + {::mf/private true} + [{:keys [file page selected on-toggle-all on-checked-page]}] + (let [pages (->> (get-in file [:data :pages]) + (map #(get-in file [:data :pages-index %]))) + page-id (:id page) + all-selected? (= selected (into #{} (map :id) pages))] + + [:div {:class (stl/css :permissions-pages-field)} + [:div {:class (stl/css :permissions-label)} + (tr "common.share-link.permissions-pages")] + [:div {:class (stl/css :permissions-control)} + (if (= 1 (count pages)) + [:div {:class (stl/css :permissions-page-row)} + [:> checkbox* {:id (dm/str "page-" page-id) + :class (stl/css :permissions-checkbox) + :label (:name page) + :checked true + :disabled true}] + [:span {:class (stl/css :permissions-current-tag)} + (tr "common.share-link.current-tag")]] + + [:* + [:div {:class (stl/css :permissions-select-all-row)} + [:> checkbox* {:id "view-all" + :class (stl/css :permissions-checkbox) + :label (tr "common.share-link.view-all") + :checked all-selected? + :on-change on-toggle-all}] + [:span {:class (stl/css :permissions-page-count)} + (tr "common.share-link.page-shared" (i18n/c (count selected)))]] + + [:ul {:class (stl/css :permissions-pages-selection)} + (for [{:keys [id name]} pages] + (let [checked? (contains? selected id) + only-selected? (and checked? (= 1 (count selected)))] + [:li {:class (stl/css :permissions-page-row) + :key (dm/str id)} + [:> checkbox* {:id (dm/str "page-" id) + :class (stl/css :permissions-checkbox) + :label name + :checked checked? + :disabled only-selected? + :on-change #(on-checked-page id %)}] + (when (= page-id id) + [:span {:class (stl/css :permissions-current-tag)} + (tr "common.share-link.current-tag")])]))]])]])) + +(mf/defc permissions-section* + {::mf/private true} + [{:keys [file page link-options on-toggle-all on-checked-page + on-comment-change on-inspect-change]}] + (let [locale (mf/deref i18n/locale) + selected (:pages link-options) + permissions (mf/with-memo [locale] + [{:id "team" :label (tr "common.share-link.team-members")} + {:id "all" :label (tr "common.share-link.all-users")}])] + + [:> collapsible-panel* {:class (stl/css :permissions-section) + :title (tr "common.share-link.manage-ops")} + [:div {:class (stl/css :permissions-content)} + [:> permissions-page-selection* {:file file + :page page + :selected selected + :on-toggle-all on-toggle-all + :on-checked-page on-checked-page}] + + [:> permissions-select-row* {:label (tr "common.share-link.permissions-can-comment") + :value (:who-comment link-options) + :options permissions + :on-change on-comment-change}] + + [:> permissions-select-row* {:label (tr "common.share-link.permissions-can-inspect") + :value (:who-inspect link-options) + :options permissions + :on-change on-inspect-change}]]])) + +(mf/defc destroy-link-confirmation* + {::mf/private true} + [{:keys [on-cancel on-destroy]}] + (let [cancel-ref (mf/use-ref nil) + + on-cancel-ref + (mf/use-fn + (fn [node] + (mf/set-ref-val! cancel-ref node))) + + on-key-down (fn [event] - (dom/prevent-default event) - (st/emit! (modal/hide)) - (modal/disallow-click-outside!)) + (when (kbd/esc? event) + (dom/stop-propagation event) + (on-cancel event)))] + + (mf/with-effect [] + (some-> (mf/ref-val cancel-ref) dom/focus!)) + + [:div {:class (stl/css :share-link-confirm) + :role "alertdialog" + :aria-modal true + :aria-labelledby "share-link-title" + :aria-describedby "share-link-confirm-description" + :tab-index "-1" + :on-key-down on-key-down} + [:div {:id "share-link-confirm-description" + :class (stl/css :share-link-confirm-description)} + (tr "common.share-link.confirm-deletion-link-description")] + [:div {:class (stl/css :share-link-confirm-actions)} + [:> button* {:variant "secondary" + :type "button" + :on-ref on-cancel-ref + :on-click on-cancel} + (tr "labels.cancel")] + [:> button* {:variant "destructive" + :type "button" + :on-click on-destroy} + (tr "common.share-link.destroy-link")]]])) + +(mf/defc share-link-content* + {::mf/private true} + [{:keys [file page on-request-destroy on-destroy-ref]}] + (let [share-links (mf/deref refs/share-links) + router (mf/deref refs/router) + route (mf/deref refs/route) + zoom-type (mf/deref refs/viewer-zoom-type) + + page-ids (dm/get-in file [:data :pages]) + all-page-ids (into #{} page-ids) + + link-options* (mf/use-state + {:pages #{(:id page)} + :who-comment "team" + :who-inspect "team"}) + link-options (deref link-options*) + + current-share-link + (mf/with-memo [share-links link-options] + (let [{:keys [pages who-comment who-inspect]} (link-params link-options)] + (d/seek #(and (= (:who-inspect %) who-inspect) + (= (:who-comment %) who-comment) + (= (:pages %) pages)) + share-links))) + + current-share-url + (mf/with-memo [current-share-link page-ids route router zoom-type] + (when (some? current-share-link) + (link->url router route page-ids zoom-type current-share-link))) on-toggle-all (fn [_event] - (reset! confirm* false) - (swap! options* - (fn [state] - (if (true? (:all-pages state)) - (-> state - (assoc :all-pages false) - (assoc :pages #{(:id page)})) - (-> state - (assoc :all-pages true) - (assoc :pages (into #{} (get-in file [:data :pages])))))))) + (swap! link-options* + (fn [{:keys [pages] :as state}] + (if (= pages all-page-ids) + (assoc state :pages #{(:id page)}) + (assoc state :pages all-page-ids))))) - on-mark-checked-page - (fn [event] - (let [target (dom/get-target event) - checked? (dom/checked? target) - page-id (uuid/parse (dom/get-data target "page-id")) - dif-pages? (not= page-id (first (:pages options))) - no-one-page (< 1 (count (:pages options))) - should-change? (or ^boolean no-one-page - ^boolean dif-pages?)] - (when ^boolean should-change? - (reset! confirm* false) - (swap! options* - (fn [{:keys [pages] :as state}] - (let [pages (if checked? - (conj pages page-id) - (disj pages page-id))] - (-> state - (assoc :pages pages) - (assoc :all-pages (= (count pages) (count page-ids)))))))))) + on-checked-page + (fn [page-id event] + (let [checked? (dom/checked? (dom/get-target event))] + (swap! link-options* + (fn [{:keys [pages] :as state}] + (let [pages (if checked? + (conj pages page-id) + (disj pages page-id))] + (if (seq pages) + (assoc state :pages pages) + state)))))) - create-link + on-create-link (fn [_] - (let [params (prepare-params options) + (let [params (link-params link-options) params (assoc params :file-id (:id file))] (st/emit! (dc/create-share-link params) (ev/event {::ev/name "create-share-link" @@ -132,192 +277,142 @@ :can-comment (:who-comment params) :can-inspect-code (:who-inspect params)})))) - copy-link - (fn [_] - (clipboard/to-clipboard current-link) - (st/emit! (ntf/show {:level :info - :type :toast - :content (tr "common.share-link.link-copied-success") - :timeout 1000}) - (ev/event {::ev/name "copy-share-link" - ::ev/origin "viewer"}))) + on-copy-current-share-url + (mf/use-fn + (mf/deps current-share-url) + (fn [_] + (copy-url-to-clipboard current-share-url))) - try-delete-link - (fn [_] - (reset! confirm* true)) - - delete-link - (fn [_] - (let [params (prepare-params options) - slink (d/seek #(= (:flags %) (:flags params)) slinks)] - (reset! confirm* false) - (st/emit! (dc/delete-share-link slink)))) - - toggle-perms-visibility - (fn [_] - (swap! perms-visible* not)) + on-destroy-current-share-link + (mf/use-fn + (mf/deps current-share-link on-request-destroy) + (fn [_] + (on-request-destroy current-share-link))) on-inspect-change (fn [value] - (reset! confirm* false) - (swap! options* assoc :who-inspect value)) + (swap! link-options* assoc :who-inspect value)) on-comment-change (fn [value] - (reset! confirm* false) - (swap! options* assoc :who-comment value))] + (swap! link-options* assoc :who-comment value))] - [:div {:class (stl/css :share-modal)} - [:div {:class (stl/css :share-link-dialog)} + [:div {:class (stl/css :share-link-content)} + [:div {:class (stl/css :share-link-form)} + (when (some? current-share-url) + [:div {:class (stl/css :share-link-url)} + [:> input-field* {:id "share-link-url-input" + :type "text" + :class (stl/css :share-link-url-input) + :value (or current-share-url "") + :placeholder (tr "common.share-link.placeholder") + :aria-label (tr "common.share-link.placeholder") + :read-only true}] + [:> icon-button* {:variant "secondary" + :class (stl/css :share-link-url-button) + :aria-label (tr "viewer.header.share.copy-link") + :icon i/clipboard + :on-click on-copy-current-share-url}]]) + + [:div {:class (stl/css :share-link-hint-row)} + [:div {:class (stl/css :share-link-hint)} (tr "common.share-link.permissions-hint")] + (if (some? current-share-link) + [:> button* {:variant "destructive" + :type "button" + :on-ref on-destroy-ref + :on-click on-destroy-current-share-link} + (tr "common.share-link.destroy-link")] + [:> button* {:variant "primary" + :type "button" + :on-click on-create-link} + (tr "common.share-link.get-link")])]] + + [:> permissions-section* {:file file + :page page + :link-options link-options + :on-toggle-all on-toggle-all + :on-checked-page on-checked-page + :on-comment-change on-comment-change + :on-inspect-change on-inspect-change}]])) + +(mf/defc share-link-dialog + {::mf/register modal/components + ::mf/register-as :share-link + ::mf/wrap-props false} + [{:keys [file page]}] + (let [link-to-destroy* (mf/use-state nil) + link-to-destroy (deref link-to-destroy*) + + dialog-ref (mf/use-ref nil) + destroy-btn-ref (mf/use-ref nil) + focus-target* (mf/use-ref nil) + + on-close + (mf/use-fn + (fn [event] + (dom/prevent-default event) + (modal/hide!))) + + on-request-destroy + (mf/use-fn + (fn [link] + (reset! link-to-destroy* link))) + + on-destroy-ref + (mf/use-fn + (fn [node] + (mf/set-ref-val! destroy-btn-ref node))) + + on-destroy-link + (fn [_] + (let [link link-to-destroy] + (mf/set-ref-val! focus-target* :dialog) + (reset! link-to-destroy* nil) + (st/emit! (dc/delete-share-link link)))) + + on-cancel-destroy + (fn [_] + (mf/set-ref-val! focus-target* :destroy-button) + (reset! link-to-destroy* nil))] + + (mf/with-effect [] + (some-> (mf/ref-val dialog-ref) dom/focus!)) + + (mf/with-effect [link-to-destroy] + (when (nil? link-to-destroy) + (when-let [target (mf/ref-val focus-target*)] + (mf/set-ref-val! focus-target* nil) + (case target + :destroy-button + (some-> (mf/ref-val destroy-btn-ref) dom/focus!) + + :dialog + (some-> (mf/ref-val dialog-ref) dom/focus!))))) + + [:div {:class (stl/css :share-link-modal)} + [:div {:ref dialog-ref + :class (stl/css :share-link-dialog) + :role "dialog" + :aria-modal true + :aria-labelledby "share-link-title" + :tab-index "-1"} [:div {:class (stl/css :share-link-header)} - [:h2 {:class (stl/css :share-link-title)} + [:h2 {:id "share-link-title" + :class (stl/css :share-link-title)} (tr "common.share-link.title")] [:> icon-button* {:variant "ghost" - :tooltip-class (stl/css :modal-close-button) + :class (stl/css :share-link-close-button) :aria-label (tr "labels.close") :icon i/close :on-click on-close}]] - [:div {:class (stl/css :modal-content)} - [:div {:class (stl/css :share-link-section)} - (when (and (not confirm?) (some? current-link)) - [:div {:class (stl/css :custon-input-wrapper)} - [:input {:class (stl/css :input-text) - :type "text" - :value (or current-link "") - :placeholder (tr "common.share-link.placeholder") - :read-only true}] - - [:> icon-button* {:variant "secondary" - :aria-label (tr "viewer.header.share.copy-link") - :icon i/clipboard - :on-click copy-link}]]) - - [:div {:class (stl/css :hint-wrapper)} - (when (not ^boolean confirm?) - [:div {:class (stl/css :hint)} (tr "common.share-link.permissions-hint")]) - (cond - (true? confirm?) - [:div {:class (stl/css :confirm-dialog)} - [:div {:class (stl/css :description)} - (tr "common.share-link.confirm-deletion-link-description")] - [:div {:class (stl/css :actions)} - [:> button* {:variant "secondary" - :type "button" - :on-click #(reset! confirm* false)} - (tr "labels.cancel")] - [:> button* {:variant "destructive" - :type "button" - :on-click delete-link} - (tr "common.share-link.destroy-link")]]] - - (some? current-link) - [:> button* {:variant "destructive" - :type "button" - :on-click try-delete-link} - (tr "common.share-link.destroy-link")] - - :else - [:> button* {:variant "primary" - :type "button" - :on-click create-link} - (tr "common.share-link.get-link")])]] - - - (when (not ^boolean confirm?) - [:div {:class (stl/css :permissions-section)} - [:button {:class (stl/css :manage-permissions) - :type "button" - :aria-expanded perms-visible? - :on-click toggle-perms-visibility} - [:span {:class (stl/css-case :icon true - :rotated perms-visible?)} - [:> icon* {:icon-id i/arrow - :class (stl/css :arrow-icon)}]] - (tr "common.share-link.manage-ops")] - - (when ^boolean perms-visible? - [:* - (let [all-selected? (:all-pages options) - pages (->> (get-in file [:data :pages]) - (map #(get-in file [:data :pages-index %]))) - selected (:pages options)] - [:div {:class (stl/css :view-mode)} - [:div {:class (stl/css :subtitle)} - (tr "common.share-link.permissions-pages")] - [:div {:class (stl/css :items)} - (if (= 1 (count pages)) - [:div {:class (stl/css :checkbox-wrapper)} - - [:label {:for (str "page-" current-page-id) - :class (stl/css-case :global/checked true)} - - [:span {:class (stl/css :checked)} - deprecated-icon/status-tick] - - (:name current-page)] - - [:input {:type "checkbox" - :id (dm/str "page-" current-page-id) - :data-page-id (dm/str current-page-id) - :on-change on-mark-checked-page - :checked true}] - [:span (str " " (tr "common.share-link.current-tag"))]] - - [:* - [:div {:class (stl/css :select-all-row)} - [:div {:class (stl/css :checkbox-wrapper)} - [:label {:for "view-all" - :class (stl/css :select-all-label)} - [:span {:class (stl/css-case :global/checked all-selected?)} - (when all-selected? - deprecated-icon/status-tick)] - (tr "common.share-link.view-all") - [:input {:type "checkbox" - :id "view-all" - :checked all-selected? - :name "pages-mode" - :on-change on-toggle-all}]]] - - [:span {:class (stl/css :count-pages)} - (tr "common.share-link.page-shared" (i18n/c (count selected)))]] - - [:ul {:class (stl/css :pages-selection)} - (for [{:keys [id name]} pages] - [:li {:class (stl/css :checkbox-wrapper :page-row) - :key (dm/str id)} - [:label {:for (dm/str "page-" id)} - [:span {:class (stl/css-case :global/checked (contains? selected id))} - (when (contains? selected id) - deprecated-icon/status-tick)] - name - (when (= current-page-id id) - [:div {:class (stl/css :current-tag)} (dm/str " " (tr "common.share-link.current-tag"))]) - [:input {:type "checkbox" - :id (dm/str "page-" id) - :data-page-id (dm/str id) - :on-change on-mark-checked-page - :checked (contains? selected id)}]]])]])]]) - - [:div {:class (stl/css :access-mode)} - [:div {:class (stl/css :subtitle)} - (tr "common.share-link.permissions-can-comment")] - [:div {:class (stl/css :items)} - [:& select - {:class (stl/css :who-comment-select) - :default-value (dm/str (:who-comment options)) - :options [{:value "team" :label (tr "common.share-link.team-members")} - {:value "all" :label (tr "common.share-link.all-users")}] - :on-change on-comment-change}]]] - [:div {:class (stl/css :inspect-mode)} - [:div {:class (stl/css :subtitle)} - (tr "common.share-link.permissions-can-inspect")] - [:div {:class (stl/css :items)} - [:& select - {:class (stl/css :who-inspect-select) - :default-value (dm/str (:who-inspect options)) - :options [{:value "team" :label (tr "common.share-link.team-members")} - {:value "all" :label (tr "common.share-link.all-users")}] - :on-change on-inspect-change}]]]])])]]])) - + [:div {:class (stl/css :share-link-scroll) + :hidden (some? link-to-destroy)} + [:> share-link-content* {:file file + :page page + :on-request-destroy on-request-destroy + :on-destroy-ref on-destroy-ref}]] + (when (some? link-to-destroy) + [:> destroy-link-confirmation* {:on-cancel on-cancel-destroy + :on-destroy on-destroy-link}])]])) diff --git a/frontend/src/app/main/ui/viewer/share_link.scss b/frontend/src/app/main/ui/viewer/share_link.scss index 7cb4e94245..1140d82bab 100644 --- a/frontend/src/app/main/ui/viewer/share_link.scss +++ b/frontend/src/app/main/ui/viewer/share_link.scss @@ -9,13 +9,112 @@ @use "ds/_utils.scss" as *; @use "ds/typography.scss" as *; -// refactor/ stays for the modal placeholder (%modal-container-base), the -// link input (%input-element) and the page checkboxes (%input-checkbox, -// %button-icon-small for their tick), which move with their own groups. +// refactor/ stays for the modal placeholder (%modal-container-base), which +// moves with its own group. @use "refactor/common-refactor.scss" as deprecated; -.share-modal { - display: block; +// --- Collapsible (shared) + +.collapsible-title { + @include use-typography("headline-small"); + + border: none; + background: none; + color: var(--color-foreground-secondary); + block-size: $sz-32; + display: flex; + align-items: center; + justify-content: flex-start; + padding: 0; + cursor: pointer; + + &:focus-visible { + outline: $b-1 solid var(--color-accent-primary); + outline-offset: px2rem(2); + } +} + +.collapsible-title-icon { + margin-inline-end: px2rem(6); +} + +// --- Permissions section + +.permissions-section { + display: flex; + flex-direction: column; + gap: var(--sp-s); +} + +.permissions-content { + display: flex; + flex-direction: column; + gap: var(--sp-s); +} + +.permissions-pages-field, +.permissions-select-row { + display: grid; + grid-template-columns: minmax(px2rem(136), max-content) 1fr; + align-items: start; + column-gap: var(--sp-s); + inline-size: 100%; +} + +.permissions-pages-field { + max-block-size: px2rem(216); + overflow: hidden auto; + scrollbar-gutter: stable; +} + +.permissions-label { + color: var(--color-foreground-secondary); + display: flex; + align-items: center; + justify-content: flex-start; + block-size: $sz-32; +} + +.permissions-control { + color: var(--color-foreground-primary); +} + +.permissions-page-row, +.permissions-select-all-row { + display: flex; + align-items: center; + gap: var(--sp-xs); + block-size: $sz-32; +} + +.permissions-page-row .permissions-checkbox, +.permissions-select-all-row .permissions-checkbox { + --input-checkbox-text-color: var(--color-foreground-primary); +} + +.permissions-current-tag { + @include use-typography("body-small"); + + color: var(--color-foreground-secondary); +} + +.permissions-select-all-row { + justify-content: space-between; +} + +.permissions-page-count { + @include use-typography("body-small"); + + color: var(--color-foreground-secondary); +} + +.permissions-pages-selection { + margin: 0; +} + +// --- Share link dialog + +.share-link-modal { position: absolute; inset-block-start: px2rem(52); inset-inline: calc(100vw - #{$sz-512}) var(--sp-m); @@ -25,7 +124,18 @@ .share-link-dialog { @extend %modal-container-base; + display: flex; + flex-direction: column; min-block-size: unset; + + &:focus { + outline: none; + } +} + +.share-link-scroll { + min-block-size: 0; + overflow-y: auto; } .share-link-header { @@ -38,14 +148,13 @@ color: var(--color-foreground-primary); } -// Tooltip wrapper of the close button: it places it in the corner. -.modal-close-button { +.share-link-close-button { position: absolute; inset-block-start: var(--sp-s); inset-inline-end: var(--sp-xs); } -.modal-content { +.share-link-content { @include use-typography("body-small"); display: flex; @@ -53,171 +162,61 @@ gap: var(--sp-xxl); } -.share-link-section { +.share-link-form { display: flex; flex-direction: column; gap: var(--sp-s); } -.hint-wrapper { +.share-link-url { + display: flex; + padding: px2rem(1); +} + +.share-link-url-input { + border-end-end-radius: 0; + border-start-end-radius: 0; +} + +.share-link-url-button { + border-start-start-radius: 0; + border-end-start-radius: 0; +} + +.share-link-hint-row { display: flex; align-items: center; gap: var(--sp-xs); } -.hint { +.share-link-hint { flex-grow: 1; color: var(--color-foreground-secondary); } -.custon-input-wrapper { +.share-link-confirm { display: flex; - align-items: center; - gap: var(--sp-xs); - border-radius: $br-8; - block-size: $sz-32; - inline-size: 100%; - background-color: var(--color-background-tertiary); -} - -.input-text { - @extend %input-element; - @include use-typography("body-small"); - - color: var(--color-foreground-primary); - padding-inline-start: var(--sp-s); - margin: 0; + flex-direction: column; flex-grow: 1; + justify-content: center; + min-block-size: 0; + overflow-y: auto; &:focus { outline: none; - border: $b-1 solid var(--color-accent-primary); } } -.description { +.share-link-confirm-description { @include use-typography("body-small"); color: var(--color-foreground-secondary); margin-block-end: var(--sp-xxl); } -.actions { +.share-link-confirm-actions { display: flex; align-items: center; gap: var(--sp-xs); justify-content: flex-end; } - -.permissions-section { - display: flex; - flex-direction: column; - gap: var(--sp-s); -} - -.manage-permissions { - @include use-typography("headline-small"); - - border: none; - background: none; - cursor: pointer; - color: var(--color-foreground-secondary); - block-size: $sz-32; - display: flex; - align-items: center; - padding: 0; -} - -.icon { - display: flex; - justify-content: center; - align-items: center; - margin-inline-end: px2rem(6); - - .arrow-icon { - flex-shrink: 0; - color: var(--color-foreground-secondary); - } - - &.rotated { - transform: rotate(90deg); - } -} - -.view-mode, -.access-mode, -.inspect-mode { - display: flex; - inline-size: 100%; -} - -.view-mode { - max-block-size: px2rem(216); - overflow: hidden auto; - scrollbar-gutter: stable; -} - -.subtitle { - color: var(--color-foreground-secondary); - display: flex; - align-items: center; - justify-content: flex-start; - inline-size: px2rem(136); - block-size: $sz-32; -} - -.items { - flex-grow: 1; - color: var(--color-foreground-primary); -} - -.select-all-row { - display: flex; - align-items: center; - gap: var(--sp-xs); - justify-content: space-between; - block-size: $sz-32; - border-block-end: $b-1 solid var(--color-background-quaternary); -} - -.select-all-label { - color: var(--color-foreground-primary); -} - -.pages-selection { - margin: 0; -} - -.page-row { - border-block-end: $b-1 solid var(--color-background-quaternary); - - &:last-child { - border-block-end: none; - } -} - -.count-pages, -.current-tag { - @include use-typography("body-small"); - - color: var(--color-foreground-secondary); -} - -.checkbox-wrapper { - @extend %input-checkbox; - - block-size: $sz-32; - padding: 0; - - span.checked { - background-color: var(--color-accent-primary); - border: $b-1 solid var(--color-accent-primary); - - // The legacy tick icon has no class; it moves with the checkboxes. - svg { - @extend %button-icon-small; - - stroke: var(--color-background-primary); - } - } -} diff --git a/frontend/src/app/main/ui/workspace.cljs b/frontend/src/app/main/ui/workspace.cljs index 6d761be5f6..0711cfbc3e 100644 --- a/frontend/src/app/main/ui/workspace.cljs +++ b/frontend/src/app/main/ui/workspace.cljs @@ -60,7 +60,7 @@ (mf/defc workspace-content* {::mf/private true} - [{:keys [file layout page wglobal]}] + [{:keys [file layout page wglobal scroll-store]}] (let [palete-size (mf/use-state nil) selected (mf/deref refs/selected-shapes) @@ -144,6 +144,7 @@ :file-id (get file :id) :page-id page-id :file file + :scroll-store scroll-store :selected selected :section options-mode :drawing-tool (get drawing :tool)}])])) @@ -187,9 +188,14 @@ (mf/defc workspace-inner* {::mf/private true} [{:keys [page-id file-id file layout wglobal]}] - (let [page-ref (mf/with-memo [file-id page-id] - (make-page-ref file-id page-id)) - page (mf/deref page-ref)] + (let [page-ref (mf/with-memo [file-id page-id] + (make-page-ref file-id page-id)) + page (mf/deref page-ref) + + ;; NOTE: lives here because the sidebar unmounts on every page + ;; switch (the loader replaces `workspace-content*` until the + ;; new page is ready), so the scroll positions must outlive it + scroll-store* (mf/use-var {})] (mf/with-effect [] (let [focus-out #(st/emit! (dw/workspace-focus-lost)) @@ -205,7 +211,8 @@ [:> workspace-content* {:file file :page page :wglobal wglobal - :layout layout}] + :layout layout + :scroll-store scroll-store*}] [:> workspace-loader*]))) (mf/defc workspace* diff --git a/frontend/src/app/main/ui/workspace/colorpicker.cljs b/frontend/src/app/main/ui/workspace/colorpicker.cljs index 97c5ecd01d..08d9734da0 100644 --- a/frontend/src/app/main/ui/workspace/colorpicker.cljs +++ b/frontend/src/app/main/ui/workspace/colorpicker.cljs @@ -39,6 +39,7 @@ [app.main.ui.workspace.colorpicker.color-tokens :refer [token-section*]] [app.main.ui.workspace.colorpicker.gradients :refer [gradients*]] [app.main.ui.workspace.colorpicker.harmony :refer [harmony-selector*]] + [app.main.ui.workspace.colorpicker.hsl :as hsl] [app.main.ui.workspace.colorpicker.hsva :refer [hsva-selector*]] [app.main.ui.workspace.colorpicker.libraries :refer [libraries*]] [app.main.ui.workspace.colorpicker.ramp :refer [ramp-selector*]] @@ -88,10 +89,10 @@ ;; saturation 0 → 1. For L: fix current saturation, sweep ;; lightness 0 → 0.5 (pure hue) → 1. All computed at the ;; current hue. - [_ cur-hsl-s cur-hsl-l] (cc/rgb->hsl rgb) - hsl-sat-from [h 0.0 cur-hsl-l] - hsl-sat-to [h 1.0 cur-hsl-l] - lightness-mid [h cur-hsl-s 0.5] + [hsl-h cur-hsl-s cur-hsl-l] (hsl/components current-color) + hsl-sat-from [hsl-h 0.0 cur-hsl-l] + hsl-sat-to [hsl-h 1.0 cur-hsl-l] + lightness-mid [hsl-h cur-hsl-s 0.5] format-hsl (fn [[h s l]] (str/fmt "hsl(%s, %s, %s)" @@ -214,7 +215,7 @@ (mf/use-fn (mf/deps current-color drag?) (fn [color] - (let [color (merge current-color color) + (let [color (dc/merge-color-components current-color color) color (dc/materialize-color-components color)] (st/emit! (dc/update-colorpicker-color color (not drag?)))))) diff --git a/frontend/src/app/main/ui/workspace/colorpicker/color_inputs.cljs b/frontend/src/app/main/ui/workspace/colorpicker/color_inputs.cljs index 5d7d72cf71..ab4481f4fd 100644 --- a/frontend/src/app/main/ui/workspace/colorpicker/color_inputs.cljs +++ b/frontend/src/app/main/ui/workspace/colorpicker/color_inputs.cljs @@ -12,6 +12,7 @@ [app.common.types.color :as cc] [app.main.ui.ds.controls.input :refer [input*]] [app.main.ui.ds.foundations.assets.icon :as i] + [app.main.ui.workspace.colorpicker.hsl :as hsl] [app.util.dom :as dom] [app.util.keyboard :as kbd] [rumext.v2 :as mf])) @@ -41,11 +42,7 @@ ;; in sync with the inputs. hsb-mode (or mode :hsb) - ;; Compute HSL from current RGB (derived; not stored on the color map) - [_hsl-h hsl-s hsl-l] - (if (and red green blue) - (cc/rgb->hsl [red green blue]) - [0 0 0]) + [hsl-h hsl-s hsl-l] (hsl/components color) refs {:hex (mf/use-ref nil) :r (mf/use-ref nil) @@ -104,17 +101,9 @@ :h h :s s :v v :r r :g g :b b})) - ;; HSL changes: recompute RGB/HSV from the new HSL triple, - ;; reusing the current hue when only S or L changes. - (#{:hsl-s :hsl-l} property) - (let [new-s (if (= property :hsl-s) val hsl-s) - new-l (if (= property :hsl-l) val hsl-l) - [r g b] (cc/hsl->rgb [hue new-s new-l]) - hex (cc/rgb->hex [r g b]) - [h s v] (cc/hex->hsv hex)] - (on-change {:hex hex - :h h :s s :v v - :r r :g g :b b})) + (and (= hsb-mode :hsl) + (#{:h :hsl-s :hsl-l} property)) + (on-change (hsl/update-color color property val)) :else (let [{:keys [h s v]} (merge color (hash-map property val)) @@ -159,6 +148,7 @@ (fn [] (doseq [ref-key (keys refs)] (let [property-val (case ref-key + :h (if (= hsb-mode :hsl) hsl-h hue) :hsl-s hsl-s :hsl-l hsl-l (get color ref-key)) @@ -240,11 +230,11 @@ :property "Hue" :aria-label "Hue" :max 360 - :default-value hue + :default-value hsl-h :on-change (on-change-property :h 360) :on-key-down (on-key-down-property :h 360)}] [:> input* {:id "saturation-value" - :ref (:s refs) + :ref (:hsl-s refs) :type "number" :min 0 :icon i/character-s @@ -252,9 +242,9 @@ :aria-label "Saturation" :max 100 :step 1 - :default-value saturation - :on-change (on-change-property :s 100) - :on-key-down (on-key-down-property :s 100)}] + :default-value (mth/precision (* hsl-s 100) 2) + :on-change (on-change-property :hsl-s 100) + :on-key-down (on-key-down-property :hsl-s 100)}] [:> input* {:id "lightness-value" :ref (:hsl-l refs) :type "number" diff --git a/frontend/src/app/main/ui/workspace/colorpicker/hsl.cljs b/frontend/src/app/main/ui/workspace/colorpicker/hsl.cljs new file mode 100644 index 0000000000..31c95f1a9c --- /dev/null +++ b/frontend/src/app/main/ui/workspace/colorpicker/hsl.cljs @@ -0,0 +1,23 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.main.ui.workspace.colorpicker.hsl + (:require + [app.common.types.color :as cc])) + +(defn components + [{:keys [hsl r g b]}] + (or hsl (cc/rgb->hsl [(or r 0) (or g 0) (or b 0)]))) + +(defn update-color + [color property value] + (let [index (case property :h 0 :hsl-s 1 :hsl-l 2) + hsl (assoc (components color) index value) + hex (cc/rgb->hex (cc/hsl->rgb hsl)) + [r g b] (cc/hex->rgb hex) + [h s v] (cc/hex->hsv hex)] + ;; RGB cannot retain the chosen hue and saturation at black or white. + {:hsl hsl :hex hex :h h :s s :v v :r r :g g :b b})) diff --git a/frontend/src/app/main/ui/workspace/colorpicker/hsva.cljs b/frontend/src/app/main/ui/workspace/colorpicker/hsva.cljs index 5194488b5e..446c88f8f8 100644 --- a/frontend/src/app/main/ui/workspace/colorpicker/hsva.cljs +++ b/frontend/src/app/main/ui/workspace/colorpicker/hsva.cljs @@ -8,20 +8,15 @@ (:require-macros [app.main.style :as stl]) (:require [app.common.types.color :as cc] + [app.main.ui.workspace.colorpicker.hsl :as hsl] [app.main.ui.workspace.colorpicker.slider-selector :refer [slider-selector*]] [rumext.v2 :as mf])) (mf/defc hsva-selector* [{:keys [color disable-opacity mode on-change on-start-drag on-finish-drag]}] - (let [{hue :h saturation :s value :v alpha :alpha - r-val :r g-val :g b-val :b} color + (let [{hue :h saturation :s value :v alpha :alpha} color hsl-mode? (= mode :hsl) - ;; Current HSL derived from RGB — used as the starting point - ;; for HSL saturation/lightness slider values and for - ;; recomputing the color when either is dragged. - [_ hsl-s hsl-l] (if (and r-val g-val b-val) - (cc/rgb->hsl [r-val g-val b-val]) - [0 0 0]) + [hsl-h hsl-s hsl-l] (hsl/components color) ;; HSB math — current default behavior. handle-change-hsv @@ -41,14 +36,7 @@ handle-change-hsl (fn [key] (fn [new-value] - (let [new-s (if (= key :hsl-s) new-value hsl-s) - new-l (if (= key :hsl-l) new-value hsl-l) - [r g b] (cc/hsl->rgb [hue new-s new-l]) - hex (cc/rgb->hex [r g b]) - [h s v] (cc/hex->hsv hex)] - (on-change {:hex hex - :h h :s s :v v - :r r :g g :b b})))) + (on-change (hsl/update-color color key new-value)))) on-change-opacity (fn [new-alpha] (on-change {:alpha new-alpha}))] [:div {:class (stl/css :hsva-selector)} @@ -58,8 +46,10 @@ {:class (stl/css :hsva-bar) :type :hue :max-value 360 - :value hue - :on-change (handle-change-hsv :h) + :value (if hsl-mode? hsl-h hue) + :on-change (if hsl-mode? + (handle-change-hsl :h) + (handle-change-hsv :h)) :on-start-drag on-start-drag :on-finish-drag on-finish-drag}]] [:div {:class (stl/css :hsva-row)} diff --git a/frontend/src/app/main/ui/workspace/colorpicker/ramp.cljs b/frontend/src/app/main/ui/workspace/colorpicker/ramp.cljs index a8ddf66070..1e9b915141 100644 --- a/frontend/src/app/main/ui/workspace/colorpicker/ramp.cljs +++ b/frontend/src/app/main/ui/workspace/colorpicker/ramp.cljs @@ -97,6 +97,7 @@ (mf/deps internal-color on-change) (fn [saturation value] (let [color (-> internal-color + (dissoc :hsl) (assoc :s saturation) (assoc :v value) (enrich-color-map))] @@ -108,6 +109,7 @@ (mf/deps internal-color on-change) (fn [hue] (let [color (-> internal-color + (dissoc :hsl) (assoc :h hue) enrich-color-map)] (reset! internal-color* color) diff --git a/frontend/src/app/main/ui/workspace/context_menu.cljs b/frontend/src/app/main/ui/workspace/context_menu.cljs index 6a11dfd988..817bc040da 100644 --- a/frontend/src/app/main/ui/workspace/context_menu.cljs +++ b/frontend/src/app/main/ui/workspace/context_menu.cljs @@ -873,6 +873,10 @@ {::mf/private true} [{:keys [mdata]}] (let [{:keys [type index grid-id]} mdata + + objects (deref refs/workspace-page-objects) + in-copy? (dwsl/grid-in-copy? objects (get objects grid-id)) + do-delete-track (mf/use-fn (mf/deps grid-id type index) @@ -905,14 +909,14 @@ (if (= type :column) [:* - [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.column.duplicate") :on-click do-duplicate-track}] + [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.column.duplicate") :on-click do-duplicate-track :disabled in-copy?}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.column.add-before") :on-click do-add-track-before}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.column.add-after") :on-click do-add-track-after}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.column.delete") :on-click do-delete-track}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.column.delete-shapes") :on-click do-delete-track-shapes}]] [:* - [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.row.duplicate") :on-click do-duplicate-track}] + [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.row.duplicate") :on-click do-duplicate-track :disabled in-copy?}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.row.add-before") :on-click do-add-track-before}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.row.add-after") :on-click do-add-track-after}] [:> menu-entry* {:title (tr "workspace.context-menu.grid-track.row.delete") :on-click do-delete-track}] @@ -949,6 +953,9 @@ grid-edition (mf/deref grid-edition-ref) has-copied-tracks? (some? (:copied-tracks grid-edition)) + objects (deref refs/workspace-page-objects) + in-copy? (dwsl/grid-in-copy? objects (get objects grid-id)) + do-merge-cells (mf/use-fn (mf/deps grid-id cells) @@ -1002,7 +1009,7 @@ [:> menu-entry* {:title (tr "workspace.context-menu.grid-cells.paste-tracks") :on-click do-paste-tracks - :disabled (not has-copied-tracks?)}]])) + :disabled (or (not has-copied-tracks?) in-copy?)}]])) (def guide-color-presets diff --git a/frontend/src/app/main/ui/workspace/main_menu.cljs b/frontend/src/app/main/ui/workspace/main_menu.cljs index b9564e5d5b..f8612e6642 100644 --- a/frontend/src/app/main/ui/workspace/main_menu.cljs +++ b/frontend/src/app/main/ui/workspace/main_menu.cljs @@ -43,6 +43,7 @@ [app.main.ui.workspace.branch-switcher :as bsw] [app.main.ui.workspace.menu-shortcuts :refer [shortcuts*]] [app.plugins.register :as preg] + [app.render-wasm.api.webgl :as webgl] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] [app.util.keyboard :as kbd] @@ -954,8 +955,9 @@ (dom/reload-current-window true)) (fn [_] (st/emit! (ntf/error (tr "errors.generic")))))) - (st/emit! (ev/event {::ev/name ev-name - ::ev/origin "workspace:menu"}) + (st/emit! (ev/event (assoc (webgl/get-environment) + ::ev/name ev-name + ::ev/origin "workspace:menu")) (ptk/data-event ::ev/force-persist {}))))) open-plugins-manager diff --git a/frontend/src/app/main/ui/workspace/plugins.cljs b/frontend/src/app/main/ui/workspace/plugins.cljs index cd73c45664..86e237fdc9 100644 --- a/frontend/src/app/main/ui/workspace/plugins.cljs +++ b/frontend/src/app/main/ui/workspace/plugins.cljs @@ -111,12 +111,13 @@ plugin-url* (mf/use-state "") plugin-url (deref plugin-url*) - input-status* (mf/use-state nil) ;; :error-url :error-manifest :success + input-status* (mf/use-state nil) ;; :error-url :error-manifest :error-global :success input-status (deref input-status*) error-url? (= :error-url input-status) error-manifest? (= :error-manifest input-status) - error? (or error-url? error-manifest?) + error-global? (= :error-global input-status) + error? (or error-url? error-manifest? error-global?) permissions (mf/deref refs/permissions) user-can-edit? (get permissions :can-edit) @@ -159,7 +160,10 @@ (fn [err] (.error js/console err) (reset! fetching-manifest? false) - (reset! input-status* :error-url)))))) + (reset! input-status* + (if (= ::preg/global-not-allowed (:type (ex-data err))) + :error-global + :error-url))))))) on-open-plugin (mf/use-fn @@ -236,6 +240,10 @@ [:div {:class (stl/css-case :info true :error error?)} (tr "workspace.plugins.error.manifest")]) + (when error-global? + [:div {:class (stl/css-case :info true :error error?)} + (tr "workspace.plugins.error.global-not-allowed")]) + (when-not (empty? plugins-state) [:> i18n/tr-html* {:class (stl/css :discover) diff --git a/frontend/src/app/main/ui/workspace/sidebar.cljs b/frontend/src/app/main/ui/workspace/sidebar.cljs index 8137784dc9..7d222f0f43 100644 --- a/frontend/src/app/main/ui/workspace/sidebar.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar.cljs @@ -38,6 +38,7 @@ [app.main.ui.workspace.sidebar.history :refer [history-toolbox*]] [app.main.ui.workspace.sidebar.layers :refer [layers-toolbox*]] [app.main.ui.workspace.sidebar.options :refer [options-toolbox*]] + [app.main.ui.workspace.sidebar.scale :refer [scale-sidebar*]] [app.main.ui.workspace.sidebar.shortcuts :refer [shortcuts-container*]] [app.main.ui.workspace.sidebar.sitemap :refer [sitemap*]] [app.main.ui.workspace.sidebar.versions :refer [versions-toolbox*]] @@ -102,6 +103,7 @@ [:article {:class (stl/css :layers-tab)} [:> sitemap* {:layout layout :height sitemap-height + :scroll-store scroll-store :collapsed sitemap-collapsed? :on-toggle-collapsed on-toggle-sitemap-collapsed}] @@ -119,11 +121,10 @@ (mf/defc left-sidebar* {::mf/memo true} - [{:keys [layout file tokens-lib tokens-status active-tokens resolved-active-tokens]}] + [{:keys [layout file tokens-lib tokens-status active-tokens resolved-active-tokens scroll-store]}] (let [options-mode (mf/deref refs/options-mode-global) project (mf/deref refs/project) file-id (get file :id) - scroll-store* (mf/use-var {}) design-tokens? (features/use-feature "design-tokens/v1") ;; a design review is read-only: it shows the pages and layers alone @@ -209,7 +210,7 @@ [:div {:class (stl/css :left-sidebar-content)} [:> layers-content* {:layout layout :width width - :scroll-store scroll-store*}]] + :scroll-store scroll-store}]] :else [:div {:class (stl/css :left-sidebar-content)} @@ -225,7 +226,7 @@ :assets [:> assets-toolbox* {:size (- width 58) :file-id file-id - :scroll-store scroll-store*}] + :scroll-store scroll-store}] :tokens [:> tokens-sidebar-tab* @@ -233,12 +234,12 @@ :tokens-status tokens-status :active-tokens active-tokens :resolved-active-tokens resolved-active-tokens - :scroll-store scroll-store*}] + :scroll-store scroll-store}] :layers [:> layers-content* {:layout layout :width width - :scroll-store scroll-store*}])]])]])) + :scroll-store scroll-store}])]])]])) ;; --- Right Sidebar (Component) @@ -296,6 +297,7 @@ (let [reviewing? (some? (mf/deref refs/pull-request-preview)) is-comments? (and (not reviewing?) (= drawing-tool :comments)) is-history? (contains? layout :document-history) + is-scale? (contains? layout :scale-text) is-inspect? (= section :inspect) is-debug? (= section :debug) @@ -305,6 +307,7 @@ can-be-expanded? (and (not is-comments?) (not is-history?) + (not is-scale?) is-inspect? (= current-section :code)) @@ -376,6 +379,9 @@ is-history? [:> history-content* {}] + is-scale? + [:> scale-sidebar* {}] + :else (let [props (mf/spread-props props {:on-change-section on-change-section @@ -383,7 +389,7 @@ [:> options-toolbox* props]))]]]])) (mf/defc sidebar* - [{:keys [layout file file-id page-id section drawing-tool selected]}] + [{:keys [layout file file-id page-id section drawing-tool selected scroll-store]}] (let [tokens-lib (mf/deref refs/tokens-lib) @@ -428,6 +434,7 @@ [:> left-sidebar* {:layout layout :file file :page-id page-id + :scroll-store scroll-store :tokens-lib tokens-lib :tokens-status tokens-status :active-tokens active-tokens-force-set diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs index f3c9764e15..387e0c9ac2 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs @@ -757,7 +757,9 @@ on-library-change (mf/use-fn (fn [id] - (swap! filters* assoc :file-id id :term "" :path ""))) + ;; Keep the search term: it describes what the user is looking + ;; for, not where, so it applies to the newly selected library. + (swap! filters* assoc :file-id id :path ""))) on-search-term-change (mf/use-fn diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs index cd48557514..4dd5a2c761 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs @@ -17,6 +17,7 @@ [app.main.ui.ds.buttons.icon-button :refer [icon-button*]] [app.main.ui.ds.foundations.assets.icon :as i] [app.main.ui.exports.assets] + [app.main.ui.exports.preview :refer [export-preview*]] [app.util.dom :as dom] [app.util.i18n :refer [c tr]] [app.util.keyboard :as kbd] @@ -79,7 +80,7 @@ scale-enabled? (mf/use-fn (fn [export] - (#{:png :jpeg :webp} (:type export)))) + (contains? de/raster-export-types (:type export)))) on-download (mf/use-fn @@ -274,4 +275,9 @@ :disabled in-progress?} (if in-progress? (tr "workspace.options.exporting-object") - (tr "workspace.options.export-object" (c (count shapes-with-exports))))])])])) + (tr "workspace.options.export-object" (c (count shapes-with-exports))))]) + + (when (and (= 1 (count ids)) (vector? exports) (seq exports)) + [:> export-preview* {:key (first ids) + :object-id (first ids) + :exports exports}])])])) diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.cljs index 61ee90bf07..034d209ac1 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.cljs @@ -52,6 +52,9 @@ Evaluated once at module load time; cf/flags is immutable after startup." (contains? cf/flags :token-typography-row)) +(def ^:private text-lists-flag-enabled? + (contains? cf/flags :text-editor-wasm-v1-lists)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Sub-components ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -230,6 +233,37 @@ :allow-empty true :options options}]])) +(mf/defc list-options* + [{:keys [values on-change on-blur]}] + (let [list-style-type (:list-style-type values) + ;; Mixed selections leave no option selected + selected (if (= list-style-type :multiple) + nil + (d/nilv list-style-type "none")) + options + (mf/with-memo [] + [{:value "none" + :id "list-style-none" + :label (tr "workspace.options.text-options.list-none") + :icon i/remove} + {:value "disc" + :id "list-style-disc" + :label (tr "workspace.options.text-options.list-bullets") + :icon i/view-as-list}]) + + handle-change + (mf/use-fn + (mf/deps on-change on-blur) + (fn [value] + (on-change {:list-style-type value}) + (when (some? on-blur) (on-blur))))] + + [:div {:class (stl/css :list-options)} + [:> radio-buttons* {:selected selected + :on-change handle-change + :name "list-options" + :options options}]])) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -276,6 +310,8 @@ (get n-values :grow-type)) (identical? (get o-values :letter-spacing) (get n-values :letter-spacing)) + (identical? (get o-values :list-style-type) + (get n-values :list-style-type)) (identical? (get o-values :line-height) (get n-values :line-height)) (identical? (get o-values :text-align) @@ -360,6 +396,10 @@ ;; --- Helpers multiple? (->> values vals (d/seek #(= % :multiple))) + ;; v3 can be toggled at runtime, so it is read on render + text-editor-v3? (features/use-feature "text-editor-wasm/v1") + text-lists-enabled? (and text-lists-flag-enabled? text-editor-v3?) + apply-token! (mf/use-fn (mf/deps ids typography-tokens) @@ -596,4 +636,7 @@ [:div {:class (stl/css :text-decoration-options)} [:> vertical-align* common-props] [:> text-decoration-options* (mf/spread-props common-props {:token-applied current-token-name})] - [:> text-direction-options* common-props]])])])) + [:> text-direction-options* common-props]]) + + (when (and more-options-open? text-lists-enabled?) + [:> list-options* common-props])])])) diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.scss index 4b39b0663c..7d114ab03a 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/text.scss @@ -59,7 +59,8 @@ .text-direction-options, .vertical-align-options, .grow-options, -.text-decoration-options { +.text-decoration-options, +.list-options { block-size: $sz-32; } diff --git a/frontend/src/app/main/ui/workspace/sidebar/scale.cljs b/frontend/src/app/main/ui/workspace/sidebar/scale.cljs new file mode 100644 index 0000000000..a62fda8f9c --- /dev/null +++ b/frontend/src/app/main/ui/workspace/sidebar/scale.cljs @@ -0,0 +1,331 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.main.ui.workspace.sidebar.scale + "Right sidebar panel of the scale content mode (K shortcut)." + (:require-macros [app.main.style :as stl]) + (:require + [app.common.data :as d] + [app.common.geom.shapes :as gsh] + [app.common.math :as mth] + [app.common.types.component :as ctk] + [app.main.data.workspace :as dw] + [app.main.features :as features] + [app.main.refs :as refs] + [app.main.store :as st] + [app.main.ui.components.title-bar :refer [title-bar*]] + [app.main.ui.ds.controls.combobox :refer [combobox*]] + [app.main.ui.ds.controls.numeric-input :refer [numeric-input*]] + [app.main.ui.ds.foundations.assets.icon :as i] + [app.main.ui.ds.product.panel-title :refer [panel-title*]] + [app.main.ui.ds.tooltip :refer [tooltip*]] + [app.main.ui.formats :as fmt] + [app.util.i18n :refer [tr]] + [cuerdas.core :as str] + [rumext.v2 :as mf])) + +(def ^:private scale-presets + [0.5 0.75 1 1.5 2 3 4]) + +(def ^:private anchors + [{:value [:left :top] + :class (stl/css :anchor-top-left) + :label #(tr "workspace.scale.anchor.top-left")} + {:value [:center :top] + :class (stl/css :anchor-top-center) + :label #(tr "workspace.scale.anchor.top-center")} + {:value [:right :top] + :class (stl/css :anchor-top-right) + :label #(tr "workspace.scale.anchor.top-right")} + {:value [:left :center] + :class (stl/css :anchor-center-left) + :label #(tr "workspace.scale.anchor.center-left")} + {:value [:center :center] + :class (stl/css :anchor-center) + :label #(tr "workspace.scale.anchor.center")} + {:value [:right :center] + :class (stl/css :anchor-center-right) + :label #(tr "workspace.scale.anchor.center-right")} + {:value [:left :bottom] + :class (stl/css :anchor-bottom-left) + :label #(tr "workspace.scale.anchor.bottom-left")} + {:value [:center :bottom] + :class (stl/css :anchor-bottom-center) + :label #(tr "workspace.scale.anchor.bottom-center")} + {:value [:right :bottom] + :class (stl/css :anchor-bottom-right) + :label #(tr "workspace.scale.anchor.bottom-right")}]) + +(def ^:private default-anchor + [:center :center]) + +(def ^:private factor-pattern + #"^(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)$") + +(defn- factor->string + [factor] + (let [precision (cond + (< factor 0.01) 4 + (< factor 0.1) 3 + :else 2)] + (or (fmt/format-number factor {:precision precision}) + "1"))) + +(defn- parse-factor + "Parses a scale typed by the user, like `2`, `1.5x` or `0,5x`. + Returns nil when the value is not a positive number, optionally + followed by an `x`." + [value] + (let [text (-> (str value) + (str/trim) + (str/lower) + (str/replace #"x$" "") + (str/trim) + (str/replace "," "."))] + (when (re-matches factor-pattern text) + (let [factor (d/parse-double text)] + (when (and (mth/finite? factor) (pos? factor)) + factor))))) + +(defn scale-state-for-selection + "Starts a scale session at 1x for the selected IDs." + [selected] + {:selection selected :factor 1}) + +(defn selection-scale-factor + "Returns 1x when the stored scale belongs to a different selection." + [scale-state selected] + (if (= selected (:selection scale-state)) + (:factor scale-state) + 1)) + +(defn selection-anchor + "Returns the saved anchor, or the center for a new non-empty selection." + [anchor-state selected] + (when (seq selected) + (if (= selected (:selection anchor-state)) + (:anchor anchor-state) + default-anchor))) + +(defn calculate-scale-after-dimension-change + "Updates the current scale factor by a width or height change ratio." + [factor dimension new-dimension] + (* factor (/ new-dimension dimension))) + +(defn- apply-preview-modifiers + [shapes modifiers wasm-modifiers wasm?] + (mapv (fn [shape] + (let [id (:id shape)] + (if wasm? + (gsh/apply-transform shape (get wasm-modifiers id)) + (gsh/transform-shape shape (get-in modifiers [id :modifiers]))))) + shapes)) + +(defn- scale-input-operation + [factor value] + (when-let [new-factor (parse-factor value)] + (when-not (mth/close? new-factor factor) + {:multiplier (/ new-factor factor) + :factor new-factor}))) + +(mf/defc anchor-point-selector* + {::mf/private true} + [{:keys [value disabled group-name on-change]}] + [:div {:class (stl/css :anchor-grid) + :role "radiogroup" + :aria-label (tr "workspace.scale.anchor-point")} + (for [{anchor-value :value + anchor-class :class + get-label :label} anchors] + (let [selected? (= anchor-value value) + label (get-label)] + [:> tooltip* {:key (str (first anchor-value) "-" (second anchor-value)) + :content label + :class (str anchor-class " " (stl/css :anchor-point-tooltip))} + [:label {:class [(stl/css :anchor-point) + (stl/css-case :center-area (= anchor-value default-anchor) + :disabled disabled)]} + [:input {:type "radio" + :class (stl/css :anchor-input) + :name group-name + :checked selected? + :disabled disabled + :aria-label label + :on-change #(on-change anchor-value)}] + [:span {:class (stl/css :anchor-mark)}]]]))]) + +(mf/defc scale-sidebar* + [] + (let [selected (mf/deref refs/selected-shapes) + shapes-ref (mf/with-memo [selected] + (refs/objects-by-id selected)) + shapes-data (mf/deref shapes-ref) + + workspace-modifiers (mf/deref refs/workspace-modifiers) + wasm-modifiers (mf/deref refs/workspace-wasm-modifiers) + wasm? (features/active-feature? @st/state "render-wasm/v1") + + selected-shapes (mf/with-memo [shapes-data] + (remove ctk/is-variant-container? shapes-data)) + + shapes (mf/with-memo [selected-shapes workspace-modifiers wasm-modifiers wasm?] + (apply-preview-modifiers selected-shapes + workspace-modifiers + (into {} wasm-modifiers) + wasm?)) + + rect (mf/with-memo [shapes] + (when (seq shapes) + (gsh/shapes->rect shapes))) + + width (get rect :width) + height (get rect :height) + size (when rect (max width height)) + disabled (or (nil? size) (mth/almost-zero? size)) + + width-disabled (or disabled (mth/almost-zero? (d/nilv width 0))) + height-disabled (or disabled (mth/almost-zero? (d/nilv height 0))) + + scale-state* (mf/use-state (scale-state-for-selection selected)) + scale-state (deref scale-state*) + factor (selection-scale-factor scale-state selected) + + factor-str (factor->string factor) + factor-id (str factor-str) + + anchor* (mf/use-state nil) + anchor-map (deref anchor*) + + anchor (selection-anchor anchor-map selected) + + width-id (mf/use-id) + height-id (mf/use-id) + scale-id (mf/use-id) + size-title-id (mf/use-id) + scale-title-id (mf/use-id) + group-id (mf/use-id) + + revision* (mf/use-state 0) + revision (deref revision*) + + scale-options + (mf/with-memo [] + (mapv (fn [factor] + (let [value (factor->string factor)] + {:id value + :label (str value "x")})) + scale-presets)) + + default-selected + (if (some #(= factor-id (:id %)) scale-options) + factor-id + (str factor-id "x")) + + reset-scale-input + (mf/use-fn + #(swap! revision* inc)) + + scale-by + (mf/use-fn + (mf/deps size anchor factor selected) + (fn [multiplier new-factor] + (when (and (some? size) + (pos? size) + (pos? multiplier)) + (st/emit! (dw/scale-selected multiplier + (or anchor default-anchor))) + (reset! scale-state* {:selection selected :factor new-factor})))) + + on-width-change + (mf/use-fn + (mf/deps width factor scale-by) + (fn [value] + (when (and (number? value) (some? width) (pos? width) (pos? value)) + (scale-by (/ value width) + (calculate-scale-after-dimension-change factor width value))))) + + on-height-change + (mf/use-fn + (mf/deps height factor scale-by) + (fn [value] + (when (and (number? value) (some? height) (pos? height) (pos? value)) + (scale-by (/ value height) + (calculate-scale-after-dimension-change factor height value))))) + + on-scale-change + (mf/use-fn + (mf/deps factor scale-by reset-scale-input) + (fn [value] + (if-let [{:keys [multiplier factor]} (scale-input-operation factor value)] + (scale-by multiplier factor) + ;; Normalize the display without scaling for invalid or unchanged input. + (reset-scale-input)))) + + on-anchor-change + (mf/use-fn + (mf/deps selected) + (fn [anchor] + (reset! anchor* {:selection selected :anchor anchor}))) + + on-close + (mf/use-fn + #(st/emit! (dw/remove-layout-flag :scale-text)))] + + (mf/with-layout-effect [selected] + (when-not (= selected (:selection (deref scale-state*))) + (reset! scale-state* (scale-state-for-selection selected)) + (reset! anchor* nil))) + + [:div {:class (stl/css :scale-sidebar)} + [:> panel-title* {:class (stl/css :scale-title) + :text (tr "workspace.scale.title") + :on-close on-close}] + + [:div {:class (stl/css :scale-content)} + [:section {:class (stl/css :scale-row) + :aria-labelledby size-title-id} + [:div {:id size-title-id} + [:> title-bar* {:collapsable false + :title (tr "workspace.options.size")}]] + [:div {:class (stl/css :scale-fields)} + [:div {:class (stl/css :scale-field)} + [:> numeric-input* {:id width-id + :icon i/character-w + :property (tr "workspace.options.width") + :min 0.01 + :disabled width-disabled + :placeholder "--" + :value width + :on-change on-width-change}]] + [:div {:class (stl/css :scale-field)} + [:> numeric-input* {:id height-id + :icon i/character-h + :property (tr "workspace.options.height") + :min 0.01 + :disabled height-disabled + :placeholder "--" + :value height + :on-change on-height-change}]]]] + + [:section {:class (stl/css :scale-row) + :aria-labelledby scale-title-id} + [:div {:id scale-title-id} + [:> title-bar* {:collapsable false + :title (tr "workspace.scale.scale-and-anchor")}]] + [:div {:class (stl/css :scale-fields)} + [:div {:class (stl/css :scale-field)} + [:> combobox* {:key (str revision "-" factor-id) + :id scale-id + :icon i/scale + :disabled disabled + :options scale-options + :default-selected default-selected + :on-change on-scale-change}]] + [:div {:class (stl/css :scale-field)} + [:> anchor-point-selector* {:value anchor + :disabled disabled + :group-name (str "scale-anchor-" group-id) + :on-change on-anchor-change}]]]]]])) diff --git a/frontend/src/app/main/ui/workspace/sidebar/scale.scss b/frontend/src/app/main/ui/workspace/sidebar/scale.scss new file mode 100644 index 0000000000..f9dc4abd64 --- /dev/null +++ b/frontend/src/app/main/ui/workspace/sidebar/scale.scss @@ -0,0 +1,156 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) KALEIDOS SUBSIDIARY SL + +@use "ds/_borders.scss" as *; +@use "ds/_sizes.scss" as *; +@use "ds/_utils.scss" as *; +@use "ds/typography.scss" as t; + +.scale-sidebar { + display: flex; + flex-direction: column; + block-size: 100%; + overflow-y: auto; + background-color: var(--color-background-primary); +} + +.scale-title { + margin: var(--sp-s) var(--sp-s) 0; +} + +.scale-content { + display: flex; + flex-direction: column; + gap: var(--sp-l); + padding: var(--sp-m) var(--sp-m) var(--sp-l); +} + +.scale-row { + display: flex; + flex-direction: column; + gap: var(--sp-xs); +} + +.scale-fields { + display: grid; + grid-template-columns: 1fr 1fr; + align-items: start; + gap: var(--sp-xs); +} + +.scale-field { + position: relative; + min-inline-size: 0; +} + +// --- Anchor point selector ---------------------------------------- + +.anchor-grid { + --anchor-side-size: #{$sz-24}; + --anchor-center-size: #{px2rem(60)}; + + display: grid; + grid-template-areas: + "top-left top-center top-right" + "center-left center center-right" + "bottom-left bottom-center bottom-right"; + grid-template-columns: var(--anchor-side-size) var(--anchor-center-size) var(--anchor-side-size); + grid-template-rows: var(--anchor-side-size) var(--anchor-center-size) var(--anchor-side-size); + inline-size: fit-content; + block-size: fit-content; + border-radius: $br-8; + background-color: var(--color-background-tertiary); +} + +.anchor-grid .anchor-point-tooltip { + inline-size: 100%; + block-size: 100%; +} + +.anchor-point { + position: relative; + display: grid; + place-items: center; +} + +.anchor-top-left { + grid-area: top-left; +} + +.anchor-top-center { + grid-area: top-center; +} + +.anchor-top-right { + grid-area: top-right; +} + +.anchor-center-left { + grid-area: center-left; +} + +.anchor-center { + grid-area: center; +} + +.anchor-center-right { + grid-area: center-right; +} + +.anchor-bottom-left { + grid-area: bottom-left; +} + +.anchor-bottom-center { + grid-area: bottom-center; +} + +.anchor-bottom-right { + grid-area: bottom-right; +} + +.center-area { + border-radius: $br-8; + background-color: var(--color-background-primary); +} + +.anchor-input { + position: absolute; + inline-size: $sz-1; + block-size: $sz-1; + margin: 0; + opacity: 0; + pointer-events: none; +} + +.anchor-mark { + inline-size: px2rem(10); + block-size: px2rem(10); + border-radius: px2rem(6); + outline: $b-2 solid transparent; + background-color: var(--color-foreground-secondary); +} + +.anchor-point:hover .anchor-mark { + background-color: var(--color-foreground-primary); +} + +.anchor-input:checked + .anchor-mark { + outline-color: var(--color-accent-primary-muted); + background-color: var(--color-accent-primary); +} + +.anchor-input:focus-visible + .anchor-mark { + outline-color: var(--color-accent-primary); +} + +.anchor-point.disabled .anchor-mark { + background-color: var(--color-foreground-disabled); +} + +.anchor-point.disabled .anchor-input:checked + .anchor-mark { + background-color: var(--color-accent-primary-muted); +} diff --git a/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs b/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs index 1902c9798f..386ed63b73 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs @@ -7,11 +7,11 @@ (ns app.main.ui.workspace.sidebar.scroll "Scroll position save/restore for sidebar panels. - Panels unmount on every tab switch, so positions that must survive it - cannot live in component state. The store is created once in - `left-sidebar*` (which stays mounted while tabs swap, unlike - `layers-content*` and the toolboxes) with `mf/use-var` and threaded to - the panels as a prop. Shape: {[panel id] scroll-top}." + Panels unmount on every tab switch, and the whole sidebar unmounts on + every page switch, so positions that must survive it cannot live in + component state. The store is created once in `workspace-inner*` (which + stays mounted while pages and tabs swap) with `mf/use-var` and threaded + to the panels as a prop. Shape: {[panel id] scroll-top}." (:require [app.util.dom :as dom] [app.util.timers :as tm] diff --git a/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs b/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs index 51d4c398ca..b375763c3f 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs @@ -23,6 +23,7 @@ [app.main.ui.ds.foundations.assets.icon :as i :refer [icon*]] [app.main.ui.hooks :as hooks] [app.main.ui.notifications.badge :refer [badge-notification]] + [app.main.ui.workspace.sidebar.scroll :as sc] [app.render-wasm.api :as wasm.api] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] @@ -291,10 +292,16 @@ ;; --- Sitemap Toolbox (mf/defc sitemap* - [{:keys [height collapsed on-toggle-collapsed]}] + [{:keys [height collapsed on-toggle-collapsed scroll-store]}] (let [file (mf/deref ref:file-with-pages) file-id (get file :id) project-id (get file :project-id) + content-ref (mf/use-ref nil) + + on-scroll (mf/use-fn + (mf/deps scroll-store file-id) + (fn [event] + (sc/save-scroll! scroll-store [:sitemap file-id] event))) on-create (mf/use-fn (mf/deps file-id project-id) @@ -308,6 +315,15 @@ ;; a design review is read-only for everybody, editors included reviewing? (some? (mf/deref refs/pull-request-preview))] + ;; NOTE: a layout effect, so the saved position is back before the + ;; selected page item runs its `scroll-into-view-if-needed!` effect. + ;; With nothing saved the list is left alone, so the selected page + ;; still scrolls into view when the file opens. + (mf/with-layout-effect [file-id collapsed] + (when-not ^boolean collapsed + (when-some [saved (get @scroll-store [:sitemap file-id])] + (sc/restore-scroll! (mf/ref-val content-ref) saved)))) + [:div {:class (stl/css :sitemap) :style {:--height (dm/str height "px")}} @@ -328,6 +344,8 @@ :icon i/add}])] (when-not ^boolean collapsed - [:div {:class (stl/css :sitemap-content)} + [:div {:class (stl/css :sitemap-content) + :ref content-ref + :on-scroll on-scroll} [:> pages-list* {:key (dm/str (:id file)) :file file}]])])) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs index 477d7327ea..115fbddd45 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs @@ -281,19 +281,28 @@ (d/update-when result :error wte/resolve-error-message))) (rx/subs! (fn [{:keys [error value]}] (let [touched? (get-in @form [:touched input-name])] - (when touched? - (if error - (if (csu/group-name-conflict-error? error token-name) - (swap! form assoc-in [:extra-errors ""] {:message error}) - (do - (swap! form assoc-in [:extra-errors input-name] {:message error}) - (swap! form assoc-in [:data :color-result] "") - (reset! hint* {:message error :type "error"}))) + (cond + ;; Untouched field (form just opened) + (not touched?) + (when-not error + (swap! form assoc-in [:data :color-result] value) + (when (csu/show-hint-on-open? (get-in @form [:data input-name]) value) + (reset! hint* {:message (tr "workspace.tokens.resolved-value" (dwtf/format-token-value value)) + :type "hint"}))) - (let [message (tr "workspace.tokens.resolved-value" (dwtf/format-token-value value))] - (swap! form update :extra-errors dissoc input-name) - (swap! form assoc-in [:data :color-result] value) - (reset! hint* {:message message :type "hint"}))))))))] + error + (if (csu/group-name-conflict-error? error token-name) + (swap! form assoc-in [:extra-errors ""] {:message error}) + (do + (swap! form assoc-in [:extra-errors input-name] {:message error}) + (swap! form assoc-in [:data :color-result] "") + (reset! hint* {:message error :type "error"}))) + + :else + (let [message (tr "workspace.tokens.resolved-value" (dwtf/format-token-value value))] + (swap! form update :extra-errors dissoc input-name) + (swap! form assoc-in [:data :color-result] value) + (reset! hint* {:message message :type "hint"})))))))] (fn [] (rx/dispose! subs)))) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs index a98e51ce74..730ced7b99 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs @@ -285,16 +285,25 @@ ((:error/fn error) (:error/value error)))))) (rx/subs! (fn [{:keys [error value]}] (let [touched? (get-in @form [:touched name])] - (when touched? - (if error - (if (csu/group-name-conflict-error? error token-name) - (swap! form assoc-in [:extra-errors ""] {:message error}) - (do - (swap! form assoc-in [:extra-errors name] {:message error}) - (reset! hint* {:message error :type "error"}))) - (let [message (tr "workspace.tokens.resolved-value" value)] - (swap! form update :extra-errors dissoc name) - (reset! hint* {:message message :type "hint"}))))))))] + (cond + ;; Untouched field (form just opened) + (not touched?) + (when (and (not error) + (csu/show-hint-on-open? (get-in @form [:data name]) value)) + (reset! hint* {:message (tr "workspace.tokens.resolved-value" value) + :type "hint"})) + + error + (if (csu/group-name-conflict-error? error token-name) + (swap! form assoc-in [:extra-errors ""] {:message error}) + (do + (swap! form assoc-in [:extra-errors name] {:message error}) + (reset! hint* {:message error :type "error"}))) + + :else + (let [message (tr "workspace.tokens.resolved-value" value)] + (swap! form update :extra-errors dissoc name) + (reset! hint* {:message message :type "hint"})))))))] (fn [] (rx/dispose! subs)))) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs index 7833bd95ea..8838adc844 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs @@ -175,17 +175,26 @@ (d/update-when result :error wte/resolve-error-assoc-message))) (rx/subs! (fn [{:keys [error value]}] (let [touched? (get-in @form [:touched input-name])] - (when touched? - (if error - (let [error' (:message error)] - (if (csu/group-name-conflict-error? error' token-name) - (swap! form assoc-in [:extra-errors ""] error') - (do - (swap! form assoc-in [:extra-errors input-name] error') - (reset! hint* {:message error' :type "error"})))) - (let [message (tr "workspace.tokens.resolved-value" value)] - (swap! form update :extra-errors dissoc input-name) - (reset! hint* {:message message :type "hint"}))))))))] + (cond + ;; Untouched field (form just opened) + (not touched?) + (when (and (not error) + (csu/show-hint-on-open? (get-in @form [:data input-name]) value)) + (reset! hint* {:message (tr "workspace.tokens.resolved-value" value) + :type "hint"})) + + error + (let [error' (:message error)] + (if (csu/group-name-conflict-error? error' token-name) + (swap! form assoc-in [:extra-errors ""] error') + (do + (swap! form assoc-in [:extra-errors input-name] error') + (reset! hint* {:message error' :type "error"})))) + + :else + (let [message (tr "workspace.tokens.resolved-value" value)] + (swap! form update :extra-errors dissoc input-name) + (reset! hint* {:message message :type "hint"})))))))] (fn [] (rx/dispose! subs)))) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs index b7dd512b7f..4a057d72bd 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs @@ -255,16 +255,25 @@ (d/update-when result :error wte/resolve-error-message))) (rx/subs! (fn [{:keys [error value]}] (let [touched? (get-in @form [:touched input-name])] - (when touched? - (if error - (if (csu/group-name-conflict-error? error token-name) - (swap! form assoc-in [:extra-errors ""] {:message error}) - (do - (swap! form assoc-in [:extra-errors input-name] {:message error}) - (reset! hint* {:message error :type "error"}))) - (let [message (tr "workspace.tokens.resolved-value" value)] - (swap! form update :extra-errors dissoc input-name) - (reset! hint* {:message message :type "hint"}))))))))] + (cond + ;; Untouched field (form just opened) + (not touched?) + (when (and (not error) + (csu/show-hint-on-open? (get-in @form [:data input-name]) value)) + (reset! hint* {:message (tr "workspace.tokens.resolved-value" value) + :type "hint"})) + + error + (if (csu/group-name-conflict-error? error token-name) + (swap! form assoc-in [:extra-errors ""] {:message error}) + (do + (swap! form assoc-in [:extra-errors input-name] {:message error}) + (reset! hint* {:message error :type "error"}))) + + :else + (let [message (tr "workspace.tokens.resolved-value" value)] + (swap! form update :extra-errors dissoc input-name) + (reset! hint* {:message message :type "hint"})))))))] (fn [] (rx/dispose! subs)))) @@ -309,7 +318,11 @@ resolve-stream (mf/with-memo [token] - (if-let [value (get-in token [:value input-name])] + ;; An alias token keeps its reference as a plain string value + (if-let [value (if (= input-name :reference) + (let [value (:value token)] + (when (string? value) value)) + (get-in token [:value input-name]))] (rx/behavior-subject value) (rx/subject))) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs index 75e2317814..4213b5d5d2 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs @@ -2,6 +2,7 @@ (:require [app.common.data.macros :as dm] [app.common.types.token :as cto] + [app.main.data.workspace.tokens.format :as dwtf] [app.util.i18n :refer [tr]] [cuerdas.core :as str])) @@ -126,6 +127,14 @@ (let [translated-string (tr "errors.tokens.name-collision" token-name)] (= error translated-string))) +(defn show-hint-on-open? + "True when an untouched field (form just opened) shows the resolved + value: only when it differs from the written one, so a plain `16` + gets no `Resolved value: 16` hint." + [input-value resolved-value] + (not= (str/lower (str/trim (dwtf/format-token-value input-value))) + (str/lower (str/trim (dwtf/format-token-value resolved-value))))) + (defn- referenced-token-name "Name of the token `reference` points to, when `reference` is exactly one `{name}` reference." diff --git a/frontend/src/app/main/ui/workspace/tokens/management/group.cljs b/frontend/src/app/main/ui/workspace/tokens/management/group.cljs index b1c6b0342d..8359921362 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/group.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/group.cljs @@ -57,6 +57,31 @@ :shadow i/drop-shadow "add")) +(defn- type-unfolded? + "A token type's section only stays open for the file and set it was + opened in, so switching sets doesn't carry open sections over." + [unfolded-token-types file-id set-id type] + (and (= (:file-id unfolded-token-types) file-id) + (= (:set-id unfolded-token-types) set-id) + (contains? (set (:types unfolded-token-types)) type))) + +(defn- editing-text? + [editor-state edition objects] + (or (not (empty? editor-state)) + (and (some? edition) + (= :text (:type (get objects edition)))))) + +(defn- pill-click-action + "Clicking a token pill applies or removes the token on the selection, + or warns while a text is being edited. Number tokens and an empty + selection do nothing." + [token selected-shapes is-editing-text] + (cond + (= :number (:type token)) nil + (empty? selected-shapes) nil + is-editing-text :warn-text-edition + :else :toggle)) + (def ^:private schema:token-group [:map [:type :keyword] @@ -80,17 +105,13 @@ current-file (mf/deref refs/file) - is-same-file-set? (and (= (:file-id unfolded-token-types-state) (:id current-file)) - (= (:set-id unfolded-token-types-state) selected-token-set-id)) - is-type-unfolded (and is-same-file-set? - (contains? (set (:types unfolded-token-types-state)) type)) + is-type-unfolded + (type-unfolded? unfolded-token-types-state (:id current-file) selected-token-set-id type) - editing-ref (mf/deref refs/workspace-editor-state) - edition (mf/deref refs/selected-edition) - objects (mf/deref refs/workspace-page-objects) - not-editing? (and (empty? editing-ref) - (not (and (some? edition) - (= :text (:type (get objects edition)))))) + editing-ref (mf/deref refs/workspace-editor-state) + edition (mf/deref refs/selected-edition) + objects (mf/deref refs/workspace-page-objects) + is-editing-text (editing-text? editing-ref edition objects) can-edit-tokens? (mf/use-ctx ctx/can-edit-tokens?) @@ -151,20 +172,22 @@ on-token-pill-click (mf/use-fn - (mf/deps not-editing? selected-ids tokens-lib selected-token-set-id) + (mf/deps is-editing-text selected-shapes selected-ids tokens-lib selected-token-set-id) (fn [event token] (let [token (ctob/get-token tokens-lib selected-token-set-id (:id token))] (dom/stop-propagation event) - ;; Number tokens can't be applied via button click - (when (not= (:type token) :number) - (if (and not-editing? (seq selected-shapes)) - (st/emit! (dwta/toggle-token {:token token - :shape-ids selected-ids})) - (when (seq selected-shapes) - (st/emit! (ntf/show {:content (tr "workspace.tokens.error-text-edition") - :type :toast - :level :warning - :timeout 3000}))))))))] + (case (pill-click-action token selected-shapes is-editing-text) + :toggle + (st/emit! (dwta/toggle-token {:token token + :shape-ids selected-ids})) + + :warn-text-edition + (st/emit! (ntf/show {:content (tr "workspace.tokens.error-text-edition") + :type :toast + :level :warning + :timeout 3000})) + + nil))))] (mf/use-effect (fn [] diff --git a/frontend/src/app/main/ui/workspace/tokens/sets/lists.cljs b/frontend/src/app/main/ui/workspace/tokens/sets/lists.cljs index 358d611703..0ee9c45400 100644 --- a/frontend/src/app/main/ui/workspace/tokens/sets/lists.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/sets/lists.cljs @@ -10,6 +10,7 @@ (:require [app.common.data.macros :as dm] [app.common.types.tokens-lib :as ctob] + [app.common.uuid :as uuid] [app.main.data.workspace.tokens.library-edit :as dwtl] [app.main.store :as st] [app.main.ui.context :as ctx] @@ -134,13 +135,16 @@ on-collapse-click (mf/use-fn + (mf/deps path on-toggle-collapse) (fn [event] (dom/prevent-default event) (dom/stop-propagation event) (on-toggle-collapse path))) on-double-click - (mf/use-fn (mf/deps id) #(on-start-edition id)) + (mf/use-fn + (mf/deps id on-start-edition) + #(on-start-edition id)) on-checkbox-click (mf/use-fn @@ -240,7 +244,7 @@ on-double-click (mf/use-fn - (mf/deps id is-new) + (mf/deps id is-new on-start-edition) (fn [] (when-not is-new (on-start-edition id)))) @@ -358,7 +362,13 @@ (fn [path] (swap! collapsed-paths* #(if (contains? % path) (disj % path) - (conj % path)))))] + (conj % path))))) + + ;; The set being created is not persisted yet, so its id is generated + ;; on every walk of the tree. Keep one for the lifetime of this + ;; creation so the node can be keyed by identity like the rest. + new-node-id + (mf/use-memo (mf/deps new-path) #(dm/str (uuid/next)))] (mf/with-effect [] (let [sub (rx/subs! (fn [paths'] @@ -378,7 +388,7 @@ (cond ^boolean is-group [:> sets-tree-set-group* - {:key index + {:key id :id id :label (peek path) :is-editing (= edition-id id) @@ -400,7 +410,7 @@ ^boolean is-new [:> sets-tree-set* - {:key index + {:key new-node-id :id id :set token-set :label "" @@ -420,7 +430,7 @@ :else [:> sets-tree-set* - {:key index + {:key id :id id :set token-set :label (peek path) diff --git a/frontend/src/app/main/ui/workspace/tokens/themes/create_modal.cljs b/frontend/src/app/main/ui/workspace/tokens/themes/create_modal.cljs index 3053141215..cf4b48566c 100644 --- a/frontend/src/app/main/ui/workspace/tokens/themes/create_modal.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/themes/create_modal.cljs @@ -260,6 +260,23 @@ (ctos/make-tokens-status :active-theme-ids #{(:id theme)} :active-set-ids (into #{} (map ctob/get-id sets))))) +(defn- toggle-set + "Ticks the set in the theme being edited, or unticks it when it is + ticked already." + [theme tokens-lib set-id] + (let [set (ctob/get-set tokens-lib set-id)] + (ctob/toggle-set theme (ctob/get-name set)))) + +(defn- toggle-set-group + "Ticks every set in the group for the theme being edited. When some + or all of them are ticked already, unticks them all." + [theme sets group-path] + (let [lib (make-lib-with-theme theme sets) + set-names (into #{} (map ctob/get-name) (ctob/get-sets-at-path lib group-path))] + (if (some (:sets theme) set-names) + (ctob/disable-sets theme set-names) + (ctob/enable-sets theme set-names)))) + (mf/defc edit-create-theme* [{:keys [change-view theme on-save is-editing has-prev-view]}] (let [ordered-token-sets (mf/deref refs/workspace-ordered-token-sets) @@ -325,33 +342,15 @@ on-toggle-token-set (mf/use-fn - (mf/deps current-theme tokens-lib) + (mf/deps tokens-lib) (fn [set-id] - (let [set (ctob/get-set tokens-lib set-id)] - (swap! current-theme* #(ctob/toggle-set % (ctob/get-name set)))))) + (swap! current-theme* toggle-set tokens-lib set-id))) on-toggle-token-set-group (mf/use-fn - (mf/deps current-theme ordered-token-sets) + (mf/deps ordered-token-sets) (fn [group-path] - (swap! current-theme* (fn [theme'] - ; TODOstatus ver si se puede añadir una función en files.tokens o types.tokens - ; según si necesitamos al final el status o no (que creo que no) - (let [lib' (make-lib-with-theme theme' ordered-token-sets) - status' (make-status-with-theme lib' theme') - active? (cfo/sets-at-path-all-active? status' lib' group-path) - sets-at-path (ctob/get-sets-at-path lib' group-path) - set-names (into #{} (map ctob/get-name) sets-at-path)] - (if (contains? #{:all :partial} active?) - (ctob/disable-sets theme' set-names) - (ctob/enable-sets theme' set-names))))))) - - on-click-token-set - (mf/use-fn - (mf/deps on-toggle-token-set) - (fn [set-id] - (let [set (ctob/get-set lib-with-theme set-id)] - (on-toggle-token-set (ctob/get-name set)))))] + (swap! current-theme* toggle-set-group ordered-token-sets group-path)))] [:div {:class (stl/css :themes-modal-wrapper)} [:> heading* {:level 2 :typography "headline-medium" :class (stl/css :themes-modal-title)} @@ -379,7 +378,7 @@ :token-sets token-sets :is-token-set-active token-set-active? :is-token-set-group-active token-set-group-active? - :on-select on-click-token-set + :on-select on-toggle-token-set :can-edit false :on-toggle-token-set on-toggle-token-set :on-toggle-token-set-group on-toggle-token-set-group diff --git a/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs b/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs index 931fb6e4df..9f5ac07459 100644 --- a/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs @@ -167,8 +167,9 @@ (mf/defc frame-grid* {::mf/wrap [mf/memo]} [{:keys [zoom transform selected focus vbox clip-rulers] :or {clip-rulers false}}] - (let [frames (->> (mf/deref refs/workspace-frames) - (filter has-grid?)) + (let [frames (mf/deref refs/workspace-frames) + frames (mf/with-memo [frames] + (into [] (filter has-grid?) frames)) transforming (when (some? transform) selected)] [:g.grid-display {:style {:pointer-events "none"} diff --git a/frontend/src/app/main/ui/workspace/viewport_wasm.cljs b/frontend/src/app/main/ui/workspace/viewport_wasm.cljs index c2a24a0d3e..23a319f991 100644 --- a/frontend/src/app/main/ui/workspace/viewport_wasm.cljs +++ b/frontend/src/app/main/ui/workspace/viewport_wasm.cljs @@ -16,6 +16,7 @@ [app.common.types.component :as ctk] [app.common.types.shape :as cts] [app.common.types.shape.layout :as ctl] + [app.main.data.event :as ev] [app.main.data.modal :as modal] [app.main.data.workspace :as dw] [app.main.data.workspace.transforms :as dwt] @@ -59,6 +60,7 @@ [app.main.ui.workspace.viewport.viewport-ref :as vp-ref :refer [create-viewport-ref]] [app.main.ui.workspace.viewport.widgets :as widgets] [app.render-wasm.api :as wasm.api] + [app.render-wasm.api.webgl :as webgl] [app.render-wasm.rulers-state :as rs] [app.util.debug :as dbg] [app.util.text-editor :as ted] @@ -71,6 +73,10 @@ ;; --- Viewport +;; Retries after the first failed WebGL context init before showing the +;; "WebGL unavailable" modal. +(def ^:private ^:const max-init-retries 3) + (defn- apply-modifiers-to-objects [objects modifiers] (->> modifiers @@ -466,21 +472,28 @@ (let [timeout-id-ref (volatile! nil) unmounted? (volatile! false) modal-shown? (volatile! false) + init-error (volatile! nil) show-unavailable (fn [] (when-not (or @unmounted? @modal-shown?) (vreset! modal-shown? true) (reset! canvas-init? false) - (st/emit! (modal/show {:type :webgl-unavailable})))) + (st/emit! (modal/show {:type :webgl-unavailable}) + (ev/event (assoc (webgl/get-environment) + ::ev/name "webgl-unavailable" + ::ev/origin "workspace" + :hint (or @init-error "no webgl2 context")))))) try-init (fn try-init [retries] (when-not @unmounted? + (vreset! init-error nil) (let [init? (try (wasm.api/init-canvas-context canvas) (catch :default e (js/console.error "Error initializing canvas context:" e) + (vreset! init-error (ex-message e)) false))] (cond init? @@ -496,7 +509,7 @@ (->> @wasm.api/module (p/fmap (fn [ready?] (when ready? - (try-init 3))))) + (try-init max-init-retries))))) (fn [] (vreset! unmounted? true) (when-let [timeout-id @timeout-id-ref] diff --git a/frontend/src/app/plugins.cljs b/frontend/src/app/plugins.cljs index 921cf740b9..f10e4213d4 100644 --- a/frontend/src/app/plugins.cljs +++ b/frontend/src/app/plugins.cljs @@ -8,6 +8,7 @@ "RPC for plugins runtime." (:require ["@penpot/plugins-runtime" :as runtime] + [app.main.data.event :as ev] [app.main.errors :as errors] [app.main.features :as features] [app.main.store :as st] @@ -20,14 +21,26 @@ [app.plugins.register :as preg] [app.plugins.ruler-guides :as rg] [app.plugins.shape :as shape] + [app.util.globals :as ug] [beicon.v2.core :as rx] [potok.v2.core :as ptk])) +(defn- runtime-available? + [] + (fn? (unchecked-get ug/global "ɵloadPlugin"))) + (defn init-plugins-runtime [] (runtime/initPluginsRuntime (fn [plugin-id] (api/create-context plugin-id))) - ;; Signal that runtime is ready - (preg/signal-runtime-ready)) + ;; Signal even when the runtime failed to start: MCP initialization waits + ;; for this promise and would otherwise hang forever. + (preg/signal-runtime-ready) + (runtime-available?)) + +(defn- runtime-unavailable-event + [] + (ev/event {::ev/name "plugins-runtime-unavailable" + :message (some-> (runtime/getPluginsRuntimeError) .-message)})) (defn initialize [] @@ -41,8 +54,10 @@ (rx/observe-on :async) (rx/filter #(features/active-feature? @st/state "plugins/runtime")) (rx/take 1) - (rx/tap init-plugins-runtime) - (rx/ignore))))) + (rx/map (fn [_] + (when-not (init-plugins-runtime) + (runtime-unavailable-event)))) + (rx/filter some?))))) ;; Prevent circular dependency (set! flex/shape-proxy? shape/shape-proxy?) diff --git a/frontend/src/app/plugins/file.cljs b/frontend/src/app/plugins/file.cljs index 59d222512b..28c4f9a0a6 100644 --- a/frontend/src/app/plugins/file.cljs +++ b/frontend/src/app/plugins/file.cljs @@ -54,7 +54,7 @@ :else (do (swap! data assoc :label value :created-by "user") - (->> (rp/cmd! :update-file-snapshot {:id (:id @data) :label value}) + (->> (rp/cmd! :update-file-snapshot {:id (:id @data) :file-id file-id :label value}) (rx/take 1) (rx/subs! #(st/emit! (se/event plugin-id "rename-version" :file-id file-id)))))))} @@ -95,7 +95,7 @@ :else (let [version-id (:id @data)] - (->> (rp/cmd! :delete-file-snapshot {:id version-id}) + (->> (rp/cmd! :delete-file-snapshot {:id version-id :file-id file-id}) (rx/map (constantly nil)) (rx/subs! resolve reject))))))) @@ -111,15 +111,22 @@ (u/reject-not-valid reject :pin "Only auto-saved versions can be pinned") :else - (let [params {:id (:id @data) - :label (ct/format-inst (:created-at @data) :localized-date)}] + (let [params {:id (:id @data) + :file-id file-id + :label (ct/format-inst (:created-at @data) :localized-date)}] (->> (rx/zip (rp/cmd! :get-team-users {:file-id file-id}) (rp/cmd! :update-file-snapshot params)) (rx/subs! - (fn [[users data]] + (fn [[users _]] (let [users (d/index-by :id users)] (st/emit! (se/event plugin-id "pin-version" :file-id file-id)) - (resolve (file-version-proxy plugin-id file-id users @data)))) + ;; the command answers `true`: the pinned row is the + ;; current one with the new label, as a user version + (resolve (file-version-proxy plugin-id file-id users + (assoc @data + :label (:label params) + :created-by "user" + :deleted-at nil))))) reject)))))))))) (defn file-proxy? [p] diff --git a/frontend/src/app/plugins/library.cljs b/frontend/src/app/plugins/library.cljs index faec7fad16..8fccd73b83 100644 --- a/frontend/src/app/plugins/library.cljs +++ b/frontend/src/app/plugins/library.cljs @@ -1022,7 +1022,8 @@ (let [file (u/locate-file file-id) component (u/locate-library-component file-id id) root (ctf/get-component-root (:data file) component)] - (when (some? root) + ;; A deleted component keeps its root only in its own objects, not on a page + (when (and (some? root) (not (:deleted component))) (shape/shape-proxy plugin-id file-id (:main-instance-page component) (:id root))))) :isVariant diff --git a/frontend/src/app/plugins/management.cljs b/frontend/src/app/plugins/management.cljs index 91671991f8..fe1662a11f 100644 --- a/frontend/src/app/plugins/management.cljs +++ b/frontend/src/app/plugins/management.cljs @@ -46,6 +46,14 @@ (update-fn state) state)))) +;; Dashboard events are logged by the backend, which reads the origin from +;; the `x-event-origin` header that `rp/cmd!` builds from the params meta. +(defn- with-origin + "Tags RPC `params` with the plugin or MCP origin, as `se/add-event` does + for frontend events." + [plugin-id params] + (se/add-event params plugin-id)) + (defn- refresh-restored-project [team-id project-id] (if (dashboard-of-team? @st/state team-id) @@ -65,8 +73,9 @@ (defn- restore-deleted-files "Uses the dashboard's Trash operation and waits for its final result." - [team-id ids] - (->> (rp/cmd! ::sse/restore-deleted-team-files {:team-id team-id :ids ids}) + [plugin-id team-id ids] + (->> (rp/cmd! ::sse/restore-deleted-team-files + (with-origin plugin-id {:team-id team-id :ids ids})) (rx/filter sse/end-of-stream?) (rx/take 1) (rx/map sse/get-payload) @@ -252,7 +261,7 @@ [plugin-id team-id options] (if-let [name (valid-name (obj/get options "name"))] (request plugin-id "manage:projects" - #(->> (rp/cmd! :create-project {:team-id team-id :name name}) + #(->> (rp/cmd! :create-project (with-origin plugin-id {:team-id team-id :name name})) (rx/map (fn [project] (assoc project :count 0))) (rx/tap (fn [project] (update-dashboard team-id @@ -309,7 +318,7 @@ :else (set-property plugin-id data :name name - (dtm/update-team {:id (:id @data) :name name})))))} + (dtm/update-team (with-origin plugin-id {:id (:id @data) :name name}))))))} :listProjects (fn [] (list-projects plugin-id (:id @data))) @@ -355,7 +364,7 @@ :else (set-property plugin-id data :name name - (dd/rename-project {:id (:id @data) :name name})))))} + (dd/rename-project (with-origin plugin-id {:id (:id @data) :name name}))))))} :pinned {:get #(boolean (:is-pinned @data)) @@ -370,9 +379,10 @@ :else (set-property plugin-id data :is-pinned value - (set-project-pin {:id (:id @data) - :team-id (:team-id @data) - :is-pinned value}))))} + (set-project-pin (with-origin plugin-id + {:id (:id @data) + :team-id (:team-id @data) + :is-pinned value})))))} :listFiles (fn [] @@ -395,7 +405,7 @@ (let [ids (into #{} (comp (filter (fn [file] (= id (:project-id file)))) (map :id)) files)] (if (seq ids) - (restore-deleted-files team-id ids) + (restore-deleted-files plugin-id team-id ids) (rx/throw (js/Error. "Cannot restore a project without recoverable files")))))) (rx/tap (fn [_] (swap! data dissoc :deleted-at))) (rx/mapcat (fn [_] (refresh-restored-project team-id id))))))) @@ -406,7 +416,7 @@ (request plugin-id "content:write" #(let [{:keys [id team-id]} @data features (set/difference (:features @st/state #{}) cfeat/frontend-only-features)] - (->> (rp/cmd! :create-file {:project-id id :name name :features features}) + (->> (rp/cmd! :create-file (with-origin plugin-id {:project-id id :name name :features features})) (rx/map (fn [file] (-> (dissoc file :data) (assoc :team-id team-id)))) (rx/tap (fn [file] (update-dashboard team-id (fn [state] (ptk/update (dd/file-created file) state))))) @@ -420,8 +430,9 @@ (js/Promise.reject (js/Error. "Expected a name with 1 to 250 characters")) (request plugin-id "manage:projects" #(let [{:keys [id team-id]} @data] - (->> (rp/cmd! :duplicate-project (cond-> {:project-id id} - (some? name) (assoc :name (valid-name name)))) + (->> (rp/cmd! :duplicate-project (with-origin plugin-id + (cond-> {:project-id id} + (some? name) (assoc :name (valid-name name))))) (rx/mapcat dd/project-with-file-count) (rx/tap (fn [project] (update-dashboard team-id (fn [state] (ptk/update (dd/project-duplicated project) state))))) @@ -432,7 +443,7 @@ (if-let [team-id (uuid/parse* (obj/get team "id"))] (request plugin-id "manage:teams" #(let [{:keys [id] :as project} @data] - (->> (rp/cmd! :move-project {:project-id id :team-id team-id}) + (->> (rp/cmd! :move-project (with-origin plugin-id {:project-id id :team-id team-id})) (rx/tap (fn [_] (swap! data assoc :team-id team-id) (update-dashboard (:team-id project) (fn [state] (update state :projects dissoc id))))) @@ -443,7 +454,7 @@ (fn [] (request plugin-id "manage:delete" #(let [{:keys [id team-id]} @data] - (->> (rp/cmd! :delete-project {:id id}) + (->> (rp/cmd! :delete-project (with-origin plugin-id {:id id})) (rx/tap (fn [_] (update-dashboard team-id (fn [state] (update state :projects dissoc id))))) (rx/map (constantly nil))))))))) @@ -480,7 +491,7 @@ :else (set-property plugin-id data :name name - (dd/rename-file {:id (:id @data) :name name})))))} + (dd/rename-file (with-origin plugin-id {:id (:id @data) :name name}))))))} :shared {:get #(boolean (:is-shared @data)) @@ -495,7 +506,7 @@ :else (set-property plugin-id data :is-shared value - (dd/set-file-shared {:id (:id @data) :is-shared value}))))} + (dd/set-file-shared (with-origin plugin-id {:id (:id @data) :is-shared value})))))} :open (fn [] @@ -506,7 +517,7 @@ :restore (fn [] (request plugin-id "content:write" - #(->> (restore-deleted-files (:team-id @data) #{(:id @data)}) + #(->> (restore-deleted-files plugin-id (:team-id @data) #{(:id @data)}) (rx/tap (fn [_] (swap! data dissoc :deleted-at :will-be-deleted-at))) (rx/mapcat (fn [_] (refresh-restored-project (:team-id @data) (:project-id @data))))))) @@ -519,7 +530,7 @@ (->> (rp/cmd! :get-project {:id project-id}) (rx/mapcat (fn [target] (if (= team-id (:team-id target)) - (rp/cmd! :move-files {:ids #{id} :project-id project-id}) + (rp/cmd! :move-files (with-origin plugin-id {:ids #{id} :project-id project-id})) (rx/throw (js/Error. "Cannot move a file to a project in another team"))))) (rx/tap (fn [_] (update-dashboard team-id (fn [state] (ptk/update (dd/move-files {:ids #{id} :project-id project-id}) state))) @@ -534,8 +545,9 @@ (js/Promise.reject (js/Error. "Expected a name with 1 to 250 characters")) (request plugin-id "content:write" #(let [{:keys [id team-id]} @data] - (->> (rp/cmd! :duplicate-file (cond-> {:file-id id} - (some? name) (assoc :name (valid-name name)))) + (->> (rp/cmd! :duplicate-file (with-origin plugin-id + (cond-> {:file-id id} + (some? name) (assoc :name (valid-name name))))) (rx/map (fn [file] (-> (dissoc file :data) (assoc :team-id team-id)))) (rx/tap (fn [file] (update-dashboard team-id (fn [state] (ptk/update (dd/file-created file) state))))) @@ -545,7 +557,7 @@ (fn [] (request plugin-id "manage:delete" #(let [{:keys [id project-id team-id]} @data] - (->> (rp/cmd! :delete-file {:id id}) + (->> (rp/cmd! :delete-file (with-origin plugin-id {:id id})) (rx/tap (fn [_] (update-dashboard team-id (fn [state] @@ -576,7 +588,7 @@ (fn [options] (if-let [name (valid-name (obj/get options "name"))] (request plugin-id "manage:teams" - #(->> (rp/cmd! :create-team {:name name :features features/global-enabled-features}) + #(->> (rp/cmd! :create-team (with-origin plugin-id {:name name :features features/global-enabled-features})) (rx/tap (fn [_] (st/emit! (dtm/fetch-teams)))) (rx/map (partial team-proxy plugin-id)))) (js/Promise.reject (js/Error. "Expected a name with 1 to 250 characters")))) diff --git a/frontend/src/app/plugins/register.cljs b/frontend/src/app/plugins/register.cljs index 3c919eed50..5c7c4b4fe6 100644 --- a/frontend/src/app/plugins/register.cljs +++ b/frontend/src/app/plugins/register.cljs @@ -16,6 +16,7 @@ [app.main.store :as st] [app.util.object :as obj] [beicon.v2.core :as rx] + [cuerdas.core :as str] [promesa.core :as p])) ;; Needs to be here because moving it to `app.main.data.workspace.mcp` will @@ -29,6 +30,33 @@ ;; Keeps the rollbacks not reported yet across reloads "allow:localstorage"}) +;; Permissions only granted to plugins that pass `global-allowed?` +(def global-permissions + #{"allow:global" "manage:projects" "manage:teams" "manage:delete"}) + +(def ^:private local-hosts + #{"localhost" "127.0.0.1" "[::1]" "::1"}) + +(defn- local-url? + "True when the url host is a loopback name, ignoring case and a + trailing dot." + [url] + (contains? local-hosts (some-> url u/uri :host str/lower (str/rtrim ".")))) + +(defn global-requested? + "True when the plugin asks for global scope or a management permission." + [{:keys [scope permissions]}] + (or (= "global" scope) + (boolean (some global-permissions permissions)))) + +(defn global-allowed? + "True when the plugin may run with global scope and use the management + API. The feature is still in progress, so only the bundled MCP plugin + and plugins served from localhost get it." + [{:keys [plugin-id host]}] + (or (= plugin-id mcp-plugin-id) + (local-url? host))) + ;; Promise that resolves when plugins runtime is initialized. ;; Lives here to avoid circular dependency: workspace.mcp -> app.plugins -> app.plugins.api -> workspace (defonce ^:private runtime-ready-promise (p/deferred)) @@ -121,9 +149,18 @@ :code code :icon icon :permissions (into #{} (map str) permissions)})] - (if (sm/validate ctp/schema:registry-entry manifest) - manifest - (.error js/console (clj->js (sm/explain ctp/schema:registry-entry manifest)))))) + (cond + (not (sm/validate ctp/schema:registry-entry manifest)) + (.error js/console (clj->js (sm/explain ctp/schema:registry-entry manifest))) + + (and (global-requested? manifest) + (not (global-allowed? manifest))) + (throw (ex-info "Global plugins can only be loaded from localhost" + {:type ::global-not-allowed + :host origin})) + + :else + manifest))) (defn subscribe-registry! "Subscribes f, called with no arguments on every registry change. @@ -273,13 +310,16 @@ (.error js/console "Failed to remove plugin:" err)))))))) (defn has-permission? - "True when the plugin was granted `permission`." + "True when the plugin was granted `permission`. The permissions in + `global-permissions` also need the plugin to pass `global-allowed?`." [plugin-id permission] (cond (= plugin-id "00000000-0000-0000-0000-000000000000") true (= plugin-id mcp-plugin-id) (contains? mcp-permissions permission) - :else (let [{:keys [permissions]} (dm/get-in @registry [:data plugin-id])] - (contains? permissions permission)))) + :else (let [{:keys [permissions] :as plugin} (dm/get-in @registry [:data plugin-id])] + (and (contains? permissions permission) + (or (not (contains? global-permissions permission)) + (global-allowed? plugin)))))) (defn- read-only-reason "Why plugin writes must fail in `state`, or nil when they can go ahead. diff --git a/frontend/src/app/plugins/shape.cljs b/frontend/src/app/plugins/shape.cljs index a313ae1ee6..3e771b5852 100644 --- a/frontend/src/app/plugins/shape.cljs +++ b/frontend/src/app/plugins/shape.cljs @@ -1535,8 +1535,8 @@ (let [objects (u/locate-objects file-id page-id) shape (u/locate-shape file-id page-id id)] (when (ctn/in-any-component? objects shape) - (let [[root component] (u/locate-component objects shape)] - (shape-proxy plugin-id (:component-file root) (:main-instance-page component) (:id root)))))) + (let [root (ctn/get-instance-root objects shape)] + (shape-proxy plugin-id file-id page-id (:id root)))))) :componentHead (fn [] @@ -1551,8 +1551,10 @@ (let [objects (u/locate-objects file-id page-id) shape (u/locate-shape file-id page-id id)] (when (ctn/in-any-component? objects shape) - (when-let [[head component] (u/locate-head-component objects shape)] - (lib-component-proxy plugin-id (:component-file head) (:id component)))))) + (let [[head component] (u/locate-head-component objects shape)] + ;; The component is nil when its library is not available + (when (some? component) + (lib-component-proxy plugin-id (:component-file head) (:id component))))))) :detach (fn [] diff --git a/frontend/src/app/plugins/system_events.cljs b/frontend/src/app/plugins/system_events.cljs index 716852d152..99d163e768 100644 --- a/frontend/src/app/plugins/system_events.cljs +++ b/frontend/src/app/plugins/system_events.cljs @@ -10,31 +10,35 @@ [app.main.store :as st] [app.plugins.register :as r])) +(defn- get-origin + [plugin-id] + (if (= plugin-id r/mcp-plugin-id) "mcp" "plugin")) + +(defn- get-context + "Returns the audit context of an action run by a plugin: the file it ran + in (none on the dashboard) and, for plugins other than the MCP, the + plugin name and URL. Events drop the nil values." + [state plugin-id] + (let [file-id (:current-file-id state)] + (if (= plugin-id r/mcp-plugin-id) + {:file-id file-id} + (let [plugin-data (r/get-plugin-data state plugin-id)] + {:file-id file-id + :plugin-name (:name plugin-data) + :plugin-url (:url plugin-data)})))) + ;; Formats an event from the plugin system (defn event [plugin-id name & {:as props}] - (if (= plugin-id r/mcp-plugin-id) - (-> props - (assoc ::ev/name name) - (assoc ::ev/origin "mcp") - (ev/event)) - - (let [plugin-data (r/get-plugin-data @st/state plugin-id)] - (-> props - (assoc ::ev/name name) - (assoc ::ev/origin "plugin") - (assoc ::ev/context - {:plugin-name (:name plugin-data) - :plugin-url (:url plugin-data)}) - (ev/event))))) + (-> props + (assoc ::ev/name name) + (assoc ::ev/origin (get-origin plugin-id)) + (assoc ::ev/context (get-context @st/state plugin-id)) + (ev/event))) (defn add-event [event plugin-id] - (let [plugin-data (r/get-plugin-data @st/state plugin-id)] - (with-meta - event - (if (= plugin-id r/mcp-plugin-id) - {::ev/origin "mcp"} - {::ev/origin "plugin" - ::ev/context {:plugin-name (:name plugin-data) - :plugin-url (:url plugin-data)}})))) + (with-meta + event + {::ev/origin (get-origin plugin-id) + ::ev/context (get-context @st/state plugin-id)})) diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index aab9321c6c..9e75946757 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -16,6 +16,7 @@ [app.common.fonts :as cfnt] [app.common.logging :as log] [app.common.math :as mth] + [app.common.render-wasm.api.modifiers :as modifiers] [app.common.render-wasm.api.props :as props] [app.common.render-wasm.api.select :as wselect] [app.common.render-wasm.helpers :as h] @@ -33,6 +34,7 @@ [app.common.types.text :as txt] [app.common.uuid :as uuid] [app.config :as cf] + [app.main.data.event :as-alias ev] [app.main.data.helpers :as dsh] [app.main.data.notifications :as ntf] [app.main.data.render-wasm :as drw] @@ -228,13 +230,6 @@ (def ^:const UUID-U8-SIZE 16) (def ^:const UUID-U32-SIZE (/ UUID-U8-SIZE 4)) -;; FIXME: Migrate this as we adjust the DTO structure in wasm -(def ^:const MODIFIER-U8-SIZE 40) -(def ^:const MODIFIER-U32-SIZE (/ MODIFIER-U8-SIZE 4)) -(def ^:const MODIFIER-TRANSFORM-U8-OFFSET-SIZE 16) -(def ^:const INPUT-MODIFIER-U8-SIZE 44) -(def ^:const INPUT-MODIFIER-U32-SIZE (/ INPUT-MODIFIER-U8-SIZE 4)) - (def ^:const GRID-LAYOUT-ROW-U8-SIZE 8) (def ^:const GRID-LAYOUT-COLUMN-U8-SIZE 8) (def ^:const GRID-LAYOUT-CELL-U8-SIZE 36) @@ -1129,6 +1124,10 @@ [hidden] (h/call wasm/internal-module "_set_shape_hidden" hidden)) +(defn set-shape-hide-fill-on-export + [hide] + (h/call wasm/internal-module "_set_shape_hide_fill_on_export" hide)) + (defn clear-shape-fills! "Clear the fills of the currently-selected shape (call `use-shape` first). Equivalent to `set-shape-fills` with an empty collection." @@ -2096,87 +2095,25 @@ (defn set-structure-modifiers [entries] - (when (and (initialized?) (not ^boolean (empty? entries))) - (let [size (mem/get-alloc-size entries 44) - offset (mem/alloc->offset-32 size) - heapu32 (mem/get-heap-u32) - heapf32 (mem/get-heap-f32)] - - - (reduce (fn [offset {:keys [type parent id index value]}] - (-> offset - (mem.h32/write-u32 heapu32 (sr/translate-structure-modifier-type type)) - (mem.h32/write-u32 heapu32 (d/nilv index 0)) - (mem.h32/write-uuid heapu32 parent) - (mem.h32/write-uuid heapu32 id) - (mem.h32/write-f32 heapf32 value))) - offset - entries) - - (h/call wasm/internal-module "_set_structure_modifiers")))) + (when (initialized?) + (modifiers/set-structure-modifiers entries))) ;; Axes the pixel grid rounds, as `propagate_modifiers` expects them. -(def ^:private pixel-precision - {:disabled 0 - :both 1 - :only-x 2 - :only-y 3}) - -(defn- pixel-precision-mode - "Encodes the pixel grid snapping for the renderer. `snap-ignore-axis` - names the axis to leave alone (`:x`, `:y` or nil)." - [snap-pixel? snap-ignore-axis] - (pixel-precision - (cond - (not snap-pixel?) :disabled - (= :x snap-ignore-axis) :only-y - (= :y snap-ignore-axis) :only-x - :else :both))) - (defn propagate-modifiers - "Propagates geometry modifiers through the WASM shape tree. + "Propagates geometry transforms when the renderer is ready. - Rounds the resulting geometry to the pixel grid when `snap-pixel?` is set, - skipping the axis named by `snap-ignore-axis` (`:x`, `:y` or nil). + Args: + entries: Collection of [uuid {:transform matrix :kind kind}]. + snap-pixel?: Whether to round geometry to the pixel grid. + snap-ignore-axis: Axis to leave unsnapped, :x, :y or nil. - Always returns a vector. When the context is not ready (lost / mid-reload) - or `entries` is empty, returns `[]` so callers never receive `nil` (which - would trip `set-modifiers`' vector assert)." + Returns: + A vector of [uuid matrix], or [] for an unavailable renderer or empty + entries." [entries snap-pixel? snap-ignore-axis] - (if-not (and (initialized?) (not ^boolean (empty? entries))) - [] - (let [size (mem/get-alloc-size entries INPUT-MODIFIER-U8-SIZE) - offset (mem/alloc->offset-32 size) - heapf32 (mem/get-heap-f32) - heapu32 (mem/get-heap-u32)] - - (reduce (fn [offset [id data]] - (let [transform (:transform data) - kind (:kind data)] - (-> offset - (mem.h32/write-uuid heapu32 id) - (mem.h32/write-matrix heapf32 transform) - (mem.h32/write-u32 heapu32 (sr/translate-transform-entry-kind kind))))) - offset - entries) - - (let [precision (pixel-precision-mode snap-pixel? snap-ignore-axis) - offset (-> (h/call wasm/internal-module "_propagate_modifiers" precision) - (mem/->offset-32)) - heapf32 (mem/get-heap-f32) - heapu32 (mem/get-heap-u32) - length (aget heapu32 offset) - max-offset (+ offset 1 (* length MODIFIER-U32-SIZE)) - result (loop [result (transient []) - offset (inc offset)] - (if (< offset max-offset) - (let [entry (dr/read-modifier-entry heapu32 heapf32 offset)] - (recur (conj! result entry) - (+ offset MODIFIER-U32-SIZE))) - (persistent! result)))] - - (mem/free) - result)))) + (if (initialized?) + (modifiers/propagate-modifiers entries snap-pixel? snap-ignore-axis) + [])) (defn get-selection-rect [entries] @@ -2211,48 +2148,29 @@ (defn clean-modifiers [] (when (initialized?) - (h/call wasm/internal-module "_clean_modifiers"))) + (modifiers/clean-modifiers))) (defn set-modifiers-start - "Enter interactive transform mode (drag / resize / rotate). Enables - fast-mode effect skipping in the renderer and activates an atlas - backdrop so tiles do not appear sequentially or flicker while the - gesture is in progress." + "Enables interactive rendering for shape transforms when the renderer is ready." [] (when (initialized?) - (h/call wasm/internal-module "_set_modifiers_start"))) + (modifiers/set-modifiers-start))) (defn set-modifiers-end - "Leave interactive transform mode. Cancels any pending async render - scheduled under it; the caller is expected to trigger a full-quality - render (via `request-render`) once the gesture is committed." + "Ends interactive rendering for shape transforms when the renderer is ready. + + A subsequent render produces the final full-quality image." [] (when (initialized?) - (h/call wasm/internal-module "_set_modifiers_end"))) + (modifiers/set-modifiers-end))) (defn set-modifiers - [modifiers] + [entries] (when (initialized?) - ;; Assert only when we would touch WASM; callers may still build - ;; modifiers while the context is unavailable. - (assert (vector? modifiers) "expected a vector for `set-modifiers`") - - (let [length (count modifiers)] - (when (pos? length) - (let [offset (mem/alloc->offset-32 (* MODIFIER-U8-SIZE length)) - heapu32 (mem/get-heap-u32) - heapf32 (mem/get-heap-f32)] - - (reduce (fn [offset [id transform]] - (-> offset - (mem.h32/write-uuid heapu32 id) - (mem.h32/write-matrix heapf32 transform))) - offset - modifiers) - - (h/call wasm/internal-module "_set_modifiers") - - (request-render "set-modifiers")))))) + (assert (vector? entries) "expected a vector for `set-modifiers`") + (when (seq entries) + (modifiers/set-modifiers entries) + (request-render "set-modifiers")))) (defn initialize-viewport [base-objects zoom vbox & @@ -2389,6 +2307,17 @@ (set-render-options! new-dpr) (resize-viewbox css-w css-h))))) +(defn- emit-webgl-event + "Send a WebGL audit event with the environment details. Built as a plain + data event so this namespace does not load `app.main.data.event`, which + the worker build cannot run." + [event-name props] + (st/emit! (ptk/data-event ::ev/event + (merge (webgl/get-environment) + props + {::ev/name event-name + ::ev/origin "workspace"})))) + (defn- on-webgl-context-lost [event] (dom/prevent-default event) @@ -2405,7 +2334,8 @@ :type :toast :level :warning :timeout 5000})) - (st/emit! (drw/context-lost))) + (st/emit! (drw/context-lost)) + (emit-webgl-event "webgl-context-lost" {})) (defn- on-webgl-context-restored [event] @@ -2422,6 +2352,7 @@ ;; only after the full reload so layout/viewport sync runs ;; against a live renderer with `wasm/ready?` true. (st/emit! (drw/context-restored)) + (emit-webgl-event "webgl-context-restored" {}) (listen-tiles-render-complete-once! end-context-loss-overlay!) (st/async-emit! (ntf/show {:content (tr "webgl.webgl-context-recovered.toast") @@ -2472,6 +2403,7 @@ (let [{:keys [context handle]} result] (set! wasm/gl-context-handle handle) (set! wasm/gl-context context) + (webgl/remember-gpu-info context) ;; `init-context` already ran `_init` through `_set_browser` in ;; order. Publish the handles, then mark live before sizing so the @@ -2609,6 +2541,9 @@ (p/catch (fn [cause] (ug/dispatch! (ug/event "penpot:wasm:reload-failed")) + (emit-webgl-event "webgl-reload-failed" + {:hint (ex-message cause) + :code (some-> (ex-data cause) :code name)}) (reset! wasm/reloading? false) (reset! wasm/context-lost? true) (clear-canvas) @@ -2916,8 +2851,9 @@ (defn render-shape-pixels "Renders a shape subtree to encoded image bytes on the GPU export surface. `format` is :png, :jpeg or :webp; jpeg is flattened onto white on the Rust - side, since it has no alpha channel." - [shape-id scale format] + side, since it has no alpha channel. With `export?` the root drops its + fills when it hides them on export." + [shape-id scale format export?] (when (initialized?) (let [buffer (uuid/get-u32 shape-id) offset (h/call wasm/internal-module "_render_shape_pixels" @@ -2926,7 +2862,8 @@ (aget buffer 2) (aget buffer 3) scale - (sr/translate-raster-format format))] + (sr/translate-raster-format format) + export?)] (read-raster-result offset)))) (defn render-shape-raster diff --git a/frontend/src/app/render_wasm/api/webgl.cljs b/frontend/src/app/render_wasm/api/webgl.cljs index 3fe8a7b0ba..cde2622727 100644 --- a/frontend/src/app/render_wasm/api/webgl.cljs +++ b/frontend/src/app/render_wasm/api/webgl.cljs @@ -11,6 +11,7 @@ [app.common.math :as mth] [app.common.render-wasm.helpers :as h] [app.common.render-wasm.wasm :as wasm] + [cuerdas.core :as str] [promesa.core :as p])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -314,3 +315,73 @@ void main() { (resolve false))) (set! (.-src img) uri)))) (p/resolved false))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Environment +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- get-gpu-family + "Normalize a GPU renderer string to a GPU family. Telemetry sends the + family, not the full string, which can help fingerprint a device." + [renderer] + (let [renderer (str/lower renderer) + has? (fn [& parts] (some #(str/includes? renderer %) parts))] + (cond + (has? "swiftshader") "swiftshader" + (has? "llvmpipe" "softpipe") "llvmpipe" + (has? "microsoft basic render") "microsoft-basic-render" + (has? "apple") "apple" + (has? "nvidia" "geforce" "quadro") "nvidia" + (has? "amd" "radeon") "amd" + (has? "intel") "intel" + (has? "adreno") "adreno" + (has? "mali") "mali" + (has? "powervr") "powervr" + :else "other"))) + +(defn- get-gpu-backend + "Graphics API behind the WebGL implementation (ANGLE names it in the + renderer string), or nil when the string does not say." + [renderer] + (let [renderer (str/lower renderer) + has? (fn [& parts] (some #(str/includes? renderer %) parts))] + (cond + (has? "direct3d11" "d3d11") "d3d11" + (has? "direct3d9" "d3d9") "d3d9" + (has? "metal") "metal" + (has? "vulkan") "vulkan" + (has? "opengl") "opengl"))) + +(defn- read-gpu-info + "GPU details of a live WebGL context, or nil when it is lost." + [^js gl] + (when-not (.isContextLost gl) + (let [^js ext (.getExtension gl "WEBGL_debug_renderer_info") + renderer (or (when ext + (.getParameter gl (.-UNMASKED_RENDERER_WEBGL ext))) + (.getParameter gl (.-RENDERER gl)))] + (when (string? renderer) + {:gpu-family (get-gpu-family renderer) + :gpu-backend (get-gpu-backend renderer) + :max-texture-size (.getParameter gl (.-MAX_TEXTURE_SIZE gl))})))) + +(defonce ^:private gpu-info + (volatile! nil)) + +(defn remember-gpu-info + "Keep the GPU details of `gl`, a context the renderer just created, so + later events can send them after that context is lost." + [gl] + (when-let [info (try (read-gpu-info gl) (catch :default _ nil))] + (vreset! gpu-info info))) + +(defn get-environment + "Environment details for WebGL telemetry: GPU family, GPU backend and max + texture size when already remembered, device memory, CPU cores and + whether the app runs inside Electron. Never creates a WebGL context." + [] + (let [^js navigator js/navigator] + (assoc @gpu-info + :device-memory (.-deviceMemory navigator) + :cpu-cores (.-hardwareConcurrency navigator) + :electron (str/includes? (.-userAgent navigator) "Electron/")))) diff --git a/frontend/src/app/render_wasm/deserializers.cljs b/frontend/src/app/render_wasm/deserializers.cljs index 882c6f22be..0441298220 100644 --- a/frontend/src/app/render_wasm/deserializers.cljs +++ b/frontend/src/app/render_wasm/deserializers.cljs @@ -5,28 +5,8 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.render-wasm.deserializers (:require - [app.common.data :as d] [app.common.geom.matrix :as gmt] - [app.common.geom.point :as gpt] - [app.common.uuid :as uuid])) - -(defn read-modifier-entry - [heapu32 heapf32 offset] - (let [id1 (aget heapu32 (+ offset 0)) - id2 (aget heapu32 (+ offset 1)) - id3 (aget heapu32 (+ offset 2)) - id4 (aget heapu32 (+ offset 3)) - - a (aget heapf32 (+ offset 4)) - b (aget heapf32 (+ offset 5)) - c (aget heapf32 (+ offset 6)) - d (aget heapf32 (+ offset 7)) - e (aget heapf32 (+ offset 8)) - f (aget heapf32 (+ offset 9))] - - (d/vec2 (uuid/from-unsigned-parts id1 id2 id3 id4) - (gmt/matrix a b c d e f)))) - + [app.common.geom.point :as gpt])) (defn read-selection-rect [heapf32 offset] diff --git a/frontend/src/app/render_wasm/shape.cljs b/frontend/src/app/render_wasm/shape.cljs index dfbcf37bb4..7190062f9f 100644 --- a/frontend/src/app/render_wasm/shape.cljs +++ b/frontend/src/app/render_wasm/shape.cljs @@ -191,6 +191,9 @@ :hidden (api/set-shape-hidden v) + :hide-fill-on-export + (api/set-shape-hide-fill-on-export (boolean v)) + :shapes (api/set-shape-children v) diff --git a/frontend/src/app/render_wasm/text_editor.cljs b/frontend/src/app/render_wasm/text_editor.cljs index 851f08fbb8..322e12b59e 100644 --- a/frontend/src/app/render_wasm/text_editor.cljs +++ b/frontend/src/app/render_wasm/text_editor.cljs @@ -24,7 +24,7 @@ (def multiple-state-multiple (sr/translate-multiple-state :multiple)) -(def ^:const TEXT_EDITOR_STYLES_METADATA_SIZE (* 31 4)) +(def ^:const TEXT_EDITOR_STYLES_METADATA_SIZE (* 33 4)) (def ^:const TEXT_EDITOR_STYLES_FILL_SOLID 0) (def ^:const TEXT_EDITOR_STYLES_FILL_LINEAR_GRADIENT 1) (def ^:const TEXT_EDITOR_STYLES_FILL_RADIAL_GRADIENT 2) @@ -325,6 +325,8 @@ font-weight-value (aget heap-i32 (+ u32-offset 24)) line-height-value (aget heap-f32 (+ u32-offset 29)) letter-spacing-value (aget heap-f32 (+ u32-offset 30)) + list-style-type-state (aget heap-u32 (+ u32-offset 31)) + list-style-type-value (aget heap-u32 (+ u32-offset 32)) font-id (fonts/uuid->font-id font-family-id-value) font-style-value (sr/untranslate-font-style (text-editor-get-style-property font-style-state font-style-raw-value)) font-variant-id-computed (text-editor-compute-font-variant-id font-id font-weight-value font-style-value) @@ -356,6 +358,7 @@ :text-direction (sr/untranslate-text-direction (text-editor-get-style-property text-direction-state text-direction-value)) :text-decoration (sr/untranslate-text-decoration (text-editor-get-style-property text-decoration-state text-decoration-value)) :text-transform (sr/untranslate-text-transform (text-editor-get-style-property text-transform-state text-transform-value)) + :list-style-type (sr/untranslate-list-style-type (text-editor-get-style-property list-style-type-state list-style-type-value)) ;; WASM reports size/weight as numbers, but the rest of Penpot (and the backend schema) expects strings. :line-height (let [height (text-editor-get-style-property line-height-state line-height-value)] (if (= height :multiple) height (str height))) diff --git a/frontend/src/app/util/code_gen/style_css.cljs b/frontend/src/app/util/code_gen/style_css.cljs index 2534404394..fa04b1e89c 100644 --- a/frontend/src/app/util/code_gen/style_css.cljs +++ b/frontend/src/app/util/code_gen/style_css.cljs @@ -13,12 +13,14 @@ [app.common.geom.shapes.bounds :as gsb] [app.common.geom.shapes.points :as gpo] [app.common.text :as legacy.txt] + [app.common.types.color :as cc] [app.common.types.shape.layout :as ctl] [app.common.types.text :as types.text] [app.main.ui.shapes.text.styles :as sts] [app.util.code-gen.common :as cgc] [app.util.code-gen.style-css-formats :refer [format-value format-shadow->css]] [app.util.code-gen.style-css-values :refer [get-value]] + [app.util.object :as obj] [cuerdas.core :as str])) ;; @@ -230,6 +232,11 @@ body { [(str/kebab key) (unchecked-get properties key)]))) nil)) +(defn- list-paragraph? + [node] + (and (= "paragraph" (:type node)) + (types.text/list-paragraph? node))) + (defn node->css [shape shape-selector node] (let [properties @@ -241,7 +248,9 @@ body { (sts/generate-paragraph-set-styles shape) (:paragraph "paragraph") - (sts/generate-paragraph-styles shape node) + (cond-> (sts/generate-paragraph-styles shape node) + (list-paragraph? node) + (obj/set! "listStyleType" (:list-style-type node))) (sts/generate-text-styles shape node))] (dm/fmt @@ -249,15 +258,47 @@ body { (dm/str shape-selector " ." (:$id node)) (format-js-styles properties nil)))) +(defn- marker-color + "The bullet takes its span's text color. A `::marker` can't paint a gradient, + so a gradient fill gives its first stop." + [shape span] + (if-let [gradient (or (-> span :fills first :fill-color-gradient) + (:fill-color-gradient span))] + (let [{:keys [color opacity]} (first (:stops gradient)) + [r g b a] (cc/hex->rgba color opacity)] + (str/format "rgba(%s, %s, %s, %s)" r g b a)) + (obj/get (sts/generate-text-styles shape span) "color"))) + +(defn- list-marker-css + "Paragraphs set `font-size: 0`, so the bullet takes its size (and color) from + the item's first span, as it does on the canvas." + [shape shape-selector node] + (let [span (first (:children node)) + font-size (d/nilv (:font-size span) (:font-size types.text/default-text-attrs)) + color (marker-color shape span)] + (dm/fmt + ".% {\n%\n}" + (dm/str shape-selector " ." (:$id node) "::marker") + (format-js-styles (cond-> #js {:fontSize (dm/str font-size "px")} + (some? color) (obj/set! "color" color)) + nil)))) + (defn generate-text-css [shape] - (let [selector (cgc/shape->selector shape)] - (->> shape - :content - (legacy.txt/index-content) - (types.text/node-seq) - (map #(node->css shape selector %)) - (str/join "\n")))) + (let [selector (cgc/shape->selector shape) + nodes (->> shape + :content + (legacy.txt/index-content) + (types.text/node-seq)) + node-rules (mapcat (fn [node] + (cond-> [(node->css shape selector node)] + (list-paragraph? node) + (conj (list-marker-css shape selector node)))) + nodes) + list-rules (when (some list-paragraph? nodes) + [(dm/fmt ".% ul {\n%\n}" selector + (format-js-styles #js {:margin 0 :padding 0} nil))])] + (str/join "\n" (concat node-rules list-rules)))) (defn get-shape-css-selector ([objects shape] diff --git a/frontend/src/app/worker.cljs b/frontend/src/app/worker.cljs index 5df69ac365..be8c80a3c4 100644 --- a/frontend/src/app/worker.cljs +++ b/frontend/src/app/worker.cljs @@ -11,6 +11,7 @@ [app.common.schema :as sm] [app.common.types.objects-map] [app.util.object :as obj] + [app.worker.component-thumbnails] [app.worker.exports] [app.worker.impl :as impl] [app.worker.index] diff --git a/frontend/src/app/worker/component_thumbnails.cljs b/frontend/src/app/worker/component_thumbnails.cljs new file mode 100644 index 0000000000..94dd83120c --- /dev/null +++ b/frontend/src/app/worker/component_thumbnails.cljs @@ -0,0 +1,148 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.worker.component-thumbnails + "Renders component thumbnails on the worker WASM module from the shapes + of their mains, which the main thread sends. Sends each render back to + be shown, and uploads the ones asked for." + (:require + [app.common.data.macros :as dm] + [app.common.logging :as log] + [app.common.math :as mth] + [app.common.render-wasm.wasm :as wasm] + [app.common.thumbnails :as thc] + [app.common.uri :as u] + [app.config :as cf] + [app.render-wasm.api :as wasm.api] + [app.util.http :as http] + [app.util.webapi :as wapi] + [app.worker.impl :as impl] + [app.worker.wasm :as wwasm] + [beicon.v2.core :as rx])) + +;; Bounds for a thumbnail's longest side, in pixels. Full-size renders of big +;; components hold the worker for seconds, delaying its other work. +(def ^:private ^:const min-target-size 200) +(def ^:private ^:const max-target-size 512) + +(def ^:private ^:const canvas-size 100) + +(defn- thumbnail-scale + [longest-side] + (if (pos? longest-side) + (-> 1 + (mth/max (/ min-target-size longest-side)) + (mth/min (/ max-target-size longest-side))) + 1)) + +(defn- render-component-root + [objects root-id] + (let [{ext-w :width ext-h :height} (wasm.api/get-shape-extrect root-id) + {sel-w :width sel-h :height} (dm/get-in objects [root-id :selrect]) + longest-side (mth/max (or ext-w sel-w) (or ext-h sel-h)) + scale (thumbnail-scale longest-side) + ;; A thumbnail is a preview, not an export: never drop the root's + ;; fills, even when it hides them on export. + bytes (wasm.api/render-shape-pixels root-id scale :png false)] + (when (and (some? bytes) (pos? (.-length ^js bytes))) + (wapi/create-blob bytes "image/png")))) + +(defn- render-batch + "Renders each of `requests` from the shapes of its main, in `:objects`, + to a PNG blob in one job on the worker WASM module. Emits each request + that rendered, with its `:blob`. Never fails: an error would end the + stream of requests, and with it every later render, so it is logged and + the batch ends." + [requests] + (let [objects (into {} (mapcat :objects) requests)] + (->> (rx/from @wwasm/module) + (rx/mapcat + (fn [_] + (rx/create + (fn [subs] + (let [canvas (js/OffscreenCanvas. canvas-size canvas-size) + vbox {:x 0 :y 0 :width canvas-size :height canvas-size}] + (if (wasm.api/init-canvas-context canvas) + (try + (reset! wasm/disable-request-render? true) + ;; `:on-render` runs once the images and fonts are in. + (wasm.api/initialize-viewport + objects 1 vbox + :force-sync true + :on-render + (fn [] + (try + (doseq [{:keys [root-id] :as request} requests] + (when-let [blob (render-component-root objects root-id)] + (rx/push! subs (assoc request :blob blob)))) + (rx/end! subs) + (catch :default cause + (rx/error! subs cause)) + (finally + (wasm.api/clear-canvas))))) + (catch :default cause + (wasm.api/clear-canvas) + (rx/error! subs cause))) + (rx/error! subs (ex-info "unable to init wasm canvas" {:roots (count requests)}))) + nil))))) + (rx/catch (fn [cause] + (log/error :hint "error rendering component thumbnails" + :roots (count requests) + :cause cause) + (rx/empty)))))) + +(defn- upload-thumbnail + [file-id object-id blob] + (let [params {:file-id file-id :object-id object-id :tag "component"}] + (->> (http/send! {:method :post + :uri (u/join cf/public-uri "api/main/methods/create-file-object-thumbnail") + :credentials "include" + :query params + :body (http/form-data (assoc params :media blob))}) + (rx/subs! (fn [response] + (when-not (http/success? response) + (log/wrn :hint "unable to upload component thumbnail" + :object-id object-id :status (:status response)))) + (fn [cause] + (log/wrn :hint "unable to upload component thumbnail" + :object-id object-id :cause cause)))))) + +;; The stream the main thread reads the renders from; a new +;; `:thumbnails/watch-components` ends the previous one. +(defonce ^:private output* (atom nil)) + +(defn- add-render + "Sends `render` to the main thread to be shown, and uploads it when it + asks for it." + [{:keys [file-id page-id root-id blob persist?]}] + (let [object-id (thc/fmt-object-id file-id page-id root-id "component")] + (some-> @output* (rx/push! {:object-id object-id :blob blob})) + (when persist? + (upload-thumbnail file-id object-id blob)))) + +;; The requests from the main thread, rendered in batches, one at a time. +;; A batch takes the requests up to a 300 ms pause. +(defonce ^:private requests + (let [subject (rx/subject)] + (->> subject + (rx/buffer-until (rx/debounce 300 subject)) + ;; TODO: keep one request per main, so a main asked for twice in a + ;; batch renders and uploads once. + (rx/mapcat #(wwasm/submit (partial render-batch %))) + (rx/subs! add-render)) + subject)) + +(defmethod impl/handler :thumbnails/render-component + [message _] + (rx/push! requests (select-keys message [:file-id :page-id :root-id :objects :persist?])) + nil) + +(defmethod impl/handler :thumbnails/watch-components + [_ _] + (let [output (rx/subject)] + (some-> @output* rx/end!) + (reset! output* output) + output)) diff --git a/frontend/src/app/worker/impl.cljs b/frontend/src/app/worker/impl.cljs index c7f65dc1b7..aa060c7a6b 100644 --- a/frontend/src/app/worker/impl.cljs +++ b/frontend/src/app/worker/impl.cljs @@ -8,7 +8,8 @@ "Dispatcher for messages received from the main thread." (:require [app.common.logging :as log] - [app.config :as cf])) + [app.config :as cf] + [app.main.store :as st])) (log/set-level! :info) @@ -43,3 +44,9 @@ (set! cf/build-date build-date)) nil) + +(defmethod handler :fonts/sync + [{:keys [fonts]}] + ;; The render looks the team's custom fonts up in the store + (swap! st/state assoc :fonts fonts) + nil) diff --git a/frontend/src/app/worker/thumbnails.cljs b/frontend/src/app/worker/thumbnails.cljs index bed81a00e9..70f8933303 100644 --- a/frontend/src/app/worker/thumbnails.cljs +++ b/frontend/src/app/worker/thumbnails.cljs @@ -7,7 +7,6 @@ (ns app.worker.thumbnails (:require ["react-dom/server" :as rds] - [app.common.data.macros :as dm] [app.common.geom.rect :as grc] [app.common.geom.shapes.bounds :as gsb] [app.common.logging :as log] @@ -107,37 +106,6 @@ ;; WASM RENDERING ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(mf/defc svg-wrapper* - {::mf/private true} - [{:keys [uri background width height]}] - [:svg {:version "1.1" - :xmlns "http://www.w3.org/2000/svg" - :xmlnsXlink "http://www.w3.org/1999/xlink" - - :style {:width "100%" - :height "100%" - :background background} - :fill "none" - :viewBox (dm/str "0 0 " width " " height)} - [:image {:xlinkHref uri - :width width - :height height}]]) - -(defn- blob->uri - [blob] - (.readAsDataURL (js/FileReaderSync.) blob)) - -(defn- render-canvas-blob - [canvas width height background] - (->> (.convertToBlob ^js canvas) - (p/fmap (fn [blob] - (rds/renderToStaticMarkup - (mf/element svg-wrapper* - #js {:uri (blob->uri blob) - :width width - :height height - :background background})))))) - (defn- render-thumbnail-with-wasm [{:keys [file-id revn width] :as _message}] (->> (rx/from @wwasm/module) @@ -170,13 +138,13 @@ (wasm.api/render-sync-shape (:id frame)) (wasm.api/render-sync)) - (->> (render-canvas-blob canvas width height bgcolor) - (p/fnly (fn [data cause] + (->> (.convertToBlob ^js canvas #js {:type "image/png"}) + (p/fnly (fn [blob cause] (wasm.api/clear-canvas) (if cause (rx/error! subs cause) (rx/push! subs - {:data data + {:blob blob :file-id file-id :revn revn})) (rx/end! subs))))))) diff --git a/frontend/test/frontend_tests/code_gen_style_test.cljs b/frontend/test/frontend_tests/code_gen_style_test.cljs index fb10dfef85..937df0b8ca 100644 --- a/frontend/test/frontend_tests/code_gen_style_test.cljs +++ b/frontend/test/frontend_tests/code_gen_style_test.cljs @@ -10,14 +10,17 @@ Each test guards against a concrete bug found in the CSS/HTML generation of layout children and text shapes." (:require + ["react-dom/server" :as rds] [app.common.geom.matrix :as gmt] [app.common.geom.point :as gpt] [app.common.geom.rect :as grc] [app.common.uuid :as uuid] + [app.main.ui.shapes.text.html-text :as html-text] [app.util.code-gen.markup-html :as html] [app.util.code-gen.style-css :as css] [cljs.test :refer [deftest is testing] :include-macros true] - [cuerdas.core :as str])) + [cuerdas.core :as str] + [rumext.v2 :as mf])) ;; --- Builders ------------------------------------------------------------ @@ -194,3 +197,110 @@ (is (str/includes? markup "root-0") "the text nodes must expose their $id as a class for the CSS to apply") (is (str/includes? markup "root-0-paragraph-set-0-paragraph-0"))))) + +;; --- Text lists ---------------------------------------------------------- + +(defn- list-paragraph + [text list-style-type] + (cond-> {:type "paragraph" + :children [{:text text + :font-size "20" + :fills [{:fill-color "#ff0000" :fill-opacity 1}]}]} + (some? list-style-type) + (assoc :list-style-type list-style-type))) + +(defn- text-shape-with + [paragraphs] + {:id (uuid/next) :name "Text" :type :text + :parent-id uuid/zero :frame-id uuid/zero + :selrect (grc/make-rect 0 0 100 20) + :points (pts 0 0 100 20) + :grow-type :fixed + :content {:type "root" + :children [{:type "paragraph-set" :children paragraphs}]}}) + +(defn- block-tags + "The p/ul/li tags of `markup`, in order." + [markup] + (->> (re-seq #"<(/?)(p|ul|li)[ >]" markup) + (map (fn [[_ close tag]] (str close tag))))) + +(deftest text-markup-groups-list-paragraphs-into-lists + (let [text (text-shape-with [(list-paragraph "a" nil) + (list-paragraph "b" "disc") + (list-paragraph "c" "disc") + (list-paragraph "d" "none") + (list-paragraph "e" "disc")]) + markup (html/generate-markup (objects text) [text])] + (is (= ["p" "/p" "ul" "li" "/li" "li" "/li" "/ul" "p" "/p" "ul" "li" "/li" "/ul"] + (block-tags markup))) + (testing "list items keep the paragraph class and direction" + (is (str/includes? markup "
      • "))))) + +(deftest text-markup-without-lists-has-no-list-tags + (let [text (text-shape-with [(list-paragraph "a" nil) (list-paragraph "b" "none")]) + markup (html/generate-markup (objects text) [text])] + (is (= ["p" "/p" "p" "/p"] (block-tags markup))))) + +(deftest text-css-styles-list-items + (let [text (text-shape-with [(list-paragraph "a" nil) + (list-paragraph "b" "disc")]) + css (css/generate-text-css text)] + (testing "the list item rule sets the list style" + (is (re-find #"(?s)\.root-0-paragraph-set-0-paragraph-1 \{[^}]*list-style-type: disc;" css)) + (is (not (re-find #"(?s)\.root-0-paragraph-set-0-paragraph-0 \{[^}]*list-style-type" css)))) + (testing "the marker gets the first span's size and color" + (is (re-find #"(?s)\.root-0-paragraph-set-0-paragraph-1::marker \{[^}]*font-size: 20px;[^}]*color: rgba\(255, 0, 0, 1\);" css))) + (testing "the list drops the browser's spacing" + (is (= 1 (count (re-seq #" ul \{" css)))) + (is (re-find #"(?s) ul \{\s*margin: 0;\s*padding: 0;\s*\}" css))))) + +(deftest text-css-without-lists-has-no-list-rules + (let [css (css/generate-text-css (text-shape-with [(list-paragraph "a" nil)]))] + (is (not (str/includes? css "list-style"))) + (is (not (str/includes? css " ul {"))) + (is (not (str/includes? css "::marker"))))) + +(deftest rendered-text-keeps-paragraphs-for-lists + (testing "outside code mode list paragraphs still render as

        " + (let [text (text-shape-with [(list-paragraph "b" "disc")]) + markup (rds/renderToStaticMarkup + (mf/element html-text/text-shape* #js {:shape text :grow-type :fixed}))] + (is (= ["p" "/p"] (block-tags markup)))))) + +(defn- marker-rule + "The ::marker rule generated for a single list item made of `span`." + [span] + (let [css (css/generate-text-css + (text-shape-with [{:type "paragraph" + :list-style-type "disc" + :children [span]}]))] + (re-find #"(?s)::marker \{[^}]*\}" css))) + +(deftest text-css-colors-the-marker-like-its-text + (testing "a half-opaque fill keeps its opacity" + (is (str/includes? (marker-rule {:text "a" :font-size "20" + :fills [{:fill-color "#ff0000" :fill-opacity 0.5}]}) + "color: rgba(255, 0, 0, 0.5);"))) + (testing "a gradient fill uses its first stop" + (is (str/includes? (marker-rule {:text "a" :font-size "20" + :fills [{:fill-color-gradient + {:type :linear + :start-x 0 :start-y 0 :end-x 1 :end-y 1 :width 1 + :stops [{:color "#00ff00" :opacity 0.8 :offset 0} + {:color "#0000ff" :opacity 1 :offset 1}]}}]}) + "color: rgba(0, 255, 0, 0.8);"))) + (testing "a legacy span keeps its color on the node" + (is (str/includes? (marker-rule {:text "a" :font-size "20" + :fill-color "#0000ff" :fill-opacity 1}) + "color: rgba(0, 0, 255, 1);"))) + (testing "a span without a font size falls back to the default" + (is (str/includes? (marker-rule {:text "a" + :fills [{:fill-color "#ff0000" :fill-opacity 1}]}) + "font-size: 14px;")))) + +(deftest text-markup-keeps-the-direction-of-list-items + (let [text (text-shape-with [(assoc (list-paragraph "a" "disc") :text-direction "rtl")]) + markup (html/generate-markup (objects text) [text])] + (is (str/includes? markup "

      • {:profile {:id (uuid/next)}} + (some? current-team-id) + (assoc :current-team-id current-team-id) + + (some? default-team-id) + (assoc-in [:profile :default-team-id] default-team-id) + + (some? route-team-id) + (assoc :route {:data {:name :dashboard-recent} + :params {:path {} :query {:team-id (str route-team-id)}} + :query-params {:team-id (str route-team-id)}}))) + +(defn- ^:async emitted + "Runs `event` against `state` and returns every event it emitted, once + the event's stream terminates. + + The stream is a subject the caller of `ptk/watch` never sees: `go-to- + dashboard-search` keeps listening to the router after navigating and + completes only when it receives one `::rt/navigated`, which is the event + it waits for in the app to focus the search box. Pushing it here makes + that helper terminate like the rest, so every event of the family is + measured the same way, subscribe and await termination. + + `observe` forces asynchronous delivery even for a synchronous source and + rejects when the stream does not terminate in time, so a helper that + stops closing its stream fails here instead of passing on the first + event." + [event state] + (let [stream (rx/subject) + events (atom []) + done (async/observe (ptk/watch event state stream) + :on-next #(swap! events conj %))] + ;; `observe` subscribes inside its promise executor, so the subject + ;; already has a listener; a subject drops values nobody hears. + (rx/push! stream (rt/navigated {:data {:name :dashboard-search}} false)) + (await done) + @events)) + +(defn- nav-params + "The params of the last navigation to `route-id`, or nil when the event + emitted none." + [events route-id] + (some->> events + (filter #(and (= ::rt/navigate (ptk/type %)) + (= route-id (:id @%)))) + (last) + (deref) + (:params))) + +(t/deftest ^:async go-to-workspace-takes-the-team-of-the-route-on-screen + ;; The opened file belongs to the team of the route. The personal + ;; default would open it under another team. + (let [route-team-id (uuid/next) + default-team-id (uuid/next) + events (await (emitted (dcm/go-to-workspace :file-id file-id) + (state {:route-team-id route-team-id + :default-team-id default-team-id})))] + (t/is (= route-team-id (:team-id (nav-params events :workspace)))))) + +(t/deftest ^:async go-to-workspace-takes-the-team-initialized-in-the-store + (let [current-team-id (uuid/next) + events (await (emitted (dcm/go-to-workspace :file-id file-id) + (state {:current-team-id current-team-id + :route-team-id (uuid/next) + :default-team-id (uuid/next)})))] + (t/is (= current-team-id (:team-id (nav-params events :workspace)))))) + +(t/deftest ^:async go-to-workspace-takes-the-profile-default-when-no-team-is-known + (let [default-team-id (uuid/next) + events (await (emitted (dcm/go-to-workspace :file-id file-id) + (state {:default-team-id default-team-id})))] + (t/is (= default-team-id (:team-id (nav-params events :workspace)))))) + +(t/deftest ^:async go-to-workspace-keeps-the-team-the-caller-passes + (let [team-id (uuid/next) + events (await (emitted (dcm/go-to-workspace :team-id team-id :file-id file-id) + (state {:current-team-id (uuid/next) + :route-team-id (uuid/next) + :default-team-id (uuid/next)})))] + (t/is (= team-id (:team-id (nav-params events :workspace)))))) + +(t/deftest ^:async go-to-dashboard-recent-keeps-the-default-sentinel + (let [default-team-id (uuid/next) + events (await (emitted (dcm/go-to-dashboard-recent :team-id :default) + (state {:current-team-id (uuid/next) + :route-team-id (uuid/next) + :default-team-id default-team-id})))] + (t/is (= default-team-id (:team-id (nav-params events :dashboard-recent)))))) + +(t/deftest ^:async go-to-dashboard-deleted-keeps-the-default-sentinel + ;; The sentinel is a second copy of the branch in `-recent`; without its + ;; own test, a divergence here would navigate to the current team. + (let [default-team-id (uuid/next) + events (await (emitted (dcm/go-to-dashboard-deleted :team-id :default) + (state {:current-team-id (uuid/next) + :route-team-id (uuid/next) + :default-team-id default-team-id})))] + (t/is (= default-team-id (:team-id (nav-params events :dashboard-deleted)))))) + +(t/deftest ^:async go-to-dashboard-recent-resolves-a-nil-team-through-the-route + (let [route-team-id (uuid/next) + events (await (emitted (dcm/go-to-dashboard-recent) + (state {:route-team-id route-team-id + :default-team-id (uuid/next)})))] + (t/is (= route-team-id (:team-id (nav-params events :dashboard-recent)))))) + +(def ^:private dashboard-helpers + "The rest of the family, with the route each one navigates to." + [[dcm/go-to-dashboard-files :dashboard-files] + [dcm/go-to-dashboard-search :dashboard-search] + [dcm/go-to-dashboard-libraries :dashboard-libraries] + [dcm/go-to-dashboard-fonts :dashboard-fonts] + [dcm/go-to-dashboard-members :dashboard-members] + [dcm/go-to-dashboard-invitations :dashboard-invitations] + [dcm/go-to-dashboard-webhooks :dashboard-webhooks] + [dcm/go-to-dashboard-settings :dashboard-settings] + [dcm/go-to-dashboard-deleted :dashboard-deleted]]) + +(t/deftest ^:async dashboard-helpers-take-the-team-of-the-route-on-screen + (doseq [[helper route-id] dashboard-helpers] + (let [route-team-id (uuid/next) + events (await (emitted (helper) + (state {:route-team-id route-team-id + :default-team-id (uuid/next)})))] + (t/is (= route-team-id (:team-id (nav-params events route-id))) + (str "route: " route-id))))) + +(t/deftest ^:async dashboard-helpers-take-the-profile-default-when-no-team-is-known + (doseq [[helper route-id] dashboard-helpers] + (let [default-team-id (uuid/next) + events (await (emitted (helper) + (state {:default-team-id default-team-id})))] + (t/is (= default-team-id (:team-id (nav-params events route-id))) + (str "route: " route-id))))) + +(t/deftest ^:async navigations-still-fire-when-no-team-can-be-resolved + ;; Documented last resort: a profile without a default team is a broken + ;; or transient state, and there is nothing to resolve. The navigation + ;; still fires, so the not-found page appears with its exit in the + ;; header instead of the click doing nothing. + (let [events (await (emitted (dcm/go-to-dashboard-members) (state {})))] + (t/is (some? (nav-params events :dashboard-members))) + (t/is (nil? (:team-id (nav-params events :dashboard-members)))))) diff --git a/frontend/test/frontend_tests/data/dashboard_test.cljs b/frontend/test/frontend_tests/data/dashboard_test.cljs index 2283d06825..45eaa9151b 100644 --- a/frontend/test/frontend_tests/data/dashboard_test.cljs +++ b/frontend/test/frontend_tests/data/dashboard_test.cljs @@ -27,7 +27,7 @@ "Runs handle-change-team-organization and returns recorded i18n/tr calls." [notification organization] (let [team-id (uuid/next) - current-url (str "https://penpot.example.com/#/dashboard/recent?team-id=" team-id) + current-query-string (str "screen=dashboard-recent&team-id=" team-id) state {:current-team-id team-id :teams {team-id {:id team-id :name "Team-Apple"}}} event (dcm/handle-change-team-organization @@ -43,11 +43,11 @@ rp/cmd! (mock/stub (fn [cmd params] (if (= [:check-nitrate-sso - {:team-id team-id :url current-url}] + {:team-id team-id :query-string current-query-string}] [cmd params]) (rx/of {:authorized true}) (rx/throw (ex-info "unexpected RPC" {:cmd cmd :params params}))))) - rt/get-current-href (mock/stub (constantly current-url))} + rt/get-current-query-string (mock/stub (constantly current-query-string))} (await (async/observe (ptk/watch event state nil) :on-next (fn [ev] (t/is (contains? #{::ntf/show ::dtm/fetch-teams} @@ -120,7 +120,7 @@ (t/async done (let [team-id (uuid/next) organization {:id (uuid/next) :name "OrgA"} - current-url (str "https://penpot.example.com/#/dashboard/recent?team-id=" team-id) + current-query-string (str "screen=dashboard-recent&team-id=" team-id) redirect-url "https://idp.example.com/authorize" state {:current-team-id team-id} event (dcm/handle-change-team-organization @@ -131,11 +131,11 @@ rp/cmd! (mock/stub (fn [cmd params] (if (= [:check-nitrate-sso - {:team-id team-id :url current-url}] + {:team-id team-id :query-string current-query-string}] [cmd params]) (rx/of {:authorized false :redirect-uri redirect-url}) (rx/throw (ex-info "unexpected RPC" {:cmd cmd :params params}))))) - rt/get-current-href (mock/stub (constantly current-url))} + rt/get-current-query-string (mock/stub (constantly current-query-string))} (fn [done'] (->> (ptk/watch event state nil) (rx/reduce conj []) @@ -152,7 +152,7 @@ (t/async done (let [team-id (uuid/next) organization-id (uuid/next) - current-url (str "https://penpot.example.com/#/workspace?team-id=" team-id) + current-query-string (str "screen=workspace&team-id=" team-id) redirect-url "https://idp.example.com/authorize" state {:current-team-id team-id :teams {team-id {:id team-id @@ -164,11 +164,11 @@ rp/cmd! (mock/stub (fn [cmd params] (if (= [:check-nitrate-sso - {:team-id team-id :url current-url}] + {:team-id team-id :query-string current-query-string}] [cmd params]) (rx/of {:authorized false :redirect-uri redirect-url}) (rx/throw (ex-info "unexpected RPC" {:cmd cmd :params params}))))) - rt/get-current-href (mock/stub (constantly current-url))} + rt/get-current-query-string (mock/stub (constantly current-query-string))} (fn [done'] (->> (ptk/watch event state nil) (rx/reduce conj []) @@ -330,7 +330,7 @@ (t/is (= [[:wait #{file-id}] [:duplicate-file]] @calls)) (t/is (nil? (:notification @store)) "nothing is told before the copy exists") (rx/push! answer {:id (uuid/next)}) - (await (async/wait-for #(some? (:notification @store)) "the user is told")) + (await (async/wait-for-store store #(some? (:notification @store)) "the user is told")) (await (async/settle)) (t/is (= "translated:dashboard.duplicate-unsaved-changes" (get-in @store [:notification :content]))) @@ -354,7 +354,7 @@ (await (async/wait-for #(= 1 @sent) "the copy is requested")) (t/is (nil? (:notification @store))) (rx/push! answer {:id (uuid/next)}) - (await (async/wait-for #(some? (:notification @store)) "the user is told")) + (await (async/wait-for-store store #(some? (:notification @store)) "the user is told")) (t/is (= "translated:dashboard.success-duplicate-file" (get-in @store [:notification :content]))) (t/is (= :success (get-in @store [:notification :level]))))))) diff --git a/frontend/test/frontend_tests/data/exports_assets_test.cljs b/frontend/test/frontend_tests/data/exports_assets_test.cljs index 18277c2f37..53965531d5 100644 --- a/frontend/test/frontend_tests/data/exports_assets_test.cljs +++ b/frontend/test/frontend_tests/data/exports_assets_test.cljs @@ -300,7 +300,7 @@ wasm.api/export-target (mock/stub (constantly :too-large)) i18n/tr (mock/stub #(str "translated:" %))} (ptk/emit! store (de/request-simple-export {:export one})) - (await (async/wait-for #(some? (:notification @store)) "the toast shows")) + (await (async/wait-for-store store #(some? (:notification @store)) "the toast shows")) (t/is (empty? @cmd-calls) "no exporter RPC") (t/is (= "translated:errors.export-too-large" (get-in @store [:notification :content]))) @@ -418,14 +418,14 @@ (reset! downloaded [filename mtype uri]))) st/ongoing-tasks (atom #{})} (ptk/emit! store (de/request-multiple-export {:exports exports :name "page"})) - (await (async/wait-for #(= "running" (get-in @store [:export :status])) - "export is running")) + (await (async/wait-for-store store #(= "running" (get-in @store [:export :status])) + "export is running")) (rx/push! worker-bus {:type :progress :done 1 :total 2}) - (await (async/wait-for #(= 1 (get-in @store [:export :progress])) - "first progress lands")) + (await (async/wait-for-store store #(= 1 (get-in @store [:export :progress])) + "first progress lands")) (ptk/emit! store (de/cancel-export)) - (await (async/wait-for #(= "cancelled" (get-in @store [:export :status])) - "export is cancelled")) + (await (async/wait-for-store store #(= "cancelled" (get-in @store [:export :status])) + "export is cancelled")) (rx/push! worker-bus {:type :progress :done 2 :total 2}) (rx/push! worker-bus {:type :done :filename "page.zip" diff --git a/frontend/test/frontend_tests/data/fonts_test.cljs b/frontend/test/frontend_tests/data/fonts_test.cljs new file mode 100644 index 0000000000..0bbd806bff --- /dev/null +++ b/frontend/test/frontend_tests/data/fonts_test.cljs @@ -0,0 +1,158 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.data.fonts-test + "Tests for the upload metadata resolution in app.main.data.fonts." + (:require + [app.common.uuid :as uuid] + [app.main.data.fonts :as df] + [app.main.data.uploads :as uploads] + [app.main.repo :as rp] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock])) + +(t/deftest resolve-font-metadata-prefers-opentype-values + (t/testing "opentype.js values win over the filename" + (t/is (= {:font-family "Inter" + :font-weight 700 + :font-style "normal" + :variant-name "Bold"} + (df/resolve-font-metadata {:family "Inter" :variant "Bold"} + "Inter.ttf"))))) + +(t/deftest resolve-font-metadata-falls-back-to-filename + (t/testing "derives weight and style from the filename when the variant is missing" + (t/is (= {:font-family "Pretendard" + :font-weight 600 + :font-style "normal" + :variant-name nil} + (df/resolve-font-metadata {:family "Pretendard" :variant nil} + "Pretendard-SemiBold.otf")))) + + (t/testing "derives weight and style from the filename when the variant is blank" + (t/is (= {:font-family "Lato" + :font-weight 300 + :font-style "normal" + :variant-name nil} + (df/resolve-font-metadata {:family "Lato" :variant ""} + "Lato-Light.ttf")))) + + (t/testing "derives the family from the filename when it is missing" + (t/is (= {:font-family "Blender Inter" + :font-weight 700 + :font-style "normal" + :variant-name nil} + (df/resolve-font-metadata {:family nil :variant nil} + "Blender-Inter-Bold.ttf")))) + + (t/testing "derives family, weight and style from the filename when nothing is available" + (t/is (= {:font-family "Pretendard" + :font-weight 600 + :font-style "normal" + :variant-name nil} + (df/resolve-font-metadata {} "Pretendard-SemiBold.woff2")))) + + (t/testing "derives italic from the filename" + (t/is (= {:font-family "Roboto" + :font-weight 400 + :font-style "italic" + :variant-name nil} + (df/resolve-font-metadata {} "Roboto-Italic.ttf")))) + + (t/testing "separates a style token glued to a weight token" + (t/is (= {:font-family "Roboto" + :font-weight 700 + :font-style "italic" + :variant-name nil} + (df/resolve-font-metadata {:family "Roboto" :variant nil} + "Roboto-BoldItalic.ttf")))) + + (t/testing "treats an oblique variant as italic" + (t/is (= {:font-family "Roboto" + :font-weight 700 + :font-style "italic" + :variant-name "BoldOblique"} + (df/resolve-font-metadata {:family "Roboto" :variant "BoldOblique"} + "Roboto.ttf")))) + + (t/testing "handles filenames without extension" + (t/is (= {:font-family "Lato" + :font-weight 300 + :font-style "normal" + :variant-name nil} + (df/resolve-font-metadata {} "Lato-Light"))))) + +(t/deftest ^:async upload-font-variant-sends-variant-name + (await + (mock/with-mocks* + {rp/cmd! mock/rpc-cmd-mock + uploads/upload-blob-chunked + (fn [_blob & _opts] (rx/of {:session-id (uuid/next)}))} + (let [item {:data {"font/ttf" (js/Uint8Array. 4)} + :team-id (uuid/next) + :font-id (uuid/next) + :font-family "Roboto" + :font-weight 700 + :font-style "italic" + :variant-name "BoldOblique"}] + (await (async/->promise (df/upload-font-variant item))) + (let [{:keys [cmd params]} (first @mock/rpc-calls)] + (t/is (= :create-font-variant cmd)) + (t/is (= "BoldOblique" (:variant-name params))))) + (await (async/settle))))) + +(defn- fake-font + "A minimal opentype.js-like font object exposing only what + prepare-font-entry reads." + [names] + #js {:getEnglishName (fn [k] (get names k)) + :tables #js {:hhea #js {:ascender 800 :descender -200} + :os2 #js {:usWinAscent 800 + :usWinDescent 200 + :sTypoAscender 800 + :sTypoDescender -200 + :fsSelection 0}}}) + +(t/deftest prepare-font-entry-falls-back-when-metadata-is-missing + (t/testing "parsed font with a missing subfamily" + (let [result (df/prepare-font-entry + {:font (fake-font {"preferredFamily" "Pretendard"}) + :type "font/ttf" + :name "Pretendard-SemiBold.otf" + :data (js/Uint8Array. 4)})] + (t/is (= "Pretendard" (:font-family result))) + (t/is (= 600 (:font-weight result))) + (t/is (= "normal" (:font-style result))) + (t/is (nil? (:variant-name result))) + (t/is (false? (:height-warning? result))) + (t/is (= {:name "Pretendard-SemiBold.otf" :type "font/ttf"} + (select-keys (:content result) [:name :type]))))) + + (t/testing "unparseable woff2 entry is resolved from the filename" + (let [result (df/prepare-font-entry + {:font nil + :type "font/woff2" + :name "Pretendard-SemiBold.woff2" + :data (js/Uint8Array. 4)})] + (t/is (= "Pretendard" (:font-family result))) + (t/is (= 600 (:font-weight result))) + (t/is (= "normal" (:font-style result))) + (t/is (false? (:height-warning? result))))) + + (t/testing "a blank first candidate does not shadow a valid second one" + (let [result (df/prepare-font-entry + {:font (fake-font {"preferredFamily" " " + "preferredSubfamily" " " + "fontSubfamily" "Bold"}) + :type "font/ttf" + :name "Roboto-Light.ttf" + :data (js/Uint8Array. 4)})] + (t/is (= "Roboto" (:font-family result))) + (t/is (= 700 (:font-weight result))) + (t/is (= "normal" (:font-style result))) + (t/is (= "Bold" (:variant-name result)))))) diff --git a/frontend/test/frontend_tests/data/imports_test.cljs b/frontend/test/frontend_tests/data/imports_test.cljs index e173ec3671..9a58d315dd 100644 --- a/frontend/test/frontend_tests/data/imports_test.cljs +++ b/frontend/test/frontend_tests/data/imports_test.cljs @@ -210,6 +210,112 @@ (t/testing "what the container occupies is kept apart" (t/is (= [10000 10000 10000] (mapv :container-size @seen)))))) +(t/deftest ^:async the-references-of-a-container-are-read-one-at-a-time + (let [file-id (uuid/next) + media-id (str (uuid/next)) + thumb-id (str (uuid/next)) + seen (atom []) + in-flight (atom 0) + peak (atom 0) + entries + #js [#js {:filename "manifest.json" + :compressedSize 100} + #js {:filename (str "files/" file-id ".json") + :compressedSize 200} + #js {:filename (str "files/" file-id "/media/one.json") + :compressedSize 10 + :content (str "{\"mediaId\": \"" media-id "\"}")} + #js {:filename (str "files/" file-id "/thumbnails/t/p/f.json") + :compressedSize 10 + :content (str "{\"thumbnailId\": \"" thumb-id "\"}")} + #js {:filename (str "objects/" media-id ".json") + :compressedSize 20} + #js {:filename (str "objects/" media-id ".png") + :compressedSize 1000} + #js {:filename (str "objects/" thumb-id ".json") + :compressedSize 5} + #js {:filename (str "objects/" thumb-id ".png") + :compressedSize 40}]] + + (await + (mock/with-mocks* + {http/send! (slurp-stub zip-magic) + uz/get-entry (mock/stub (fn [_ _] + (p/resolved #js {:content (manifest [file-id "One"])}))) + uz/get-entries (mock/stub (fn [_] (p/resolved entries))) + ;; every read counts how many are open at the same time + uz/read-as-text + (mock/stub (fn [entry] + (swap! in-flight inc) + (swap! peak max @in-flight) + (->> (p/resolved (.-content entry)) + (p/fmap (fn [content] + (swap! in-flight dec) + content))))) + uz/close (mock/stub (fn [_] nil))} + + (await (hva/observe (imp/analyze [{:uri uri :name "package.penpot" :size 10000}]) + {:on-next #(swap! seen conj %)})))) + + (t/testing "a container of many references never opens more than one read" + ;; a container of a big file holds thousands of references, and every + ;; read opens its own decompression stream: reading them all at once + ;; is what breaks the analysis in the browser + (t/is (= 1 @peak))) + + (t/testing "every reference is still counted" + ;; 200 + 10 + 10 of its own, and 20 + 1000 and 5 + 40 of the two + ;; objects its media and its thumbnail point at + (t/is (= [1285] (mapv :size @seen)))))) + +(t/deftest ^:async a-reference-that-cannot-be-read-leaves-its-file-without-size + (let [readable-id (uuid/next) + broken-id (uuid/next) + object-id (str (uuid/next)) + broken-path (str "files/" broken-id "/media/two.json") + seen (atom []) + entries + #js [#js {:filename "manifest.json" + :compressedSize 100} + #js {:filename (str "files/" readable-id ".json") + :compressedSize 200} + #js {:filename (str "files/" readable-id "/media/one.json") + :compressedSize 10 + :content (str "{\"mediaId\": \"" object-id "\"}")} + #js {:filename (str "objects/" object-id ".json") + :compressedSize 20} + #js {:filename (str "objects/" object-id ".png") + :compressedSize 1000} + #js {:filename (str "files/" broken-id ".json") + :compressedSize 400} + #js {:filename broken-path + :compressedSize 10}]] + + (await + (mock/with-mocks* + {http/send! (slurp-stub zip-magic) + uz/get-entry (mock/stub (fn [_ _] + (p/resolved #js {:content (manifest [readable-id "Readable"] + [broken-id "Broken"])}))) + uz/get-entries (mock/stub (fn [_] (p/resolved entries))) + uz/read-as-text + (mock/stub (fn [entry] + (if (= broken-path (.-filename entry)) + (p/rejected (ex-info "cannot read the entry" {:path broken-path})) + (p/resolved (.-content entry))))) + uz/close (mock/stub (fn [_] nil))} + + (await (hva/observe (imp/analyze [{:uri uri :name "package.penpot" :size 10000}]) + {:on-next #(swap! seen conj %)})))) + + (t/testing "one unreadable reference does not fail the package" + (t/is (= [readable-id broken-id] (mapv :file-id @seen))) + (t/is (every? #(= :success (:status %)) @seen))) + + (t/testing "the file whose reference could not be read has no size" + ;; a number that misses one of its objects is worse than no number + (t/is (= [1230 nil] (mapv :size @seen)))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPORTING A PACKAGE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/frontend/test/frontend_tests/data/nitrate_test.cljs b/frontend/test/frontend_tests/data/nitrate_test.cljs index 2ea4442157..bb539cccfb 100644 --- a/frontend/test/frontend_tests/data/nitrate_test.cljs +++ b/frontend/test/frontend_tests/data/nitrate_test.cljs @@ -8,6 +8,7 @@ (:require [app.common.time :as ct] [app.common.uri :as u] + [app.config :as cf] [app.main.data.event :as ev] [app.main.data.modal :as modal] [app.main.data.nitrate :as dnt] @@ -367,7 +368,7 @@ state {:teams {(:id team) team}}] (await (mock/with-mocks* - {rt/get-current-href (mock/stub (constantly "http://localhost/#/dashboard")) + {rt/get-current-token (mock/stub (constantly "?screen=dashboard&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d")) rp/cmd! (mock/stub (fn [cmd _params] (case cmd @@ -394,3 +395,32 @@ (t/is (= ::dt/fetch-teams (ptk/type (first emitted)))) (t/is (= {:modal-show [:no-permission-modal {:type :no-organizations-create}]} (second emitted))))) + +(def ^:private sso-query-string + "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") + +(defn- ^:async observe-retry-organization-sso + "Runs `retry-organization-sso` with the given answer from the backend and + resolves to the uri the user is sent to." + [rpc-answer] + (let [emitted (atom []) + event (dnt/retry-organization-sso {:team-id "team-1" + :organization-id "org-1" + :query-string sso-query-string})] + (await + (mock/with-mocks* + {rp/cmd! (mock/stub (fn [_cmd _params] (rx/of rpc-answer)))} + (await (async/observe (ptk/watch event {} nil) + :on-next #(swap! emitted conj %))))) + ;; `nav-raw` exposes its destination through `@event`. + (-> @emitted first deref :uri))) + +(t/deftest ^:async retry-organization-sso-lands-on-the-route-of-the-token + (t/testing "without a redirect uri the user goes back to the route of the token" + (let [target (await (observe-retry-organization-sso {:authorized true}))] + (t/is (= (str cf/public-uri "?" sso-query-string) target)))) + + (t/testing "the redirect uri of the backend wins over the route of the token" + (let [target (await (observe-retry-organization-sso {:authorized false + :redirect-uri "https://idp.example.com/authorize"}))] + (t/is (= "https://idp.example.com/authorize" target))))) diff --git a/frontend/test/frontend_tests/data/persistence_lock_test.cljs b/frontend/test/frontend_tests/data/persistence_lock_test.cljs index 41ec6d38f9..7e908d9ab2 100644 --- a/frontend/test/frontend_tests/data/persistence_lock_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_lock_test.cljs @@ -83,7 +83,7 @@ (thp/open-file file-id page-id (thp/file-data page-id shape-id "B")) (thp/rename-commit file-id page-id shape-id "A" "B") ::dps/force-persist) - (await (async/wait-for #(plk/locked? @store) "the failed save locks the workspace")) + (await (async/wait-for-store store #(plk/locked? @store) "the failed save locks the workspace")) (reset! made []) (ptk/emit! store (dch/commit-changes {:redo-changes [(thp/rename page-id shape-id "C")] @@ -156,7 +156,7 @@ (thp/file-data page-id shape-id "B") (thp/file-data page-id lshape-id "Y"))) (ptk/emit! store linked commit ::dps/force-persist) - (await (async/wait-for #(plk/locked? @store) "this file's failed save locks the workspace")) + (await (async/wait-for-store store #(plk/locked? @store) "this file's failed save locks the workspace")) (reset! made []) (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) (await (async/wait-for #(seq @made) "the library's rename is undone")) @@ -164,8 +164,8 @@ (t/is (= "X" (thp/shape-name store lib-id page-id lshape-id))) (t/is (= "A" (thp/shape-name store file-id page-id shape-id))) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the undo is saved")) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the undo is saved")) (rx/dispose! sub) (t/is (= 3 (count @requests))) (t/is (= lib-id (get-in (nth @requests 2) [1 :id]))) diff --git a/frontend/test/frontend_tests/data/persistence_retry_test.cljs b/frontend/test/frontend_tests/data/persistence_retry_test.cljs index 1f79811018..dcab8e593c 100644 --- a/frontend/test/frontend_tests/data/persistence_retry_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_retry_test.cljs @@ -50,11 +50,11 @@ (mock/with-mocks* {rx/timer (mock/stub instant-backoff)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) - (= :invalid-save-response - (get-in @store [:persistence :error :code])) - (= 1 (count (get-in @store [:persistence :queue])))) - "invalid response fails the save")) + (await (async/wait-for-store store #(and (= :error (get-in @store [:persistence :status])) + (= :invalid-save-response + (get-in @store [:persistence :error :code])) + (= 1 (count (get-in @store [:persistence :queue])))) + "invalid response fails the save")) (t/is (= :error (get-in @store [:persistence :status]))) (t/is (= :invalid-save-response (get-in @store [:persistence :error :code]))) (t/is (= 1 (count (get-in @store [:persistence :queue])))) @@ -97,9 +97,9 @@ (mock/with-mocks* {rx/timer (mock/stub instant-backoff)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "retry saves the file")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "retry saves the file")) (t/is (= :saved (get-in @store [:persistence :status]))) (t/is (empty? (get-in @store [:persistence :queue]))) (t/is (= 2 (count @requests)) "failed send plus one retry") @@ -124,9 +124,9 @@ (mock/with-mocks* {rx/timer (mock/stub instant-backoff)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - (str "a " status " is retried and the save lands"))) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + (str "a " status " is retried and the save lands"))) (t/is (= 2 (count @requests)) "the proxy error plus one retry") (t/is (empty? @failures) "a retried proxy error never reaches the failure toast")))) (fn [_ _] @@ -158,9 +158,9 @@ (mock/with-mocks* {rx/timer (mock/stub instant-backoff)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - (str "a busy " (:status response) " is retried and the save lands"))) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + (str "a busy " (:status response) " is retried and the save lands"))) (t/is (= 2 (count @requests)) "the busy answer plus one retry") (t/is (empty? @failures) "a retried busy answer never reaches the failure toast")))) (fn [_ _] @@ -197,7 +197,7 @@ hints (atom [])] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [ms] @@ -209,13 +209,13 @@ (js/Promise.resolve nil)) i18n/tr (fn ([key] (str "translated:" key)) ([key & _] (str "translated:" key)))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (get-in @store [:persistence :attempts])) - "the first answer starts the retries")) + (await (async/wait-for-store store #(= 1 (get-in @store [:persistence :attempts])) + "the first answer starts the retries")) (t/is (= "translated:errors.save-retrying-busy" (get-in @store [:notification :content]))) (doseq [n [2 3 4]] (rx/push! timer-s :tick) - (await (async/wait-for #(= n (count @requests)) "the retry is sent"))) + (await (async/wait-for #(= n (count @requests)) "the retry is sent" {:signal requests$}))) (await (async/wait-for #(seq @hints) "the slow-cycle notice")) (t/is (= ["translated:errors.save-still-retrying-busy"] @hints)) (rx/end! timer-s)))) @@ -234,8 +234,8 @@ (mock/with-mocks* {rx/timer (mock/stub instant-backoff)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the save fails")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the save fails")) (t/is (= :server-error (get-in @store [:persistence :error :cause-type]))) (t/is (= 1 (count @requests)) "no retry")))) (fn [_ _] @@ -251,14 +251,17 @@ (let [delays (atom [])] (await (with-persistence - (^:async fn [{:keys [file-id failures requests store]}] + (^:async fn [{:keys [file-id failures requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [ms] (swap! delays conj ms) (instant-backoff ms)))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) + ;; The predicate reads both the store and the request counter, so it + ;; passes the merged signal; the wrapper only provides the store one. (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) (= 4 (count @requests))) - "retries exhaust into terminal error")) + "retries exhaust into terminal error" + {:signal (rx/merge requests$ (ptk/input-stream store))})) (t/is (= :error (get-in @store [:persistence :status]))) (t/is (= :save-failed (get-in @store [:persistence :error :code]))) (t/is (= 1 (count (get-in @store [:persistence :queue])))) @@ -277,20 +280,20 @@ timer-s (rx/subject)] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [ms] (swap! delays conj ms) timer-s))} ;; Phase 1 — first send fails transiently; the retry pends on ;; the hand-fired timer. (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "transient failure retries")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "transient failure retries")) (t/is (= 1 (count @requests))) ;; Phase 2 — the online signal resends the head under the live ;; episode (status stays :retrying), hanging in flight. (ptk/emit! store (#'dps/resume-on-online)) - (await (async/wait-for #(= 2 (count @requests)) "online resends")) + (await (async/wait-for #(= 2 (count @requests)) "online resends" {:signal requests$})) (t/is (= :retrying (get-in @store [:persistence :status]))) (t/is (= 1 (get-in @store [:persistence :attempts]))) ;; Phase 3 — the pending timer fires into the in-flight request: @@ -318,8 +321,8 @@ (mock/with-mocks* {rx/timer (mock/stub (fn [_] (rx/subject)))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "transient failure retries")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "transient failure retries")) (t/is (= 1 (count @requests))) (ptk/emit! store (#'dps/persist-commit (peek (get-in @store [:persistence :queue])) @@ -348,13 +351,13 @@ ;; Phase 1 — first send fails transiently; the retry pends on ;; the hand-fired timer. (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "transient failure retries")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "transient failure retries")) (t/is (= 1 (count @requests))) ;; Phase 2 — a new edit joins the queue without resending. (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 2 (count (get-in @store [:persistence :queue]))) - "second edit is queued")) + (await (async/wait-for-store store #(= 2 (count (get-in @store [:persistence :queue]))) + "second edit is queued")) (await (async/settle)) (t/is (= 1 (count @requests)) "the edit does not resend the head") (t/is (= 1 (get-in @store [:persistence :attempts])) "no attempt is spent") @@ -362,9 +365,9 @@ (t/is (= :retrying (get-in @store [:persistence :status]))) ;; Phase 3 — the timer fires: the head saves, then the new edit. (rx/push! timer-s :tick) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "both edits save")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "both edits save")) (t/is (= 3 (count @requests)) "failed send, retry, then the new edit") (let [[first-id retry-id edit-id] (map (comp :commit-id second) @requests)] (t/is (= first-id retry-id) "the retry resends the head") @@ -391,17 +394,17 @@ {rx/timer (mock/stub (fn [_] timer-s)) i18n/tr (mock/stub #(str "translated:" %))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (get-in @store [:persistence :attempts])) - "first attempt fails into retrying")) + (await (async/wait-for-store store #(= 1 (get-in @store [:persistence :attempts])) + "first attempt fails into retrying")) (let [notice (get @store :notification)] (t/is (map? notice) "a single notice is visible, never stacked")) (t/is (= :persistence-reconnecting (get-in @store [:notification :tag]))) (t/is (= "translated:errors.save-retrying" (get-in @store [:notification :content]))) (rx/push! timer-s :tick) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (nil? (get @store :notification))) - "save hides the notice")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (nil? (get @store :notification))) + "save hides the notice")) (t/is (= :saved (get-in @store [:persistence :status]))) (t/is (nil? (get @store :notification)) "recovery is silent") (rx/end! timer-s)))) @@ -421,8 +424,8 @@ (mock/with-mocks* {rx/timer (mock/stub instant-backoff)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "retries exhaust into terminal error")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "retries exhaust into terminal error")) (t/is (= :error (get-in @store [:persistence :status]))) (t/is (nil? (get @store :notification)) "the terminal path hides the reconnect notice")))) @@ -440,23 +443,23 @@ second-s (atom nil)] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [_] timer-s))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "transient failure retries")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "transient failure retries")) (t/is (= 1 (count @requests))) (ptk/emit! store (#'dps/resume-on-online)) - (await (async/wait-for #(= 2 (count @requests)) "online resends")) + (await (async/wait-for #(= 2 (count @requests)) "online resends" {:signal requests$})) (t/is (= 2 (count @requests))) (t/is (= :retrying (get-in @store [:persistence :status]))) (t/is (= 1 (get-in @store [:persistence :attempts]))) (rx/push! @second-s {:revn 1}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "resumed save lands")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "resumed save lands")) (t/is (= :saved (get-in @store [:persistence :status]))) (ptk/emit! store (#'dps/resume-on-online)) (await (async/settle)) @@ -485,20 +488,20 @@ timer-s (rx/subject)] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [_] timer-s))} ;; Phase 1 — first send fails transiently; the retry pends on ;; the hand-fired timer. (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "transient failure retries")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "transient failure retries")) (t/is (= 1 (count @requests))) ;; Phase 2 — the retry timer fires and resends the head, ;; hanging in flight. (rx/push! timer-s :tick) - (await (async/wait-for #(= 2 (count @requests)) "retry resends")) + (await (async/wait-for #(= 2 (count @requests)) "retry resends" {:signal requests$})) (t/is (= :retrying (get-in @store [:persistence :status]))) ;; Phase 3 — online arrives mid-flight: silent, never a third send. (ptk/emit! store (#'dps/resume-on-online)) @@ -558,10 +561,10 @@ ;; Phase 2 — the save lands, resetting the stall clock; a new ;; edit stalls again, over a governor window later. (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "first save lands")) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "first save lands")) (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 2 (count @requests)) "second edit is sent")) + (await (async/wait-for #(= 2 (count @requests)) "second edit is sent" {:signal requests$})) (reset! clock (+ 300001 errors/report-window-ms 300001)) (rx/push! ticks :tick) (await (async/wait-for #(= 2 (count (audit-events @events))) @@ -594,9 +597,9 @@ (swap! timeouts conj ms) (timeout 0 with ob)))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the retry saves the file")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the retry saves the file")) (t/is (= 2 (count @requests)) "hung send plus one retry") (t/is (apply = (map (comp :commit-id second) @requests)) "both sends carry the same commit id") @@ -612,12 +615,15 @@ (defn- slow-cycle-timer "Timer double: the backoff delays fire at once, and every slow cycle waits - on a subject collected in `cycles`, so the test fires them one by one." - [cycles] + on a subject collected in `cycles`, so the test fires them one by one. When + `cycles$` is given, each new subject is also emitted there, so a wait can + resolve on the cycle being armed instead of polling the atom." + [cycles & [cycles$]] (fn [ms] (if (= ms dps/slow-retry-delay-ms) (let [cycle-s (rx/subject)] (swap! cycles conj cycle-s) + (when (some? cycles$) (rx/push! cycles$ cycle-s)) cycle-s) (rx/of :tick)))) @@ -626,18 +632,22 @@ ;; again, and saves once the network is back. Proves: a session outlasting ;; the backoff still recovers on its own. (t/deftest ^:async a-spent-backoff-keeps-trying-on-a-slow-cycle - (let [calls (atom 0) - cycles (atom [])] + (let [calls (atom 0) + cycles (atom []) + cycles$ (rx/subject)] (await (with-persistence (^:async fn [{:keys [file-id failures requests store]}] (await (mock/with-mocks* - {rx/timer (mock/stub (slow-cycle-timer cycles))} + {rx/timer (mock/stub (slow-cycle-timer cycles cycles$))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) + ;; The predicate reads both the store and the cycle counter, so it + ;; passes the merged signal; the wrapper only provides the store one. (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) (= 1 (count @cycles))) - "the spent backoff arms the slow cycle")) + "the spent backoff arms the slow cycle" + {:signal (rx/merge cycles$ (ptk/input-stream store))})) (t/is (= 4 (count @requests))) (t/is (= 1 (count @failures))) @@ -652,9 +662,9 @@ (ptk/emit! store #(assoc % :notification {:tag errors/persistence-failed-tag :level :error})) (rx/push! (last @cycles) :tick) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the slow attempt saves")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the slow attempt saves")) (t/is (= 6 (count @requests))) (t/is (apply = (map (comp :commit-id second) @requests)) "every attempt carries the same commit id") @@ -675,17 +685,20 @@ (t/deftest ^:async the-slow-cycle-keeps-the-workspace-locked-until-a-save-lands (let [calls (atom 0) cycles (atom []) + cycles$ (rx/subject) attempt-s (rx/subject)] (await (with-persistence (^:async fn [{:keys [file-id store]}] (await (mock/with-mocks* - {rx/timer (mock/stub (slow-cycle-timer cycles))} + {rx/timer (mock/stub (slow-cycle-timer cycles cycles$))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) + ;; Same as above: the predicate also reads the cycle counter. (await (async/wait-for #(and (= 1 (count @cycles)) (some? (get-in @store [:workspace-global :persistence-lock]))) - "the spent backoff locks the workspace")) + "the spent backoff locks the workspace" + {:signal (rx/merge cycles$ (ptk/input-stream store))})) (t/is (true? (get-in @store [:workspace-global :read-only?]))) (t/is (dps/save-retrying? (:persistence @store))) @@ -697,8 +710,8 @@ "a slow attempt in flight keeps the lock") (rx/push! attempt-s {:revn 1}) - (await (async/wait-for #(nil? (get-in @store [:workspace-global :persistence-lock])) - "the landed save lifts the lock")) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-global :persistence-lock])) + "the landed save lifts the lock")) (t/is (false? (get-in @store [:workspace-global :read-only?])))))) (fn [_ _] (if (< (swap! calls inc) 5) @@ -725,9 +738,9 @@ (t/is (= 1 (count (get-in @store [:persistence :queue])))) (rx/push! (last @cycles) :tick) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - (str status " recovers on the slow cycle"))) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + (str status " recovers on the slow cycle"))) (t/is (= 5 (count @requests))) (t/is (apply = (map (comp :commit-id second) @requests)) "the slow cycle saves the held commit")))) @@ -747,8 +760,8 @@ (mock/with-mocks* {rx/timer (mock/stub (slow-cycle-timer cycles))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the save fails")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the save fails")) (await (async/settle)) (t/is (= 1 (count @requests))) (t/is (empty? @cycles))))) @@ -773,8 +786,8 @@ {rx/timer (mock/stub (slow-cycle-timer cycles)) ct/now (mock/stub #(ct/inst @clock))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - (str slow-count " slow failures spend the backoff"))) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + (str slow-count " slow failures spend the backoff"))) (await (async/settle)) (t/is (= (if armed? 4 slow-count) (count @requests)) "the backoff sends no attempt past the limit") @@ -784,8 +797,8 @@ (when-not armed? (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(empty? (get-in @store [:persistence :queue])) - "recovery releases the held edit")) + (await (async/wait-for-store store #(empty? (get-in @store [:persistence :queue])) + "recovery releases the held edit")) (t/is (= slow-count (count @requests)) "the released edit is not sent again"))))) (fn [_ _] (when (<= (swap! calls inc) slow-count) @@ -799,7 +812,7 @@ cycles (atom [])] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (slow-cycle-timer cycles)) @@ -809,9 +822,9 @@ (reset! clock (+ dps/retry-give-up-ms 1000)) (rx/push! (last @cycles) :tick) - (await (async/wait-for #(= 5 (count @requests)) "the last slow attempt")) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the attempt fails")) + (await (async/wait-for #(= 5 (count @requests)) "the last slow attempt" {:signal requests$})) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the attempt fails")) (await (async/settle)) (t/is (= 1 (count @cycles)) "no further cycle is armed") (t/is (= 1 (count (get-in @store [:persistence :queue])))) @@ -829,7 +842,7 @@ cycles (atom [])] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (slow-cycle-timer cycles)) @@ -843,9 +856,9 @@ (t/is (dps/save-retrying? (:persistence @store))) (rx/push! (last @cycles) :tick) (await (async/wait-for #(= (+ 4 dps/max-slow-failures) (count @requests)) - "the last slow attempt is sent")) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the last slow attempt fails")) + "the last slow attempt is sent" {:signal requests$})) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the last slow attempt fails")) (await (async/settle)) (t/is (= dps/max-slow-failures (count @cycles)) "no further cycle is armed") (t/is (nil? (get-in @store [:persistence :recovering]))) @@ -873,9 +886,9 @@ (await (async/wait-for #(= 1 (count @cycles)) "the slow cycle is armed")) (ptk/emit! store (ptk/data-event :app.main.data.workspace/finalize-workspace)) (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "recovery saves the held edit")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "recovery saves the held edit")) (t/is (= 5 (count @requests))) (t/is (apply = (map (comp :commit-id second) @requests)) "recovery sends the same commit, not a released queue")))) @@ -902,9 +915,9 @@ (ptk/emit! store #(assoc-in % [:permissions :can-edit] false) (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(= :save-permission-denied - (get-in @store [:persistence :error :code])) - "the viewer cannot send the edit")) + (await (async/wait-for-store store #(= :save-permission-denied + (get-in @store [:persistence :error :code])) + "the viewer cannot send the edit")) (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) (await (async/settle)) (t/is (= :error (get-in @store [:persistence :status]))) @@ -935,8 +948,8 @@ (await (async/wait-for #(= 1 (count @timers)) "the failure schedules a retry")) (ptk/emit! store (ptk/data-event :app.main.data.workspace/finalize-workspace)) - (await (async/wait-for #(get-in @store [:persistence :suspended]) - "leaving the workspace suspends retries")) + (await (async/wait-for-store store #(get-in @store [:persistence :suspended]) + "leaving the workspace suspends retries")) (rx/push! (first @timers) :tick) (await (async/settle)) (t/is (= 1 (count @requests)) "the pending retry stays silent") @@ -944,9 +957,9 @@ (t/is (= 1 (count (get-in @store [:persistence :queue])))) (ptk/emit! store (ptk/data-event :app.main.data.workspace/workspace-initialized)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "opening a file saves the held edit")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "opening a file saves the held edit")) (t/is (= 2 (count @requests))) (t/is (apply = (map (comp :commit-id second) @requests)) "the same commit is sent again") @@ -966,19 +979,19 @@ first-s (rx/subject)] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [ms] (swap! timers conj ms) (rx/subject)))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the edit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the edit is sent" {:signal requests$})) (ptk/emit! store (ptk/data-event :app.main.data.workspace/finalize-workspace)) - (await (async/wait-for #(get-in @store [:persistence :suspended]) - "leaving the workspace suspends retries")) + (await (async/wait-for-store store #(get-in @store [:persistence :suspended]) + "leaving the workspace suspends retries")) (rx/error! first-s (ex-info "offline" {:type :offline})) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "the failure keeps the edit queued")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "the failure keeps the edit queued")) (await (async/settle)) (t/is (empty? @timers) "no backoff is scheduled") @@ -987,9 +1000,9 @@ (t/is (= 1 (count @requests)) "the online event does not resend") (ptk/emit! store (ptk/data-event :app.main.data.workspace/workspace-initialized)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "opening a file saves the held edit")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "opening a file saves the held edit")) (t/is (= 2 (count @requests)))))) (fn [_ _] (if (= 1 (swap! calls inc)) @@ -1006,27 +1019,27 @@ last-s (rx/subject)] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (slow-cycle-timer cycles))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 4 (count @requests)) "the last retry is sent")) + (await (async/wait-for #(= 4 (count @requests)) "the last retry is sent" {:signal requests$})) (ptk/emit! store (ptk/data-event :app.main.data.workspace/finalize-workspace)) - (await (async/wait-for #(get-in @store [:persistence :suspended]) - "leaving the workspace suspends retries")) + (await (async/wait-for-store store #(get-in @store [:persistence :suspended]) + "leaving the workspace suspends retries")) (rx/error! last-s (ex-info "offline" {:type :offline})) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the spent backoff halts the queue")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the spent backoff halts the queue")) (await (async/settle)) (t/is (empty? @cycles) "no slow cycle is armed") (t/is (= 1 (count (get-in @store [:persistence :queue])))) (ptk/emit! store (ptk/data-event :app.main.data.workspace/workspace-initialized)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "opening a file saves the held edit")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "opening a file saves the held edit")) (t/is (= 5 (count @requests)))))) (fn [_ _] (let [n (swap! calls inc)] diff --git a/frontend/test/frontend_tests/data/persistence_rollback_test.cljs b/frontend/test/frontend_tests/data/persistence_rollback_test.cljs index 3efff3cb85..69a971ce21 100644 --- a/frontend/test/frontend_tests/data/persistence_rollback_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_rollback_test.cljs @@ -66,7 +66,7 @@ (t/deftest ^:async a-rejected-save-undoes-the-rejected-commit-and-every-later-one (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store failures]}] + (^:async fn [{:keys [file-id response requests requests$ store failures]}] (let [page-id (uuid/next) shape-id (uuid/next) other-id (uuid/next) @@ -84,8 +84,11 @@ :index 3}) (assoc-in [:workspace-local :selected] #{shape-id}))) (ptk/emit! store commit1 commit2 ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the first two commits are sent")) + (await (async/wait-for #(= 1 (count @requests)) "the first two commits are sent" {:signal requests$})) (ptk/emit! store commit3 other ::dps/force-persist) + ;; This test drives the pipeline step by step: these two waits are the + ;; pause that lets the persistence side effect advance before the next + ;; emit, not an effect to observe, so they keep the poll. (await (async/wait-for #(= 3 (count (get-in @store [:persistence :queue]))) "the third commit and the other file's edit are queued")) (ptk/emit! store commit4) @@ -93,7 +96,7 @@ "the fourth commit is buffered")) (rx/push! response (thp/rejection (:id @commit2))) - (await (async/wait-for #(= 2 (count @requests)) "the other file's edit is sent next")) + (await (async/wait-for #(= 2 (count @requests)) "the other file's edit is sent next" {:signal requests$})) (t/is (= "B" (thp/shape-name store file-id page-id shape-id)) "the file holds the commits the server kept") @@ -107,8 +110,8 @@ (t/is (= 1 (count @failures)) "the user is told once") (t/is (= other-id (get-in (second @requests) [1 :id]))) (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains"))))))) ;; Scenario: an edit is buffered while a save is in flight and the save comes ;; back rejected, so the rollback drops the edit; the 200 ms pending notice @@ -118,19 +121,19 @@ (t/deftest ^:async a-pending-notice-after-a-rollback-leaves-the-status-saved (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) commit1 (thp/rename-commit file-id page-id shape-id "A" "B") commit2 (thp/rename-commit file-id page-id shape-id "B" "C")] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "C"))) (ptk/emit! store commit1 ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the first commit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the first commit is sent" {:signal requests$})) (ptk/emit! store commit2) (rx/push! response (thp/rejection (:id @commit1))) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :buffer]))) - "the rollback drops both commits")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :buffer]))) + "the rollback drops both commits")) ;; The notice the dropped edit scheduled (ptk/emit! store (#'dps/update-status :pending)) (await (async/settle)) @@ -145,24 +148,24 @@ (t/deftest ^:async a-rollback-keeps-a-peer-change-to-the-same-attribute (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) commit1 (thp/rename-commit file-id page-id shape-id "A" "B") commit2 (thp/attr-commit file-id page-id shape-id :opacity 0.5 0.9)] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "B" :opacity 0.9))) (ptk/emit! store commit1 commit2) - (await (async/wait-for #(= 2 (count (get-in @store [:persistence :buffer]))) - "both edits are buffered")) + (await (async/wait-for-store store #(= 2 (count (get-in @store [:persistence :buffer]))) + "both edits are buffered")) (ptk/emit! store (thp/remote-commit file-id 1 [(thp/set-attr page-id shape-id :opacity 0.3)])) (await (async/settle)) (t/is (= 0.9 (thp/shape-attr store file-id page-id shape-id :opacity)) "the local edit stays on top of the peer's while pending") (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the edits are sent")) + (await (async/wait-for #(= 1 (count @requests)) "the edits are sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit2))) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the rollback settles")) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the rollback settles")) (t/is (= 0.3 (thp/shape-attr store file-id page-id shape-id :opacity)) "the peer's opacity, as on the server") (t/is (= "B" (thp/shape-name store file-id page-id shape-id)) @@ -177,7 +180,7 @@ (t/deftest ^:async a-rollback-ignores-peer-changes-older-than-the-pending-commits (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) commit0 (thp/attr-commit file-id page-id shape-id :opacity 0.5 0.7) @@ -185,22 +188,22 @@ commit2 (thp/rename-commit file-id page-id shape-id "B" "C")] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "A" :opacity 0.7))) (ptk/emit! store commit0) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) "the edit is buffered")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) "the edit is buffered")) (ptk/emit! store (thp/remote-commit file-id 1 [(thp/set-attr page-id shape-id :opacity 0.3)])) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the opacity edit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the opacity edit is sent" {:signal requests$})) (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the opacity edit is saved")) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the opacity edit is saved")) (t/is (empty? (get-in @store [:persistence :remote-changes file-id])) "nothing pending, nothing kept") (ptk/emit! store #(assoc-in % [:files file-id :data :pages-index page-id :objects shape-id :name] "C") commit1 commit2 ::dps/force-persist) - (await (async/wait-for #(= 2 (count @requests)) "the renames are sent")) + (await (async/wait-for #(= 2 (count @requests)) "the renames are sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit2))) - (await (async/wait-for #(= "B" (thp/shape-name store file-id page-id shape-id)) - "the rejected rename is undone")) + (await (async/wait-for-store store #(= "B" (thp/shape-name store file-id page-id shape-id)) + "the rejected rename is undone")) (t/is (= 0.7 (thp/shape-attr store file-id page-id shape-id :opacity)) "the saved opacity edit stays")))))) @@ -216,7 +219,7 @@ (t/deftest ^:async a-rejection-reverts-the-pending-commits-of-the-same-action-in-other-files (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [lib-id (uuid/next) page-id (uuid/next) shape-id (uuid/next) @@ -231,9 +234,9 @@ (thp/file-data page-id shape-id "C") (thp/file-data page-id lshape-id "W"))) (ptk/emit! store commit1 commit2 before linked after ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "this file's save is sent")) + (await (async/wait-for #(= 1 (count @requests)) "this file's save is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit2))) - (await (async/wait-for #(= 2 (count @requests)) "the library's save is sent next")) + (await (async/wait-for #(= 2 (count @requests)) "the library's save is sent next" {:signal requests$})) (t/is (= "B" (thp/shape-name store file-id page-id shape-id))) (t/is (= "Y" (thp/shape-name store lib-id page-id lshape-id)) "the library goes back to the edit made before the action") @@ -241,8 +244,8 @@ (t/is (= [[(thp/rename page-id lshape-id "Y")]] (sent-changes requests 1)) "only the library's edit made before the action is saved") (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains"))))))) ;; Scenario: one action makes a commit in a library and one in this file. ;; The library's is saved first; then this file's save comes back with the @@ -251,7 +254,7 @@ (t/deftest ^:async a-rejection-undoes-the-saved-commits-of-the-same-action-with-a-new-commit (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [lib-id (uuid/next) page-id (uuid/next) shape-id (uuid/next) @@ -263,22 +266,22 @@ (thp/file-data page-id shape-id "B") (thp/file-data page-id lshape-id "Y"))) (ptk/emit! store linked commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the library's save is sent first")) + (await (async/wait-for #(= 1 (count @requests)) "the library's save is sent first" {:signal requests$})) (t/is (= lib-id (get-in (first @requests) [1 :id]))) (rx/push! response {:revn 1}) - (await (async/wait-for #(= 2 (count @requests)) "this file's save is sent")) + (await (async/wait-for #(= 2 (count @requests)) "this file's save is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "a new commit undoes the library's")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "a new commit undoes the library's")) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 3 (count @requests)) "the new commit is sent")) + (await (async/wait-for #(= 3 (count @requests)) "the new commit is sent" {:signal requests$})) (t/is (= lib-id (get-in (nth @requests 2) [1 :id]))) (t/is (= [[(thp/rename page-id lshape-id "X")]] (sent-changes requests 2))) (t/is (= "A" (thp/shape-name store file-id page-id shape-id))) (t/is (= "X" (thp/shape-name store lib-id page-id lshape-id))) (rx/push! response {:revn 2}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains"))))))) ;; Scenario: as above, while the user has an undo transaction open. Proves: ;; the undo commit gets its own group, not the transaction's, so a later @@ -286,7 +289,7 @@ (t/deftest ^:async the-undo-of-a-saved-commit-stays-out-of-an-open-transaction (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [lib-id (uuid/next) page-id (uuid/next) shape-id (uuid/next) @@ -298,21 +301,21 @@ (thp/file-data page-id shape-id "B") (thp/file-data page-id lshape-id "Y"))) (ptk/emit! store linked commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the library's save is sent first")) + (await (async/wait-for #(= 1 (count @requests)) "the library's save is sent first" {:signal requests$})) (rx/push! response {:revn 1}) - (await (async/wait-for #(= 2 (count @requests)) "this file's save is sent")) + (await (async/wait-for #(= 2 (count @requests)) "this file's save is sent" {:signal requests$})) (ptk/emit! store (dwu/start-undo-transaction (js/Symbol) :timeout 0)) (let [tx-group (get-in @store [:workspace-undo :transaction :commit-group])] (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "a new commit undoes the library's")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "a new commit undoes the library's")) (t/is (some? tx-group)) (t/is (not= tx-group (get-in @store [:persistence :buffer 0 :undo-group]))) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 3 (count @requests)) "the new commit is sent")) + (await (async/wait-for #(= 3 (count @requests)) "the new commit is sent" {:signal requests$})) (rx/push! response {:revn 2}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains")))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains")))))))) ;; Scenario: as above, but the library is no longer loaded when this file's ;; save comes back. Proves: the undo of the library's saved commit is still @@ -320,7 +323,7 @@ (t/deftest ^:async a-rejection-queues-the-undo-of-a-saved-commit-of-a-file-not-loaded (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [lib-id (uuid/next) page-id (uuid/next) shape-id (uuid/next) @@ -332,20 +335,20 @@ (thp/file-data page-id shape-id "B") (thp/file-data page-id lshape-id "Y"))) (ptk/emit! store linked commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the library's save is sent first")) + (await (async/wait-for #(= 1 (count @requests)) "the library's save is sent first" {:signal requests$})) (rx/push! response {:revn 1}) - (await (async/wait-for #(= 2 (count @requests)) "this file's save is sent")) + (await (async/wait-for #(= 2 (count @requests)) "this file's save is sent" {:signal requests$})) (ptk/emit! store #(update % :files dissoc lib-id)) (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(= 3 (count @requests)) "the undo is queued and sent")) + (await (async/wait-for #(= 3 (count @requests)) "the undo is queued and sent" {:signal requests$})) (t/is (= lib-id (get-in (nth @requests 2) [1 :id]))) (t/is (= [[(thp/rename page-id lshape-id "X")]] (sent-changes requests 2))) (t/is (empty? (get-in @store [:persistence :buffer]))) (t/is (nil? (get-in @store [:files lib-id])) "no local data is made up") (t/is (= "A" (thp/shape-name store file-id page-id shape-id))) (rx/push! response {:revn 2}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains"))))))) ;; Scenario: one action makes two commits in this file and the save keeps ;; the first and rejects the second. Proves: the kept commit of the action @@ -353,7 +356,7 @@ (t/deftest ^:async a-rejection-undoes-the-kept-commits-of-the-same-action (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) group (uuid/next) @@ -361,18 +364,18 @@ commit2 (grouped (thp/attr-commit file-id page-id shape-id :opacity 1 0.5) group)] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "B" :opacity 0.5))) (ptk/emit! store commit1 commit2 ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the action is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the action is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit2))) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "a new commit undoes the kept one")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "a new commit undoes the kept one")) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 2 (count @requests)) "the new commit is sent")) + (await (async/wait-for #(= 2 (count @requests)) "the new commit is sent" {:signal requests$})) (t/is (= [[(thp/rename page-id shape-id "A")]] (sent-changes requests 1))) (t/is (= "A" (thp/shape-name store file-id page-id shape-id))) (t/is (= 1 (thp/shape-attr store file-id page-id shape-id :opacity))) (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains"))))))) ;; Scenario: applying a token to five shapes makes three commits inside one ;; undo transaction, all sent in one save; the server keeps the first two @@ -382,7 +385,7 @@ (t/deftest ^:async a-rejection-reverts-every-commit-of-an-undo-transaction (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-ids (into [] (repeatedly 5 uuid/next)) shapes (map #(cts/setup-shape {:id % :type :rect :name "R" :opacity 1 @@ -416,14 +419,14 @@ (apply-to (subvec shape-ids 4 5)) (dwu/commit-undo-transaction tx-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the action is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the action is sent" {:signal requests$})) (let [groups (get-in (first @requests) [1 :changes-with-metadata])] (t/is (= 3 (count groups)) "the save carries the three commits") (rx/push! response (thp/rejection (:commit-id (nth groups 2))))) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "a new commit undoes the kept ones")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "a new commit undoes the kept ones")) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 2 (count @requests)) "the new commit is sent")) + (await (async/wait-for #(= 2 (count @requests)) "the new commit is sent" {:signal requests$})) (doseq [id shape-ids] (t/is (= 1 (thp/shape-attr store file-id page-id id :opacity))) (t/is (nil? (thp/shape-attr store file-id page-id id :applied-tokens)))) @@ -436,8 +439,8 @@ (get-in (second @requests) [1 :changes-with-metadata]))) "the new commit puts the kept opacity back on the server") (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the queue drains"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the queue drains"))))))) ;; UI state after a rollback. @@ -447,7 +450,7 @@ (t/deftest ^:async a-rollback-leaves-the-edition-of-a-removed-shape (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) text-id (uuid/next) @@ -458,10 +461,10 @@ (thp/add-shape page-id text))) #(assoc-in % [:workspace-local :edition] text-id)) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the new text is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the new text is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(nil? (get-in @store [:workspace-local :edition])) - "the rollback leaves edition mode")) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-local :edition])) + "the rollback leaves edition mode")) (t/is (nil? (thp/shape-attr store file-id page-id text-id :id)))))))) ;; Scenario: the user edits the nodes of a saved path and the rejected commit @@ -472,7 +475,7 @@ (t/deftest ^:async a-rollback-leaves-the-edition-of-a-changed-path (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) commit (thp/rename-commit file-id page-id shape-id "A" "B")] @@ -485,10 +488,10 @@ (assoc-in [:workspace-drawing :object] {:id shape-id :content [:rejected]}))) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the edit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the edit is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(nil? (get-in @store [:workspace-local :edition])) - "the rollback leaves edition mode")) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-local :edition])) + "the rollback leaves edition mode")) (t/is (= "A" (thp/shape-name store file-id page-id shape-id))) (t/is (nil? (get-in @store [:workspace-local :edit-path shape-id])) "the editor's state is dropped") @@ -502,7 +505,7 @@ (t/deftest ^:async a-rollback-keeps-the-edition-of-an-unchanged-shape (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) other-id (uuid/next) @@ -514,10 +517,10 @@ (thp/add-shape page-id other))) #(assoc-in % [:workspace-local :edition] other-id)) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the edit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the edit is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(= "A" (thp/shape-name store file-id page-id shape-id)) - "the rollback lands")) + (await (async/wait-for-store store #(= "A" (thp/shape-name store file-id page-id shape-id)) + "the rollback lands")) (await (async/settle)) (t/is (= other-id (get-in @store [:workspace-local :edition])))))))) @@ -527,7 +530,7 @@ (t/deftest ^:async a-rollback-closes-the-color-picker-of-a-removed-shape (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) rect-id (uuid/next) @@ -540,10 +543,10 @@ (assoc-in [:workspace-local :selected] #{rect-id}) (assoc ::modal/modal {:id (uuid/next) :type :colorpicker :props {}}))) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the new rectangle is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the new rectangle is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit))) - (await (async/wait-for #(empty? (get-in @store [:workspace-local :selected])) - "the rollback deselects the removed shape")) + (await (async/wait-for-store store #(empty? (get-in @store [:workspace-local :selected])) + "the rollback deselects the removed shape")) (t/is (nil? (::modal/modal @store)) "the color picker is closed")))))) ;; Scenario: the user undoes a rename with Ctrl+Z and the server rejects the @@ -552,7 +555,7 @@ (t/deftest ^:async a-rejected-undo-puts-the-change-and-the-history-back (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) item (thp/undo-entry (thp/rename-commit file-id page-id shape-id "A" "B"))] @@ -560,19 +563,19 @@ (thp/open-file file-id page-id (thp/file-data page-id shape-id "B")) #(assoc % :workspace-undo {:items [item] :index 0})) (ptk/emit! store dwu/undo) - (await (async/wait-for #(= "A" (thp/shape-name store file-id page-id shape-id)) - "the rename is undone")) + (await (async/wait-for-store store #(= "A" (thp/shape-name store file-id page-id shape-id)) + "the rename is undone")) (t/is (= -1 (get-in @store [:workspace-undo :index]))) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the undo is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the undo is sent" {:signal requests$})) (rx/push! response (thp/rejection (get-in (first @requests) [1 :changes-with-metadata 0 :commit-id]))) - (await (async/wait-for #(= "B" (thp/shape-name store file-id page-id shape-id)) - "the rename is back")) + (await (async/wait-for-store store #(= "B" (thp/shape-name store file-id page-id shape-id)) + "the rename is back")) (t/is (= [item] (get-in @store [:workspace-undo :items])) "the history stays") (t/is (= 0 (get-in @store [:workspace-undo :index])) "it points at the rename again") - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the rollback settles"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the rollback settles"))))))) ;; Version restores. @@ -598,7 +601,7 @@ (try (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [page-id (uuid/next) shape-id (uuid/next) ;; A change type with no handler, so its undo throws @@ -614,7 +617,7 @@ (thp/shape-name store file-id page-id shape-id)]))))] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "B"))) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the rename is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the rename is sent" {:signal requests$})) (rx/push! response (thp/rejection (:id @commit))) (await (async/wait-for #(= 1 @reloads) "the rollback reloads the file")) (await (async/settle)) @@ -639,7 +642,7 @@ (t/deftest ^:async an-answer-from-before-a-restore-leaves-the-reloaded-file-alone (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store failures]}] + (^:async fn [{:keys [file-id response requests requests$ store failures]}] (let [page-id (uuid/next) shape-id (uuid/next) commit1 (thp/rename-commit file-id page-id shape-id "A" "B") @@ -649,10 +652,10 @@ (thp/open-file file-id page-id (thp/file-data page-id shape-id "D")) #(assoc-in % [:files file-id :vern] 0)) (ptk/emit! store commit1 commit2 ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the first two renames are sent")) + (await (async/wait-for #(= 1 (count @requests)) "the first two renames are sent" {:signal requests$})) (ptk/emit! store commit3 ::dps/force-persist) - (await (async/wait-for #(= 2 (count (get-in @store [:persistence :queue]))) - "the third rename is queued")) + (await (async/wait-for-store store #(= 2 (count (get-in @store [:persistence :queue]))) + "the third rename is queued")) (ptk/emit! store (restore-version file-id (thp/file-data page-id shape-id "Restored"))) (ptk/emit! store (thp/remote-commit file-id 1 [(thp/set-attr page-id shape-id :opacity 0.5)])) @@ -660,9 +663,9 @@ (t/is (= "Restored" (thp/shape-name store file-id page-id shape-id)) "a peer's change does not replay the renames") (rx/push! response (thp/rejection (:id @commit2))) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the queue drains")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the queue drains")) (t/is (= "Restored" (thp/shape-name store file-id page-id shape-id)) "the file keeps the restored version") (t/is (= 0.5 (thp/shape-attr store file-id page-id shape-id :opacity))) @@ -677,20 +680,20 @@ (t/deftest ^:async a-save-refused-after-a-restore-is-dropped (await (thp/with-persistence - (^:async fn [{:keys [file-id response requests store failures]}] + (^:async fn [{:keys [file-id response requests requests$ store failures]}] (let [page-id (uuid/next) shape-id (uuid/next)] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "B")) #(assoc-in % [:files file-id :vern] 0)) (ptk/emit! store (thp/rename-commit file-id page-id shape-id "A" "B") ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the rename is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the rename is sent" {:signal requests$})) (ptk/emit! store (restore-version file-id (thp/file-data page-id shape-id "Restored"))) (rx/error! response (ex-info "A different version has been restored for the file." {:type :validation :code :vern-conflict})) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the refused save is dropped")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the refused save is dropped")) (t/is (empty? @failures)) (t/is (nil? (get-in @store [:workspace-global :persistence-lock]))) (t/is (= "Restored" (thp/shape-name store file-id page-id shape-id)))))))) @@ -783,7 +786,7 @@ [prepare answer] (let [flashes (atom [])] (-> (thp/with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (await (mock/with-mocks* {errors/flash (fn [& {:as params}] @@ -795,7 +798,7 @@ commit1 (thp/rename-commit file-id page-id shape-id "A" "B")] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "B"))) (ptk/emit! store commit1 ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the rename is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the rename is sent" {:signal requests$})) (ptk/emit! store #(prepare % file-id)) (rx/push! response (answer (thp/rejection (:id @commit1)))) (await (async/wait-for #(seq @flashes) "the notice is flashed"))))))) diff --git a/frontend/test/frontend_tests/data/persistence_test.cljs b/frontend/test/frontend_tests/data/persistence_test.cljs index 544a41ee03..9f23fb317f 100644 --- a/frontend/test/frontend_tests/data/persistence_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_test.cljs @@ -366,8 +366,8 @@ ;; Phase 1 — two commits share the first request; its landing starts the second. (reset! clock 290000) (rx/push! response {:revn 1}) - (await (async/wait-for #(= 1 (count (get-in @store [:persistence :queue]))) - "first save lands, second request starts")) + (await (async/wait-for-store store #(= 1 (count (get-in @store [:persistence :queue]))) + "first save lands, second request starts")) (reset! clock 300001) (rx/push! ticks :tick) (await (async/settle)) @@ -380,9 +380,9 @@ (t/is (= 1 (count @reports)) "The second request has now stalled") (rx/push! response {:revn 2}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "second save lands")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "second save lands")) (t/is (= :saved (get-in @store [:persistence :status]))) ;; Phase 3 — answering saves the file: the tick stays silent. (reset! clock 1000000) @@ -471,7 +471,7 @@ timer-s (rx/subject)] (await (with-watchdog - (^:async fn [{:keys [clock ticks reports causes requests store file-id]}] + (^:async fn [{:keys [clock ticks reports causes requests requests$ store file-id]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [_] timer-s)) @@ -485,12 +485,12 @@ ;; Phase 1 — first send fails transiently; the episode parks ;; on the hand-fired timer. (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :retrying (get-in @store [:persistence :status])) - "transient failure retries")) + (await (async/wait-for-store store #(= :retrying (get-in @store [:persistence :status])) + "transient failure retries")) (t/is (= 1 (count @requests))) ;; Phase 2 — the timer fires and the resend goes out, hanging. (rx/push! timer-s :tick) - (await (async/wait-for #(= 2 (count @requests)) "retry resends")) + (await (async/wait-for #(= 2 (count @requests)) "retry resends" {:signal requests$})) (t/is (= :retrying (get-in @store [:persistence :status]))) ;; Phase 3 — past the deadline the watchdog reports the stall ;; once, keeping the queued edit. @@ -537,9 +537,9 @@ (ptk/emit! store (local-commit file-id) #(assoc-in % [:permissions :can-edit] false) ::dps/force-persist) - (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) - (= 1 (count (get-in @store [:persistence :queue])))) - "permission loss fails the save")) + (await (async/wait-for-store store #(and (= :error (get-in @store [:persistence :status])) + (= 1 (count (get-in @store [:persistence :queue])))) + "permission loss fails the save")) (t/is (= :error (get-in @store [:persistence :status]))) (t/is (= 1 (count (get-in @store [:persistence :queue]))) "the undeliverable edit is retained, so waiters keep rejecting") @@ -557,18 +557,18 @@ (ptk/emit! store (local-commit file-id) #(assoc-in % [:permissions :can-edit] false) ::dps/force-persist) - (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) - (= 1 (count (get-in @store [:persistence :queue])))) - "the save fails terminally")) + (await (async/wait-for-store store #(and (= :error (get-in @store [:persistence :status])) + (= 1 (count (get-in @store [:persistence :queue])))) + "the save fails terminally")) ;; Edit rights come back: a team-role change is one of the two real ;; recovery triggers (the other is opening a file, whose event does ;; real work and is not usable as a bare signal here). (ptk/emit! store #(assoc-in % [:permissions :can-edit] true) (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "re-entering the file settles the queue")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "re-entering the file settles the queue")) (t/is (nil? (get-in @store [:persistence :error])) "the halt is cleared once the user comes back") (t/is (empty? @requests) @@ -594,8 +594,8 @@ commit #(assoc-in % [:permissions :can-edit] false) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "lost permission fails the save")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "lost permission fails the save")) (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) (await (async/settle)) @@ -608,9 +608,9 @@ (ptk/emit! store #(assoc-in % [:permissions :can-edit] true) (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "edit rights back release the edit")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "edit rights back release the edit")) (t/is (= "A" (thp/shape-name store file-id page-id shape-id)) "the file shows the server's data") (t/is (empty? (get-in @store [:workspace-undo :items])) @@ -636,20 +636,20 @@ (ptk/emit! store #(assoc-in % [:workspace-global :read-only?] true) (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(some? (get-in @store [:workspace-global :persistence-lock])) - "the failed save locks the workspace")) + (await (async/wait-for-store store #(some? (get-in @store [:workspace-global :persistence-lock])) + "the failed save locks the workspace")) (t/is (true? (get-in @store [:workspace-global :read-only?]))) (t/is (true? (get-in @store [:workspace-global :persistence-lock :read-only?]))) (ptk/emit! store (dwc/set-workspace-read-only false)) - (await (async/wait-for #(false? (get-in @store [:workspace-global :persistence-lock :read-only?])) - "the requested mode is kept for later")) + (await (async/wait-for-store store #(false? (get-in @store [:workspace-global :persistence-lock :read-only?])) + "the requested mode is kept for later")) (t/is (true? (get-in @store [:workspace-global :read-only?])) "the workspace stays read-only while saves fail") (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(nil? (get-in @store [:workspace-global :persistence-lock])) - "recovery lifts the lock")) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-global :persistence-lock])) + "recovery lifts the lock")) (t/is (false? (get-in @store [:workspace-global :read-only?])) "the workspace takes the mode the user asked for")) server-error))) @@ -665,8 +665,8 @@ (ptk/emit! store (local-commit file-id) #(assoc-in % [:permissions :can-edit] false) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "lost permission fails the save")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "lost permission fails the save")) (await (async/settle)) (t/is (nil? (get-in @store [:workspace-global :persistence-lock]))) (t/is (nil? (get-in @store [:workspace-global :read-only?]))) @@ -680,18 +680,18 @@ (with-persistence (^:async fn [{:keys [file-id store]}] (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(some? (get-in @store [:workspace-global :persistence-lock])) - "the failed save locks the workspace")) + (await (async/wait-for-store store #(some? (get-in @store [:workspace-global :persistence-lock])) + "the failed save locks the workspace")) (ptk/emit! store (drw/context-lost) (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(nil? (get-in @store [:workspace-global :persistence-lock])) - "recovery lifts the save lock")) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-global :persistence-lock])) + "recovery lifts the save lock")) (t/is (true? (get-in @store [:workspace-global :read-only?])) "the lost context keeps the workspace read-only") (ptk/emit! store (drw/context-restored)) - (await (async/wait-for #(false? (get-in @store [:workspace-global :read-only?])) - "the restored context brings back the first mode"))) + (await (async/wait-for-store store #(false? (get-in @store [:workspace-global :read-only?])) + "the restored context brings back the first mode"))) server-error))) ;; Scenario: two files have pending edits and the first send fails @@ -703,7 +703,7 @@ sends (atom 0)] (await (with-persistence - (^:async fn [{:keys [requests store]}] + (^:async fn [{:keys [requests requests$ store]}] (ptk/emit! store #(assoc % :files (into (:files %) (map (fn [id] [id {:id id :revn 0}])) @@ -711,17 +711,17 @@ (local-commit (nth files 0)) (local-commit (nth files 1)) ::dps/force-persist) - (await (async/wait-for #(seq @requests) "the first file is sent")) + (await (async/wait-for #(seq @requests) "the first file is sent" {:signal requests$})) (let [failed-file (get-in (second (first @requests)) [:id])] - (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) - (not (empty? (get-in @store [:persistence :queue])))) - "the terminal failure retains the queue")) + (await (async/wait-for-store store #(and (= :error (get-in @store [:persistence :status])) + (not (empty? (get-in @store [:persistence :queue])))) + "the terminal failure retains the queue")) ;; The user acts again: recovery releases the failed file and drains ;; the rest. (ptk/emit! store (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the released file frees the rest of the queue")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the released file frees the rest of the queue")) (t/is (= 2 (count @requests)) "the other file was saved after the failed one was released") (t/is (= 1 (count (filter #(= failed-file (get-in (second %) [:id])) @requests))) @@ -747,8 +747,8 @@ i18n/tr (mock/stub #(str "translated:" %))} (ptk/emit! store (local-commit file-id) ::dps/force-persist) (.error response (ex-info "Raw transport details" {:type cause-type})) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "failed request errors the save")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "failed request errors the save")) (t/is (= :error (get-in @store [:persistence :status]))) (let [data (get-in @store [:persistence :error])] (ptk/handle-error (assoc data ::errors/instance (ex-info "Save failed" data)))) @@ -857,8 +857,8 @@ :index {} :run-id (uuid/next) :status :saving}) (dps/initialize-persistence)) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "dangling commit errors instead of skipping")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "dangling commit errors instead of skipping")) (t/is (= :error (get-in @store [:persistence :status]))) (t/is (= [id] (vec (get-in @store [:persistence :queue])))) (t/is (empty? @requests))))))) @@ -870,7 +870,7 @@ (t/deftest ^:async initialization-recovers-an-unsent-commit-with-a-dangling-run-id (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [commit (pcm/queue-entry [@(local-commit file-id)]) id (:id commit)] (ptk/emit! store @@ -878,12 +878,12 @@ :index {id commit} :run-id (uuid/next) :status :saving}) (dps/initialize-persistence)) - (await (async/wait-for #(= 1 (count @requests)) "dangling commit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "dangling commit is sent" {:signal requests$})) (t/is (= 1 (count @requests))) (rx/push! response {:revn 1}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "recovered save lands")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "recovered save lands")) (t/is (= :saved (get-in @store [:persistence :status]))) (t/is (empty? (get-in @store [:persistence :queue])))))))) @@ -892,15 +892,15 @@ reinitializes persistence: the pending request must not be sent again." [interrupt] (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (apply ptk/emit! store (local-commit file-id) ::dps/force-persist interrupt) (ptk/emit! store (dps/initialize-persistence)) - (await (async/wait-for #(= 1 (count @requests)) "only the active request is sent")) + (await (async/wait-for #(= 1 (count @requests)) "only the active request is sent" {:signal requests$})) (t/is (= 1 (count @requests))) (rx/push! response {:revn 1}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "active request completes")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "active request completes")) (t/is (= :saved (get-in @store [:persistence :status]))) (t/is (empty? (get-in @store [:persistence :queue])))))) @@ -925,15 +925,15 @@ (ptk/emit! store (local-commit file-id) #(assoc-in % [:permissions :can-edit] false) ::dps/force-persist) - (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) - (= 1 (count (get-in @store [:persistence :queue])))) - "permission loss fails the save")) + (await (async/wait-for-store store #(and (= :error (get-in @store [:persistence :status])) + (= 1 (count (get-in @store [:persistence :queue])))) + "permission loss fails the save")) ;; Permission comes back and a file is opened again. (ptk/emit! store #(assoc-in % [:permissions :can-edit] true) (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "recovery releases the undeliverable edit")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "recovery releases the undeliverable edit")) (t/is (empty? @requests) "the undeliverable edit is never replayed, not even once the permission is back") @@ -950,9 +950,9 @@ (^:async fn [{:keys [file-id response store]}] (ptk/emit! store (local-commit file-id) ::dps/force-persist) (.error response (ex-info "Validation failed" {:type :validation})) - (await (async/wait-for #(and (= :error (get-in @store [:persistence :status])) - (= 1 (count (get-in @store [:persistence :queue])))) - "the save fails terminally")) + (await (async/wait-for-store store #(and (= :error (get-in @store [:persistence :status])) + (= 1 (count (get-in @store [:persistence :queue])))) + "the save fails terminally")) (ptk/emit! store (dps/initialize-persistence)) (await (async/settle)) (t/is (= :error (get-in @store [:persistence :status])) @@ -971,13 +971,13 @@ (ptk/emit! store (local-commit file-id) ::dps/force-persist (ptk/data-event ::dps/error)) (rx/push! response {:revn 1}) - (await (async/wait-for #(= 1 (count (get-in @store [:persistence :queue]))) - "acknowledged commit stays queued")) + (await (async/wait-for-store store #(= 1 (count (get-in @store [:persistence :queue]))) + "acknowledged commit stays queued")) (t/is (= 1 (count (get-in @store [:persistence :queue])))) (ptk/emit! store (dps/initialize-persistence)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "recovery completes without resending")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "recovery completes without resending")) (t/is (= 1 (count @requests)) "The acknowledged changes must not be sent again") (t/is (= :saved (get-in @store [:persistence :status]))) (t/is (empty? (get-in @store [:persistence :queue]))))))) @@ -989,19 +989,19 @@ (t/deftest ^:async recovery-sends-again-a-save-with-an-unknown-outcome (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [commit (assoc (pcm/queue-entry [@(local-commit file-id)]) ::dps/request-id (uuid/next)) id (:id commit)] (ptk/emit! store #(assoc % :persistence {:queue (conj #queue [] id) :index {id commit} :status :saving}) (dps/initialize-persistence)) - (await (async/wait-for #(= 1 (count @requests)) "the commit is sent again")) + (await (async/wait-for #(= 1 (count @requests)) "the commit is sent again" {:signal requests$})) (t/is (= id (:commit-id (second (first @requests))))) (rx/push! response {:revn 1}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the resent commit saves")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the resent commit saves")) (t/is (= 1 (count @requests)))))))) ;; Scenario: an edit is queued before persistence even initializes. On @@ -1011,18 +1011,21 @@ (t/deftest ^:async initialization-flushes-buffered-edits-without-duplicating-them (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (ptk/emit! store (local-commit file-id) (dps/initialize-persistence) ::dps/force-persist) + ;; The predicate reads both the store and the request counter, so it + ;; passes the merged signal; the wrapper only provides the store one. (await (async/wait-for #(and (= 1 (count @requests)) (= 1 (count (get-in @store [:persistence :queue])))) - "buffered edit is sent once")) + "buffered edit is sent once" + {:signal (rx/merge requests$ (ptk/input-stream store))})) (t/is (= 1 (count @requests))) (t/is (= 1 (count (get-in @store [:persistence :queue])))) (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "buffered edit saves")) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "buffered edit saves")) (t/is (= :saved (get-in @store [:persistence :status]))))))) ;; Scenario: the transport answers synchronously (immediate observable). @@ -1033,9 +1036,9 @@ (with-persistence (^:async fn [{:keys [file-id store]}] (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "synchronous save completes")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "synchronous save completes")) (t/is (= :saved (get-in @store [:persistence :status]))) (t/is (empty? (get-in @store [:persistence :queue])))) (fn [_ _] (rx/of {:revn 1}))))) @@ -1049,12 +1052,12 @@ (t/deftest ^:async unsaved-edits-keep-the-leave-page-guard (await (with-persistence - (^:async fn [{:keys [file-id response requests store tasks]}] + (^:async fn [{:keys [file-id response requests requests$ store tasks]}] (ptk/emit! store (local-commit file-id)) (await (async/wait-for #(contains? @tasks :persistence) "a buffered edit turns the guard on")) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(seq @requests) "the edit is sent")) + (await (async/wait-for #(seq @requests) "the edit is sent" {:signal requests$})) (t/is (contains? @tasks :persistence) "the guard stays on while the save is in flight") (rx/push! response {:revn 1}) (await (async/wait-for #(not (contains? @tasks :persistence)) @@ -1068,17 +1071,17 @@ (t/deftest ^:async edits-buffered-during-a-save-keep-the-leave-page-guard (await (with-persistence - (^:async fn [{:keys [file-id response requests store tasks]}] + (^:async fn [{:keys [file-id response requests requests$ store tasks]}] (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the first edit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the first edit is sent" {:signal requests$})) (ptk/emit! store (local-commit file-id)) (rx/push! response {:revn 1}) - (await (async/wait-for #(empty? (get-in @store [:persistence :queue])) - "the first save lands")) + (await (async/wait-for-store store #(empty? (get-in @store [:persistence :queue])) + "the first save lands")) (await (async/settle)) (t/is (contains? @tasks :persistence) "the buffered edit keeps the guard on") (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 2 (count @requests)) "the buffered edit is sent")) + (await (async/wait-for #(= 2 (count @requests)) "the buffered edit is sent" {:signal requests$})) (rx/push! response {:revn 2}) (await (async/wait-for #(not (contains? @tasks :persistence)) "the second save turns the guard off")) @@ -1095,15 +1098,15 @@ (ptk/emit! store (local-commit file-id) #(assoc-in % [:permissions :can-edit] false) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the save fails terminally")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the save fails terminally")) (await (async/settle)) (t/is (contains? @tasks :persistence) "the retained edit keeps the guard on") (ptk/emit! store #(assoc-in % [:permissions :can-edit] true) (ptk/data-event :app.main.data.common/change-team-role)) - (await (async/wait-for #(empty? (get-in @store [:persistence :queue])) - "recovery releases the edit")) + (await (async/wait-for-store store #(empty? (get-in @store [:persistence :queue])) + "recovery releases the edit")) (await (async/wait-for #(not (contains? @tasks :persistence)) "the release turns the guard off")) (t/is (= :saved (get-in @store [:persistence :status]))))))) @@ -1118,13 +1121,13 @@ {dps/leave-signals (fn [] leave-s)} (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (ptk/emit! store (local-commit file-id)) (await (async/settle)) (t/is (empty? @requests) "the edit waits in the buffer") (rx/push! leave-s :hidden) (await (async/wait-for #(= 1 (count @requests)) - "leaving sends the edit at once")) + "leaving sends the edit at once" {:signal requests$})) (t/is (= 1 (count @requests)))))))))) ;; Scenario: nothing is buffered when the user leaves. Proves: a leave @@ -1163,13 +1166,13 @@ (let [done (atom false)] (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (await (mock/with-mocks* {refs/persistence (l/derived :persistence store)} (ptk/emit! store (local-commit file-id)) (wait-files-saved store #{file-id} 60000 done) - (await (async/wait-for #(= 1 (count @requests)) "the edit is sent at once")) + (await (async/wait-for #(= 1 (count @requests)) "the edit is sent at once" {:signal requests$})) (await (async/settle)) (t/is (false? @done) "the wait holds while the save is in flight") (rx/push! response {:revn 1}) @@ -1182,12 +1185,12 @@ (let [done (atom false)] (await (with-persistence - (^:async fn [{:keys [file-id requests store]}] + (^:async fn [{:keys [file-id requests requests$ store]}] (await (mock/with-mocks* {refs/persistence (l/derived :persistence store)} (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the other file's save hangs")) + (await (async/wait-for #(= 1 (count @requests)) "the other file's save hangs" {:signal requests$})) (wait-files-saved store #{(uuid/next)} 60000 done) (await (async/wait-for #(true? @done) "the wait ends at once"))))))))) @@ -1226,8 +1229,8 @@ (^:async fn [{:keys [file-id store]}] (let [events (save-failed-events store)] (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= :error (get-in @store [:persistence :status])) - "the save fails")) + (await (async/wait-for-store store #(= :error (get-in @store [:persistence :status])) + "the save fails")) (await (async/wait-for #(seq @events) "the event is emitted")) (await (async/settle)) (t/is (= 1 (count @events))) @@ -1248,23 +1251,23 @@ (t/deftest ^:async buffered-commits-stay-in-the-store-until-queued (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [commit (local-commit file-id)] (ptk/emit! store commit) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "the edit is buffered")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "the edit is buffered")) (t/is (= [(:id @commit)] (mapv :id (get-in @store [:persistence :buffer])))) (t/is (empty? (get-in @store [:persistence :queue]))) (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(seq @requests) "the flush sends the edit")) + (await (async/wait-for #(seq @requests) "the flush sends the edit" {:signal requests$})) (t/is (empty? (get-in @store [:persistence :buffer]))) (t/is (= [(:id @commit)] (->> (get-in @store [:persistence :queue]) (mapcat #(get-in @store [:persistence :index % :commits])) (mapv :id)))) (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the save lands"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the save lands"))))))) ;; Scenario: an edit waits in the buffer when another user's change to the ;; same attribute arrives. The server will apply the remote change first and @@ -1279,8 +1282,8 @@ local (thp/rename-commit file-id page-id shape-id "Original" "Local")] (ptk/emit! store (thp/open-file file-id page-id (thp/file-data page-id shape-id "Local"))) (ptk/emit! store local) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "the local edit is buffered")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "the local edit is buffered")) (ptk/emit! store (thp/remote-commit file-id 1 [(thp/rename page-id shape-id "Remote")])) (await (async/settle)) (t/is (= "Local" (thp/shape-name store file-id page-id shape-id)))))))) @@ -1299,8 +1302,8 @@ (thp/file-data page-id shape-id "A") (thp/file-data page-id shape-id "Y"))) (ptk/emit! store (thp/rename-commit lib-id page-id shape-id "X" "Y")) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "the library edit is buffered")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "the library edit is buffered")) (ptk/emit! store (thp/remote-commit file-id 1 [(thp/set-attr page-id shape-id :opacity 0.3)])) (await (async/settle)) (t/is (= 0.3 (thp/shape-attr store file-id page-id shape-id :opacity))) @@ -1362,7 +1365,7 @@ (t/deftest ^:async a-repair-answered-before-its-message-goes-under-the-saved-commit (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [session-id (uuid/next) page-id (uuid/next) shape-id (uuid/next) @@ -1372,18 +1375,18 @@ repair [(set-root-children page-id [shape-id])]] (open-with-new-child store file-id session-id page-id shape-id child) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the new child is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the new child is sent" {:signal requests$})) (rx/push! response {:revn 0 :lagged [(lagged-row file-id 1 repair-session-id repair) (lagged-row file-id 2 session-id (:redo-changes @commit))]}) - (await (async/wait-for #(get-in @store [:persistence :index (:id @commit) ::dps/awaiting]) - "the saved commit waits for the repair")) + (await (async/wait-for-store store #(get-in @store [:persistence :index (:id @commit) ::dps/awaiting]) + "the saved commit waits for the repair")) (t/is (= [(:id @commit)] (vec (get-in @store [:persistence :queue])))) (ptk/emit! store (thp/remote-commit file-id 1 repair)) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the saved commit is acknowledged once the repair arrives")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the saved commit is acknowledged once the repair arrives")) (t/is (= [shape-id (:id child)] (root-children store file-id page-id)) "the repair goes under the new child, as on the server")))))) @@ -1392,7 +1395,7 @@ (t/deftest ^:async a-save-answer-does-not-wait-for-changes-already-received (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [session-id (uuid/next) page-id (uuid/next) shape-id (uuid/next) @@ -1402,14 +1405,14 @@ repair [(set-root-children page-id [shape-id])]] (open-with-new-child store file-id session-id page-id shape-id child) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the new child is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the new child is sent" {:signal requests$})) (ptk/emit! store (thp/remote-commit file-id 1 repair)) (rx/push! response {:revn 0 :lagged [(lagged-row file-id 1 repair-session-id repair) (lagged-row file-id 2 session-id (:redo-changes @commit))]}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the save lands at once")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the save lands at once")) (t/is (= [shape-id (:id child)] (root-children store file-id page-id)))))))) ;; Scenario: the save answer lists a repair whose message never reaches the @@ -1426,7 +1429,7 @@ (try (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (await (mock/with-mocks* {rx/timer (mock/stub (fn [ms] @@ -1441,13 +1444,13 @@ commit (add-child-commit file-id page-id child)] (open-with-new-child store file-id session-id page-id shape-id child) (ptk/emit! store commit ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the new child is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the new child is sent" {:signal requests$})) (rx/push! response {:revn 0 :lagged [(lagged-row file-id 1 repair-session-id [(set-root-children page-id [shape-id])]) (lagged-row file-id 2 session-id (:redo-changes @commit))]}) - (await (async/wait-for #(get-in @store [:persistence :index (:id @commit) ::dps/awaiting]) - "the saved commit waits for the repair")) + (await (async/wait-for-store store #(get-in @store [:persistence :index (:id @commit) ::dps/awaiting]) + "the saved commit waits for the repair")) (t/is (zero? @reloads)) (rx/push! timeout-s :tick) (await (async/wait-for #(= 1 @reloads) "the file reloads")) @@ -1465,19 +1468,19 @@ (t/deftest ^:async a-save-sends-each-commit-as-its-own-group (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [commit1 (local-commit file-id) commit2 (local-commit file-id)] (ptk/emit! store commit1 commit2 ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "both edits go in one save")) + (await (async/wait-for #(= 1 (count @requests)) "both edits go in one save" {:signal requests$})) (let [[_ params] (first @requests)] (t/is (not (contains? params :changes))) (t/is (= [{:commit-id (:id @commit1) :changes (:redo-changes @commit1)} {:commit-id (:id @commit2) :changes (:redo-changes @commit2)}] (:changes-with-metadata params)))) (rx/push! response {:revn 1}) - (await (async/wait-for #(= :saved (get-in @store [:persistence :status])) - "the save lands"))))))) + (await (async/wait-for-store store #(= :saved (get-in @store [:persistence :status])) + "the save lands"))))))) ;; Scenario: a save is in flight while a later edit waits in the buffer, and ;; something waits for the edits to be saved (as a library sync does before @@ -1487,20 +1490,20 @@ (t/deftest ^:async waiting-for-the-save-covers-the-buffered-edits (await (with-persistence - (^:async fn [{:keys [file-id response requests store]}] + (^:async fn [{:keys [file-id response requests requests$ store]}] (let [released? #(dps/terminal-status? (:persistence @store))] (ptk/emit! store (local-commit file-id) ::dps/force-persist) - (await (async/wait-for #(= 1 (count @requests)) "the first edit is sent")) + (await (async/wait-for #(= 1 (count @requests)) "the first edit is sent" {:signal requests$})) (ptk/emit! store (local-commit file-id)) - (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) - "the second edit waits in the buffer")) + (await (async/wait-for-store store #(seq (get-in @store [:persistence :buffer])) + "the second edit waits in the buffer")) (rx/push! response {:revn 1}) - (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) - (empty? (get-in @store [:persistence :queue]))) - "the first save lands")) + (await (async/wait-for-store store #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + "the first save lands")) (t/is (seq (get-in @store [:persistence :buffer]))) (t/is (not (released?)) "the buffered edit is not saved yet") (ptk/emit! store ::dps/force-persist) - (await (async/wait-for #(= 2 (count @requests)) "the buffered edit is sent")) + (await (async/wait-for #(= 2 (count @requests)) "the buffered edit is sent" {:signal requests$})) (rx/push! response {:revn 2}) (await (async/wait-for released? "the waiters are released"))))))) diff --git a/frontend/test/frontend_tests/data/team_test.cljs b/frontend/test/frontend_tests/data/team_test.cljs index 9bf84efc79..d05041c0db 100644 --- a/frontend/test/frontend_tests/data/team_test.cljs +++ b/frontend/test/frontend_tests/data/team_test.cljs @@ -5,9 +5,13 @@ (ns frontend-tests.data.team-test (:require [app.common.uuid :as uuid] + [app.main.data.modal :as modal] [app.main.data.team :as dtm] [app.main.repo :as rp] + [app.main.router :as rt] [app.main.ui.dashboard.team :as dtui] + [app.util.dom :as dom] + [app.util.session-state :as ss] [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] [frontend-tests.helpers.async :as async] @@ -27,6 +31,89 @@ :on-next #(swap! emitted conj %)))) (.then (fn [_] {:team-id team-id :events @emitted}))))) +(t/deftest ^:async check-and-create-team-redirects-to-sso-and-remembers-the-action + (let [team-id "00000000-0000-4000-8000-000000000001" + organization-id "00000000-0000-4000-8000-000000000002" + profile-id "00000000-0000-4000-8000-000000000003" + dashboard-url "https://penpot.example/#/dashboard-recent?team-id=current-team" + callback-url (str dashboard-url "&pending-action-id=pending-id") + team {:id team-id + :organization {:id organization-id + :name "Product" + :owner-id profile-id}} + requests (atom []) + saved-actions (atom []) + emitted (atom [])] + (await + (mock/with-mocks* + {rp/cmd! (mock/stub + (fn [command params] + (swap! requests conj [command params]) + (case command + :get-teams (rx/of [team]) + :check-nitrate-sso + (rx/of {:authorized false + :redirect-uri "https://sso.example.test/login"})))) + rt/get-current-href (mock/stub (constantly dashboard-url)) + dom/append-query-param (mock/stub (fn [_url _key _value] callback-url)) + uuid/next (mock/stub (constantly "pending-id")) + ss/save-pending-action! (mock/stub (fn [& args] + (swap! saved-actions conj args))) + rt/nav-raw (fn [& args] {:nav-raw args}) + modal/show (mock/stub (fn [& args] {:modal-show args}))} + (await (async/observe + (ptk/watch (dtm/check-and-create-team team-id) + {:profile {:id profile-id}} + nil) + :on-next #(swap! emitted conj %))))) + + (t/is (= [[:get-teams nil] + [:check-nitrate-sso {:team-id team-id :url callback-url}]] + @requests)) + (t/is (= [["pending-id" {:type :create-team-in-organization + :team-id team-id}]] + @saved-actions)) + (t/is (= {:nav-raw [:uri "https://sso.example.test/login"]} + (last @emitted))) + (t/is (not-any? #(contains? % :modal-show) @emitted)))) + +(t/deftest ^:async check-and-create-team-shows-form-after-sso-is-authorized + (let [team-id "00000000-0000-4000-8000-000000000001" + organization-id "00000000-0000-4000-8000-000000000002" + profile-id "00000000-0000-4000-8000-000000000003" + dashboard-url "https://penpot.example/#/dashboard-recent?team-id=current-team" + callback-url (str dashboard-url "&pending-action-id=pending-id") + team {:id team-id + :organization {:id organization-id + :name "Product" + :owner-id profile-id}} + requests (atom []) + emitted (atom [])] + (await + (mock/with-mocks* + {rp/cmd! (mock/stub + (fn [command params] + (swap! requests conj [command params]) + (case command + :get-teams (rx/of [team]) + :check-nitrate-sso (rx/of {:authorized true})))) + rt/get-current-href (mock/stub (constantly dashboard-url)) + dom/append-query-param (mock/stub (fn [_url _key _value] callback-url)) + uuid/next (mock/stub (constantly "pending-id")) + modal/show (mock/stub (fn [& args] {:modal-show args}))} + (await (async/observe + (ptk/watch (dtm/check-and-create-team team-id) + {:profile {:id profile-id}} + nil) + :on-next #(swap! emitted conj %))))) + + (t/is (= [:check-nitrate-sso {:team-id team-id :url callback-url}] + (last @requests))) + (t/is (= {:modal-show [:team-form {:organization-id organization-id + :organization-name "Product"}]} + (last @emitted))) + (t/is (= 2 (count @emitted))))) + (t/deftest ^:async fetch-invitations-clears-the-list-on-permission-error ;; The pending invitations are an admin only list: a member that cannot ;; invite gets an insufficient permissions error. That must not become an @@ -75,7 +162,16 @@ (t/deftest owner-invitation-link-is-copyable-only-by-the-team-owner ;; An admin cannot create nor promote an owner invitation, so the backend ;; does not mint its token for them: the action is not offered either. - (t/is (true? (dtui/invitation-link-copyable? true :owner))) - (t/is (false? (dtui/invitation-link-copyable? false :owner))) - (t/is (true? (dtui/invitation-link-copyable? false :editor))) - (t/is (true? (dtui/invitation-link-copyable? false :viewer)))) + (t/is (true? (dtui/invitation-link-copyable? true true :owner))) + (t/is (false? (dtui/invitation-link-copyable? true false :owner))) + (t/is (true? (dtui/invitation-link-copyable? true false :editor))) + (t/is (true? (dtui/invitation-link-copyable? true false :viewer)))) + +(t/deftest invitation-link-is-not-copyable-without-the-invite-policy + ;; The link invites as much as sending the invitation does, so a profile + ;; that the organization invite policy leaves out cannot copy it, not even + ;; the team owner. + (t/is (false? (dtui/invitation-link-copyable? false true :owner))) + (t/is (false? (dtui/invitation-link-copyable? false true :editor))) + (t/is (false? (dtui/invitation-link-copyable? false false :editor))) + (t/is (false? (dtui/invitation-link-copyable? false false :viewer)))) diff --git a/frontend/test/frontend_tests/data/workspace_colors_test.cljs b/frontend/test/frontend_tests/data/workspace_colors_test.cljs index 5f5b25fc1e..5e8b8c6ce6 100644 --- a/frontend/test/frontend_tests/data/workspace_colors_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_colors_test.cljs @@ -6,8 +6,10 @@ (ns frontend-tests.data.workspace-colors-test (:require + [app.common.types.color :as cc] [app.common.uuid :as uuid] [app.main.data.workspace.colors :as dwc] + [app.main.ui.workspace.colorpicker.hsl :as hsl] [cljs.test :as t :include-macros true] [frontend-tests.helpers.async :as async] [potok.v2.core :as ptk])) @@ -84,6 +86,56 @@ event (dwc/update-colorpicker-color {} true)] (t/is (some? (ptk/watch event valid-state nil))))) +(t/deftest hsl-boundary-editing + (let [original (dwc/split-color-components {:color "#0d5af2" :opacity 1}) + expected (hsl/components original)] + (doseq [lightness [0 1]] + (let [boundary (hsl/update-color original :hsl-l lightness) + restored (hsl/update-color boundary :hsl-l 0.5)] + (t/is (= expected (hsl/components restored))) + (t/is (= "#0d5af2" (:hex restored))) + (t/is (every? js/Number.isFinite (map restored [:r :g :b]))))) + (let [gray (hsl/update-color original :hsl-s 0) + green (hsl/update-color gray :h 120) + saturated (hsl/update-color green :hsl-s 0.5)] + (t/is (= [120 0.5 0.5] (:hsl saturated)))))) + +(t/deftest hsl-draft-stays-in-colorpicker + (let [color (dwc/materialize-color-components + (hsl/update-color (dwc/split-color-components + {:color "#0d5af2" :opacity 1}) :hsl-l 0)) + state {:colorpicker {:type :color :current-color color}} + changed (ptk/update (dwc/update-colorpicker-color + {:hex "#ff0000" :r 255 :g 0 :b 0 :h 0 :s 1 :v 255} false) + state)] + (t/is (not (contains? (get-in changed [:colorpicker :current-color]) :hsl))) + (t/is (not (contains? (dwc/get-color-from-colorpicker-state (:colorpicker state)) :hsl))) + (let [gradient {:type :linear-gradient :current-color color + :editing-stop 0 :stops [(assoc color :offset 0)] + :gradient {} :opacity 1}] + (t/is (not (contains? (get-in (dwc/get-color-from-colorpicker-state gradient) + [:gradient :stops 0]) :hsl)))))) + +(t/deftest hsl-white-is-finite + (t/is (= [0 0 1] (cc/rgb->hsl [255 255 255])))) + +(t/deftest hsl-gradient-echo-preserves-draft + (doseq [[property value] [[:hsl-l 0] [:hsl-l 1] [:hsl-s 0]]] + (let [original (dwc/split-color-components {:color "#0d5af2" :opacity 1}) + color (assoc (dwc/materialize-color-components + (merge original (hsl/update-color original property value))) :offset 0) + state {:colorpicker {:type :linear-gradient :current-color color + :editing-stop 0 :stops [color] + :gradient {:type :linear :shape-id nil} :opacity 1}} + data (dwc/get-color-from-colorpicker-state (:colorpicker state)) + echoed (ptk/update (dwc/update-colorpicker data) state) + changed (ptk/update (dwc/update-colorpicker-color {:hex "#ff0000"} false) state)] + (t/is (= (:hsl color) (get-in echoed [:colorpicker :current-color :hsl]))) + (when (= property :hsl-s) + (t/is (= "#406bbf" (:hex (hsl/update-color + (get-in echoed [:colorpicker :current-color]) :hsl-s 0.5))))) + (t/is (not (contains? (get-in changed [:colorpicker :stops 0]) :hsl)))))) + (t/deftest ^:async color-change-updates-the-shapes-of-a-property-in-one-event ;; Changing a color in a selection of many shapes must not commit once per ;; shape: each commit makes the page objects a new map, and everything that diff --git a/frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs b/frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs new file mode 100644 index 0000000000..11f140f151 --- /dev/null +++ b/frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs @@ -0,0 +1,60 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.data.workspace-libraries-load-test + "Tests for a library of the open file that fails to load." + (:require + [app.common.uuid :as uuid] + [app.main.data.workspace.libraries :as dwl] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [potok.v2.core :as ptk])) + +(defn- error-of-type + [type] + (ex-info "http error" {:type type})) + +(t/deftest broken-library-error-only-for-backend-errors + ;; the backend failed to load the library file + (t/is (true? (dwl/broken-library-error? (error-of-type :server-error)))) + ;; transient or transport errors never disconnect a library + (doseq [type [:network :offline :bad-gateway :service-unavailable + :gateway-error :rate-limit nil]] + (t/is (false? (dwl/broken-library-error? (error-of-type type))) + (str "type " type))) + (t/is (false? (dwl/broken-library-error? (js/Error. "boom"))))) + +(defn- watch-events + "Synchronously collects the events the watch of `event` emits." + [event state] + (let [events (atom [])] + (->> (ptk/watch event state (rx/empty)) + (rx/subs! #(swap! events conj %))) + @events)) + +(t/deftest library-load-failed-disconnects-a-direct-library + (let [file-id (uuid/next) + library-id (uuid/next) + library {:id library-id :name "Broken" :is-indirect false} + events (watch-events (dwl/library-load-failed file-id library (error-of-type :server-error)) + {})] + ;; the library is unlinked from the file and the user is told + (t/is (= [::dwl/detach-library :app.main.data.notifications/show] + (mapv ptk/type events))))) + +(t/deftest library-load-failed-drops-an-indirect-library + (let [file-id (uuid/next) + library-id (uuid/next) + other-id (uuid/next) + library {:id library-id :name "Broken" :is-indirect true} + state {:files {library-id {:id library-id} + other-id {:id other-id}}} + events (watch-events (dwl/library-load-failed file-id library (error-of-type :server-error)) + state)] + ;; an indirect library is not linked to the file: only drop it from the state + (t/is (= 1 (count events))) + (t/is (= {other-id {:id other-id}} + (:files (ptk/update (first events) state)))))) diff --git a/frontend/test/frontend_tests/data/workspace_media_test.cljs b/frontend/test/frontend_tests/data/workspace_media_test.cljs index cabdd54abb..03e80547c3 100644 --- a/frontend/test/frontend_tests/data/workspace_media_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_media_test.cljs @@ -8,12 +8,24 @@ "Integration tests for the chunked-upload logic in app.main.data.workspace.media." (:require + [app.common.path-names :as cpn] + [app.common.test-helpers.files :as cthf] + [app.common.test-helpers.ids-map :as cthi] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.workspace.media :as media] + [app.main.store :as st] [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] - [frontend-tests.helpers.http :as http])) + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.http :as http] + [frontend-tests.helpers.mock :as mock] + [frontend-tests.helpers.state :as ths] + [potok.v2.core :as ptk] + [tubax.core :as tubax])) + +(t/use-fixtures :each + {:before cthi/reset-idmap!}) ;; --------------------------------------------------------------------------- ;; Local helpers @@ -187,3 +199,62 @@ (http/restore-fetch! orig) (t/is (= 3 @chunk-calls)) (done)))))))) + +;; --------------------------------------------------------------------------- +;; SVG component creation: any file extension must be stripped from both +;; the component name and the main instance root shape name. +;; --------------------------------------------------------------------------- + +(def ^:private svg-content + (str "" + "" + "")) + +;; The `sax` XML parser used by `tubax/xml->clj` is not available in the +;; ESM test build, so the parse step is mocked with an equivalent tree. +;; Everything else (name handling included) runs for real. +(def ^:private parsed-svg + {:tag :svg + :attrs {:xmlns "http://www.w3.org/2000/svg" + :width "100" :height "100" + :viewBox "0 0 100 100"} + :content [{:tag :rect + :attrs {:x "0" :y "0" :width "100" :height "100"} + :content []}]}) + +(t/deftest ^:async upload-svg-component-strips-svg-extension + (t/testing "creating a component from an uploaded SVG removes the extension from both names" + ;; Any extension must be stripped (not only ".svg"), including + ;; uppercase ones, extensions in the middle of the name, and + ;; filenames with no extension at all. + (doseq [[filename expected] + [["my-component.svg" "my-component"] + ["icon.SVG" "icon"] + ["my.svg.icon.svg" "my.svg.icon"] + ["design" "design"]]] + (let [file (cthf/sample-file :file1) + store (ths/setup-store file) + blob (js/File. #js [svg-content] + filename + #js {:type "image/svg+xml"})] + (await + (mock/with-mocks* + {st/state store + st/emit! (mock/stub (fn [& events] (apply ptk/emit! store events))) + tubax/xml->clj (mock/stub (fn [_source] parsed-svg))} + ;; The SVG is placed at the viewport center; without a vbox the + ;; geometry setup would receive nil coordinates. + (ptk/emit! store #(assoc % :workspace-local {:vbox {:x 0 :y 0 :width 800 :height 600}})) + (ptk/emit! store (media/upload-media-components {:file-id (:id file) + :blobs [blob]})) + (await (async/wait-for #(seq (get-in @store [:files (:id file) :data :components])) + "component created")) + (let [data (get-in @store [:files (:id file) :data]) + component (first (vals (:components data))) + root (get-in data [:pages-index (:main-instance-page component) + :objects (:main-instance-id component)])] + (t/is (some? component) (str "component not created for " filename)) + (t/is (some? root) (str "root not created for " filename)) + (t/is (= expected (:name component))) + (t/is (= expected (:name root))) + (t/is (= expected (cpn/merge-path-item (:path component) (:name component))))))))))) diff --git a/frontend/test/frontend_tests/data/workspace_thumbnails_wasm_test.cljs b/frontend/test/frontend_tests/data/workspace_thumbnails_wasm_test.cljs new file mode 100644 index 0000000000..ca29015e73 --- /dev/null +++ b/frontend/test/frontend_tests/data/workspace_thumbnails_wasm_test.cljs @@ -0,0 +1,74 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.data.workspace-thumbnails-wasm-test + "The main thread side of the component thumbnails the worker renders + and uploads: what it sends the worker, and how it shows the renders + that come back." + (:require + [app.common.test-helpers.compositions :as ctho] + [app.common.test-helpers.files :as cthf] + [app.common.test-helpers.ids-map :as cthi] + [app.main.data.workspace.thumbnails :as dwt] + [app.main.data.workspace.thumbnails-wasm :as dwt.wasm] + [app.main.worker :as mw] + [app.util.webapi :as wapi] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock] + [frontend-tests.helpers.state :as ths] + [potok.v2.core :as ptk])) + +(t/deftest ^:async the-worker-gets-the-shapes-of-the-main + (cthi/reset-idmap!) + (let [file (-> (cthf/sample-file :file) + (ctho/add-frame-with-child :root :child) + (ctho/add-frame :other)) + state @(ths/setup-store file) + page-id (cthf/current-page-id file) + sent (atom [])] + (await + (mock/with-mocks* + {mw/emit! (mock/stub (fn [message] (swap! sent conj message) (rx/empty)))} + (ptk/effect (dwt.wasm/render-thumbnail (:id file) page-id (cthi/id :root) :persist? true) + state nil))) + (t/is (= [{:cmd :thumbnails/render-component + :file-id (:id file) + :page-id page-id + :root-id (cthi/id :root) + :persist? true}] + (mapv #(dissoc % :objects) @sent))) + (t/is (= #{(cthi/id :root) (cthi/id :child)} + (set (keys (:objects (first @sent)))))))) + +(t/deftest ^:async a-missing-main-sends-nothing + (cthi/reset-idmap!) + (let [file (cthf/sample-file :file) + state @(ths/setup-store file) + sent (atom [])] + (await + (mock/with-mocks* + {mw/emit! (mock/stub (fn [message] (swap! sent conj message) (rx/empty)))} + (ptk/effect (dwt.wasm/render-thumbnail (:id file) (cthf/current-page-id file) nil) + state nil))) + (t/is (empty? @sent)))) + +(t/deftest ^:async worker-renders-are-shown + (let [renders (rx/subject) + events (atom [])] + (await + (mock/with-mocks* + {mw/ask-many! (mock/stub (fn [_] (rx/observe-on :async renders))) + wapi/create-uri (mock/stub (fn [_] "blob:test"))} + (let [done (async/observe (dwt.wasm/component-thumbnail-events) + :on-next #(swap! events conj %))] + (rx/push! renders {:object-id "a" :blob (wapi/create-blob "a" "image/png")}) + (rx/push! renders {:object-id "b" :blob (wapi/create-blob "b" "image/png")}) + (await (async/wait-for #(= 2 (count @events)) "both renders shown")) + (t/is (every? (ptk/type? ::dwt/assoc-thumbnail) @events)) + (rx/end! renders) + (await done)))))) diff --git a/frontend/test/frontend_tests/data/workspace_undo_test.cljs b/frontend/test/frontend_tests/data/workspace_undo_test.cljs index 1794b13a90..830f5c94e9 100644 --- a/frontend/test/frontend_tests/data/workspace_undo_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_undo_test.cljs @@ -267,8 +267,8 @@ (ptk/event ::dw/reload-current-file)) (await (async/settle)) (ptk/emit! store (rename-page (cthf/current-page-id file) "Renamed")) - (await (async/wait-for #(seq (get-in @store [:workspace-undo :items])) - "the commit reaches the undo history")) + (await (async/wait-for-store store #(seq (get-in @store [:workspace-undo :items])) + "the commit reaches the undo history")) (await (async/settle)) (t/is (= 1 (count (get-in @store [:workspace-undo :items])))) (finally diff --git a/frontend/test/frontend_tests/data/workspace_variants_test.cljs b/frontend/test/frontend_tests/data/workspace_variants_test.cljs index d596add6c2..b293ba2560 100644 --- a/frontend/test/frontend_tests/data/workspace_variants_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_variants_test.cljs @@ -35,6 +35,26 @@ (thv/add-variant-two-properties :v01 :c01 :m01 :c02 :m02) (cthf/add-sample-page :page2))) +(defn- nine-property-file-with-empty-middle + "Nine properties; Property 5 is empty on every variant." + [] + (let [names (mapv #(str "Property " %) (range 1 10)) + props1 (mapv (fn [n i] + {:name n :value (if (= i 4) "" (str "a" (inc i)))}) + names + (range)) + props2 (mapv (fn [n i] + {:name n :value (if (= i 4) "" (str "b" (inc i)))}) + names + (range))] + (-> (cthf/sample-file :file1 :page-label :page1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02) + (cthc/update-component :c01 {:variant-properties props1}) + (cthc/update-component :c02 {:variant-properties props2}) + (cths/update-shape :m01 :variant-name (ctv/properties-to-name props1)) + (cths/update-shape :m02 :variant-name (ctv/properties-to-name props2)) + (cthf/add-sample-page :page2)))) + (defn- run [file events done assert-fn] (ths/run-store @@ -124,6 +144,24 @@ (t/is (= "crítica" (-> props second :value))) (t/is (= (ctv/properties-to-name props) (variant-name file :m01))))))))) +(t/deftest remove-empty-properties-preserves-order-past-eight + (t/async + done + (let [file (nine-property-file-with-empty-middle) + variant-id (cthi/id :v01) + expected ["Property 1" "Property 2" "Property 3" "Property 4" + "Property 6" "Property 7" "Property 8" "Property 9"]] + (run file + [(dwv/remove-empty-properties variant-id)] + done + (fn [state] + (let [file (ths/get-file-from-state state) + names1 (mapv :name (:variant-properties (cthc/get-component file :c01))) + names2 (mapv :name (:variant-properties (cthc/get-component file :c02)))] + (t/is (= expected names1)) + (t/is (= expected names2)) + (t/is (not-any? #(= "Property 5" %) names1)))))))) + (t/deftest update-property-name-from-another-page (t/async done diff --git a/frontend/test/frontend_tests/data/workspace_versions_test.cljs b/frontend/test/frontend_tests/data/workspace_versions_test.cljs index 7753ba4164..0bac593c0b 100644 --- a/frontend/test/frontend_tests/data/workspace_versions_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_versions_test.cljs @@ -220,3 +220,74 @@ (t/is (nil? (get-in result [:workspace-versions :backup])))) (finally (rx/dispose! sub))))) + +;; The snapshot RPC commands require the `file-id` (they use it to prune the +;; partitions of `file_change`), so every workspace version event sends the +;; one of the file that is open. + +(defn- version-event-request + "Run a workspace version event with `file-id` open and `version-id` in the + history, and return the `[command params]` of the RPC request it sends." + [file-id version-id event] + (let [requests (atom []) + state {:current-file-id file-id + :workspace-versions {:data [{:id version-id + :created-at (ct/now) + :created-by "system"}]}}] + (mock/with-mocks* + {rp/cmd! (mock/stub (fn [cmd params] + (swap! requests conj [cmd params]) + (rx/of true)))} + (->> (ptk/watch event state (rx/empty)) + (rx/subs! (fn [_]))) + (first @requests)))) + +(t/deftest ^:async renaming-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + [cmd params] (await (version-event-request + file-id version-id + (versions/rename-version version-id "name")))] + (t/is (= :update-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))))) + +(t/deftest ^:async deleting-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + [cmd params] (await (version-event-request + file-id version-id + (versions/delete-version version-id)))] + (t/is (= :delete-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))))) + +(t/deftest ^:async locking-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + [cmd params] (await (version-event-request + file-id version-id + (versions/lock-version version-id)))] + (t/is (= :lock-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))))) + +(t/deftest ^:async unlocking-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + [cmd params] (await (version-event-request + file-id version-id + (versions/unlock-version version-id)))] + (t/is (= :unlock-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))))) + +(t/deftest ^:async pinning-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + [cmd params] (await (version-event-request + file-id version-id + (versions/pin-version version-id)))] + (t/is (= :update-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))))) diff --git a/frontend/test/frontend_tests/errors_governor_test.cljs b/frontend/test/frontend_tests/errors_governor_test.cljs index a51f775a43..035188c5ff 100644 --- a/frontend/test/frontend_tests/errors_governor_test.cljs +++ b/frontend/test/frontend_tests/errors_governor_test.cljs @@ -174,6 +174,18 @@ (t/is (= 1 (count events))) (t/is (= 1 (:occurrences (deref (first events))))))) +(t/deftest submit-report-carries-the-active-renderer + (let [submit (fn [state hint] + (with-redefs [st/state (atom state)] + (capture-reports! + #(errors/submit-report :event-name "handled-exception" + :report "report" + :hint hint + :cause (error-cause :hint hint)))))] + (t/is (= "wasm" (:renderer (deref (first (submit {:features #{"render-wasm/v1"}} "a")))))) + (t/is (= "svg" (:renderer (deref (first (submit {:features #{}} "b")))))) + (t/is (nil? (:renderer (deref (first (submit {} "c")))))))) + (t/deftest invalid-report-does-not-consume-a-reservation (let [cause (error-cause :type :network :hint "boom") events (capture-reports! diff --git a/frontend/test/frontend_tests/fonts_test.cljs b/frontend/test/frontend_tests/fonts_test.cljs index dad9adf043..2be018f0a4 100644 --- a/frontend/test/frontend_tests/fonts_test.cljs +++ b/frontend/test/frontend_tests/fonts_test.cljs @@ -281,28 +281,28 @@ (t/is (nil? (#'df/parse-font-mtype nil))) (t/is (nil? (#'df/parse-font-mtype (bytes->buffer [0 1]))))) -(t/deftest prepare-font-defaults-missing-subfamily +(t/deftest prepare-font-entry-defaults-missing-subfamily (let [font (fake-ot-font {:family "Satoshi" :variant nil :hhea #js {:ascender 1000 :descender 300} :os2 #js {:usWinAscent 1000 :usWinDescent 300 :sTypoAscender 800 :sTypoDescender 200 :fsSelection 0}}) - entry (#'df/prepare-font {:font font - :type "font/otf" - :name "Satoshi-Regular.otf" - :data (bytes->buffer [79 84 84 79 0 0 0 0])})] + entry (df/prepare-font-entry {:font font + :type "font/otf" + :name "Satoshi-Regular.otf" + :data (bytes->buffer [79 84 84 79 0 0 0 0])})] (t/is (= "Satoshi" (:font-family entry))) (t/is (= 400 (:font-weight entry))) (t/is (= "normal" (:font-style entry))) (t/is (false? (:height-warning? entry))))) -(t/deftest prepare-font-tolerates-missing-tables +(t/deftest prepare-font-entry-tolerates-missing-tables (let [font (fake-ot-font {:family "Satoshi" :variant "Regular" :hhea nil :os2 nil}) - entry (#'df/prepare-font {:font font - :type "font/ttf" - :name "Satoshi-Regular.ttf" - :data (bytes->buffer [0 1 0 0 0 0 0 0])})] + entry (df/prepare-font-entry {:font font + :type "font/ttf" + :name "Satoshi-Regular.ttf" + :data (bytes->buffer [0 1 0 0 0 0 0 0])})] (t/is (= "Satoshi" (:font-family entry))) (t/is (false? (:height-warning? entry))))) @@ -351,3 +351,25 @@ (t/is (= 700 (:font-weight entry))) (t/is (= "normal" (:font-style entry)))) (t/is (empty? @emitted)))) + +(t/deftest ^:async process-upload-keeps-bold-and-bolditalic-woff2 + ;; Regression for the review on #11951: the bold-italic file resolved to + ;; style "normal", collided with the bold-only file on the (family, + ;; weight, style) join key and silently overwrote it. + (let [team-id (uuid/next) + blobs [#js {:name "Inter-Bold.woff2"} + #js {:name "Inter-BoldItalic.woff2"}] + emitted (atom []) + result (atom nil)] + (await (mock/with-mocks* + {wa/read-file-as-array-buffer + (mock/stub (fn [_] (rx/of (bytes->buffer [119 79 70 50 0 0 0 0])))) + st/emit! (mock/stub (fn [& evts] (swap! emitted into evts) nil))} + (await (async/observe (df/process-upload blobs team-id) + :on-next #(reset! result %))) + (await (async/settle)))) + (t/is (= 2 (count (vals @result)))) + (t/is (every? #(= "Inter" (:font-family %)) (vals @result))) + (t/is (every? #(= 700 (:font-weight %)) (vals @result))) + (t/is (= #{"normal" "italic"} (set (map :font-style (vals @result))))) + (t/is (empty? @emitted)))) diff --git a/frontend/test/frontend_tests/helpers/async.cljs b/frontend/test/frontend_tests/helpers/async.cljs index 1e52a49c25..1af96d9b27 100644 --- a/frontend/test/frontend_tests/helpers/async.cljs +++ b/frontend/test/frontend_tests/helpers/async.cljs @@ -2,7 +2,7 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) KALEIDOS INC Sucursal en España SL +;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns frontend-tests.helpers.async "Async-first plumbing for ClojureScript tests. @@ -18,7 +18,8 @@ scheduled before it." (:require [beicon.v2.core :as rx] - [cljs.test :as t :include-macros true])) + [cljs.test :as t :include-macros true] + [potok.v2.core :as ptk])) (defn ->promise "Coerces a single-value observable into a js/Promise that resolves @@ -78,15 +79,111 @@ (when on-complete (on-complete)) (settle resolve nil)))))))) +(def ^:private default-timeout-ms + "Ceiling of a wait, in real milliseconds. With a signal it only bounds + the failure: a wait that resolves on an event never reaches it. Without + a signal it is the poll budget." + 5000) + +(defn- ^:async poll + [pred msg expired?] + (cond + (pred) nil + @expired? (t/is false (str "Timed out waiting for: " msg)) + :else (do (await (settle)) + (await (poll pred msg expired?))))) + +(defn- ^:async wait-poll + [pred msg timeout-ms] + (let [expired? (atom false) + timer (js/setTimeout #(reset! expired? true) timeout-ms)] + (await (poll pred msg expired?)) + (js/clearTimeout timer))) + +(defn- cause-text + "Readable, bounded text for a signal error: its message when it has one, + otherwise the printed value inside a `try` (a cyclic value would not print), + and never more than 500 characters." + [cause] + (let [text (or (ex-message cause) + (try + (pr-str cause) + (catch :default _ "#")))] + (if (> (count text) 500) + (str (subs text 0 500) "…") + text))) + +(defn- wait-signal + "Resolves when `pred` holds, re-checking it on every emission of `signal`. + + The timeout only bounds failure: it records the failure and resolves, so + the caller never hangs. A signal that ends or fails without the predicate + holding is reported with its own reason. The subscription is disposed on + every path." + [pred msg signal timeout-ms] + (js/Promise. + (fn [resolve] + (let [done? (atom false) + sub* (atom nil) + timer* (atom nil) + stop (fn [] + (when-let [s @sub*] (rx/dispose! s)) + (when-let [t @timer*] (js/clearTimeout t))) + fail (fn [text] + (when-not @done? + (reset! done? true) + (stop) + (t/is false text) + (resolve nil))) + finish (fn [] + (when-not @done? + (reset! done? true) + (stop) + (resolve nil)))] + (reset! timer* + (js/setTimeout #(fail (str "Timed out waiting for: " msg)) + timeout-ms)) + (reset! sub* + (rx/subs! + (fn [_] (when (pred) (finish))) + (fn [cause] + (if (pred) + (finish) + (fail (str "Signal failed while waiting for: " msg ": " + (cause-text cause))))) + (fn [] + (if (pred) + (finish) + (fail (str "Signal ended while waiting for: " msg)))) + signal)) + ;; A source that emits while subscribing already settled the wait. + (when @done? + (rx/dispose! @sub*)) + nil)))) + (defn ^:async wait-for - "Resolves once `pred` holds, checking immediately and then once per - macrotask (bounded: records a failure instead of hanging). Await it - before asserting effects of triggers — it holds whether the effect - lands synchronously or not." - [pred msg & [{:keys [max-ticks] :or {max-ticks 50}}]] - (if (pred) - nil - (if (<= max-ticks 0) - (t/is false (str "Timed out waiting for: " msg)) - (do (await (settle)) - (await (wait-for pred msg {:max-ticks (dec max-ticks)})))))) + "Resolves once `pred` holds, checking immediately and then re-checking it + on every emission of `:signal` (or once per macrotask when no signal is + given), up to `:timeout-ms` of real time (default 5000). + + With a signal the ceiling bounds failure only: the wait resolves on the + event that changes the state, not after a number of event-loop turns. + Without one it is a poll budget, and turns are not a unit of time (50 + turns are ~54 ms idle and much less ground under load). Records a single + failure instead of hanging: `Timed out waiting for: ` when the ceiling + expires, or the signal's own reason when it ends or fails (see + `wait-signal`)." + [pred msg & [{:keys [signal timeout-ms] + :or {timeout-ms default-timeout-ms}}]] + (when-not (pred) + (if (some? signal) + (await (wait-signal pred msg signal timeout-ms)) + (await (wait-poll pred msg timeout-ms))))) + +(defn wait-for-store + "Resolves once `pred` holds over `store`, re-checking it on every store + event. The store notifies its subscribers after applying the update, so the + predicate already sees the new state. One signal only: a predicate that also + reads another source passes a merged signal to `wait-for` itself." + [store pred msg] + (wait-for pred msg {:signal (ptk/input-stream store)})) diff --git a/frontend/test/frontend_tests/helpers/async_test.cljs b/frontend/test/frontend_tests/helpers/async_test.cljs new file mode 100644 index 0000000000..869fffdd58 --- /dev/null +++ b/frontend/test/frontend_tests/helpers/async_test.cljs @@ -0,0 +1,178 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.helpers.async-test + "Contract of the wait primitives: a wait with a signal resolves on the + event that changes the state, not on an event-loop budget, and the + ceiling only bounds the failure." + (:require + [app.common.time :as ct] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock] + [potok.v2.core :as ptk])) + +;; The old budget: 50 event-loop turns. A delivery after it used to be lost. +(def ^:private delivery-after-ticks 100) + +(def ^:private reported + "Messages of the failures recorded in a throwaway test environment, so the + failure reasons can be asserted without failing the run." + (atom [])) + +(defmethod t/report [::silent :fail] + ;; The never-true case asserts that the helper records a failure: count it + ;; in the throwaway environment and keep the message, but do not print it. + [m] + (swap! reported conj (:message m)) + (t/inc-report-counter! :fail)) + +(defn- deliver-after! + "Calls `f` once the event loop has turned `n` times, one per macrotask." + [n f] + (if (pos? n) + (js/setTimeout #(deliver-after! (dec n) f) 0) + (f))) + +(defn- turn-probe + "Schedules one macrotask and returns `:ran?`, a reader for whether it + already ran, and `:stop`, which cancels it. A wait that polls always + consumes at least one macrotask, so a resolved probe still false proves the + wait did not poll." + [] + (let [ran (atom false) + id (js/setTimeout #(reset! ran true) 0)] + {:ran? (fn [] @ran) + :stop (fn [] (js/clearTimeout id))})) + +(defn- inc-counter + [] + (ptk/reify ::inc-counter + ptk/UpdateEvent + (update [_ state] + (update state :counter inc)))) + +(t/deftest ^:async a-signal-resolves-a-late-delivery + ;; The delivery lands after the old 50-turn budget. Without a signal the + ;; wait used to give up first; with one it resolves when the event arrives. + ;; Counting the predicate evaluations separates the two paths: a poll + ;; evaluates it once per turn, a signal exactly twice. + (let [ready (atom false) + evals (atom 0) + sig (rx/subject)] + (deliver-after! delivery-after-ticks + #(do (reset! ready true) + (rx/push! sig :ready))) + (await (async/wait-for #(do (swap! evals inc) (true? @ready)) + "the late delivery" + {:signal sig})) + (t/is (= 2 @evals) "one initial check and one signal emission") + (t/is (true? @ready)))) + +(t/deftest ^:async a-store-signal-resolves-after-the-update + ;; The store notifies its subscribers after applying the update, so the + ;; predicate is already true when the wait re-checks it. + (let [store (ptk/store {:state {:counter 0}}) + {:keys [ran? stop]} (turn-probe)] + (try + (let [done (async/wait-for-store store + #(= 1 (:counter @store)) + "the update")] + (ptk/emit! store (inc-counter)) + (await done) + (t/is (= 1 (:counter @store))) + (t/is (false? (ran?)) "resolved on the event, not on a poll")) + (finally + (stop) + (rx/dispose! store))))) + +(t/deftest ^:async a-late-predicate-without-signal + ;; Same late delivery, with nothing to listen to: the ceiling has to be + ;; time, not turns, for the wait to survive it. + (let [ready (atom false)] + (deliver-after! delivery-after-ticks #(reset! ready true)) + (await (async/wait-for #(true? @ready) "the late delivery")) + (t/is (true? @ready)))) + +(t/deftest ^:async a-never-true-predicate-records-one-failure + (let [failures (atom nil) + started (inst-ms (ct/now))] + (binding [t/*current-env* (t/empty-env ::silent)] + (await (async/wait-for (constantly false) "the impossible" {:timeout-ms 100})) + (reset! failures (get-in (t/get-current-env) [:report-counters :fail]))) + (t/is (= 1 @failures) "one failure, not one per turn") + ;; A safety net at 10x the 100 ms ceiling, so a hung wait fails instead of + ;; stalling the run; it does not measure precision. + (t/is (< (- (inst-ms (ct/now)) started) 1000) "bounded by the ceiling"))) + +(t/deftest ^:async a-frozen-clock-still-times-out-in-real-time + ;; The watchdog fixtures freeze `ct/now`: the ceiling must be real time, + ;; so a never-true wait still fails instead of hanging on a frozen clock. + (let [failures (atom nil) + started (js/Date.now)] + (binding [t/*current-env* (t/empty-env ::silent)] + (await (mock/with-mocks* + {ct/now (mock/stub (fn [] (ct/inst 0)))} + (await (async/wait-for (constantly false) + "the impossible on a frozen clock" + {:timeout-ms 100})))) + (reset! failures (get-in (t/get-current-env) [:report-counters :fail]))) + (t/is (= 1 @failures) "one failure, not one per turn") + (t/is (< (- (js/Date.now) started) 1000) "bounded by the ceiling"))) + +(t/deftest ^:async an-immediate-predicate-does-not-yield + (let [{:keys [ran? stop]} (turn-probe)] + (try + (await (async/wait-for (constantly true) "the immediate")) + (t/is (false? (ran?)) "an already-true predicate does not wait") + (finally + (stop))))) + +(t/deftest ^:async a-resolved-wait-disposes-its-signal + (let [evals (atom 0) + sig (rx/subject) + pred #(do (swap! evals inc) (>= @evals 2)) + done (async/wait-for pred "the second check" {:signal sig})] + (rx/push! sig :first) + (await done) + (let [before @evals] + (rx/push! sig :second) + (t/is (= before @evals) "a resolved wait stops listening")))) + +(t/deftest ^:async a-signal-that-ends-records-why + (reset! reported []) + (let [sig (rx/subject)] + (binding [t/*current-env* (t/empty-env ::silent)] + (let [done (async/wait-for (constantly false) "the closed signal" {:signal sig})] + (rx/end! sig) + (await done)))) + (t/is (= ["Signal ended while waiting for: the closed signal"] @reported) + "an ended signal is not reported as a timeout")) + +(t/deftest ^:async a-signal-that-errors-records-why + (reset! reported []) + (let [sig (rx/subject)] + (binding [t/*current-env* (t/empty-env ::silent)] + (let [done (async/wait-for (constantly false) "the broken signal" {:signal sig})] + (rx/error! sig (ex-info "boom" {:where :test})) + (await done)))) + (t/is (some #(str/includes? % "Signal failed while waiting for: the broken signal") + @reported) + "an errored signal is not reported as a timeout")) + +(t/deftest ^:async a-signal-error-bounds-a-long-cause + (reset! reported []) + (let [sig (rx/subject) + cause (apply str (repeat 600 "x"))] + (binding [t/*current-env* (t/empty-env ::silent)] + (let [done (async/wait-for (constantly false) "the long cause" {:signal sig})] + (rx/error! sig cause) + (await done)))) + (let [text (first @reported)] + (t/is (str/includes? text "Signal failed while waiting for: the long cause")) + (t/is (<= (count text) 600) "the cause text is bounded"))) diff --git a/frontend/test/frontend_tests/helpers/persistence.cljs b/frontend/test/frontend_tests/helpers/persistence.cljs index 70e3db744a..f7f4da7fbf 100644 --- a/frontend/test/frontend_tests/helpers/persistence.cljs +++ b/frontend/test/frontend_tests/helpers/persistence.cljs @@ -50,18 +50,21 @@ (`observe-on :async`): await each effect — via `wait-for` — before asserting it." [f & [respond]] - (let [file-id (uuid/next) - response (rx/subject) - failures (atom []) - requests (atom []) - tasks (atom #{}) - store (ptk/store {:state {:current-file-id file-id - :permissions {:can-edit true} - :files {file-id {:id file-id :revn 0}}} - :on-error #(t/is false (str %))})] + (let [file-id (uuid/next) + response (rx/subject) + failures (atom []) + requests (atom []) + requests$ (rx/subject) + tasks (atom #{}) + store (ptk/store {:state {:current-file-id file-id + :permissions {:can-edit true} + :files {file-id {:id file-id :revn 0}}} + :on-error #(t/is false (str %))})] (mock/with-mocks* {rp/cmd! (mock/stub (fn [cmd params] - (swap! requests conj [cmd params]) + (let [entry [cmd params]] + (swap! requests conj entry) + (rx/push! requests$ entry)) (if respond (respond cmd params) (->> response (rx/take 1) (rx/observe-on :async))))) @@ -73,9 +76,11 @@ (try (ptk/emit! store (dps/initialize-persistence)) (await (f {:file-id file-id :response response :failures failures - :requests requests :store store :tasks tasks})) + :requests requests :requests$ requests$ :store store + :tasks tasks})) (finally (rx/dispose! store) + (rx/end! requests$) (rx/end! response)))))) (defn set-attr diff --git a/frontend/test/frontend_tests/logic/comp_watcher_test.cljs b/frontend/test/frontend_tests/logic/comp_watcher_test.cljs index b9767480ba..51696f2397 100644 --- a/frontend/test/frontend_tests/logic/comp_watcher_test.cljs +++ b/frontend/test/frontend_tests/logic/comp_watcher_test.cljs @@ -87,8 +87,8 @@ #(assoc % :fills [{:fill-color "#ff0000" :fill-opacity 1}]))) - (await (async/wait-for #(empty? (copy-children store)) - "the copy to lose its rect")) + (await (async/wait-for-store store #(empty? (copy-children store)) + "the copy to lose its rect")) (let [file' (ths/get-file-from-state @store)] (t/is (= (cthi/id :outside) diff --git a/frontend/test/frontend_tests/logic/components_and_tokens.cljs b/frontend/test/frontend_tests/logic/components_and_tokens.cljs index dee8c555e1..2e77189666 100644 --- a/frontend/test/frontend_tests/logic/components_and_tokens.cljs +++ b/frontend/test/frontend_tests/logic/components_and_tokens.cljs @@ -9,6 +9,7 @@ [app.common.math :as mth] [app.common.test-helpers.components :as cthc] [app.common.test-helpers.compositions :as ctho] + [app.common.test-helpers.files :as cthf] [app.common.test-helpers.ids-map :as cthi] [app.common.test-helpers.shapes :as cths] [app.common.test-helpers.tokens :as ctht] @@ -36,6 +37,7 @@ (defn- setup-base-file [] (-> (ctht/sample-file-with-tokens + :page-label :page1 :lib-fn #(-> % (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :test-token-set) :name "test-token-set")) @@ -439,3 +441,141 @@ (tohs/run-store-async store step2 events identity)))) + +(defn- setup-file-with-stale-values + "Adds :frame2 (with a :rect1 child) and :frame3 in :page1, and :frame4 + in :page2, whose stored radius differs from the resolved token value." + [] + (-> (setup-base-file) + (ctho/add-frame-with-child :frame2 :rect1) + (ctho/add-frame :frame3) + (ctht/apply-token-to-shape :frame2 "test-token-2" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10) + (ctht/apply-token-to-shape :rect1 "test-token-3" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10) + (ctht/apply-token-to-shape :frame3 "test-token-2" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10) + (cthf/add-sample-page :page2) + (ctho/add-frame :frame4) + (ctht/apply-token-to-shape :frame4 "test-token-2" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10) + (cthf/switch-to-page :page1))) + +(t/deftest scoped-propagation-updates-only-the-scoped-tree + (t/async + done + (let [;; ==== Setup + file (setup-file-with-stale-values) + store (ths/setup-store file) + + ;; ==== Action + events [(dwtp/propagate-workspace-tokens + {:page-id (cthf/current-page-id file) + :root-ids [(cthi/id :frame2)]})]] + + (tohs/run-store-async + store done events + (fn [new-state] + (let [;; ==== Get + file' (ths/get-file-from-state new-state) + frame2' (cths/get-shape file' :frame2) + rect1' (cths/get-shape file' :rect1) + frame3' (cths/get-shape file' :frame3) + frame4' (cths/get-shape file' :frame4 :page-label :page2)] + + ;; ==== Check + (t/is (= (:r1 frame2') 50)) + (t/is (= (:r1 rect1') 75)) + (t/is (= (:r1 frame3') 10)) + (t/is (= (:r1 frame4') 10)))))))) + +(t/deftest unscoped-propagation-updates-every-shape + (t/async + done + (let [;; ==== Setup + file (setup-file-with-stale-values) + store (ths/setup-store file) + + ;; ==== Action + events [(dwtp/propagate-workspace-tokens)]] + + (tohs/run-store-async + store done events + (fn [new-state] + (let [;; ==== Get + file' (ths/get-file-from-state new-state) + frame2' (cths/get-shape file' :frame2) + rect1' (cths/get-shape file' :rect1) + frame3' (cths/get-shape file' :frame3) + frame4' (cths/get-shape file' :frame4 :page-label :page2)] + + ;; ==== Check + (t/is (= (:r1 frame2') 50)) + (t/is (= (:r1 rect1') 75)) + (t/is (= (:r1 frame3') 50)) + (t/is (= (:r1 frame4') 50)))))))) + +(defn- setup-file-with-stale-components + "Adds :component1 (from :frame1) and :component2 (from :frame2), and a + plain :frame3, all of them with a stored radius that differs from the + resolved token value." + [] + (-> (setup-base-file) + (ctht/apply-token-to-shape :frame1 "test-token-1" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10) + (cthc/make-component :component1 :frame1) + (ctho/add-frame :frame2) + (ctht/apply-token-to-shape :frame2 "test-token-2" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10) + (cthc/make-component :component2 :frame2) + (ctho/add-frame :frame3) + (ctht/apply-token-to-shape :frame3 "test-token-3" [:r1 :r2 :r3 :r4] [:r1 :r2 :r3 :r4] 10))) + +(t/deftest instantiate-component-applies-token-values + (t/async + done + (let [;; ==== Setup + file (setup-file-with-stale-components) + store (ths/setup-store file) + + ;; ==== Action + events [(dwl/instantiate-component (:id file) + (cthi/id :component1) + (geom/point 0 0))]] + + (tohs/run-store-async + store done events + (fn [new-state] + (let [;; ==== Get + copy' (dsh/lookup-shape new-state (first (dsh/lookup-selected new-state))) + file' (ths/get-file-from-state new-state) + frame3' (cths/get-shape file' :frame3)] + + ;; ==== Check + (t/is (= (get-in copy' [:applied-tokens :r1]) "test-token-1")) + (t/is (= (:r1 copy') 25)) + (t/is (= (:r4 copy') 25)) + (t/is (= (:r1 frame3') 10)))))))) + +(t/deftest component-swap-applies-token-values + (t/async + done + (let [;; ==== Setup + file (-> (setup-file-with-stale-components) + (cthc/instantiate-component :component1 :c-frame1)) + store (ths/setup-store file) + + ;; ==== Action + events [(dwl/component-swap (cths/get-shape file :c-frame1) + (:id file) + (cthi/id :component2) + true)]] + + (tohs/run-store-async + store done events + (fn [new-state] + (let [;; ==== Get + copy' (dsh/lookup-shape new-state (first (dsh/lookup-selected new-state))) + file' (ths/get-file-from-state new-state) + frame3' (cths/get-shape file' :frame3)] + + ;; ==== Check + (t/is (= (:component-id copy') (cthi/id :component2))) + (t/is (= (get-in copy' [:applied-tokens :r1]) "test-token-2")) + (t/is (= (:r1 copy') 50)) + (t/is (= (:r4 copy') 50)) + (t/is (= (:r1 frame3') 10)))))))) diff --git a/frontend/test/frontend_tests/logic/copying_and_duplicating_test.cljs b/frontend/test/frontend_tests/logic/copying_and_duplicating_test.cljs index 7939b903df..b096a96417 100644 --- a/frontend/test/frontend_tests/logic/copying_and_duplicating_test.cljs +++ b/frontend/test/frontend_tests/logic/copying_and_duplicating_test.cljs @@ -12,16 +12,19 @@ [app.common.test-helpers.ids-map :as cthi] [app.common.test-helpers.shapes :as cths] [app.common.types.component :as ctk] + [app.common.types.shape.layout :as ctl] [app.common.uuid :as uuid] [app.main.data.workspace :as dw] [app.main.data.workspace.colors :as dc] [app.main.data.workspace.libraries :as dwl] [app.main.data.workspace.pages :as dwp] [app.main.data.workspace.selection :as dws] + [app.main.data.workspace.shape-layout :as dwsl] [cljs.test :as t :include-macros true] [frontend-tests.helpers.mock :as mock] [frontend-tests.helpers.pages :as thp] - [frontend-tests.helpers.state :as ths])) + [frontend-tests.helpers.state :as ths] + [potok.v2.core :as ptk])) (t/use-fixtures :each {:before thp/reset-idmap!}) @@ -653,3 +656,160 @@ "Pasted text shapes have nil position-data")))))) done))) + +;; Grid tracks in component copies + +(defn- add-grid-with-child + "Adds a grid board with one rect child and assigns the child to a cell." + [file grid-label rect-label & {:keys [parent-label params]}] + (let [file (-> file + (ctho/add-frame grid-label + (merge {:name "grid" + :parent-label parent-label + :layout :grid + :layout-grid-dir :row + :layout-grid-columns [] + :layout-grid-rows [] + :layout-grid-cells {}} + params)) + (cths/add-sample-shape rect-label :parent-label grid-label)) + grid (->> (cths/get-shape file grid-label) + (#(ctl/assign-cells % (:objects (cthf/current-page file)))))] + (reduce (fn [file attr] (cths/update-shape file grid-label attr (get grid attr))) + file + [:layout-grid-columns :layout-grid-rows :layout-grid-cells]))) + +(defn- grid-root-file + "Main component whose root board is the grid, and a copy of it." + [] + (-> (cthf/sample-file :file1 :page-label :page-1) + (add-grid-with-child :grid-main :rect-main) + (cthc/make-component :comp :grid-main) + (cthc/instantiate-component :comp :grid-copy :children-labels [:rect-copy]))) + +(defn- grid-nested-file + "Main component with a wrapper board that contains the grid, and a copy." + [] + (-> (cthf/sample-file :file1 :page-label :page-1) + (ctho/add-frame :wrapper-main :name "wrapper") + (add-grid-with-child :grid-main :rect-main :parent-label :wrapper-main) + (cthc/make-component :comp :wrapper-main) + (cthc/instantiate-component :comp :wrapper-copy :children-labels [:grid-copy :rect-copy]))) + +(defn- grid-user-added-in-copy-file + "Copy of a wrapper board, with a plain grid (no :shape-ref) added inside + the copy. Only its parent is a copy." + [] + (let [file (-> (grid-nested-file) + (add-grid-with-child :grid-user :rect-user :parent-label :wrapper-copy))] + (assert (nil? (:shape-ref (cths/get-shape file :grid-user)))) + file)) + +(defn- grid-variant-file + "Variant main whose root board is the grid. The root has :variant-id and + no :shape-ref." + [] + (let [file (-> (cthf/sample-file :file1 :page-label :page-1) + (cths/add-sample-shape :variant :type :frame :is-variant-container true)) + variant-id (cthi/id :variant)] + (-> file + (add-grid-with-child :grid-main :rect-main + :parent-label :variant + :params {:variant-id variant-id :variant-name "Value1"}) + (cthc/make-component :comp :grid-main) + (cthc/update-component :comp {:variant-id variant-id + :variant-properties [{:name "Property 1" :value "Value1"}]})))) + +(defn- copied-tracks-event + [grid-id] + (ptk/reify ::set-copied-tracks + ptk/UpdateEvent + (update [_ state] + (assoc-in state [:workspace-grid-edition grid-id :copied-tracks] + {:track-indices [0] :type :column :grid-id grid-id})))) + +(defn- run-grid-track-events + [file make-events check-fn] + (t/async done + (mock/with-mocks {uuid/next cthi/next-uuid} + (fn [done'] + (let [store (ths/setup-store file) + events (make-events file)] + (ths/run-store + store done' events + (fn [new-state] + (check-fn (cthf/current-page file) + (ths/get-file-from-state new-state)))))) + done))) + +(defn- duplicate-events + [grid-label] + (fn [file] + [(dwsl/duplicate-layout-track [(:id (cths/get-shape file grid-label))] :column 0)])) + +(defn- paste-events + [grid-label] + (fn [file] + (let [grid-id (:id (cths/get-shape file grid-label))] + [(copied-tracks-event grid-id) + (dwsl/paste-grid-tracks grid-id)]))) + +(defn- objects-of + [file] + (:objects (cthf/current-page file))) + +(defn- column-count + [file label] + (count (:layout-grid-columns (cths/get-shape file label)))) + +(defn- check-blocked + [file make-events grid-label] + (run-grid-track-events + file make-events + (fn [page file'] + (t/is (= (:objects page) (objects-of file'))) + (t/is (= (column-count file grid-label) (column-count file' grid-label)))))) + +(defn- check-works + [file make-events grid-label] + (run-grid-track-events + file make-events + (fn [page file'] + (t/is (> (count (objects-of file')) (count (:objects page)))) + (t/is (> (column-count file' grid-label) (column-count file grid-label)))))) + +(t/deftest duplicate-track-blocked-in-copy-root-grid + (check-blocked (grid-root-file) (duplicate-events :grid-copy) :grid-copy)) + +(t/deftest paste-tracks-blocked-in-copy-root-grid + (check-blocked (grid-root-file) (paste-events :grid-copy) :grid-copy)) + +(t/deftest duplicate-track-blocked-in-nested-copy-grid + (check-blocked (grid-nested-file) (duplicate-events :grid-copy) :grid-copy)) + +(t/deftest paste-tracks-blocked-in-nested-copy-grid + (check-blocked (grid-nested-file) (paste-events :grid-copy) :grid-copy)) + +(t/deftest duplicate-track-blocked-in-user-added-grid-inside-copy + (check-blocked (grid-user-added-in-copy-file) (duplicate-events :grid-user) :grid-user)) + +(t/deftest paste-tracks-blocked-in-user-added-grid-inside-copy + (check-blocked (grid-user-added-in-copy-file) (paste-events :grid-user) :grid-user)) + +(t/deftest duplicate-track-works-in-main-grid + (check-works (grid-root-file) (duplicate-events :grid-main) :grid-main)) + +(t/deftest paste-tracks-works-in-main-grid + (check-works (grid-root-file) (paste-events :grid-main) :grid-main)) + +(t/deftest duplicate-track-works-in-main-nested-grid + (check-works (grid-nested-file) (duplicate-events :grid-main) :grid-main)) + +(t/deftest paste-tracks-works-in-main-nested-grid + (check-works (grid-nested-file) (paste-events :grid-main) :grid-main)) + +(t/deftest duplicate-track-works-in-variant-main-grid + (check-works (grid-variant-file) (duplicate-events :grid-main) :grid-main)) + +(t/deftest paste-tracks-works-in-variant-main-grid + (check-works (grid-variant-file) (paste-events :grid-main) :grid-main)) diff --git a/frontend/test/frontend_tests/logic/drag_on_save_failure_test.cljs b/frontend/test/frontend_tests/logic/drag_on_save_failure_test.cljs index 7ccf1c9d9c..fb336a2a9e 100644 --- a/frontend/test/frontend_tests/logic/drag_on_save_failure_test.cljs +++ b/frontend/test/frontend_tests/logic/drag_on_save_failure_test.cljs @@ -64,7 +64,7 @@ #(assoc-in % [:workspace-local :selected] (d/ordered-set (:id rect))) (start rect)) (rx/push! ms/mouse-position (gpt/point 160 260)) - (await (async/wait-for #(seq (:workspace-modifiers @store)) "the drag shows a preview")) + (await (async/wait-for-store store #(seq (:workspace-modifiers @store)) "the drag shows a preview")) (ptk/emit! store stop) (await (async/settle)) (finally @@ -147,8 +147,8 @@ (await (move store)) (ptk/emit! store stop) ;; A box drawing ends 100 ms after its stop - (await (async/wait-for #(nil? (get-in @store [:workspace-drawing :object])) - "the drawing ends" {:max-ticks 500})) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-drawing :object])) + "the drawing ends")) (await (async/settle)) (finally (rx/push! ms/mouse-position nil) @@ -158,21 +158,21 @@ (defn- ^:async drag-box [store] (rx/push! ms/mouse-position (gpt/point 160 180)) - (await (async/wait-for #(< 50 (get-in @store [:workspace-drawing :object :width] 0)) - "the box follows the pointer"))) + (await (async/wait-for-store store #(< 50 (get-in @store [:workspace-drawing :object :width] 0)) + "the box follows the pointer"))) (defn- ^:async drag-curve [store] (doseq [x [110 130 150 170]] (rx/push! ms/mouse-position (gpt/point x (+ x 20))) - (await (async/wait-for #(some (fn [p] (= x (:x p))) - (get-in @store [:workspace-drawing :object ::dwdc/points])) - "the curve takes the point")))) + (await (async/wait-for-store store #(some (fn [p] (= x (:x p))) + (get-in @store [:workspace-drawing :object ::dwdc/points])) + "the curve takes the point")))) (defn- ^:async click-path "Stands for the pen clicks: the drawn path gets two nodes." [store] - (await (async/wait-for #(some? (get-in @store [:workspace-drawing :object])) "the pen starts")) + (await (async/wait-for-store store #(some? (get-in @store [:workspace-drawing :object])) "the pen starts")) (ptk/emit! store #(assoc-in % [:workspace-drawing :object :content] (path/content [{:command :move-to :params {:x 100 :y 100}} {:command :line-to :params {:x 180 :y 160}}])))) diff --git a/frontend/test/frontend_tests/logic/scale_selected_test.cljs b/frontend/test/frontend_tests/logic/scale_selected_test.cljs new file mode 100644 index 0000000000..4c05636ece --- /dev/null +++ b/frontend/test/frontend_tests/logic/scale_selected_test.cljs @@ -0,0 +1,212 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.logic.scale-selected-test + "Tests for the scale-selected event used by the scale sidebar panel." + (:require + [app.common.data :as d] + [app.common.geom.point :as gpt] + [app.common.geom.rect :as grc] + [app.common.geom.shapes :as gsh] + [app.common.test-helpers.compositions :as ctho] + [app.common.test-helpers.files :as cthf] + [app.common.test-helpers.ids-map :as cthi] + [app.common.test-helpers.shapes :as cths] + [app.common.types.modifiers :as ctm] + [app.common.types.text :as txt] + [app.main.data.workspace.selection :as dws] + [app.main.data.workspace.transforms :as dwt] + [app.main.ui.workspace.sidebar.scale :as scale] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.state :as ths] + [frontend-tests.helpers.wasm :as thw])) + +(def ^:private parse-factor + @#'scale/parse-factor) + +(def ^:private scale-input-operation + @#'scale/scale-input-operation) + +(def ^:private apply-preview-modifiers + @#'scale/apply-preview-modifiers) + +(t/use-fixtures :each + {:before (fn [] + (cthi/reset-idmap!) + (thw/setup-wasm-mocks!)) + :after thw/teardown-wasm-mocks!}) + +(defn- shape-rect + [file label] + (-> (cths/get-shape file label) + :points + grc/points->rect)) + +(defn- assert-rect + [rect x y width height] + (t/is (= x (:x rect))) + (t/is (= y (:y rect))) + (t/is (= width (:width rect))) + (t/is (= height (:height rect)))) + +(t/deftest scale-sidebar-factor-starts-at-one-for-a-new-selection + (let [selection #{:shape} + state (scale/scale-state-for-selection selection)] + (t/is (= 1 (scale/selection-scale-factor state selection))))) + +(t/deftest scale-sidebar-factor-resets-after-deselecting-and-reselecting + (let [selection #{:shape} + scaled {:selection selection :factor 1.5} + deselected (scale/scale-state-for-selection #{}) + reselected (scale/scale-state-for-selection selection)] + (t/is (= 1 (scale/selection-scale-factor deselected #{}))) + (t/is (= 1 (scale/selection-scale-factor reselected selection))) + (t/is (= 1 (scale/selection-scale-factor scaled #{}))))) + +(t/deftest scale-sidebar-factor-is-independent-of-canvas-resize + (let [selection #{:shape} + state {:selection selection :factor 1.5}] + (t/is (= 1.5 (scale/selection-scale-factor state selection))))) + +(t/deftest scale-sidebar-factor-updates-after-dimension-input + (t/is (= 1.5 (scale/calculate-scale-after-dimension-change 1 100 150)))) + +(t/deftest scale-sidebar-size-uses-live-resize-modifiers + (let [file (-> (cthf/sample-file :file1) + (ctho/add-rect :rect1 :x 0 :y 0 :width 100 :height 50)) + shape (cths/get-shape file :rect1) + resize (ctm/resize-modifiers (gpt/point 2 2) (gpt/point 0 0)) + modifiers {(:id shape) {:modifiers resize}} + wasm-mtx (ctm/modifiers->transform resize) + normal (apply-preview-modifiers [shape] modifiers {} false) + wasm (apply-preview-modifiers [shape] {} {(:id shape) wasm-mtx} true)] + (t/is (= 200 (:width (gsh/shapes->rect normal)))) + (t/is (= 100 (:height (gsh/shapes->rect normal)))) + (t/is (= 200 (:width (gsh/shapes->rect wasm)))) + (t/is (= 100 (:height (gsh/shapes->rect wasm)))))) + +(t/deftest scale-sidebar-defaults-to-center-anchor-for-a-selection + (t/is (= [:center :center] + (scale/selection-anchor nil #{:shape}))) + (t/is (nil? (scale/selection-anchor nil #{})))) + +(t/deftest scale-sidebar-uses-the-user-selected-anchor + (let [selection #{:shape} + state {:selection selection :anchor [:left :top]}] + (t/is (= [:left :top] + (scale/selection-anchor state selection))))) + +(t/deftest scale-sidebar-does-not-keep-the-anchor-between-selections + (let [previous-selection #{:shape-1} + current-selection #{:shape-2} + state {:selection previous-selection :anchor [:left :top]}] + (t/is (= [:center :center] + (scale/selection-anchor state current-selection))))) + +(t/deftest scale-sidebar-parses-custom-scale-factors + (doseq [[value expected] [["1.5x" 1.5] + ["0,5x" 0.5] + [" 2 X " 2]]] + (t/is (= expected (parse-factor value)) + (str "parses " value)))) + +(t/deftest scale-sidebar-rejects-invalid-custom-scale-factors + (doseq [value ["0" "-1" "not a number"]] + (t/is (nil? (parse-factor value)) + (str "rejects " value)))) + +(t/deftest scale-sidebar-does-not-scale-invalid-or-unchanged-input + (t/is (nil? (scale-input-operation 1 "0"))) + (t/is (nil? (scale-input-operation 1 "not a number"))) + (t/is (nil? (scale-input-operation 1 "1x")))) + +(t/deftest scale-sidebar-builds-an-operation-for-a-new-scale-factor + (t/is (= {:multiplier 1.5 :factor 1.5} + (scale-input-operation 1 "1.5x")))) + +(t/deftest scale-selected-keeps-the-center-anchor-in-place + (t/async + done + (let [file (-> (cthf/sample-file :file1) + (ctho/add-rect :rect1 + :x 100 :y 100 :width 100 :height 50)) + rect (cths/get-shape file :rect1) + store (ths/setup-store file)] + + (ths/run-store + store done + [(dws/select-shape (:id rect)) + (dwt/scale-selected 2 [:center :center])] + (fn [state] + (let [file' (ths/get-file-from-state state)] + (assert-rect (shape-rect file' :rect1) 50 75 200 100))))))) + +(t/deftest scale-selected-keeps-the-top-left-anchor-in-place + (t/async + done + (let [file (-> (cthf/sample-file :file1) + (ctho/add-rect :rect1 + :x 100 :y 100 :width 100 :height 50)) + rect (cths/get-shape file :rect1) + store (ths/setup-store file)] + + (ths/run-store + store done + [(dws/select-shape (:id rect)) + (dwt/scale-selected 3 [:left :top])] + (fn [state] + (let [file' (ths/get-file-from-state state)] + (assert-rect (shape-rect file' :rect1) 100 100 300 150))))))) + +(t/deftest scale-selected-scales-strokes-and-text + (t/async + done + (let [file (-> (cthf/sample-file :file1) + (ctho/add-rect :rect1 + :x 0 :y 0 :width 100 :height 100 + :strokes [{:stroke-color "#000000" + :stroke-opacity 1 + :stroke-width 2 + :stroke-alignment :inner}]) + (ctho/add-text :text1 "Hello" :text-params {:x 200 :y 0})) + rect (cths/get-shape file :rect1) + text (cths/get-shape file :text1) + store (ths/setup-store file)] + + (ths/run-store + store done + [(dws/select-shapes (d/ordered-set (:id rect) (:id text))) + (dwt/scale-selected 2 [:left :top])] + (fn [state] + (let [file' (ths/get-file-from-state state) + rect' (cths/get-shape file' :rect1) + text' (cths/get-shape file' :text1) + text-node (first (txt/node-seq txt/is-text-node? (:content text')))] + + (assert-rect (shape-rect file' :rect1) 0 0 200 200) + (t/is (= 4 (get-in rect' [:strokes 0 :stroke-width])) + "stroke width scales with the selection") + (t/is (= "28" (:font-size text-node)) + "font size scales with the selection"))))))) + +(t/deftest scale-selected-ignores-constraints-in-the-legacy-renderer + (t/async + done + (let [file (ctho/add-frame-with-child + (cthf/sample-file :file1) :frame1 :child1 + :frame-params {:x 0 :y 0 :width 200 :height 200} + :child-params {:x 50 :y 50 :width 50 :height 50}) + frame (cths/get-shape file :frame1) + store (ths/setup-store file {:renderer :svg})] + + (ths/run-store + store done + [(dws/select-shape (:id frame)) + (dwt/scale-selected 2 [:left :top])] + (fn [state] + (let [file' (ths/get-file-from-state state)] + (assert-rect (shape-rect file' :frame1) 0 0 400 400) + (assert-rect (shape-rect file' :child1) 100 100 100 100))))))) diff --git a/frontend/test/frontend_tests/main_errors_test.cljs b/frontend/test/frontend_tests/main_errors_test.cljs index 348e8419c1..89b9b6bf73 100644 --- a/frontend/test/frontend_tests/main_errors_test.cljs +++ b/frontend/test/frontend_tests/main_errors_test.cljs @@ -23,6 +23,7 @@ - save failure notification – sticky toast carrying the error report - delegated save failures – causes handled by the handler for their type" (:require + [app.common.exceptions :as ex] [app.common.uuid :as uuid] [app.main.data.event :as ev] [app.main.data.persistence :as dps] @@ -43,6 +44,15 @@ [frontend-tests.helpers.mock :as mock] [potok.v2.core :as ptk])) +(defn- quiet-throwable + "Console stub for the tests that run a real error handler end to end. + Those handlers print the cause with `ex/print-throwable` before notifying, + which is what a developer wants in the app; in a test it only floods the + run log, and one of them replays the same error thousands of times. No + test asserts the printed text, so the stub changes nothing but the noise." + [& _] + nil) + ;; --------------------------------------------------------------------------- ;; stale-asset-error? ;; --------------------------------------------------------------------------- @@ -247,7 +257,8 @@ cause (ex-info "http error" {:type type :hint "http error"})] (await (mock/with-mocks* - {st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + {ex/print-throwable quiet-throwable + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) rt/get-current-href (constantly "https://penpot.example.com/#/workspace") tm/schedule (mock/stub (fn [f] (f))) st/format-last-events (mock/stub (fn [& _] (throw (ex-info "must not be called" {}))))} @@ -303,7 +314,8 @@ cause (ex-info "http error" {:type :offline :hint "http error"})] (await (mock/with-mocks* - {st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + {ex/print-throwable quiet-throwable + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) rt/get-current-href (constantly "https://penpot.example.com/#/workspace") tm/schedule (mock/stub (fn [f] (mock/asap f))) st/format-last-events (mock/stub (fn [& _] (throw (ex-info "must not be called" {}))))} @@ -588,6 +600,9 @@ ;; being told they have no access to the file. ;; --------------------------------------------------------------------------- +(def ^:private workspace-query-string + "screen=workspace&team-id=b8f8bb52-8b70-8144-8004-4a5085f0bdc9") + (def ^:private workspace-href "https://penpot.example.com/#/workspace?team-id=b8f8bb52-8b70-8144-8004-4a5085f0bdc9") @@ -609,7 +624,8 @@ (fn [_command _params] (rx/of {:authorized false :redirect-uri "https://idp.example.com/authorize"}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))} (errors/on-error (sso-required-error)) (await (async/settle)) @@ -625,14 +641,15 @@ (swap! rpc-calls conj {:command command :params params}) (rx/of {:authorized false :redirect-uri "https://idp.example.com/authorize"}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) st/emit! mock/noop} (errors/on-error (sso-required-error)) (await (async/settle)) (t/is (= [{:command :check-nitrate-sso :params {:team-id "b8f8bb52-8b70-8144-8004-4a5085f0bdc9" :organization-id organization-id - :url workspace-href}}] + :query-string workspace-query-string}}] @rpc-calls))))))) (t/deftest ^:async already-satisfied-organization-sso-retries-the-location @@ -643,7 +660,8 @@ {rp/cmd! (mock/stub (fn [_command _params] (rx/of {:authorized true :reason :sso-satisfied}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))} (errors/on-error (sso-required-error)) (await (async/settle)) @@ -657,7 +675,8 @@ {rp/cmd! (mock/stub (fn [_command _params] (rx/of {:authorized false :redirect-uri nil}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) rt/assign-exception (fn [error] (reset! assigned* error) (ptk/data-event ::assigned error))} @@ -675,7 +694,8 @@ {rp/cmd! (mock/stub (fn [_command _params] (rx/of {:authorized true :reason :no-team-access}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) rt/assign-exception (fn [error] (reset! assigned* error) (ptk/data-event ::assigned error))} @@ -690,7 +710,8 @@ (await (mock/with-mocks* {rp/cmd! (mock/stub (fn [_command _params] (rx/of {:authorized true}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) rt/assign-exception (fn [error] (ptk/data-event ::assigned error)) st/async-emit! (fn [& emitted] (swap! events into emitted))} (errors/on-error (sso-required-error)) @@ -706,7 +727,8 @@ {rp/cmd! (mock/stub (fn [_command _params] (swap! rpc-calls inc) (rx/empty))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) rt/assign-exception (fn [error] (reset! assigned* error) (ptk/data-event ::assigned error))} @@ -724,7 +746,8 @@ {rp/cmd! (mock/stub (fn [_command _params] (swap! rpc-calls inc) (rx/empty))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) st/emit! mock/noop} (errors/on-error (sso-required-error)) (errors/on-error (sso-required-error)) @@ -758,6 +781,8 @@ rt/get-current-href (constantly workspace-href) + rt/get-current-query-string + (constantly workspace-query-string) rt/assign-exception (fn [error] @@ -1015,7 +1040,8 @@ (fn [_command _params] (rx/of {:authorized false :redirect-uri "https://idp.example.com/authorize"}))) - rt/get-current-href (constantly workspace-href) + rt/get-current-href (constantly workspace-href) + rt/get-current-query-string (constantly workspace-query-string) st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))} (errors/flash-persistence (ex-info "SSO required" (sso-required-error))) (await (async/settle)) @@ -1060,7 +1086,8 @@ (t/testing "a version restored elsewhere reloads the file instead of notifying" (let [events (atom []) scheduled (atom [])] - (with-redefs [st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + (with-redefs [ex/print-throwable quiet-throwable + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) tm/schedule (mock/stub (fn [callback] (swap! scheduled conj callback)))] (errors/flash-persistence (ex-info "A different version has been restored" @@ -1138,7 +1165,8 @@ :uri "https://design.penpot.app/api/main/methods/get-teams"})] (await (mock/with-mocks* - {st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + {ex/print-throwable quiet-throwable + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) rt/get-current-href (constantly "https://penpot.example.com/#/workspace") rt/assign-exception (fn [error] (swap! assigned conj error) error) tm/schedule (mock/stub (fn [f] (f))) diff --git a/frontend/test/frontend_tests/plugins/creation_parent_test.cljs b/frontend/test/frontend_tests/plugins/creation_parent_test.cljs index 6fe12ebc82..7d94361d91 100644 --- a/frontend/test/frontend_tests/plugins/creation_parent_test.cljs +++ b/frontend/test/frontend_tests/plugins/creation_parent_test.cljs @@ -79,7 +79,7 @@ context ""))) id (obj/get created "$id")] - (await (async/wait-for #(some? (get-in @store (conj path id))) "created shape committed")) + (await (async/wait-for-store store #(some? (get-in @store (conj path id))) "created shape committed")) (let [created-shape (get-in @store (conj path id))] (t/is (= uuid/zero (:parent-id created-shape)) (str creation " / " layout)) (t/is (= uuid/zero (:frame-id created-shape)) (str creation " / " layout)) @@ -90,19 +90,19 @@ (t/is (= 1 (count (filter #{:create-file-media-object-from-url} (map first @uploads)))))) (ptk/emit! store dwu/undo) - (await (async/wait-for #(nil? (get-in @store (conj path id))) "creation undone")) + (await (async/wait-for-store store #(nil? (get-in @store (conj path id))) "creation undone")) (t/is (nil? (get-in @store (conj path id)))) (t/is (empty? (get-in @store (into path [(thi/id :board) :shapes])))) (ptk/emit! store dwu/redo) - (await (async/wait-for #(some? (get-in @store (conj path id))) "creation redone")) + (await (async/wait-for-store store #(some? (get-in @store (conj path id))) "creation redone")) (set! (.-x created) 1000) (set! (.-y created) 1200) - (await (async/wait-for #(= [1000 1200] ((juxt :x :y) (get-in @store (conj path id)))) "position updated")) + (await (async/wait-for-store store #(= [1000 1200] ((juxt :x :y) (get-in @store (conj path id)))) "position updated")) (t/is (= [1000 1200] ((juxt :x :y) (get-in @store (conj path id))))) (let [^js page (.-currentPage context) ^js board (.getShapeById page (str (thi/id :board)))] (.appendChild board created)) - (await (async/wait-for #(= (thi/id :board) (get-in @store (into path [id :parent-id]))) "explicit parent applied")) + (await (async/wait-for-store store #(= (thi/id :board) (get-in @store (into path [id :parent-id]))) "explicit parent applied")) (t/is (= (thi/id :board) (get-in @store (into path [id :parent-id])))))))))) (t/deftest ^:async interactive-creation-keeps-coordinate-based-parents @@ -143,16 +143,16 @@ position nil)}] (ptk/emit! store (get events creation))) (let [id (if (= creation :component) @id-ref svg-id)] - (await (async/wait-for #(some? (get-in @store (conj path id))) "interactive shape committed")) + (await (async/wait-for-store store #(some? (get-in @store (conj path id))) "interactive shape committed")) (t/is (= parent (get-in @store (into path [id :parent-id])))) (t/is (= parent (get-in @store (into path [id :frame-id])))) (t/is (= #{id} (set (get-in @store [:workspace-local :selected])))) (when (= creation :component) - (await (async/wait-for #(= :move (get-in @store [:workspace-local :transform])) "interactive move started")) + (await (async/wait-for-store store #(= :move (get-in @store [:workspace-local :transform])) "interactive move started")) (t/is (= :move (get-in @store [:workspace-local :transform]))) (rx/push! ms/mouse-position position) (await (async/settle)) (ptk/emit! store :interrupt) - (await (async/wait-for #(nil? (get-in @store [:workspace-local :transform])) "interactive move finished")) + (await (async/wait-for-store store #(nil? (get-in @store [:workspace-local :transform])) "interactive move finished")) (t/is (nil? (get-in @store [:workspace-local :transform]))) (t/is (= parent (get-in @store (into path [id :parent-id]))))))))))) diff --git a/frontend/test/frontend_tests/plugins/file_test.cljs b/frontend/test/frontend_tests/plugins/file_test.cljs index e6f2780035..a71b336943 100644 --- a/frontend/test/frontend_tests/plugins/file_test.cljs +++ b/frontend/test/frontend_tests/plugins/file_test.cljs @@ -6,8 +6,14 @@ (ns frontend-tests.plugins.file-test (:require + [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.main.repo :as rp] [app.plugins.file :as file] - [cljs.test :as t :include-macros true])) + [app.plugins.register :as r] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) (t/deftest file-version-created-at-returns-stored-date (let [created-at (js/Date.) @@ -19,3 +25,109 @@ :label "Version" :created-at created-at})] (t/is (identical? created-at (.-createdAt version))))) + +(defn- make-version + [file-id data] + (file/file-version-proxy (str (uuid/next)) file-id {} data)) + +(defn- record-cmd! + "Mock for `rp/cmd!` that records the calls and answers an observable with + the shape each command expects: a list of team users, and `true` for the + snapshot update (`update-file-snapshot` answers a boolean)." + [calls] + (fn [cmd params] + (swap! calls conj [cmd params]) + (rx/of (if (= :get-team-users cmd) [] true)))) + +;; The snapshot RPC commands require the `file-id` (they use it to prune the +;; partitions of `file_change`); the plugin proxy is the other door to them and +;; sends it on rename, remove and pin. + +(t/deftest ^:async renaming-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + calls (atom [])] + + (await + (mock/with-mocks* + {r/check-permission (constantly true) + rp/cmd! (mock/stub (record-cmd! calls))} + + (let [version (make-version file-id + {:id version-id + :label "Version" + :created-at (ct/now) + :created-by "user"})] + (aset version "label" "renamed") + (let [[cmd params] (last @calls)] + (t/is (= :update-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))) + (t/is (= "renamed" (:label params))))))))) + +(t/deftest ^:async removing-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + calls (atom [])] + + (await + (mock/with-mocks* + {r/check-permission (constantly true) + rp/cmd! (mock/stub (record-cmd! calls))} + + (let [version (make-version file-id + {:id version-id + :label "Version" + :created-at (ct/now) + :created-by "user"})] + (await (.remove version)) + (let [[cmd params] (last @calls)] + (t/is (= :delete-file-snapshot cmd)) + (t/is (= version-id (:id params))) + (t/is (= file-id (:file-id params))))))))) + +(t/deftest ^:async pinning-a-version-sends-the-current-file-id + (let [file-id (uuid/next) + version-id (uuid/next) + calls (atom [])] + + (await + (mock/with-mocks* + {r/check-permission (constantly true) + rp/cmd! (mock/stub (record-cmd! calls))} + + (let [version (make-version file-id + {:id version-id + :label "Version" + :created-at (ct/now) + :created-by "system"})] + (await (.pin version)) + (t/is (= [[:get-team-users file-id] + [:update-file-snapshot file-id]] + (mapv (fn [[cmd params]] [cmd (:file-id params)]) @calls)))))))) + +(t/deftest ^:async file-version-proxy-pin-resolves-the-pinned-version + ;; `pin` answers with a proxy, not with the boolean the snapshot command + ;; returns: the autosave becomes a user version with a new label. + (let [file-id (uuid/next) + version-id (uuid/next) + created-at (ct/now) + calls (atom [])] + (await + (mock/with-mocks* + {r/check-permission (constantly true) + rp/cmd! (mock/stub (record-cmd! calls))} + (let [version (make-version file-id + {:id version-id + :label "Version" + :created-at created-at + :created-by "system"}) + pinned (await (.pin version))] + (t/is (file/file-version-proxy? pinned)) + (t/is (= [[:get-team-users file-id] + [:update-file-snapshot file-id]] + (mapv (fn [[cmd params]] [cmd (:file-id params)]) @calls))) + (t/is (= (ct/format-inst created-at :localized-date) + (.-label pinned))) + (t/is (false? (.-isAutosave pinned))) + (t/is (identical? created-at (.-createdAt pinned)))))))) diff --git a/frontend/test/frontend_tests/plugins/library_test.cljs b/frontend/test/frontend_tests/plugins/library_test.cljs index 982a62c566..9df7e4e5ff 100644 --- a/frontend/test/frontend_tests/plugins/library_test.cljs +++ b/frontend/test/frontend_tests/plugins/library_test.cljs @@ -77,8 +77,8 @@ component (.component copy)] (t/is (true? (get-in @store [:files (:id file) :data :components (thi/id :restorable) :deleted]))) (.restore component) - (await (async/wait-for #(not (get-in @store [:files (:id file) :data :components (thi/id :restorable) :deleted])) - "component restored")) + (await (async/wait-for-store store #(not (get-in @store [:files (:id file) :data :components (thi/id :restorable) :deleted])) + "component restored")) (.restore component) (await (async/settle)) (t/is (= (thi/id :main) @@ -203,6 +203,46 @@ (await (async/settle)) (t/is (empty? @events)))))))) +(t/deftest ^:async main-instance-of-a-deleted-component-is-null + (let [file (file-with-deleted-component) + store (ths/setup-store file {:renderer :svg})] + (await + (mock/with-mocks* + {st/state store + shape/lib-component-proxy library/lib-component-proxy} + (let [copy (shape/shape-proxy plugin-id (:id file) (thi/id :main-page) (thi/id :copy)) + component (.component copy)] + (t/is (some? component)) + (t/is (nil? (.mainInstance component)))))))) + +(t/deftest ^:async component-root-of-a-copy-on-another-page-than-its-main + (let [file (-> (cthf/sample-file :file :page-label :main-page) + (cths/add-sample-shape :main :type :frame) + (cths/add-sample-shape :child :parent-label :main) + (cthc/make-component :component :main) + (cthf/add-sample-page :other-page) + (cthc/instantiate-component :component :copy :children-labels [:copy-child])) + store (ths/setup-store file {:renderer :svg})] + (await + (mock/with-mocks* + {st/state store} + (let [copy-child (shape/shape-proxy plugin-id (:id file) (thi/id :other-page) (thi/id :copy-child)) + root (.componentRoot copy-child)] + (t/is (= (str (thi/id :copy)) (.-id root))) + (t/is (= (str (thi/id :copy)) (.-id (.componentHead copy-child))))))))) + +(t/deftest ^:async component-of-a-copy-whose-library-is-missing-is-null + (let [file (-> (file-with-component) + (cths/update-shape :copy :component-file (uuid/next))) + store (ths/setup-store file {:renderer :svg})] + (await + (mock/with-mocks* + {st/state store + shape/lib-component-proxy library/lib-component-proxy} + (let [copy (shape/shape-proxy plugin-id (:id file) (thi/id :main-page) (thi/id :copy))] + (t/is (nil? (.component copy))) + (t/is (= (str (thi/id :copy)) (.-id (.componentRoot copy))))))))) + (t/deftest library-asset-proxies-expose-library-id (let [file-id (random-uuid) id (random-uuid)] diff --git a/frontend/test/frontend_tests/plugins/management_test.cljs b/frontend/test/frontend_tests/plugins/management_test.cljs index 15d04f8f7f..e9637d4d88 100644 --- a/frontend/test/frontend_tests/plugins/management_test.cljs +++ b/frontend/test/frontend_tests/plugins/management_test.cljs @@ -8,6 +8,7 @@ (:require [app.common.uuid :as uuid] [app.main.data.dashboard :as dd] + [app.main.data.event :as ev] [app.main.data.team :as dtm] [app.main.repo :as rp] [app.main.store :as st] @@ -80,6 +81,15 @@ ([command] (swap! requests conj [command]) (respond command nil)) ([command params] (swap! requests conj [command params]) (respond command params))))) +(defn- origins + "Maps each recorded command that is not a read to the origin of its params." + [requests] + (into {} + (keep (fn [[command params]] + (when-not (re-find #"^get-" (name command)) + [command (::ev/origin (meta params))]))) + requests)) + (defn- emit-stub "Mock `st/emit!` applying update functions to `state` and recording events." [state events] @@ -231,18 +241,20 @@ (props team* ["id" "name" "isDefault"]))) (t/is (= [[:create-team "Brand"]] (mapv (fn [[command params]] [command (:name params)]) @requests))) + (t/is (= {:create-team "plugin"} (origins @requests))) (t/is (= 1 (count (filter #(= ::dtm/fetch-teams (ptk/type %)) @events)))) (t/is (= "Expected a name with 1 to 250 characters" (await (message (.createTeam (management/create-context plugin-id) #js {:name ""})))))))))) (t/deftest ^:async setters-update-values-after-the-backend-succeeds - (let [state (atom (throwing dashboard)) - events (atom [])] + (let [state (atom (throwing dashboard)) + events (atom []) + requests (atom [])] (await (mock/with-mocks* {st/state state st/emit! (emit-stub state events) - rp/cmd! (mock/stub (fn [& _] (->> (rx/of {}) (rx/observe-on :async))))} + rp/cmd! (cmd-stub requests (fn [& _] (->> (rx/of {}) (rx/observe-on :async))))} (let [team* (management/team-proxy plugin-id team) project* (management/project-proxy plugin-id project) file* (management/file-proxy plugin-id file)] @@ -260,7 +272,10 @@ (t/is (true? (obj/get project* "pinned"))) (t/is (= "Login" (obj/get file* "name"))) (t/is (true? (obj/get file* "shared"))) - (t/is (= 5 (count @events)))))))) + (t/is (= 5 (count @events))) + (t/is (= {:update-team "plugin" :rename-project "plugin" :update-project-pin "plugin" + :rename-file "plugin" :set-file-shared "plugin"} + (origins @requests)))))))) (t/deftest ^:async stale-object-setter-errors-stay-out-of-the-global-handler (doseq [[constructor property value original] @@ -340,6 +355,7 @@ (t/is (= {"name" "Login" "teamId" (str team-id)} (props file* ["name" "teamId"]))) (t/is (= 2 (obj/get parent "fileCount"))) (t/is (= [:create-project {:team-id team-id :name "Icons"}] (first @requests))) + (t/is (= {:create-project "plugin" :create-file "plugin"} (origins @requests))) (if (= initial dashboard) (do (t/is (contains? (:projects @state) (:id created-project))) (t/is (contains? (:files @state) (:id created-file)))) @@ -383,7 +399,10 @@ [:move-project {:project-id other-id :team-id other-id}] [:delete-file {:id file-id}] [:delete-project {:id other-id}]] - @requests))))))) + @requests)) + (t/is (= {:duplicate-file "plugin" :duplicate-project "plugin" :move-files "plugin" + :move-project "plugin" :delete-file "plugin" :delete-project "plugin"} + (origins @requests)))))))) (t/deftest ^:async methods-reject-invalid-input-and-missing-permissions (let [requests (atom [])] @@ -475,7 +494,8 @@ (doseq [proxy [(management/project-proxy r/mcp-plugin-id project) (management/file-proxy r/mcp-plugin-id file)]] (t/is (nil? (await (.remove proxy))))) - (t/is (= [[:delete-project {:id project-id}] [:delete-file {:id file-id}]] @requests)))))) + (t/is (= [[:delete-project {:id project-id}] [:delete-file {:id file-id}]] @requests)) + (t/is (= {:delete-project "mcp" :delete-file "mcp"} (origins @requests))))))) (t/deftest ^:async deleted-objects-can-be-listed-and-restored (let [state (atom ready) @@ -532,6 +552,7 @@ rp/cmd! (mock/stub (fn [command params] (t/is (= ::sse/restore-deleted-team-files command)) (t/is (= {:team-id team-id :ids #{file-id}} params)) + (t/is (= "plugin" (::ev/origin (meta params)))) (rx/observe-on :async response)))} (let [promise (.then (.restore (management/file-proxy plugin-id file)) #(reset! completed? true))] diff --git a/frontend/test/frontend_tests/plugins/register_test.cljs b/frontend/test/frontend_tests/plugins/register_test.cljs index a4399ab6af..afa0cfe304 100644 --- a/frontend/test/frontend_tests/plugins/register_test.cljs +++ b/frontend/test/frontend_tests/plugins/register_test.cljs @@ -347,3 +347,71 @@ :permissions #js ["content:read"]})] (t/is (nil? (:scope manifest))) (t/is (= #{"content:read"} (:permissions manifest))))) + +;; --- global plugins restriction --- + +(defn- global-not-allowed-error + "The ex-data type thrown by `(f)`, or nil when it does not throw." + [f] + (try + (f) + nil + (catch :default cause + (:type (ex-data cause))))) + +(t/deftest global-plugins-are-allowed-for-mcp-and-localhost + (t/is (preg/global-allowed? {:plugin-id preg/mcp-plugin-id + :host "https://design.penpot.app/plugins/mcp/"})) + (doseq [host ["http://localhost:4400/" "http://127.0.0.1:4400/" "http://[::1]:4400/" + "http://LOCALHOST:4400/" "http://LocalHost/" "http://localhost.:4400/"]] + (t/is (preg/global-allowed? {:plugin-id "local" :host host}) host)) + (t/is (not (preg/global-allowed? {:plugin-id "remote" :host "https://plugins.example.com/"}))) + (t/is (not (preg/global-allowed? {:plugin-id "remote" :host "https://localhost.example.com/"})))) + +(t/deftest parse-manifest-rejects-remote-global-scope + (t/is (= ::preg/global-not-allowed + (global-not-allowed-error + #(preg/parse-manifest + "https://plugins.example.com/manifest.json" + #js {:name "Remote global plugin" + :code "plugin.js" + :scope "global" + :permissions #js ["content:read"]}))))) + +(t/deftest parse-manifest-rejects-remote-management-permissions + (doseq [permission ["manage:projects" "manage:teams" "manage:delete" "allow:global"]] + (t/is (= ::preg/global-not-allowed + (global-not-allowed-error + #(preg/parse-manifest + "https://plugins.example.com/manifest.json" + #js {:name "Remote management plugin" + :code "plugin.js" + :permissions #js ["content:read" permission]}))) + permission))) + +(t/deftest parse-manifest-keeps-remote-workspace-plugins + (let [manifest (preg/parse-manifest + "https://plugins.example.com/manifest.json" + #js {:name "Remote workspace plugin" + :code "plugin.js" + :permissions #js ["content:read"]})] + (t/is (= #{"content:read"} (:permissions manifest))))) + +(t/deftest ^:async stored-remote-plugins-lose-global-permissions + (await + (mock/with-mocks* + {rp/cmd! (record-cmd-mock (fn [_ _] (rx/of {:ok true})))} + (let [permissions #{"content:read" "allow:global" "manage:projects"}] + (preg/install-plugin! {:plugin-id "reg-global-remote" + :host "https://plugins.example.com/" + :scope "global" + :permissions permissions}) + (preg/install-plugin! {:plugin-id "reg-global-local" + :host "http://localhost:4400/" + :scope "global" + :permissions permissions}) + (t/is (preg/has-permission? "reg-global-remote" "content:read")) + (t/is (not (preg/has-permission? "reg-global-remote" "allow:global"))) + (t/is (not (preg/has-permission? "reg-global-remote" "manage:projects"))) + (t/is (preg/has-permission? "reg-global-local" "allow:global")) + (t/is (preg/has-permission? "reg-global-local" "manage:projects")))))) diff --git a/frontend/test/frontend_tests/plugins/runtime_unavailable_test.cljs b/frontend/test/frontend_tests/plugins/runtime_unavailable_test.cljs new file mode 100644 index 0000000000..1c2d16bc56 --- /dev/null +++ b/frontend/test/frontend_tests/plugins/runtime_unavailable_test.cljs @@ -0,0 +1,57 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.plugins.runtime-unavailable-test + (:require + [app.main.data.plugins :as dp] + [app.main.store :as st] + [app.util.globals :as ug] + [app.util.object :as obj] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock] + [potok.v2.core :as ptk] + [promesa.core :as p])) + +(def ^:private test-manifest + {:plugin-id "test-plugin" + :name "Test plugin" + :description "A plugin used by the runtime availability tests" + :host "https://example.com" + :code ""}) + +(defn- with-load-plugin + "Calls dp/open-plugin! with a mocked st/emit! and the given ɵloadPlugin + value on the global object, returning the emitted events." + [load-plugin-fn] + (let [global ug/global + had-plugin? (obj/contains? global "ɵloadPlugin") + previous (obj/get global "ɵloadPlugin") + emitted (atom [])] + (if (fn? load-plugin-fn) + (obj/set! global "ɵloadPlugin" load-plugin-fn) + (obj/unset! global "ɵloadPlugin")) + (-> (mock/with-mocks* + {st/emit! (fn [event & events] (swap! emitted into (cons event events)))} + (dp/open-plugin! test-manifest nil)) + (p/then (fn [_] @emitted)) + (p/finally (fn [] + (if had-plugin? + (obj/set! global "ɵloadPlugin" previous) + (obj/unset! global "ɵloadPlugin"))))))) + +(t/deftest ^:async load-plugin-notifies-when-runtime-is-unavailable + (let [emitted (await (with-load-plugin nil)) + types (mapv ptk/type emitted)] + (t/is (some #{:app.main.data.notifications/show} types) + "emits a notification when the plugin runtime is unavailable") + (t/is (some #{::dp/remove-current-plugin} types) + "still removes the plugin from the open list"))) + +(t/deftest ^:async load-plugin-does-not-notify-when-runtime-is-available + (let [emitted (await (with-load-plugin (fn [_ _] (p/resolved nil)))) + types (mapv ptk/type emitted)] + (t/is (not-any? #{:app.main.data.notifications/show} types) + "does not emit a notification when the plugin runtime is available"))) diff --git a/frontend/test/frontend_tests/plugins/system_events_test.cljs b/frontend/test/frontend_tests/plugins/system_events_test.cljs new file mode 100644 index 0000000000..302a566f9c --- /dev/null +++ b/frontend/test/frontend_tests/plugins/system_events_test.cljs @@ -0,0 +1,52 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.plugins.system-events-test + (:require + [app.main.data.event :as ev] + [app.main.store :as st] + [app.plugins.register :as r] + [app.plugins.system-events :as se] + [cljs.test :as t :include-macros true] + [potok.v2.core :as ptk])) + +(def ^:private plugin-id "00000000-0000-0000-0000-000000000000") + +(defn- set-state + [file-id] + (set! st/state + (atom {:current-file-id file-id + :profile {:props {:plugins {:data {plugin-id {:name "Plugin" + :url "https://plugin.test"}}}}}}))) + +(t/deftest mcp-event-carries-file + (let [file-id (random-uuid)] + (set-state file-id) + (let [data (deref (se/event r/mcp-plugin-id "create-version" :file-id file-id))] + (t/is (= "mcp" (::ev/origin data))) + (t/is (= {:file-id file-id} (::ev/context data))) + (t/is (= file-id (:file-id data)))))) + +(t/deftest plugin-event-carries-file-and-plugin + (let [file-id (random-uuid)] + (set-state file-id) + (t/is (= {:file-id file-id + :plugin-name "Plugin" + :plugin-url "https://plugin.test"} + (::ev/context (deref (se/event plugin-id "add-flow"))))))) + +(t/deftest mcp-add-event-carries-file + (let [file-id (random-uuid)] + (set-state file-id) + (t/is (= {::ev/origin "mcp" + ::ev/context {:file-id file-id}} + (meta (se/add-event (ptk/reify ::test) r/mcp-plugin-id)))))) + +(t/deftest event-has-no-file-on-the-dashboard + (set-state nil) + (t/is (nil? (-> (deref (se/event r/mcp-plugin-id "create-project")) + ::ev/context + :file-id)))) diff --git a/frontend/test/frontend_tests/plugins/tokens_test.cljs b/frontend/test/frontend_tests/plugins/tokens_test.cljs index a8e5033cef..2f0fc21855 100644 --- a/frontend/test/frontend_tests/plugins/tokens_test.cljs +++ b/frontend/test/frontend_tests/plugins/tokens_test.cljs @@ -154,8 +154,8 @@ (mock/with-mocks* {st/state store st/stream (ptk/input-stream store)} (t/is (= (str (:id file)) (.. context -tokensSource -id))) (set! (.-tokensSource context) source-proxy) - (await (async/wait-for #(= (:id source) (get-in @store [:files (:id file) :data :tokens-source])) - "library token source")) + (await (async/wait-for-store store #(= (:id source) (get-in @store [:files (:id file) :data :tokens-source])) + "library token source")) (t/is (= (str (:id source)) (.. context -tokensSource -id))) (t/is (= (datafy (ctht/get-tokens-status source)) (datafy (get-in @store [:files (:id file) :data :tokens-status])))) @@ -175,8 +175,8 @@ (await (mock/with-mocks* {st/state store st/stream (ptk/input-stream store)} (set! (.-tokensSource context) (.-local context)) - (await (async/wait-for #(nil? (get-in @store [:files (:id file) :data :tokens-source])) - "local token source")) + (await (async/wait-for-store store #(nil? (get-in @store [:files (:id file) :data :tokens-source])) + "local token source")) (let [^js catalog (.. context -local -tokens)] (t/is (= (str (:id file)) (.. context -tokensSource -id))) (t/is (true? (.-isEditableTokens catalog))) @@ -194,16 +194,16 @@ (mock/with-mocks* {st/state store st/stream (ptk/input-stream store)} (st/emit! (tohs/watch-undo-stack)) (set! (.-tokensSource context) source-proxy) - (await (async/wait-for #(seq (get-in @store [:workspace-undo :items])) - "token source undo entry")) + (await (async/wait-for-store store #(seq (get-in @store [:workspace-undo :items])) + "token source undo entry")) (st/emit! dwu/undo) - (await (async/wait-for #(nil? (get-in @store [:files (:id file) :data :tokens-source])) - "undo token source")) + (await (async/wait-for-store store #(nil? (get-in @store [:files (:id file) :data :tokens-source])) + "undo token source")) (t/is (= (datafy (ctht/get-tokens-status file)) (datafy (get-in @store [:files (:id file) :data :tokens-status])))) (st/emit! dwu/redo) - (await (async/wait-for #(= (:id source) (get-in @store [:files (:id file) :data :tokens-source])) - "redo token source")) + (await (async/wait-for-store store #(= (:id source) (get-in @store [:files (:id file) :data :tokens-source])) + "redo token source")) (t/is (= (datafy (ctht/get-tokens-status source)) (datafy (get-in @store [:files (:id file) :data :tokens-status])))))))) @@ -379,13 +379,13 @@ ^js theme (aget (.-themes catalog) 0)] (st/emit! (tohs/watch-undo-stack)) (.toggleActive theme) - (await (async/wait-for #(seq (get-in @store [:workspace-undo :items])) - "theme toggle undo entry")) + (await (async/wait-for-store store #(seq (get-in @store [:workspace-undo :items])) + "theme toggle undo entry")) (t/is (false? (.-active theme))) (st/emit! dwu/undo) - (await (async/wait-for #(= (datafy status) - (datafy (get-in @store [:files (:id file) :data :tokens-status]))) - "undo external theme toggle")) + (await (async/wait-for-store store #(= (datafy status) + (datafy (get-in @store [:files (:id file) :data :tokens-status]))) + "undo external theme toggle")) (t/is (tokens-unchanged? store source))))))) (t/deftest ^:async external-token-source-allows-applying-tokens-to-shapes @@ -413,10 +413,10 @@ (.applyToSelected token #js ["fill"]) (doseq [label [:rect1 :rect2 :rect3]] (let [shape-path [:files (:id file) :data :pages-index page-id :objects (cthi/id label)]] - (await (async/wait-for #(= "color.primary" (get-in @store (conj shape-path :applied-tokens :fill))) - (str "apply external token to " label))) - (await (async/wait-for #(= "#ff0000" (get-in @store (conj shape-path :fills 0 :fill-color))) - (str "resolve external token on " label))))) + (await (async/wait-for-store store #(= "color.primary" (get-in @store (conj shape-path :applied-tokens :fill))) + (str "apply external token to " label))) + (await (async/wait-for-store store #(= "#ff0000" (get-in @store (conj shape-path :fills 0 :fill-color))) + (str "resolve external token on " label))))) (t/is (= (ctht/get-tokens-lib file) (get-in @store [:files (:id file) :data :tokens-lib]))) (t/is (= (datafy (ctht/get-tokens-status file)) (datafy (get-in @store [:files (:id file) :data :tokens-status])))) @@ -728,9 +728,9 @@ ;; token and resolves to a uniform width, which the strokes setter accepts. (t/testing "the uniform strokeWidth alias is still accepted" (.applyToken ellipse token #js ["strokeWidth"]) - (await (async/wait-for - #(= "width.sm" (get-in @store (conj path :stroke-width-top))) - "uniform stroke width token applied")) + (await (async/wait-for-store store + #(= "width.sm" (get-in @store (conj path :stroke-width-top))) + "uniform stroke width token applied")) (t/is (= "width.sm" (get-in @store (conj path :stroke-width-top))))))) (t/deftest token-attr?-rejects-unknown-input @@ -897,6 +897,38 @@ (t/is (= token-id (:token-id @captured))) (t/is (= ["Inter" "Arial"] (get-in @captured [:attrs :value]))))))) +(t/deftest shadow-token-value-round-trips + (let [set-id (cthi/new-id! :token-set) + token-id (cthi/new-id! :shadow-token) + shadow [{:offset-x "1" :offset-y "2" :blur "4" :spread "0" :color "#000000" :inset false}] + file (-> (cthf/sample-file :file1 :page-label :page1) + (ctht/add-tokens-lib) + (ctht/update-tokens-lib + #(-> % + (ctob/add-set + (ctob/make-token-set :id set-id + :name "shadows")) + (ctob/add-token + set-id + (ctob/make-token :id token-id + :name "shadow.card" + :type :shadow + :value shadow))))) + store (ths/setup-store file) + _ (set! st/state store) + _ (set! st/stream (ptk/input-stream store)) + file-id (:id file) + captured (atom nil)] + (with-redefs [r/check-permission (constantly true) + u/check-editable-tokens (constantly nil) + dwtl/update-token (mock/stub (fn [_ _ attrs] + (reset! captured attrs) + :update-token)) + st/emit! mock/noop] + (let [token (ptok/token-proxy "plugin-id" file-id set-id token-id)] + (set! (.-value token) (.-value token)) + (t/is (= shadow (:value @captured))))))) + (t/deftest typography-token-resolved-value-is-plugin-array-shape (let [token (ctob/make-token {:name "type.body" diff --git a/frontend/test/frontend_tests/render_wasm/call_test.cljs b/frontend/test/frontend_tests/render_wasm/call_test.cljs new file mode 100644 index 0000000000..17d4331519 --- /dev/null +++ b/frontend/test/frontend_tests/render_wasm/call_test.cljs @@ -0,0 +1,62 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.render-wasm.call-test + "Unit tests for the `app.common.render-wasm.helpers/call` macro." + (:require + [app.common.render-wasm.helpers :as h] + [cljs.test :as t :include-macros true])) + +(defn- fake-module + "Module whose `_fail` export throws like a WASM trap after moving the + shadow stack pointer down 64 bytes, leaking the frame as a trap does." + [error-code] + (let [sp (atom 1000) + restored (atom [])] + {:sp sp + :restored restored + :module #js {"stackSave" (fn [] @sp) + "stackRestore" (fn [v] + (swap! restored conj v) + (reset! sp v)) + "_read_error_code" (fn [] error-code) + "_ok" (fn [x] (* 2 x)) + "_fail" (fn [] + (swap! sp - 64) + (throw (js/Error. "unreachable")))}})) + +(t/deftest failed-call-restores-stack-pointer + (let [{:keys [module sp restored]} (fake-module 0x02)] + (dotimes [_ 3] + (t/is (thrown? js/Error (h/call module "_fail")))) + (t/is (= 1000 @sp)) + (t/is (= [1000 1000 1000] @restored)))) + +(t/deftest failed-call-keeps-error-classification + (let [{:keys [module]} (fake-module 0x01) + data (try + (h/call module "_fail") + nil + (catch :default cause + (ex-data cause)))] + (t/is (= :wasm-error (:type data))) + (t/is (= :non-blocking (:code data))) + (t/is (= "_fail" (:fn data))))) + +(t/deftest successful-call-does-not-restore + (let [{:keys [module restored]} (fake-module 0)] + (t/is (= 42 (h/call module "_ok" 21))) + (t/is (empty? @restored)))) + +(t/deftest module-without-stack-methods-still-reports-errors + (let [module #js {"_read_error_code" (fn [] 0x02) + "_fail" (fn [] (throw (js/Error. "unreachable")))} + data (try + (h/call module "_fail") + nil + (catch :default cause + (ex-data cause)))] + (t/is (= :panic (:code data))))) diff --git a/frontend/test/frontend_tests/render_wasm/modifiers_test.cljs b/frontend/test/frontend_tests/render_wasm/modifiers_test.cljs new file mode 100644 index 0000000000..ad66bb7e05 --- /dev/null +++ b/frontend/test/frontend_tests/render_wasm/modifiers_test.cljs @@ -0,0 +1,130 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.render-wasm.modifiers-test + (:require + [app.common.geom.matrix :as gmt] + [app.common.render-wasm.api.modifiers :as modifiers] + [app.common.render-wasm.wasm :as wasm] + [app.common.test-helpers.render-wasm :as thw] + [app.render-wasm.api :as api] + [app.util.timers :as timers] + [cljs.test :as t :include-macros true])) + +(def id-a #uuid "f1234567-89ab-cdef-fedc-ba9876543210") +(def id-b #uuid "12345678-abcd-ef01-2345-6789abcdef01") +(def transform (gmt/matrix 1 0 0 1 24 -16)) + +(defn- with-renderer + [options f] + (let [{:keys [module] :as env} (thw/create-module options) + frames (atom [])] + (with-redefs [wasm/internal-module module + wasm/serializers thw/serializers + wasm/context-initialized? true + wasm/context-lost? (atom false) + wasm/reloading? (atom false) + wasm/disable-request-render? (atom false) + wasm/internal-frame-id nil + timers/raf (fn [callback] + (swap! frames conj callback) + (count @frames))] + (try + (f (assoc env :frames frames)) + (finally + (with-redefs [wasm/context-initialized? false] + (doseq [callback @frames] + (callback 0)))))))) + +(t/deftest unavailable-renderers-preserve-existing-modifiers + (doseq [[initialized lost reloading] [[false false false] + [true true false] + [true false true]] + interactive [false true]] + (with-renderer + {} + (fn [{:keys [state frames]}] + (modifiers/set-modifiers [[id-a transform]]) + (modifiers/set-structure-modifiers + [{:type :add-children :parent id-a :id id-b :value 0}]) + (when interactive + (modifiers/set-modifiers-start)) + (let [before @state] + (with-redefs [wasm/context-initialized? initialized + wasm/context-lost? (atom lost) + wasm/reloading? (atom reloading)] + (t/is (= [] (api/propagate-modifiers + [[id-a {:transform transform :kind :child}]] true nil))) + (api/set-modifiers nil) + (api/set-structure-modifiers + [{:type :add-children :parent id-b :id id-a :index 9 :value 0}]) + (api/clean-modifiers) + (api/set-modifiers-start) + (api/set-modifiers-end) + (t/is (= before @state)) + (t/is (empty? @frames)))))))) + +(t/deftest applying-a-preview-schedules-a-render + (with-renderer + {} + (fn [{:keys [state frames]}] + (api/set-modifiers []) + (t/is (empty? @frames)) + (t/is (thrown? js/Error (api/set-modifiers nil))) + (api/set-modifiers [[id-a transform] [id-b transform]]) + (t/is (= [[id-a transform] [id-b transform]] (:preview @state))) + (t/is (= 1 (count @frames)))))) + +(t/deftest propagation-and-structure-updates-leave-rendering-explicit + (let [output [[id-b transform]]] + (with-renderer + {:output output} + (fn [{:keys [state frames]}] + (let [entries [[id-a {:transform transform :kind :child}]]] + (t/is (= output (api/propagate-modifiers entries true :x))) + (t/is (= {:precision 3 :entries entries} (:propagated @state)))) + (api/set-structure-modifiers + [{:type :add-children :parent id-a :id id-b :index 2 :value 0}]) + (t/is (= [{:type :add-children :parent id-a :id id-b :index 2 :value 0}] + (:structure @state))) + (t/is (empty? @frames)))))) + +(t/deftest interactive-mode-and-preview-cleanup-have-independent-effects + (let [first-preview [[id-a transform]] + first-structure [{:type :add-children :parent id-a :id id-b :index 2 :value 0}] + second-preview [[id-b (gmt/matrix 1 0 0 1 -48 32)]] + second-structure [{:type :remove-children :parent id-b :id id-a :index 1 :value 0}] + state-keys [:interactive :preview :structure]] + (with-renderer + {} + (fn [{:keys [state frames]}] + (api/set-modifiers-start) + (modifiers/set-modifiers first-preview) + (modifiers/set-structure-modifiers first-structure) + (t/is (= {:interactive true :preview first-preview :structure first-structure} + (select-keys @state state-keys))) + + (api/clean-modifiers) + (t/is (= {:interactive true :preview [] :structure []} + (select-keys @state state-keys))) + + (modifiers/set-modifiers second-preview) + (modifiers/set-structure-modifiers second-structure) + (api/set-modifiers-end) + (t/is (= {:interactive false :preview second-preview :structure second-structure} + (select-keys @state state-keys))) + (t/is (empty? @frames)))))) + +(t/deftest a-failed-preview-does-not-schedule-a-render + (with-renderer + {:set-error 2} + (fn [{:keys [frames]}] + (let [error (try + (api/set-modifiers [[id-a transform]]) + (catch :default cause cause))] + (t/is (= "_set_modifiers" (:fn (ex-data error)))) + (t/is (= :panic (:code (ex-data error)))) + (t/is (empty? @frames)))))) diff --git a/frontend/test/frontend_tests/render_wasm/serialization_test.cljs b/frontend/test/frontend_tests/render_wasm/serialization_test.cljs index b204bc13b6..88fba33b67 100644 --- a/frontend/test/frontend_tests/render_wasm/serialization_test.cljs +++ b/frontend/test/frontend_tests/render_wasm/serialization_test.cljs @@ -92,15 +92,6 @@ path? (conj [:path id (:content shape)]))))) cases)) -(t/deftest tail-cases-flags-are-consistent - (doseq [{:keys [shape tail? svg? path?]} tail-cases] - (t/is (boolean? tail?) "tail? is a boolean") - (t/is (= tail? (boolean (or svg? path?))) "tail? matches the write flags") - (t/is (= svg? (some? (:svg-attrs shape))) "svg? matches the shape") - (t/is (= path? (boolean (and (contains? #{:path :bool} (:type shape)) - (some? (:content shape))))) - "path? matches the shape"))) - (t/deftest needs-shape-tail-matches-spec-table (doseq [{:keys [shape tail?]} tail-cases] (t/is (= tail? (serialize-shape/needs-shape-tail? shape)) diff --git a/frontend/test/frontend_tests/router_test.cljs b/frontend/test/frontend_tests/router_test.cljs index c6f39433dd..10d5713541 100644 --- a/frontend/test/frontend_tests/router_test.cljs +++ b/frontend/test/frontend_tests/router_test.cljs @@ -6,6 +6,7 @@ (ns frontend-tests.router-test (:require + [app.config :as cf] [app.main.router :as rt] [cljs.test :as t :include-macros true])) @@ -43,3 +44,28 @@ (t/is (nil? (rt/match test-routes ""))) (t/is (nil? (rt/match test-routes "?team-id=team-1"))) (t/is (nil? (rt/match test-routes "?screen=nope&team-id=team-1")))) + +(t/deftest token->query-string-drops-the-separator + ;; The history token is `location.search`, so it carries the `?` that + ;; separates it from the path; a query string does not carry it. + (t/is (= "screen=workspace&team-id=t1" + (rt/token->query-string "?screen=workspace&team-id=t1"))) + (t/is (= "" (rt/token->query-string "?"))) + ;; A value already without separator is not touched: no double strip. + (t/is (= "screen=workspace" (rt/token->query-string "screen=workspace"))) + (t/is (= "" (rt/token->query-string ""))) + (t/is (= "" (rt/token->query-string nil)))) + +(t/deftest token->url-joins-the-token-to-the-public-uri + (let [base (str cf/public-uri)] + (t/is (= (str base "?screen=workspace") + (rt/token->url "?screen=workspace"))) + (t/is (= (str base "?screen=workspace") + (rt/token->url "screen=workspace"))) + (t/is (= base (rt/token->url ""))) + (t/is (= base (rt/token->url nil))))) + +(t/deftest get-current-query-string-comes-from-the-current-token + (with-redefs [rt/get-current-token (constantly "?screen=workspace&team-id=t1")] + (t/is (= "screen=workspace&team-id=t1" + (rt/get-current-query-string))))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index f0eb597672..5ff1e7db87 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -8,16 +8,19 @@ [frontend-tests.browser-history-test] [frontend-tests.code-gen-style-test] [frontend-tests.composable-tests.comp.sync-test] + [frontend-tests.config-test] [frontend-tests.copy-as-svg-test] [frontend-tests.data.auth-test] [frontend-tests.data.bottom-bars-test] [frontend-tests.data.comments-batches-test] [frontend-tests.data.comments-filters-test] + [frontend-tests.data.common-test] [frontend-tests.data.dashboard-test] [frontend-tests.data.event-test] [frontend-tests.data.exports-assets-test] [frontend-tests.data.exports-files-test] [frontend-tests.data.fix-deleted-fonts-test] + [frontend-tests.data.fonts-test] [frontend-tests.data.imports-test] [frontend-tests.data.jobs-test] [frontend-tests.data.modal-test] @@ -41,6 +44,7 @@ [frontend-tests.data.workspace-context-menu-test] [frontend-tests.data.workspace-focus-test] [frontend-tests.data.workspace-interactions-test] + [frontend-tests.data.workspace-libraries-load-test] [frontend-tests.data.workspace-libraries-sync-test] [frontend-tests.data.workspace-mcp-test] [frontend-tests.data.workspace-media-test] @@ -52,6 +56,7 @@ [frontend-tests.data.workspace-tasks-test] [frontend-tests.data.workspace-texts-test] [frontend-tests.data.workspace-thumbnails-test] + [frontend-tests.data.workspace-thumbnails-wasm-test] [frontend-tests.data.workspace-undo-test] [frontend-tests.data.workspace-undo-translation-test] [frontend-tests.data.workspace-variant-rename-test] @@ -62,6 +67,7 @@ [frontend-tests.errors-test] [frontend-tests.fonts-test] [frontend-tests.helpers-shapes-test] + [frontend-tests.helpers.async-test] [frontend-tests.logic.bool-move-test] [frontend-tests.logic.comp-remove-swap-slots-test] [frontend-tests.logic.comp-watcher-test] @@ -81,6 +87,7 @@ [frontend-tests.logic.path-helpers-test] [frontend-tests.logic.path-lifecycle-test] [frontend-tests.logic.path-tools-test] + [frontend-tests.logic.scale-selected-test] [frontend-tests.logic.sidebar-transform-coalescing-test] [frontend-tests.logic.update-position-test] [frontend-tests.logic.wasm-modifiers-nil-id-test] @@ -104,13 +111,17 @@ [frontend-tests.plugins.parser-test] [frontend-tests.plugins.read-only-test] [frontend-tests.plugins.register-test] + [frontend-tests.plugins.runtime-unavailable-test] [frontend-tests.plugins.shape-bugfixes-test] + [frontend-tests.plugins.system-events-test] [frontend-tests.plugins.text-test] [frontend-tests.plugins.tokens-test] [frontend-tests.plugins.user-test] [frontend-tests.plugins.utils-test] [frontend-tests.plugins.value-objects-test] [frontend-tests.render-dimensions-test] + [frontend-tests.render-wasm.call-test] + [frontend-tests.render-wasm.modifiers-test] [frontend-tests.render-wasm.process-objects-test] [frontend-tests.render-wasm.serialization-test] [frontend-tests.render-wasm.shape-proxy-test] @@ -134,7 +145,10 @@ [frontend-tests.tokens.referenced-token-value-test] [frontend-tests.tokens.shadow-form-test] [frontend-tests.tokens.style-dictionary-test] + [frontend-tests.tokens.theme-create-modal-test] + [frontend-tests.tokens.token-application-test] [frontend-tests.tokens.token-errors-test] + [frontend-tests.tokens.token-group-test] [frontend-tests.tokens.token-parsing-test] [frontend-tests.tokens.token-path-test] [frontend-tests.tokens.token-pill-test] @@ -155,11 +169,13 @@ [frontend-tests.ui.drawarea-test] [frontend-tests.ui.ds-controls-numeric-input-test] [frontend-tests.ui.error-boundary-test] + [frontend-tests.ui.export-preview-test] [frontend-tests.ui.file-restored-modal-test] [frontend-tests.ui.gradient-handlers-test] [frontend-tests.ui.history-test] [frontend-tests.ui.inspect-geometry-test] [frontend-tests.ui.inspect-stroke-tokens-test] + [frontend-tests.ui.inspect-text-list-test] [frontend-tests.ui.jobs-progress-test] [frontend-tests.ui.layout-container-multiple-test] [frontend-tests.ui.mcp-authorize-test] @@ -177,6 +193,7 @@ [frontend-tests.ui.sortable-auto-scroll-test] [frontend-tests.ui.static-test] [frontend-tests.ui.stroke-menu-test] + [frontend-tests.ui.subscription-test] [frontend-tests.ui.team-container-test] [frontend-tests.ui.text-attrs-multiple-test] [frontend-tests.ui.viewport-top-bar-test] @@ -195,7 +212,9 @@ [frontend-tests.util.code-gen-markup-svg-wasm-test] [frontend-tests.util.dom.dnd-test] [frontend-tests.worker-snap-test] + [frontend-tests.worker.component-thumbnails-test] [frontend-tests.worker.exports-test] + [frontend-tests.worker.fonts-sync-test] [frontend-tests.worker.index-test] [frontend-tests.worker.messages-test] [frontend-tests.worker.trim-test] @@ -216,11 +235,13 @@ ['frontend-tests.basic-shapes-test 'frontend-tests.code-gen-style-test 'frontend-tests.composable-tests.comp.sync-test + 'frontend-tests.config-test 'frontend-tests.copy-as-svg-test 'frontend-tests.data.auth-test 'frontend-tests.data.bottom-bars-test 'frontend-tests.data.comments-batches-test 'frontend-tests.data.comments-filters-test + 'frontend-tests.data.common-test 'frontend-tests.data.dashboard-test 'frontend-tests.data.event-test 'frontend-tests.data.fix-deleted-fonts-test @@ -237,9 +258,12 @@ 'frontend-tests.data.store-test 'frontend-tests.data.exports-assets-test 'frontend-tests.data.exports-files-test + 'frontend-tests.data.fonts-test 'frontend-tests.data.imports-test 'frontend-tests.data.jobs-test + 'frontend-tests.worker.component-thumbnails-test 'frontend-tests.worker.exports-test + 'frontend-tests.worker.fonts-sync-test 'frontend-tests.worker.index-test 'frontend-tests.worker.messages-test 'frontend-tests.worker.trim-test @@ -254,6 +278,7 @@ 'frontend-tests.data.workspace-context-menu-test 'frontend-tests.data.workspace-focus-test 'frontend-tests.data.workspace-interactions-test + 'frontend-tests.data.workspace-libraries-load-test 'frontend-tests.data.workspace-libraries-sync-test 'frontend-tests.data.workspace-mcp-test 'frontend-tests.data.workspace-media-test @@ -264,6 +289,7 @@ 'frontend-tests.data.workspace-shortcuts-test 'frontend-tests.data.workspace-texts-test 'frontend-tests.data.workspace-thumbnails-test + 'frontend-tests.data.workspace-thumbnails-wasm-test 'frontend-tests.data.workspace-undo-test 'frontend-tests.data.workspace-undo-translation-test 'frontend-tests.data.workspace-variant-rename-test @@ -274,6 +300,7 @@ 'frontend-tests.errors-test 'frontend-tests.fonts-test 'frontend-tests.helpers-shapes-test + 'frontend-tests.helpers.async-test 'frontend-tests.logic.bool-move-test 'frontend-tests.logic.comp-remove-swap-slots-test 'frontend-tests.logic.comp-watcher-test @@ -293,6 +320,7 @@ 'frontend-tests.logic.path-lifecycle-test 'frontend-tests.logic.path-tools-test 'frontend-tests.logic.pasting-in-containers-test + 'frontend-tests.logic.scale-selected-test 'frontend-tests.main.refs-test 'frontend-tests.main-errors-test 'frontend-tests.logic.sidebar-transform-coalescing-test @@ -316,12 +344,16 @@ 'frontend-tests.plugins.parser-test 'frontend-tests.plugins.read-only-test 'frontend-tests.plugins.register-test + 'frontend-tests.plugins.runtime-unavailable-test 'frontend-tests.plugins.shape-bugfixes-test + 'frontend-tests.plugins.system-events-test 'frontend-tests.plugins.text-test 'frontend-tests.plugins.tokens-test 'frontend-tests.plugins.user-test 'frontend-tests.plugins.utils-test 'frontend-tests.plugins.value-objects-test + 'frontend-tests.render-wasm.call-test + 'frontend-tests.render-wasm.modifiers-test 'frontend-tests.render-wasm.process-objects-test 'frontend-tests.render-wasm.serialization-test 'frontend-tests.render-wasm.shape-proxy-test @@ -343,7 +375,10 @@ 'frontend-tests.tokens.referenced-token-value-test 'frontend-tests.tokens.shadow-form-test 'frontend-tests.tokens.style-dictionary-test + 'frontend-tests.tokens.theme-create-modal-test + 'frontend-tests.tokens.token-application-test 'frontend-tests.tokens.token-errors-test + 'frontend-tests.tokens.token-group-test 'frontend-tests.tokens.token-parsing-test 'frontend-tests.tokens.token-path-test 'frontend-tests.tokens.token-pill-test @@ -366,11 +401,13 @@ 'frontend-tests.ui.drawarea-test 'frontend-tests.ui.ds-controls-numeric-input-test 'frontend-tests.ui.error-boundary-test + 'frontend-tests.ui.export-preview-test 'frontend-tests.ui.file-restored-modal-test 'frontend-tests.ui.gradient-handlers-test 'frontend-tests.ui.history-test 'frontend-tests.ui.inspect-geometry-test 'frontend-tests.ui.inspect-stroke-tokens-test + 'frontend-tests.ui.inspect-text-list-test 'frontend-tests.ui.jobs-progress-test 'frontend-tests.ui.review-details-test 'frontend-tests.ui.review-markers-test @@ -390,6 +427,7 @@ 'frontend-tests.ui.sortable-auto-scroll-test 'frontend-tests.ui.static-test 'frontend-tests.ui.stroke-menu-test + 'frontend-tests.ui.subscription-test 'frontend-tests.ui.team-container-test 'frontend-tests.ui.text-attrs-multiple-test 'frontend-tests.ui.viewport-top-bar-test diff --git a/frontend/test/frontend_tests/tokens/import_export_test.cljs b/frontend/test/frontend_tests/tokens/import_export_test.cljs index 982c6a7cae..315cd1c851 100644 --- a/frontend/test/frontend_tests/tokens/import_export_test.cljs +++ b/frontend/test/frontend_tests/tokens/import_export_test.cljs @@ -9,8 +9,59 @@ [app.common.json :as json] [app.common.types.tokens-lib :as ctob] [app.main.data.workspace.tokens.import-export :as dwti] + [app.util.i18n :as i18n] [beicon.v2.core :as rx] - [cljs.test :as t :include-macros true])) + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private import-warnings-notification @#'dwti/import-warnings-notification) + +;; Echoes the key and its arguments, so the test sees what went into each +;; string. Both arities, like `tr`: a call site compiled for one of them +;; looks up that arity on the replacement. +(defn- tr-stub + ([key] key) + ([key & args] + (str key "(" (str/join ", " (map #(if (i18n/c? %) @% %) args)) ")"))) + +(t/deftest import-warnings-notification-test + (with-redefs [i18n/tr tr-stub] + (let [unknown-types {"core.motion.fast" "duration" + "core.motion.slow" "duration" + "core.odd<1>" "x"} + unsupported [{:property "$extensions" :path "core.color.old"} + {:property "$extensions" :path "core.color"} + {:property "$value" :path "core.color."}]] + + (t/testing "only unknown types" + (let [{:keys [content detail]} (import-warnings-notification unknown-types nil)] + (t/is (= "workspace.tokens.unknown-token-type-message" content)) + (t/is (str/includes? detail "workspace.tokens.unknown-token-type-section(duration, labels.warning-count(2))")) + (t/is (str/includes? detail "core.motion.fast")) + (t/is (str/includes? detail "core.motion.slow")) + (t/is (not (str/includes? detail "workspace.tokens.unsupported-token-data-section"))))) + + (t/testing "only unsupported data" + (let [{:keys [content detail]} (import-warnings-notification nil unsupported)] + (t/is (= "workspace.tokens.unsupported-token-data-message" content)) + (t/is (str/includes? detail "workspace.tokens.unsupported-token-data-section($extensions, labels.warning-count(2))")) + (t/is (str/includes? detail "workspace.tokens.unsupported-token-data-section($value, labels.warning-count(1))")) + (t/is (str/includes? detail "core.color.old")) + (t/is (not (str/includes? detail "workspace.tokens.unknown-token-type-section"))))) + + (t/testing "both list their sections, and the unsupported data picks the message" + (let [{:keys [content detail]} (import-warnings-notification unknown-types unsupported)] + (t/is (= "workspace.tokens.unsupported-token-data-message" content)) + (t/is (str/includes? detail "workspace.tokens.unknown-token-type-section(duration, labels.warning-count(2))")) + (t/is (str/includes? detail "workspace.tokens.unsupported-token-data-section($extensions, labels.warning-count(2))")))) + + (t/testing "types and paths from the file are escaped" + (let [{:keys [detail]} (import-warnings-notification unknown-types unsupported)] + (t/is (str/includes? detail "workspace.tokens.unknown-token-type-section(x<y>, labels.warning-count(1))")) + (t/is (str/includes? detail "core.odd<1>")) + (t/is (str/includes? detail "core.color.<odd>")) + (t/is (not (str/includes? detail ""))) + (t/is (not (str/includes? detail "x")))))))) (t/deftest import-file-stream-test (t/async @@ -104,3 +155,24 @@ color.value tries to reference missing, which is not defined."))) (t/is (instance? ctob/TokensLib tokens-lib)) (t/is (= "{missing}" (:value (ctob/get-token-by-name tokens-lib "core" "color")))) (done)))))))) + +(t/deftest import-dtcg-2025-json-test + (t/async + done + (t/testing "imports DTCG 2025.10 object values" + (let [json (-> {"core" {"space" {"$type" "dimension" + "sm" {"$value" {"value" 4 "unit" "px"}}} + "color" {"$type" "color" + "brand" {"$value" {"colorSpace" "srgb" "components" [1 0 0]} + "$extensions" {"com.figma" {"scopes" ["ALL"]}}}}} + "$metadata" {"tokenSetOrder" ["core"]}} + (json/encode {:type :json-verbose}))] + (->> (rx/of json) + (dwti/import-file-stream "") + (rx/subs! (fn [tokens-lib] + (t/is (= "4px" (:value (ctob/get-token-by-name tokens-lib "core" "space.sm")))) + (t/is (= "srgb(1, 0, 0)" (:value (ctob/get-token-by-name tokens-lib "core" "color.brand")))) + (done)) + (fn [err] + (t/do-report {:type :error :message "Stream error" :actual err}) + (done)))))))) diff --git a/frontend/test/frontend_tests/tokens/logic/token_actions_test.cljs b/frontend/test/frontend_tests/tokens/logic/token_actions_test.cljs index fdd14413f2..e23c51b7ff 100644 --- a/frontend/test/frontend_tests/tokens/logic/token_actions_test.cljs +++ b/frontend/test/frontend_tests/tokens/logic/token_actions_test.cljs @@ -1541,6 +1541,177 @@ (t/testing "regular shape doesn't get spacing attributes" (t/is (nil? (:applied-tokens rect-regular'))))))))))) +(def ^:private design-tab-tokens + [{:name "color.primary" :value "red" :type :color} + {:name "opacity.half" :value "0.5" :type :opacity} + {:name "dimensions.lg" :value "100" :type :dimensions} + {:name "sizing.min" :value "50" :type :sizing}]) + +(defn- setup-file-with-design-tab-tokens + [& {:as shapes}] + (update-in (setup-file-with-tokens shapes) + [:data :tokens-lib] + #(reduce (fn [lib token] + (ctob/add-token lib (cthi/id :set-a) (ctob/make-token token))) + % + design-tab-tokens))) + +(t/deftest test-toggle-token-expands-group + (t/testing "toggling a token on a group with expand-with-children applies it to the children" + (t/async + done + (let [file (-> (setup-file-with-design-tab-tokens) + (ctho/add-group :group-1) + (cths/add-sample-shape :rect-in-group-1 :parent-label :group-1) + (cths/add-sample-shape :rect-in-group-2 :parent-label :group-1)) + store (ths/setup-store file) + events [(dwta/toggle-token {:token (toht/get-token file "color.primary") + :shape-ids [(cthi/id :group-1)] + :expand-with-children true})]] + (tohs/run-store-async + store done events + (fn [new-state] + (let [file' (ths/get-file-from-state new-state) + group-1' (cths/get-shape file' :group-1) + rect-in-group-1' (cths/get-shape file' :rect-in-group-1) + rect-in-group-2' (cths/get-shape file' :rect-in-group-2)] + (t/is (= "color.primary" (:fill (:applied-tokens rect-in-group-1')))) + (t/is (= "color.primary" (:fill (:applied-tokens rect-in-group-2')))) + (t/is (= "#ff0000" (get-in rect-in-group-1' [:fills 0 :fill-color]))) + (t/is (nil? (:applied-tokens group-1')))))))))) + +(t/deftest test-apply-token-from-input-fill-and-stroke + (t/testing "the fill and stroke rows apply a color token to their attribute only" + (t/async + done + (let [file (setup-file-with-design-tab-tokens) + store (ths/setup-store file) + token (toht/get-token file "color.primary") + events [(dwta/apply-token-from-input {:token token + :attrs #{:fill} + :shape-ids [(cthi/id :rect-1)] + :expand-with-children true}) + (dwta/apply-token-from-input {:token token + :attrs #{:stroke-color} + :shape-ids [(cthi/id :rect-2)] + :expand-with-children true})]] + (tohs/run-store-async + store done events + (fn [new-state] + (let [file' (ths/get-file-from-state new-state) + rect-1' (cths/get-shape file' :rect-1) + rect-2' (cths/get-shape file' :rect-2)] + (t/testing "fill row" + (t/is (= {:fill "color.primary"} (:applied-tokens rect-1'))) + (t/is (= "#ff0000" (get-in rect-1' [:fills 0 :fill-color])))) + (t/testing "stroke row" + (t/is (= {:stroke-color "color.primary"} (:applied-tokens rect-2'))) + (t/is (= "#ff0000" (get-in rect-2' [:strokes 0 :stroke-color]))))))))))) + +(t/deftest test-apply-token-from-input-keeps-applied-token + (t/testing "choosing the token a shape already has keeps it applied, unlike toggling" + (t/async + done + (let [file (-> (setup-file-with-design-tab-tokens) + (toht/apply-token-to-shape :rect-1 "color.primary" #{:fill})) + store (ths/setup-store file) + events [(dwta/apply-token-from-input {:token (toht/get-token file "color.primary") + :attrs #{:fill} + :shape-ids [(cthi/id :rect-1)] + :expand-with-children true})]] + (tohs/run-store-async + store done events + (fn [new-state] + (let [file' (ths/get-file-from-state new-state) + rect-1' (cths/get-shape file' :rect-1)] + (t/is (= "color.primary" (:fill (:applied-tokens rect-1')))) + (t/is (= "#ff0000" (get-in rect-1' [:fills 0 :fill-color])))))))))) + +(t/deftest test-apply-token-from-input-measures-and-opacity + (t/testing "the opacity, position and size fields apply the token to their attribute only" + (t/async + done + (let [file (setup-file-with-design-tab-tokens) + store (ths/setup-store file) + rect-3 (cths/get-shape file :rect-3) + events [(dwta/apply-token-from-input {:token (toht/get-token file "opacity.half") + :attrs #{:opacity} + :shape-ids [(cthi/id :rect-1)]}) + (dwta/apply-token-from-input {:token (toht/get-token file "dimensions.lg") + :attrs #{:x} + :shape-ids [(cthi/id :rect-2)]}) + (dwta/apply-token-from-input {:token (toht/get-token file "dimensions.lg") + :attrs #{:width} + :shape-ids [(cthi/id :rect-3)]})]] + (tohs/run-store-async + store done events + (fn [new-state] + (let [file' (ths/get-file-from-state new-state) + rect-1' (cths/get-shape file' :rect-1) + rect-2' (cths/get-shape file' :rect-2) + rect-3' (cths/get-shape file' :rect-3)] + (t/testing "opacity" + (t/is (= {:opacity "opacity.half"} (:applied-tokens rect-1'))) + (t/is (= 0.5 (:opacity rect-1')))) + (t/testing "position" + (t/is (= {:x "dimensions.lg"} (:applied-tokens rect-2'))) + (t/is (= 100 (:x rect-2')))) + (t/testing "size" + (t/is (= {:width "dimensions.lg"} (:applied-tokens rect-3'))) + (t/is (= 100 (:width rect-3'))) + (t/is (= (:height rect-3) (:height rect-3'))))))))))) + +(t/deftest test-apply-token-from-input-layout-sizing-limit + (t/testing "the min width field of a flex child sets that limit only" + (t/async + done + (let [file (-> (setup-file-with-design-tab-tokens) + (ctho/add-frame-with-child :frame-layout :rect-in-layout + {:frame-params {:layout :flex}})) + store (ths/setup-store file) + events [(dwta/apply-token-from-input {:token (toht/get-token file "sizing.min") + :attrs #{:layout-item-min-w} + :shape-ids [(cthi/id :rect-in-layout)]})]] + (tohs/run-store-async + store done events + (fn [new-state] + (let [file' (ths/get-file-from-state new-state) + rect-in-layout' (cths/get-shape file' :rect-in-layout)] + (t/is (= {:layout-item-min-w "sizing.min"} (:applied-tokens rect-in-layout'))) + (t/is (= 50 (:layout-item-min-w rect-in-layout'))) + (t/is (nil? (:layout-item-max-w rect-in-layout')))))))))) + +(t/deftest test-apply-token-on-color-selected + (t/testing "a color token chosen in the selection colors goes on each shape's fill or stroke" + (t/async + done + (let [file (setup-file-with-design-tab-tokens) + store (ths/setup-store file) + operations [{:prop :fill :shape-id (cthi/id :rect-1) :index 0} + {:prop :stroke :shape-id (cthi/id :rect-2) :index 0} + {:prop :content :shape-id (cthi/id :text-1) :index 0} + {:prop :shadow :shape-id (cthi/id :rect-3) :index 0}] + events [(dwta/apply-token-on-color-selected operations + (toht/get-token file "color.primary"))]] + (tohs/run-store-async + store done events + (fn [new-state] + (let [file' (ths/get-file-from-state new-state) + rect-1' (cths/get-shape file' :rect-1) + rect-2' (cths/get-shape file' :rect-2) + rect-3' (cths/get-shape file' :rect-3) + text-1' (cths/get-shape file' :text-1)] + (t/testing "fill" + (t/is (= {:fill "color.primary"} (:applied-tokens rect-1'))) + (t/is (= "#ff0000" (get-in rect-1' [:fills 0 :fill-color])))) + (t/testing "stroke" + (t/is (= {:stroke-color "color.primary"} (:applied-tokens rect-2'))) + (t/is (= "#ff0000" (get-in rect-2' [:strokes 0 :stroke-color])))) + (t/testing "text goes on the fill" + (t/is (= {:fill "color.primary"} (:applied-tokens text-1')))) + (t/testing "shadows take no color token" + (t/is (nil? (:applied-tokens rect-3'))))))))))) + (t/deftest test-detach-styles-color (t/testing "applying a color token to a shape with color styles should detach the styles" (t/async diff --git a/frontend/test/frontend_tests/tokens/logic/tokens_source_test.cljs b/frontend/test/frontend_tests/tokens/logic/tokens_source_test.cljs index 6813fd1da4..08b77f5073 100644 --- a/frontend/test/frontend_tests/tokens/logic/tokens_source_test.cljs +++ b/frontend/test/frontend_tests/tokens/logic/tokens_source_test.cljs @@ -80,9 +80,8 @@ "Set `library` as tokens source and wait for the canvas to use it." [store library] (ptk/emit! store (dwtl/set-tokens-source (:id library))) - (await (async/wait-for #(= "#00ff00" (fill-color store)) - "the fill uses the library value of `red`" - {:max-ticks 200}))) + (await (async/wait-for-store store #(= "#00ff00" (fill-color store)) + "the fill uses the library value of `red`"))) (t/deftest ^:async switching-tokens-source-updates-the-canvas (let [library (setup-library) @@ -92,9 +91,8 @@ (t/is (= "#7b2222" (fill-color store))) (ptk/emit! store (dwtl/set-tokens-source (:id library))) - (await (async/wait-for #(= "#00ff00" (fill-color store)) - "the fill uses the library value of `red`" - {:max-ticks 200})) + (await (async/wait-for-store store #(= "#00ff00" (fill-color store)) + "the fill uses the library value of `red`")) (t/is (= "#00ff00" (fill-color store)) "Setting the library as tokens source resolves `red` from the library") @@ -102,9 +100,8 @@ (await (async/settle)) (t/is (contains? (active-set-ids store) (cthi/id :local-set)) "Going back to the local tokens keeps the local set active") - (await (async/wait-for #(= "#7b2222" (fill-color store)) - "the fill uses the local value of `red` again" - {:max-ticks 200})) + (await (async/wait-for-store store #(= "#7b2222" (fill-color store)) + "the fill uses the local value of `red` again")) (t/is (= "#7b2222" (fill-color store)) "Going back to the local tokens resolves `red` from the file again") @@ -116,16 +113,26 @@ store (ths/setup-store file {:libraries [library]})] (await (use-library-tokens store library)) - (await - (mock/with-mocks* - {rp/cmd! (mock/stub (fn [_ _] (rx/observe-on :async (rx/of []))))} - ;; The same events the libraries dialog emits on unlink - (ptk/emit! store - (dwl/unlink-file-from-library (:id file) (:id library)) - (dwtl/set-tokens-source (:id file))) - (await (async/wait-for #(= "#7b2222" (fill-color store)) - "the fill uses the local value of `red` again" - {:max-ticks 200})))) + (let [calls (atom []) + calls$ (rx/subject)] + (await + (mock/with-mocks* + {rp/cmd! (mock/stub (fn [cmd _] + (swap! calls conj cmd) + (rx/push! calls$ cmd) + (rx/observe-on :async (rx/of []))))} + ;; The same events the libraries dialog emits on unlink + (ptk/emit! store + (dwl/unlink-file-from-library (:id file) (:id library)) + (dwtl/set-tokens-source (:id file))) + ;; The unlink event dissocs the library before its watch runs, so the + ;; store cannot tell when the chain ended. The chain's second call + ;; (`get-file-libraries`) is the proof that it ran inside the mock; + ;; wait for it, and keep the store signal for the token resolution. + (await (async/wait-for #(and (= "#7b2222" (fill-color store)) + (some #{:get-file-libraries} @calls)) + "the unlink flow completes and the local tokens resolve" + {:signal (rx/merge calls$ (ptk/input-stream store))}))))) (t/is (nil? (tokens-source store))) (t/is (contains? (active-set-ids store) (cthi/id :local-set)) diff --git a/frontend/test/frontend_tests/tokens/theme_create_modal_test.cljs b/frontend/test/frontend_tests/tokens/theme_create_modal_test.cljs new file mode 100644 index 0000000000..f87cde47e3 --- /dev/null +++ b/frontend/test/frontend_tests/tokens/theme_create_modal_test.cljs @@ -0,0 +1,79 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.tokens.theme-create-modal-test + "The sets list of the theme editor: which sets it shows as ticked, and + what ticking a set or a group does to the theme." + (:require + [app.common.files.tokens :as cfo] + [app.common.types.tokens-lib :as ctob] + [app.common.types.tokens-status :as ctos] + [app.main.ui.workspace.tokens.themes.create-modal :as create-modal] + [cljs.test :as t :include-macros true])) + +(def ^:private toggle-set @#'create-modal/toggle-set) +(def ^:private toggle-set-group @#'create-modal/toggle-set-group) +(def ^:private make-lib-with-theme @#'create-modal/make-lib-with-theme) +(def ^:private make-status-with-theme @#'create-modal/make-status-with-theme) + +(defn- sample-lib + "A library with the set `light` and the group `brand` holding the sets + `a` and `b` and the subgroup `x` with the set `c`." + [] + (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set :name "light")) + (ctob/add-set (ctob/make-token-set :name "brand/a")) + (ctob/add-set (ctob/make-token-set :name "brand/b")) + (ctob/add-set (ctob/make-token-set :name "brand/x/c")))) + +(defn- sample-theme + [& set-names] + (ctob/make-token-theme :name "Light" :sets (set set-names))) + +(defn- set-id + [lib set-name] + (ctob/get-id (ctob/get-set-by-name lib set-name))) + +(t/deftest toggle-set-ticks-and-unticks-by-id + (let [lib (sample-lib) + id (set-id lib "light") + ticked (toggle-set (sample-theme) lib id)] + (t/is (= #{"light"} (:sets ticked))) + (t/is (= #{} (:sets (toggle-set ticked lib id)))))) + +(t/deftest toggle-set-keeps-the-other-sets + (let [lib (sample-lib)] + (t/is (= #{"brand/a" "light"} + (:sets (toggle-set (sample-theme "brand/a") lib (set-id lib "light"))))))) + +(t/deftest toggle-set-group-ticks-all-its-sets-when-none-is-ticked + (let [sets (ctob/get-sets (sample-lib))] + (t/is (= #{"light" "brand/a" "brand/b" "brand/x/c"} + (:sets (toggle-set-group (sample-theme "light") sets ["brand"])))))) + +(t/deftest toggle-set-group-unticks-all-its-sets-when-some-are-ticked + (let [sets (ctob/get-sets (sample-lib))] + (t/is (= #{"light"} + (:sets (toggle-set-group (sample-theme "light" "brand/a") sets ["brand"])))))) + +(t/deftest toggle-set-group-unticks-all-its-sets-when-only-a-nested-one-is-ticked + (let [sets (ctob/get-sets (sample-lib))] + (t/is (= #{"light"} + (:sets (toggle-set-group (sample-theme "light" "brand/x/c") sets ["brand"])))))) + +(t/deftest toggle-set-group-unticks-all-its-sets-when-all-are-ticked + (let [sets (ctob/get-sets (sample-lib))] + (t/is (= #{} + (:sets (toggle-set-group (sample-theme "brand/a" "brand/b" "brand/x/c") sets ["brand"])))))) + +(t/deftest the-editor-shows-the-sets-of-the-theme-as-ticked + (let [lib (sample-lib) + theme (sample-theme "brand/a") + lib' (make-lib-with-theme theme (ctob/get-sets lib)) + status (make-status-with-theme lib' theme)] + (t/is (ctos/set-active? status (set-id lib "brand/a"))) + (t/is (not (ctos/set-active? status (set-id lib "light")))) + (t/is (= :partial (cfo/sets-at-path-all-active? status lib' ["brand"]))))) diff --git a/frontend/test/frontend_tests/tokens/token_application_test.cljs b/frontend/test/frontend_tests/tokens/token_application_test.cljs new file mode 100644 index 0000000000..1e23f373af --- /dev/null +++ b/frontend/test/frontend_tests/tokens/token_application_test.cljs @@ -0,0 +1,114 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.tokens.token-application-test + (:require + [app.common.test-helpers.compositions :as tho] + [app.common.test-helpers.files :as thf] + [app.common.test-helpers.ids-map :as thi] + [app.common.test-helpers.shapes :as ths] + [app.common.types.token :as cto] + [app.common.types.tokens-lib :as ctob] + [app.main.data.workspace.tokens.application :as dwta] + [cljs.test :as t :include-macros true])) + +(t/use-fixtures :each thi/test-fixture) + +(def ^:private target-shapes @#'dwta/target-shapes) +(def ^:private toggle-action @#'dwta/toggle-action) + +(def ^:private radius + (ctob/make-token :name "radius.md" :type :border-radius :value "8")) + +(def ^:private color + (ctob/make-token :name "brand" :type :color :value "#ff0000")) + +(def ^:private spacing + (ctob/make-token :name "spacing.md" :type :spacing :value "16")) + +(def ^:private stroke-width + (ctob/make-token :name "stroke.md" :type :stroke-width :value "2")) + +(defn- with-token + [shape token attributes] + (cto/apply-token-to-shape {:shape shape :token token :attributes attributes})) + +(defn- toggle + "What toggling the token decides: remove it or apply it, and on which + attributes. How the shapes then change is covered by + `token-actions-test`." + [token attrs shapes] + (select-keys (toggle-action token attrs shapes) [:action :attributes])) + +(t/deftest toggle-from-the-pill + (t/testing "applies the token when no selected shape has it" + (t/is (= {:action :apply :attributes cto/border-radius-keys} + (toggle radius nil [(ths/sample-shape :rect1) + (ths/sample-shape :rect2)])))) + + (t/testing "removes the token when any selected shape has it on any attribute" + (t/is (= {:action :unapply :attributes cto/border-radius-keys} + (toggle radius nil [(ths/sample-shape :rect1) + (with-token (ths/sample-shape :rect2) radius #{:r1})])))) + + (t/testing "another token on the same attributes does not count" + (let [other (ctob/make-token :name "radius.lg" :type :border-radius :value "16") + shape (with-token (ths/sample-shape :rect1) other cto/border-radius-keys)] + (t/is (= :apply (:action (toggle radius nil [shape])))))) + + (t/testing "a color token goes on the fill but comes off the stroke too" + (t/is (= {:action :apply :attributes #{:fill}} + (toggle color nil [(ths/sample-shape :rect1)]))) + (t/is (= {:action :unapply :attributes cto/color-keys} + (toggle color nil [(with-token (ths/sample-shape :rect1) color #{:stroke-color})]))))) + +(t/deftest toggle-on-chosen-attributes + (t/testing "a partial application is completed, not removed" + (let [shape (with-token (ths/sample-shape :rect1) stroke-width + #{:stroke-width-right :stroke-width-bottom})] + (t/is (= {:action :apply :attributes cto/per-side-stroke-width-keys} + (toggle stroke-width cto/per-side-stroke-width-keys [shape]))))) + + (t/testing "a shape without the token completes the others" + (let [shapes [(with-token (ths/sample-shape :rect1) radius cto/border-radius-keys) + (ths/sample-shape :rect2)]] + (t/is (= :apply (:action (toggle radius cto/border-radius-keys shapes)))))) + + (t/testing "removes the token once every shape has it on every attribute" + (t/is (= {:action :unapply :attributes cto/border-radius-keys} + (toggle radius + cto/border-radius-keys + [(with-token (ths/sample-shape :rect1) radius cto/border-radius-keys) + (with-token (ths/sample-shape :rect2) radius cto/border-radius-keys)]))))) + +(t/deftest toggle-spacing + (t/testing "without attributes a spacing token is split between layouts and their children" + (t/is (= {:action :apply-spacing} + (toggle spacing nil [(ths/sample-shape :rect1)])))) + + (t/testing "on chosen attributes it goes on those" + (t/is (= {:action :apply :attributes #{:m1}} + (toggle spacing #{:m1} [(ths/sample-shape :rect1)]))))) + +(defn- objects-with-group + [] + (-> (thf/sample-file :file1 :page-label :page1) + (tho/add-group :group1) + (ths/add-sample-shape :rect1 :parent-label :group1) + (ths/add-sample-shape :rect2 :parent-label :group1) + (tho/add-rect :rect3) + (thf/current-page) + (:objects))) + +(t/deftest target-shapes-of-the-selection + (let [objects (objects-with-group) + ids [(thi/id :group1) (thi/id :rect3)]] + (t/testing "a selected group stands for its children" + (t/is (= [(thi/id :rect1) (thi/id :rect2) (thi/id :rect3)] + (map :id (target-shapes objects ids true))))) + + (t/testing "without expanding, the group stays selected" + (t/is (= ids (map :id (target-shapes objects ids false))))))) diff --git a/frontend/test/frontend_tests/tokens/token_group_test.cljs b/frontend/test/frontend_tests/tokens/token_group_test.cljs new file mode 100644 index 0000000000..ce92049110 --- /dev/null +++ b/frontend/test/frontend_tests/tokens/token_group_test.cljs @@ -0,0 +1,69 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.tokens.token-group-test + (:require + [app.common.test-helpers.ids-map :as thi] + [app.common.test-helpers.shapes :as ths] + [app.common.types.tokens-lib :as ctob] + [app.main.data.workspace.tokens.library-edit :as dwtl] + [app.main.ui.workspace.tokens.management.group :as wtg] + [cljs.test :as t :include-macros true])) + +(t/use-fixtures :each thi/test-fixture) + +(def ^:private type-unfolded? @#'wtg/type-unfolded?) +(def ^:private editing-text? @#'wtg/editing-text?) +(def ^:private pill-click-action @#'wtg/pill-click-action) +(def ^:private unfolded-token-types @#'dwtl/make-unfolded-token-types-state) + +(def ^:private radius + (ctob/make-token :name "radius.md" :type :border-radius :value "8")) + +(def ^:private number + (ctob/make-token :name "scale.md" :type :number :value "2")) + +(t/deftest section-unfolded + (let [unfolded (unfolded-token-types (thi/new-id! :file1) (thi/new-id! :set1) #{:color})] + (t/testing "open in the file and set it was opened in" + (t/is (type-unfolded? unfolded (thi/id :file1) (thi/id :set1) :color))) + + (t/testing "closed in another set or file" + (t/is (not (type-unfolded? unfolded (thi/id :file1) (thi/new-id! :set2) :color))) + (t/is (not (type-unfolded? unfolded (thi/new-id! :file2) (thi/id :set1) :color)))) + + (t/testing "other types stay closed" + (t/is (not (type-unfolded? unfolded (thi/id :file1) (thi/id :set1) :spacing)))))) + +(t/deftest text-edition + (let [text (ths/sample-shape :text1 :type :text) + rect (ths/sample-shape :rect1) + objects {(:id text) text (:id rect) rect}] + (t/testing "nothing is being edited" + (t/is (not (editing-text? nil nil objects)))) + + (t/testing "a text is being edited" + (t/is (editing-text? nil (:id text) objects)) + (t/is (editing-text? {(:id text) :editor-state} nil objects))) + + (t/testing "editing another shape isn't text edition" + (t/is (not (editing-text? nil (:id rect) objects)))))) + +(t/deftest pill-click + (let [selection [(ths/sample-shape :rect1)]] + (t/testing "applies or removes the token on the selection" + (t/is (= :toggle (pill-click-action radius selection false)))) + + (t/testing "warns while a text is being edited" + (t/is (= :warn-text-edition (pill-click-action radius selection true)))) + + (t/testing "does nothing without a selection" + (t/is (nil? (pill-click-action radius [] false))) + (t/is (nil? (pill-click-action radius [] true)))) + + (t/testing "number tokens can't be applied with a click" + (t/is (nil? (pill-click-action number selection false))) + (t/is (nil? (pill-click-action number selection true)))))) diff --git a/frontend/test/frontend_tests/ui/export_preview_test.cljs b/frontend/test/frontend_tests/ui/export_preview_test.cljs new file mode 100644 index 0000000000..3fdfe65652 --- /dev/null +++ b/frontend/test/frontend_tests/ui/export_preview_test.cljs @@ -0,0 +1,296 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.ui.export-preview-test + (:require + ["jsdom" :refer [JSDOM]] + ["react" :as react] + ["react-dom/client" :as rdc] + ["react-dom/server" :as rds] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [app.main.data.exports.wasm :as wasm.exports] + [app.main.store :as st] + [app.main.ui.exports.preview :as preview] + [app.main.ui.hooks :as hooks] + [app.main.ui.inspect.exports :as inspect-exports] + [app.main.ui.workspace.sidebar.options.menus.exports :as exports] + [app.render-wasm.api :as wasm.api] + [app.util.globals :as globals] + [app.util.storage :as storage] + [app.util.webapi :as wapi] + [cljs.test :as t :include-macros true] + [cuerdas.core :as str] + [frontend-tests.helpers.mock :as mock] + [rumext.v2 :as mf])) + +(defn- render-menu + [shapes presets] + (with-redefs [hooks/use-portal-container (fn ([] nil) ([_] nil))] + (rds/renderToStaticMarkup + (mf/element exports/exports-menu* + #js {:ids (mapv :id shapes) + :shapes shapes + :type (if (= 1 (count shapes)) :rect :multiple) + :values {:exports presets} + :fileId (uuid/next) + :pageId (uuid/next)})))) + +(defn- ^:async with-mounted-menu + [presets test-fn] + (let [dom (JSDOM. "
        ") + window (.-window dom) + document (.-document window) + old-window (.-window js/globalThis) + old-document (.-document js/globalThis) + old-act (.-IS_REACT_ACT_ENVIRONMENT js/globalThis) + old-state @st/state + file-id (uuid/next) + page-id (uuid/next) + shape (cts/setup-shape {:type :rect :width 100 :height 50 :exports presets}) + container (.getElementById document "root")] + (set! (.-window js/globalThis) window) + (set! (.-document js/globalThis) document) + (set! (.-IS_REACT_ACT_ENVIRONMENT js/globalThis) true) + (reset! st/state {:current-file-id file-id :current-page-id page-id + :files {file-id {:data {:pages-index {page-id {:objects {(:id shape) shape}}}}}} + :viewer {:pages {page-id {:objects {(:id shape) shape}}}}}) + (try + (await + (mock/with-mocks* {globals/document document globals/window window + storage/user (storage/create-storage nil "export-preview-test")} + (let [root (rdc/createRoot container) + render-selection + (fn [section] + (.render root + (case section + :design (mf/element exports/exports-menu* + #js {:ids [(:id shape)] :shapes [shape] :type :rect + :values {:exports presets} :fileId file-id :pageId page-id}) + :preview (mf/element preview/export-preview* + #js {:objectId (:id shape) :exports presets}) + (:inspect :viewer) (mf/html + [:& inspect-exports/exports + {:shapes [shape] :type :rect :file-id file-id :page-id page-id + :from (if (= section :viewer) :viewer :workspace)}]) + nil)))] + (try + (await (react/act #(render-selection :design))) + (await (test-fn container render-selection)) + (finally + (await (react/act #(.unmount root)))))))) + (finally + (reset! st/state old-state) + (.close window) + (set! (.-window js/globalThis) old-window) + (set! (.-document js/globalThis) old-document) + (set! (.-IS_REACT_ACT_ENVIRONMENT js/globalThis) old-act))))) + +(t/deftest ^:async reopening-export-keeps-the-last-preview-expansion + (await + (with-mounted-menu + [{:type :png :scale 1 :suffix ""}] + (^:async fn [container _render-selection] + (let [preview-button #(.querySelector container "button[aria-controls^='export-preview-']") + export-button (.querySelector container "button")] + (await (react/act #(.click (preview-button)))) + (t/is (= "true" (.getAttribute (preview-button) "aria-expanded"))) + (await (react/act #(.click export-button))) + (t/is (nil? (preview-button))) + (await (react/act #(.click export-button))) + (t/is (= "true" (.getAttribute (preview-button) "aria-expanded"))) + (await (react/act #(.click (preview-button)))) + (await (react/act #(.click export-button))) + (await (react/act #(.click export-button))) + (t/is (= "false" (.getAttribute (preview-button) "aria-expanded")))))))) + +(t/deftest ^:async preview-preset-uses-the-penpot-select + (await + (with-mounted-menu + [{:type :png :scale 1 :suffix ""} {:type :pdf :scale 1 :suffix ""}] + (^:async fn [container _render-selection] + (await (react/act #(.click (.querySelector container "button[aria-controls^='export-preview-']")))) + (let [panel (.querySelector container "div[id^='export-preview-']") + select (.querySelector panel "button[role='combobox']")] + (t/is (some? select)) + (t/is (nil? (.querySelector panel "select"))) + (when select + (await (react/act #(.click select))) + (let [option (aget (.querySelectorAll panel "[role='option']") 1)] + (await (react/act #(.click option))) + (t/is (str/includes? (.-textContent panel) "100 × 50 · PDF"))))))))) + +(t/deftest ^:async reselecting-a-layer-keeps-the-preview-expansion + (await + (with-mounted-menu + [{:type :png :scale 1 :suffix ""}] + (^:async fn [container render-selection] + (let [preview-button #(.querySelector container "button[aria-controls^='export-preview-']")] + (await (react/act #(.click (preview-button)))) + (await (react/act #(render-selection nil))) + (t/is (nil? (preview-button))) + (await (react/act #(render-selection :design))) + (t/is (= "true" (.getAttribute (preview-button) "aria-expanded"))) + (await (react/act #(.click (preview-button)))) + (await (react/act #(render-selection nil))) + (await (react/act #(render-selection :design))) + (t/is (= "false" (.getAttribute (preview-button) "aria-expanded")))))))) + +(t/deftest ^:async inspect-shows-the-preview-and-keeps-its-expansion-between-tabs + (await + (with-mounted-menu + [{:type :png :scale 1 :suffix ""}] + (^:async fn [container render-selection] + (let [preview-button #(.querySelector container "button[aria-controls^='export-preview-']")] + (await (react/act #(.click (preview-button)))) + (await (react/act #(render-selection :inspect))) + (t/is (some? (preview-button))) + (when (preview-button) + (t/is (= "true" (.getAttribute (preview-button) "aria-expanded"))) + (t/is (some? (.querySelector container "svg[id^='screenshot-']"))) + (await (react/act #(.click (preview-button)))) + (await (react/act #(render-selection :design))) + (t/is (= "false" (.getAttribute (preview-button) "aria-expanded"))))))))) + +(t/deftest ^:async viewer-inspect-previews-the-viewer-page-without-workspace-objects + (await + (with-mounted-menu + [{:type :png :scale 1 :suffix ""}] + (^:async fn [container render-selection] + (await (react/act #(swap! st/state assoc :files {} :features #{"render-wasm/v1"}))) + (await (react/act #(render-selection :viewer))) + (let [button (.querySelector container "button[aria-controls^='export-preview-']")] + (t/is (some? button)) + (when button + (await (react/act #(.click button))) + (t/is (some? (.querySelector container "svg[id^='screenshot-']"))) + (t/is (str/includes? (.-textContent container) "100 × 50 · PNG")))))))) + +(t/deftest single-shape-with-export-offers-a-preview + (let [presets [{:type :png :scale 1 :suffix ""}] + shape (cts/setup-shape {:type :rect :exports presets}) + markup (render-menu [shape] presets)] + (t/is (str/includes? markup "aria-controls=\"export-preview-")) + (t/is (str/includes? markup "aria-expanded=\"false\"")))) + +(t/deftest preview-requires-an-export-preset + (let [shape (cts/setup-shape {:type :rect})] + (t/is (not (str/includes? (render-menu [shape] []) + "aria-controls=\"export-preview-"))))) + +(t/deftest multiple-selection-does-not-imply-a-combined-export-preview + (let [presets [{:type :png :scale 1 :suffix ""}] + shapes [(cts/setup-shape {:type :rect :exports presets}) + (cts/setup-shape {:type :rect :exports presets})]] + (t/is (not (str/includes? (render-menu shapes presets) + "aria-controls=\"export-preview-"))))) + +(t/deftest preview-keeps-the-export-size-while-bounding-the-render-size + (doseq [type [:png :jpeg :webp]] + (let [settings (preview/preview-settings {:width 1200 :height 800} + {:type type :scale 4})] + (t/is (= {:type type :scale 0.2 :width 4800 :height 3200} settings))))) + +(t/deftest preview-respects-small-raster-export-scales + (t/is (= {:type :png :scale 0.5 :width 50 :height 25} + (preview/preview-settings {:width 100 :height 50} {:type :png :scale 0.5})))) + +(t/deftest vector-previews-ignore-a-raster-preset-scale + (doseq [type [:svg :pdf]] + (t/is (= {:type :png :scale 1 :width 100 :height 50} + (preview/preview-settings {:width 100 :height 50} {:type type :scale 4}))))) + +(t/deftest empty-export-bounds-do-not-create-an-infinite-render-scale + (t/is (= {:type :png :scale 1 :width 0 :height 0} + (preview/preview-settings {:width 0 :height 0} {:type :png :scale 1})))) + +(t/deftest classic-preview-matches-the-rounded-svg-export-dimensions + (let [shape (cts/setup-shape {:type :frame :x 282 :y 300 + :width 400.000015 :height 260.000015}) + bounds (preview/preview-bounds {(:id shape) shape} shape false) + settings (preview/preview-settings bounds {:type :png :scale 1})] + (t/is (= 400 (:width settings))) + (t/is (= 260 (:height settings))))) + +(t/deftest wasm-preview-uses-native-bounds-without-rounding-away-real-fractions + (let [shape (cts/setup-shape {:type :frame :width 400 :height 260})] + (with-redefs [wasm.api/initialized? (constantly true) + wasm.api/get-shape-extrect (fn [_] {:width 400.5 :height 260.25})] + (let [bounds (preview/preview-bounds {(:id shape) shape} shape true) + settings (preview/preview-settings bounds {:type :png :scale 2})] + (t/is (= 801 (:width settings))) + (t/is (= 521 (:height settings))))))) + + +(defn- wait-for-preview + [] + (react/act #(js/Promise. (fn [resolve] (js/setTimeout resolve 250))))) + +(defn- ^:async with-native-preview + [test-fn] + (let [next-uri* (atom 0) + revoked* (atom [])] + (await + (mock/with-mocks* {wasm.api/initialized? (constantly true) + wasm.api/get-shape-extrect (fn [_] (hash-map :width 100 :height 50)) + wasm.exports/export-image-uri (fn [_] (str "blob:preview-" (swap! next-uri* inc))) + wapi/revoke-uri #(swap! revoked* conj %)} + (await + (with-mounted-menu + [{:type :png :scale 1 :suffix ""}] + (^:async fn [container render-selection] + (await (react/act #(swap! st/state assoc :features #{"render-wasm/v1"}))) + (await (react/act #(render-selection :preview))) + (await (react/act #(.click (.querySelector container "button[aria-controls^='export-preview-']")))) + (await (wait-for-preview)) + (await (test-fn container render-selection revoked*))))))))) + +(t/deftest ^:async native-preview-keeps-the-image-until-the-refresh-is-ready + (await + (with-native-preview + (^:async fn [container render-selection revoked*] + (let [image-src #(.getAttribute (.querySelector container "img") "src") + original (image-src) + file-id (:current-file-id @st/state) + page-id (:current-page-id @st/state) + path [:files file-id :data :pages-index page-id :objects] + shape-id (-> (get-in @st/state path) keys first)] + (await (react/act #(swap! st/state update-in (conj path shape-id :x) inc))) + (t/is (some? (.querySelector container "img"))) + (t/is (empty? @revoked*)) + (when (.querySelector container "img") + (t/is (= original (image-src)))) + (await (wait-for-preview)) + (t/is (not= original (image-src))) + (t/is (= [original] @revoked*)) + (let [replacement (image-src)] + (await (react/act #(render-selection nil))) + (t/is (= [original replacement] @revoked*)))))))) + +(t/deftest ^:async unrelated-parent-renders-keep-the-native-preview + (await + (with-native-preview + (^:async fn [container render-selection revoked*] + (let [original (.getAttribute (.querySelector container "img") "src")] + (await (react/act #(render-selection :preview))) + (await (wait-for-preview)) + (t/is (= original (.getAttribute (.querySelector container "img") "src"))) + (t/is (empty? @revoked*))))))) + +(t/deftest ^:async closing-native-preview-cancels-a-pending-refresh + (await + (with-native-preview + (^:async fn [container _render-selection revoked*] + (let [original (.getAttribute (.querySelector container "img") "src") + file-id (:current-file-id @st/state) + page-id (:current-page-id @st/state) + path [:files file-id :data :pages-index page-id :objects] + shape-id (-> (get-in @st/state path) keys first)] + (await (react/act #(swap! st/state update-in (conj path shape-id :x) inc))) + (await (react/act #(.click (.querySelector container "button[aria-controls^='export-preview-']")))) + (await (wait-for-preview)) + (t/is (nil? (.querySelector container "img"))) + (t/is (= [original] @revoked*))))))) diff --git a/frontend/test/frontend_tests/ui/inspect_text_list_test.cljs b/frontend/test/frontend_tests/ui/inspect_text_list_test.cljs new file mode 100644 index 0000000000..067ee6f178 --- /dev/null +++ b/frontend/test/frontend_tests/ui/inspect_text_list_test.cljs @@ -0,0 +1,61 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.ui.inspect-text-list-test + (:require + ["react-dom/server" :as rds] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.main.ui.hooks :as hooks] + [app.main.ui.inspect.attributes.text :as attributes-text] + [app.main.ui.inspect.styles.panels.text :as styles-text] + [cljs.test :as t :include-macros true] + [cuerdas.core :as str] + [frontend-tests.helpers.mock :as mock] + [rumext.v2 :as mf])) + +(defn- text-shape + [paragraph-attrs] + {:id (uuid/next) + :type :text + :content {:type "root" + :children [{:type "paragraph-set" + :children [(merge {:type "paragraph" + :children [{:text "Item" + :font-id "sourcesanspro" + :font-family "sourcesanspro" + :font-size "14"}]} + paragraph-attrs)]}]}}) + +(defn- render-panel + [panel shapes] + (rds/renderToStaticMarkup + (mf/element panel + #js {:shapes shapes + :resolvedTokens {} + :colorSpace "hex" + :onFontShorthand identity}))) + +(t/deftest text-panels-show-the-list-style + (with-redefs [cf/flags (conj cf/flags :inspect-styles) + hooks/use-portal-container (mock/stub (fn [_] nil))] + (let [shapes [(text-shape {:list-style-type "disc"})]] + (t/testing "styles tab" + (let [markup (render-panel styles-text/text-panel* shapes)] + (t/is (str/includes? markup "List Style")) + (t/is (str/includes? markup ">disc<")))) + (t/testing "computed tab" + (let [markup (render-panel attributes-text/text-panel* shapes)] + (t/is (str/includes? markup "List Style")) + (t/is (str/includes? markup "inspect.attributes.typography.list-style-type.disc"))))))) + +(t/deftest text-panels-hide-the-list-style-without-a-list + (with-redefs [cf/flags (conj cf/flags :inspect-styles) + hooks/use-portal-container (mock/stub (fn [_] nil))] + (doseq [paragraph-attrs [{} {:list-style-type "none"}] + panel [styles-text/text-panel* attributes-text/text-panel*]] + (let [markup (render-panel panel [(text-shape paragraph-attrs)])] + (t/is (not (str/includes? markup "List Style"))))))) diff --git a/frontend/test/frontend_tests/ui/routes_test.cljs b/frontend/test/frontend_tests/ui/routes_test.cljs index 68ef15c5d1..08c2203187 100644 --- a/frontend/test/frontend_tests/ui/routes_test.cljs +++ b/frontend/test/frontend_tests/ui/routes_test.cljs @@ -38,15 +38,17 @@ (fn [command params] (t/is (= :check-nitrate-sso command)) (t/is (= team-id (:team-id params))) + (t/is (= "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d" + (:query-string params))) (swap! rpc-calls inc) (rx/of {:authorized true}))) st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))} (fn [done'] - (#'routes/check-sso-and-navigate match true "https://penpot.example.com/#/workspace") + (#'routes/check-sso-and-navigate match true "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") (reset! now (ct/plus @now #js {:minutes 4 :seconds 59})) - (#'routes/check-sso-and-navigate match true "https://penpot.example.com/#/workspace") + (#'routes/check-sso-and-navigate match true "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") (t/is (= 1 @rpc-calls)) (t/is (= 2 (count @events))) (done')) @@ -67,9 +69,9 @@ (rx/of {:authorized true}))) st/emit! mock/noop} (fn [done'] - (#'routes/check-sso-and-navigate match true "https://penpot.example.com/#/workspace") + (#'routes/check-sso-and-navigate match true "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") (reset! now (ct/plus @now #js {:minutes 5})) - (#'routes/check-sso-and-navigate match true "https://penpot.example.com/#/workspace") + (#'routes/check-sso-and-navigate match true "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") (t/is (= 2 @rpc-calls)) (done')) done)))) @@ -91,8 +93,8 @@ (fn [& emitted] (swap! events into emitted)))} (fn [done'] - (#'routes/check-sso-and-navigate match true "https://penpot.example.com/#/workspace") - (#'routes/check-sso-and-navigate match true "https://penpot.example.com/#/workspace") + (#'routes/check-sso-and-navigate match true "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") + (#'routes/check-sso-and-navigate match true "screen=workspace&team-id=1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d") (t/is (= 2 @rpc-calls)) (t/is (= 2 (count @events))) (done')) diff --git a/frontend/test/frontend_tests/ui/shortcuts_labels_test.cljs b/frontend/test/frontend_tests/ui/shortcuts_labels_test.cljs index 3f3715d1d7..3df04c3ccf 100644 --- a/frontend/test/frontend_tests/ui/shortcuts_labels_test.cljs +++ b/frontend/test/frontend_tests/ui/shortcuts_labels_test.cljs @@ -75,3 +75,11 @@ (ss/translation-keyname :sc :stale-id))) (t/is (= "shortcuts.preview-frame" (ss/translation-keyname :sc :preview-frame)))))) + +(t/deftest unit-move-commands-match-their-direction + (doseq [shortcuts [wsc/shortcuts psc/shortcuts] + dir ["up" "down" "left" "right"] + :let [id (keyword (str "move-unit-" dir)) + command (:command (get shortcuts id)) + commands (if (string? command) [command] command)]] + (t/is (= dir (first commands)) (str id " is bound to the " dir " arrow")))) diff --git a/frontend/test/frontend_tests/ui/subscription_test.cljs b/frontend/test/frontend_tests/ui/subscription_test.cljs new file mode 100644 index 0000000000..8724d47c31 --- /dev/null +++ b/frontend/test/frontend_tests/ui/subscription_test.cljs @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.ui.subscription-test + (:require + [app.main.ui.dashboard.subscription :as ds] + [cljs.test :as t :include-macros true])) + +(t/deftest team-plan-is-enterprise-while-team-belongs-to-an-organization + (doseq [owner-plan ["professional" "unlimited"]] + (t/is (= "enterprise" + (ds/get-team-plan-type {:organization {:id "org-1"} + :subscription {:type owner-plan + :status "active"}}))))) + +(t/deftest team-plan-returns-to-owner-plan-after-leaving-organization + (doseq [owner-plan ["professional" "unlimited"]] + (t/is (= owner-plan + (ds/get-team-plan-type {:subscription {:type owner-plan + :status "active"}}))))) + +(t/deftest team-plan-is-visible-when-owner-has-professional-or-unlimited-plan + (doseq [owner-plan ["professional" "unlimited"]] + (t/is (true? (ds/show-team-plan? {:subscription {:type owner-plan + :status "active"}}))))) + +(t/deftest team-plan-is-hidden-when-owner-of-team-outside-organization-has-enterprise-plan + ;; The team `:subscription` carries the plan of the owner, so the result + ;; is the same for the owner and for any other member. + (doseq [owner-plan ["enterprise" "nitrate"] + status ["active" "trialing" "past_due"]] + (t/is (false? (ds/show-team-plan? {:subscription {:type owner-plan + :status status}}))))) + +(t/deftest team-plan-is-visible-for-team-in-organization-whatever-the-owner-plan + (doseq [owner-plan ["professional" "unlimited" "enterprise" "nitrate"]] + (t/is (true? (ds/show-team-plan? {:subscription {:type owner-plan + :status "active"} + :organization {:id "org-1"}}))))) + +(t/deftest team-plan-is-visible-when-owner-enterprise-plan-is-no-longer-paid + (doseq [status ["unpaid" "canceled"]] + (t/is (true? (ds/show-team-plan? {:subscription {:type "enterprise" + :status status}}))))) diff --git a/frontend/test/frontend_tests/ui/team_container_test.cljs b/frontend/test/frontend_tests/ui/team_container_test.cljs index b16d0b089d..99649f0485 100644 --- a/frontend/test/frontend_tests/ui/team_container_test.cljs +++ b/frontend/test/frontend_tests/ui/team_container_test.cljs @@ -7,8 +7,10 @@ (ns frontend-tests.ui.team-container-test "`team-container*` renders the same hooks with and without a team id. React throws when the hook count changes between two renders of the - same component, so these tests mount it and render it again; the - documents come from jsdom since the runner has no DOM." + same component, so these tests mount it and render it again with each + id of the sequence; the documents come from jsdom since the runner has + no DOM. The guard that compares the route id with the store team is + covered too: it decides whether the children are painted at all." (:require ["jsdom" :refer [JSDOM]] ["react-dom" :as rd] @@ -30,51 +32,115 @@ (js/Promise. (fn [resolve] (js/setImmediate resolve)))) (defn- ^:async render-with - "Mounts `team-container*` once and renders it again with each of - `team-ids`, in order. Resolves to the errors React reported and the - markup left in the document. + "Mounts `team-container*` once, with `team` as the store's team, and + renders it again with each of `team-ids`, in order. Resolves to + `{:renders [...] :console-errors [...]}`: one entry per render, holding + the markup the document had right after that render and the errors + React reported up to that point, plus everything React logged to + `console.error` while the sequence ran. + + Per render matters: a render error makes React drop the subtree, so + reading the document once at the end tells `renders nothing on purpose` + and `rendered and then crashed` apart from the errors alone. + + The console is part of the evidence because a hook-order violation is + reported there and as an internal React error, not through the root's + error callbacks: React logs it and recovers, so the callbacks stay + empty and the markup looks right. react-dom reads the global `window` while it renders, and its scheduler does so again on a later turn, so the jsdom one stays installed until - the root is unmounted and the scheduler has run. - - The effect is replaced by a hook that does nothing: the real one raises - for an invalid id, and what is under test is the render. It must still - take a hook slot, because React only compares the hooks of two renders - when the first one had any." - [team-ids] - (let [errors (volatile! []) - on-error (fn [error _info] (vswap! errors conj error)) - window (.-window (JSDOM. "
        ")) - container (.getElementById (.-document window) "root") - root (rdc/createRoot container #js {:onUncaughtError on-error - :onCaughtError on-error - :onRecoverableError on-error}) - child (mf/html [:span "inside"]) - previous (.-window js/globalThis)] + the root is unmounted and the scheduler has run." + [team team-ids] + (let [errors (volatile! []) + rendered (volatile! []) + console-errors (volatile! []) + on-error (fn [error _info] (vswap! errors conj error)) + window (.-window (JSDOM. "
        ")) + container (.getElementById (.-document window) "root") + root (rdc/createRoot container #js {:onUncaughtError on-error + :onCaughtError on-error + :onRecoverableError on-error}) + child (mf/html [:span "inside"]) + previous (.-window js/globalThis) + console (.-console js/globalThis) + console-error (.-error console) + torn-down (volatile! false) + teardown (fn [] + (when-not @torn-down + (vreset! torn-down true) + (rd/flushSync #(.unmount root)) + (.close window)))] (set! (.-window js/globalThis) window) + (set! (.-error console) + (fn [& args] (vswap! console-errors conj (str/join " " (mapv str args))))) (try - (with-redefs [refs/team (l/atom {:id team-id :permissions {:can-edit true}}) - mf/useEffect (fn [_f _deps] (mf/use-ref nil))] - (doseq [id team-ids] - (rd/flushSync - (fn [] - (.render root (mf/element (deref #'ui/team-container*) - #js {:teamId id :children child})))))) - (let [markup (.-innerHTML container)] - (rd/flushSync #(.unmount root)) - (await (next-immediate)) - {:errors @errors - :markup markup}) + ;; The root goes down from `finally` even when a render throws, and + ;; before the scheduler turn that closes the window behind it. + (try + ;; `with-redefs` restores the vars when the block ends, so every + ;; render under it must be synchronous. `flushSync` is; a promise + ;; is not, and defers past the restore. + (with-redefs [refs/team (l/atom team) + mf/useEffect (fn [_f _deps] (mf/use-ref nil))] + (doseq [id team-ids] + (rd/flushSync + (fn [] + (.render root (mf/element (deref #'ui/team-container*) + #js {:teamId id :children child})))) + (vswap! rendered conj {:markup (.-innerHTML container) + :errors @errors}))) + (finally + (teardown))) + (await (next-immediate)) (finally - (set! (.-window js/globalThis) previous))))) + (set! (.-error console) console-error) + (set! (.-window js/globalThis) previous))) + {:renders @rendered + :console-errors @console-errors})) + +(defn- messages + [result] + (mapv #(.-message %) (:errors result))) (t/deftest ^:async team-id-arriving-after-the-first-render-keeps-the-hooks - (let [{:keys [errors markup]} (await (render-with [nil team-id]))] - (t/is (empty? (map #(.-message %) errors))) - (t/is (str/includes? markup "inside")))) + (let [{:keys [renders console-errors]} + (await (render-with {:id team-id :permissions {:can-edit true}} + [nil team-id]))] + (t/is (empty? console-errors)) + (t/is (every? #(empty? (messages %)) renders)) + (t/is (str/includes? (:markup (last renders)) "inside")))) (t/deftest ^:async team-id-leaving-after-the-first-render-keeps-the-hooks - (let [{:keys [errors markup]} (await (render-with [team-id nil]))] - (t/is (empty? (map #(.-message %) errors))) - (t/is (= "" markup)))) + ;; This direction is not what React flags: a render that calls no hooks + ;; at all slips through its bookkeeping, so the guard is covered here + ;; and the hook regression is the `nil` -> uuid case above. + (let [{:keys [renders console-errors]} + (await (render-with {:id team-id :permissions {:can-edit true}} + [team-id nil]))] + (t/is (empty? console-errors)) + (t/is (every? #(empty? (messages %)) renders)) + (t/is (str/includes? (:markup (first renders)) "inside")) + ;; The guard drops the children on purpose, so the second render is + ;; empty for its own reason and not because React removed the subtree. + (t/is (= "" (:markup (last renders)))))) + +(t/deftest ^:async team-container-renders-nothing-when-the-route-team-is-not-the-store-team + ;; A valid uuid that does not match the team held in the store must not + ;; paint: the providers would otherwise wrap another team's children. + (let [{:keys [renders console-errors]} + (await (render-with {:id (uuid/next) :permissions {:can-edit true}} + [team-id])) + result (last renders)] + (t/is (empty? console-errors)) + (t/is (empty? (messages result))) + (t/is (= "" (:markup result))))) + +(t/deftest ^:async team-container-renders-the-team-when-the-profile-has-no-permissions + (let [{:keys [renders console-errors]} + (await (render-with {:id team-id :permissions nil} + [team-id])) + result (last renders)] + (t/is (empty? console-errors)) + (t/is (empty? (messages result))) + (t/is (str/includes? (:markup result) "inside")))) diff --git a/frontend/test/frontend_tests/util/code_gen_markup_svg_wasm_test.cljs b/frontend/test/frontend_tests/util/code_gen_markup_svg_wasm_test.cljs index 62df62eb59..83e981636f 100644 --- a/frontend/test/frontend_tests/util/code_gen_markup_svg_wasm_test.cljs +++ b/frontend/test/frontend_tests/util/code_gen_markup_svg_wasm_test.cljs @@ -55,12 +55,20 @@ :fn "_render_shape_svg"}))) (t/deftest get-svg-markup-swallows-wasm-errors - (with-redefs [wasm.api/initialized? (constantly true) - wasm.api/render-shape-svg (fn [_ _] (wasm-panic))] - (t/is (= "" (svg.wasm/get-svg-markup [(uuid/next)])))) - (with-redefs [wasm.api/initialized? (constantly true) - wasm.api/render-shapes-svg (fn [_ _] (wasm-panic))] - (t/is (= "" (svg.wasm/get-svg-markup [(uuid/next) (uuid/next)]))))) + ;; The helper logs the non-fatal failure with `console.error` and returns + ;; empty. This test provokes two panics on purpose, so the dumped exception + ;; is noise: silence it for the body, then restore it. + (let [console-error (.-error js/console)] + (set! (.-error js/console) (fn [& _] nil)) + (try + (with-redefs [wasm.api/initialized? (constantly true) + wasm.api/render-shape-svg (fn [_ _] (wasm-panic))] + (t/is (= "" (svg.wasm/get-svg-markup [(uuid/next)])))) + (with-redefs [wasm.api/initialized? (constantly true) + wasm.api/render-shapes-svg (fn [_ _] (wasm-panic))] + (t/is (= "" (svg.wasm/get-svg-markup [(uuid/next) (uuid/next)])))) + (finally + (set! (.-error js/console) console-error))))) (t/deftest get-formatted-svg-markup-safe-on-empty (with-redefs [wasm.api/initialized? (constantly false)] diff --git a/frontend/test/frontend_tests/worker/component_thumbnails_test.cljs b/frontend/test/frontend_tests/worker/component_thumbnails_test.cljs new file mode 100644 index 0000000000..a721275427 --- /dev/null +++ b/frontend/test/frontend_tests/worker/component_thumbnails_test.cljs @@ -0,0 +1,95 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.worker.component-thumbnails-test + "The worker batches the component thumbnails the main thread asks for, + renders the batches one at a time, sends each render back and uploads + the ones asked for." + (:require + [app.common.thumbnails :as thc] + [app.common.uuid :as uuid] + [app.util.http :as http] + [app.util.webapi :as wapi] + [app.worker.component-thumbnails :as wct] + [app.worker.impl :as impl] + [app.worker.wasm :as wwasm] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private file-id (uuid/next)) +(def ^:private page-id (uuid/next)) + +(defn- request-render + [root-id] + (impl/handler {:cmd :thumbnails/render-component + :file-id file-id + :page-id page-id + :root-id root-id + :objects {}} + nil)) + +(defn- sleep + [ms] + (js/Promise. (fn [resolve] (js/setTimeout resolve ms)))) + +(t/deftest ^:async a-burst-of-requests-renders-as-one-batch + (let [jobs (atom 0)] + (await + (mock/with-mocks* + {wwasm/submit (mock/stub (fn [_] (swap! jobs inc) (rx/empty)))} + (run! request-render (repeatedly 30 uuid/next)) + (await (async/wait-for #(pos? @jobs) "the batch")) + ;; Longer than the pause that would start a second batch + (await (sleep 400)))) + (t/is (= 1 @jobs)))) + +(defn- ^:async render-with-module + "Renders a batch of one main with the WASM module mocked as `module`. + Returns what came out and whether it ended." + [module] + (let [out (atom {:values [] :ended? false :error nil})] + (await + (mock/with-mocks* + {wwasm/module module} + (await (async/observe (#'wct/render-batch [{:root-id (uuid/next) :objects {}}]) + :on-next #(swap! out update :values conj %) + :on-error #(swap! out assoc :error %) + :on-complete #(swap! out assoc :ended? true))))) + @out)) + +(t/deftest ^:async a-failed-render-ends-its-batch-quietly + ;; A failed module load, and a failed job: the tests have no WASM render + (doseq [module [(delay (js/Promise.reject (ex-info "no module" {}))) + (delay (js/Promise.resolve nil))]] + (t/is (= {:values [] :ended? true :error nil} + (await (render-with-module module)))))) + +(t/deftest ^:async a-render-is-shown-and-uploaded-when-asked + (let [output (rx/subject) + shown (atom []) + requests (atom []) + file-id (uuid/next) + page-id (uuid/next) + edited (uuid/next) + undone (uuid/next) + blob (wapi/create-blob "png" "image/png")] + (rx/sub! output #(swap! shown conj %)) + (reset! @#'wct/output* output) + (await + (mock/with-mocks* + {http/send! (mock/stub (fn [request] (swap! requests conj request) (rx/of {:status 200})))} + (#'wct/add-render {:file-id file-id :page-id page-id :root-id edited :blob blob :persist? true}) + (#'wct/add-render {:file-id file-id :page-id page-id :root-id undone :blob blob :persist? false}))) + (reset! @#'wct/output* nil) + (t/is (= [{:object-id (thc/fmt-object-id file-id page-id edited "component") :blob blob} + {:object-id (thc/fmt-object-id file-id page-id undone "component") :blob blob}] + @shown)) + (t/is (= [{:file-id file-id + :object-id (thc/fmt-object-id file-id page-id edited "component") + :tag "component"}] + (mapv :query @requests))))) diff --git a/frontend/test/frontend_tests/worker/fonts_sync_test.cljs b/frontend/test/frontend_tests/worker/fonts_sync_test.cljs new file mode 100644 index 0000000000..6cdf76313d --- /dev/null +++ b/frontend/test/frontend_tests/worker/fonts_sync_test.cljs @@ -0,0 +1,54 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.worker.fonts-sync-test + "The main thread keeps the worker's copy of the team's custom fonts + current, for the render there to find them." + (:require + [app.common.uuid :as uuid] + [app.main.data.fonts :as df] + [app.main.store :as st] + [app.main.worker :as mw] + [app.worker.impl :as impl] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock] + [okulary.core :as l])) + +(defn- font-variant + [] + (let [id (uuid/next)] + {:id id + :font-id (uuid/next) + :font-family "Custom" + :font-weight 400 + :font-style "normal" + :ttf-file-id (uuid/next)})) + +(t/deftest ^:async the-worker-gets-the-fonts-each-time-they-change + (let [a (font-variant) + b (font-variant) + store (l/atom {}) + sent (atom [])] + (await + (mock/with-mocks* + {mw/emit! (mock/stub (fn [message] (swap! sent conj message) (rx/empty)))} + (add-watch store ::sync @#'df/sync-worker-fonts) + (swap! store assoc :fonts {(:id a) a}) + (swap! store assoc :selected #{}) + (swap! store update :fonts assoc (:id b) b))) + (t/is (= [{:cmd :fonts/sync :fonts {(:id a) a}} + {:cmd :fonts/sync :fonts {(:id a) a (:id b) b}}] + @sent)))) + +(t/deftest ^:async the-worker-keeps-the-fonts-it-gets + (let [a (font-variant) + b (font-variant)] + (await + (mock/with-mocks* + {st/state (l/atom {:fonts {(:id a) a}})} + (impl/handler {:cmd :fonts/sync :fonts {(:id b) b}}) + (t/is (= {(:id b) b} (:fonts @st/state))))))) diff --git a/frontend/test/performance_tests/render_wasm/artifacts_test.cljs b/frontend/test/performance_tests/render_wasm/artifacts_test.cljs new file mode 100644 index 0000000000..4e4efbf696 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/artifacts_test.cljs @@ -0,0 +1,566 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.artifacts-test + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + ["worker_threads" :as workers] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [cljs.test :as t :include-macros true])) + +(def ^:private temporary-root (atom nil)) +(def ^:private selection {:default-features? true :features []}) +(def ^:private provenance {:git {:sha "test-revision" :dirty false}}) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-artifacts-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- create-build + [] + (artifacts/create-build @temporary-root selection provenance)) + +(defn- write-file + [file content] + (fs/mkdirSync (path/dirname file) #js {:recursive true}) + (fs/writeFileSync file content "utf8")) + +(defn- complete-files + [build] + (write-file (:cases-path build) "module.exports = {};") + (write-file (path/join (:browser-dir build) "browser.js") "export default {};") + (write-file (path/join (:renderer-dir build) "render-wasm.js") "export default {};") + (fs/writeFileSync (path/join (:renderer-dir build) "render-wasm.wasm") + (js/Buffer.from #js [0 97 115 109 1 0 0 0])) + (write-file (path/join (:renderer-dir build) "shared.js") "export default {};") + (write-file (:index-path build) "selected build") + build) + +(defn- ready-build + [] + (let [build (complete-files (create-build))] + (artifacts/write-manifest build (assoc (:manifest build) :status :ready)))) + +(defn- failure + [f] + (try + (f) + nil + (catch :default error error))) + +(t/deftest frontend-root-is-independent-of-caller-cwd + (let [expected (artifacts/frontend-root) + previous (.cwd js/process)] + (try + (.chdir js/process @temporary-root) + (t/is (= expected (artifacts/frontend-root))) + (t/is (.isFile (fs/statSync (path/join expected "deps.edn")))) + (t/is (.isDirectory (fs/statSync + (path/join expected "performance" "benches" "render_wasm")))) + (finally + (.chdir js/process previous))))) + +(t/deftest create-build-retains-its-initial-release-evidence + (let [selected {:default-features? false :features ["feature-one" "feature-two"]} + build (artifacts/create-build @temporary-root selected provenance) + manifest (transit/decode-str (fs/readFileSync (:manifest-path build) "utf8"))] + (t/is (re-matches #"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + (:id build))) + (t/is (= (path/join (artifacts/managed-root @temporary-root) "builds" (:id build)) + (:dir build))) + (t/is (= (path/join (:dir build) "build.transit") (:manifest-path build))) + (t/is (= (path/join (:dir build) "index.html") (:index-path build))) + (t/is (= manifest (:manifest build))) + (t/is (= #{:type :id :status :created-at :updated-at :functional :provenance} + (set (keys manifest)))) + (t/is (= {:type :renderer-build :status :building} + (select-keys manifest [:type :status]))) + (t/is (= {:mode :release :default-features? false + :features ["feature-one" "feature-two"] :cljs-profile-marks false} + (:functional manifest))) + (t/is (= provenance (:provenance manifest))) + (t/is (= (:created-at manifest) (.toISOString (js/Date. (:created-at manifest))))) + (t/is (= (:created-at manifest) (:updated-at manifest))) + (t/is (= "" (fs/readFileSync (:log-path build) "utf8"))) + (t/is (= (path/join (:dir build) "cases.cjs") (:cases-path build))) + (t/is (.isDirectory (fs/statSync (:browser-dir build)))) + (t/is (.isDirectory (fs/statSync (:renderer-dir build)))))) + +(t/deftest ready-build-loads-all-selected-paths + (let [build (ready-build) + loaded (artifacts/load-build @temporary-root (:id build))] + (t/is (= build loaded)) + (t/is (= (path/join (:dir build) "index.html") (:index-path loaded))))) + +(t/deftest complete-building-files-can-be-checked-before-publishing-ready + (let [build (complete-files (create-build))] + (t/is (= build (artifacts/check-files build))) + (t/is (= :building (:status (transit/decode-str + (fs/readFileSync (:manifest-path build) "utf8"))))) + (t/is (= ::artifacts/not-ready + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + +(t/deftest incomplete-building-files-are-rejected-before-publishing-ready + (doseq [relative ["cases.cjs" "browser/browser.js" "renderer/render-wasm.js" + "renderer/render-wasm.wasm" "renderer/shared.js" "index.html"]] + (let [build (complete-files (create-build))] + (fs/unlinkSync (path/join (:dir build) relative)) + (t/is (some? (failure #(artifacts/check-files build))) relative) + (t/is (= :building (:status (transit/decode-str + (fs/readFileSync (:manifest-path build) "utf8")))))))) + +(t/deftest building-and-failed-builds-are-not-loadable-and-retain-evidence + (let [building (create-build) + failed (create-build) + log "renderer compilation failed\n"] + (fs/appendFileSync (:log-path failed) log "utf8") + (artifacts/write-manifest failed (assoc (:manifest failed) :status :failed + :failure {:phase "renderer" :message "compile failed"})) + (doseq [build [building failed]] + (t/is (= ::artifacts/not-ready + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build)))))))) + (t/is (= :failed (:status (transit/decode-str + (fs/readFileSync (:manifest-path failed) "utf8"))))) + (t/is (= log (fs/readFileSync (:log-path failed) "utf8"))) + (t/is (fs/existsSync (:dir building))))) + +(t/deftest stale-references-cannot-reopen-terminal-builds + (doseq [status [:ready :failed]] + (let [build (create-build) + terminal (artifacts/write-manifest build (assoc (:manifest build) :status status)) + saved (fs/readFileSync (:manifest-path build) "utf8")] + (t/is (= terminal (artifacts/write-manifest terminal (:manifest terminal)))) + (doseq [next-status [:building :ready :failed]] + (let [attempt (assoc (:manifest terminal) :status next-status :extra "changed")] + (t/is (= ::artifacts/terminal-build + (:type (ex-data (failure #(artifacts/write-manifest build attempt)))))))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")))))) + +(t/deftest unencodable-update-keeps-the-last-manifest-and-log + (let [build (create-build) + saved (fs/readFileSync (:manifest-path build) "utf8")] + (fs/appendFileSync (:log-path build) "saved diagnostic\n" "utf8") + (t/is (some? (failure #(artifacts/write-manifest + build (assoc (:manifest build) :callback (fn [] nil)))))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8"))) + (t/is (= "saved diagnostic\n" (fs/readFileSync (:log-path build) "utf8"))) + (t/is (= #{"browser" "renderer" "build.log" "build.transit"} + (set (array-seq (fs/readdirSync (:dir build)))))))) + +(t/deftest malformed-builds-cannot-replace-saved-manifests + (doseq [side [:current :replacement]] + (let [build (create-build) + valid (:manifest build) + malformed (dissoc valid :functional) + proposed (if (= :replacement side) malformed valid)] + (when (= :current side) + (fs/writeFileSync (:manifest-path build) (codec/encode-str malformed) "utf8")) + (let [saved (fs/readFileSync (:manifest-path build) "utf8") + error (failure #(artifacts/write-manifest build proposed))] + (t/is (= ::artifacts/invalid-manifest (:type (ex-data error))) (name side)) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")) (name side)))))) + +(def ^:private manifest-reader + (str "const {parentPort, workerData} = require('worker_threads');" + "const fs = require('fs'); const stop = new Int32Array(workerData.stop);" + "let reads = 0; const errors = [];" + "function read() { try { JSON.parse(fs.readFileSync(workerData.path, 'utf8'));" + "reads++; } catch (error) { if (errors.length < 5) errors.push(error.message); } }" + "read(); parentPort.postMessage('ready');" + "while (Atomics.load(stop, 0) === 0) read();" + "parentPort.postMessage({reads, errors});")) + +(defn- worker-message + [^js worker] + (js/Promise. + (fn [resolve reject] + (letfn [(received [value] + (.removeListener worker "error" failed) + (resolve value)) + (failed [error] + (.removeListener worker "message" received) + (reject error))] + (.once worker "message" received) + (.once worker "error" failed))))) + +(t/deftest ^:async manifest-replacements-never-expose-a-partial-file + (let [build (create-build) + stop (js/SharedArrayBuffer. 4) + ^js worker (workers/Worker. manifest-reader + #js {:eval true + :workerData #js {:path (:manifest-path build) :stop stop}})] + (try + (await (worker-message worker)) + (let [result (worker-message worker)] + (dotimes [n 24] + (artifacts/write-manifest + build (assoc (:manifest build) :progress n + :diagnostic (apply str (repeat (+ 8192 (* n 1024)) "evidence "))))) + (js/Atomics.store (js/Int32Array. stop) 0 1) + (let [^js observed (await result)] + (t/is (> (.-reads observed) 1) "a separate thread read during replacement") + (t/is (empty? (array-seq (.-errors observed))) + "every observed manifest remained a complete JSON document"))) + (t/is (= 23 (:progress (transit/decode-str + (fs/readFileSync (:manifest-path build) "utf8"))))) + (finally + (await (.terminate worker)))))) + +(t/deftest malformed-manifests-and-incomplete-builds-are-rejected + (doseq [change [{:functional []} {:type :run-record} {:id "another-build"} + {:status :unknown}]] + (let [build (ready-build)] + (fs/writeFileSync (:manifest-path build) (codec/encode-str (merge (:manifest build) change)) + "utf8") + (t/is (= ::artifacts/unsupported-manifest + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + (doseq [relative ["cases.cjs" "browser/browser.js" "renderer/render-wasm.js" + "renderer/render-wasm.wasm" "renderer/shared.js" "index.html"]] + (let [build (ready-build)] + (fs/unlinkSync (path/join (:dir build) relative)) + (t/is (some? (failure #(artifacts/load-build @temporary-root (:id build)))) relative)))) + +(t/deftest saved-builds-validate-current-shapes + (let [build (ready-build) + manifest (select-keys (:manifest build) + [:type :id :status :created-at :updated-at :functional :provenance])] + (fs/writeFileSync (:manifest-path build) (codec/encode-str manifest) "utf8") + (t/is (= manifest (:manifest (artifacts/load-build @temporary-root (:id build))))) + (fs/writeFileSync (:manifest-path build) + (codec/encode-str (dissoc manifest :functional)) "utf8") + (t/is (= ::artifacts/unsupported-manifest + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + +(t/deftest build-environment-roundtrips-without-inventing-missing-evidence + (t/is (not (contains? (:manifest (create-build)) :environment))) + (doseq [environment [{} {"CFLAGS" "" "RUSTFLAGS" "-C\u001fopt-level=3"}]] + (let [build (complete-files (create-build)) + ready (artifacts/write-manifest build (assoc (:manifest build) + :status :ready :environment environment)) + saved (artifacts/load-build @temporary-root (:id ready))] + (t/is (= environment (get-in saved [:manifest :environment])))))) + +(t/deftest build-environment-rejects-non-string-keys-and-values + (doseq [environment [nil [] {:CFLAGS ""} {"CFLAGS" nil} {"CFLAGS" 0}]] + (let [build (create-build) + saved (fs/readFileSync (:manifest-path build) "utf8") + cause (failure #(artifacts/write-manifest build (assoc (:manifest build) :environment environment)))] + (t/is (= ::artifacts/invalid-manifest (:type (ex-data cause)))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")))))) + +(t/deftest ids-cannot-select-or-delete-outside-paths + (let [build (ready-build) + outside (path/join @temporary-root "outside.txt")] + (write-file outside "keep this") + (doseq [id [nil "" "." ".." "../outside.txt" "safe/../../outside.txt" + "..\\outside.txt" "/tmp/outside" "id%2fother"]] + (t/is (= ::artifacts/invalid-id + (:type (ex-data (failure #(artifacts/load-build @temporary-root id)))))) + (t/is (= ::artifacts/invalid-id + (:type (ex-data (failure #(artifacts/remove-builds + @temporary-root [(:id build) id]))))))) + (t/is (= build (artifacts/load-build @temporary-root (:id build)))) + (t/is (= "keep this" (fs/readFileSync outside "utf8"))))) + +(t/deftest safe-non-uuid-ids-can-load-existing-builds + (let [build (ready-build) + id "retained-build_1" + dir (path/join (path/dirname (:dir build)) id)] + (fs/renameSync (:dir build) dir) + (fs/writeFileSync (path/join dir "build.transit") + (codec/encode-str (assoc (:manifest build) :id id)) "utf8") + (t/is (= id (:id (artifacts/load-build @temporary-root id)))) + (t/is (= dir (:dir (artifacts/load-build @temporary-root id)))))) + +(t/deftest build-and-artifact-symlinks-cannot-escape-the-managed-root + (let [build (ready-build) + outside (path/join @temporary-root "outside.txt")] + (write-file outside "private") + (fs/unlinkSync (:cases-path build)) + (fs/symlinkSync outside (:cases-path build)) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/check-files build)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build)))))))) + (let [build (ready-build) + outside (path/join @temporary-root "outside-build")] + (fs/renameSync (:dir build) outside) + (fs/symlinkSync outside (:dir build) "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/check-files build)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + +(t/deftest category-purge-unlinks-only-known-outputs-and-keeps-directories + (doseq [category [:builds :runs :all]] + (let [root (fs/mkdtempSync (path/join @temporary-root (str (name category) "-"))) + building (complete-files (artifacts/create-build root selection provenance)) + failed (artifacts/create-build root selection provenance) + managed (artifacts/managed-root root) + run-dir (path/join managed "runs" "group" "saved-run") + outputs ["build.transit" "build.log" "cases.cjs" "index.html" + "browser/browser.js" "renderer/render-wasm.js" + "renderer/render-wasm.wasm" "renderer/render-wasm.wasm.map" + "renderer/shared.js" "renderer/cargo-build.json" + "renderer/worker/render.js"] + boot (path/join managed "runner.cjs") + outside (path/join root "explicit-output" "run.transit")] + (artifacts/write-manifest failed (assoc (:manifest failed) :status :failed)) + (doseq [file outputs] + (write-file (path/join (:dir building) file) "build output")) + (doseq [file ["run.log" "run.transit" "shot.png" "SHOT.PNG"]] + (write-file (path/join run-dir file) "run output")) + (doseq [dir ["run.log" "run.transit" "directory.PNG"]] + (write-file (path/join managed "runs" dir "keep.txt") "keep directory")) + (write-file (path/join managed "runs" "no-marker" "only.PNG") "screenshot") + (write-file (path/join run-dir "record.transit") "unrelated run file") + (write-file (path/join (:dir building) "custom" "build.log") "unrelated build file") + (write-file boot "boot executable") + (write-file outside "unmanaged") + (artifacts/purge root category) + (doseq [dir [(:dir building) (:dir failed) (:browser-dir building) + (:renderer-dir building) run-dir]] + (t/is (.isDirectory (fs/statSync dir)))) + (doseq [file outputs] + (t/is (= (= category :runs) (fs/existsSync (path/join (:dir building) file))))) + (t/is (= (= category :runs) (fs/existsSync (:manifest-path failed)))) + (doseq [file ["run.log" "run.transit" "shot.png" "SHOT.PNG"]] + (t/is (= (= category :builds) (fs/existsSync (path/join run-dir file))))) + (t/is (= (= category :builds) + (fs/existsSync (path/join managed "runs" "no-marker" "only.PNG")))) + (doseq [dir ["run.log" "run.transit" "directory.PNG"]] + (t/is (fs/existsSync (path/join managed "runs" dir "keep.txt")))) + (t/is (fs/existsSync (path/join run-dir "record.transit"))) + (t/is (fs/existsSync (path/join (:dir building) "custom" "build.log"))) + (t/is (= "boot executable" (fs/readFileSync boot "utf8"))) + (t/is (= "unmanaged" (fs/readFileSync outside "utf8"))) + (artifacts/purge root category)))) + +(t/deftest comparison-purge-unlinks-only-feature-comparison-evidence + (doseq [category [:comparisons :all]] + (let [root (fs/mkdtempSync (path/join @temporary-root (str (name category) "-"))) + managed (artifacts/managed-root root) + build-file (path/join managed "builds" "saved" "build.transit") + ab-dir (path/join managed "feature-comparisons" "ab-1") + config-dir (path/join ab-dir "configs" "00") + edge-dir (path/join ab-dir "comparisons") + run-dir (path/join managed "runs" "saved") + outside (path/join root "unmanaged" "feature-comparison.transit") + outputs [["feature-comparison.log" ab-dir] + ["feature-comparison.transit" ab-dir] + ["run.log" config-dir] + ["run.transit" config-dir] + ["shot.PNG" config-dir] + ["00__10.transit" edge-dir]] + kept [["notes.txt" ab-dir] + ["keep.txt" config-dir] + ["keep.txt" edge-dir]]] + (doseq [[file dir] outputs + :let [path (path/join dir file)]] + (write-file path "comparison output")) + (doseq [[file dir] kept] + (write-file (path/join dir file) "unrelated")) + (write-file (path/join managed "feature-comparisons" "keep.txt") + "unrelated root") + (write-file (path/join managed "feature-comparisons" "ab-1" "directory.PNG" + "keep.txt") + "keep directory") + (write-file build-file "retained build") + (write-file (path/join run-dir "run.transit") "standalone run") + (write-file outside "unmanaged") + (artifacts/purge root category) + (doseq [[file dir] outputs] + (t/is (not (fs/existsSync (path/join dir file))) (str category " " file))) + (doseq [[file dir] kept] + (t/is (fs/existsSync (path/join dir file)) (str category " " file))) + (t/is (= "unrelated root" + (fs/readFileSync (path/join managed "feature-comparisons" "keep.txt") + "utf8"))) + (t/is (fs/existsSync (path/join ab-dir "directory.PNG" "keep.txt"))) + (t/is (.isDirectory (fs/statSync ab-dir))) + (t/is (.isDirectory (fs/statSync config-dir))) + (t/is (.isDirectory (fs/statSync edge-dir))) + (t/is (= (= category :comparisons) + (fs/existsSync (path/join run-dir "run.transit"))) + (str category " standalone run")) + (t/is (= (= category :comparisons) (fs/existsSync build-file)) + (str category " retained build")) + (when (= category :comparisons) + (t/is (= "standalone run" + (fs/readFileSync (path/join run-dir "run.transit") "utf8"))) + (t/is (= "retained build" (fs/readFileSync build-file "utf8")))) + (t/is (= "unmanaged" (fs/readFileSync outside "utf8")))))) + +(t/deftest purge-follows-category-and-nested-directory-symlinks-and-stops-cycles + (let [managed (artifacts/managed-root @temporary-root)] + (fs/mkdirSync managed #js {:recursive true}) + (doseq [category [:builds :runs]] + (let [outside (path/join @temporary-root (str "outside-" (name category))) + detached (path/join @temporary-root (str "detached-" (name category))) + output (if (= category :builds) "build.transit" "run.transit") + saved (path/join outside "keep.txt") + link (path/join managed (name category))] + (write-file saved "keep symlink target") + (write-file (path/join outside "saved" output) "output, not a parsed record") + (write-file (path/join detached output) "linked output") + (write-file (path/join detached "keep.txt") "unrelated") + (fs/symlinkSync outside (path/join outside "saved" "cycle") "dir") + (fs/symlinkSync detached (path/join outside "linked") "dir") + (fs/symlinkSync outside link "dir") + (artifacts/purge @temporary-root category) + (t/is (.isSymbolicLink (fs/lstatSync link))) + (t/is (.isSymbolicLink (fs/lstatSync (path/join outside "saved" "cycle")))) + (t/is (.isSymbolicLink (fs/lstatSync (path/join outside "linked")))) + (t/is (not (fs/existsSync (path/join outside "saved" output)))) + (t/is (not (fs/existsSync (path/join detached output)))) + (t/is (fs/existsSync (path/join detached "keep.txt"))) + (t/is (= "keep symlink target" (fs/readFileSync saved "utf8"))))))) + +(t/deftest managed-root-symlink-cannot-create-or-delete-outside-the-frontend + (let [root (path/join @temporary-root "frontend") + managed (path/join root "target" "performance" "render-wasm") + outside (path/join @temporary-root "outside-managed") + build-log (path/join outside "builds" "retained" "build.log") + run-file (path/join outside "runs" "retained" "record.transit")] + (write-file build-log "outside build") + (write-file run-file "outside run") + (fs/mkdirSync (path/dirname managed) #js {:recursive true}) + (fs/symlinkSync outside managed "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/managed-root root)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/create-build root selection provenance)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/purge root :all)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/remove-builds root ["retained"])))))) + (t/is (= "outside build" (fs/readFileSync build-log "utf8"))) + (t/is (= "outside run" (fs/readFileSync run-file "utf8"))) + (t/is (= #{"retained"} (set (array-seq (fs/readdirSync (path/join outside "builds")))))))) + +(t/deftest escaping-target-ancestor-is-rejected-before-the-managed-path-exists + (let [root (path/join @temporary-root "frontend-ancestor") + outside (path/join @temporary-root "outside-target")] + (fs/mkdirSync root) + (fs/mkdirSync outside) + (fs/symlinkSync outside (path/join root "target") "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/create-build root selection provenance)))))) + (t/is (empty? (array-seq (fs/readdirSync outside)))))) + +(t/deftest create-build-rejects-an-escaping-builds-parent-before-writing-files + (let [managed (artifacts/managed-root @temporary-root) + outside (path/join @temporary-root "outside-builds") + saved (path/join outside "retained" "build.log")] + (write-file saved "keep external build") + (fs/mkdirSync managed #js {:recursive true}) + (fs/symlinkSync outside (path/join managed "builds") "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/create-build + @temporary-root selection provenance)))))) + (t/is (= "keep external build" (fs/readFileSync saved "utf8"))) + (t/is (= #{"retained"} (set (array-seq (fs/readdirSync outside))))))) + +(t/deftest remove-builds-cleans-selected-ids-and-keeps-directories-and-unrelated-files + (let [removed (ready-build) + kept (ready-build) + managed (artifacts/managed-root @temporary-root) + run (path/join managed "runs" "saved" "run.transit") + outside (path/join @temporary-root "outside" "keep.txt") + browser (path/join @temporary-root "linked-browser") + link (path/join managed "builds" "outside-link")] + (write-file run "keep run") + (write-file outside "keep target") + (write-file (path/join (:dir removed) "screenshot.PNG") "keep build screenshot") + (write-file (path/join (path/dirname outside) "build.transit") "linked build output") + (write-file (path/join browser "browser.js") "linked browser output") + (write-file (path/join browser "keep.js") "unrelated") + (fs/symlinkSync browser (path/join (path/dirname outside) "browser") "dir") + (fs/symlinkSync (path/dirname outside) link "dir") + (artifacts/remove-builds @temporary-root [(:id removed) "outside-link" "absent-build"]) + (t/is (.isDirectory (fs/statSync (:dir removed)))) + (t/is (not (fs/existsSync (:manifest-path removed)))) + (t/is (not (fs/existsSync (:cases-path removed)))) + (t/is (fs/existsSync (path/join (:dir removed) "screenshot.PNG"))) + (t/is (.isSymbolicLink (fs/lstatSync link))) + (t/is (not (fs/existsSync (path/join link "build.transit")))) + (t/is (not (fs/existsSync (path/join browser "browser.js")))) + (t/is (fs/existsSync (path/join browser "keep.js"))) + (t/is (= kept (artifacts/load-build @temporary-root (:id kept)))) + (t/is (= "keep run" (fs/readFileSync run "utf8"))) + (t/is (= "keep target" (fs/readFileSync outside "utf8"))))) + +(t/deftest inside-builds-resolves-symlink-ancestors-with-missing-output-components + (let [managed (artifacts/managed-root @temporary-root) + builds (path/join managed "builds") + outside (path/join @temporary-root "external-builds") + alias (path/join @temporary-root "build-alias")] + (t/is (artifacts/inside-builds? @temporary-root (path/join builds "future" "run"))) + (fs/mkdirSync managed #js {:recursive true}) + (fs/mkdirSync outside) + (fs/symlinkSync outside builds "dir") + (fs/symlinkSync builds alias "dir") + (doseq [output [builds outside (path/join builds "new" "run") + (path/join outside "new" "run") (path/join alias "new" "run")]] + (t/is (artifacts/inside-builds? @temporary-root output))) + (t/is (not (artifacts/inside-builds? @temporary-root (str outside "-neighbor/run")))) + (t/is (not (artifacts/inside-builds? @temporary-root (path/join managed "runs" "new")))))) + +(defn- descriptor + [id] + {:id id + :scene :artifact-only + :scene-description "A descriptor supplied only by this build" + :scene-seed 42 + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {}}) + +(defn- write-catalog + [build loaded-id response] + (write-file + (:cases-path build) + (str "const identity = " (js/JSON.stringify (codec/encode-str {:build-id loaded-id})) ";" + "const response = " (js/JSON.stringify (codec/encode-str response)) ";" + "module.exports = {buildIdentity() {return identity;}," + "collectCases(encodedSelection) {return response;}};"))) + +(t/deftest catalogs-use-the-selected-commonjs-module-with-isolated-scope + (let [first-build (ready-build) + second-build (ready-build) + first-case (descriptor :artifact-only/first) + second-case (descriptor :artifact-only/second) + selection {:master-seed 42}] + (write-catalog first-build (:id first-build) + {:build-id (:id first-build) :cases [first-case]}) + (write-catalog second-build (:id second-build) + {:build-id (:id second-build) :cases [second-case]}) + (t/is (= [first-case] (catalog/collect first-build selection))) + (t/is (= [second-case] (catalog/collect second-build selection))) + (t/is (= [first-case] (catalog/collect first-build selection))))) + +(t/deftest catalog-rejects-both-loaded-and-response-identity-mismatches + (doseq [boundary [:loaded :response]] + (let [build (ready-build)] + (write-catalog build (if (= boundary :loaded) "wrong-build" (:id build)) + {:build-id (if (= boundary :response) "wrong-build" (:id build)) + :cases [(descriptor :artifact-only/load)]}) + (t/is (= ::identity/mismatch + (:type (ex-data (failure #(catalog/collect build {:master-seed 42}))))))))) + +(t/deftest catalog-rejects-malformed-descriptors-and-empty-responses + (doseq [cases [[(assoc (descriptor :artifact-only/load) :scene-seed -1)] [] nil]] + (let [build (ready-build)] + (write-catalog build (:id build) {:build-id (:id build) :cases cases}) + (t/is (some? (failure #(catalog/collect build {:master-seed 42}))))))) diff --git a/frontend/test/performance_tests/render_wasm/boolean_test.cljs b/frontend/test/performance_tests/render_wasm/boolean_test.cljs new file mode 100644 index 0000000000..ccea94ce2e --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/boolean_test.cljs @@ -0,0 +1,214 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.boolean-test + (:require + [app.common.geom.point :as gpt] + [app.common.geom.shapes.intersect :as gint] + [app.common.types.path :as path] + [app.common.types.path.segment :as segm] + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as b] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(t/deftest difference-of-overlapping-rects-keeps-hierarchy-and-bounds + (let [red {:fill-color "#ff0000" :fill-opacity 1} + blue {:fill-color "#0000ff" :fill-opacity 1} + instance (b/scene {:seed 1} + (b/bool :l-shape {:bool-type :difference :name "L"} + (b/rect {:x 0 :y 0 :width 100 :height 100 :fills [red]}) + (b/rect {:x 50 :y 50 :width 100 :height 100 :fills [blue]}))) + bool (first (helpers/children-of instance uuid/zero)) + children (helpers/children-of instance (:id bool))] + (t/is (= :bool (:type bool))) + (t/is (= :difference (:bool-type bool))) + (t/is (= (:id bool) (snapshot/ref-id instance :l-shape))) + (t/is (= 2 (count children))) + (t/is (= [(:id (first children)) (:id (second children))] (:shapes bool))) + (t/is (= [(:id bool) (:id bool)] + [(:parent-id (first children)) (:parent-id (second children))])) + (t/is (= [uuid/zero uuid/zero] + [(:frame-id (first children)) (:frame-id (second children))])) + (t/is (= [0 0 100 100] (helpers/selrect bool))))) + +(t/deftest difference-excludes-points-in-removed-overlap + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100}))) + bool (first (helpers/children-of instance uuid/zero)) + lines (segm/path->lines bool)] + (t/is (true? (gint/is-point-inside-nonzero? (gpt/point 25 25) lines)) + "a point in the kept region stays inside the content") + (t/is (false? (gint/is-point-inside-nonzero? (gpt/point 75 75) lines)) + "a point in the subtracted overlap is excluded from the content"))) + +(t/deftest union-derives-canonical-bounds + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :union} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= :union (:bool-type bool))) + (t/is (= [0 0 150 150] (helpers/selrect bool))))) + +(t/deftest difference-picks-the-first-child-as-style-head + (let [red {:fill-color "#ff0000" :fill-opacity 1} + blue {:fill-color "#0000ff" :fill-opacity 1} + blur {:id (uuid/custom 3 3) :type :layer-blur :value 4 :hidden false} + difference (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 10 :height 10 + :fills [red] :blur blur}) + (b/rect {:x 5 :y 0 :width 10 :height 10 + :fills [blue]}))) + diff-bool (first (helpers/children-of difference uuid/zero))] + (t/is (= [red] (:fills diff-bool))) + (t/is (= blur (:blur diff-bool))))) + +(t/deftest supplied-styles-override-inherited + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :union :fills []} + (b/rect {:x 0 :y 0 :width 10 :height 10 + :fills [{:fill-color "#ff0000" :fill-opacity 1}]}) + (b/rect {:x 5 :y 0 :width 10 :height 10 + :fills [{:fill-color "#0000ff" :fill-opacity 1}]}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= [] (:fills bool))))) + +(t/deftest nil-styles-do-not-count-as-supplied + (let [red {:fill-color "#ff0000" :fill-opacity 1} + blue {:fill-color "#0000ff" :fill-opacity 1} + instance (b/scene {:seed 1} + (b/bool {:bool-type :union :fills nil} + (b/rect {:x 0 :y 0 :width 10 :height 10 :fills [red]}) + (b/rect {:x 5 :y 0 :width 10 :height 10 :fills [blue]}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= [blue] (:fills bool))))) + +(t/deftest single-child-bool-uses-the-child-path + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 10 :y 20 :width 30 :height 40}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= 1 (count (:shapes bool)))) + (t/is (= [10 20 30 40] (helpers/selrect bool))) + (t/is (= (:selrect bool) (path/calc-selrect (:content bool)))))) + +(t/deftest nested-bools-finalize-inside-out + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :union} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100})) + (b/rect {:x 200 :y 0 :width 50 :height 50}))) + outer (first (helpers/children-of instance uuid/zero)) + inner (first (helpers/children-of instance (:id outer)))] + (t/is (= :bool (:type inner))) + (t/is (= [0 0 100 100] (helpers/selrect inner))) + (t/is (= [0 0 250 100] (helpers/selrect outer))) + (t/is (= (:selrect outer) (path/calc-selrect (:content outer)))))) + +(t/deftest bool-inside-frame-keeps-frame-ids + (let [instance (b/scene {:seed 1} + (b/frame {:x 0 :y 0 :width 200 :height 200} + (b/bool {:bool-type :union} + (b/rect {:x 10 :y 10 :width 10 :height 10}) + (b/rect {:x 30 :y 10 :width 10 :height 10})))) + frame (first (helpers/children-of instance uuid/zero)) + bool (first (helpers/children-of instance (:id frame))) + rects (helpers/children-of instance (:id bool))] + (t/is (= (:id frame) (:parent-id bool))) + (t/is (= (:id frame) (:frame-id bool))) + (t/is (= [(:id frame) (:id frame)] (mapv :frame-id rects))) + (t/is (= [(:id bool) (:id bool)] (mapv :parent-id rects))))) + +(t/deftest bool-inside-group-derives-group-bounds + (let [instance (b/scene {:seed 1} + (b/group {} + (b/bool {:bool-type :union} + (b/rect {:x 10 :y 20 :width 30 :height 40}) + (b/rect {:x 50 :y 80 :width 10 :height 10})))) + group (first (helpers/children-of instance uuid/zero)) + bool (first (helpers/children-of instance (:id group)))] + (t/is (= :bool (:type bool))) + (t/is (= [10 20 50 70] (helpers/selrect group))) + (t/is (= uuid/zero (:frame-id bool))))) + +(t/deftest empty-bool-is-rejected + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool {:bool-type :union})))] + (t/is (= ::b/empty-bool (:type data))))) + +(t/deftest caught-empty-bool-still-fails-finish + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (b/bool {:bool-type :union}) + (catch :default _ nil))))] + (t/is (= ::b/unfinished-container + (:type (helpers/failure-data #(b/finish! state))))))) + +(t/deftest bool-geometry-attrs-are-rejected + (doseq [attrs [{:bool-type :union :x 1} + {:bool-type :union :selrect {:x 0 :y 0 :width 1 :height 1}} + {:bool-type :union :content []}]] + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool attrs (b/rect))))] + (t/is (= ::b/bool-geometry (:type data)) (str attrs)) + (t/is (contains? #{:x :selrect :content} (:key data)) (str attrs))))) + +(t/deftest bool-transform-attrs-are-rejected + (doseq [attrs [{:bool-type :union :rotation 45} + {:bool-type :union :transform "m"} + {:bool-type :union :transform-inverse "m"} + {:bool-type :union :flip-x true} + {:bool-type :union :flip-y true}]] + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool attrs (b/rect))))] + (t/is (= ::b/bool-transform (:type data)) (str attrs))))) + +(t/deftest bool-type-is-required-and-validated + (t/is (= ::b/bool-type + (:type (helpers/failure-data #(b/scene {:seed 1} (b/bool {:name "B"} (b/rect))))))) + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool {:bool-type :merge} (b/rect))))] + (t/is (= ::b/bool-type (:type data))) + (t/is (= :merge (:bool-type data))) + (t/is (= #{:union :difference :intersection :exclude} (set (:supported data)))))) + +(t/deftest frame-child-is-rejected + (let [data (helpers/failure-data + #(b/scene {:seed 1} + (b/bool {:bool-type :union} + (b/frame {:x 0 :y 0 :width 10 :height 10} + (b/rect)))))] + (t/is (= ::b/frame-in-bool (:type data))) + (t/is (uuid? (:child data))))) + +(t/deftest bool-generation-ignores-the-renderer-override + (let [original path/wasm:calc-bool-content] + (try + (set! path/wasm:calc-bool-content + (fn [& _] + (throw (ex-info "renderer override ran during generation" + {:type ::unexpected-renderer-call})))) + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= [0 0 100 100] (helpers/selrect bool)))) + (finally + (set! path/wasm:calc-bool-content original))))) + +(t/deftest non-map-bool-attrs-are-rejected + (t/is (= ::b/invalid-attrs + (:type (helpers/failure-data #(b/scene {:seed 1} (b/bool :hero (b/rect))))))) + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (b/bool :hero (b/rect)) + (catch :default _ nil))))] + (t/is (= 1 (count (get-in @state [:objects uuid/zero :shapes])))))) diff --git a/frontend/test/performance_tests/render_wasm/browser_test.cljs b/frontend/test/performance_tests/render_wasm/browser_test.cljs new file mode 100644 index 0000000000..e0b88af31d --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/browser_test.cljs @@ -0,0 +1,328 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.browser-test + (:require + ["fs" :as fs] + ["http" :as http] + ["net" :as net] + ["os" :as os] + ["path" :as path] + ["playwright" :as playwright] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.browser :as browser] + [benches.render-wasm.node.runner.server :as server] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) +(def ^:private build-id "owned-browser-fixture") +(def ^:private viewport {:width 320 :height 240 :dpr 1}) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-browser-test-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(t/deftest gpu-container-selects-the-vulkan-launch-backend + (let [env (.-env js/process) + previous (.-PENPOT_GPU env)] + (try + (doseq [value [nil "0" "true" "1"]] + (if (nil? value) + (js-delete env "PENPOT_GPU") + (set! (.-PENPOT_GPU env) value)) + (let [args (set (:args (browser/launch-settings)))] + (t/is (= (= value "1") (contains? args "--use-angle=vulkan"))) + (t/is (= (= value "1") (contains? args "--use-gl=angle"))) + (t/is (contains? args "--enable-unsafe-swiftshader")))) + (finally + (if (undefined? previous) + (js-delete env "PENPOT_GPU") + (set! (.-PENPOT_GPU env) previous)))))) + +(defn- ^:async open-host + [actual-id ready?] + (let [identity-text (.stringify js/JSON (transit/encode-str {:build-id actual-id}))] + (fs/writeFileSync (path/join @temporary-root "index.html") + (str "browser ownership"))) + (await (server/open {:id build-id :dir @temporary-root} {:startup-timeout-ms 3000}))) + +(defn- request-status + [base-url request-path] + (js/Promise. + (fn [resolve reject] + (let [address (js/URL. base-url) + req (http/get #js {:hostname (.-hostname address) :port (.-port address) + :path request-path :agent false} + (fn [^js response] + (.once response "error" reject) + (.once response "end" #(resolve (.-statusCode response))) + (.resume response)))] + (.once ^js req "error" reject) + (.setTimeout ^js req 3000 #(.destroy ^js req (js/Error. "HTTP test timed out"))))))) + +(defn- socket-status + [url] + (js/Promise. + (fn [resolve _] + (let [address (js/URL. url) + ^js socket (net/createConnection #js {:host (.-hostname address) + :port (js/Number (.-port address))})] + (.once socket "connect" #(do (.destroy socket) (resolve :open))) + (.once socket "error" #(do (.destroy socket) (resolve (.-code ^js %)))) + (.setTimeout socket 3000 #(do (.destroy socket) (resolve :timeout))))))) + +(defn- ^:async rejected + [promise] + (try + (await promise) + (t/is false "expected browser work to reject") + nil + (catch :default cause cause))) + +(t/deftest ^:async page-and-context-close-failures-both-survive + (let [page-error (js/Error. "page disposal failed") + context-error (js/Error. "context disposal failed") + owner {:page #js {:isClosed (constantly false) + :close #(js/Promise.reject page-error)} + :context #js {:close #(js/Promise.reject context-error)}} + cause (await (rejected (browser/close-page owner)))] + (t/is (= [page-error context-error] (mapv :error (errors/get-errors cause)))))) + +(t/deftest ^:async browser-disposal-collects-failures-and-repeated-close-shares-the-result + (doseq [stop-fails? [false true]] + (let [page-error (js/Error. "page disposal failed") + context-error (js/Error. "context disposal failed") + process-error (js/Error. "process disposal failed") + disconnect-error (js/Error. "CDP disconnect failed") + controller (js/AbortController.) + running (js/Promise. + (fn [_ reject] + (.addEventListener (.-signal controller) "abort" + #(reject (if stop-fails? + process-error + (ex-info "cancelled" {:type :process-cancelled})))))) + owner {:controller controller :process running :shutdown (atom nil) + :user-data-dir @temporary-root + :page #js {:isClosed (constantly false) :close #(js/Promise.reject page-error)} + :context #js {:close #(js/Promise.reject context-error)} + :connection (atom (js/Promise.resolve + #js {:isConnected (constantly true) + :close #(if stop-fails? + (js/Promise.reject disconnect-error) + (js/Promise.resolve nil))}))} + closing (browser/close owner)] + (t/is (identical? closing (browser/close owner))) + (if stop-fails? + (let [cause (await (rejected closing))] + (t/is (= [page-error context-error process-error disconnect-error] + (mapv :error (errors/get-errors cause))))) + (t/is (nil? (await closing))))))) + + +(t/deftest ^:async unresolved-cancellation-retains-the-browser-profile + (let [profile (path/join @temporary-root "profile.info") + controller (js/AbortController.) + failure (errors/combine + [{:error (ex-info "Command cancelled" {:type :process-cancelled})} + {:error (errors/fatal-error "Command group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil)}]) + running (js/Promise. (fn [_ reject] + (.addEventListener (.-signal controller) "abort" #(reject failure)))) + owner {:controller controller :process running :shutdown (atom nil) + :connection (atom nil) :user-data-dir @temporary-root}] + (fs/writeFileSync profile "retained") + (let [cause (try (await (browser/close owner)) nil (catch :default cause cause))] + (t/is (errors/fatal? cause)) + (t/is (process/unresolved? cause)) + (t/is (= @temporary-root (:profile-path (errors/evidence :cleanup cause)))) + (t/is (= "retained" (when (fs/existsSync profile) (fs/readFileSync profile "utf8"))))))) + + +(t/deftest ^:async process-stop-retains-other-errors-after-cancellation + (let [controller (js/AbortController.) + probe (ex-info "Probe failed" {:type :process-probe}) + failure (errors/combine [{:error (ex-info "Command cancelled" {:type :process-cancelled})} + {:error probe}]) + running (js/Promise. (fn [_ reject] + (.addEventListener (.-signal controller) "abort" #(reject failure)))) + owner {:controller controller :process running :shutdown (atom nil) + :connection (atom nil) :user-data-dir @temporary-root} + cause (try (await (browser/close owner)) nil (catch :default cause cause))] + (t/is (= [probe] (mapv :error (errors/get-errors cause)))))) + +(t/deftest ^:async browser-startup-retains-a-failure-of-its-disposal + (let [run-process process/run + close browser/close + cleanup-error (js/Error. "startup disposal failed")] + (await + (mock/with-mocks* + {process/run (fn [options] + (run-process (assoc options :command "/bin/sh" :args ["-c" "exit 17"]))) + browser/close (fn [owner] + (.then (close owner) + (fn [_] (throw cleanup-error)) + (fn [_] (throw cleanup-error))))} + (let [cause (await (rejected (browser/launch {:startup-timeout-ms 10000}))) + entries (errors/get-errors cause)] + (t/is (= ::browser/startup-cleanup (:type (ex-data cause)))) + (t/is (= ["Command exited unsuccessfully" "startup disposal failed"] + (mapv #(errors/message-of (:error %)) entries)))))))) + +(defn- ^:async check-stopped + [{:keys [endpoint user-data-dir]}] + (t/is (not (fs/existsSync user-data-dir)) "owned Chromium profile was removed") + (t/is (= "ECONNREFUSED" (await (socket-status endpoint))) "the owned CDP listener is gone")) + +(defn- ^:async close-fixture + [owner page host] + (try + (when page + (try + (await (browser/close-page page)) + (catch :default _ nil))) + (when owner + (try + (await (browser/close owner)) + (catch :default _ nil))) + (finally + (await (server/close host))))) + +(t/deftest ^:async startup-process-details-survive-browser-failure-wrappers + (let [run-process process/run] + (doseq [exit-code [0 17]] + (await + (mock/with-mocks* + {process/run (fn [options] + (run-process (assoc options :command "/bin/sh" + :args ["-c" (str "exit " exit-code)])))} + (let [cause (await (rejected (browser/launch {:startup-timeout-ms 10000}))) + saved (-> (errors/evidence :browser-startup cause) + transit/encode-str + transit/decode-str) + command (if (zero? exit-code) + (.executablePath (.-chromium playwright)) + "/bin/sh")] + (t/is (= {:command command :exit-code exit-code} + (select-keys saved [:command :exit-code]))) + (t/is (= ::browser/endpoint-exit (:type (ex-data cause)))) + (t/is (= "Chromium exited before its CDP endpoint became ready" (errors/message-of cause))) + (t/is (= 1 (count (errors/get-errors cause)))))))))) + +(t/deftest ^:async successful-close-removes-the-profile-and-cdp-listener + (let [host (await (open-host build-id true)) + owner-ref (atom nil) + page-ref (atom nil)] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000})) + _ (reset! owner-ref owner) + page (await (browser/new-page owner host {:viewport viewport + :startup-timeout-ms 30000}))] + (reset! page-ref page) + (t/is (= 200 (await (request-status (:endpoint owner) "/json/version")))) + (t/is (= viewport + (js->clj (await (.evaluate ^js (:page page) + "({width:innerWidth,height:innerHeight,dpr:devicePixelRatio})")) + :keywordize-keys true))) + (await (browser/close-page page)) + (reset! page-ref nil) + (await (browser/close owner)) + (await (check-stopped owner)) + (t/is (not (.isConnected ^js (:browser owner)))) + (t/is (= 200 (await (request-status (:base-url host) "/"))) "browser shutdown leaves its host live") + (await (server/close host)) + (t/is (= "ECONNREFUSED" (await (socket-status (:base-url host)))))) + (finally + (await (close-fixture @owner-ref @page-ref host)))))) + +(t/deftest ^:async identity-mismatch-rejects-and-the-caller-closes-the-owned-browser + (let [host (await (open-host "wrong-compiled-build" true)) + owner-ref (atom nil)] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000})) + _ (reset! owner-ref owner) + cause (await (rejected (browser/new-page + owner host + {:viewport viewport :startup-timeout-ms 30000}))) + facts (ex-data cause)] + (t/is (= ::identity/mismatch (:type facts))) + (t/is (= "wrong-compiled-build" (:actual-id facts))) + (await (browser/close owner)) + (reset! owner-ref nil) + (await (check-stopped owner))) + (finally + (await (close-fixture @owner-ref nil host)))))) + +(t/deftest ^:async abort-stops-a-synchronously-blocked-page + (let [host (await (open-host build-id true)) + controller (js/AbortController.) + owner-ref (atom nil) + page-ref (atom nil) + started-ref (atom nil) + started (js/Promise. (fn [resolve _] (reset! started-ref resolve)))] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000 + :signal (.-signal controller)})) + _ (reset! owner-ref owner) + page-owner (await (browser/new-page + owner host + {:viewport viewport + :startup-timeout-ms 30000 + :signal (.-signal controller)})) + ^js page (:page page-owner)] + (reset! page-ref page-owner) + (await (.exposeFunction page "__blockingStarted" (fn [] (@started-ref)))) + (let [blocking (.evaluate page "window.__blockingStarted(); while (true) {}")] + (.catch blocking (fn [_] nil)) + (await (browser/within (fn [] started) + {:deadline (process/deadline 5000) :phase :test-blocking-start})) + (let [cause (await (rejected (browser/within #(.evaluate page "1") + {:deadline (process/deadline 1000) + :phase :test-blocked-page})))] + (t/is (= ::browser/timeout (:type (ex-data cause))))) + (.abort controller) + (reset! page-ref nil) + (await (browser/close owner)) + (reset! owner-ref nil) + (await (check-stopped owner)) + (t/is (not (.isConnected ^js (:browser owner)))) + (t/is (instance? js/Error (await (rejected blocking)))))) + (finally + (.abort controller) + (await (close-fixture @owner-ref @page-ref host)))))) + +(t/deftest ^:async abort-during-readiness-cleans-the-starting-browser + (let [host (await (open-host build-id false)) + controller (js/AbortController.) + owner-ref (atom nil) + requested (js/Promise. + (fn [resolve _] + (.once ^js (:server host) "request" (fn [_] (resolve nil)))))] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000 + :signal (.-signal controller)})) + _ (reset! owner-ref owner) + page (browser/new-page owner host {:viewport viewport + :startup-timeout-ms 30000 + :signal (.-signal controller)})] + (.catch page (fn [_] nil)) + (await (browser/within (fn [] requested) + {:deadline (process/deadline 30000) :phase :test-host-request})) + (.abort controller (ex-info "caller cancelled browser readiness" {:type ::caller-abort})) + (let [cause (await (rejected page))] + (t/is (= ::caller-abort (:type (ex-data cause))))) + (await (browser/close owner)) + (reset! owner-ref nil) + (await (check-stopped owner))) + (finally + (.abort controller) + (await (close-fixture @owner-ref nil host)))))) diff --git a/frontend/test/performance_tests/render_wasm/build_test.cljs b/frontend/test/performance_tests/render_wasm/build_test.cljs new file mode 100644 index 0000000000..97e0d4f732 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/build_test.cljs @@ -0,0 +1,306 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.build-test + "Real preparation failures in temporary frontend/render-wasm checkouts. + + A scripted compiler keeps these tests independent of Cargo and Skia while + exercising the process, filesystem and failure-retention boundary. Release + integration checks cover compiler success and Cargo feature forwarding." + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.build :as build] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(defn- frontend-root + [] + (path/join @temporary-root "frontend")) + +(defn- renderer-root + [] + (path/join @temporary-root "render-wasm")) + +(t/use-fixtures :each + {:before (fn [] + (reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-build-test-"))) + (fs/mkdirSync (frontend-root)) + (fs/mkdirSync (renderer-root))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(def ^:private renderer-wrapper + (str "#!/usr/bin/env node\n" + "const fs = require('node:fs');\n" + "const path = require('node:path');\n" + "const {spawnSync} = require('node:child_process');\n" + "const dir = path.dirname(process.env.FRONTEND_DEST);\n" + "const observation = {\n" + " cwd: process.cwd(),\n" + " manifest: fs.readFileSync(path.join(dir, 'build.transit'), 'utf8'),\n" + " log: fs.readFileSync(path.join(dir, 'build.log'), 'utf8'),\n" + " created: JSON.parse(fs.readFileSync(path.join(dir, 'created.json'), 'utf8'))\n" + "};\n" + "fs.writeFileSync(path.join(dir, 'renderer-observation.json'), JSON.stringify(observation));\n" + "const result = spawnSync('flock', [\n" + " '--exclusive', path.join(__dirname, 'build.lock'),\n" + " process.execPath, path.join(__dirname, 'compiler.cjs')\n" + "], {stdio: 'inherit'});\n" + "if (result.error) throw result.error;\n" + "process.exitCode = result.status === null ? 1 : result.status;\n")) + +(def ^:private failing-compiler + (str "const fs = require('node:fs');\n" + "const path = require('node:path');\n" + "fs.writeFileSync(path.join(process.env.FRONTEND_DEST, 'partial.js'), 'partial renderer output\\n');\n" + "process.stdout.write('compiler stdout before failure\\n');\n" + "process.stderr.write('compiler stderr before failure\\n');\n" + "process.exitCode = 23;\n")) + +(def ^:private sleeping-compiler + (str "const fs = require('node:fs');\n" + "const path = require('node:path');\n" + "fs.writeFileSync(path.join(process.env.FRONTEND_DEST, 'lock-acquired.txt'), 'held');\n" + "process.stdout.write('compiler lock acquired\\n');\n" + "process.stderr.write('waiting in compiler\\n');\n" + "setInterval(() => {}, 1000);\n")) + +(defn- install-renderer + [compiler] + (fs/writeFileSync (path/join (renderer-root) "build") renderer-wrapper #js {:mode 493}) + (fs/writeFileSync (path/join (renderer-root) "compiler.cjs") compiler "utf8")) + +(defn- read-manifest + [build-ref] + (transit/decode-str (fs/readFileSync (:manifest-path build-ref) "utf8"))) + +(defn- record-created + [build-ref] + ;; These reads must succeed before the executable can read created.json. + (let [observation {:id (:id build-ref) + :manifest (fs/readFileSync (:manifest-path build-ref) "utf8") + :log (fs/readFileSync (:log-path build-ref) "utf8") + :renderer-started? (fs/existsSync (path/join (:dir build-ref) "renderer-observation.json"))}] + (fs/writeFileSync (path/join (:dir build-ref) "created.json") + (js/JSON.stringify (clj->js observation)) "utf8"))) + +(defn- retained-build + [cause] + (let [id (:build-id (ex-data cause)) + dir (path/join (frontend-root) "target" "performance" "render-wasm" "builds" id)] + {:id id :dir dir + :manifest-path (path/join dir "build.transit") + :log-path (path/join dir "build.log") + :renderer-dir (path/join dir "renderer")})) + +(defn- ^:async rejected + [promise] + (try + (let [result (await promise)] + (t/is false (str "Expected preparation to fail, got " (pr-str result))) + nil) + (catch :default cause cause))) + +(defn- complete + [promise done] + (.then promise + (fn [_] (done)) + (fn [cause] + (t/is false (str "Unexpected test error: " cause)) + (done)))) + +(t/deftest ^:async renderer-facts-retain-string-environment-keys-and-missing-evidence + (let [bin (path/join @temporary-root "bin") + original-path (.. js/process -env -PATH)] + (fs/mkdirSync bin) + (fs/writeFileSync (path/join bin "clojure") "#!/bin/sh\nexit 24\n" #js {:mode 493}) + (try + (set! (.. js/process -env -PATH) (str bin path/delimiter original-path)) + (doseq [environment [::missing {} {"RUSTFLAGS" "-C\u001fopt-level=3" "CFLAGS" ""}]] + (let [facts (cond-> {:features [] :profile {:opt_level "3" :debug_assertions false}} + (not= ::missing environment) (assoc :environment environment))] + (install-renderer + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, " + (js/JSON.stringify (js/JSON.stringify (clj->js facts))) ");\n")) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + manifest (read-manifest (retained-build cause))] + (t/is (= :cljs (:phase (ex-data cause)))) + (t/is (= 24 (:exit-code (ex-data (ex-cause cause))))) + (if (= ::missing environment) + (t/is (not (contains? manifest :environment))) + (t/is (= environment (:environment manifest))))))) + (finally + (set! (.. js/process -env -PATH) original-path))))) + +(t/deftest ^:async explicit-selection-preserves-disabled-cargo-defaults + (let [bin (path/join @temporary-root "bin") + original-path (.. js/process -env -PATH)] + (fs/mkdirSync bin) + (fs/writeFileSync (path/join bin "clojure") "#!/bin/sh\nexit 24\n" #js {:mode 493}) + (try + (set! (.. js/process -env -PATH) (str bin path/delimiter original-path)) + (install-renderer + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, " + (js/JSON.stringify + (js/JSON.stringify + (clj->js {:features ["a"] + :profile {:opt_level "3" :debug_assertions false}}))) + ");\n")) + (let [cause (await (rejected (build/prepare + {:root (frontend-root) + :selection {:default-features? false :features ["a"]} + :build-timeout-ms 10000 + :on-created record-created}))) + manifest (read-manifest (retained-build cause))] + (t/is (= :cljs (:phase (ex-data cause)))) + (t/is (= {:default-features? false :features ["a"]} + (get-in manifest [:provenance :feature-selection]))) + (t/is (= ["a"] (get-in manifest [:functional :features]))) + (t/is (false? (get-in manifest [:functional :default-features?])))) + (finally + (set! (.. js/process -env -PATH) original-path))))) + +(t/deftest real-git-outside-a-repository-has-unknown-provenance + (t/async done + (complete + ((^:async fn [] + (let [git (await (build/git-provenance (frontend-root) + {:deadline (process/deadline 10000)}))] + (t/is (= {:sha nil :dirty nil} git))))) + done))) + + +(t/deftest ^:async git-provenance-propagates-unresolved-process-failures + (let [failure (errors/fatal-error "Git process may still be running" + {:type :process-unresolved :unresolved-process true + :command "git" :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil)] + (await + (mock/with-mocks* + {process/run (fn [_] (js/Promise.reject failure))} + (let [cause (try + (await (build/git-provenance (frontend-root) {:deadline (process/deadline 1000)})) + nil + (catch :default cause cause))] + (t/is (errors/fatal? cause)) + (t/is (process/unresolved? cause))))))) + +(t/deftest nonzero-renderer-retains-created-evidence-output-and-failed-manifest + (t/async done + (complete + ((^:async fn [] + (install-renderer failing-compiler) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + data (ex-data cause) + child-data (ex-data (ex-cause cause)) + retained (retained-build cause) + manifest (read-manifest retained) + log (fs/readFileSync (:log-path retained) "utf8") + observation (js->clj (js/JSON.parse + (fs/readFileSync (path/join (:dir retained) "renderer-observation.json") "utf8")) + :keywordize-keys true) + created (:created observation) + initial (transit/decode-str (:manifest created)) + at-renderer (transit/decode-str (:manifest observation))] + (t/is (= ::build/preparation-failed (:type data))) + (t/is (= :renderer (:phase data))) + (t/is (= (:log-path retained) (:log-path data))) + (t/is (= :process-exit (:type child-data))) + (t/is (= 23 (:exit-code child-data))) + (t/is (= (path/join (renderer-root) "build") (:command child-data))) + (t/is (= (renderer-root) (:cwd observation))) + (t/is (= (:id retained) (:id created) (:id manifest) + (:id initial) (:id at-renderer))) + (t/is (= :building (:status initial) (:status at-renderer))) + (t/is (false? (:renderer-started? created))) + (t/is (= "" (:log created))) + (t/is (str/includes? (:log observation) "Renderer compiling\n")) + (t/is (= :failed (:status manifest))) + (t/is (= "renderer" (get-in manifest [:failure :phase]))) + (t/is (= "Command exited unsuccessfully" (get-in manifest [:failure :message]))) + (t/is (= [:run-command] (mapv :operation (get-in manifest [:failure :errors])))) + (t/is (= :process-exit (get-in manifest [:failure :cause 0 :type]))) + (t/is (= {:sha nil :dirty nil} (get-in manifest [:provenance :git]))) + (t/is (str/includes? log "compiler stdout before failure\n")) + (t/is (str/includes? log "compiler stderr before failure\n")) + (t/is (= "partial renderer output\n" + (fs/readFileSync (path/join (:renderer-dir retained) "partial.js") "utf8")))))) + done))) + +(t/deftest failed-manifest-save-makes-the-preparation-error-fatal + (t/async done + (complete + ((^:async fn [] + (install-renderer + (str failing-compiler + "fs.unlinkSync(path.join(path.dirname(process.env.FRONTEND_DEST), 'build.transit'));\n")) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + data (ex-data cause) + entries (errors/get-errors cause)] + (t/is (errors/fatal? cause)) + (t/is (string? (:manifest-error data))) + (t/is (= :process-exit (:type (ex-data (:error (first entries)))))) + (t/is (= 2 (count entries))) + (t/is (= :write-manifest (:operation (second entries)))) + (t/is (= (:manifest-path (retained-build cause)) (:path (second entries)))) + (t/is (str/includes? (errors/message-of (:error (second entries))) "ENOENT"))))) + done))) + +(t/deftest timed-out-lock-holder-retains-its-attempt-and-retry-gets-a-new-id + (t/async done + (complete + ((^:async fn [] + (install-renderer sleeping-compiler) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 3000 + :on-created record-created}))) + retained (retained-build cause) + manifest (read-manifest retained) + before (fs/readFileSync (:manifest-path retained) "utf8") + log (fs/readFileSync (:log-path retained) "utf8")] + (t/is (= ::build/preparation-failed (:type (ex-data cause)))) + (t/is (= :renderer (:phase (ex-data cause)))) + (t/is (= (:log-path retained) (:log-path (ex-data cause)))) + (t/is (= :process-timeout (:type (ex-data (ex-cause cause))))) + (t/is (= (:id retained) (:id manifest))) + (t/is (= :failed (:status manifest))) + (t/is (= "renderer" (get-in manifest [:failure :phase]))) + (t/is (= :process-timeout (get-in manifest [:failure :cause 0 :type]))) + (t/is (= "held" (fs/readFileSync (path/join (:renderer-dir retained) "lock-acquired.txt") "utf8"))) + (t/is (str/includes? log "compiler lock acquired\n")) + (t/is (str/includes? log "waiting in compiler\n")) + ;; The retry must acquire the same flock before its compiler can exit. + (install-renderer failing-compiler) + (let [retry-cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + retry (retained-build retry-cause) + retry-log (fs/readFileSync (:log-path retry) "utf8")] + (t/is (not= (:id retained) (:id retry))) + (t/is (= :renderer (:phase (ex-data retry-cause)))) + (t/is (= 23 (:exit-code (ex-data (ex-cause retry-cause))))) + (t/is (= :failed (:status (read-manifest retry)))) + (t/is (str/includes? retry-log "compiler stderr before failure\n")) + (t/is (= before (fs/readFileSync (:manifest-path retained) "utf8"))) + (t/is (= log (fs/readFileSync (:log-path retained) "utf8"))) + (t/is (= #{(:id retained) (:id retry)} + (set (array-seq (fs/readdirSync (path/dirname (:dir retained))))))))))) + done))) diff --git a/frontend/test/performance_tests/render_wasm/builder_test.cljs b/frontend/test/performance_tests/render_wasm/builder_test.cljs new file mode 100644 index 0000000000..4e2194b4f8 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/builder_test.cljs @@ -0,0 +1,288 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.builder-test + "Tests canonical scene construction and the shared seeded number generator. + Fixed random sequences preserve scene draws during generator extraction." + (:require + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as b] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :as random] + [benches.render-wasm.scenes.rects] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(defn- build + ([] (build {})) + ([params] ((:build (decl/registered-scene :rects)) (merge {:seed 42 :count 5} params)))) + +(defn- add-tile + "Helper defined outside a scene scope, called from inside one." + [index] + (b/rect [:tile index] {:x (* 10 index)})) + +(t/deftest repeatable-builds-are-identical + (t/is (= (build) (build))) + (t/is (not= (build) (build {:seed 43})))) + +(t/deftest shared-generator-retains-scene-draws-and-independent-state + (doseq [[seed expected] [[0 [1144304738 1416247 958946056 627933444 2007157716]] + [42 [2581720956 1925393290 3661312704 2876485805 750819978]] + [4294967295 [3850105811 813802916 3073704848 4054706436 3630262831]]]] + (let [first-source (random/create seed) + second-source (random/create seed)] + (t/is (= expected (mapv #(* 4294967296 %) (repeatedly 5 first-source)))) + (t/is (= expected (mapv #(* 4294967296 %) (repeatedly 5 second-source)))) + (t/is (every? #(and (<= 0 %) (< % 1)) (repeatedly 1000 first-source))))) + (let [rng (:rng @(b/start {:seed 42}))] + (t/is (= [2581720956 1925393290 3661312704 2876485805 750819978] + (mapv #(* 4294967296 %) (repeatedly 5 rng)))))) + +(t/deftest shared-generator-rejects-invalid-seeds + (doseq [seed [-1 4294967296 1.5 nil false "42" js/NaN js/Infinity]] + (t/is (= ::random/invalid-seed + (:type (helpers/failure-data #(random/create seed))))))) + +(t/deftest rectangle-workload-keeps-default-count + (let [instance (build {:seed 42 :count 1000}) + rects (helpers/root-shapes instance)] + (t/is (= 1000 (count rects))) + (t/is (< 1 (count (distinct (map :x rects))))))) + +(t/deftest default-workload-is-canonical + (let [instance (build {:count 25}) + rects (helpers/root-shapes instance)] + (t/is (= 25 (count rects))) + (t/is (every? #(= :rect (:type %)) rects)) + (t/is (every? (fn [rect] + (and (= 1 (count (:fills rect))) + (= 1 (count (:strokes rect))) + (= 10 (:stroke-width (first (:strokes rect)))) + (= :center (:stroke-alignment (first (:strokes rect)))))) + rects)) + (t/is (every? (fn [rect] + (and (re-matches #"^#[0-9a-f]{6}$" (:fill-color (first (:fills rect)))) + (<= 0.1 (:fill-opacity (first (:fills rect))) 0.999) + (every? #(<= 0 % 24) [(:r1 rect) (:r2 rect) (:r3 rect) (:r4 rect)]))) + rects)))) + +(t/deftest geometry-is-page-relative-and-in-range + (let [params {:seed 7 :count 40 :width 800 :height 600 :min-size 5 :max-size 15} + instance (build params) + rects (helpers/root-shapes instance)] + (t/is (every? #(<= 0 (:x %) 800) rects)) + (t/is (every? #(<= 0 (:y %) 600) rects)) + (t/is (every? #(<= 5 (:width %) 15) rects)) + (t/is (every? #(<= 5 (:height %) 15) rects)) + (t/is (= 800 (get-in instance [:objects uuid/zero :width]))) + (t/is (= 600 (get-in instance [:objects uuid/zero :height]))))) + +(t/deftest labels-are-snapshot-local + (let [instance (b/scene {:seed 1} + (b/rect) + (b/rect :hero {}) + (b/rect [:tile 1] {})) + refs (:refs instance) + children (helpers/child-ids instance uuid/zero)] + (t/is (= #{:hero [:tile 1]} (set (keys refs)))) + (t/is (= (get refs :hero) (second children))) + (t/is (= (get refs [:tile 1]) (nth children 2))))) + +(t/deftest attrs-override-generated-values + (let [instance (b/scene {:seed 1 + :defaults {:rect {:x (b/gen-int 0 100)}}} + (b/rect {:x 7 :fills []})) + rect (first (helpers/root-shapes instance))] + (t/is (= 7 (:x rect))) + (t/is (= [] (:fills rect))))) + +(t/deftest shapes-use-their-defaults-before-drawing-ids + (doseq [[type low high construct] [[:rect 0.1 0.2 #(b/rect %)] + [:path 0.3 0.4 #(b/path %)] + [:circle 0.5 0.6 #(b/circle %)]] + attrs [{} {:opacity 0.75}]] + (let [rng (random/create 7) + opacity (random/round3 (random/rng-float rng low high)) + id (random/rng-uuid rng) + instance (b/scene {:seed 7 + :defaults {:rect {:opacity (b/gen-float 0.1 0.2)} + :path {:opacity (b/gen-float 0.3 0.4)} + :circle {:opacity (b/gen-float 0.5 0.6)}}} + (construct attrs)) + shape (first (helpers/root-shapes instance))] + (t/is (= {:type type :id id :opacity (get attrs :opacity opacity)} + (select-keys shape [:type :id :opacity])))))) + +(t/deftest scene-and-shape-expressions-evaluate-once-in-order + (let [events (atom []) + instance (b/scene (do (swap! events conj :params) {:seed 1}) + (b/rect (do (swap! events conj :label) :hero) + (do (swap! events conj :attrs) {:x 11}))) + hero (get-in instance [:objects (get (:refs instance) :hero)])] + (t/is (= [:params :label :attrs] @events)) + (t/is (= 11 (:x hero))))) + +(t/deftest ordinary-forms-work-inside-the-scope + (let [instance (b/scene {:seed 1} + (let [first-id (b/rect)] + (t/is (uuid? first-id))) + (doseq [index (range 3)] + (add-tile index)) + (b/rect :last {:x 99})) + refs (:refs instance) + rects (helpers/root-shapes instance)] + (t/is (= 5 (count rects))) + (t/is (= [0 10 20] + (mapv #(:x (get-in instance [:objects (get refs [:tile %])])) + (range 3)))) + (t/is (= 99 (:x (get-in instance [:objects (get refs :last)])))))) + +(t/deftest duplicate-labels-are-rejected + (let [data (helpers/failure-data + #(b/scene {:seed 1} + (b/rect :dup {}) + (b/rect :dup {})))] + (t/is (= ::b/duplicate-label (:type data))) + (t/is (= :dup (:label data))))) + +(t/deftest failed-builds-leak-no-state + (let [clean (build)] + (helpers/failure-data #(b/scene {:seed 99} + (b/rect :dup {}) + (b/rect :dup {}))) + (t/is (= clean (build))))) + +(t/deftest rect-outside-scope-is-rejected + (let [data (helpers/failure-data #(b/rect))] + (t/is (= ::b/outside-scope (:type data))))) + +(t/deftest nested-scopes-are-rejected + (let [data (helpers/failure-data + #(b/scene {:seed 1} + (b/scene {:seed 2} + (b/rect))))] + (t/is (= ::b/nested-scope (:type data))))) + +(t/deftest invalid-seeds-are-rejected + (t/is (= ::b/invalid-seed + (:type (helpers/failure-data #(b/scene {} (b/rect)))))) + (t/is (= ::b/invalid-seed + (:type (helpers/failure-data #(b/scene {:seed -1} (b/rect)))))) + (t/is (= ::b/invalid-seed + (:type (helpers/failure-data #(b/scene {:seed 4294967296} (b/rect))))))) + +(t/deftest runtime-attrs-maps-are-resolved + (let [attrs {:x 11} + instance (b/scene {:seed 1} + (b/rect attrs)) + rect (first (helpers/root-shapes instance))] + (t/is (= 11 (:x rect))))) + +(t/deftest single-label-form-is-supported + (let [instance (b/scene {:seed 1} + (b/rect :solo)) + id (get (:refs instance) :solo)] + (t/is (uuid? id)) + (t/is (= [id] (helpers/child-ids instance uuid/zero))))) + +(t/deftest built-snapshot-feeds-the-scene-api + (let [instance (b/scene {:seed 1} + (b/rect :hero {})) + order (snapshot/upload-order instance) + hero-id (snapshot/ref-id instance :hero)] + (t/is (= [uuid/zero hero-id] (mapv :id order))))) + +(t/deftest invalid-labels-are-rejected + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/rect "hero" {})))] + (t/is (= ::b/invalid-label (:type data))))) + +(t/deftest protected-keys-cannot-be-overridden + (let [other (uuid/custom 9 9) + instance (b/scene {:seed 1} + (b/rect {:type :circle :id other :parent-id other})) + rect (first (helpers/root-shapes instance))] + (t/is (= :rect (:type rect))) + (t/is (not= other (:id rect))) + (t/is (= uuid/zero (:parent-id rect))))) + +(t/deftest root-protected-keys-cannot-be-overridden + (let [other (uuid/custom 9 8) + instance (b/scene {:seed 1 + :root {:id other :type :rect :width 10}} + (b/rect)) + root (get-in instance [:objects uuid/zero])] + (t/is (= uuid/zero (:id root))) + (t/is (= :frame (:type root))) + (t/is (= 10 (:width root))))) + +(t/deftest attrs-override-generated-fills + (let [fill {:fill-color "#ffffff" :fill-opacity 1} + instance (b/scene {:seed 1} + (b/rect {:fills [fill]})) + rect (first (helpers/root-shapes instance))] + (t/is (= [fill] (:fills rect))))) + +(t/deftest generator-helpers-produce-expected-values + (let [instance (b/scene {:seed 3 + :defaults {:rect {:opacity (b/gen-float 0.5 0.6) + :x (b/gen-one-of [4 5]) + :r1 (b/gen-int 3 5) + :fills (b/gen-vector (b/gen-fill))}}} + (b/rect)) + rect (first (helpers/root-shapes instance))] + (t/is (<= 0.5 (:opacity rect) 0.6)) + (t/is (contains? #{4 5} (:x rect))) + (t/is (<= 3 (:r1 rect) 4)) + (t/is (re-matches #"^#[0-9a-f]{6}$" (:fill-color (first (:fills rect))))))) + +(t/deftest empty-one-of-is-rejected + (let [data (helpers/failure-data #(b/gen-one-of []))] + (t/is (= ::b/no-values (:type data))))) + +(t/deftest circle-creates-ellipse-type + (let [instance (b/scene {:seed 1} + (b/circle {:x 10 :y 20 :width 30 :height 40})) + circle (first (helpers/root-shapes instance))] + (t/is (= :circle (:type circle))) + (t/is (= [10 20 30 40] [(:x circle) (:y circle) (:width circle) (:height circle)])) + (t/is (= 1 (count (helpers/child-ids instance uuid/zero)))))) + +(t/deftest circle-uses-circle-defaults + (let [instance (b/scene {:seed 3 + :defaults {:circle {:x (b/gen-int 10 12)}}} + (b/circle)) + circle (first (helpers/root-shapes instance))] + (t/is (contains? #{10 11} (:x circle))))) + +(t/deftest circle-ignores-rect-defaults + (let [instance (b/scene {:seed 1 + :defaults {:rect {:x 999}}} + (b/circle)) + circle (first (helpers/root-shapes instance))] + (t/is (not= 999 (:x circle))))) + +(t/deftest circle-geometry-is-page-relative-and-in-range + (let [instance (b/scene {:seed 7 + :defaults {:circle {:x (b/gen-int 0 800) + :y (b/gen-int 0 600) + :width (b/gen-int 5 15) + :height (b/gen-int 5 15)}}} + (doseq [_ (range 40)] + (b/circle))) + circles (helpers/root-shapes instance)] + (t/is (= 40 (count circles))) + (t/is (every? #(<= 0 (:x %) 800) circles)) + (t/is (every? #(<= 0 (:y %) 600) circles)) + (t/is (every? #(<= 5 (:width %) 15) circles)) + (t/is (every? #(<= 5 (:height %) 15) circles)))) + +(t/deftest circle-preserves-radius-attrs + (let [instance (b/scene {:seed 1} + (b/circle {:x 0 :y 0 :width 20 :height 20 :rx 5 :ry 7})) + circle (first (helpers/root-shapes instance))] + (t/is (= [5 7] [(:rx circle) (:ry circle)])))) diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs new file mode 100644 index 0000000000..648ac8be43 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -0,0 +1,2197 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) KALEIDOS SUBSIDIARY SL + +// Our CLJS test suite already covers the internals: +// - runner_test.cljs tests parse-args, +// - build_test.cljs tests preparation with fake compilers, +// - feature_comparison_test.cljs tests orchestration. +// Those compile to an :esm bundle with :runtime :custom and run in-process from +// target/performance-tests/test.js. +// +// It cannot test the assembled executable, so here we test: +// - exit codes of the real process, including 130/143 for SIGINT/SIGTERM, +// - stdout and stderr separation as the shell sees it, +// - that runner.cjs finds commands.cjs next to itself, +// - that help, compare, and merge never load execution dependencies, +// - release-compiled file and manifest behavior on the real filesystem, + +// Opt-in: build the release runner and the test bundle first: +// +// pnpm run build:renderer-benchmarks:runner +// pnpm run build:renderer-benchmarks:tests +// RENDER_WASM_RUNNER_BOOT=target/performance/render-wasm/runner.cjs \ +// node --test test/performance_tests/render_wasm/cli_integration.cjs +// +// The normal unit test build does not build this CLI. + +const assert = require("node:assert/strict"); +const { execFile, spawn, spawnSync } = require("node:child_process"); +const { once } = require("node:events"); +const { setTimeout: pause } = require("node:timers/promises"); +const { randomUUID } = require("node:crypto"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { before, test } = require("node:test"); +const { pathToFileURL } = require("node:url"); +const { promisify } = require("node:util"); + +const execute = promisify(execFile); +const testOptions = { timeout: 60000 }; +let bootstrap; +let frontendRoot; +let transit; + +before(async () => { + assert.ok( + process.env.RENDER_WASM_RUNNER_BOOT, + "Set RENDER_WASM_RUNNER_BOOT to the release-compiled runner.cjs", + ); + bootstrap = path.resolve(process.env.RENDER_WASM_RUNNER_BOOT); + assert.ok( + fs.existsSync(bootstrap) && fs.statSync(bootstrap).isFile(), + "Run pnpm run build:renderer-benchmarks:runner before the CLI tests", + ); + bootstrap = fs.realpathSync(bootstrap); + frontendRoot = path.resolve(path.dirname(bootstrap), "../../.."); + assert.ok(fs.statSync(path.join(frontendRoot, "deps.edn")).isFile()); + assert.ok( + fs + .statSync( + path.join(frontendRoot, "performance", "benches", "render_wasm"), + ) + .isDirectory(), + ); + assert.ok(fs.statSync(path.join(frontendRoot, "node_modules")).isDirectory()); + + // Load the actual Transit JS library emitted by Shadow, without starting + // either the CLJS test runner or the production CLI in this process. + const transitModule = path.join( + frontendRoot, + "target/performance-tests/cljs-runtime/com.cognitect.transit.js", + ); + assert.ok( + fs.existsSync(transitModule), + "Run pnpm run build:renderer-benchmarks:tests before the CLI tests", + ); + await import(pathToFileURL(transitModule).href); + transit = globalThis.com.cognitect.transit; +}); + +function createFrontend(t) { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "render-wasm-cli-")); + t.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const root = path.join(temporary, "frontend"); + const managed = path.join(root, "target", "performance", "render-wasm"); + const renderer = path.join(temporary, "render-wasm"); + const runner = path.join(managed, "runner.cjs"); + fs.mkdirSync(path.join(root, "performance", "benches", "render_wasm"), { + recursive: true, + }); + fs.writeFileSync(path.join(root, "deps.edn"), "{}\n"); + fs.mkdirSync(managed, { recursive: true }); + fs.copyFileSync(bootstrap, runner); + fs.copyFileSync( + path.join(path.dirname(bootstrap), "commands.cjs"), + path.join(managed, "commands.cjs"), + ); + fs.symlinkSync( + path.join(frontendRoot, "node_modules"), + path.join(root, "node_modules"), + "dir", + ); + fs.mkdirSync(renderer); + fs.mkdirSync(path.join(managed, "compiler-cache")); + fs.writeFileSync( + path.join(managed, "compiler-cache", "retained.cache"), + "cache\n", + ); + return { + managed, + runner, + compiler: path.join(renderer, "build"), + rendererRoot: renderer, + builds: path.join(managed, "builds"), + runs: path.join(managed, "runs"), + comparisons: path.join(managed, "feature-comparisons"), + env: {}, + }; +} + +async function runCli(fixture, args) { + try { + const output = await execute( + process.execPath, + [...(fixture.nodeArgs ?? []), fixture.runner, ...args], + { + cwd: os.tmpdir(), + encoding: "utf8", + timeout: 15000, + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + env: { ...process.env, ...fixture.env }, + }, + ); + return { exitCode: 0, ...output }; + } catch (error) { + // A timeout, signal or spawn failure must fail the test, rather than + // masquerade as the expected nonzero CLI exit. + if (!Number.isInteger(error.code) || error.killed || error.signal) { + throw error; + } + return { + exitCode: error.code, + stdout: error.stdout, + stderr: error.stderr, + }; + } +} + +function readManifest(file) { + return transit.reader("json").read(fs.readFileSync(file, "utf8")); +} + +function requestText(command, options, color = "never") { + const k = transit.keyword; + return transit.writer("json").write(transit.map([ + k("command"), command, + k("console"), transit.map([k("level"), k("info"), k("color"), k(color)]), + k("options"), transit.map(Object.entries(options).flatMap(([key, value]) => [k(key), value])), + ])); +} + +test("deferred transport carries a complete request and numeric status", testOptions, async (t) => { + const fixture = createFrontend(t); + const capture = path.join(fixture.managed, "request.transit"); + fs.writeFileSync(path.join(fixture.managed, "commands.cjs"), ` + module.exports.execute = (text) => { + if (typeof text !== "string") throw new Error("expected Transit text"); + require("node:fs").writeFileSync(${JSON.stringify(capture)}, text); + return Promise.resolve(37); + }; + `); + const result = await runCli(fixture, ["run", "--build", "ready", "--warmups", "0", "--quiet", "--no-color"]); + assert.equal(result.exitCode, 37, result.stderr); + const k = transit.keyword; + const request = readManifest(capture); + assert.equal(request.get(k("command")), "run"); + assert.equal(request.get(k("console")).get(k("level")).toString(), ":warn"); + assert.equal(request.get(k("console")).get(k("color")).toString(), ":never"); + const options = request.get(k("options")); + assert.equal(options.get(k("build")).get(k("kind")).toString(), ":existing"); + assert.equal(options.get(k("build")).get(k("id")), "ready"); + assert.equal(options.get(k("warmups")), 0); + assert.equal(options.get(k("repetitions")), 10); + assert.equal(options.get(k("startup-timeout-ms")), 30000); +}); + +test("decoded requests reject incomplete, unsupported and policy-invalid commands before acquisition", testOptions, async (t) => { + const fixture = createFrontend(t); + fs.unlinkSync(path.resolve(fixture.managed, "../../../deps.edn")); + const entry = path.join(fixture.managed, "receive.cjs"); + fs.writeFileSync(entry, ` + require("./commands.cjs").execute(process.env.COMMAND_REQUEST) + .then(code => { process.exitCode = code; }); + `); + const before = tree(fixture.managed); + const measurement = { seed: 42, filter: null, warmups: 3, repetitions: 10, + "timeout-ms": 30000, "startup-timeout-ms": 30000, "output-dir": null, + screenshots: false, "require-gpu": false }; + const ab = { ...measurement, features: ["a"], "max-features": 3, confirm: false, + "dry-run": false, "build-timeout-ms": 300000, "allow-build-differences": false, + "allow-scene-differences": false, "allow-all-differences": false }; + const k = transit.keyword; + const build = { "default-features": true, features: [], "build-timeout-ms": 300000 }; + for (const [text, diagnostic] of [ + ["invalid Transit", /JSON/], + [requestText("build", {}), /default-features|build-timeout-ms/], + [requestText("purge", { target: k("all") }, "always"), /unsupported-command/], + [requestText("build", { ...build, features: ["profile"] }), /diagnostic-features/], + [requestText("run", { ...measurement, remove: false, + build: transit.map([k("kind"), k("prepare"), ...Object.entries({ ...build, features: ["stats"] }).flatMap(([key, value]) => [k(key), value])]) }), /diagnostic-features/], + [requestText("ab", ab), /confirmation-required/], + [requestText("ab", { ...ab, features: ["a", "b"], "max-features": 1, "dry-run": true }), /too-many-features/], + ]) { + fixture.env.COMMAND_REQUEST = text; + const result = await runCli({ ...fixture, runner: entry }, []); + assert.equal(result.exitCode, 1, result.stderr); + assert.equal(result.stdout, ""); + assert.match(result.stderr, diagnostic); + assert.equal((result.stderr.match(/✖/g) ?? []).length, 1); + assert.deepEqual(tree(fixture.managed), before); + } +}); + +test("final publication failures select a failing status in local and deferred commands", testOptions, async (t) => { + for (const command of [["--help"], ["build"]]) { + const fixture = createFrontend(t); + if (command[0] === "build") installPreparedCompiler(fixture, false); + const guard = path.join(fixture.managed, "writer.cjs"); + fs.writeFileSync(guard, `process.stdout.write = () => { throw new Error("publication unavailable"); };`); + fixture.nodeArgs = ["--require", guard]; + const result = await runCli(fixture, command); + assert.equal(result.exitCode, 1, result.stderr); + assert.match(result.stderr, /publication unavailable/); + assert.equal((result.stderr.match(/✖/g) ?? []).length, 1); + if (command[0] === "build") { + const [id] = fs.readdirSync(fixture.builds); + assert.equal(readManifest(path.join(fixture.builds, id, "build.transit")).get(transit.keyword("status")).toString(), ":ready"); + } + } +}); + +test("merge and purge execute with unavailable process, HTTP and browser dependencies", testOptions, async (t) => { + const fixture = createFrontend(t); + const guard = path.join(fixture.managed, "offline.cjs"); + fs.writeFileSync(guard, ` + const Module = require("node:module"); + const load = Module._load; + Module._load = function (id, ...args) { + if (["playwright", "http", "node:http", "child_process", "node:child_process"].includes(id)) { + throw new Error("Unavailable dependency: " + id); + } + return load.call(this, id, ...args); + }; + `); + fixture.nodeArgs = ["--require", guard]; + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + for (const [file, runId, value] of [[base, "base", 10], [additional, "additional", 12]]) { + fs.writeFileSync(file, runFixtureText({ runId, caseId: "case/load", value, gitSha: "same" })); + } + const merged = await runCli(fixture, ["merge", base, additional]); + assert.equal(merged.exitCode, 0, merged.stderr); + assert.match(merged.stdout, /^Run .+\nOutput .+\n$/); + seedBuilds(fixture); + const purged = await runCli(fixture, ["purge", "builds"]); + assert.equal(purged.exitCode, 0, purged.stderr); + for (const id of fs.readdirSync(fixture.builds)) { + assert.deepEqual(fs.readdirSync(path.join(fixture.builds, id)), ["keep.txt"]); + } +}); + +function manifestText(id, status, type) { + const k = transit.keyword; + return transit + .writer("json") + .write( + transit.map([k("type"), k(type), k("id"), id, k("status"), k(status)]), + ); +} + +function seedBuilds(fixture) { + for (const status of ["building", "failed", "ready"]) { + const id = randomUUID(); + const directory = path.join(fixture.builds, id); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "build.transit"), + manifestText(id, status, "renderer-build"), + ); + fs.writeFileSync(path.join(directory, "build.log"), `${status} build\n`); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + } +} + +function seedRuns(fixture) { + for (const status of ["complete", "failed"]) { + const id = randomUUID(); + const directory = path.join(fixture.runs, id); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "run.transit"), + `${status} run output\n`, + ); + fs.writeFileSync(path.join(directory, "run.log"), "run\n"); + fs.writeFileSync(path.join(directory, "case-load.PNG"), "screenshot\n"); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + } +} + +function seedComparisons(fixture) { + const id = randomUUID(); + const directory = path.join(fixture.comparisons, id); + const config = path.join(directory, "configs", "00"); + const edges = path.join(directory, "comparisons"); + fs.mkdirSync(config, { recursive: true }); + fs.mkdirSync(edges, { recursive: true }); + fs.mkdirSync(path.join(config, "shot.PNG"), { recursive: true }); + fs.writeFileSync( + path.join(directory, "feature-comparison.transit"), + "comparison\n", + ); + fs.writeFileSync( + path.join(directory, "feature-comparison.log"), + "progress\n", + ); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + for (const file of ["run.transit", "run.log", "case-load.PNG"]) { + fs.writeFileSync(path.join(config, file), "configuration output\n"); + } + fs.writeFileSync(path.join(config, "keep.txt"), "keep\n"); + fs.writeFileSync(path.join(config, "shot.PNG", "keep.txt"), "keep\n"); + for (const file of ["00__10.transit", "keep.txt"]) { + fs.writeFileSync(path.join(edges, file), "edge output\n"); + } + fs.writeFileSync(path.join(fixture.comparisons, "keep.txt"), "keep\n"); +} + +function installCargoMetadata(fixture, definitions) { + const manifest = path.join(fixture.rendererRoot, "Cargo.toml"); + const bin = path.join(path.dirname(fixture.rendererRoot), "fixture-bin"); + const metadata = path.join(fixture.rendererRoot, "metadata.json"); + const cargo = path.join(bin, "cargo"); + fs.writeFileSync( + manifest, + "[package]\nname = 'fixture'\nversion = '0.1.0'\n", + ); + fs.writeFileSync( + path.join(fixture.rendererRoot, "Cargo.lock"), + "version = 4\n", + ); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync( + metadata, + JSON.stringify({ + packages: [{ manifest_path: manifest, features: definitions }], + }), + ); + fs.writeFileSync( + cargo, + `#!/usr/bin/env node\nconst fs = require("node:fs");\nprocess.stdout.write(fs.readFileSync(process.env.CARGO_METADATA_FIXTURE, "utf8"));\n`, + { mode: 0o755 }, + ); + fixture.env.PATH = `${bin}${path.delimiter}${process.env.PATH}`; + fixture.env.CARGO_METADATA_FIXTURE = metadata; + return cargo; +} + +function installPreparedCompiler(fixture, breakLog, identityId = null) { + const k = transit.keyword; + const m = (pairs) => transit.map(pairs); + const marker = "RENDER_FIXTURE_BUILD_ID"; + const identity = transit + .writer("json") + .write( + m([k("build-id"), identityId ?? marker, k("cljs-profile-marks"), false]), + ); + const descriptor = m([ + k("id"), + k("fixture/load"), + k("scene"), + k("fixture"), + k("scene-description"), + "fixture", + k("scene-seed"), + 42, + k("params"), + m([]), + k("view"), + m([k("scale"), 1, k("x"), 0, k("y"), 0]), + k("context"), + k("fresh"), + k("preparation"), + m([]), + ]); + const catalog = transit + .writer("json") + .write(m([k("build-id"), identityId ?? marker, k("cases"), [descriptor]])); + const frontend = path.resolve(fixture.managed, "../../.."); + const host = path.join( + frontend, + "performance", + "benches", + "render_wasm", + "browser", + ); + const bin = path.join(fixture.rendererRoot, "fixture-bin"); + fs.mkdirSync(host, { recursive: true }); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(host, "index.html"), ""); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require('node:fs'); const path = require('node:path'); +fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, JSON.stringify({features: ['default'], profile: {opt_level: '3', debug_assertions: false}})); +for (const file of ['render-wasm.js', 'render-wasm.wasm', 'shared.js']) fs.writeFileSync(path.join(process.env.FRONTEND_DEST, file), 'fixture'); +`, + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(bin, "clojure"), + `#!/usr/bin/env node +const fs = require('node:fs'); const path = require('node:path'); +const id = process.env.RENDER_BENCH_BUILD_ID; +const identity = ${JSON.stringify(identity)}.replaceAll(${JSON.stringify(marker)}, id); +const catalog = ${JSON.stringify(catalog)}.replaceAll(${JSON.stringify(marker)}, id); +fs.writeFileSync(process.env.RENDER_BENCH_CASES_PATH, 'module.exports = {buildIdentity() {return ' + JSON.stringify(identity) + ';}, collectCases() {return ' + JSON.stringify(catalog) + ';}};'); +const log = path.join(path.dirname(process.env.RENDER_BENCH_CASES_PATH), 'build.log'); +const sabotage = ${breakLog ? "'fs.unlinkSync(' + JSON.stringify(log) + '); fs.mkdirSync(' + JSON.stringify(log) + ');'" : "''"}; +fs.writeFileSync(path.join(process.env.RENDER_BENCH_BROWSER_DIR, 'browser.js'), 'const fs = require("node:fs"); module.exports = {buildIdentity() {' + sabotage + 'return ' + JSON.stringify(identity) + ';}};'); +`, + { mode: 0o755 }, + ); + fixture.env.PATH = `${bin}${path.delimiter}${process.env.PATH}`; +} + + +test("unresolved compiler ownership exits while retaining its build and lock", testOptions, async (t) => { + for (const [command, interrupt] of [["build", null], ["run", "SIGINT"], ["ab", "SIGTERM"]]) { + const fixture = createFrontend(t); + if (command === "ab") installCargoMetadata(fixture, { default: [], a: [] }); + const groupFile = path.join(fixture.managed, "compiler-group.pid"); + const signalsFile = path.join(fixture.managed, "termination-signals.log"); + const lockFile = path.join(fixture.rendererRoot, "compiler.lock"); + const guard = path.join(fixture.managed, "unresolved-process.cjs"); + fs.writeFileSync(fixture.compiler, `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const { spawn } = require("node:child_process"); +fs.writeFileSync(path.join(process.env.FRONTEND_DEST, "partial.js"), "retained compiler output"); +const worker = spawn("flock", ["--exclusive", ${JSON.stringify(lockFile)}, process.execPath, "-e", + "process.stdout.write('compiler ready'); setInterval(() => {}, 1000);"], { stdio: "inherit" }); +worker.on("exit", (code) => { process.exitCode = code; }); +`, { mode: 0o755 }); + fs.writeFileSync(guard, ` +const fs = require("node:fs"); +const cp = require("node:child_process"); +const spawn = cp.spawn; +const kill = process.kill; +const interrupt = ${JSON.stringify(interrupt)}; +let group; +cp.spawn = function (command, ...args) { + const child = spawn.call(this, command, ...args); + if (command === ${JSON.stringify(fixture.compiler)} && child.pid) { + group = child.pid; + fs.writeFileSync(${JSON.stringify(groupFile)}, String(group)); + if (interrupt) child.stdout.once("data", () => process.kill(process.pid, interrupt)); + } + return child; +}; +process.kill = function (pid, signal) { + if (pid === -group && ["SIGTERM", "SIGKILL"].includes(signal)) { + fs.appendFileSync(${JSON.stringify(signalsFile)}, signal + "\\n"); + return true; + } + return kill.apply(this, arguments); +}; +require("node:module").syncBuiltinESMExports(); +`); + fixture.nodeArgs = ["--require", guard]; + const args = command === "ab" ? ["ab", "--features", "a", "--filter", "rects/load"] : [command]; + if (!interrupt) args.push("--build-timeout-ms", "1000"); + if (command === "run") args.push("--remove"); + args.push("--no-color"); + try { + const result = await runCli(fixture, args); + assert.equal(result.exitCode, interrupt === "SIGINT" ? 130 : interrupt === "SIGTERM" ? 143 : 1, result.stderr); + assert.match(result.stderr, /unresolved/); + assert.match(result.stderr, /pgid/); + assert.ok(result.stderr.includes(fixture.compiler), result.stderr); + assert.deepEqual(fs.readFileSync(signalsFile, "utf8").trim().split("\n"), ["SIGTERM", "SIGKILL"]); + const group = Number(fs.readFileSync(groupFile, "utf8")); + assert.ok(process.kill(group, 0)); + assert.equal(spawnSync("flock", ["--nonblock", lockFile, "true"], { timeout: 1000 }).status, 1); + const ids = fs.readdirSync(fixture.builds); + assert.equal(ids.length, 1); + const directory = path.join(fixture.builds, ids[0]); + const manifest = readManifest(path.join(directory, "build.transit")); + assert.equal(manifest.get(transit.keyword("status")).toString(), ":failed"); + assert.equal(manifest.get(transit.keyword("failure")).get(transit.keyword("unresolved-process")), true); + assert.equal(fs.readFileSync(path.join(directory, "renderer", "partial.js"), "utf8"), "retained compiler output"); + if (command === "ab") { + const k = transit.keyword; + const [id] = fs.readdirSync(fixture.comparisons); + const comparison = readManifest(path.join(fixture.comparisons, id, "feature-comparison.transit")); + assert.equal(comparison.get(k("status")).toString(), ":interrupted"); + assert.deepEqual(comparison.get(k("configurations")).map((entry) => entry.get(k("status")).toString()), + [":failed", ":unattempted"]); + assert.deepEqual(comparison.get(k("edges")).map((entry) => entry.get(k("status")).toString()), [":unattempted"]); + } + } finally { + if (fs.existsSync(groupFile)) { + const group = Number(fs.readFileSync(groupFile, "utf8")); + assert.ok(Number.isSafeInteger(group) && group > 0); + try { + process.kill(-group, "SIGKILL"); + } catch (error) { + if (error.code !== "ESRCH") throw error; + } + const deadline = Date.now() + 3000; + while (spawnSync("flock", ["--nonblock", lockFile, "true"], { timeout: 1000 }).status === 1) { + assert.ok(Date.now() < deadline, "Owned fixture lock did not release"); + await pause(10); + } + } + } + } +}); + +function runFixtureText({ + runId, + caseId, + value, + environment = { node: "v24" }, + mode = "release", + features = [], + scene = "case", + sceneSeed = 7, + count = 10, + fingerprint = "a".repeat(64), + view = { scale: 1, x: 0, y: 0 }, + context = "fresh", + warmups = 0, + repetitions = 1, + gitSha = runId, +}) { + const k = transit.keyword; + const map = (pairs) => transit.map(pairs); + const event = map([ + k("command"), k("fixture/work"), + k("metrics"), map([k("command/elapsed"), [value]]), + ]); + const preparation = map([ + k("preparation-id"), + `${runId}-preparation`, + k("case-id"), + k(caseId), + k("warmup?"), + false, + k("fingerprint"), + map([k("algorithm"), k("sha-256"), k("digest"), fingerprint]), + k("effective-graphics"), + map([k("webgl2"), true]), + k("diagnostics"), + map([]), + k("phases"), + [map([ + k("phase"), k("preparation"), + k("preparation-id"), `${runId}-preparation`, + k("outcome"), k("completed"), + k("events"), [], + ])], + ]); + const makeAttempt = (attemptId, warmup) => + map([ + k("attempt-id"), + attemptId, + k("case-id"), + k(caseId), + k("preparation-id"), + `${runId}-preparation`, + k("warmup?"), + warmup, + k("outcome"), + k("completed"), + k("phases"), + [map([ + k("phase"), k("main"), + k("preparation-id"), `${runId}-preparation`, + k("outcome"), k("completed"), + k("events"), [event], + ])], + ]); + const attempts = [ + ...Array.from({ length: warmups }, (_, index) => makeAttempt(index, true)), + ...Array.from({ length: repetitions }, (_, index) => + makeAttempt(warmups + index, false), + ), + ]; + const record = map([ + k("type"), + k("run-record"), + k("run-id"), + runId, + k("started-at"), + "2026-10-26T00:00:00.000Z", + k("ended-at"), + "2026-10-26T00:01:00.000Z", + k("plan"), + map([ + k("cases"), + [ + map([ + k("id"), + k(caseId), + k("scene"), + k(scene), + k("scene-seed"), + sceneSeed, + k("params"), + map([k("count"), count]), + k("view"), + map(Object.entries(view).flatMap(([key, entry]) => [k(key), entry])), + k("context"), + k(context), + k("preparation"), + map([]), + ]), + ], + k("warmups"), + warmups, + k("repetitions"), + repetitions, + k("timeout-policy"), + map([k("case-ms"), 30000]), + ]), + k("metadata"), + map([ + k("git"), + map([k("sha"), gitSha, k("dirty"), false]), + k("build"), + map([ + k("functional"), + map([k("mode"), k(mode), k("features"), features]), + ]), + k("environment"), + map( + Object.entries(environment).flatMap(([key, entry]) => [k(key), entry]), + ), + k("scored?"), + true, + k("diagnostics"), + map([]), + ]), + k("analysis"), + map([ + k("draws"), + 2000, + k("seed"), + 1885695600, + k("confidence"), + 0.95, + k("min-ci"), + 10, + k("min-p95"), + 100, + k("min-p99"), + 500, + ]), + k("quantities"), + map([k("command/elapsed"), map([k("unit"), k("ms")])]), + k("preparations"), + [preparation], + k("attempts"), + attempts, + k("case-outcomes"), + map([k(caseId), map([k("outcome"), k("completed")])]), + k("remaining"), + map([k(caseId), map([k("warmups"), 0, k("repetitions"), 0])]), + k("termination"), + map([k("reason"), k("completed")]), + ]); + return transit.writer("json").write(record); +} + +function tree(directory) { + return fs + .readdirSync(directory, { withFileTypes: true }) + .sort((a, b) => a.name.localeCompare(b.name)) + .map((entry) => { + const file = path.join(directory, entry.name); + return [ + entry.name, + entry.isDirectory() ? tree(file) : fs.readFileSync(file), + ]; + }); +} + +const failingCompiler = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const directory = path.dirname(process.env.FRONTEND_DEST); +fs.copyFileSync( + path.join(directory, "build.transit"), + path.join(directory, "before-compiler.transit") +); +fs.writeFileSync( + path.join(process.env.FRONTEND_DEST, "partial.js"), + "partial renderer output\\n" +); +process.stdout.write("compiler stdout before failure\\n"); +process.stderr.write("compiler stderr before failure\\n"); +process.exitCode = 23; +`; + +test( + "build prints its allocation and one failure block with retained compiler evidence", + testOptions, + async (t) => { + const fixture = createFrontend(t); + fs.writeFileSync(fixture.compiler, failingCompiler, { mode: 0o755 }); + + const args = ["build", "--build-timeout-ms", "10000"]; + const result = await runCli(fixture, args); + assert.equal(result.exitCode, 1); + const [buildLine] = result.stdout.trim().split("\n"); + const id = buildLine.slice("Build ".length); + assert.match( + id, + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i, + ); + assert.equal( + result.stdout, + `Build ${id}\nOutput ${path.join(fixture.builds, id)}\n`, + ); + assert.deepEqual(fs.readdirSync(fixture.builds), [id]); + + const directory = path.join(fixture.builds, id); + const logPath = path.join(directory, "build.log"); + const initial = readManifest( + path.join(directory, "before-compiler.transit"), + ); + const manifest = readManifest(path.join(directory, "build.transit")); + const k = transit.keyword; + assert.equal(initial.get(k("id")), id); + assert.equal(initial.get(k("status")).toString(), ":building"); + assert.equal(manifest.get(k("id")), id); + assert.equal(manifest.get(k("status")).toString(), ":failed"); + const failure = manifest.get(k("failure")); + assert.equal(failure.get(k("phase")), "renderer"); + assert.equal(failure.get(k("exit-code")), 23); + assert.equal(failure.get(k("command")), fixture.compiler); + assert.equal( + failure.get(k("cause"))[0].get(k("type")).toString(), + ":process-exit", + ); + assert.equal( + fs.readFileSync(path.join(directory, "renderer", "partial.js"), "utf8"), + "partial renderer output\n", + ); + + const log = fs.readFileSync(logPath, "utf8"); + for (const output of [log, result.stderr]) { + assert.ok(output.includes("compiler stdout before failure\n")); + assert.ok(output.includes("compiler stderr before failure\n")); + } + assert.match(result.stderr, /renderer build preparation failed/); + assert.match( + result.stderr, + /type\s+:benches.render-wasm.node.runner.build\/preparation-failed/, + ); + assert.match(result.stderr, new RegExp(`build\\s+${id}`)); + assert.ok(result.stderr.includes(logPath)); + assert.match(result.stderr, /phase\s+:renderer/); + assert.match(result.stderr, /exit-code\s+23/); + assert.ok(result.stderr.includes(fixture.compiler), result.stderr); + assert.equal(result.stderr.split("✖ ").length - 1, 1); + assert.doesNotMatch(result.stderr, /^compiler (stdout|stderr)/m); + assert.match(log, /✖ renderer build preparation failed/); + }, +); + +test( + "build retains selected and loaded identities in saved and terminal failures", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const actualId = `other\n\u001b${"x".repeat(2048)}`; + installPreparedCompiler(fixture, false, actualId); + + const result = await runCli(fixture, [ + "build", + "--no-color", + "--build-timeout-ms", + "10000", + ]); + + assert.equal(result.exitCode, 1, result.stderr); + const id = result.stdout.match(/^Build ([0-9a-f-]+)$/m)?.[1]; + assert.ok(id, result.stdout); + const k = transit.keyword; + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + const failure = manifest.get(k("failure")); + assert.equal(manifest.get(k("status")).toString(), ":failed"); + assert.equal(failure.get(k("phase")), "identity"); + assert.equal( + failure.get(k("cause"))[0].get(k("type")).toString(), + ":benches.render-wasm.identity/mismatch", + ); + assert.equal(failure.get(k("expected-id")), id); + assert.equal(failure.get(k("actual-id")), actualId.slice(0, 1024)); + assert.match(result.stderr, new RegExp(`expected-id\\s+"${id}"`)); + const actualField = result.stderr.match(/^ actual-id\s+(.+)$/m)?.[1]; + assert.ok(actualField, result.stderr); + assert.ok(actualField.includes("other"), actualField); + assert.ok(actualField.length <= 1024, actualField.length); + assert.ok( + [...actualField].every((character) => character.charCodeAt(0) >= 32), + actualField, + ); + }, +); + +test( + "build failure diagnostics retain semantic and storage causes", + testOptions, + async (t) => { + for (const [source, reason, broken] of [ + [ + `#!/usr/bin/env node +const fs = require('node:fs'); +fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, JSON.stringify({features: [], profile: {opt_level: '0', debug_assertions: false}})); +`, + /Cargo did not produce an optimized renderer/, + ], + [ + failingCompiler + + "\nfs.unlinkSync(require('node:path').join(require('node:path').dirname(process.env.FRONTEND_DEST), 'build.transit'));\n", + /Command exited unsuccessfully/, + "manifest", + ], + [ + `#!/usr/bin/env node +const fs = require('node:fs'); +const path = require('node:path'); +const log = path.join(path.dirname(process.env.FRONTEND_DEST), 'build.log'); +fs.unlinkSync(log); fs.mkdirSync(log); +process.stdout.write('compiler output\\n'); +`, + /cannot append benchmark log/, + "log", + ], + ]) { + const fixture = createFrontend(t); + fs.writeFileSync(fixture.compiler, source, { mode: 0o755 }); + const result = await runCli(fixture, ["build"]); + const [id] = fs.readdirSync(fixture.builds); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /✖ renderer build preparation failed/); + assert.match(result.stderr, reason); + assert.ok(result.stderr.includes(id)); + assert.ok( + result.stderr.includes(path.join(fixture.builds, id, "build.log")), + ); + if (broken !== "log") { + const log = fs.readFileSync( + path.join(fixture.builds, id, "build.log"), + "utf8", + ); + assert.match(log, /✖ renderer build preparation failed/); + assert.match(log, reason); + } + if (broken === "manifest") { + assert.match(result.stderr, /manifest-error\s+.*ENOENT/); + } else { + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + assert.equal( + manifest.get(transit.keyword("status")).toString(), + ":failed", + ); + } + } + }, +); + +test( + "build completion retains a usable ready build or the original log failure", + testOptions, + async (t) => { + for (const breakLog of [false, true]) { + const fixture = createFrontend(t); + installPreparedCompiler(fixture, breakLog); + const result = await runCli(fixture, ["build"]); + const [id] = fs.readdirSync(fixture.builds); + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + assert.equal(result.exitCode, breakLog ? 1 : 0, result.stderr); + assert.equal( + manifest.get(transit.keyword("status")).toString(), + breakLog ? ":failed" : ":ready", + ); + if (breakLog) { + assert.match(result.stderr, /cannot append benchmark log/); + assert.ok( + result.stderr.includes(path.join(fixture.builds, id, "build.log")), + ); + } else { + assert.match(result.stderr, /ready/); + } + } + }, +); + +test( + "output flags control progress, raw output and console colors", + testOptions, + async (t) => { + for (const [flags, phases, raw, colors] of [ + [[], true, false, false], + [["--verbose"], true, true, false], + [["--quiet"], false, false, false], + [["--color"], true, false, true], + [["--no-color"], true, false, false], + ]) { + const fixture = createFrontend(t); + fs.writeFileSync(fixture.compiler, failingCompiler, { mode: 0o755 }); + const result = await runCli(fixture, ["build", ...flags]); + assert.equal(result.exitCode, 1, result.stderr); + assert.match(result.stdout, /^Build [^\n]+\nOutput \/[^\n]+\n$/); + assert.equal( + /^(?:\u001b\[[0-9;]*m)?Renderer compiling/m.test(result.stderr), + phases, + ); + assert.equal( + result.stderr.includes("· compiler stdout before failure"), + raw, + ); + assert.equal(result.stderr.includes("\u001b["), colors); + assert.match(result.stderr, /! Git provenance unavailable:/); + const [id] = fs.readdirSync(fixture.builds); + const log = fs.readFileSync( + path.join(fixture.builds, id, "build.log"), + "utf8", + ); + assert.match(log, /^compiler stdout before failure$/m); + assert.match(log, /^Renderer compiling$/m); + assert.doesNotMatch(log, /\u001b\[/); + } + }, +); + +test( + "ab retains its log and names the nested build tail on failure", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + fs.writeFileSync(fixture.compiler, failingCompiler, { mode: 0o755 }); + const result = await runCli(fixture, [ + "ab", + "--features", + "a", + "--filter", + "rects/load", + ]); + assert.equal(result.exitCode, 1, result.stderr); + const [idLine, outputLine, statusLine] = result.stdout.trim().split("\n"); + assert.match(idLine, /^Comparison [0-9a-f-]+$/); + const directory = outputLine.slice("Output ".length); + assert.equal(statusLine, "Status failed"); + const logPath = path.join(directory, "feature-comparison.log"); + assert.equal(result.stderr.split("✖ ").length - 1, 1); + assert.ok(result.stderr.includes(logPath)); + assert.match(result.stderr, /tail-log\s+.*build\.log/); + assert.match( + fs.readFileSync(logPath, "utf8"), + /compiler stderr before failure/, + ); + await runCli(fixture, ["purge", "comparisons"]); + assert.equal(fs.existsSync(logPath), false); + }, +); + +test( + "ab prints status once and preserves the build failure when its log becomes unwritable", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const output = path.join(fixture.managed, "comparison-output"); + fixture.env.COMPARISON_LOG = path.join(output, "feature-comparison.log"); + fs.writeFileSync( + fixture.compiler, + failingCompiler + + "\nfs.unlinkSync(process.env.COMPARISON_LOG); fs.mkdirSync(process.env.COMPARISON_LOG);\n", + { mode: 0o755 }, + ); + const result = await runCli(fixture, [ + "ab", + "--features", + "a", + "--filter", + "rects/load", + "--output-dir", + output, + ]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout.split("Status failed\n").length - 1, 1); + assert.equal(result.stdout.trim().split("\n").length, 3); + assert.match(result.stderr, /✖ renderer build preparation failed/); + }, +); + +test( + "run and ab retain signal exit codes during preparation", + testOptions, + async (t) => { + for (const command of ["run", "ab"]) { + for (const [signal, code] of [ + ["SIGINT", 130], + ["SIGTERM", 143], + ]) { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const marker = path.join(fixture.managed, "compiler-started"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node\nconst fs = require('node:fs'); fs.writeFileSync(${JSON.stringify(marker)}, 'ready'); setInterval(() => {}, 1000);\n`, + { mode: 0o755 }, + ); + const args = + command === "ab" + ? ["ab", "--features", "a", "--filter", "rects/load"] + : ["run"]; + const failPublication = command === "run" && signal === "SIGINT"; + if (failPublication) { + const writer = path.join(fixture.managed, "writer.cjs"); + fs.writeFileSync(writer, `process.stdout.write = () => { throw new Error("publication unavailable"); };`); + fixture.nodeArgs = ["--require", writer]; + } + const child = spawn(process.execPath, [...(fixture.nodeArgs ?? []), fixture.runner, ...args], { + cwd: os.tmpdir(), + env: { ...process.env, ...fixture.env }, + stdio: ["ignore", "pipe", "pipe"], + }); + t.after(() => { + if (child.exitCode === null) child.kill("SIGKILL"); + }); + const exited = once(child, "exit"); + const closed = once(child, "close"); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const deadline = Date.now() + 10000; + while ( + !fs.existsSync(marker) && + child.exitCode === null && + Date.now() < deadline + ) + await pause(10); + assert.ok(fs.existsSync(marker), stderr); + child.kill(signal); + assert.deepEqual(await exited, [code, null], stderr); + await closed; + if (command === "ab") { + assert.match(stdout, /^Status interrupted$/m); + assert.equal(stdout.trim().split("\n").length, 3); + } + if (failPublication) assert.match(stderr, /publication unavailable/); + } + } + }, +); + +test( + "run records an early retained-build failure in its allocated log", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const output = path.join(fixture.managed, "early-run"); + const result = await runCli(fixture, [ + "run", + "--build", + "missing", + "--output-dir", + output, + ]); + assert.equal(result.exitCode, 1); + const logPath = path.join(output, "run.log"); + assert.ok(result.stderr.includes(logPath)); + assert.match(fs.readFileSync(logPath, "utf8"), /✖ /); + }, +); + +test( + "ab dry-run prints the Gray plan without creating comparison output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + fixture.env.PENPOT_GPU = "0"; + installCargoMetadata(fixture, { default: [], a: [], b: ["a"] }); + const retained = tree(fixture.managed); + + const result = await runCli(fixture, [ + "ab", + "--features", + "b,a", + "--filter", + "rects/load", + "--dry-run", + "--require-gpu", + ]); + + assert.equal(result.exitCode, 0); + assert.equal(result.stderr, ""); + assert.match(result.stdout, /^Feature comparison plan/); + for (const part of [ + "Features: a, b", + "00: ", + "01: b", + "11: a, b", + "10: a", + "00 -> 10: a", + "00 -> 01: b", + "01 -> 11: a", + "10 -> 11: b", + "Cases: 1", + ]) { + assert.ok(result.stdout.includes(part), part); + } + assert.equal(fs.existsSync(fixture.comparisons), false); + assert.deepEqual(tree(fixture.managed), retained); + }, +); + +test( + "ab validates requested features before creating comparison output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const retained = tree(fixture.managed); + + const result = await runCli(fixture, [ + "ab", + "--features", + "missing", + "--filter", + "rects/load", + "--dry-run", + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /unknown requested Cargo features/); + assert.equal(fs.existsSync(fixture.comparisons), false); + assert.deepEqual(tree(fixture.managed), retained); + }, +); + +test("purge deletes only the selected artifacts", testOptions, async (t) => { + const fixture = createFrontend(t); + const retained = tree(fixture.managed); + seedBuilds(fixture); + seedRuns(fixture); + seedComparisons(fixture); + const runs = tree(fixture.runs); + const comparisons = tree(fixture.comparisons); + + assert.deepEqual(await runCli(fixture, ["purge", "builds"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + for (const id of fs.readdirSync(fixture.builds)) { + assert.deepEqual(fs.readdirSync(path.join(fixture.builds, id)), [ + "keep.txt", + ]); + } + assert.deepEqual(tree(fixture.runs), runs); + assert.deepEqual(tree(fixture.comparisons), comparisons); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + seedBuilds(fixture); + const builds = tree(fixture.builds); + assert.deepEqual(await runCli(fixture, ["purge", "runs"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + for (const id of fs.readdirSync(fixture.runs)) { + assert.deepEqual(fs.readdirSync(path.join(fixture.runs, id)), ["keep.txt"]); + } + assert.deepEqual(tree(fixture.builds), builds); + assert.deepEqual(tree(fixture.comparisons), comparisons); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + const buildsBeforeComparisonPurge = tree(fixture.builds); + const runsBeforeComparisonPurge = tree(fixture.runs); + assert.deepEqual(await runCli(fixture, ["purge", "comparisons"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + const [comparisonId] = fs.readdirSync(fixture.comparisons); + const comparison = path.join(fixture.comparisons, comparisonId); + for (const file of [ + "feature-comparison.log", + "feature-comparison.transit", + "configs/00/run.transit", + "configs/00/run.log", + "configs/00/case-load.PNG", + "comparisons/00__10.transit", + ]) { + assert.equal(fs.existsSync(path.join(comparison, file)), false, file); + } + for (const file of [ + "keep.txt", + "configs/00/keep.txt", + "configs/00/shot.PNG/keep.txt", + "comparisons/keep.txt", + ]) { + assert.equal(fs.existsSync(path.join(comparison, file)), true, file); + } + assert.deepEqual(tree(fixture.builds), buildsBeforeComparisonPurge); + assert.deepEqual(tree(fixture.runs), runsBeforeComparisonPurge); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + seedRuns(fixture); + seedComparisons(fixture); + assert.deepEqual(await runCli(fixture, ["purge", "all"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + for (const root of [fixture.builds, fixture.runs]) { + for (const id of fs.readdirSync(root)) { + assert.deepEqual(fs.readdirSync(path.join(root, id)), ["keep.txt"]); + } + } + for (const id of fs.readdirSync(fixture.comparisons)) { + const directory = path.join(fixture.comparisons, id); + if (id === "keep.txt") continue; + for (const file of [ + "feature-comparison.transit", + "configs/00/run.transit", + "configs/00/run.log", + "configs/00/case-load.PNG", + "comparisons/00__10.transit", + ]) { + assert.equal(fs.existsSync(path.join(directory, file)), false, file); + } + assert.equal(fs.existsSync(path.join(directory, "keep.txt")), true); + } + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); +}); + +test( + "run reserves nested output before the compiler starts", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const output = path.join(fixture.managed, "missing", "parents", "output"); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require("node:fs"); +if (!fs.statSync(${JSON.stringify(path.join(output, "run.log"))}).isFile()) process.exit(99); +fs.writeFileSync(${JSON.stringify(marker)}, "reserved"); +process.exit(23); +`, + { mode: 0o755 }, + ); + + const result = await runCli(fixture, ["run", "--output-dir", output]); + + assert.equal(result.exitCode, 1); + assert.match( + result.stdout, + new RegExp(`^Run [0-9a-f-]+\\nOutput ${output}\\n$`), + ); + assert.equal(fs.readFileSync(marker, "utf8"), "reserved"); + assert.equal(fs.existsSync(path.join(output, "run.transit")), false); + }, +); + +test( + "run rejects invalid output parents before invoking the compiler", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const parent = path.join(fixture.managed, "not-a-directory"); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync(parent, "keep"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +require("node:fs").writeFileSync(${JSON.stringify(marker)}, "ran"); +process.exit(23); +`, + { mode: 0o755 }, + ); + + const result = await runCli(fixture, [ + "run", + "--output-dir", + path.join(parent, "output"), + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /ENOTDIR|cannot create run home/); + assert.equal(fs.existsSync(marker), false); + assert.equal(fs.readFileSync(parent, "utf8"), "keep"); + }, +); + +test( + "run rejects --remove outputs inside builds including directory aliases", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const marker = path.join(fixture.managed, "compiler-ran"); + const alias = path.join(path.dirname(fixture.managed), "build-alias"); + fs.mkdirSync(fixture.builds); + fs.symlinkSync(fixture.builds, alias, "dir"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +require("node:fs").writeFileSync(${JSON.stringify(marker)}, "ran"); +process.exit(23); +`, + { mode: 0o755 }, + ); + + for (const root of [fixture.builds, alias]) { + const output = path.join(root, "missing", "output"); + const result = await runCli(fixture, [ + "run", + "--remove", + "--output-dir", + output, + ]); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /--remove.*inside builds/); + assert.equal(fs.existsSync(output), false); + } + assert.equal(fs.existsSync(marker), false); + }, +); + +test( + "malformed purge categories leave all files intact", + testOptions, + async (t) => { + const fixture = createFrontend(t); + seedBuilds(fixture); + seedRuns(fixture); + const original = tree(fixture.managed); + + for (const args of [ + ["purge"], + ["purge", "unknown"], + ["purge", "builds", "runs"], + ]) { + const result = await runCli(fixture, args); + assert.equal(result.exitCode, 1, args.join(" ")); + assert.equal(result.stdout, ""); + assert.match( + result.stderr, + /Usage: runner purge/, + ); + assert.deepEqual(tree(fixture.managed), original, args.join(" ")); + } + }, +); + +test( + "required GPU access rejects run and ab before allocating output or checking builds", + testOptions, + async (t) => { + for (const command of [ + ["run"], + ["ab", "--features", "a", "--filter", "rects/load"], + ]) { + const fixture = createFrontend(t); + fixture.env.PENPOT_GPU = ""; + const original = tree(fixture.managed); + const result = await runCli(fixture, [ + ...command, + "--require-gpu", + "--output-dir", + path.join(fixture.managed, "gpu-run"), + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /PENPOT_GPU=1/); + assert.match(result.stderr, /\.\/manage\.sh run-devenv --gpu/); + assert.match(result.stderr, /pnpm run render-bench/); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "run validates an explicit output dir before preparing a build", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require("node:fs"); +fs.writeFileSync(${JSON.stringify(marker)}, "ran"); +`, + { mode: 0o755 }, + ); + const output = path.join(fixture.managed, "existing-output"); + fs.mkdirSync(output); + const original = tree(fixture.managed); + + const result = await runCli(fixture, [ + "run", + "--output-dir", + output, + "--warmups", + "0", + "--repetitions", + "1", + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /output directory already exists/); + assert.equal(fs.existsSync(marker), false); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test( + "run rejects retained-build feature selection before any work", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require("node:fs"); +fs.writeFileSync(${JSON.stringify(marker)}, "ran"); +`, + { mode: 0o755 }, + ); + const original = tree(fixture.managed); + + for (const flags of [["--features", "functional"], ["--build-timeout-ms", "1000"]]) { + const result = await runCli(fixture, ["run", "--build", "retained-id", ...flags]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /cannot be combined/); + assert.equal(fs.existsSync(marker), false); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "help and compare work without a frontend checkout or execution dependencies", + testOptions, + async (t) => { + const temporary = fs.mkdtempSync( + path.join(os.tmpdir(), "render-wasm-offline-"), + ); + t.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const runner = path.join(temporary, "runner.cjs"); + const guard = path.join(temporary, "guard.cjs"); + fs.copyFileSync(bootstrap, runner); + fs.symlinkSync( + path.join(frontendRoot, "node_modules"), + path.join(temporary, "node_modules"), + "dir", + ); + fs.writeFileSync( + guard, + ` + const Module = require("node:module"); + const load = Module._load; + Module._load = function (id, ...args) { + if (["playwright", "http", "node:http", "child_process", "node:child_process"].includes(id)) { + throw new Error("Offline command imported execution dependency: " + id); + } + return load.call(this, id, ...args); + }; + `, + ); + const baseline = path.join(temporary, "baseline.transit"); + const candidate = path.join(temporary, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), + ); + const fixture = { runner, nodeArgs: ["--require", guard] }; + + const help = await runCli(fixture, ["--help"]); + assert.equal(help.exitCode, 0, help.stderr); + assert.match(help.stdout, /runner compare/); + assert.match(help.stdout, /runner merge/); + const abHelp = await runCli(fixture, ["ab", "--help"]); + assert.equal(abHelp.exitCode, 0, abHelp.stderr); + assert.match(abHelp.stdout, /runner ab --features/); + assert.match(abHelp.stdout, /--max-features/); + const mergeHelp = await runCli(fixture, ["merge", "--help"]); + assert.equal(mergeHelp.exitCode, 0, mergeHelp.stderr); + assert.match(mergeHelp.stdout, /runner merge BASE ADDITIONAL/); + for (const args of [["ab"], ["ab", "--dry-run"]]) { + const invalid = await runCli(fixture, args); + assert.equal(invalid.exitCode, 1, invalid.stderr); + assert.match(invalid.stderr, /at least one requested feature/); + assert.doesNotMatch( + invalid.stderr, + /execution dependency|Cannot find module/, + ); + } + const comparison = await runCli(fixture, ["compare", baseline, candidate]); + assert.equal(comparison.exitCode, 0, comparison.stderr); + assert.match(comparison.stdout, /median change/); + for (const flag of [ + "--allow-build-differences", + "--allow-scene-differences", + "--allow-all-differences", + ]) { + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + flag, + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /median change/); + } + assert.equal(fs.existsSync(path.join(temporary, "commands.cjs")), false); + }, +); + +test( + "help lists run and compare without touching managed storage", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const original = tree(fixture.managed); + + const result = await runCli(fixture, ["--help"]); + + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /runner run/); + assert.match(result.stdout, /runner compare/); + assert.match(result.stdout, /runner merge/); + const compareHelp = await runCli(fixture, ["compare", "--help"]); + assert.equal(compareHelp.exitCode, 0, compareHelp.stderr); + assert.match(compareHelp.stdout, /--cases/); + assert.match(compareHelp.stdout, /--allow-build-differences/); + assert.match(compareHelp.stdout, /--allow-scene-differences/); + assert.match(compareHelp.stdout, /--allow-all-differences/); + assert.match(result.stdout, /-q, --quiet/); + assert.match(result.stdout, /-v, --verbose/); + assert.match(result.stdout, /--\[no-\]color/); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test( + "compare and merge identify unreadable saved inputs", + testOptions, + async (t) => { + const cases = [ + { + command: "compare", + role: "baseline", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "corrupt", + message: /JSON/, + type: null, + }, + { + command: "compare", + role: "baseline", + kind: "unsupported", + message: /unsupported benchmark format/, + type: /result\/unsupported-format/, + }, + { + command: "compare", + role: "candidate", + kind: "invalid", + message: /invalid schema:run/, + type: /result\/invalid-record/, + }, + { + command: "compare", + role: "candidate", + kind: "directory", + message: /EISDIR/, + type: null, + }, + { + command: "merge", + role: "base", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "merge", + role: "additional", + kind: "corrupt", + message: /JSON/, + type: null, + }, + ]; + for (const { command, role, kind, message, type } of cases) { + await t.test(`${command}: ${role} ${kind}`, async (t) => { + const fixture = createFrontend(t); + const valid = path.join(fixture.managed, "valid.transit"); + const input = path.join(fixture.managed, `${kind}.transit`); + fs.writeFileSync( + valid, + runFixtureText({ runId: "valid", caseId: "case/load", value: 10 }), + ); + if (kind === "directory") { + fs.mkdirSync(input); + } else if (kind !== "missing") { + const text = + kind === "corrupt" + ? "not JSON" + : kind === "unsupported" + ? "{}" + : transit + .writer("json") + .write( + transit.map([ + transit.keyword("type"), + transit.keyword("run-record"), + ]), + ); + fs.writeFileSync(input, text); + } + const inputs = + role === "baseline" || role === "base" + ? [input, valid] + : [valid, input]; + const result = await runCli(fixture, [ + command, + ...inputs, + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, message); + assert.ok(result.stderr.includes(input), result.stderr); + assert.match(result.stderr, new RegExp(`role\\s+:${role}\\b`)); + assert.match(result.stderr, /operation\s+:read-record\b/); + if (type) assert.match(result.stderr, type); + }); + } + }, +); + +test( + "compare bounds and escapes the reported input path", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const prefix = 'missing\n"\\\u001b'; + const prefixLength = JSON.stringify(prefix).length - 1; + const input = `${prefix}${"x".repeat(1023 - prefixLength)}😀.transit`; + const result = await runCli(fixture, [ + "compare", + input, + "unused.transit", + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /role\s+:baseline\b/); + const reportedPath = result.stderr.match(/^ path\s+(.+)$/m)?.[1]; + assert.ok(reportedPath, result.stderr); + assert.match(reportedPath, /^"missing\\n\\"\\\\\\u001b/); + assert.ok(reportedPath.length <= 1024, reportedPath.length); + assert.ok(reportedPath.endsWith("x"), reportedPath); + }, +); + +test( + "compare reads saved run files without starting resources", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), + ); + const original = tree(fixture.managed); + + const result = await runCli(fixture, ["compare", baseline, candidate]); + + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /Baseline/); + assert.match(result.stdout, /case\/load/); + assert.match(result.stdout, /median change/); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test( + "compare supports explicit pairs and all-differences from the CLI", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "b/pan", + value: 12, + environment: { node: "v25" }, + }), + ); + const original = tree(fixture.managed); + + const incompatible = await runCli(fixture, [ + "compare", + baseline, + candidate, + ]); + assert.equal(incompatible.exitCode, 1); + assert.deepEqual(tree(fixture.managed), original); + + const selected = ["compare", baseline, candidate, "--cases", "a/pan,b/pan"]; + const strict = await runCli(fixture, selected); + assert.equal(strict.exitCode, 1); + assert.match(strict.stderr, /incompatible saved runs/); + + const allowed = await runCli(fixture, [ + ...selected, + "--allow-all-differences", + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /:allow-all-differences true/); + assert.match(allowed.stdout, /ACKNOWLEDGED DIFFERENCE/); + assert.match(allowed.stdout, /:rule :all/); + assert.match(allowed.stdout, /Pair a\/pan,b\/pan/); + assert.deepEqual(tree(fixture.managed), original); + + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "a/pan", + value: 12, + environment: { node: "v25" }, + }), + ); + const automatic = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-all-differences", + ]); + assert.equal(automatic.exitCode, 0, automatic.stderr); + assert.match(automatic.stdout, /:allow-all-differences true/); + assert.match(automatic.stdout, /ACKNOWLEDGED DIFFERENCE/); + assert.match(automatic.stdout, /Pair a\/pan,a\/pan/); + }, +); + +test( + "compare applies build and scene permissions separately and together", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + + const build = "--allow-build-differences"; + const scene = "--allow-scene-differences"; + for (const scenario of [ + { + changes: { mode: "debug", features: ["fixture"] }, + allowed: [build], + denied: [[], [scene]], + fields: ["functional-build", "effective-features"], + rules: ["build"], + }, + { + changes: { + scene: "other", + sceneSeed: 8, + count: 20, + fingerprint: "b".repeat(64), + }, + allowed: [scene], + denied: [[], [build]], + fields: ["scene", "fingerprint"], + rules: ["scene"], + }, + { + changes: { mode: "debug", count: 20 }, + allowed: [build, scene], + denied: [[], [build], [scene]], + fields: ["functional-build", "scene"], + rules: ["build", "scene"], + }, + ]) { + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "case/load", + value: 12, + ...scenario.changes, + }), + ); + const original = tree(fixture.managed); + for (const flags of scenario.denied) { + const rejected = await runCli(fixture, [ + "compare", + baseline, + candidate, + ...flags, + ]); + assert.equal(rejected.exitCode, 1, rejected.stderr); + assert.match(rejected.stderr, /incompatible saved runs/); + } + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + ...scenario.allowed, + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /:allow-all-differences false/); + assert.match(allowed.stdout, /ACKNOWLEDGED DIFFERENCE/); + for (const flag of scenario.allowed) { + assert.ok(allowed.stdout.includes(`:${flag.slice(2)} true`)); + } + for (const field of scenario.fields) { + assert.ok(allowed.stdout.includes(`:field :${field}`)); + } + for (const rule of scenario.rules) { + assert.ok(allowed.stdout.includes(`:rule :${rule}`)); + } + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "scoped compare permissions keep execution settings and runtime environment strict", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + for (const [field, changes] of [ + ["execution", { context: "reuse" }], + ["execution", { view: { scale: 2, x: 0, y: 0 } }], + ["environment", { environment: { node: "v25" } }], + ]) { + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "case/load", + value: 12, + ...changes, + }), + ); + const original = tree(fixture.managed); + const rejected = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-build-differences", + "--allow-scene-differences", + ]); + assert.equal(rejected.exitCode, 1); + assert.match(rejected.stderr, /incompatible saved runs/); + assert.match(rejected.stderr, new RegExp(`difference\\s+:${field}`)); + + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-all-differences", + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.ok(allowed.stdout.includes(`:field :${field}`)); + assert.match(allowed.stdout, /:rule :all/); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test("compare rejects --force without an alias", testOptions, async (t) => { + const fixture = createFrontend(t); + const original = tree(fixture.managed); + for (const flags of [["--force"], ["--force", "--allow-all-differences"]]) { + const rejected = await runCli(fixture, [ + "compare", + "missing-baseline.transit", + "missing-candidate.transit", + ...flags, + ]); + assert.equal(rejected.exitCode, 1); + assert.match(rejected.stderr, /Unknown option.*--force/); + assert.doesNotMatch(rejected.stderr, /ENOENT/); + assert.deepEqual(tree(fixture.managed), original); + } +}); + +test( + "merge writes a validated run home with appended measured attempts", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + const baseText = runFixtureText({ + runId: "base", + caseId: "case/load", + value: 10, + warmups: 1, + repetitions: 2, + gitSha: "same-sha", + }); + const additionalText = runFixtureText({ + runId: "additional", + caseId: "case/load", + value: 12, + warmups: 1, + repetitions: 3, + gitSha: "same-sha", + }); + fs.writeFileSync(base, baseText); + fs.writeFileSync(additional, additionalText); + const output = path.join(fixture.managed, "merged-run"); + + const result = await runCli(fixture, [ + "merge", + base, + additional, + "--output-dir", + output, + ]); + + assert.equal(result.exitCode, 0, result.stderr); + const runId = result.stdout.match(/Run ([0-9a-f-]{36})/)[1]; + assert.ok(fs.existsSync(path.join(output, "run.transit"))); + assert.equal(fs.readFileSync(base, "utf8"), baseText); + assert.equal(fs.readFileSync(additional, "utf8"), additionalText); + + const k = transit.keyword; + const merged = readManifest(path.join(output, "run.transit")); + const plan = merged.get(k("plan")); + const attempts = merged.get(k("attempts")); + const remaining = merged.get(k("remaining")).get(k("case/load")); + const mergeInfo = merged + .get(k("metadata")) + .get(k("diagnostics")) + .get(k("merge")); + + assert.equal(merged.get(k("run-id")), runId); + assert.equal(plan.get(k("warmups")), 1); + assert.equal(plan.get(k("repetitions")), 5); + assert.deepEqual( + attempts.map((attempt) => attempt.get(k("attempt-id"))), + [0, 1, 2, 3, 4, 5], + ); + assert.deepEqual( + attempts.map((attempt) => attempt.get(k("warmup?"))), + [true, false, false, false, false, false], + ); + assert.equal(remaining.get(k("warmups")), 0); + assert.equal(remaining.get(k("repetitions")), 0); + assert.equal(mergeInfo.get(k("base-run-id")), "base"); + assert.equal(mergeInfo.get(k("additional-run-id")), "additional"); + assert.equal(mergeInfo.get(k("additional-warmups-excluded")), 1); + assert.match( + fs.readFileSync(path.join(output, "run.log"), "utf8"), + /merge base \+ additional/, + ); + }, +); + +test( + "merge rejects incompatible runs before creating output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + fs.writeFileSync( + base, + runFixtureText({ + runId: "base", + caseId: "case/load", + value: 10, + gitSha: "sha-a", + }), + ); + fs.writeFileSync( + additional, + runFixtureText({ + runId: "additional", + caseId: "case/load", + value: 12, + gitSha: "sha-b", + }), + ); + const output = path.join(fixture.managed, "merged-run"); + const original = tree(fixture.managed); + + const result = await runCli(fixture, [ + "merge", + base, + additional, + "--output-dir", + output, + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /different git provenance/); + assert.equal(fs.existsSync(output), false); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test("merge retains rename and temporary-file cleanup failures", testOptions, async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + for (const [file, runId] of [[base, "base"], [additional, "additional"]]) { + fs.writeFileSync(file, runFixtureText({ runId, caseId: "case/load", value: 10, gitSha: "same" })); + } + const output = path.join(fixture.managed, "merged-run"); + const record = path.join(output, "run.transit"); + const guard = path.join(fixture.managed, "storage-failures.cjs"); + fs.writeFileSync(guard, ` + const fs = require("node:fs"); + const rename = fs.renameSync; + const remove = fs.rmSync; + const write = fs.writeFileSync; + const record = ${JSON.stringify(record)}; + fs.writeFileSync = function (file) { + if (String(file).startsWith(record + ".") && String(file).endsWith(".tmp")) { + write.call(fs, record, "previous complete record"); + } + return write.apply(this, arguments); + }; + fs.renameSync = function (source, destination) { + if (destination === record) throw new Error("record rename failed"); + return rename.apply(this, arguments); + }; + fs.rmSync = function (file) { + if (String(file).startsWith(record + ".") && String(file).endsWith(".tmp")) { + throw new Error("temporary record removal failed"); + } + return remove.apply(this, arguments); + }; + `); + fixture.nodeArgs = ["--require", guard]; + + const result = await runCli(fixture, ["merge", base, additional, "--output-dir", output, "--no-color"]); + + assert.equal(result.exitCode, 1, result.stderr); + assert.match(result.stderr, /record rename failed/); + assert.match(result.stderr, /temporary record removal failed/); + assert.ok(result.stderr.indexOf("record rename failed") < result.stderr.indexOf("temporary record removal failed"), result.stderr); + assert.equal(fs.readFileSync(record, "utf8"), "previous complete record"); + assert.match(result.stderr, /rename-temporary/); + assert.match(result.stderr, /remove-temporary/); + assert.ok(result.stderr.includes(record), result.stderr); +}); + +test("merge refuses an existing output directory", testOptions, async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + fs.writeFileSync( + base, + runFixtureText({ + runId: "base", + caseId: "case/load", + value: 10, + gitSha: "same-sha", + }), + ); + fs.writeFileSync( + additional, + runFixtureText({ + runId: "additional", + caseId: "case/load", + value: 12, + gitSha: "same-sha", + }), + ); + const output = path.join(fixture.managed, "existing-run"); + fs.mkdirSync(output); + fs.writeFileSync(path.join(output, "keep.txt"), "keep\n"); + + const result = await runCli(fixture, [ + "merge", + base, + additional, + "--output-dir", + output, + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /already exists/); + assert.equal( + fs.readFileSync(path.join(output, "keep.txt"), "utf8"), + "keep\n", + ); +}); diff --git a/frontend/test/performance_tests/render_wasm/cli_test.cljs b/frontend/test/performance_tests/render_wasm/cli_test.cljs new file mode 100644 index 0000000000..c001463051 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/cli_test.cljs @@ -0,0 +1,278 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.cli-test + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.cli.resolve :as cli] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private measurement-defaults + {:seed 42 :filter nil :warmups 3 :repetitions 10 + :timeout-ms 30000 :startup-timeout-ms 30000 :output-dir nil + :screenshots false :require-gpu false}) + +(def ^:private comparison-defaults + {:allow-build-differences false :allow-scene-differences false + :allow-all-differences false}) + +(defn- resolved + [command options] + {:command command :console {:level :info :color :auto} :options options}) + +(defn- failure + [argv] + (try + (cli/parse-resolve argv) + nil + (catch :default cause cause))) + +(t/deftest commands-resolve-complete-requests + (doseq [[argv expected] + [[["build"] + (resolved "build" {:default-features true :features [] :build-timeout-ms 300000})] + [["run"] + (resolved "run" (assoc measurement-defaults :remove false + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000}))] + [["run" "--build" "ready-build"] + (resolved "run" (assoc measurement-defaults :remove false + :build {:kind :existing :id "ready-build"}))] + [["ab" "--features" "cache" "--dry-run"] + (resolved "ab" (merge measurement-defaults comparison-defaults + {:features ["cache"] :max-features 3 :confirm false + :dry-run true :build-timeout-ms 300000}))] + [["compare" "base" "candidate"] + (resolved "compare" (assoc comparison-defaults :baseline-path "base" + :candidate-path "candidate" :case-pair nil))] + [["compare" "base" "candidate" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"] + (resolved "compare" {:baseline-path "base" :candidate-path "candidate" :case-pair nil + :allow-build-differences true :allow-scene-differences true + :allow-all-differences true})] + [["ab" "--features" "cache" "--filter" "rects" "--require-gpu" "--screenshots" + "--allow-build-differences" "--allow-scene-differences" "--allow-all-differences"] + (resolved "ab" (assoc measurement-defaults :filter "rects" :require-gpu true + :screenshots true :features ["cache"] :max-features 3 + :confirm false :dry-run false :build-timeout-ms 300000 + :allow-build-differences true :allow-scene-differences true + :allow-all-differences true))] + [["merge" "base" "extra"] + (resolved "merge" {:base-path "base" :additional-path "extra" :output-dir nil})] + [["merge" " ./base " " ./extra " "--output-dir" " ./merged "] + (resolved "merge" {:base-path " ./base " :additional-path " ./extra " + :output-dir " ./merged "})] + [["purge" "comparisons"] (resolved "purge" {:target :comparisons})] + [["--help"] (resolved "help" {:topic nil})]]] + (t/testing (pr-str argv) + (t/is (= expected (cli/parse-resolve argv)))))) + +(t/deftest console-flags-compose-with-every-command + (doseq [argv [["build"] ["run"] ["ab" "--features" "cache" "--dry-run"] + ["compare" "base" "candidate"] ["merge" "base" "extra"] + ["purge" "runs"] ["--help"]] + [flags console] [[["-q" "--no-color"] {:level :warn :color :never} + [["-v" "--color"] {:level :debug :color :always}]]]] + (t/is (= console (:console (cli/parse-resolve (into argv flags)))))) + (doseq [argv [["run" "-qv"] ["build" "--quiet" "--verbose"] + ["compare" "--help" "-qv"] ["--help" "-qv"]]] + (let [error (failure argv)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "mutually exclusive"))))) + +(t/deftest explicit-build-and-measurement-values-survive-resolution + (t/is (= (resolved "build" {:default-features false :features ["a" "b"] + :build-timeout-ms 1000}) + (cli/parse-resolve ["--" "build" "--no-default-features" + "--features=b,a,b" "--build-timeout-ms" "1e3"]))) + (t/is (= {:command "run" :console {:level :info :color :never} + :options {:build {:kind :prepare :default-features false + :features ["a" "b"] :build-timeout-ms 5000} + :seed 0 :filter " rects " :warmups 0 :repetitions 2 + :timeout-ms 1000 :startup-timeout-ms 2000 :output-dir " ./runs " + :screenshots true :remove true :require-gpu true}} + (cli/parse-resolve + ["run" "--no-default-features" "--features" " b,a,b " + "--build-timeout-ms" "5000" "--seed" "0" "--filter" " rects " + "--warmups" "0" "--repetitions" "2" "--timeout-ms" "1000" + "--startup-timeout-ms" "2000" "--output-dir" " ./runs " + "--screenshots" "--remove" "--require-gpu" "--no-color"])))) + +(t/deftest retained-build-conflicts-use-supplied-flags + (doseq [flags [["--features" "cache"] ["--default-features"] + ["--no-default-features"] ["--build-timeout-ms=300000"]] + argv [(into ["run" "--build" "ready"] flags) + (into (into ["run"] flags) ["--build=ready"])]] + (t/is (str/includes? (ex-message (failure argv)) "--build cannot be combined"))) + (t/is (= (resolved "run" (assoc measurement-defaults :remove false :warmups 0 + :build {:kind :existing :id "ready"})) + (cli/parse-resolve ["run" "--build" "ready" "--warmups" "0"])))) + +(t/deftest parsers-reject-options-owned-by-other-commands + (doseq [[argv flags] + [[["build"] ["--warmups" "--cases" "--confirm"]] + [["run"] ["--cases" "--dry-run" "--allow-build-differences"]] + [["ab" "--features" "cache" "--dry-run"] + ["--build" "--remove" "--default-features" "--no-default-features" "--cases"]] + [["compare" "base" "candidate"] ["--output-dir" "--seed" "--features"]] + [["merge" "base" "extra"] ["--warmups" "--cases" "--allow-all-differences"]] + [["purge" "runs"] ["--features" "--seed" "--only-runs"]]] + flag flags] + (doseq [args [(conj argv flag) (conj argv "--help" flag)]] + (let [error (failure args)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "Unknown option")))))) + +(t/deftest command-first-syntax-and-positional-arguments-are-checked + (doseq [argv [[] ["unknown" "--help"] ["--help" "run"] ["-q" "build"] + ["build" "extra"] ["run" "extra"] + ["compare"] ["compare" "one"] ["compare" "one" "two" "three"] + ["compare" "" "candidate"] ["merge" "base"] ["merge" "base" " "] + ["run" "--build" "../escape"] ["run" "--output-dir" " "]]] + (t/testing (pr-str argv) + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv))))))) + (doseq [target ["builds" "runs" "comparisons" "all"]] + (t/is (= (resolved "purge" {:target (keyword target)}) + (cli/parse-resolve ["purge" target])))) + (doseq [args [[] ["unknown"] ["runs" "builds"]]] + (let [error (failure (into ["purge"] args))] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "runner purge builds|runs|comparisons|all"))))) + +(t/deftest argument-errors-include-command-and-validation-details + (let [data (ex-data (failure ["unknown" "--help"]))] + (t/is (= "unknown" (:command data))) + (t/is (= "runner COMMAND [options]" (:usage data)))) + (doseq [[argv path] [[["compare" "" "candidate"] [:options :baseline-path]] + [["merge" "base" " "] [:options :additional-path]]]] + (let [data (ex-data (failure argv))] + (t/is (= ::cli/invalid-args (:type data))) + (t/is (= ::request/invalid-request (:cause-type data))) + (t/is (some #(= path (:in %)) (:errors (::sm/explain data))))))) + +(t/deftest literal-paths-selectors-and-option-looking-values-are-preserved + (t/is (= (resolved "compare" (assoc comparison-defaults :baseline-path " ./base " + :candidate-path " ./candidate " + :case-pair [" rects/load " "other/load"])) + (cli/parse-resolve ["compare" " ./base " " ./candidate " + "--cases" " rects/load ,other/load"]))) + (t/is (= (resolved "compare" (assoc comparison-defaults :baseline-path "--seed" + :candidate-path "-candidate" :case-pair nil)) + (cli/parse-resolve ["compare" "--" "--seed" "-candidate"]))) + (doseq [text ["" " " "--allow-all-differences" "--" "-h"]] + (t/is (= (resolved "run" (assoc measurement-defaults :filter text :remove false + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000})) + (cli/parse-resolve ["run" "--filter" text]))))) + +(t/deftest malformed-selector-pairs-fail-before-record-loading + (doseq [text ["" "one" "one,two,three" ",two" "one," " ,two" "one, "]] + (let [error (failure ["compare" "base" "candidate" (str "--cases=" text)])] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "--cases")))) + (t/is (= ::cli/invalid-args + (:type (ex-data (failure ["compare" "base" "candidate" "--cases"])))))) + +(t/deftest numeric-values-use-integer-and-timer-bounds + (doseq [[flag values] + [["--seed" ["0" "4294967295"]] + ["--warmups" ["0" "9007199254740991"]] + ["--repetitions" ["1" "9007199254740991"]] + ["--timeout-ms" ["1" "2147483647"]] + ["--startup-timeout-ms" ["1" "2147483647"]] + ["--build-timeout-ms" ["1" " 1000 " "1e3" "0x3e8" "0o1750" "0b1111101000" "2147483647"]]] + value values] + (t/is (= (js/Number value) + (let [options (:options (cli/parse-resolve ["run" flag value]))] + (if (= flag "--build-timeout-ms") + (get-in options [:build :build-timeout-ms]) + (get options (keyword (subs flag 2)))))))) + (doseq [[flag values] + [["--seed" ["-1" "4294967296" "1.5"]] + ["--warmups" ["-1" "9007199254740992" "0.5"]] + ["--repetitions" ["0" "9007199254740992" "1.5"]] + ["--timeout-ms" ["0" "2147483648" "1.5"]] + ["--startup-timeout-ms" ["0" "2147483648"]] + ["--build-timeout-ms" ["0" "2147483648" "1.5" "1000ms"]]] + value (concat values ["" " " "NaN" "Infinity"])] + (t/testing (str flag " " value) + (t/is (= ::cli/invalid-args (:type (ex-data (failure ["run" flag value])))))))) + +(t/deftest feature-selection-and-ab-policy-use-domain-rules + (doseq [command ["build" "run" "ab"] + [feature message] [["stats" "diagnostic Cargo features"] + ["profile" "diagnostic Cargo features"] + ["profile-macros" "diagnostic Cargo features"] + ["profile-raf" "diagnostic Cargo features"] + ["" "invalid Cargo feature selection"] + ["a,,b" "invalid Cargo feature selection"] + ["a/b" "invalid Cargo feature selection"]]] + (let [argv (cond-> [command "--features" feature] (= command "ab") (conj "--dry-run")) + error (failure argv)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) message)))) + (doseq [argv [["ab"] ["ab" "--features" "cache"] + ["ab" "--features" "cache" "--filter" " "] + ["ab" "--features" "a,b,c,d" "--dry-run"] + ["ab" "--features" "a,b,c,d" "--max-features" "4" "--filter" "rects"]]] + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv)))))) + (doseq [limit ["" " " "0" "7" "1.5" "NaN" "Infinity" "3x"]] + (t/is (= ::cli/invalid-args + (:type (ex-data (failure ["ab" "--features" "cache" "--dry-run" + "--max-features" limit])))))) + (doseq [flags [["--confirm"] ["--dry-run"] ["--filter" "rects"]]] + (t/is (= ["a" "b" "c"] + (get-in (cli/parse-resolve (into ["ab" "--features" "c,b,a,c"] flags)) + [:options :features])))) + (let [error (failure ["ab" "--features" "cache"])] + (t/is (str/includes? (ex-message (ex-cause error)) "--confirm")))) + +(t/deftest help-skips-required-command-arguments-and-keeps-syntax-checks + (doseq [command ["build" "run" "ab" "compare" "merge" "purge"]] + (t/is (= (resolved "help" {:topic command}) + (cli/parse-resolve [command "--help"])))) + (t/is (= (assoc (resolved "help" {:topic "compare"}) + :console {:level :debug :color :never}) + (cli/parse-resolve ["compare" "--help" "--verbose" "--no-color"]))) + (doseq [argv [["--help" "--unknown"] ["run" "--help" "--warmups" "bad"] + ["compare" "--help" "--cases="] ["ab" "--help" "--features"]]] + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv))))))) + +(t/deftest help-shows-command-options-and-schema-defaults + (doseq [[topic flags] + [["build" #{"--[no-]default-features" "--features" "--build-timeout-ms"}] + ["run" #{"--build" "--[no-]default-features" "--features" "--build-timeout-ms" + "--seed" "--filter" "--warmups" "--repetitions" "--timeout-ms" + "--startup-timeout-ms" "--output-dir" "--screenshots" "--require-gpu" "--remove"}] + ["ab" #{"--features" "--max-features" "--confirm" "--dry-run" "--build-timeout-ms" + "--seed" "--filter" "--warmups" "--repetitions" "--timeout-ms" + "--startup-timeout-ms" "--output-dir" "--screenshots" "--require-gpu" + "--allow-build-differences" "--allow-scene-differences" "--allow-all-differences"}] + ["compare" #{"--cases" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"}] + ["merge" #{"--output-dir"}] + ["purge" #{}]]] + (let [text (cli/get-help topic)] + (t/is (str/includes? text (str "Usage: runner " topic))) + (t/is (= (into #{"--help" "--quiet" "--verbose" "--[no-]color"} flags) + (set (re-seq #"--(?:\[no-\])?[a-z][a-z-]*" text)))))) + (t/is (= ["build" "run" "ab" "compare" "merge" "purge"] + (mapv second (re-seq #"(?m)^ runner ([a-z]+)" (cli/get-help nil))))) + (doseq [[topic pattern] [["build" #"--build-timeout-ms MS\s+300000"] + ["run" #"--warmups N\s+3"] + ["run" #"--seed N\s+42"] + ["run" #"--repetitions N\s+10"] + ["run" #"--timeout-ms MS\s+30000\s+Case deadline"] + ["ab" #"--timeout-ms MS\s+30000\s+Case deadline"] + ["run" #"--features FEATURES\s+\[\]"] + ["run" #"--\[no-\]default-features\s+true"] + ["run" #"--build-timeout-ms MS\s+300000"] + ["ab" #"--max-features N\s+3"] + ["merge" #"--output-dir DIR\s+nil"] + ["compare" #"--cases A,B\s+nil"]]] + (t/is (re-find pattern (cli/get-help topic))))) diff --git a/frontend/test/performance_tests/render_wasm/commands_test.cljs b/frontend/test/performance_tests/render_wasm/commands_test.cljs new file mode 100644 index 0000000000..a2b60587bd --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/commands_test.cljs @@ -0,0 +1,520 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.commands-test + (:require + [app.common.render-wasm.wasm :as wasm] + [app.common.schema :as sm] + [benches.render-wasm.builder :as builder] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.runtime.camera :as camera] + [benches.render-wasm.runtime.commands :as cmd :refer [defcommand]] + [cljs.test :as t])) + +(defcommand move + "Adds the supplied offsets to the command context's view." + [rtx {:keys [dx dy]}] + (-> rtx + (update-in [:view :x] + dx) + (update-in [:view :y] + dy))) + +(defcommand selected-renders + {:measured-variant false} + [rtx] + (-> rtx + (camera/render-sync*) + (move {:dx 10 :dy 5}) + (camera/render-sync) + (camera/render-sync*))) + +(defcommand unexpected-promise + [rtx cause] + (if cause + (js/Promise.reject cause) + (js/Promise.resolve rtx))) + +(defcommand reject-offset + [rtx offset] + {:pre [(pos? offset)]} + ((get-in rtx [:hooks :render-sync])) + (update-in rtx [:view :x] + offset)) + +(defcommand replace-owner + [rtx] + (assoc-in rtx [:hooks :check] (fn []))) + +(defcommand async-work + {:async [complete fail]} + [rtx work] + (work rtx complete fail)) + +(defcommand selected-async + {:async [complete fail] :measured-variant false} + [rtx] + (-> (camera/render-sync* rtx) + (move {:dx 10 :dy 5}) + (camera/render-sync*) + (.then complete) + (.catch fail))) + +(defn- context + [clock] + {:case {:id :commands/example} + :scene (builder/scene {:seed 1}) + :view {:scale 1 :x 0 :y 0} + :hooks {:now #(deref clock) + :check (fn []) + :frame #(js/Promise.resolve (vswap! clock + 16)) + :sleep #(do (vswap! clock + %) (js/Promise.resolve nil)) + :render (fn [_ _] (vswap! clock + 7) wasm/FRAME_TYPE_FULL) + :render-sync #(vswap! clock + 7) + :render-from-cache #(vswap! clock + 2) + :set-view (fn [_]) + :set-view-start (fn []) + :set-view-end #(vswap! clock + 3)}}) + +(t/deftest ^:async measurement-uses-core-time + (let [clock (volatile! 0) + rtx (assoc-in (context clock) [:hooks :render-sync] + #(do + (vswap! clock + 7) + (js/queueMicrotask (fn [] (vswap! clock + 500))))) + {:keys [outcome events context]} + (await + (cmd/execute + rtx + (fn [rtx] + (let [input (-> (js/Promise.resolve rtx) + (.then (fn [rtx] + (vswap! clock + 1000) + rtx)))] + (-> (camera/render-sync* input) + (.then (fn [rtx] + (vswap! clock + 2000) + rtx)))))))] + (t/is (= :completed outcome)) + (t/is (= (:view rtx) (:view context))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + events)) + (t/is (= 3507 @clock)))) + +(t/deftest ^:async composite-preserves-view-and-selected-observations + (doseq [promised [false true]] + (let [clock (volatile! 0) + cost (volatile! 0) + rtx (assoc-in (context clock) [:hooks :render-sync] + #(vswap! clock + (vswap! cost + 7))) + {:keys [outcome context events]} + (await (cmd/execute rtx #(selected-renders + (if promised (js/Promise.resolve %) %))))] + (t/is (= :completed outcome)) + (t/is (= {:scale 1 :x 10 :y 5} (:view context))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}} + {:command ::camera/render-sync + :metrics {:command/elapsed [21]}}] + events)) + (t/is (= 42 @clock))))) + +(t/deftest measured-command-documentation + (t/is (.startsWith (:doc (meta #'camera/render-sync*)) + "Measured variant of `render-sync`.")) + (t/is (= "Measured variant of `unexpected-promise`." + (:doc (meta #'unexpected-promise*))))) + +(t/deftest ^:async zero-duration-is-a-raw-observation + (let [rtx (context (volatile! 0)) + result (await (cmd/execute rtx #(move* % {:dx 2 :dy 3})))] + (t/is (= :completed (:outcome result))) + (t/is (= {:scale 1 :x 2 :y 3} (get-in result [:context :view]))) + (t/is (= [{:command ::move :metrics {:command/elapsed [0]}}] + (:events result))))) + +(t/deftest ^:async async-completion-keeps-endpoints-and-raw-evidence + (doseq [command [async-work async-work*] + supplied [false true]] + (let [clock (volatile! 0) + rtx (context clock) + metrics {:renderer/call-elapsed [1 2 4]} + trace {:steps [1 2 3]} + result (await + (cmd/execute + rtx + (fn [rtx] + (command + (-> (js/Promise.resolve rtx) + (.then #(do (vswap! clock + 1000) %))) + (fn [rtx complete _fail] + (vswap! clock + 7) + (let [end @clock + evidence {:metrics metrics :trace trace} + output (assoc-in rtx [:view :x] 10)] + (if supplied + (-> (js/Promise.resolve nil) + (.then (fn [_] + (vswap! clock + 500) + (complete output (assoc evidence :end-ms end))))) + (do + (complete output evidence) + (js/queueMicrotask #(vswap! clock + 500))))))))))] + (t/is (= :completed (:outcome result))) + (t/is (= 10 (get-in result [:context :view :x]))) + (t/is (= 1507 @clock)) + (t/is (= (if (= command async-work*) + [{:command ::async-work + :metrics (assoc metrics :command/elapsed [7]) + :trace trace}] + []) + (:events result))))) + (let [result (await (cmd/execute (context (volatile! 0)) selected-async))] + (t/is (= :completed (:outcome result))) + (t/is (= {:scale 1 :x 10 :y 5} (get-in result [:context :view]))) + (t/is (= [::camera/render-sync ::camera/render-sync] + (mapv :command (:events result)))))) + +(t/deftest ^:async invalid-trace-stops-the-command-and-retains-earlier-measurements + (doseq [trace [{:handle (fn [] nil)} (repeat 0) (nth (iterate vector 0) 40)]] + (let [run (fn [rtx] + (async-work* rtx (fn [rtx complete _] (complete rtx {:trace trace})))) + result (await (cmd/execute (context (volatile! 0)) + (fn [rtx] (.then (camera/render-sync* rtx) run))))] + (t/is (= :failed (:outcome result))) + (t/is (= "command trace cannot cross Transit" (errors/message-of (:error result)))) + (t/is (= [::camera/render-sync] (mapv :command (:events result))))))) + +(t/deftest ^:async async-completion-retains-invalid-numeric-observations + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (complete rtx {:end-ms js/Infinity + :metrics {:sample/raw [-1 0 js/NaN js/Infinity]}}))))) + values (get-in result [:events 0 :metrics :sample/raw])] + (t/is (= :completed (:outcome result))) + (t/is (= [js/Infinity] (get-in result [:events 0 :metrics :command/elapsed]))) + (t/is (= [-1 0] (subvec values 0 2))) + (t/is (js/Number.isNaN (nth values 2))) + (t/is (= js/Infinity (nth values 3))))) + +(t/deftest ^:async async-commands-use-the-first-terminal-signal + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete fail] + (complete (assoc-in rtx [:view :x] 10)) + (complete nil) + (fail (js/Error. "late failure")) + (throw (js/Error. "late throw"))))))] + (t/is (= :completed (:outcome result))) + (t/is (= 10 (get-in result [:context :view :x]))) + (t/is (= [{:command ::async-work :metrics {:command/elapsed [0]}}] (:events result))) + (t/is (= [:complete :fail :fail] (mapv :signal (:diagnostics result))))) + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (js/queueMicrotask (fn [] (complete (assoc-in rtx [:view :x] 20)))) + rtx))))] + (t/is (= :completed (:outcome result))) + (t/is (= 20 (get-in result [:context :view :x])))) + (let [unhandled (atom []) + observe #(swap! unhandled conj %)] + (.on js/process "unhandledRejection" observe) + (try + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (complete rtx) + (js/Promise.reject (js/Error. "rejected after completion"))))))] + (await (js/Promise. #(js/setImmediate %))) + (t/is (= :completed (:outcome result))) + (t/is (= [{:command ::async-work :metrics {:command/elapsed [0]}}] (:events result))) + (t/is (= [] @unhandled))) + (finally + (.removeListener js/process "unhandledRejection" observe))))) + +(t/deftest ^:async async-failures-retain-causes-and-earlier-events + (doseq [mode [:throw :reject :fail]] + (let [cause (ex-info "renderer failed" {:partial {:slices [{:duration-ms 3}]}}) + result (await + (cmd/execute + (context (volatile! 0)) + #(-> % + (camera/render-sync*) + (async-work* (fn [rtx complete fail] + (case mode + :throw (throw cause) + :reject (js/Promise.reject cause) + :fail (do (fail cause) (complete rtx))))))))] + (t/is (= :failed (:outcome result))) + (t/is (identical? cause (:error result))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [7]}}] + (:events result)))))) + +(t/deftest ^:async malformed-async-completion-fails-before-recording + (doseq [[output evidence expected] + [[identity {:end-ms nil} ::cmd/invalid-completion] + [identity {:metrics {:sample/raw []}} ::cmd/invalid-completion] + [identity {:metrics {:sample/raw ["bad"]}} ::cmd/invalid-completion] + [identity {:metrics {:command/elapsed [123]}} ::cmd/invalid-completion] + [#(assoc-in % [:view :scale] 0) {} ::cmd/invalid-context] + [#(assoc-in % [:hooks :check] (fn [])) {} ::cmd/ownership-lost]]] + (let [result (await (cmd/execute (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (complete (output rtx) evidence)))))] + (t/is (= :failed (:outcome result))) + (t/is (= expected (:type (ex-data (:error result))))) + (t/is (= [] (:events result)))))) + +(t/deftest ^:async async-completion-checks-the-owner-guard + (doseq [cause [(ex-info "deadline expired" {:phase :deadline}) + (ex-info "owner superseded" {::errors/stale true})]] + (let [blocked (volatile! nil) + rtx (assoc-in (context (volatile! 0)) [:hooks :check] + #(when @blocked (throw @blocked))) + result (await (cmd/execute rtx #(async-work* % (fn [rtx complete _fail] + (vreset! blocked cause) + (complete rtx)))))] + (t/is (= :failed (:outcome result))) + (t/is (identical? cause (:error result))) + (t/is (= [] (:events result)))))) + +(t/deftest ^:async late-completion-cannot-record-in-another-execution + (let [callbacks (volatile! nil) + pending (volatile! nil) + first-run (await + (cmd/execute + (context (volatile! 0)) + (fn [rtx] + (vreset! pending + (.catch (async-work* rtx (fn [rtx complete fail] + (vreset! callbacks [rtx complete fail]))) + errors/as-failure)) + rtx))) + [rtx complete fail] @callbacks + next-run (await (cmd/execute (:context first-run) + (fn [next-rtx] + (complete rtx) + (fail (js/Error. "late failure")) + (camera/render-sync* next-rtx)))) + cause (await @pending)] + (t/is (= :completed (:outcome first-run))) + (t/is (= [] (:events first-run))) + (t/is (= ::cmd/inactive-execution (:type (ex-data cause)))) + (t/is (= :completed (:outcome next-run))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [7]}}] + (:events next-run))))) + +(t/deftest ^:async camera-gestures-return-the-view-and-renderer-evidence + (doseq [[command options target command-id measured] + [[camera/pan {:dx 10 :dy 5 :steps 2 :settle-ms 5} {:scale 1 :x 10 :y 5} ::camera/pan false] + [camera/pan* {:dx 10 :dy 5 :steps 2 :settle-ms 5} {:scale 1 :x 10 :y 5} ::camera/pan true] + [camera/zoom {:factor 2 :steps 2 :settle-ms 5} {:scale 2 :x 0 :y 0} ::camera/zoom false] + [camera/zoom* {:factor 2 :steps 2 :settle-ms 5} {:scale 2 :x 0 :y 0} ::camera/zoom true]]] + (let [clock (volatile! 0) + view (volatile! nil) + remaining (volatile! [wasm/FRAME_TYPE_PARTIAL wasm/FRAME_TYPE_FULL]) + rtx (update (context clock) :hooks merge + {:set-view #(vreset! view %) + :render (fn [_ _] + (vswap! clock + 7) + (let [frame-type (first @remaining)] + (vswap! remaining rest) + (when (= wasm/FRAME_TYPE_FULL frame-type) + (js/queueMicrotask #(vswap! clock + 500))) + frame-type))}) + result (await (cmd/execute rtx #(command % options)))] + (t/is (= :completed (:outcome result))) + (t/is (= target @view (get-in result [:context :view]))) + (t/is (= 574 @clock)) + (if measured + (do + (t/is (= command-id (get-in result [:events 0 :command]))) + (t/is (= {:command/elapsed [74] + :renderer/call-elapsed [7 7] + :renderer/cache-call-elapsed [2 2] + :camera/finalization-to-full [33]} + (get-in result [:events 0 :metrics]))) + (t/is (= 74 (get-in result [:events 0 :trace :end-ms])))) + (t/is (= [] (:events result)))) + (vreset! clock 0) + (let [invalid (await (cmd/execute rtx #(command % (assoc options :steps 0))))] + (t/is (= :failed (:outcome invalid))) + (t/is (= ::cmd/invalid-arguments (:type (ex-data (:error invalid))))) + (t/is (= [] (:events invalid))) + (t/is (zero? @clock)))))) + +(t/deftest ^:async load-and-restore-finish-at-the-renderer-endpoint + (doseq [[command restore measured] [[camera/load false false] [camera/load* false true] + [camera/restore true false] [camera/restore* true true]]] + (let [clock (volatile! 0) + view (volatile! nil) + target {:scale 2 :x 30 :y -10} + rtx (update (context clock) :hooks merge + {:set-view #(vreset! view %) + :render (fn [_ _] + (vswap! clock + 7) + (js/queueMicrotask #(vswap! clock + 500)) + wasm/FRAME_TYPE_FULL)}) + result (await (cmd/execute rtx #(if restore + (command (js/Promise.resolve %) target) + (command (js/Promise.resolve %)))))] + (t/is (= :completed (:outcome result))) + (t/is (= (if restore target (:view rtx)) (get-in result [:context :view]))) + (when restore (t/is (= target @view))) + (t/is (= (if restore 510 523) @clock)) + (when measured + (let [event (first (:events result))] + (t/is (= (if restore ::camera/restore ::camera/load) (:command event))) + (t/is (= [(if restore 10 23)] (get-in event [:metrics :command/elapsed]))) + (t/is (= [7] (get-in event [:metrics :renderer/call-elapsed]))) + (t/is (= (if restore 10 23) (get-in event [:trace :end-ms]))) + (when-not restore + (t/is (= [23] (get-in event [:metrics :renderer/prepared-scene-to-full]))))))))) + +(t/deftest ^:async invalid-camera-targets-fail-before-changing-renderer-state + (doseq [[command options view] + [[camera/pan* {:dx sm/max-safe-int :dy 0 :steps 1 :settle-ms 0} {:scale 1 :x 1 :y 0}] + [camera/zoom* {:factor sm/max-safe-int :steps 1 :settle-ms 0} {:scale 2 :x 0 :y 0}] + [camera/restore* {:scale 1 :x 10 :y 0 :viewport {:width 800}} {:scale 1 :x 0 :y 0}]]] + (let [clock (volatile! 0) + applied (volatile! view) + rtx (-> (context clock) + (assoc :view view) + (assoc-in [:hooks :set-view] #(vreset! applied %))) + result (await (cmd/execute rtx #(command % options)))] + (t/is (= :failed (:outcome result))) + (t/is (= ::cmd/invalid-arguments (:type (ex-data (:error result))))) + (t/is (= view @applied)) + (t/is (zero? @clock))))) + +(t/deftest ^:async declared-reference-cases-execute-selected-work-and-reset-the-view + (let [clock (volatile! 0) + descriptor (first (cases/collect-cases {:master-seed 42 :filter "rects/blocking-full"})) + result (await (cmd/execute (assoc (context clock) :case descriptor) + (:run! (decl/registered-case (:id descriptor)))))] + (t/is (= :completed (:outcome result))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [7]}}] (:events result)))) + (let [clock (volatile! 0) + descriptor (first (cases/collect-cases {:master-seed 42 :filter "rects/mixed-pans"})) + registered (decl/registered-case (:id descriptor)) + applied (volatile! (:view descriptor)) + initial (-> (context clock) (assoc :case descriptor) + (assoc-in [:hooks :set-view] #(vreset! applied %)))] + (loop [rtx initial remaining 2] + (when (pos? remaining) + (let [main (await (cmd/execute rtx (:run! registered))) + _ (t/is (= {:scale 1 :x 0 :y 80} @applied (get-in main [:context :view]))) + reset (await (cmd/execute (:context main) (:reset registered)))] + (t/is (= :completed (:outcome main) (:outcome reset))) + (t/is (= [::camera/pan ::camera/pan] (mapv :command (:events main)))) + (t/is (= [::camera/restore] (mapv :command (:events reset)))) + (t/is (= (:view descriptor) @applied (get-in reset [:context :view]))) + (recur (:context reset) (dec remaining))))))) + +(t/deftest ^:async small-positive-zoom-scales-stay-positive-and-reach-the-target + (doseq [[scale factor] [[1 1e-20] [js/Number.MIN_VALUE 1]]] + (let [applied (volatile! {:scale scale :x 0 :y 0}) + minimum (volatile! scale) + rtx (-> (context (volatile! 0)) + (assoc :view @applied) + (assoc-in [:hooks :set-view] + #(do (vreset! applied %) + (vswap! minimum min (:scale %))))) + result (await (cmd/execute rtx #(camera/zoom* % {:factor factor :steps 2 :settle-ms 0})))] + (t/is (= :completed (:outcome result))) + (t/is (pos? @minimum)) + (t/is (= (* scale factor) (:scale @applied) (get-in result [:context :view :scale])))))) + +(t/deftest ^:async failed-work-retains-earlier-observations + (let [unhandled (atom []) + observe #(swap! unhandled conj %)] + (.on js/process "unhandledRejection" observe) + (try + (doseq [cause [nil (js/Error. "rejected core")]] + (let [rtx (context (volatile! 0)) + result (await (cmd/execute rtx #(-> % + (camera/render-sync*) + (unexpected-promise* cause))))] + (t/is (= :failed (:outcome result))) + (t/is (= ::cmd/async-result (:type (ex-data (:error result))))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + (:events result))))) + (await (js/Promise. #(js/setImmediate %))) + (t/is (= [] @unhandled)) + (finally + (.removeListener js/process "unhandledRejection" observe))))) + +(t/deftest ^:async execution-failures-preserve-causes-and-earlier-observations + (doseq [cause [(js/Error. "renderer failed") + "renderer disconnected" + {:reason :renderer-disconnected} + (errors/fatal-error "recording failed" {} (js/Error. "storage"))]] + (let [clock (volatile! 0) + rtx (assoc-in (context clock) [:hooks :render-sync] + #(if (zero? @clock) + (vswap! clock + 7) + (throw cause))) + result (await (cmd/execute rtx #(-> % + (camera/render-sync*) + (camera/render-sync*)))) + error (:error result)] + (t/is (= :failed (:outcome result))) + (t/is (errors/failure? error)) + (if (errors/failure? cause) + (t/is (identical? cause error)) + (t/is (= cause (:thrown (ex-data error))))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + (:events result)))))) + +(t/deftest ^:async invalid-boundaries-fail-the-execution + (doseq [[input command expected] + [[#(assoc-in % [:view :scale] 0) camera/render-sync* ::cmd/invalid-context] + [#(assoc % :module #js {}) camera/render-sync* ::cmd/invalid-context] + [#(update % :hooks dissoc :render-sync) camera/render-sync* ::cmd/invalid-context] + [#(update % :hooks dissoc :frame) camera/render-sync* ::cmd/invalid-context] + [#(assoc-in % [:hooks :sleep] 0) camera/render-sync* ::cmd/invalid-context] + [identity #(reject-offset* % -1) ::cmd/invalid-arguments]]] + (let [clock (volatile! 0) + now-reads (volatile! 0) + rtx (assoc-in (context clock) [:hooks :now] + #(do (vswap! now-reads inc) @clock)) + result (await (cmd/execute rtx #(command (input %))))] + (t/is (= :failed (:outcome result))) + (t/is (= expected (:type (ex-data (:error result))))) + (t/is (= [] (:events result))) + (t/is (zero? @now-reads)) + (t/is (zero? @clock))))) + +(t/deftest ^:async commands-enforce-execution-ownership + (let [rtx (context (volatile! 0)) + result (await (cmd/execute rtx replace-owner*)) + cause (await (.then (camera/render-sync* rtx) (fn [_] nil) identity))] + (t/is (= :failed (:outcome result))) + (t/is (= ::cmd/ownership-lost (:type (ex-data (:error result))))) + (t/is (= [] (:events result))) + (t/is (= ::cmd/missing-execution (:type (ex-data cause)))))) + +(t/deftest ^:async completed-context-starts-a-new-execution + (let [first-result (await (cmd/execute (context (volatile! 0)) + #(move* % {:dx 10 :dy 5}))) + rtx (:context first-result) + late-error (await (.then (camera/render-sync* rtx) + (fn [_] nil) + identity)) + next-result (await (cmd/execute rtx camera/render-sync*))] + (t/is (= ::cmd/inactive-execution (:type (ex-data late-error)))) + (t/is (= :completed (:outcome next-result))) + (t/is (= {:scale 1 :x 10 :y 5} (get-in next-result [:context :view]))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + (:events next-result))))) diff --git a/frontend/test/performance_tests/render_wasm/compare_test.cljs b/frontend/test/performance_tests/render_wasm/compare_test.cljs new file mode 100644 index 0000000000..7809f03713 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/compare_test.cljs @@ -0,0 +1,278 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.compare-test + (:require + ["fs" :as fs] + ["node:stream" :as stream] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.report.compare :as compare] + [benches.render-wasm.report.compare :as report-compare] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-compare-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- metric + [value] + {:command :fixture/work :metrics {:command/elapsed [value]}}) + +(defn- case-definition + [case-id scene] + {:id case-id + :scene scene + :scene-description "comparison scene" + :scene-seed 7 + :params {:count 10} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {}}) + +(defn- make-run + [run-id case-id value & {:keys [environment functional fingerprint scene case-settings] + :or {environment {:node "v24" :browser "Fake"} + functional {:mode :release} + fingerprint (apply str (repeat 64 "a")) + scene :case}}] + (let [preparation-id (str run-id "-preparation") + run (result/create-run + {:run-id run-id + :started-at "2026-10-26T00:00:00.000Z" + :plan {:cases [(merge (case-definition case-id scene) case-settings)] + :warmups 0 + :repetitions 1 + :timeout-policy {:case-ms 30000}} + :metadata {:git {:sha run-id :dirty false} + :build {:functional functional} + :environment environment + :scored? true}}) + run (result/record-preparation + run + {:preparation-id preparation-id + :case-id case-id :warmup? false + :fingerprint {:algorithm :sha-256 + :digest fingerprint} + :effective-graphics {:webgl2 true} + :diagnostics {} + :phases [{:phase :preparation :preparation-id preparation-id + :outcome :completed :events []}]}) + run (result/record-attempt + run + {:case-id case-id + :preparation-id preparation-id + :warmup? false + :outcome :completed + :phases [{:phase :main :preparation-id preparation-id + :outcome :completed :events [(metric value)]}]})] + (-> run + (result/record-case-outcome case-id {:outcome :completed}) + (result/finish-run "2026-10-26T00:01:00.000Z" {:reason :completed})))) + +(defn- write-run + [file run] + (fs/writeFileSync file (result/encode run) "utf8") + file) + +(defn- failure + [f] + (try + (f) + nil + (catch :default cause cause))) + +(t/deftest compares-saved-records-from-the-caller-cwd + (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12)) + (let [previous (.cwd js/process)] + (try + (.chdir js/process @temporary-root) + (let [{:keys [comparison text]} (compare/execute "baseline.transit" + "candidate.transit" {})] + (t/is (= :comparison (:type comparison))) + (t/is (= {:allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + (:permissions comparison))) + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (= 2 (get-in comparison [:pairs 0 :metrics [:main :fixture/work :command/elapsed] :absolute :value]))) + (t/is (str/includes? text "Baseline")) + (t/is (str/includes? text "Candidate")) + (t/is (str/includes? text "case/load")) + (t/is (str/includes? text "median change"))) + (finally + (.chdir js/process previous))))) + +(t/deftest cases-selects-one-explicit-baseline-candidate-pair + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :a/pan 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :b/pan 12)) + {:keys [text]} (compare/execute baseline candidate + {:case-pair ["a/pan" "b/pan"]})] + (t/is (str/includes? text "Pair a/pan,b/pan")))) + +(t/deftest cases-requires-scene-permission-for-different-scenes + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :a/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :b/load 12 :scene :other + :fingerprint (apply str (repeat 64 "b")))) + selector {:case-pair ["a/load" "b/load"]}] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate selector)))))) + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate + (assoc selector :allow-build-differences true))))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate (assoc selector :allow-scene-differences true))] + (t/is (= {:allow-build-differences false + :allow-scene-differences true + :allow-all-differences false} + (:permissions comparison))) + (t/is (= #{:scene :fingerprint} + (set (map :field (get-in comparison [:pairs 0 :mismatches]))))) + (t/is (every? #(and (:waived? %) (= :scene (:rule %))) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) + (t/is (str/includes? text "Pair a/load,b/load"))))) + +(t/deftest build-permission-allows-build-differences-only + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :functional {:mode :debug :features ["fixture"]}))] + (doseq [options [{} {:allow-scene-differences true}]] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate options))))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate {:allow-build-differences true})] + (t/is (= {:allow-build-differences true + :allow-scene-differences false + :allow-all-differences false} + (:permissions comparison))) + (t/is (= #{:functional-build :effective-features} + (set (map :field (get-in comparison [:pairs 0 :mismatches]))))) + (t/is (every? #(and (:waived? %) (= :build (:rule %))) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? text ":rule :build"))))) + +(t/deftest build-and-scene-permissions-combine + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 :scene :other + :functional {:mode :debug}))] + (doseq [options [{:allow-build-differences true} {:allow-scene-differences true}]] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate options))))))) + (let [{:keys [comparison]} + (compare/execute baseline candidate {:allow-build-differences true + :allow-scene-differences true})] + (t/is (= {:allow-build-differences true + :allow-scene-differences true + :allow-all-differences false} + (:permissions comparison))) + (t/is (= {:scene :scene :functional-build :build} + (into {} (map (juxt :field :rule)) + (get-in comparison [:pairs 0 :mismatches]))))))) + +(t/deftest comparison-failure-identifies-incompatible-fields + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :environment {:node "v25" :browser "Fake"})) + cause (failure #(compare/execute baseline candidate {})) + text (atom "") + stderr (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))})] + (try + (t/is (some? cause)) + (t/is (= 1 (execution/fail cause {:color :never} {:stderr stderr}))) + (t/is (str/includes? @text "incompatible saved runs")) + (t/is (str/includes? @text ":environment")) + (finally + (log/configure {}))))) + +(t/deftest scoped-permissions-keep-view-and-runtime-checks-strict + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10))] + (doseq [[field settings] + [[:execution {:case-settings {:view {:scale 2 :x 0 :y 0}}}] + [:environment {:environment {:node "v25" :browser "Fake"}}]]] + (let [candidate (write-run (path/join @temporary-root "candidate.transit") + (apply make-run "candidate" :case/load 12 (mapcat identity settings))) + cause (failure #(compare/execute baseline candidate + {:allow-build-differences true + :allow-scene-differences true}))] + (t/is (= ::report-compare/incompatible (:type (ex-data cause)))) + (t/is (= [field] (mapv :field (:mismatches (ex-data cause))))) + (t/is (every? (comp not :waived?) (:mismatches (ex-data cause)))))))) + +(t/deftest all-permission-allows-runtime-differences-and-displays-them + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :environment {:node "v25" :browser "Fake"}))] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate {})))))) + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate {:force true})))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate {:allow-all-differences true})] + (t/is (= {:allow-build-differences false + :allow-scene-differences false + :allow-all-differences true} + (:permissions comparison))) + (t/is (= [{:field :environment + :baseline {:node "v24" :browser "Fake"} + :candidate {:node "v25" :browser "Fake"} + :waived? true :rule :all}] + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/starts-with? text "Permissions ")) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) + (t/is (str/includes? text ":rule :all"))))) + +(t/deftest unreadable-records-fail-at-the-boundary + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12))] + (let [file (path/join @temporary-root "comparison.transit") + record (:comparison (compare/execute baseline candidate {}))] + (fs/writeFileSync file (result/encode record)) + (t/is (thrown-with-msg? js/Error #"expected a saved run record" + (compare/execute baseline file {})))) + (fs/writeFileSync (path/join @temporary-root "broken.transit") "not transit") + (t/is (thrown? js/Error (compare/execute baseline + (path/join @temporary-root "broken.transit") + {}))) + (fs/writeFileSync (path/join @temporary-root "malformed.transit") + (transit/encode-str (dissoc (make-run "malformed" :case/load 12) :plan))) + (doseq [options [{} {:allow-all-differences true}]] + (t/is (= ::result/invalid-record + (:type (ex-data (failure #(compare/execute baseline + (path/join @temporary-root "malformed.transit") + options))))))) + (t/is (thrown? js/Error (compare/execute baseline + (path/join @temporary-root "missing.transit") + {}))))) diff --git a/frontend/test/performance_tests/render_wasm/containers_test.cljs b/frontend/test/performance_tests/render_wasm/containers_test.cljs new file mode 100644 index 0000000000..19d68dc833 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/containers_test.cljs @@ -0,0 +1,251 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.containers-test + (:require + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as builder] + [benches.render-wasm.random :as random] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(t/deftest frame-attaches-children-in-order + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 100 :height 100 :name "F"} + (builder/rect {:x 10 :y 10 :width 5 :height 5}) + (builder/rect {:x 20 :y 20 :width 5 :height 5}))) + frame (first (helpers/children-of instance uuid/zero)) + rects (helpers/children-of instance (:id frame))] + (t/is (= :frame (:type frame))) + (t/is (= [(:id frame)] (helpers/child-ids instance uuid/zero))) + (t/is (= 2 (count rects))) + (t/is (every? #(= (:id frame) (:parent-id %)) rects)) + (t/is (every? #(= (:id frame) (:frame-id %)) rects)) + (t/is (= [10 20] (mapv :x rects))) + (t/is (= [10 20] (mapv :y rects))) + (t/is (= [100 100] [(:width frame) (:height frame)])))) + +(t/deftest nested-frames-keep-frame-ids + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 100 :height 100 :name "Outer"} + (builder/frame {:x 10 :y 10 :width 50 :height 50 :name "Inner"} + (builder/rect {:x 20 :y 20 :width 5 :height 5})))) + outer (first (helpers/children-of instance uuid/zero)) + inner (first (helpers/children-of instance (:id outer))) + rect (first (helpers/children-of instance (:id inner)))] + (t/is (= uuid/zero (:frame-id outer))) + (t/is (= (:id outer) (:parent-id inner))) + (t/is (= (:id outer) (:frame-id inner))) + (t/is (= (:id inner) (:parent-id rect))) + (t/is (= (:id inner) (:frame-id rect))) + (t/is (= [20 20] [(:x rect) (:y rect)])))) + +(t/deftest group-derives-bounds-from-children + (let [instance (builder/scene {:seed 1} + (builder/group {} + (builder/rect {:x 10 :y 20 :width 30 :height 40}) + (builder/rect {:x 50 :y 80 :width 10 :height 10}))) + group (first (helpers/children-of instance uuid/zero)) + rects (helpers/children-of instance (:id group))] + (t/is (= :group (:type group))) + (t/is (= [10 20 50 70] + [(:x group) (:y group) (:width group) (:height group)])) + (t/is (every? #(= uuid/zero (:frame-id %)) rects)) + (t/is (every? #(= (:id group) (:parent-id %)) rects)))) + +(t/deftest nested-groups-derive-inside-out + (let [instance (builder/scene {:seed 1} + (builder/group {:name "Outer"} + (builder/rect {:x 0 :y 0 :width 10 :height 10}) + (builder/group {:name "Inner"} + (builder/rect {:x 100 :y 100 :width 10 :height 10})))) + outer (first (helpers/children-of instance uuid/zero)) + inner (second (helpers/children-of instance (:id outer)))] + (t/is (= :group (:type inner))) + (t/is (= [100 100 10 10] + [(:x inner) (:y inner) (:width inner) (:height inner)])) + (t/is (= [0 0 110 110] + [(:x outer) (:y outer) (:width outer) (:height outer)])))) + +(t/deftest group-inside-frame-points-at-the-frame + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 200 :height 200} + (builder/group {} + (builder/rect {:x 10 :y 10 :width 10 :height 10})))) + frame (first (helpers/children-of instance uuid/zero)) + group (first (helpers/children-of instance (:id frame))) + rect (first (helpers/children-of instance (:id group)))] + (t/is (= (:id frame) (:frame-id group))) + (t/is (= (:id frame) (:frame-id rect))) + (t/is (= (:id group) (:parent-id rect))))) + +(t/deftest masked-group-takes-the-first-child-geometry + (let [instance (builder/scene {:seed 1} + (builder/group {:masked-group true} + (builder/rect {:x 10 :y 20 :width 30 :height 40}) + (builder/rect {:x 0 :y 0 :width 500 :height 500}))) + group (first (helpers/children-of instance uuid/zero))] + (t/is (true? (:masked-group group))) + (t/is (= [10 20 30 40] + [(:x group) (:y group) (:width group) (:height group)])))) + +(t/deftest container-labels-are-snapshot-local + (let [instance (builder/scene {:seed 1} + (builder/frame :hero {:x 0 :y 0 :width 10 :height 10} + (builder/group [:g 1] {} + (builder/rect)))) + refs (:refs instance)] + (t/is (= #{:hero [:g 1]} (set (keys refs)))) + (t/is (= (get refs :hero) (first (helpers/child-ids instance uuid/zero)))) + (t/is (= (get refs [:g 1]) (first (helpers/child-ids instance (get refs :hero))))) + (t/is (= 1 (count (helpers/child-ids instance uuid/zero)))))) + +(t/deftest container-expressions-evaluate-once-in-order + (let [events (atom []) + instance (builder/scene {:seed 1} + (builder/frame [:frame (do (swap! events conj :label) 3)] + (do (swap! events conj :attrs) + {:x 0 :y 0 :width 20 :height 20}) + (swap! events conj :body) + (builder/rect {:x 10}))) + frame-id (get (:refs instance) [:frame 3])] + (t/is (= [:attrs :label :body] @events)) + (t/is (= [10] (mapv :x (helpers/children-of instance frame-id)))))) + +(t/deftest invalid-container-attrs-retain-seeded-construction-order + (doseq [construct [#(builder/frame :hero :invalid) + #(builder/group :hero :invalid) + #(builder/bool :hero :invalid)]] + (let [rng (random/create 7) + failed-id (random/rng-uuid rng) + next-id (random/rng-uuid rng) + failure (volatile! nil) + state (helpers/run-scope + {:seed 7} + (fn [] + (vreset! failure (helpers/failure-data construct)) + (builder/rect))) + instance (builder/finish! state)] + (t/is (= {:type ::builder/invalid-attrs :id failed-id} @failure)) + (t/is (= [next-id] (helpers/child-ids instance uuid/zero)))))) + +(t/deftest empty-group-is-rejected + (let [data (helpers/failure-data #(builder/scene {:seed 1} (builder/group {})))] + (t/is (= ::builder/empty-group (:type data))))) + +(t/deftest group-geometry-attrs-are-rejected + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/group {:x 1} + (builder/rect))))] + (t/is (= ::builder/group-geometry (:type data))) + (t/is (= :x (:key data))))) + +(t/deftest frame-requires-bounds + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame {:name "F"} + (builder/rect))))] + (t/is (= ::builder/frame-bounds (:type data))))) + +(t/deftest runtime-attrs-expressions-are-accepted + (let [frame-attrs {:x 0 :y 0 :width 10 :height 10} + instance (builder/scene {:seed 1} + (builder/frame frame-attrs + (builder/rect))) + frame (first (helpers/children-of instance uuid/zero))] + (t/is (= :frame (:type frame))) + (t/is (= 10 (:width frame))))) + +(t/deftest non-map-attrs-are-rejected + (t/is (= ::builder/invalid-attrs + (:type (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame :hero (builder/rect))))))) + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (builder/frame :hero (builder/rect)) + (catch :default _ nil))))] + (t/is (= 1 (count (helpers/child-ids @state uuid/zero)))))) + +(t/deftest non-numeric-frame-bounds-are-rejected + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width "100" :height 10} + (builder/rect))))] + (t/is (= ::builder/frame-bounds (:type data))) + (t/is (= :width (:invalid data))))) + +(t/deftest duplicate-container-labels-are-rejected + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame :dup {:x 0 :y 0 :width 10 :height 10} + (builder/rect)) + (builder/group :dup {} + (builder/rect))))] + (t/is (= ::builder/duplicate-label (:type data))) + (t/is (= :dup (:label data))))) + +(t/deftest frame-inside-group-keeps-frame-ids + (let [instance (builder/scene {:seed 1} + (builder/group {} + (builder/frame {:x 0 :y 0 :width 50 :height 50} + (builder/rect)))) + group (first (helpers/children-of instance uuid/zero)) + frame (first (helpers/children-of instance (:id group))) + rect (first (helpers/children-of instance (:id frame)))] + (t/is (= (:id group) (:parent-id frame))) + (t/is (= uuid/zero (:frame-id frame))) + (t/is (= (:id frame) (:parent-id rect))) + (t/is (= (:id frame) (:frame-id rect))))) + +(t/deftest group-of-container-derives-bounds + (let [instance (builder/scene {:seed 1} + (builder/group {} + (builder/group {} + (builder/rect {:x 5 :y 5 :width 10 :height 10})))) + outer (first (helpers/children-of instance uuid/zero)) + inner (first (helpers/children-of instance (:id outer)))] + (t/is (= [5 5 10 10] + [(:x outer) (:y outer) (:width outer) (:height outer)])) + (t/is (= [5 5 10 10] + [(:x inner) (:y inner) (:width inner) (:height inner)])) + (t/is (= 1 (count (helpers/child-ids instance (:id outer))))))) + +(t/deftest empty-frame-is-allowed + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 10 :height 10})) + frame (first (helpers/children-of instance uuid/zero))] + (t/is (= [] (helpers/child-ids instance (:id frame)))))) + +(t/deftest caught-empty-group-still-fails-finish + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (builder/group {}) + (catch :default _ nil))))] + (t/is (= ::builder/unfinished-container + (:type (helpers/failure-data #(builder/finish! state))))))) + +(t/deftest container-failure-restores-parent-and-marks-unfinished + (let [state (helpers/run-scope {:seed 1} + (fn [] + (builder/rect {:x 0}) + (try + (builder/frame {:x 0 :y 0 :width 10 :height 10} + (builder/rect :dup {:x 1}) + (builder/rect :dup {:x 2})) + (catch :default _ nil)) + (builder/rect {:x 3}))) + root-children (helpers/child-ids @state uuid/zero) + last-child (get-in @state [:objects (last root-children)])] + (t/is (= 3 (count root-children))) + (t/is (= :rect (:type last-child))) + (t/is (= 3 (:x last-child))) + (t/is (= 1 (count (:unfinished @state)))) + (t/is (= ::builder/unfinished-container + (:type (helpers/failure-data #(builder/finish! state))))))) diff --git a/frontend/test/performance_tests/render_wasm/contracts_test.cljs b/frontend/test/performance_tests/render_wasm/contracts_test.cljs new file mode 100644 index 0000000000..0804f0e388 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/contracts_test.cljs @@ -0,0 +1,445 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.contracts-test + (:require + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [cljs.test :as t :include-macros true])) + +(defrecord Point [x y]) + +(def ^:private body-calls + (atom [])) + +(defn- touch! + [rtx] + (swap! body-calls conj :touched) + rtx) + +(decl/defscene :contracts-scene + {:description "Synthetic scene for contract tests" + :params-schema [:map {:closed true}]} + [params] + params) + +(decl/defcase :contracts-scene/case + {:params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh} + [ctx] + (touch! ctx)) + +(defn- failure-data + [f] + (try + (f) + nil + (catch :default cause + (ex-data cause)))) + +(defn- rect-cases + ([] (rect-cases 42 "rects/")) + ([master-seed filter] + (cases/collect-cases {:master-seed master-seed :filter filter}))) + +(defn- with-synthetic-scene + "Registers a synthetic scene and case under `scene-id`, runs `f`, then + unregisters both so the live registry survives." + [scene-id case-params f] + (decl/register-scene! {:id scene-id + :ns "contracts-test" + :description "synthetic" + :params-schema [:map {:closed true} [:count [:int {:min 1 :max 100000}]]] + :build (fn [params] params)}) + (decl/register-case! {:id (keyword (name scene-id) "case") + :scene scene-id + :ns "contracts-test" + :params case-params + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (try + (f) + (finally + (decl/unregister-case! (keyword (name scene-id) "case")) + (decl/unregister-scene! scene-id)))) + +(t/deftest defscene-registers-metadata-and-build + (let [scene (decl/registered-scene :rects)] + (t/is (= :rects (:id scene))) + (t/is (= "benches.render-wasm.scenes.rects" (:ns scene))) + (t/is (= "Seeded rectangles with translucent fills and centered strokes" + (:description scene))) + (t/is (vector? (:params-schema scene))) + (t/is (fn? (:build scene))))) + +(t/deftest rects-cases-are-collected-in-declaration-order + (let [collected (rect-cases)] + (t/is (= [:rects/load :rects/pan :rects/zoom :rects/blocking-full :rects/mixed-pans] (mapv :id collected))))) + +(t/deftest collected-cases-survive-the-wire + (doseq [scene-id [:rects :paths :plus :masks]] + (let [collected (cases/collect-cases {:master-seed 42 + :filter (str (name scene-id) "/")}) + decoded (transit/decode-str (transit/encode-str collected))] + (t/is (= collected decoded)) + (doseq [case-desc decoded] + (t/is (= case-desc (decl/check-collected-case case-desc))) + (t/is (not-any? #(contains? case-desc %) + [:run! :ns]) + (str (:id case-desc))))))) + +(t/deftest declared-preparation-survives-collection-and-wire + (with-synthetic-scene :contracts-prepared {:count 3} + (fn [] + (let [preparation {:render :none :settle-ms 25} + registered (decl/registered-case :contracts-prepared/case)] + (decl/register-case! (assoc registered + :preparation (assoc preparation :callback identity) + :reset identity)) + (let [collected (cases/collect-cases {:master-seed 42 :filter "contracts-prepared/"}) + decoded (transit/decode-str (transit/encode-str collected))] + (t/is (= [:contracts-prepared/case] (mapv :id decoded))) + (t/is (= collected decoded)) + (t/is (= preparation (:preparation (first decoded)))) + (t/is (= (first decoded) (decl/check-collected-case (first decoded))))) + (decl/register-case! (assoc registered :preparation {:asset {:loader identity}})) + (t/is (= ::cases/invalid-case + (:type (failure-data #(cases/collect-cases {:master-seed 42 :filter "contracts-prepared/"}))))))))) + +(t/deftest rects-cases-carry-declared-view-context-and-preparation + (let [collected (rect-cases)] + (t/is (= {:rects/load :fresh :rects/pan :reuse :rects/zoom :reuse + :rects/blocking-full :fresh :rects/mixed-pans :reuse} + (into {} (map (juxt :id :context)) collected))) + (doseq [case-desc collected] + (t/is (= {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100} + (:params case-desc)) + (str (:id case-desc))) + (t/is (= {:scale 1 :x 0 :y 0} (:view case-desc)) (str (:id case-desc))) + (t/is (= (if (#{:rects/load :rects/blocking-full} (:id case-desc)) {:render :none} {}) + (:preparation case-desc)) (str (:id case-desc)))))) + +(t/deftest rects-cases-carry-scene-metadata-and-seed + (let [collected (rect-cases)] + (doseq [case-desc collected] + (t/is (= :rects (:scene case-desc))) + (t/is (= "Seeded rectangles with translucent fills and centered strokes" + (:scene-description case-desc))) + (t/is (= 2602546180 (:scene-seed case-desc)) (str (:id case-desc))))) + (let [collected (rect-cases 43 "rects/")] + (t/is (every? #(= 2993218129 (:scene-seed %)) collected)))) + +(t/deftest derive-seed-depends-on-scene-and-params-only + (let [params {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100} + base (decl/derive-seed 42 :rects params)] + (t/is (= 2602546180 base)) + (t/is (= base (decl/derive-seed 42 :rects (into {} (reverse (seq params)))))) + (t/is (= 2993218129 (decl/derive-seed 43 :rects params))) + (t/is (not= base (decl/derive-seed 42 :rects (assoc params :count 10)))) + (t/is (not= base (decl/derive-seed 42 :paths params))))) + +(t/deftest derive-seed-rejects-keys-it-cannot-tell-apart + (t/is (= ::decl/invalid-params + (:type (failure-data #(decl/derive-seed 42 :rects {"count" 1}))))) + (t/is (= ::decl/invalid-params + (:type (failure-data #(decl/derive-seed 42 :rects {:rects/count 1}))))) + (t/is (= :rects (:scene (failure-data #(decl/derive-seed 42 :rects {"count" 1})))))) + +(t/deftest unknown-params-are-rejected + (with-synthetic-scene :contracts-unknown-params {:count 1 :bogus 2} + (fn [] + (let [data (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-unknown-params/"}))] + (t/is (= ::cases/invalid-case (:type data))) + (t/is (some? (::sm/explain data))))))) + +(t/deftest out-of-range-count-is-rejected + (with-synthetic-scene :contracts-bad-count {:count 0} + (fn [] + (let [data (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-bad-count/"}))] + (t/is (= ::cases/invalid-case (:type data))) + (t/is (some? (::sm/explain data))))))) + +(t/deftest rects-workload-ranges-are-rejected + (let [schema (:params-schema (decl/registered-scene :rects)) + good {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100}] + (t/is (true? (sm/validate schema good))) + (t/testing "reversed size range" + (t/is (false? (sm/validate schema (assoc good :min-size 100 :max-size 20))))) + (t/testing "non-finite sizes" + (doseq [bad [(assoc good :width js/Infinity) + (assoc good :height js/NaN) + (assoc good :min-size js/Infinity) + (assoc good :max-size js/NaN)]] + (t/is (false? (sm/validate schema bad)) (pr-str bad)))))) + +(t/deftest view-coordinates-are-finite + (let [good {:scale 1 :x 0 :y 0}] + (t/is (true? (sm/validate decl/schema:view good))) + (doseq [bad [(assoc good :scale js/Infinity) + (assoc good :x js/NaN) + (assoc good :y js/Infinity) + (assoc good :viewport {:width js/NaN})]] + (t/is (false? (sm/validate decl/schema:view bad)) (pr-str bad))))) + +(t/deftest transit-rejects-unencodable-values + (t/is (false? (decl/transit-round-trips? {:v (fn [] 1)}))) + (t/is (false? (decl/transit-round-trips? {:point (->Point 1 2)})))) + +(t/deftest transit-wire-preserves-json-lossy-shapes + (doseq [value [{:foo 1 "foo" 2} + {:a/b 1 :c/b 2} + {:id :rects/load :scene :rects} + {:tags [:fresh :rects/load]} + {:v js/Infinity} + {:v js/-Infinity}]] + (t/is (= value (-> value transit/encode-str transit/decode-str)) + (pr-str value)))) + +(t/deftest transit-wire-preserves-non-finite-numbers + (let [value {:nan js/NaN :positive js/Infinity :negative js/-Infinity} + decoded (-> value transit/encode-str transit/decode-str) + case-desc (assoc (first (rect-cases)) :params value)] + (t/is (number? (:nan decoded))) + (t/is (js/Number.isNaN (:nan decoded))) + (t/is (= js/Infinity (:positive decoded))) + (t/is (= js/-Infinity (:negative decoded))) + (t/is (decl/transit-round-trips? value)) + (t/is (nil? (failure-data #(decl/check-collected-case case-desc)))))) + +(t/deftest unencodable-collected-values-are-rejected + (let [good (first (rect-cases))] + (doseq [bad [(assoc good :params {:run (fn [] 1)}) + (assoc good :params {:point (->Point 1 2)})]] + (t/is (= ::decl/non-serializable-case + (:type (failure-data #(decl/check-collected-case bad)))) + (pr-str bad))))) + +(t/deftest duplicate-scene-id-from-another-namespace-is-rejected + (t/is (= ::decl/duplicate-scene + (:type (failure-data + #(decl/register-scene! {:id :rects + :ns "another-namespace" + :description "x" + :params-schema [:map] + :build (fn [params] params)})))))) + +(t/deftest scene-replacement-preserves-order-and-unregister-removes + (let [ids [:contracts-first :contracts-middle :contracts-last] + scene {:ns "contracts-test" :description "first" + :params-schema [:map] :build identity} + get-scenes #(filterv (comp (set ids) :id) (decl/scenes))] + (try + (doseq [id ids] + (decl/register-scene! (assoc scene :id id))) + (decl/register-scene! (assoc scene :id :contracts-middle :description "second")) + (t/is (= ids (mapv :id (get-scenes)))) + (t/is (= ["first" "second" "first"] (mapv :description (get-scenes)))) + (decl/unregister-scene! :contracts-middle) + (t/is (= [:contracts-first :contracts-last] (mapv :id (get-scenes)))) + (t/is (nil? (decl/registered-scene :contracts-middle))) + (finally + (doseq [id ids] + (decl/unregister-scene! id)))))) + +(t/deftest duplicate-case-id-from-another-namespace-is-rejected + (try + (decl/register-scene! {:id :contracts-dup + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (decl/register-case! {:id :contracts-dup/case + :scene :contracts-dup + :ns "contracts-test" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (t/is (= ::decl/duplicate-case + (:type (failure-data + #(decl/register-case! {:id :contracts-dup/case + :scene :contracts-dup + :ns "another-namespace" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}))))) + (finally + (decl/unregister-case! :contracts-dup/case) + (decl/unregister-scene! :contracts-dup)))) + +(t/deftest unknown-scene-reference-is-rejected + (try + (decl/register-case! {:id :contracts-missing/case + :scene :contracts-missing + :ns "contracts-test" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (t/is (= ::cases/unknown-scene + (:type (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-missing/"}))))) + (finally + (decl/unregister-case! :contracts-missing/case)))) + +(t/deftest case-id-must-be-namespaced-by-its-scene + (try + (decl/register-scene! {:id :contracts-other + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (decl/register-case! {:id :contracts-other/case + :scene :rects + :ns "contracts-test" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (t/is (= ::cases/invalid-case + (:type (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-other/"}))))) + (finally + (decl/unregister-case! :contracts-other/case) + (decl/unregister-scene! :contracts-other)))) + +(t/deftest scene-without-params-schema-is-rejected + (try + (decl/register-scene! {:id :contracts-noschema + :ns "contracts-test" + :description "x" + :build (fn [params] params)}) + (t/is (= ::decl/invalid-scene + (:type (failure-data #(cases/collect-cases {:master-seed 42}))))) + (finally + (decl/unregister-scene! :contracts-noschema)))) + +(t/deftest malformed-params-schema-is-rejected + (try + (decl/register-scene! {:id :contracts-badschema + :ns "contracts-test" + :description "x" + :params-schema [:map [:count]] + :build (fn [params] params)}) + (let [data (failure-data #(cases/collect-cases {:master-seed 42}))] + (t/is (= ::decl/invalid-scene (:type data))) + (t/is (= :contracts-badschema (:id data)))) + (finally + (decl/unregister-scene! :contracts-badschema)))) + +(t/deftest qualified-scene-id-is-rejected + (try + (decl/register-scene! {:id :contracts-ns/scene + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (t/is (= ::decl/invalid-scene + (:type (failure-data #(cases/collect-cases {:master-seed 42}))))) + (finally + (decl/unregister-scene! :contracts-ns/scene)))) + +(t/deftest invalid-declarations-fail-outside-the-filter + (with-synthetic-scene :contracts-global {:count 1 :bogus 2} + (fn [] + (t/is (= ::cases/invalid-case + (:type (failure-data #(cases/collect-cases {:master-seed 42 + :filter "rects/"})))))))) + +(t/deftest incomplete-case-declaration-is-rejected + (try + (decl/register-scene! {:id :contracts-incomplete + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (decl/register-case! {:id :contracts-incomplete/case + :scene :contracts-incomplete + :ns "contracts-test" + :params {} + :context :fresh}) + (t/is (= ::cases/invalid-case + (:type (failure-data #(cases/collect-cases {:master-seed 42 + :filter "contracts-incomplete/"}))))) + (finally + (decl/unregister-case! :contracts-incomplete/case) + (decl/unregister-scene! :contracts-incomplete)))) + +(t/deftest records-do-not-survive-the-wire + (let [point (->Point 1 2)] + (t/is (false? (decl/transit-round-trips? point))) + (t/is (false? (decl/transit-round-trips? {:point point}))))) + +(t/deftest case-replacement-preserves-order-and-unregister-removes + (let [ids [:contracts-replacecase/first :contracts-replacecase/middle :contracts-replacecase/last] + bench-case {:scene :contracts-replacecase :ns "contracts-test" + :params {} :view {:scale 1 :x 0 :y 0} :context :fresh} + get-cases #(filterv (comp (set ids) :id) (decl/cases))] + (try + (decl/register-scene! {:id :contracts-replacecase :ns "contracts-test" + :description "x" :params-schema [:map] :build identity}) + (doseq [id ids] + (decl/register-case! (assoc bench-case :id id))) + (decl/register-case! (assoc bench-case :id :contracts-replacecase/middle :view {:scale 2 :x 0 :y 0})) + (t/is (= ids (mapv :id (get-cases)))) + (t/is (= [1 2 1] (mapv #(get-in % [:view :scale]) (get-cases)))) + (decl/unregister-case! :contracts-replacecase/middle) + (t/is (= [:contracts-replacecase/first :contracts-replacecase/last] (mapv :id (get-cases)))) + (t/is (nil? (decl/registered-case :contracts-replacecase/middle))) + (finally + (doseq [id ids] + (decl/unregister-case! id)) + (decl/unregister-scene! :contracts-replacecase))))) + +(t/deftest invalid-master-seed-is-rejected + (doseq [seed [-1 4294967296 "42" nil 0.5 js/NaN js/Infinity js/-Infinity]] + (t/is (= ::cases/invalid-master-seed + (:type (failure-data #(cases/collect-cases {:master-seed seed})))) + (str seed)))) + +(t/deftest filter-matches-case-ids + (t/is (= [:rects/pan] + (mapv :id (cases/collect-cases {:master-seed 42 :filter "rects/pan"})))) + (t/is (= 5 (count (cases/collect-cases {:master-seed 42 :filter "rects/"}))))) + +(t/deftest empty-selection-is-rejected + (t/is (= ::cases/no-cases + (:type (failure-data #(cases/collect-cases {:master-seed 42 + :filter "missing"})))))) + +(t/deftest defcase-registers-function-without-sending-it + (reset! body-calls []) + (let [collected (cases/collect-cases {:master-seed 42 :filter "contracts-scene/"})] + (t/is (= [:contracts-scene/case] (mapv :id collected))) + (t/is (not (contains? (first collected) :run!))) + (t/is (decl/transit-round-trips? (first collected)))) + (t/is (empty? @body-calls)) + (let [entry (decl/registered-case :contracts-scene/case)] + (t/is (= "performance-tests.render-wasm.contracts-test" (:ns entry))) + (t/is (fn? (:run! entry))) + (t/is (= :runtime ((:run! entry) :runtime))) + (t/is (= [:touched] @body-calls)))) + +(t/deftest defcase-rejects-run-function-in-options + (t/is (= :benches.render-wasm.declarations/invalid-run-function + (:type (failure-data + #(decl/defcase :contracts-scene/unchecked + {:run! (fn [x y] [x y])}))))) + (t/is (nil? (decl/registered-case :contracts-scene/unchecked)))) + +(t/deftest defcase-rejects-unqualified-id + (t/is (= :benches.render-wasm.declarations/invalid-run-function + (:type (failure-data + #(decl/defcase :bare + {:params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh})))))) diff --git a/frontend/test/performance_tests/render_wasm/errors_test.cljs b/frontend/test/performance_tests/render_wasm/errors_test.cljs new file mode 100644 index 0000000000..ef300c168e --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/errors_test.cljs @@ -0,0 +1,248 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.errors-test + (:require + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(t/deftest attempt-returns-the-value-of-its-body + (t/is (= 3 (attempt 1 2 3))) + (t/is (nil? (attempt nil))) + (t/is (= {:error :not-a-failure} (attempt {:error :not-a-failure})))) + +(t/deftest composed-failures-retain-occurrence-order-and-fatal-propagation + (let [upload (js/Error. "upload failed") + end (js/Error. "ending load failed") + close (js/Error. "browser disposal failed") + save (js/Error. "saving evidence failed") + initial (errors/combine [{:phase "upload" :operation :serialize-shapes :error upload} + {:phase "upload" :operation :end-loading :error end}]) + wrapped (errors/fatal-error "preparation failed" + {::errors/stale true :partial {:setup {:upload-ms 7}}} + (errors/with-context initial "preparation failed" + {:phase "preparation" :path "scene.transit"})) + later (errors/combine [{:phase "cleanup" :operation :close-browser :error close} + {:phase "storage" :operation :save-evidence :error save}]) + later (errors/fatal-error "cleanup failed" + {:partial {:setup {:upload-ms 9}} + :command "/bin/chromium" :exit-code 17} + (errors/with-data later {:partial {:render {:frames 2}}})) + combined (errors/combine [{:error wrapped} {:error later}]) + entries (errors/get-errors combined) + deep (nth (iterate #(ex-info "wrapped" {} %) combined) 12) + wire (-> (errors/evidence :runner deep) + transit/encode-str transit/decode-str errors/project-diagnostic)] + (t/is (= [upload end close save] (mapv :error entries))) + (t/is (= [:serialize-shapes :end-loading :close-browser :save-evidence] + (mapv :operation entries))) + (t/is (= "scene.transit" (:path (first entries)))) + (t/is (= {:setup {:upload-ms 7}} (:partial (first entries)))) + (t/is (errors/stale? combined)) + (t/is (= (mapv errors/message-of [upload end close save]) (mapv :message (:errors wire)))) + (t/is (= {:command "/bin/chromium" :exit-code 17 + :partial {:setup {:upload-ms 9} :render {:frames 2}}} + (select-keys (nth (:errors wire) 2) [:command :exit-code :partial]))) + (t/is (sm/validate errors/schema:diagnostic wire)) + (t/is (errors/fatal? deep)) + (t/is (errors/fatal? (errors/combine [{:error upload} {:error later}]))) + (t/is (try + (attempt (throw combined)) + false + (catch :default cause + (= [upload end close save] (mapv :error (errors/get-errors cause)))))))) + +(t/deftest ordinary-failures-project-operation-and-path-context + (let [cause (ex-info "record unavailable" {:operation :read-record :path "saved/run.transit"}) + wire (-> (errors/evidence :storage cause) transit/encode-str transit/decode-str)] + (t/is (= {:operation :read-record :path "saved/run.transit"} + (select-keys wire [:operation :path]))) + (t/is (sm/validate errors/schema:diagnostic wire)))) + +(t/deftest attempt-returns-the-error-its-body-throws + (let [cause (ex-info "boom" {:a 1}) + result (attempt (throw cause))] + (t/is (failure? result)) + (t/is (identical? cause result)))) + +(t/deftest attempt-turns-a-thrown-value-into-an-error-that-keeps-its-text + (let [text (attempt (throw "device disconnected")) + object (attempt (throw #js {:code 7}))] + (t/is (failure? text)) + (t/is (= "device disconnected" (ex-message text))) + (t/is (failure? object)) + (t/is (str/includes? (ex-message object) "code")))) + +(t/deftest attempt-throws-fatal-failures-again + (let [cause (errors/fatal-error "write failed" {:path "a"} (js/Error. "disk"))] + (t/is (thrown-with-msg? js/Error #"write failed" + (attempt (throw cause)))) + (t/is (identical? cause (attempt :catch-fatal (throw cause)))))) + +(t/deftest attempt-treats-a-fatal-cause-deep-in-the-chain-as-fatal + (let [fatal (errors/fatal-error "write failed" {} nil) + outer (ex-info "outer" {} (ex-info "middle" {} fatal))] + (t/is (true? (errors/fatal? outer))) + (t/is (thrown-with-msg? js/Error #"outer" (attempt (throw outer)))))) + +(t/deftest fatal-marker-is-not-assumed-from-other-data + (t/is (false? (errors/fatal? (ex-info "x" {:type :anything})))) + (t/is (false? (errors/fatal? nil))) + (t/is (false? (errors/fatal? "text")))) + +(t/deftest ^:async attempt-awaits-promises-in-an-async-function + (let [value (attempt (await (js/Promise.resolve 7))) + failure (attempt (await (js/Promise.reject (ex-info "rejected" {})))) + bound (let [x (attempt :catch-fatal (await (js/Promise.resolve 1)) 2)] x)] + (t/is (= 7 value)) + (t/is (= "rejected" (ex-message failure))) + (t/is (= 2 bound)))) + +(t/deftest ^:async attempt-keeps-fatal-rejections-fatal + (let [cause (errors/fatal-error "write failed" {} nil) + outer (try + (attempt (await (js/Promise.reject cause))) + nil + (catch :default thrown thrown))] + (t/is (identical? cause outer)))) + +(t/deftest with-data-passes-non-failures-through + (t/is (= 1 (errors/with-data 1 {:a 1}))) + (t/is (nil? (errors/with-data nil {:a 1}))) + (t/is (= {:a 1} (errors/with-data {:a 1} {:a 2})))) + +(t/deftest with-data-adds-keys-under-the-failure-data + (let [cause (ex-info "boom" {:log-path "inner"}) + result (errors/with-data cause {:log-path "outer" :build-id "b"})] + (t/is (= "boom" (ex-message result))) + (t/is (= {:log-path "inner" :build-id "b"} (ex-data result))) + (t/is (identical? cause (ex-cause result))))) + +(t/deftest message-of-falls-back-to-the-string-form + (t/is (= "boom" (errors/message-of (ex-info "boom" {})))) + (t/is (= "text" (errors/message-of "text")))) + +(t/deftest context-keeps-deeper-measurements-and-native-cause-details + (let [native (ex-info "render failed" {:fn "_render" :code 17}) + cause (ex-info "interact failed" {:phase "interact" :partial {:slices [1 2]}} native) + error (errors/with-context cause "setup failed" + {:phase "setup" :partial {:slices [1] :setup {:upload-ms 3}}}) + result (errors/evidence :setup error)] + (t/is (= {:phase "setup" :partial {:slices [1 2] :setup {:upload-ms 3}}} + (ex-data error))) + (t/is (= "setup failed" (:message result))) + (t/is (some #(= {:message "render failed" :fn "_render" :code 17} %) + (:cause result))))) + +(t/deftest context-keeps-native-errors-as-diagnostic-causes + (let [error (errors/with-context (js/Error. "device lost") "restore failed" {:phase "restore"})] + (t/is (some #(= "device lost" (:message %)) (errors/describe-cause error))))) + +(t/deftest hostile-cause-renders-plain-data + (let [hostile (ex-info "boom" + {:fn (fn [] 1) :code #js {:evil 1} :type :wasm-error + :hint (str (apply str (repeat 499 "x")) "😀")} + (js/Error. "inner boom")) + result (errors/evidence :aborted hostile) + decoded (-> result transit/encode-str transit/decode-str) + outer (first (:cause decoded))] + (t/is (= "aborted" (:phase decoded))) + (t/is (= "boom" (:message decoded))) + (t/is (string? (:fn outer))) + (t/is (string? (:code outer))) + (t/is (= (apply str (repeat 499 "x")) (:hint outer))) + (t/is (some #(= "inner boom" (:message %)) (:cause decoded))))) + +(t/deftest diagnostics-retain-bounded-identity-and-process-details + (let [long-text (str (apply str (repeat 1023 "x")) "😀") + context {:expected-id long-text :actual-id "other-build" + :command "compiler" :exit-code 23 :signal "SIGTERM" + :endpoint "ws://127.0.0.1/devtools/browser/owner"} + cause (ex-info "failed" context) + decoded (-> (errors/evidence :identity cause) + errors/project-diagnostic + transit/encode-str + transit/decode-str)] + (t/is (= (assoc context :expected-id (apply str (repeat 1023 "x"))) + (select-keys decoded (keys context)))))) + +(t/deftest unprintable-values-produce-portable-diagnostics + (let [evil (js-obj)] + (js/Object.defineProperty evil "evil" + #js {:enumerable true + :get (fn [] (throw (js/Error. "evil")))}) + (let [data (errors/evidence :upload (ex-info "evil boom" {:fn evil})) + cause (errors/evidence :upload evil) + decoded (-> [data cause] transit/encode-str transit/decode-str)] + (t/is (= "unrenderable value" (get-in decoded [0 :cause 0 :fn]))) + (t/is (= "unrenderable cause" (get-in decoded [1 :cause 0 :message])))))) + +(t/deftest diagnostic-text-is-bounded-without-splitting-surrogate-pairs + (let [message (str (apply str (repeat 1023 "x")) "😀" "tail") + cause (ex-info message {:command message}) + data (-> (errors/evidence :main cause) transit/encode-str transit/decode-str) + hostile (js/Error.)] + (set! (.-message hostile) #js {:toString #(throw (js/Error. "cannot print"))}) + (t/is (= (apply str (repeat 1023 "x")) (:message data))) + (t/is (<= (count (:command data)) 1024)) + (t/is (sm/validate errors/schema:diagnostic (errors/evidence :main hostile))))) + +(t/deftest malformed-partial-data-retains-the-failure-and-representable-evidence + (let [cycle #js {}] + (aset cycle "self" cycle) + (doseq [invalid [(fn [] nil) cycle (js/AbortController.)] + composed? [false true]] + (let [failure (ex-info "phase failed" {:partial {:safe {:elapsed [0 js/Infinity js/NaN]} + :handle invalid}}) + failure (if composed? + (errors/combine [{:error failure} {:error (js/Error. "cleanup failed")}]) + failure) + data (errors/evidence :main failure) + wire (try + (-> data transit/encode-str transit/decode-str) + (catch :default _ nil))] + (t/is (= "phase failed" (:message wire))) + (t/is (= "phase failed" (get-in wire [:cause 0 :message]))) + (t/is (= 0 (get-in wire [:partial :safe :elapsed 0]))) + (t/is (= js/Infinity (get-in wire [:partial :safe :elapsed 1]))) + (t/is (js/Number.isNaN (get-in wire [:partial :safe :elapsed 2]))) + (t/is (= 1 (count (filter #(= :encode-diagnostic (:operation %)) (:errors wire))))))))) + +(t/deftest unbounded-partial-data-produces-an-encoding-diagnostic + (let [data (errors/evidence :main (ex-info "phase failed" {:partial {:trace (repeat 0)}})) + wire (-> data transit/encode-str transit/decode-str)] + (t/is (= "phase failed" (:message wire))) + (t/is (some #(= :encode-diagnostic (:operation %)) (:errors wire))))) + +(t/deftest malformed-partial-shapes-and-cyclic-causes-produce-portable-diagnostics + (doseq [partial [nil false 42 "partial"]] + (let [data (errors/evidence :main (ex-info "phase failed" {:partial partial}))] + (t/is (sm/validate errors/schema:diagnostic data)) + (t/is (some #(= :encode-diagnostic (:operation %)) (:errors data))))) + (let [cycle (js/Error. "cyclic failure")] + (set! (.-cause cycle) cycle) + (let [data (-> (errors/evidence :main cycle) transit/encode-str transit/decode-str)] + (t/is (= "cyclic failure" (:message data))) + (t/is (<= (count (:cause data)) 3))))) + +(t/deftest malformed-optional-metadata-retains-a-valid-failure-diagnostic + (doseq [flag [42 (fn [] nil)]] + (let [cause (errors/combine [{:error (ex-info "main failed" {:deadline-exceeded flag})}]) + data (errors/evidence :main cause) + wire (try (-> data transit/encode-str transit/decode-str) (catch :default _ nil))] + (t/is (= "main failed" (:message wire))) + (t/is (sm/validate errors/schema:diagnostic wire))))) + +(t/deftest non-finite-failure-details-cross-transit-as-numbers + (let [failure (ex-info "bad measurement" {:code js/NaN :hint js/Infinity :type js/-Infinity}) + decoded (-> (errors/evidence :render failure) transit/encode-str transit/decode-str) + detail (first (:cause decoded))] + (t/is (js/Number.isNaN (:code detail))) + (t/is (= js/Infinity (:hint detail))) + (t/is (= js/-Infinity (:type detail))))) diff --git a/frontend/test/performance_tests/render_wasm/execution_test.cljs b/frontend/test/performance_tests/render_wasm/execution_test.cljs new file mode 100644 index 0000000000..53e4c26264 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/execution_test.cljs @@ -0,0 +1,346 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.execution-test + (:require + ["fs" :as fs] + ["node:module" :refer [createRequire]] + ["node:stream" :as stream] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.cli.resolve :as cli] + [benches.render-wasm.node.log :as log] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def temporary-root (atom nil)) + +(defn- within + [promise] + (let [timer (volatile! nil)] + (-> (js/Promise.race + #js [promise + (js/Promise. + (fn [_ reject] + (vreset! timer (js/setTimeout #(reject (js/Error. "Test stalled")) 5000))))]) + (.finally #(js/clearTimeout @timer))))) + +(t/deftest ^:async real-signals-reach-the-callback-and-remover-restores-listeners + (let [before-int (.listenerCount js/process "SIGINT") + before-term (.listenerCount js/process "SIGTERM") + received (atom []) + next-signal (volatile! nil) + remove (execution/on-signals + (fn [signal] + (swap! received conj signal) + (@next-signal signal))) + receive (fn [signal] + (within + (js/Promise. + (fn [resolve _] + (vreset! next-signal resolve) + (.kill js/process (.-pid js/process) signal)))))] + (try + (t/is (= "SIGINT" (await (receive "SIGINT")))) + (t/is (= "SIGTERM" (await (receive "SIGTERM")))) + (t/is (= ["SIGINT" "SIGTERM"] @received)) + (finally + (remove))) + (remove) + (t/is (= before-int (.listenerCount js/process "SIGINT"))) + (t/is (= before-term (.listenerCount js/process "SIGTERM"))))) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "benchmark-execution-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- capture + [] + (let [output (atom "") + errors (atom "") + writer (fn [text] + (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))}))] + {:stdout (writer output) :stderr (writer errors) :output output :errors errors})) + +(defn- resolved + [command options] + (request/apply-defaults {:command command :console {:color :never} :options options})) + +(defn- listeners + [] + [(.listenerCount js/process "SIGINT") (.listenerCount js/process "SIGTERM")]) + + +(t/deftest ^:async sync-and-async-handlers-publish-once-and-release-listeners + (doseq [async [false true]] + (let [local (capture) + before (listeners) + request (cli/parse-resolve ["compare" "--help" "--no-color"]) + handler (fn [_ {:keys [topic]}] + (log/info "Preparing help") + {:exit-code 0 :errors [] :log-path nil :report (cli/get-help topic)}) + handler (if async + (^:async fn [context options] + (await (timers/setTimeout 1)) + (handler context options)) + handler)] + (t/is (= 0 (await (execution/execute request {"help" handler} + (assoc local :get-root #(throw (js/Error. "root unavailable"))))))) + (t/is (= (str (cli/get-help "compare") "\n") @(:output local))) + (t/is (= "Preparing help\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest ^:async handlers-use-local-root-and-services-with-complete-options + (let [local (capture) + file (path/join @temporary-root "input.txt")] + (fs/writeFileSync file "saved input" "utf8") + (t/is (= 0 + (await + (execution/execute + (cli/parse-resolve ["run" "--build" "ready" "--warmups" "0"]) + {"run" (fn [{:keys [get-root services]} {:keys [warmups build]}] + {:exit-code 0 :errors [] :log-path nil + :report (str (:id build) ":" warmups ":" + ((:read-text services) (path/join (get-root) "input.txt")))})} + (assoc local :get-root (constantly @temporary-root) + :services {:read-text #(fs/readFileSync % "utf8")}))))) + (t/is (= "ready:0:saved input\n" @(:output local))))) + +(t/deftest ^:async malformed-and-unsupported-requests-fail-before-output-allocation + (let [valid (resolved "run" {:build {:kind :existing :id "ready"}})] + (doseq [[received field handlers] + [[(dissoc valid :console) "console" nil] + [(update valid :options dissoc :warmups) "warmups" nil] + [(assoc (update valid :options dissoc :warmups) + :console {:level :warn :color :always}) "warmups" nil] + [(assoc-in valid [:options :extra] true) "extra" nil] + [(assoc-in valid [:console :color] :invalid) "color" nil] + [(assoc-in valid [:console :stdout] #js {:write #(throw (js/Error. "untrusted stream"))}) + "stdout" nil] + [valid "unsupported-command" {}]]] + (let [local (capture) + output (path/join @temporary-root (str (random-uuid))) + before (listeners) + handler (fn [_ _] + (fs/mkdirSync output) + {:exit-code 0 :errors [] :log-path nil})] + (t/is (= 1 (await (execution/execute received (or handlers {"run" handler}) + (assoc local :get-root #(fs/mkdirSync output)))))) + (t/is (str/includes? @(:errors local) field)) + (t/is (not (str/includes? @(:errors local) "\u001b["))) + (t/is (not (fs/existsSync output))) + (t/is (= before (listeners))))))) + +(t/deftest ^:async received-command-policies-reject-before-output-allocation + (doseq [[command options message] + [["build" {:features ["stats"]} "diagnostic-features"] + ["run" {:build {:kind :prepare :features ["profile"]}} "diagnostic-features"] + ["ab" {:features ["profile-raf"] :dry-run true} "diagnostic-features"] + ["ab" {:features ["cache"]} "confirmation-required"] + ["ab" {:features ["cache"] :filter " "} "confirmation-required"] + ["ab" {:features ["a" "b"] :max-features 1 :dry-run true} "too-many-features"]]] + (let [local (capture) + output (path/join @temporary-root (str (random-uuid))) + request (request/check (resolved command options))] + (t/is (= 1 (await (execution/execute + request + {command (fn [_ _] + (fs/mkdirSync output) + {:exit-code 0 :errors [] :log-path nil})} + (assoc local :get-root #(fs/mkdirSync output)))))) + (t/is (str/includes? @(:errors local) message)) + (t/is (not (fs/existsSync output)))))) + +(t/deftest ^:async gpu-policy-uses-checked-console-and-dry-run-ab-needs-no-gpu + (let [env (.-env js/process) + old (.-PENPOT_GPU env)] + (try + (js-delete env "PENPOT_GPU") + (doseq [[command options expected] + [["run" {:build {:kind :existing :id "ready"} :require-gpu true} 1] + ["ab" {:features ["cache"] :confirm true :require-gpu true} 1] + ["ab" {:features ["cache"] :dry-run true :require-gpu true} 0]]] + (let [local (capture) + request (assoc (resolved command options) :console {:level :warn :color :always})] + (t/is (= expected + (await (execution/execute + request {command (fn [_ _] {:exit-code 0 :errors [] :log-path nil + :report "dry plan"})} local)))) + (if (zero? expected) + (t/is (= "dry plan\n" @(:output local))) + (do + (t/is (str/includes? @(:errors local) "PENPOT_GPU=1")) + (t/is (str/includes? @(:errors local) "\u001b[")))))) + (finally + (if (nil? old) (js-delete env "PENPOT_GPU") (set! (.-PENPOT_GPU env) old)))))) + +(t/deftest ^:async thrown-and-rejected-failures-use-checked-console-and-release-listeners + (doseq [[handler expected] [[(fn [_ _] (throw (js/Error. "failed work"))) + "✖ failed work\n"] + [(^:async fn [_ _] + (await (timers/setTimeout 1)) + (throw (js/Error. "failed work"))) + "✖ failed work\n"] + [(^:async fn [_ _] + (await (timers/setTimeout 1)) + (throw (errors/fatal-error "failed work" {} (js/Error. "storage")))) + "✖ failed work\n fatal true\n"]]] + (let [local (capture) + before (listeners) + request (assoc (resolved "help" {:topic nil}) :console {:level :warn :color :never})] + (t/is (= 1 (await (execution/execute + request {"help" (fn [context options] + (log/info "hidden progress") + (handler context options))} local)))) + (t/is (= expected @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest ^:async final-publication-uses-the-outcomes-log-and-numeric-status + (let [local (capture) + file (path/join @temporary-root "command.log")] + (t/is (= 7 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 7 :errors [] :log-path file :report "saved report"})} + local)))) + (t/is (= "saved report\n" (fs/readFileSync file "utf8"))) + (t/is (= "saved report\n" @(:output local))) + (let [silent (capture)] + (t/is (= 0 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path nil})} silent)))) + (t/is (= "" @(:output silent)))))) + +(t/deftest ^:async final-errors-share-the-command-log-and-print-one-tail + (let [local (capture) + file (path/join @temporary-root "failures.log")] + (fs/writeFileSync file "saved evidence\n" "utf8") + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] + {:exit-code 1 :log-path file + :errors [(js/Error. "first failure") (js/Error. "second failure")]})} + local)))) + (t/is (= 2 (count (re-seq #"✖" @(:errors local))))) + (t/is (= 1 (count (re-seq #" tail\n" @(:errors local))))) + (t/is (str/includes? @(:errors local) "saved evidence")) + (t/is (str/includes? (fs/readFileSync file "utf8") "second failure")))) + +(t/deftest ^:async publication-failure-fails-the-command-and-preserves-interruption-status + (doseq [[code expected] [[0 1] [130 130] [143 143]]] + (let [local (capture) + written (atom []) + broken #js {:write (fn [text] + (swap! written conj text) + (throw (js/Error. "stdout closed")))}] + (t/is (= expected (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code code :errors [] :log-path nil :report "result"})} + (assoc local :stdout broken))))) + (t/is (= ["result\n"] @written)) + (t/is (= "✖ stdout closed\n" @(:errors local))))) + (let [local (capture)] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path @temporary-root :report "result"})} + local)))) + (t/is (str/includes? @(:errors local) "cannot append benchmark log")) + (t/is (str/includes? @(:errors local) "Could not append log")))) + +(t/deftest ^:async real-signals-retain-partial-evidence-and-fatal-errors + (doseq [[first-signal second-signal code] [["SIGINT" "SIGTERM" 130] ["SIGTERM" "SIGINT" 143]]] + (let [local (capture) + before (listeners) + file (path/join @temporary-root first-signal) + handler (fn [{:keys [signal]} _] + (js/Promise. + (fn [resolve reject] + (.addEventListener + signal "abort" + (fn [] + (-> (timers/setTimeout 1) + (.then (fn [] + (t/is (.emit js/process second-signal)) + (fs/writeFileSync file "partial evidence" "utf8") + (resolve {:exit-code 1 :log-path nil :report "partial report" + :errors [(js/Error. "aborted work") + (errors/fatal-error "fatal storage" {} (js/Error. "disk"))]}))) + (.catch reject))) + #js {:once true}) + (.kill js/process (.-pid js/process) first-signal))))] + (t/is (= code (await (within (execution/execute (resolved "help" {:topic nil}) + {"help" handler} local))))) + (t/is (= "partial evidence" (fs/readFileSync file "utf8"))) + (t/is (= "partial report\n" @(:output local))) + (t/is (= "✖ fatal storage\n fatal true\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest failure-adapters-preserve-usage-validation-and-decoder-diagnostics + (doseq [[get-cause message] + [[#(cli/parse-resolve ["purge"]) "Usage: runner purge"] + [#(request/check (update (resolved "run" {:build {:kind :existing :id "ready"}}) + :options dissoc :warmups)) "warmups"] + [#(transit/decode-str "invalid transit") nil] + [#((createRequire (path/join @temporary-root "entry.cjs")) + (path/join @temporary-root "missing.cjs")) "missing.cjs"]]] + (let [local (capture) + cause (try + (get-cause) + (t/is false "expected a boundary failure") + (catch :default cause cause))] + (t/is (= 1 (execution/fail cause {} local))) + (t/is (str/includes? @(:errors local) (or message (errors/message-of cause)))) + (t/is (= 1 (count (re-seq #"✖" @(:errors local)))))))) + +(t/deftest ^:async failure-text-escapes-controls-and-survives-a-throwing-printer + (let [local (capture) + hostile (reify IPrintWithWriter + (-pr-writer [_ _ _] (throw (js/Error. "cannot print path")))) + failure (ex-info (str "work failed\n\u001b" (apply str (repeat 2000 "x"))) + {:path hostile :reason "reason\u0007\u007f\u0085" + :mismatches [{:field [:metric (keyword "metric" "unsafe\u001b")]}]})] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] (throw failure))} local)))) + (t/is (str/includes? @(:errors local) "work failed")) + (t/is (str/includes? @(:errors local) "unrenderable value")) + (t/is (str/includes? @(:errors local) "\\u001b")) + (t/is (str/includes? @(:errors local) "\\u0007")) + (t/is (every? #(<= (count %) 1100) (str/split-lines @(:errors local)))) + (t/is (every? #(let [code (.charCodeAt % 0)] + (or (= code 10) (and (>= code 32) (or (< code 127) (> code 159))))) + (seq @(:errors local)))))) + +(t/deftest ^:async error-reporting-failure-uses-a-bounded-fallback + (doseq [[code expected] [[1 1] [130 130]]] + (let [local (capture) + fallback (atom []) + broken #js {:write (fn [text] + (if (str/starts-with? text "Could not report") + (swap! fallback conj text) + (throw (js/Error. (str "\u001b" (apply str (repeat 1100 "😀")))))))}] + (t/is (= expected (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code code :log-path nil :errors [(js/Error. "work failed")]})} + (assoc local :stderr broken))))) + (t/is (= 1 (count @fallback))) + (t/is (< (count (first @fallback)) 1100)) + (t/is (str/includes? (first @fallback) "\\u001b")) + (t/is (not (re-find #"[\uD800-\uDBFF](?![\uDC00-\uDFFF])" (first @fallback)))))) + (let [broken #js {:write #(throw (js/Error. "closed"))}] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path nil :report "result"})} + {:stdout broken :stderr broken})))))) diff --git a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs new file mode 100644 index 0000000000..67bb5aff2d --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs @@ -0,0 +1,1363 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.feature-comparison-test + (:require + ["fs" :as fs] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.feature-comparison :as feature-comparison] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-feature-plan-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- frontend-root [] (path/join @temporary-root "frontend")) +(defn- renderer-root [] (path/join @temporary-root "render-wasm")) + +(defn- write-cargo-fixture + [] + (fs/mkdirSync (frontend-root)) + (fs/mkdirSync (renderer-root)) + (fs/writeFileSync (path/join (renderer-root) "Cargo.toml") + (str "[package]\nname = 'feature-plan-fixture'\nversion = '0.1.0'\nedition = '2021'\n" + "[lib]\npath = 'lib.rs'\n[features]\ndefault = []\na = []\nb = ['a']\n")) + (fs/writeFileSync (path/join (renderer-root) "lib.rs") "") + (fs/writeFileSync (path/join (renderer-root) "Cargo.lock") + "version = 4\n[[package]]\nname = 'feature-plan-fixture'\nversion = '0.1.0'\n")) + +(defn- planning-options + [] + (:options (request/apply-defaults + {:command "ab" + :options {:features ["b" "a"] :filter "rects/load" :seed 42 :dry-run true + :warmups 0 :repetitions 1 :timeout-ms 2000 + :build-timeout-ms 10000 :startup-timeout-ms 3000}}))) + +(defn- metadata-output + [definitions] + {:stdout (js/JSON.stringify + (clj->js {"packages" [{"manifest_path" (path/join (frontend-root) "Cargo.toml") + "features" {"default" ["wrong-package"]}} + {"manifest_path" (path/join (renderer-root) "Cargo.toml") + "features" definitions}]}))}) + +(defn- error-message + [promise] + (.then promise (fn [_] nil) (fn [cause] (ex-message cause)))) + +(t/deftest one-feature-has-an-explicit-off-and-on-selection + (t/is (= {:features ["a"] + :configurations [{:slug "0" :selection {:default-features? false :features []}} + {:slug "1" :selection {:default-features? false :features ["a"]}}] + :edges [{:from "0" :to "1" :feature "a"}]} + (feature-plan/plan ["a"] {:filter "rects/load"})))) + +(t/deftest two-feature-order-and-all-edges-are-msb-first + (t/is (= {:features ["a" "b"] + :configurations [{:slug "00" :selection {:default-features? false :features []}} + {:slug "01" :selection {:default-features? false :features ["b"]}} + {:slug "11" :selection {:default-features? false :features ["a" "b"]}} + {:slug "10" :selection {:default-features? false :features ["a"]}}] + :edges [{:from "00" :to "10" :feature "a"} + {:from "00" :to "01" :feature "b"} + {:from "01" :to "11" :feature "a"} + {:from "10" :to "11" :feature "b"}]} + (feature-plan/plan ["b" "a" "a"] {:filter "rects/load"})))) + +(t/deftest three-feature-order-and-complete-graph + (let [{:keys [configurations edges]} (feature-plan/plan ["c" "a" "b"] {:filter "rects/load"}) + by-slug (into {} (map (juxt :slug #(set (get-in % [:selection :features])))) configurations)] + (t/is (= ["000" "001" "011" "010" "110" "111" "101" "100"] (mapv :slug configurations))) + (t/is (= 12 (count (set edges)))) + (doseq [{:keys [from to feature]} edges] + (t/is (not (contains? (by-slug from) feature))) + (t/is (= (conj (by-slug from) feature) (by-slug to)))))) + +(t/deftest upper-limit-still-emits-every-edge-once + (let [{:keys [configurations edges]} + (feature-plan/plan ["a" "b" "c" "d" "e" "f"] {:max-features 6 :confirm true})] + (t/is (= 64 (count (set configurations)))) + (t/is (= 192 (count (set edges)))) + (t/is (every? false? (map #(get-in % [:selection :default-features?]) configurations))))) + +(t/deftest invalid-inputs-fail-before-planning + (doseq [features [nil '("a") "a" [nil] [1] [""] ["a b"] ["a/b"] ["a.b"] [" a"]]] + (t/is (thrown? js/Error (feature-plan/plan features {:dry-run true})) (pr-str features))) + (t/is (thrown-with-msg? js/Error #"at least one" (feature-plan/plan [] {:dry-run true}))) + (doseq [limit [nil 0 7 -1 1.5 "3" js/NaN js/Infinity]] + (t/is (thrown-with-msg? js/Error #"integer from 1 to 6" + (feature-plan/plan ["a"] {:max-features limit :dry-run true})))) + (doseq [opts [{:confirm nil} {:confirm 1} {:dry-run nil} {:dry-run "true"} {:filter 1}]] + (t/is (thrown-with-msg? js/Error #"invalid feature planning options" + (feature-plan/plan ["a"] opts)))) + (t/is (thrown-with-msg? js/Error #"exceed max-features" + (feature-plan/plan ["a" "b" "c" "d"] {:dry-run true})))) + +(t/deftest confirmation-applies-to-execution-only + (doseq [filter [nil "" " "]] + (t/is (thrown-with-msg? js/Error #"requires --confirm" (feature-plan/plan ["a"] {:filter filter}))) + (t/is (= 2 (count (:configurations (feature-plan/plan ["a"] {:filter filter :dry-run true})))))) + (t/is (= 8 (count (:configurations (feature-plan/plan ["a" "b" "c"] {:filter "rects"}))))) + (t/is (thrown-with-msg? js/Error #"requires --confirm" + (feature-plan/plan ["a" "b" "c" "d"] {:max-features 4 :filter "rects"}))) + (doseq [opts [{:confirm true} {:dry-run true}]] + (t/is (= 16 (count (:configurations + (feature-plan/plan ["a" "b" "c" "d"] (assoc opts :max-features 4)))))))) + +(t/deftest manifest-validation-leaves-activation-to-cargo + (t/is (= ["a" "b"] (feature-plan/check-manifest ["b" "a"] + {"default" [] "a" [] "b" ["a" "dep:library"]}))) + (t/is (= ["a"] (feature-plan/check-manifest ["a"] {"a" []}))) + (t/is (thrown-with-msg? js/Error #"unknown requested" + (feature-plan/check-manifest ["missing"] {"default" [] "a" []}))) + (t/is (thrown-with-msg? js/Error #"empty declared Cargo defaults" + (feature-plan/check-manifest ["a"] {"default" ["a"] "a" []}))) + (doseq [definitions [nil [] {"a" "b"} {:a []} {"a" [1]}]] + (t/is (thrown-with-msg? js/Error #"invalid Cargo feature definitions" + (feature-plan/check-manifest ["a"] definitions)))) + (doseq [feature ["stats" "profile" "profile-macros" "profile-raf"]] + (t/is (thrown-with-msg? js/Error #"diagnostic Cargo features" + (feature-plan/plan [feature] {:dry-run true}))) + (t/is (thrown-with-msg? js/Error #"diagnostic Cargo features" + (feature-plan/check-manifest [feature] {"default" [] feature []})))) + ;; A declared alias is allowed here. Actual Cargo-reported diagnostics are + ;; rejected later by build validation, not by a second activation resolver. + (t/is (= ["alias"] (feature-plan/check-manifest ["alias"] {"default" [] "alias" ["stats"]})))) + +(t/deftest ^:async real-offline-metadata-and-collection-create-no-output + (write-cargo-fixture) + (let [output (path/join @temporary-root "new" "comparison") + opts (assoc (planning-options) :output-dir output) + before (vec (fs/readdirSync (renderer-root))) + ready (await (feature-comparison/prepare (frontend-root) opts nil))] + (t/is (= ["a" "b"] (get-in ready [:plan :features]))) + (t/is (= (cases/collect-cases {:master-seed 42 :filter "rects/load"}) (:cases ready))) + (t/is (= (assoc opts :features ["a" "b"]) + (:options ready))) + (t/is (= output (:output-dir ready))) + (t/is (not (fs/existsSync (path/join @temporary-root "new")))) + (t/is (= before (vec (fs/readdirSync (renderer-root))))) + (t/is (empty? (fs/readdirSync (frontend-root)))) + (let [text (feature-comparison/format-plan ready)] + (doseq [part ["a, b" "00" "01" "11" "10" "00 -> 10: a" "Cases: 1" + "0 warmups, 1 measured" "case 2000, build 10000, startup 3000"]] + (t/is (str/includes? text part)))))) + +(t/deftest ^:async metadata-selects-the-package-by-manifest-not-array-position + (write-cargo-fixture) + (fs/writeFileSync (path/join (frontend-root) "Cargo.toml") "decoy") + (await + (mock/with-mocks* + {process/run (fn [request] + (t/is (= "cargo" (:command request))) + (t/is (= ["metadata" "--offline" "--locked" "--no-deps" "--format-version" "1"] + (:args request))) + (t/is (= (renderer-root) (:cwd request))) + (t/is (not (contains? request :env))) + (js/Promise.resolve (metadata-output {"default" [] "a" [] "b" ["a"]})))} + (let [ready (await (feature-comparison/prepare (frontend-root) (planning-options) nil))] + (t/is (= ["a" "b"] (get-in ready [:plan :features]))))))) + +(t/deftest ^:async collected-custom-case-data-and-order-survive-planning + (write-cargo-fixture) + (decl/register-scene! {:id :feature-plan-fixture :ns "feature-comparison-test" + :description "Planning data fixture" + :params-schema [:map [:count :int]] + :build (fn [_] (throw (js/Error. "planning must not construct a scene")))}) + (doseq [id [:feature-plan-fixture/second :feature-plan-fixture/first]] + (decl/register-case! {:id id :scene :feature-plan-fixture :ns "feature-comparison-test" + :params {:count 17} :view {:scale 2 :x 10 :y -5} :context :reuse + :preparation {:reset :outside-timing :data [1 2]} + :run! (fn [_] (throw (js/Error. "planning must not run an operation")))})) + (try + (let [ready (await (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :filter "feature-plan-fixture/") nil)) + data (:cases ready)] + (t/is (= [:feature-plan-fixture/second :feature-plan-fixture/first] (mapv :id data))) + (t/is (= (cases/collect-cases {:master-seed 42 :filter "feature-plan-fixture/"}) data)) + (t/is (= {:params {:count 17} :view {:scale 2 :x 10 :y -5} :context :reuse + :preparation {:reset :outside-timing :data [1 2]}} + (select-keys (first data) [:params :view :context :preparation])))) + (finally + (decl/unregister-case! :feature-plan-fixture/second) + (decl/unregister-case! :feature-plan-fixture/first) + (decl/unregister-scene! :feature-plan-fixture)))) + +(t/deftest ^:async planning-propagates-manifest-and-collector-errors + (write-cargo-fixture) + (fs/writeFileSync (path/join (frontend-root) "Cargo.toml") "decoy") + (doseq [[definitions message] [[{"default" [] "a" []} "unknown requested Cargo features"] + [{"default" ["a"] "a" [] "b" []} "ab requires empty declared Cargo defaults"]]] + (await (mock/with-mocks* + {process/run (fn [_] (js/Promise.resolve (metadata-output definitions)))} + (t/is (= message (await (error-message + (feature-comparison/prepare (frontend-root) (planning-options) nil)))))))) + (t/is (= "no benchmark cases match the selection" + (await (error-message (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :filter "missing-case") nil)))))) + +(t/deftest ^:async invalid-input-and-existing-output-fail-before-cargo + (write-cargo-fixture) + (await + (mock/with-mocks* + {process/run (fn [_] (throw (js/Error. "unexpected Cargo invocation")))} + (t/is (= "output directory already exists" + (await (error-message (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :output-dir (frontend-root)) nil))))) + (t/is (= "ab requires at least one requested feature" + (await (error-message (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :features []) nil)))))))) + +(t/deftest ^:async cancellation-reaches-the-process-boundary + (write-cargo-fixture) + (let [controller (js/AbortController.)] + (.abort controller "SIGINT") + (t/is (str/includes? (await (error-message + (feature-comparison/prepare (frontend-root) (planning-options) (.-signal controller)))) + "cancelled")))) + +(defn- planned-record + [] + (let [plan (feature-plan/plan ["a" "b"] {:dry-run true})] + {:type :feature-comparison :comparison-id "ab-id" + :started-at "start" :updated-at "start" :status :planned + :options (planning-options) + :permissions {:allow-build-differences false :allow-scene-differences true :allow-all-differences false} + :cases (cases/collect-cases {:master-seed 42 :filter "rects/load"}) + :features (:features plan) + :configurations (mapv #(assoc % :status :planned) (:configurations plan)) + :edges (mapv #(assoc % :status :planned) (:edges plan)) + :warnings ["Timing values do not determine success."]})) + +(defn- completed-configuration + [configuration] + (assoc configuration + :status :completed + :effective-features ["transitive"] + :build {:id "build-id" :manifest-path "/build.transit" :log-path "/build.log"} + :run {:run-id "run-id" :record-path "/run.transit" :log-path "/run.log" + :exit-code 0})) + +(defn- completed-record + [] + (let [record (planned-record)] + (assoc record + :status :completed :updated-at "end" :ended-at "end" + :configurations (mapv completed-configuration (:configurations record)) + :edges (mapv #(assoc % :status :completed :path "/comparison.transit") + (:edges record))))) + +(defn- terminal-record + [status] + (let [record (planned-record)] + (assoc record + :status status :updated-at "end" :ended-at "end" + :configurations + (mapv (fn [[index configuration]] + (if (zero? index) + (assoc (completed-configuration configuration) + :status :failed :failure {:phase "run" :message "run interrupted"} + :reason "SIGINT") + (assoc configuration :status :unattempted + :reason "stopped after failure"))) + (map-indexed vector (:configurations record))) + :edges + (mapv (fn [[index edge]] + (if (zero? index) + (assoc edge :status :failed :path "/comparison.transit" + :failure {:phase "comparison" :message "partial"}) + (assoc edge :status :unattempted + :reason "stopped after failure"))) + (map-indexed vector (:edges record)))))) + +(defn- running-record + [] + (-> (planned-record) + (assoc :status :running) + (assoc-in [:configurations 0 :status] :building) + (assoc-in [:configurations 0 :build] + {:id "build-id" :manifest-path "/build.transit" :log-path "/build.log"}))) + +(t/deftest feature-comparison-record-round-trips-plan-and-partial-evidence + (let [record (terminal-record :interrupted)] + (t/is (= (planned-record) (result/decode (result/encode (planned-record))))) + (t/is (= record (result/decode (result/encode record)))))) + +(t/deftest feature-comparison-progress-must-match-saved-evidence + (doseq [record [(planned-record) (running-record) (completed-record) + (terminal-record :failed) (terminal-record :interrupted)]] + (t/is (= record (result/check-record record)))) + (let [completed (completed-record) + failed (terminal-record :failed) + running (running-record)] + (doseq [[record message] + [[(assoc (planned-record) :status :completed) + #"terminal feature comparison needs an end time"] + [(assoc-in completed [:configurations 0 :status] :planned) + #"still has active configurations"] + [(assoc-in completed [:edges 0 :status] :planned) #"still has active edges"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :build) configurations))) + #"configuration needs a build"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :effective-features) configurations))) + #"configuration needs effective features"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :run) configurations))) + #"configuration needs a run"] + [(update completed :configurations + (fn [configurations] + (mapv #(update % :run dissoc :exit-code) configurations))) + #"run exit code"] + [(assoc-in completed [:configurations 0 :run :exit-code] 1) + #"successful run"] + [(update-in completed [:edges 0] dissoc :path) + #"edge needs a report path"] + [(update-in failed [:configurations 0] dissoc :failure :reason) + #"configuration needs a failure or reason"] + [(update-in failed [:configurations 1] dissoc :reason) + #"configuration needs a reason"] + [(update-in failed [:edges 0] dissoc :failure :reason) + #"edge needs a failure"] + [(update-in failed [:edges 1] dissoc :reason) + #"edge needs a reason"] + [(assoc running :ended-at "end") #"unfinished feature comparison has an end time"] + [(assoc-in (planned-record) [:configurations 0 :status] :building) + #"started configurations"] + [(assoc-in (planned-record) [:edges 0 :status] :completed) + #"started edges"] + [(assoc (planned-record) :features ["b" "a"]) + #"features disagree with normalized selection"] + [(update (planned-record) :configurations + #(assoc % 1 (first %))) + #"configurations disagree with planned features"] + [(update (planned-record) :configurations pop) + #"configurations disagree with planned features"] + [(assoc-in (planned-record) [:edges 0 :to] "11") + #"edges disagree with planned configurations"] + [(update (planned-record) :cases conj (first (:cases (planned-record)))) + #"duplicate feature comparison case"]]] + (t/is (thrown-with-msg? js/Error message (result/check-record record)) + (pr-str message))))) + +(t/deftest feature-comparison-current-state-shapes-are-checked + (doseq [[location value] [[[:comparison-id] 1] [[:updated-at] nil] [[:status] :ready] + [[:ended-at] nil] [[:features] [:a]] [[:cases] [{}]] + [[:permissions :allow-scene-differences] nil] + [[:configurations 0 :status] :interrupted] + [[:configurations 0 :selection :features] "a"] + [[:configurations 0 :selection :default-features?] nil] + [[:configurations 0 :build] {:id "id"}] + [[:configurations 0 :run] {:run-id "id" :record-path "p" :log-path "l" :exit-code "0"}] + [[:configurations 0 :failure] "error"] + [[:configurations 0 :effective-features] [:a]] + [[:edges 0 :status] :running] [[:edges 0 :path] 1] [[:warnings] [nil]]]] + (t/is (thrown-with-msg? js/Error #"invalid schema:feature-comparison" + (result/check-record (assoc-in (planned-record) location value))) + (pr-str location)))) + +(defn- create-frontend-root + [] + (fs/mkdirSync (frontend-root) #js {:recursive true})) + +(defn- comparison-output-dir + [] + (path/join @temporary-root "output" "comparison")) + +(defn- fixture-case + [] + {:id :fixture/comparison-case}) + +(defn- comparison-prepared + ([features] (comparison-prepared features {})) + ([features overrides] + (let [plan (feature-plan/plan features {:dry-run true}) + output (or (:output-dir overrides) (comparison-output-dir)) + opts (merge {:default-features false + :filter "fixture/" + :seed 42 + :warmups 0 + :repetitions 1 + :timeout-ms 1000 + :build-timeout-ms 5000 + :startup-timeout-ms 1000 + :allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + overrides + {:features (:features plan) + :output-dir output})] + {:plan plan + :cases [(fixture-case)] + :output-dir output + :options opts}))) + +(defn- fake-build + [selection extra-features] + {:id (str (random-uuid)) + :manifest-path (path/join @temporary-root "builds" "build.transit") + :log-path (path/join @temporary-root "builds" "build.log") + :manifest {:functional {:mode :release + :features (vec (concat (:features selection) extra-features))} + :provenance {:feature-selection selection}}}) + +(defn- build-service + [{:keys [extra-features fail-on inspect build-fn] + :or {extra-features (constantly []) + fail-on (constantly false) + build-fn fake-build}}] + (fn [request] + (when inspect (inspect request)) + (let [selection (get request :selection) + build (build-fn selection (extra-features selection))] + (when-let [on-created (:on-created request)] + (on-created build)) + (if (fail-on selection) + (js/Promise.reject (js/Error. "fixture build failure")) + (js/Promise.resolve build))))) + +(defn- metric-entry + [value] + {:command :fixture/work :metrics {:command/elapsed [value]}}) + +(defn- fingerprint + [character] + {:algorithm :sha-256 + :digest (apply str (repeat 64 character))}) + +(defn- preparation + [case-id digest] + (let [id (str (random-uuid))] + {:preparation-id id :case-id case-id :warmup? false + :fingerprint (fingerprint digest) + :effective-graphics {:backend :fixture} + :diagnostics {} + :phases [{:phase :preparation :preparation-id id :outcome :completed :events []}]})) + +(defn- fake-run-record + [selected-build run-home cases options {:keys [kind digest metric-value] + :or {kind :completed digest "a" metric-value 10}}] + (let [run (result/create-run + {:run-id (:run-id run-home) + :started-at (.toISOString (js/Date.)) + :plan {:cases cases + :warmups 0 + :repetitions 1 + :timeout-policy {:case-ms (:timeout-ms options)}} + :metadata {:git {:sha nil :dirty nil} + :build {:functional (get-in selected-build [:manifest :functional]) + :provenance (get-in selected-build [:manifest :provenance])} + :environment {} + :scored? true}}) + failed? (= kind :failed) + run (if (= kind :zero-attempts) + run + (reduce + (fn [run case] + (let [case-id (:id case)] + (if failed? + (result/record-attempt + run + {:case-id case-id + :preparation-id nil + :warmup? false + :outcome :failed + :phases [] + :failure {:phase "main" :message "fixture failure"}}) + (let [prep (preparation case-id digest)] + (-> run + (result/record-preparation prep) + (result/record-attempt + {:case-id case-id + :preparation-id (:preparation-id prep) + :warmup? false + :outcome :completed + :phases [{:phase :main :preparation-id (:preparation-id prep) + :outcome :completed :events [(metric-entry metric-value)]}]})))))) + run + cases)) + termination (case kind + :failed {:reason :failed :failure {:phase "main" :message "fixture failure"}} + :zero-attempts {:reason :incomplete} + :interrupted {:reason :interrupted :signal "SIGINT"} + {:reason :completed}) + run (if (= :completed (:reason termination)) + (reduce #(result/record-case-outcome %1 (:id %2) {:outcome :completed}) run cases) + run)] + (result/finish-run run (.toISOString (js/Date.)) termination))) + +(defn- execute-service + [spec-or-fn] + (fn [_ run-home selected-build cases options _signal] + (let [spec (if (fn? spec-or-fn) (spec-or-fn selected-build) spec-or-fn) + run (fake-run-record selected-build run-home cases options spec) + exit-code (case (:kind spec) + :failed 1 + :zero-attempts 1 + :interrupted 130 + 0)] + (files/atomic-write (:record-path run-home) + (result/encode (result/check-record run))) + (js/Promise.resolve {:run run :home run-home :exit-code exit-code :errors []})))) + +(defn- comparison-services + [{:keys [prepare-build collect-catalog execute-run]}] + {:prepare-build prepare-build + :collect-catalog (or collect-catalog (fn [_ _] [(fixture-case)])) + :execute-run execute-run}) + +(defn- saved-comparison + [outcome] + (result/decode (fs/readFileSync (get-in outcome [:home :record-path]) "utf8"))) + +(t/deftest ^:async invalid-build-selection-or-features-stops-later-configurations + (create-frontend-root) + (doseq [[field value] [[[:provenance :feature-selection] {:default-features? false :features ["other"]}] + [[:functional :features] #{"a"}] + [[:functional :features] [:invalid]]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:build-fn (fn [selection extra] + (assoc-in (fake-build selection extra) + (into [:manifest] field) value))}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status record))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= "build" (get-in record [:configurations 0 :failure :phase]))) + (t/is (str/includes? (get-in record [:configurations 0 :failure :message]) + "disagrees with the planned selection"))))) + +(t/deftest ^:async operational-failures-save-the-stage-and-stop-later-configurations + (create-frontend-root) + (doseq [[boundary phase message] [[:catalog "catalog" "catalog rejected"] + [:run-home "run-home" "already exists"] + [:run "run" "run rejected"] + [:missing-run "run" "run executor returned no record"]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + base (comparison-services {:prepare-build (build-service {}) :execute-run (execute-service {})}) + services (case boundary + :catalog (assoc base :collect-catalog (fn [_ _] (throw (js/Error. "catalog rejected")))) + :run-home (assoc base :prepare-build + (build-service {:inspect (fn [_] + (fs/mkdirSync (path/join (:output-dir prepared) "configs" "0") + #js {:recursive true}))})) + :run (assoc base :execute-run (fn [& _] (js/Promise.reject (js/Error. "run rejected")))) + :missing-run (assoc base :execute-run (fn [& _] (js/Promise.resolve {:exit-code 1})))) + outcome (await (feature-comparison/execute (frontend-root) prepared nil {:services services})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status record))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= phase (get-in record [:configurations 0 :failure :phase]))) + (t/is (str/includes? (get-in record [:configurations 0 :failure :message]) message))))) + +(t/deftest ^:async run-failure-saves-partial-measurements-and-nested-causes + (create-frontend-root) + (let [partial {:slices [{:duration-ms 3}]} + cause (ex-info "run rejected" {:partial partial} + (ex-info "render failed" {:fn "_render" :code 17})) + prepared (comparison-prepared ["a"] {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (fn [& _] (js/Promise.reject cause))})})) + failure (get-in (saved-comparison outcome) [:configurations 0 :failure])] + (t/is (= {:phase "run" + :message "run rejected" + :partial partial} + (select-keys failure [:phase :message :partial]))) + (t/is (some #(= {:message "render failed" :fn "_render" :code 17} %) + (:cause failure))))) + +(t/deftest ^:async unavailable-or-mismatched-run-evidence-stops-later-configurations + (create-frontend-root) + (doseq [[kind message] [[:missing "run record is missing"] + [:mismatch "saved run identity disagrees with execution result"]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + execute (execute-service {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (^:async fn [root home build cases options signal] + (let [response (await (execute root home build cases options signal))] + (if (= kind :missing) + (fs/unlinkSync (:record-path home)) + (files/atomic-write (:record-path home) + (result/encode (assoc (:run response) :run-id (str (random-uuid)))))) + response))})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= "run-evidence" (get-in record [:configurations 0 :failure :phase]))) + (t/is (str/includes? (get-in record [:configurations 0 :failure :message]) message))))) + +(t/deftest ^:async returned-storage-failure-wins-over-readable-evidence-and-executor-errors + (create-frontend-root) + (doseq [[signal code] [[nil 1] ["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + execute (execute-service {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (^:async fn [root home build cases options signal-object] + (let [response (await (execute root home build cases options signal-object))] + (when signal (.abort controller signal)) + (assoc response :exit-code 1 + :errors [(js/Error. "executor failure") + (errors/fatal-error "recovered write failure" {} nil)])))})})) + record (saved-comparison outcome) + run (result/decode (fs/readFileSync (get-in record [:configurations 0 :run :record-path]) "utf8"))] + (t/is (= code (:exit-code outcome))) + (t/is (= (if signal :interrupted :failed) (:status record))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= "run-evidence" (get-in record [:configurations 0 :failure :phase]))) + (t/is (= "recovered write failure" (get-in record [:configurations 0 :failure :message]))) + (t/is (= "recovered write failure" (ex-message (first (:errors outcome)))))))) + +(t/deftest ^:async pre-aborted-comparison-preserves-all-work-as-unattempted + (create-frontend-root) + (doseq [[reason signal code] [["SIGINT" "SIGINT" 130] + ["SIGTERM" "SIGTERM" 143] + [:SIGTERM "SIGTERM" 143] + [(ex-info "stopped" {:signal "SIGTERM"}) "SIGTERM" 143] + [(js/Error. "stopped") "SIGINT" 130]]] + (let [controller (js/AbortController.) + _ (.abort controller reason) + prepared (comparison-prepared ["a" "b"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + outcome (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services {:prepare-build (build-service {}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= code (:exit-code outcome))) + (t/is (= :interrupted (:status record))) + (t/is (every? #(= :unattempted (:status %)) (concat (:configurations record) (:edges record)))) + (t/is (every? #(= (str "interrupted by " signal) (:reason %)) + (concat (:configurations record) (:edges record))))))) + +(t/deftest ^:async interrupted-build-and-catalog-preserve-stage-failures + (create-frontend-root) + (doseq [boundary [:build :catalog] + rejected [false true] + [signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + build (build-service {}) + stop (fn [] + (.abort controller signal) + (when rejected (throw (js/Error. "interrupted work")))) + services (comparison-services + {:prepare-build (if (= boundary :build) + (^:async fn [request] + (let [selected (await (build request))] + (stop) + selected)) + build) + :collect-catalog (fn [_ _] + (when (= boundary :catalog) (stop)) + (:cases prepared)) + :execute-run (execute-service {})}) + outcome (await (feature-comparison/execute (frontend-root) prepared (.-signal controller) + {:services services})) + record (saved-comparison outcome)] + (t/is (= code (:exit-code outcome))) + (t/is (= :interrupted (:status record))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= signal (get-in record [:configurations 0 :reason]))) + (t/is (= (if rejected [(name boundary)] []) + (mapv :phase (keep :failure (:configurations record))))) + (t/is (every? #(= :unattempted (:status %)) (:edges record)))))) + +(t/deftest ^:async failed-edge-evidence-save-retains-the-analysis-and-persistence-errors + (create-frontend-root) + (doseq [[kind point] [[nil :record] [:zero-attempts :record] [:zero-attempts :directory]]] + (let [prepared (comparison-prepared ["a" "b"] + {:output-dir (path/join @temporary-root (str "edge-save-" (or kind "completed") "-" (name point)))}) + write feature-comparison/write-record] + (await + (mock/with-mocks* + {feature-comparison/write-record + (fn [home record] + (if (= :failed (get-in record [:edges 0 :status])) + (throw (js/Error. "edge evidence unavailable")) + (write home record)))} + (let [response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (if (= :directory point) + (fs/writeFileSync (path/join (:output-dir prepared) "comparisons") "blocked") + (fs/mkdirSync + (path/join (:output-dir prepared) "comparisons" "00__10.transit") + #js {:recursive true})))}) + :execute-run (execute-service {:kind kind})})})) + evidence (errors/evidence :comparison (first (:errors response))) + entries (:errors evidence)] + (t/is (= 1 (:exit-code response))) + (if kind + (do + (t/is (= "configuration run made no attempts" (:message (first entries)))) + (t/is (= 1 (count (filter #(= "configuration run made no attempts" (:message %)) entries))))) + (t/is (some #(str/includes? (:message %) "EISDIR") entries))) + (t/is (some #(some (fn [cause] + (if (= :directory point) + (str/includes? (:message cause) "EEXIST") + (= "edge evidence unavailable" (:message cause)))) (:cause %)) entries)) + (t/is (= (:comparison response) (saved-comparison response))))))))) + +(t/deftest ^:async comparison-save-failures-retain-progress-errors-and-signal-codes + (create-frontend-root) + (doseq [[signal point] [[nil :terminal] ["SIGINT" :terminal] ["SIGTERM" :terminal] + ["SIGINT" :configuration] ["SIGTERM" :edge] + ["SIGINT" :run] ["SIGTERM" :run]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str "finish-" signal "-" (name point)))}) + controller (js/AbortController.) + write feature-comparison/write-record + pending (js/Error. "interrupted operation failed") + execute (execute-service (if (= :terminal point) {} {:kind :interrupted}))] + (when (and signal (= :terminal point)) (.abort controller signal)) + (await + (mock/with-mocks* + {feature-comparison/write-record + (fn [home record] + (if (case point + :terminal (contains? #{:completed :interrupted} (:status record)) + :run (some #(= :failed (:status %)) (:configurations record)) + :configuration (some #(= :unattempted (:status %)) (:configurations record)) + :edge (some #(= :unattempted (:status %)) (:edges record))) + (throw (js/Error. "comparison save failed")) + (write home record)))} + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (fn [& args] + (.then (apply execute args) + (fn [response] + (if (= :terminal point) + response + (do + (.abort controller signal) + (assoc response :errors [pending]))))))})})) + evidence (errors/evidence :comparison (first (:errors response)))] + (t/is (= (case signal "SIGINT" 130 "SIGTERM" 143 1) (:exit-code response))) + (t/is (= :running (get-in response [:comparison :status]))) + (t/is (= (:comparison response) (saved-comparison response))) + (t/is (some #(some (fn [cause] (= "comparison save failed" (:message cause))) (:cause %)) + (:errors evidence))) + (when (not= :terminal point) + (t/is (= 1 (count (filter #(= "interrupted operation failed" (:message %)) (:errors evidence)))))))))))) + + +(t/deftest ^:async unresolved-run-stops-the-feature-matrix + (create-frontend-root) + (let [prepared (comparison-prepared ["a"]) + execute (execute-service {}) + failure (errors/fatal-error "Command group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (fn [& args] + (.then (apply execute args) + #(assoc % :exit-code 1 :errors [failure])))})})) + saved (saved-comparison response)] + (t/is (= 1 (:exit-code response))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations saved)))) + (t/is (= [:unattempted] (mapv :status (:edges saved)))))) + + +(t/deftest ^:async unresolved-build-finishes-the-feature-matrix + (create-frontend-root) + (doseq [signal [nil "SIGTERM"]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str "unresolved-build-" signal))}) + controller (js/AbortController.) + prepare (build-service {}) + failure (errors/fatal-error "Compiler group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (fn [request] + (.then (prepare request) + (fn [_] + (when signal (.abort controller signal)) + (throw failure)))) + :execute-run (execute-service {})})})) + saved (saved-comparison response)] + (t/is (= (if signal 143 1) (:exit-code response))) + (t/is (= (if signal :interrupted :failed) (:status saved))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations saved)))) + (t/is (= [:unattempted] (mapv :status (:edges saved)))) + (t/is (true? (get-in saved [:configurations 0 :failure :unresolved-process]))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= (:comparison response) saved))))) + +(t/deftest ^:async unresolved-build-and-failed-comparison-save-retain-both-errors + (create-frontend-root) + (doseq [signal [nil "SIGTERM"]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str "unresolved-save-" signal))}) + controller (js/AbortController.) + write feature-comparison/write-record + prepare (build-service {}) + failure (errors/fatal-error "Compiler group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + save-error (js/Error. "comparison save failed")] + (await + (mock/with-mocks* + {feature-comparison/write-record + (fn [home record] + (if (= :failed (get-in record [:configurations 0 :status])) + (throw save-error) + (write home record)))} + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (fn [request] + (.then (prepare request) + (fn [_] + (when signal (.abort controller signal)) + (throw failure)))) + :execute-run (execute-service {})})})) + evidence (errors/evidence :comparison (first (:errors response)))] + (t/is (= (if signal 143 1) (:exit-code response))) + (t/is (= :running (get-in response [:comparison :status]))) + (t/is (= (:comparison response) (saved-comparison response))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= ["Compiler group may still be running" "cannot persist feature comparison"] + (mapv :message (:errors evidence)))) + (t/is (= 4242 (get-in evidence [:errors 0 :pgid]))) + (t/is (some #(= "comparison save failed" (:message %)) + (get-in evidence [:errors 1 :cause]))))))))) + +(t/deftest ^:async edge-report-storage-failure-preserves-failed-and-unattempted-edges + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"]) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (fs/mkdirSync + (path/join (:output-dir prepared) "comparisons" "00__10.transit") + #js {:recursive true}))}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status record))) + (t/is (every? #(= :completed (:status %)) (:configurations record))) + (t/is (= [:failed :unattempted :unattempted :unattempted] (mapv :status (:edges record)))) + (t/is (= "comparison-storage" (get-in record [:edges 0 :failure :phase]))) + (t/is (= (get-in outcome [:home :log-path]) + (:log-path (ex-data (first (:errors outcome)))))))) + +(t/deftest ^:async edge-analysis-failures-allow-independent-comparisons-to-complete + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"]) + removed (path/join (:output-dir prepared) "configs" "00" "run.transit") + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:inspect (fn [{:keys [selection]}] + (when (= ["b"] (:features selection)) + (fs/unlinkSync removed)))}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (every? #(= :completed (:status %)) (:configurations record))) + (t/is (= [:failed :failed :completed :completed] (mapv :status (:edges record)))) + (t/is (= ["comparison" "comparison"] (mapv #(get-in % [:failure :phase]) (take 2 (:edges record))))) + (doseq [edge (drop 2 (:edges record))] + (t/is (= :comparison (:type (result/decode (fs/readFileSync (:path edge) "utf8")))))))) + +(t/deftest ^:async post-run-persistence-failure-retains-the-running-configuration + (create-frontend-root) + (let [prepared (comparison-prepared ["a"]) + path (path/join (:output-dir prepared) "feature-comparison.transit") + prior (str path ".saved") + execute (execute-service {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (^:async fn [root home build cases options signal] + (let [response (await (execute root home build cases options signal))] + (fs/renameSync path prior) + (fs/mkdirSync path) + response))})})) + record (result/decode (fs/readFileSync prior "utf8")) + run (result/decode (fs/readFileSync (get-in record [:configurations 0 :run :record-path]) "utf8"))] + (t/is (= 1 (:exit-code outcome))) + (t/is (= record (:comparison outcome))) + (t/is (= :running (:status record))) + (t/is (= [:running :planned] (mapv :status (:configurations record)))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= (get-in outcome [:home :log-path]) + (:log-path (ex-data (first (:errors outcome)))))))) + +(t/deftest create-home-creates-only-a-fresh-home + (create-frontend-root) + (let [home (feature-comparison/create-home (frontend-root) "comparison-a" nil)] + (t/is (= (path/join (frontend-root) "target" "performance" "render-wasm" + "feature-comparisons" "comparison-a") + (:dir home))) + (t/is (= "" (fs/readFileSync (:log-path home) "utf8")))) + (doseq [id [nil "" "../escape" "a/b" "a\\b"]] + (t/is (thrown-with-msg? js/Error #"invalid feature comparison id" + (feature-comparison/create-home (frontend-root) id nil)))) + (let [existing (path/join @temporary-root "existing")] + (fs/mkdirSync existing) + (t/is (thrown-with-msg? js/Error #"already exists" + (feature-comparison/create-home (frontend-root) + "comparison-b" existing)))) + (let [builds (path/join (frontend-root) "target" "performance" "render-wasm" "builds") + dir (path/join builds "build-a" "nested")] + (fs/mkdirSync (path/join builds "build-a") #js {:recursive true}) + (t/is (thrown-with-msg? js/Error #"inside builds" + (feature-comparison/create-home (frontend-root) + "comparison-c" dir))))) + +(t/deftest ^:async initial-record-is-saved-before-the-first-build + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + observed (atom nil) + services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (reset! observed + (result/decode (fs/readFileSync record "utf8")))) + :fail-on (constantly true)}) + :execute-run (execute-service {})}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services services}))] + (t/is (fs/existsSync (get-in result [:home :log-path]))) + (t/is (= :running (:status @observed))) + (t/is (= :building (get-in @observed [:configurations 0 :status]))) + (t/is (nil? (get-in @observed [:configurations 0 :build]))) + (t/is (= :planned (get-in @observed [:configurations 1 :status]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async completed-configurations-and-edge-return-zero + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build + (^:async fn [request] + (let [build (await ((build-service {}) request))] + (fs/mkdirSync (path/dirname (:log-path build)) #js {:recursive true}) + (binding [log/*log-path* (:log-path build)] + (await (timers/setImmediate)) + (log/debug "Build fixture")) + build)) + :execute-run + (^:async fn [root home build cases options signal] + (binding [log/*log-path* (:log-path home)] + (let [response (await ((execute-service {}) root home build cases options signal))] + (log/debug "Run fixture") + response)))})})) + record (:comparison result) + edge (first (:edges record)) + path (:path edge) + comparison (result/decode (fs/readFileSync path "utf8"))] + (t/is (= 0 (:exit-code result))) + (t/is (= :completed (:status record))) + (let [text (fs/readFileSync (get-in result [:home :log-path]) "utf8")] + (doseq [slug ["0" "1"] + phase ["built" "running" "completed"]] + (t/is (str/includes? text (str "Configuration " slug " " phase "\n"))))) + (t/is (= ["a"] (get-in record [:configurations 1 :effective-features]))) + (t/is (= :comparison (:type comparison))) + (t/is (= {:from "0" + :to "1" + :feature "a" + :baseline-features [] + :candidate-features ["a"] + :added ["a"] + :removed []} + (:edge comparison))) + (t/is (= record (result/decode (fs/readFileSync (get-in result [:home :record-path]) + "utf8")))))) + +(t/deftest ^:async attempted-failed-run-continues-and-failed-edge-returns-one + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (execute-service + (fn [build] + (if (seq (get-in build [:manifest :functional :features])) + {} + {:kind :failed})))})})) + record (:comparison result) + edge (first (:edges record))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= :completed (get-in record [:configurations 1 :status]))) + (t/is (fs/existsSync (get-in record [:configurations 0 :run :record-path]))) + (t/is (= :failed (:status edge))) + (t/is (not (contains? edge :path))) + (t/is (str/includes? (get-in edge [:failure :message]) "no shared metric")))) + +(t/deftest ^:async zero-attempts-stop-later-configurations + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + base (execute-service {:kind :zero-attempts}) + calls (atom 0) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @calls)) + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (str/includes? (get-in record [:configurations 1 :reason]) "no attempts")) + (t/is (= :failed (get-in record [:edges 0 :status]))))) + +(t/deftest ^:async build-failure-stops-and-missing-endpoint-fails + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:fail-on #(= ["b"] (:features %))}) + :execute-run (execute-service {})})})) + record (:comparison result) + edge (first (:edges record))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :completed (get-in record [:configurations 0 :status]))) + (t/is (= :failed (get-in record [:configurations 1 :status]))) + (t/is (= :unattempted (get-in record [:configurations 2 :status]))) + (t/is (= :unattempted (get-in record [:configurations 3 :status]))) + (t/is (= :failed (:status edge))) + (t/is (not (contains? edge :path))) + (t/is (str/includes? (get-in edge [:failure :message]) "run record is missing")))) + +(t/deftest ^:async catalog-mismatch-stops-before-a-run-home + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + calls (atom 0) + base (execute-service {}) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [_ _] [{:id :other/case}]) + :execute-run execute})})) + record (:comparison result)] + (t/is (zero? @calls)) + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= "catalog" (get-in record [:configurations 0 :failure :phase]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))))) + +(t/deftest ^:async catalog-identity-failure-retains-selected-and-loaded-build-ids + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [selected _] + (identity/check (:id selected) + {:build-id "other-build"})) + :execute-run (execute-service {})})})) + record (saved-comparison outcome) + config (first (:configurations record))] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status config))) + (t/is (= (get-in config [:build :id]) (get-in config [:failure :expected-id]))) + (t/is (= "other-build" (get-in config [:failure :actual-id]))) + (t/is (= ::identity/mismatch (get-in config [:failure :cause 0 :type]))))) + +(t/deftest ^:async unacknowledged-differences-save-a-failed-edge-report + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (execute-service + (fn [build] + (if (seq (get-in build [:manifest :functional :features])) + {:digest "b"} + {:digest "a"})))})})) + record (:comparison result) + edge (first (:edges record)) + saved (result/decode (fs/readFileSync (:path edge) "utf8")) + text (format/format-feature-comparison-failures record (:home result))] + (t/is (= 1 (:exit-code result))) + (t/is (empty? (:errors result))) + (t/is (= :failed (:status record))) + (t/is (= :failed (:status edge))) + (t/is (fs/existsSync (:path edge))) + (t/is (= {:unacknowledged-differences 1} (:failure edge))) + (t/is (= :comparison (:type saved))) + (t/is (= ["a"] (get-in saved [:edge :added]))) + (t/is (str/includes? text "Comparison \"0\" -> \"1\" (feature \"a\"): 1 unacknowledged difference")) + (t/is (str/includes? text (:path edge))) + (t/is (str/includes? text (get-in result [:home :record-path]))) + (t/is (some #(and (= :fingerprint (:field %)) (not (:waived? %))) + (get-in saved [:pairs 0 :mismatches]))))) + +(t/deftest failure-summary-reports-build-reasons-and-comparison-evidence + (let [record (-> (terminal-record :failed) + (update-in [:configurations 0] dissoc :run :reason) + (assoc-in [:configurations 0 :failure] {:phase "catalog" :message "catalog rejected"}) + (assoc-in [:configurations 1 :status] :failed) + (assoc-in [:configurations 1 :reason] "run did not complete") + result/check-record) + home {:record-path "/feature-comparison.transit" :log-path "/feature-comparison.log"} + text (format/format-feature-comparison-failures record home) + edge (first (:edges record))] + (t/is (str/includes? text "Configuration \"00\": \"catalog\": \"catalog rejected\"")) + (t/is (str/includes? text "Configuration \"01\": \"run did not complete\"")) + (t/is (str/includes? text (str "Comparison " (pr-str (:from edge)) " -> " (pr-str (:to edge))))) + (t/is (str/includes? text "\"comparison\": \"partial\"")) + (t/is (str/includes? text "/build.transit")) + (t/is (str/includes? text "/build.log")) + (t/is (str/includes? text "/comparison.transit")) + (t/is (str/includes? text (:record-path home))) + (t/is (str/includes? text (:log-path home))))) + +(t/deftest failure-summary-bounds-and-escapes-details-and-names-omitted-evidence + (let [plan (feature-plan/plan ["a" "b" "c"] {:dry-run true}) + message (str "detail\n\u001b" (apply str (repeat 10000 "x")) "😀") + record (-> (planned-record) + (assoc :status :failed :ended-at "end" + :features (:features plan) + :configurations (mapv (fn [[index configuration]] + (if (< index 4) + (assoc configuration :status :failed + :failure {:phase "run" :message message}) + (completed-configuration configuration))) + (map-indexed vector (:configurations plan))) + :edges (mapv #(assoc % :status :failed :reason message) (:edges plan))) + (assoc-in [:options :features] (:features plan)) + result/check-record) + home {:record-path "/saved\n\u001b.transit" :log-path "/saved.log"} + text (format/format-feature-comparison-failures record home) + lines (str/split-lines text) + detail (second (re-find #"(?m)^ Configuration \"000\": \"run\": (.+)$" text))] + (t/is (= 8 (count (re-seq #"(?m)^ (?:Configuration|Comparison) " text)))) + (t/is (= 4 (count (re-seq #"(?m)^ Comparison " text)))) + (t/is (str/includes? text "8 more failures in the saved record")) + (t/is (str/includes? text "Record \"/saved")) + (t/is (string? detail)) + (t/is (<= (count detail) 1024)) + (t/is (every? #(<= (count %) 1100) lines)) + (t/is (every? #(>= (.charCodeAt % 0) 32) (mapcat seq lines))) + (t/is (< (count text) 12000)))) + +(t/deftest ^:async interruption-skips-comparisons-and-returns-the-signal-exit + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + controller (js/AbortController.) + base (execute-service + (fn [build] + (when (empty? (get-in build [:manifest :functional :features])) + (.abort controller "SIGINT")) + {:kind :interrupted})) + calls (atom 0) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @calls)) + (t/is (= 130 (:exit-code result))) + (t/is (= :interrupted (:status record))) + (t/is (some? (:ended-at record))) + (t/is (= "SIGINT" (get-in record [:configurations 0 :reason]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (= :unattempted (get-in record [:edges 0 :status]))) + (t/is (not (fs/existsSync (path/join (:dir (:home result)) "comparisons")))))) + +(t/deftest ^:async interruption-between-configurations-stops-before-the-next-build + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + controller (js/AbortController.) + builds (atom 0) + base (execute-service {}) + execute (fn [& args] + (let [result (apply base args)] + (.abort controller "SIGTERM") + result)) + result (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build + (build-service + {:inspect (fn [_] (swap! builds inc))}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @builds)) + (t/is (= 143 (:exit-code result))) + (t/is (= :interrupted (:status record))) + (t/is (= :completed (get-in record [:configurations 0 :status]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (str/includes? (get-in record [:configurations 1 :reason]) "SIGTERM")) + (t/is (= :unattempted (get-in record [:edges 0 :status]))) + (t/is (not (fs/existsSync (path/join (:dir (:home result)) "comparisons")))))) + +(t/deftest ^:async persistence-failure-retains-the-last-saved-state + (create-frontend-root) + (let [stdout (atom "") + prepared (comparison-prepared ["a"] {}) + record-path (path/join (:output-dir prepared) "feature-comparison.transit") + prepare (fn [request] + (fs/rmSync record-path) + (fs/mkdirSync record-path) + (let [build (fake-build (:selection request) [])] + ((:on-created request) build) + (js/Promise.resolve build))) + result (do + (log/configure {:stdout #js {:write #(swap! stdout str %)}}) + (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build prepare + :execute-run (execute-service {})})}))) + record (:comparison result)] + (t/is (= 1 (:exit-code result))) + (t/is (seq (:errors result))) + (t/is (= :running (:status record))) + (t/is (= :building (get-in record [:configurations 0 :status]))) + (t/is (nil? (get-in record [:configurations 0 :build]))) + (t/is (= :planned (get-in record [:configurations 1 :status]))))) + +(t/deftest ^:async comparison-storage-error-uses-comparison-evidence-after-a-failed-config + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services + (comparison-services + {:prepare-build + (build-service + {:inspect (fn [request] + (when (seq (get-in request [:selection :features])) + (fs/renameSync record (str record ".saved")) + (fs/mkdirSync record)))}) + :execute-run (execute-service {:kind :failed})})}))] + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in response [:comparison :configurations 0 :status]))) + (t/is (= (get-in response [:home :log-path]) + (:log-path (ex-data (first (:errors response)))))))) + +(t/deftest ^:async comparison-storage-error-keeps-its-origin-through-build-preparation + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services + {:prepare-build + (^:async fn [request] + (let [created (:on-created request)] + (await (build/prepare + (assoc request :on-created + (fn [selected] + (fs/renameSync record (str record ".saved")) + (fs/mkdirSync record) + (created selected)))))))}}))] + (t/is (= 1 (:exit-code response))) + (t/is (= (get-in response [:home :log-path]) + (:log-path (ex-data (first (:errors response)))))))) + +(t/deftest ^:async pending-real-signals-stop-edge-analysis + (create-frontend-root) + (doseq [[signal exit-code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root signal)}) + controller (js/AbortController.) + remove (execution/on-signals #(.abort controller %)) + execute (execute-service + (fn [build] + (when (seq (get-in build [:manifest :functional :features])) + (.kill js/process (.-pid js/process) signal)) + {}))] + (try + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})}))] + (await (timers/setImmediate)) + (let [saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= exit-code (:exit-code response))) + (t/is (= :interrupted (:status saved))) + (t/is (every? #(= :unattempted (:status %)) (:edges saved))))) + (finally + (remove)))))) diff --git a/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs b/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs new file mode 100644 index 0000000000..d351bf1b86 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs @@ -0,0 +1,100 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.fingerprint-test + "Tests content identity with production records, paths and collection types. + Validated scene snapshots retain child order even when maps reorder." + (:require + [app.common.geom.matrix :as matrix] + [app.common.geom.point :as point] + [app.common.transit :as transit] + [app.common.types.objects-map :as objects-map] + [app.common.types.path.impl :as path] + [app.common.types.shape :as shape] + [app.common.uuid :as uuid] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.fingerprint :as fingerprint] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [linked.map :as linked-map] + [linked.set :as linked-set])) + +(def a #uuid "00000000-0000-0000-0000-000000000001") +(def b #uuid "00000000-0000-0000-0000-000000000002") + +(defn- starting-snapshot + "Builds and validates two canonical rectangles under the root frame." + [] + (snapshot/validate! + {:objects {uuid/zero (shape/setup-shape {:id uuid/zero :type :frame :name "Root" + :x 0 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero :shapes [a b]}) + a (shape/setup-shape {:id a :type :rect :name "A" + :x 0 :y 0 :width 10 :height 10 :parent-id uuid/zero :frame-id uuid/zero}) + b (shape/setup-shape {:id b :type :rect :name "B" + :x 20 :y 20 :width 10 :height 10 :parent-id uuid/zero :frame-id uuid/zero})} + :refs {:a a :b b}})) + +(t/deftest unordered-representations-have-one-fingerprint + (let [scene (starting-snapshot) + first (assoc scene :extra {:map (array-map :a 1 :b 2) + :compound (array-map [:a 1] {:x 1 :y 2} [:b 2] #{:one :two}) + :set (set (range 20))}) + second (-> scene + (update :objects #(into {} (reverse (seq %)))) + (assoc :refs (array-map :b b :a a) + :extra {:set (set (reverse (range 20))) + :compound (array-map [:b 2] #{:two :one} [:a 1] (array-map :y 2 :x 1)) + :map (array-map :b 2 :a 1)}))] + (t/is (= (fingerprint/snapshot-text first) (fingerprint/snapshot-text second))) + (t/is (= (fingerprint/fingerprint first) (fingerprint/fingerprint second))))) + +(t/deftest ordered-collections-and-child-order-change-the-fingerprint + (let [scene (starting-snapshot)] + (t/is (not= (fingerprint/fingerprint scene) + (fingerprint/fingerprint (assoc-in scene [:objects uuid/zero :shapes] [b a])))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra [1 2])) + (fingerprint/fingerprint (assoc scene :extra [2 1])))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra (into linked-set/empty-linked-set [1 2]))) + (fingerprint/fingerprint (assoc scene :extra (into linked-set/empty-linked-set [2 1]))))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra (into linked-map/empty-linked-map [[:a 1] [:b 2]]))) + (fingerprint/fingerprint (assoc scene :extra (into linked-map/empty-linked-map [[:b 2] [:a 1]]))))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra (point/point 1 2))) + (fingerprint/fingerprint (assoc scene :extra {:x 1 :y 2})))))) + +(t/deftest unrepresentable-transit-content-never-gets-a-misleading-fingerprint + (let [scene (starting-snapshot)] + (t/is (= ::codec/unrepresentable-value + (try + (fingerprint/fingerprint (assoc scene :extra {"__proto__" "value"})) + nil + (catch :default cause (:type (ex-data cause)))))))) + +(t/deftest objects-map-materializes-before-hashing-and-production-tags-survive + (let [scene (starting-snapshot) + wrapped (update scene :objects objects-map/wrap) + data (assoc scene :extra {:point (point/point 1 2) + :matrix (matrix/matrix) + :path (path/path-data [{:command :move-to :params {:x 1 :y 2}} + {:command :line-to :params {:x 3 :y 4}}])}) + text (fingerprint/snapshot-text data)] + (t/is (= (fingerprint/fingerprint scene) (fingerprint/fingerprint wrapped))) + (t/is (not (str/includes? (fingerprint/snapshot-text wrapped) "penpot/objects-map"))) + (t/is (str/includes? text "~#shape")) + (t/is (str/includes? text "~#point")) + (t/is (str/includes? text "~#matrix")) + (t/is (str/includes? text "~#penpot/path-data")) + (t/is (= (fingerprint/fingerprint data) + (fingerprint/fingerprint (transit/decode-str (transit/encode-str data))))))) + +(t/deftest digest-matches-known-sha256-vector + ;; The verbose Transit map for {:objects {} :refs {}} is canonical text. + (t/is (= "{\"~:objects\":{},\"~:refs\":{}}" + (fingerprint/snapshot-text {:objects {} :refs {}}))) + (t/is (= {:algorithm :sha-256 + :digest "ceb57067506683f5a0e80615bc9dd6f6d16325e333864da705fa4916fbdfe52d"} + (fingerprint/fingerprint {:objects {} :refs {}})))) diff --git a/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs b/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs new file mode 100644 index 0000000000..58852a055b --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs @@ -0,0 +1,86 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.geometry-scenes-test + (:require + [app.common.types.path :as path] + [app.common.uuid :as uuid] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + ;; These 3 register the scenes, do not remove + [benches.render-wasm.scenes.masks] + [benches.render-wasm.scenes.paths] + [benches.render-wasm.scenes.plus] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(t/deftest curved-paths-use-canonical-content-and-stable-order + (let [params {:count 12 :width 1920 :height 1080 :seed 42} + build (:build (decl/registered-scene :paths)) + scene (build params) + again (build params) + shapes (helpers/children-of scene uuid/zero) + first (first shapes) + content (vec (:content first))] + (t/is (= 12 (count shapes))) + (t/is (= scene again) "same scene parameters and seed reproduce content and ids") + (t/is (path/content? (:content first))) + (t/is (= [:move-to :curve-to :line-to :close-path] + (mapv :command content))) + (t/is (pos? (:width (:selrect first)))) + (t/is (pos? (:height (:selrect first)))) + (t/is (= (mapv :id shapes) (mapv :id (rest (snapshot/upload-order scene))))) + (t/is (= scene (snapshot/validate! scene))))) + +(t/deftest plus-paths-have-two-crossing-open-lines + (let [build (:build (decl/registered-scene :plus)) + scene (build {:count 8 :width 1920 :height 1080 :seed 42}) + first (first (helpers/children-of scene uuid/zero)) + segments (vec (:content first)) + [a b c d] segments] + (t/is (= 8 (count (helpers/children-of scene uuid/zero)))) + (t/is (= [:move-to :line-to :move-to :line-to] + (mapv :command segments))) + (t/is (= (get-in a [:params :y]) (get-in b [:params :y])) + "the first line is horizontal") + (t/is (< (get-in a [:params :x]) (get-in b [:params :x]))) + (t/is (= (get-in c [:params :x]) (get-in d [:params :x])) + "the second line is vertical") + (t/is (< (get-in c [:params :y]) (get-in d [:params :y]))) + (t/is (seq (:strokes first))) + (t/is (= scene (snapshot/validate! scene))))) + +(t/deftest mask-child-order-and-descendants-are-reachable + (let [build (:build (decl/registered-scene :masks)) + scene (build {:width 1920 :height 1080 :seed 42}) + groups (helpers/children-of scene uuid/zero) + group (first groups) + children (helpers/children-of scene (:id group)) + mask (first children) + uploaded (snapshot/upload-order scene)] + (t/is (= 24 (count groups))) + (t/is (= 241 (count (:objects scene))) "root, groups and their nine children") + (t/is (= (:id group) (get-in scene [:refs [:mask 0]]))) + (t/is (true? (:masked-group group))) + (t/is (= :circle (:type mask)) "the first child is the mask") + (t/is (= 9 (count children))) + (t/is (every? #(= (:id group) (:parent-id %)) children)) + (t/is (= (:selrect mask) (:selrect group))) + (t/is (= uuid/zero (:id (first uploaded)))) + (t/is (= (:id group) (:id (second uploaded)))) + (t/is (= (:id mask) (:id (nth uploaded 2)))) + (t/is (= (count (:objects scene)) (count uploaded))) + (t/is (= scene (snapshot/validate! scene))))) + +(t/deftest camera-cases-share-each-scene-seed + (doseq [scene-id [:paths :plus :masks]] + (let [selected (->> (cases/collect-cases {:master-seed 42}) + (filter #(= scene-id (:scene %))) + vec)] + (t/is (= #{"load" "pan" "zoom"} (set (map (comp name :id) selected)))) + (t/is (= 1 (count (set (map :scene-seed selected))))) + (t/is (every? #(not (contains? % :run!)) selected))))) diff --git a/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs new file mode 100644 index 0000000000..8a1a8641a5 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs @@ -0,0 +1,1142 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.lifecycle-test + "Tests the browser bridge with fake DOM and renderer boundaries. + + It covers bridge input, ownership, disposal and `load-scene` with fake + canvases and a fake Emscripten factory. Progressive frames use an optional + clock at the browser boundary." + (:require + [app.common.render-wasm.wasm :as wasm] + [app.common.transit :as transit] + [benches.render-wasm.browser.bridge :as browser] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.measurement :as measurement] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [benches.render-wasm.runtime.camera :as camera] + [cljs.test :as t :include-macros true])) + +;; Forward declaration: entry-test helpers below build args against the +;; fake Emscripten factory URL defined alongside the fake-DOM harness. +(declare factory-url) + +;; Partial-init unwinding of `init-context` is covered in +;; `frontend-tests.render-wasm.webgl-test`, which also pins the success, +;; nil-context and registration-failure cases. + +(defn- register-phase-case + [options run] + (let [params (:params (decl/registered-case :rects/load))] + (decl/defcase :rects/lifecycle + (-> (merge {:params params + :view {:scale 1 :x 0 :y 0} + :context :fresh} + options) + (update :params #(merge params %))) + [rtx] + (run rtx)))) + +(defn- load-case + "Returns the collected rectangle case for entry tests." + [] + (first (cases/collect-cases {:master-seed 42 + :filter (if (decl/registered-case :rects/lifecycle) + "rects/lifecycle" + "rects/load")}))) + +(defn- scene-args + "Transit request for `browser/load-scene`. `overrides` change the + collected case descriptor." + [seed overrides] + (let [base (or (some #(when (= (:id %) (:id overrides)) %) + (cases/collect-cases {:master-seed 42})) + (load-case))] + (transit/encode-str + {:case (merge base {:scene-seed seed} overrides) + :module-url (factory-url) + :wasm-url "./fake.wasm"}))) + +(defn- expect-failed-args + "Calls `load-scene` with `args` (never touching DOM/FFI: validation runs + first) and asserts a failure map. Calls `done` when finished." + [args check-phase? done] + (-> (.then (browser/load-scene args) + (fn [result] + (let [m (transit/decode-str result)] + (t/is (= "failed" (:status m))) + (when check-phase? + (t/is (= "invalid-args" (:phase m)))) + (done)))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))) + +(t/deftest invalid-seed-fails-before-timers + (t/async done + (expect-failed-args (scene-args -1 {}) true done))) + +(t/deftest missing-seed-fails-before-timers + (t/async done + (expect-failed-args (transit/encode-str {:case (dissoc (load-case) :scene-seed)}) true done))) + +(t/deftest missing-case-fails-before-timers + (t/async done + (expect-failed-args (transit/encode-str {}) true done))) + +(t/deftest malformed-transit-fails-before-timers + (t/async done + (expect-failed-args "not transit" true done))) + +(t/deftest invalid-view-fails-before-timers + (t/async done + (expect-failed-args + (scene-args 42 {:view {:scale -1 :x 0 :y 0}}) + true + done))) + +(t/deftest invalid-viewport-fails-before-timers + (t/async done + (expect-failed-args + (scene-args 42 {:view {:scale 1 :x 0 :y 0 + :viewport {:width -5}}}) + true + done))) + +(t/deftest missing-compiled-body-fails-before-claim + (t/async done + (expect-failed-args (scene-args 42 {:id :rects/missing}) true done))) + +(t/deftest ^:async wrong-scene-body-rejects-the-load-request + (let [id :rects/wrong-scene + descriptor (load-case)] + (decl/register-case! {:id id :scene :other :ns "lifecycle-test" + :run! (fn [_] nil)}) + (try + (let [request (transit/encode-str {:case (assoc descriptor :id id)}) + result (transit/decode-str (await (browser/prepare-scene-request request)))] + (t/is (= "failed" (:status result))) + (t/is (= "invalid-args" (:phase result)))) + (finally + (decl/unregister-case! id))))) + +(t/deftest non-finite-view-values-fail-before-timers + (t/async done + (expect-failed-args (scene-args 42 {:view {:scale js/Infinity :x 0 :y 0}}) + true + (fn [] + (expect-failed-args (scene-args 42 {:view {:scale 1 :x js/NaN :y 0}}) + true + (fn [] + (expect-failed-args (scene-args 42 {:view {:scale 1 :x 0 :y js/Infinity}}) + true + done))))))) + +(t/deftest viewport-defaults-apply-at-use + (t/is (= {:width 1920 :height 1080 :dpr 2} + (decl/resolve-viewport {:scale 1 :x 0 :y 0}))) + (t/is (= {:width 800 :height 600 :dpr 2} + (decl/resolve-viewport {:scale 1 :x 0 :y 0 + :viewport {:width 800 :height 600}})))) + +(t/deftest dispose-resets-lifecycle-state + (set! wasm/internal-module #js {}) + (set! wasm/context-initialized? true) + (reset! wasm/context-lost? true) + (set! wasm/gl-context-handle 7) + (set! wasm/gl-context #js {}) + (set! wasm/canvas #js {}) + (let [removed (atom false) + fake #js {:remove (fn [] (reset! removed true))}] + (reset! @#'browser/canvas* fake) + (let [result (browser/dispose!)] + (t/is (= "disposed" (unchecked-get result "status"))) + (t/is (nil? wasm/internal-module)) + (t/is (false? wasm/context-initialized?)) + (t/is (false? @wasm/context-lost?)) + (t/is (nil? wasm/gl-context-handle)) + (t/is (nil? wasm/gl-context)) + (t/is (nil? wasm/canvas)) + (t/is (nil? @@#'browser/canvas*)) + (t/is (true? @removed) "canvas is removed from the DOM"))) + (t/testing "double dispose is idempotent" + (let [result (browser/dispose!)] + (t/is (= "disposed" (unchecked-get result "status"))) + (t/is (nil? wasm/internal-module)))) + (t/testing "dispose bumps the owner epoch" + (let [before @@#'browser/owner-epoch*] + (browser/dispose!) + (t/is (< before @@#'browser/owner-epoch*))))) + +(t/deftest stale-work-never-disposes-the-newer-owner + (let [guard @#'browser/guard-current! + terminate @#'browser/terminal-failure + epoch @@#'browser/owner-epoch*] + (t/testing "guard passes for the current owner" + (t/is (nil? (guard epoch "module-init")))) + (t/testing "guard throws stale once superseded" + (browser/dispose!) + (let [thrown (try + (guard epoch "module-init") + nil + (catch :default cause + cause))] + (t/is (errors/stale? thrown) + "the marker lives in ex-data, never in message text"))) + (t/testing "terminal stale resolves without disposal" + (let [before @@#'browser/owner-epoch* + result (terminate epoch + (ex-info "superseded" + {::errors/stale true + :phase "upload"}) + "upload") + after @@#'browser/owner-epoch*] + (t/is (= "stale" (:status result))) + (t/is (= before after) "no epoch bump: the newer owner is untouched"))) + (t/testing "terminal failure keeps its phase and disposes the current owner" + (let [current @@#'browser/owner-epoch* + result (terminate current (ex-info "upload failed" {:phase "upload"}) "aborted")] + (t/is (= "failed" (:status result))) + (t/is (= "upload" (:phase result))) + (t/is (< current @@#'browser/owner-epoch*) "current owner disposed"))) + (t/testing "phaseless failures use the fallback phase" + (set! wasm/internal-module nil) + (let [current @@#'browser/owner-epoch* + result (terminate current (ex-info "boom" {}) "aborted")] + (t/is (= "failed" (:status result))) + (t/is (= "aborted" (:phase result))) + (t/is (< current @@#'browser/owner-epoch*) "current owner disposed"))))) + +(t/deftest bridge-exposes-executor-entries + (doseq [key ["ping" "loadScene" "prepareScene" "runPrepared" "dispose"]] + (t/is (fn? (unchecked-get browser/bridge key)) + (str key " is callable through the browser ABI"))) + (t/is (= "ok" (unchecked-get (.ping browser/bridge) "status")))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Through-entry fake harness. +;; +;; The DOM/module boundary the fakes cover does not change with the entry: +;; fake DOM, fake Emscripten factory, call recording. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- record! + "Appends `event` (a vector) to the shared call log." + [event] + (.push (unchecked-get js/globalThis "__benchCalls") (clj->js event))) + +(defn- fake-canvas + "Numbered fake canvas identified by `id`. Removal and context loss are + recorded in the shared call log. The context reports a drawing-buffer size + distinct from any canvas size, so tests prove `read-graphics` reads it from + the context rather than the canvas attributes." + ([id] + (fake-canvas id nil)) + ([id {:keys [vendor renderer]}] + (let [debug-ext (when (and vendor renderer) + #js {:UNMASKED_VENDOR_WEBGL 1 :UNMASKED_RENDERER_WEBGL 2}) + lose-ext #js {:loseContext (fn [] (record! ["lose" id]) nil)} + context #js {:getExtension (fn [name] + (case name + "WEBGL_lose_context" lose-ext + "WEBGL_debug_renderer_info" debug-ext + nil)) + :getParameter (fn [parameter] (case parameter 1 vendor 2 renderer nil)) + :getContextAttributes (fn [] + #js {"antialias" false + "depth" true + "stencil" false + "alpha" false + "preserveDrawingBuffer" false}) + :drawingBufferWidth 1234 + :drawingBufferHeight 5678}] + #js {:id id + :width 0 + :height 0 + :style #js {} + :getContext (fn [_ _] context) + :addEventListener (fn [_ _] nil) + :remove (fn [] (record! ["remove" id]) nil)}))) + +(defn- install-fake-dom! + "Installs fake DOM, import, clock and scenario globals and returns a + `restore!` thunk that restores their previous values. Created canvases are + numbered from 1 per installation. + The import polyfill mirrors + `resources/polyfills/dynamicImport.js`, which ships with the app HTML but + not with the unit-test bundle." + [] + (let [prev-document (unchecked-get js/globalThis "document") + prev-window (unchecked-get js/globalThis "window") + prev-import (unchecked-get js/globalThis "dynamicImport") + prev-performance (unchecked-get js/globalThis "performance") + prev-now-fn (unchecked-get prev-performance "now") + prev-now-desc (js/Object.getOwnPropertyDescriptor prev-performance "now") + prev-now (unchecked-get js/globalThis "__benchNow") + prev-script (unchecked-get js/globalThis "__benchRenderScript") + prev-calls (unchecked-get js/globalThis "__benchCalls") + prev-factory (unchecked-get js/globalThis "__benchFactoryCalls") + prev-throw (unchecked-get js/globalThis "__benchThrowIn") + prev-gate (unchecked-get js/globalThis "__benchGateRaf") + prev-raf (unchecked-get js/globalThis "__benchRafCb") + next-id (atom 0) + document #js {:createElement (fn [_] + (let [id (swap! next-id inc)] + (record! ["create" id]) + (fake-canvas id))) + :body #js {:appendChild (fn [canvas] + (record! ["append" (unchecked-get canvas "id")]) + canvas)} + :querySelectorAll (fn [_] #js [])} + window #js {:requestAnimationFrame + (fn [cb] + ;; Park the first frame while `__benchGateRaf` + ;; is set so a second load can supersede a live, + ;; active owner; later frames run synchronously. + (if (and (unchecked-get js/globalThis "__benchGateRaf") + (nil? (unchecked-get js/globalThis "__benchRafCb"))) + (unchecked-set js/globalThis "__benchRafCb" cb) + (if-some [clock (unchecked-get js/globalThis "__benchNow")] + (let [timestamp (+ clock 16)] + (unchecked-set js/globalThis "__benchNow" timestamp) + (cb timestamp)) + (cb 0))) + nil)} + fake-now (fn [] + (if-some [clock (unchecked-get js/globalThis "__benchNow")] + clock + (.call prev-now-fn prev-performance))) + dynamic-import (fn [url] (js/eval (str "import(" (pr-str url) ")")))] + (unchecked-set js/globalThis "document" document) + (unchecked-set js/globalThis "window" window) + ;; Patch `now` on the real object: shadow hangs the compiled + ;; `performance.*` namespaces off the global `performance` object, so + ;; replacing that global hides every `benches.render-wasm.*` var. + (js/Object.defineProperty prev-performance "now" + #js {:value fake-now :configurable true :writable true}) + (unchecked-set js/globalThis "dynamicImport" dynamic-import) + (fn restore! [] + (unchecked-set js/globalThis "document" prev-document) + (unchecked-set js/globalThis "window" prev-window) + (if (undefined? prev-now-desc) + (js-delete prev-performance "now") + (js/Object.defineProperty prev-performance "now" prev-now-desc)) + (unchecked-set js/globalThis "__benchNow" prev-now) + (unchecked-set js/globalThis "__benchRenderScript" prev-script) + (unchecked-set js/globalThis "__benchCalls" prev-calls) + (unchecked-set js/globalThis "__benchFactoryCalls" prev-factory) + (unchecked-set js/globalThis "__benchThrowIn" prev-throw) + (unchecked-set js/globalThis "__benchGateRaf" prev-gate) + (unchecked-set js/globalThis "__benchRafCb" prev-raf) + (unchecked-set js/globalThis "dynamicImport" prev-import)))) + +(def ^:private factory-source + "ES module stubbing the Emscripten factory for entry tests. Records every + renderer call in `globalThis.__benchCalls`; `globalThis.__benchThrowIn` + names one module fn or an array of module fns that throw. + `globalThis.__benchRenderScript` holds optional [frame-type, duration-ms] + pairs that advance `__benchNow`; absent + a script, `_render` immediately returns Full. The instance trap answers + `undefined` for `then` and non-strings: promise assimilation would treat the + instance as a never-settling thenable and hang the load. A real 16 MiB + heap with bump allocation backs `_alloc_bytes`/`HEAPU8`, so the real batch + upload assembles bytes instead of throwing on stub returns." + (str "export default function (opts) {" + "globalThis.__benchFactoryCalls.push(opts || null);" + "var rec = globalThis.__benchCalls;" + "function push(e) { rec.push(e); }" + "var GL = {" + "registerContext: function () { push(['register']); return 7; }," + "makeContextCurrent: function () {}," + "deleteContext: function (h) { push(['delete', h]); }" + "};" + "var heapBuf = new ArrayBuffer(16777216);" + "var heapNext = 1024;" + "var base = {" + "GL: GL," + "HEAPU8: new Uint8Array(heapBuf)," + "HEAPU32: new Uint32Array(heapBuf)," + "HEAP32: new Int32Array(heapBuf)," + "HEAPF32: new Float32Array(heapBuf)," + "_alloc_bytes: function (size) { var p = heapNext; heapNext += ((size + 3) & ~3); return p; }," + "_free_bytes: function () {}," + "_read_error_code: function () { return 0; }," + "_render: function (timestamp, flags) {" + "push(['render', timestamp, flags]);" + "var script = globalThis.__benchRenderScript;" + "if (script && script.length) {" + "var next = script.shift(); globalThis.__benchNow += next[1]; return next[0];" + "}" + "return 2;" + "}," + "_clean_up: function () { push(['clean']); return 0; }" + "};" + "var inst = new Proxy(base, {" + "get: function (t, p) {" + "if (typeof p !== 'string') return undefined;" + "if (p === 'then') return undefined;" + "if (p in t) return t[p];" + "return function () {" + "push(['call', p]);" + "var faults = globalThis.__benchThrowIn;" + "if (Array.isArray(faults) ? faults.includes(p) : faults === p) throw new Error('fake ' + p + ' failure');" + "return 0;" + "};" + "}" + "});" + "return Promise.resolve(inst);" + "}")) + +(defn- factory-url + "Data-URL Emscripten factory for entry tests. Fully hermetic: no + filesystem or network access." + [] + (str "data:text/javascript," (js/encodeURIComponent factory-source))) + +(defn- reset-bench-globals! + "Resets the recording and scenario flags the fake factory reads." + [] + (unchecked-set js/globalThis "__benchCalls" #js []) + (unchecked-set js/globalThis "__benchFactoryCalls" #js []) + (unchecked-set js/globalThis "__benchThrowIn" nil) + (unchecked-set js/globalThis "__benchGateRaf" false) + (unchecked-set js/globalThis "__benchRafCb" nil) + (unchecked-set js/globalThis "__benchNow" nil) + (unchecked-set js/globalThis "__benchRenderScript" nil)) + +(defn- read-calls + "Call log as Clojure data." + [] + (js->clj (unchecked-get js/globalThis "__benchCalls"))) + +(defn- effect-count + "Counts recorded effects without filtering any returned evidence." + [calls effect] + (count (filter #(= effect (first %)) calls))) + +(defn- load-result + "Decodes the bridge's Transit result for `k`. A rejected bridge arrives + as a `threw` map so the test fails with the cause attached." + [args k] + (-> (browser/load-scene args) + (.then (fn [result] + (t/is (string? result) "loadScene returns Transit") + (k (transit/decode-str result)))) + (.catch (fn [cause] (k {:status "threw" :cause (str cause)}))))) + +(defn- with-entry-env + "Installs the fake DOM, resets bench globals, then calls `f` with a + `cleanup` thunk that disposes the owner and restores the previous globals. + Call `cleanup` after asserting, before `done`." + [f] + (let [restore-dom! (install-fake-dom!) + cleanup (fn [] + (try + (browser/dispose!) + (finally + (decl/unregister-case! :rects/lifecycle) + (restore-dom!))))] + (reset-bench-globals!) + (try + (register-phase-case {} camera/render-sync*) + (f cleanup) + (catch :default cause + (cleanup) + (throw cause))))) + +(t/deftest ^:async preparations-record-the-context-identity-and-software-status + (doseq [[identity software] [[{:vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan)"} false] + [{:vendor "Google" :renderer "ANGLE (SwiftShader)"} true] + [{:vendor "Mesa" :renderer "llvmpipe (LLVM)"} true] + [nil false]]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (set! (.-createElement js/document) (fn [_] (fake-canvas 1 identity))) + (let [response (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + graphics (get-in response [:preparation :effective-graphics])] + (t/is (= "ok" (:status response))) + (t/is (= (or (:vendor identity) "unknown") (:vendor graphics))) + (t/is (= (or (:renderer identity) "unknown") (:renderer graphics))) + (t/is (= software (:software graphics)))) + (finally + (cleanup)))))))) + +(t/deftest successful-load-reaches-full + (t/async done + (with-entry-env + (fn [cleanup] + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "ok" (:status m))) + (t/is (= 1 (count (get-in m [:preparation :diagnostics :setup-render :slices])))) + (t/is (= [::camera/render-sync] + (mapv :command (get-in m [:phases 0 :events])))) + (t/is (= 1001 (:shapes (:scene m))) "the canonical scene uploads whole") + (let [graphics (:effective-graphics m)] + (t/is (= 1234 (:drawing-buffer-width graphics)) + "drawing buffer width comes from the context") + (t/is (= 5678 (:drawing-buffer-height graphics)) + "drawing buffer height comes from the context") + (t/is (false? (:antialias graphics)) + "context attributes pass through") + (t/is (true? (:depth graphics)) + "context attributes pass through")) + (let [calls (read-calls)] + (t/is (= 1 (effect-count calls "create")) "one canvas per load") + (t/is (zero? (effect-count calls "remove")) "success keeps the owner live")) + (cleanup) + (done))))))) + +(t/deftest preparation-retains-progressive-frame-evidence + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchNow" 0) + (unchecked-set js/globalThis "__benchRenderScript" + #js [#js [wasm/FRAME_TYPE_PARTIAL 2] + #js [wasm/FRAME_TYPE_VIEWPORT_READY 3] + #js [wasm/FRAME_TYPE_FULL 5]]) + (-> (.loadScene browser/bridge (scene-args 7 {})) + (.then (fn [text] + (t/is (string? text) "the exported loadScene returns Transit") + (let [{:keys [status preparation scene]} (transit/decode-str text) + {:keys [slices viewport-ready-ms full-ms]} + (get-in preparation [:diagnostics :setup-render]) + retained slices + render-calls (filterv #(= "render" (first %)) (read-calls))] + (t/is (= "ok" status)) + (t/is (= {:shapes 1001 :seed 7} scene) + "the real canonical case prepares and uploads the whole scene") + (t/is (= 3 (count retained)) "all three slices cross Transit") + (t/is (= [wasm/FRAME_TYPE_PARTIAL + wasm/FRAME_TYPE_VIEWPORT_READY + wasm/FRAME_TYPE_FULL] + (mapv :frame-type retained))) + (t/is (= [0 wasm/FRAME_TYPE_PARTIAL wasm/FRAME_TYPE_VIEWPORT_READY] + (mapv :flags retained)) + "ViewportReady becomes the next flags and the drain continues to Full") + (t/is (= [16 34 53] (mapv :timestamp retained)) + "each continuation waits for the next frame") + (t/is (= [2 3 5] (mapv :duration-ms retained))) + (t/is (= render-calls + (mapv (fn [{:keys [timestamp flags]}] + ["render" timestamp flags]) + retained)) + "decoded slices retain every WASM call's timestamp and flags in order") + (t/is (= 37 viewport-ready-ms + (+ (:timestamp (nth retained 1)) + (:duration-ms (nth retained 1)))) + "viewport completion ends with the ViewportReady slice") + (t/is (= 58 full-ms + (+ (:timestamp (nth retained 2)) + (:duration-ms (nth retained 2)))) + "Full completion ends with the final slice") + (t/is (< viewport-ready-ms full-ms)) + (t/is (= [{:phase :preparation + :preparation-id (:preparation-id preparation) + :outcome :completed :events []}] + (:phases preparation)))) + (cleanup) + (done))) + (.catch (fn [cause] + (t/is false (str "exported progressive load failed: " cause)) + (cleanup) + (done)))))))) + +(t/deftest invalid-args-keep-the-current-owner + (t/async done + (with-entry-env + (fn [cleanup] + (load-result (scene-args 7 {}) + (fn [loaded] + (t/is (= "ok" (:status loaded))) + (let [owner @@#'browser/canvas* + epoch @@#'browser/owner-epoch*] + (load-result (transit/encode-str {:seed -1}) + (fn [invalid] + (t/is (= "invalid-args" (:phase invalid))) + (t/is (identical? owner @@#'browser/canvas*)) + (t/is (= epoch @@#'browser/owner-epoch*)) + (t/is (zero? (effect-count (read-calls) "remove"))) + (cleanup) + (done)))))))))) + +(t/deftest compiled-pan-finishes-at-its-final-render + (t/async done + (with-entry-env + (fn [cleanup] + (register-phase-case {:context :reuse} + #(camera/pan* % {:dx 10 :dy 5 :steps 2 :settle-ms 0})) + (load-result (scene-args 7 {}) + (fn [m] + (let [phase (first (:phases m)) + event (first (:events phase)) + trace (:trace event)] + (t/is (= "ok" (:status m))) + (t/is (= :main (:phase phase))) + (t/is (= ::camera/pan (:command event))) + (t/is (= 2 (count (:cached-slices trace)))) + (t/is (= 0 (:settling-requested-ms trace))) + (t/is (= [4] (mapv :flags (:slices trace)))) + (t/is (= [2] (mapv :frame-type (:slices trace)))) + (t/is (= 2 (effect-count (read-calls) "render")) + "preparation and camera finalization each render once")) + (cleanup) + (done))))))) + +(t/deftest ^:async reused-context-prepares-once-and-resets-each-main + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [views (atom []) + options {:dy 0 :steps 1 :settle-ms 0} + pan (fn [phase dx rtx] + (swap! views conj [phase (get-in rtx [:view :x])]) + (camera/pan* rtx (assoc options :dx dx)))] + (register-phase-case {:context :reuse + :preparation {:callback #(pan :preparation 5 %)} + :reset #(pan :reset -10 %)} + #(pan :main 10 %)) + (let [prepared (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + prep (:preparation prepared) + request (transit/encode-str {:preparation-id (:preparation-id prep)}) + first (transit/decode-str (await (browser/run-prepared request))) + second (transit/decode-str (await (browser/run-prepared request)))] + (t/is (= ["ok" "ok" "ok"] (mapv :status [prepared first second]))) + (t/is (= [[:preparation 0] [:main 5] [:reset 15] [:main 5] [:reset 15]] @views)) + (t/is (= [:preparation] (mapv :phase (:phases prep)))) + (t/is (= [::camera/pan] (mapv :command (get-in prep [:phases 0 :events])))) + (t/is (= prep (:preparation first) (:preparation second))) + (doseq [execution [first second]] + (t/is (= [:main :reset] (mapv :phase (:phases execution)))) + (t/is (= [:completed :completed] (mapv :outcome (:phases execution)))) + (t/is (= [[::camera/pan] [::camera/pan]] + (mapv #(mapv :command (:events %)) (:phases execution))))) + (t/is (= 1 (effect-count (read-calls) "create"))) + (t/is (= 6 (effect-count (read-calls) "render")) + "the callback, initial Full, and four main/reset gestures render") + (t/is (= 1 (count (filter #(= ["call" "_init_shapes_pool"] %) (read-calls))))))) + (finally + (cleanup))))))) + +(t/deftest ^:async reused-preparation-keeps-its-original-case-deadline + (await + (with-entry-env + (^:async fn [cleanup] + (try + (unchecked-set js/globalThis "__benchNow" 0) + (register-phase-case {:context :reuse :preparation {:render :none}} camera/render-sync*) + (let [request (-> (scene-args 7 {}) transit/decode-str + (assoc :timeout-ms 100) transit/encode-str) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + id (get-in prepared [:preparation :preparation-id])] + (unchecked-set js/globalThis "__benchNow" 60) + (let [first (transit/decode-str (await (browser/run-prepared + (transit/encode-str {:preparation-id id :timeout-ms 10000}))))] + (t/is (= "ok" (:status first)))) + (unchecked-set js/globalThis "__benchNow" 101) + (let [expired (transit/decode-str (await (browser/run-prepared + (transit/encode-str {:preparation-id id}))))] + (t/is (= "failed" (:status expired))) + (t/is (= "deadline" (:phase expired))))) + (finally + (cleanup))))))) + +(t/deftest ^:async fresh-contexts-prepare-and-reset-once + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [visits (atom []) + work (fn [phase rtx] + (swap! visits conj phase) + (camera/render-sync* rtx))] + (register-phase-case {:preparation {:callback #(work :preparation %)} + :reset #(work :reset %)} + #(work :main %)) + (dotimes [_ 2] + (let [prepared (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + prep (:preparation prepared) + request (transit/encode-str {:preparation-id (:preparation-id prep)}) + evidence (transit/decode-str (await (browser/run-prepared request))) + repeated (transit/decode-str (await (browser/run-prepared request)))] + (t/is (= "ok" (:status evidence))) + (t/is (= [:main :reset] (mapv :phase (:phases evidence)))) + (t/is (= [::camera/render-sync] (mapv :command (get-in prep [:phases 0 :events])))) + (t/is (= "invalid-args" (:phase repeated))))) + (t/is (= [:preparation :main :reset :preparation :main :reset] @visits)) + (t/is (= 2 (effect-count (read-calls) "create"))) + (t/is (= 2 (effect-count (read-calls) "render")))) + (finally + (cleanup))))))) + +(t/deftest ^:async preparation-policy-keeps-selected-events-and-empty-main + (doseq [context [:fresh :reuse] + render [nil :none]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (unchecked-set js/globalThis "__benchNow" 0) + (register-phase-case + {:params {:count 2} + :context context + :preparation (cond-> {:callback #(-> % (camera/render-sync*) (camera/render-sync*))} + render (assoc :render render))} + identity) + (let [evidence (transit/decode-str (await (browser/load-scene (scene-args 7 {})))) + prep (:preparation evidence) + block (first (:phases prep))] + (t/is (= "ok" (:status evidence))) + (t/is (= :completed (:outcome block))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [0]}} + {:command ::camera/render-sync :metrics {:command/elapsed [0]}}] + (:events block))) + (t/is (= [{:phase :main :preparation-id (:preparation-id prep) + :outcome :completed :events []}] + (:phases evidence))) + (t/is (= (if (= render :none) 0 1) (effect-count (read-calls) "render")))) + (finally + (cleanup)))))))) + +(t/deftest ^:async reused-context-carries-view-without-a-reset + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [views (atom [])] + (register-phase-case {:params {:count 2} :context :reuse} + (fn [rtx] + (swap! views conj (:view rtx)) + (camera/pan rtx {:dx 10 :dy 5 :steps 1 :settle-ms 0}))) + (let [prepared (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + request (transit/encode-str {:preparation-id (get-in prepared [:preparation :preparation-id])}) + first (transit/decode-str (await (browser/run-prepared request))) + second (transit/decode-str (await (browser/run-prepared request)))] + (t/is (= ["ok" "ok"] (mapv :status [first second]))) + (t/is (= [{:scale 1 :x 0 :y 0} {:scale 1 :x 10 :y 5}] @views)) + (t/is (= [[] []] (mapv #(get-in % [:phases 0 :events]) [first second]))))) + (finally + (cleanup))))))) + +(t/deftest ^:async diagnostic-encoding-keeps-phase-evidence-and-releases-its-owner + (await + (with-entry-env + (^:async fn [cleanup] + (try + (register-phase-case + {:params {:count 2} :context :reuse} + (fn [rtx] + (-> (camera/render-sync* rtx) + (.then (fn [_] + (throw (ex-info "main fixture failed" + {:command "fixture" :path "scene.transit" + :partial {:safe {:elapsed [0 js/Infinity js/NaN]} + :handle (fn [] nil)}}))))))) + (let [request (transit/encode-str (assoc (transit/decode-str (scene-args 7 {})) + :preparation-id "encoding")) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + response (await (browser/run-prepared (transit/encode-str {:preparation-id "encoding"}))) + evidence (transit/decode-str response) + attempt (measurement/attempt :rects/lifecycle "encoding" false evidence) + saved (-> (result/create-run + {:run-id "encoding" :started-at "start" + :plan {:cases [(load-case)] :warmups 0 :repetitions 1} + :metadata {:git {:sha nil :dirty nil} :build {:functional {}} + :environment {} :scored? true}}) + (result/record-preparation (assoc (:preparation prepared) :warmup? false)) + (result/record-attempt attempt) + (result/record-case-outcome :rects/lifecycle {:outcome :failed}) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode)] + (t/is (= "failed" (:status evidence))) + (t/is (= "main" (:phase evidence))) + (t/is (= "main fixture failed" (:message evidence))) + (t/is (= "main fixture failed" (get-in evidence [:cause 0 :message]))) + (t/is (= "fixture" (:command evidence))) + (t/is (= "scene.transit" (:path evidence))) + (t/is (= [::camera/render-sync] (mapv :command (get-in evidence [:phases 0 :events])))) + (t/is (= (:phases evidence) (:phases attempt))) + (t/is (= (:preparation prepared) (:preparation evidence))) + (t/is (js/Number.isNaN (get-in saved [:attempts 0 :partial :safe :elapsed 2]))) + (t/is (some #(= :encode-diagnostic (:operation %)) (get-in saved [:attempts 0 :failure :errors]))) + (t/is (= "invalid-args" + (:phase (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "encoding"})))))))) + (register-phase-case {:params {:count 2}} camera/render-sync*) + (t/is (= "ok" (:status (transit/decode-str (await (browser/load-scene (scene-args 8 {}))))))) + (finally + (cleanup))))))) + +(t/deftest ^:async malformed-preparation-partial-keeps-the-original-failure-and-setup + (await + (with-entry-env + (^:async fn [cleanup] + (try + (register-phase-case + {:params {:count 2} + :preparation {:callback (fn [rtx] + (.then (camera/render-sync* rtx) + (fn [_] (throw (ex-info "preparation failed" {:partial 42})))))}} + camera/render-sync*) + (let [request (transit/encode-str (assoc (transit/decode-str (scene-args 7 {})) + :preparation-id "malformed")) + data (transit/decode-str (await (browser/prepare-scene-request request)))] + (t/is (= "preparation failed" (:message data))) + (t/is (= "preparation" (:phase data))) + (t/is (= [:preparation] (mapv :phase (:phases data)))) + (t/is (number? (get-in data [:partial :setup :upload-ms]))) + (t/is (some #(= :encode-diagnostic (:operation %)) (:errors data))) + (t/is (= "invalid-args" + (:phase (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "malformed"})))))))) + (finally (cleanup))))))) + +(t/deftest ^:async response-encoding-failure-retains-completed-preparation-work + (await + (with-entry-env + (^:async fn [cleanup] + (let [create-element (.-createElement js/document)] + (try + (set! (.-createElement js/document) + (fn [_] + (let [canvas (fake-canvas 1 nil) + gl (.getContext canvas "webgl2")] + (aset gl "drawingBufferWidth" (fn [] nil)) + canvas))) + (let [request (transit/encode-str (assoc (transit/decode-str (scene-args 7 {})) + :preparation-id "fallback")) + data (transit/decode-str (await (browser/prepare-scene-request request)))] + (t/is (= "failed" (:status data))) + (t/is (= :encode-response (:operation data))) + (t/is (= [:preparation] (mapv :phase (:phases data)))) + (t/is (= :completed (get-in data [:phases 0 :outcome]))) + (t/is (seq (:cause data))) + (t/is (= "invalid-args" + (:phase (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "fallback"})))))))) + (set! (.-createElement js/document) create-element) + (t/is (= "ok" (:status (transit/decode-str (await (browser/load-scene (scene-args 8 {}))))))) + (finally + (set! (.-createElement js/document) create-element) + (cleanup)))))))) + +(t/deftest ^:async phase-failure-keeps-raw-evidence-and-stops-the-context + (doseq [context [:fresh :reuse] + failed-phase [:preparation :main :reset]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (unchecked-set js/globalThis "__benchNow" 0) + (let [visits (atom []) + work (fn [phase rtx] + (swap! visits conj phase) + (-> (camera/render-sync* rtx) + (.then (fn [next-rtx] + (when (= phase failed-phase) + (throw (js/Error. "phase fixture failed"))) + next-rtx))))] + (register-phase-case {:params {:count 2} :context context + :preparation {:callback #(work :preparation %)} + :reset #(work :reset %)} + #(work :main %)) + (let [request (-> (transit/decode-str (scene-args 7 {})) + (assoc :preparation-id "phase-failure") + (transit/encode-str)) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + run (transit/encode-str {:preparation-id "phase-failure"}) + evidence (if (= failed-phase :preparation) + prepared + (transit/decode-str (await (browser/run-prepared run)))) + stopped (transit/decode-str (await (browser/run-prepared run))) + blocks (:phases evidence) + diagnostic (errors/project-diagnostic evidence) + preparation (if (= failed-phase :preparation) + {:preparation-id "phase-failure" :case-id :rects/lifecycle + :warmup? false :diagnostics (get-in evidence [:partial :setup]) + :phases blocks :failure diagnostic} + (assoc (:preparation prepared) :warmup? false)) + saved (-> (cond-> + (-> (result/create-run + {:run-id "phase-failure" :started-at "start" + :plan {:cases [(load-case)] :warmups 0 :repetitions 1} + :metadata {:git {:sha nil :dirty nil} :build {:functional {}} + :environment {} :scored? true}}) + (result/record-preparation preparation)) + (or (= context :fresh) (not= failed-phase :preparation)) + (result/record-attempt + (measurement/attempt :rects/lifecycle "phase-failure" false + (if (= failed-phase :preparation) + (dissoc evidence :phases) + evidence)))) + (result/record-case-outcome :rects/lifecycle + {:outcome :failed}) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + stored (if (= failed-phase :preparation) + (first (:preparations saved)) + (first (:attempts saved)))] + (t/is (= "failed" (:status evidence))) + (t/is (= (name failed-phase) (:phase evidence))) + (t/is (= "phase fixture failed" (:message evidence))) + (t/is (= (:cause evidence) (get-in stored [:failure :cause]))) + (t/is (= blocks (:phases stored))) + (t/is (= "phase-failure" (:preparation-id stored))) + (t/is (= 1 (count (re-seq #"phase fixture failed" + (format/format-run (summary/summarize saved)))))) + (t/is (= (case failed-phase + :preparation [:preparation] + :main [:preparation :main] + :reset [:preparation :main :reset]) + @visits)) + (t/is (= (if (= failed-phase :reset) [:main :reset] [failed-phase]) + (mapv :phase blocks))) + (t/is (= :failed (:outcome (last blocks)))) + (t/is (= (repeat (count blocks) + [{:command ::camera/render-sync :metrics {:command/elapsed [0]}}]) + (map :events blocks))) + (when (= failed-phase :preparation) + (let [setup (get-in evidence [:partial :setup])] + (t/is (number? (:module-ms setup))) + (t/is (number? (:graphics-ms setup))) + (t/is (number? (:upload-ms setup))) + (t/is (string? (get-in setup [:fingerprint :digest]))))) + (when-not (= failed-phase :preparation) + (t/is (= (:preparation prepared) (:preparation evidence))) + (t/is (= :completed (get-in evidence [:preparation :phases 0 :outcome])))) + (t/is (= "invalid-args" (:phase stopped))) + (t/is (= 1 (effect-count (read-calls) "remove")))) + (register-phase-case {:params {:count 2}} camera/render-sync*) + (let [next-case (transit/decode-str (await (browser/load-scene (scene-args 8 {}))))] + (t/is (= "ok" (:status next-case))) + (t/is (= [::camera/render-sync] + (mapv :command (get-in next-case [:phases 0 :events])))))) + (finally + (cleanup)))))))) + +(t/deftest ^:async superseded-phase-failure-keeps-its-own-preparation + (doseq [failed-phase [:main :reset]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [entered (volatile! nil) + reject (volatile! nil) + started (js/Promise. (fn [resolve _reject] (vreset! entered resolve))) + work (fn [phase rtx] + (-> (camera/render-sync* rtx) + (.then (fn [next-rtx] + (if (= phase failed-phase) + (js/Promise. (fn [_resolve fail] + (vreset! reject fail) + (@entered nil))) + next-rtx)))))] + (register-phase-case {:params {:count 2} :context :reuse + :reset #(work :reset %)} + #(work :main %)) + (let [request (-> (transit/decode-str (scene-args 7 {})) + (assoc :preparation-id "first") + (transit/encode-str)) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + first-run (browser/run-prepared (transit/encode-str {:preparation-id "first"}))] + (await started) + (register-phase-case {:params {:count 2}} camera/render-sync*) + (let [next-request (-> (transit/decode-str (scene-args 8 {})) + (assoc :preparation-id "second") + (transit/encode-str)) + next-prepared (transit/decode-str (await (browser/prepare-scene-request next-request)))] + (@reject (js/Error. "late phase failure")) + (let [failed (transit/decode-str (await first-run)) + next-run (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "second"}))))] + (t/is (= "failed" (:status failed))) + (t/is (= (name failed-phase) (:phase failed))) + (t/is (= (:preparation prepared) (:preparation failed))) + (t/is (= (if (= failed-phase :main) [:main] [:main :reset]) + (mapv :phase (:phases failed)))) + (t/is (= :failed (:outcome (last (:phases failed))))) + (t/is (= (repeat (count (:phases failed)) "first") + (map :preparation-id (:phases failed)))) + (t/is (= ["ok" "ok"] (mapv :status [next-prepared next-run]))) + (t/is (= "second" (get-in next-run [:preparation :preparation-id]))))))) + (finally + (cleanup)))))))) + +(t/deftest rejected-camera-body-releases-owner + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" "_render_from_cache") + (register-phase-case {:context :reuse} + #(camera/pan* % {:dx 10 :dy 5 :steps 2 :settle-ms 0})) + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "main" (:phase m))) + (t/is (= 1 (effect-count (read-calls) "remove"))) + (t/is (nil? @@#'browser/canvas*)) + (cleanup) + (done))))))) + +(t/deftest rejected-import-resolves-module-init-failure + (t/async done + ;; Needs the entry env for the import polyfill, even though the import + ;; itself fails before touching the DOM. + (with-entry-env + (fn [cleanup] + (load-result (transit/encode-str + {:case (assoc (load-case) :scene-seed 7) + :module-url "data:text/javascript,this is not valid javascript((("}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "module-init" (:phase m))) + (t/is (= {} (get-in m [:partial :setup]))) + (cleanup) + (done))))))) + +(defn- assert-wasm-cause + "Shared cause assertions for WASM failures through `load-scene`: plain-data + vector, wire contract, failing fn name, fake message and WASM code." + [cause fn-name message-pattern] + (t/is (vector? cause) "cause is plain-data vector") + (t/is (decl/transit-round-trips? cause) "cause satisfies wire contract") + (t/is (some #(= fn-name (:fn %)) cause) "cause names the failing WASM fn") + (t/is (some #(and (string? (:message %)) + (re-find message-pattern (:message %))) + cause) + "fake message preserved") + (t/is (some #(let [code (:code %)] + (and (some? code) + (re-find #"wasm-critical" (str code)))) + cause) + "WASM code preserved")) + +(t/deftest set-browser-failure-releases-registered-context + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" "_set_browser") + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "graphics-init" (:phase m))) + (t/is (= {} (get-in m [:partial :setup]))) + (t/is (= "graphics init failed" (:message m)) + "top message stays phase-labeled") + (assert-wasm-cause (:cause m) + "_set_browser" + #"fake _set_browser failure") + (let [calls (read-calls)] + (t/is (= 1 (effect-count calls "register"))) + (t/is (= 1 (effect-count calls "clean")) "renderer state is released") + (t/is (some #{["delete" 7]} calls) "the registered handle is deleted") + (t/is (= 1 (effect-count calls "lose")) "the browser context is released") + (t/is (= 1 (effect-count calls "remove")) "the canvas is removed")) + (t/is (nil? @@#'browser/canvas*) "no owner left behind") + (cleanup) + (done))))))) + +(t/deftest upload-failure-keeps-wasm-cause + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" "_set_view") + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "upload" (:phase m))) + (t/is (= "upload failed" (:message m)) + "top message stays phase-labeled") + (assert-wasm-cause (:cause m) + "_set_view" + #"fake _set_view failure") + (let [setup (get-in m [:partial :setup])] + (t/is (number? (:module-ms setup))) + (t/is (number? (:graphics-ms setup))) + (t/is (string? (get-in setup [:fingerprint :digest])))) + (cleanup) + (done))))))) + +(t/deftest ^:async upload-retains-batch-and-end-loading-failures-in-order + (doseq [functions [["_set_shapes_batch" "_end_loading"] ["_end_loading"]]] + (await + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" (clj->js functions)) + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "upload failed" (:message m))) + (t/is (= (mapv {"_set_shapes_batch" :serialize-shapes "_end_loading" :end-loading} functions) + (mapv :operation (:errors m)))) + (t/is (= functions (mapv #(get-in % [:cause 0 :fn]) (:errors m)))) + (t/is (decl/transit-round-trips? m)) + (cleanup)))))))) + +(t/deftest nested-stale-marker-does-not-resolve-stale + (let [terminate @#'browser/terminal-failure + current @@#'browser/owner-epoch* + nested (ex-info "upload failed" + {:phase "upload"} + (ex-info "superseded" + {::errors/stale true + :phase "upload"})) + result (terminate current nested "aborted") + m result] + (t/is (= "failed" (:status m)) "only the top level decides staleness") + (t/is (= "upload" (:phase m))) + (t/is (vector? (:cause m)) "cause still rendered"))) + +(t/deftest overlapping-load-keeps-the-newer-owner + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchGateRaf" true) + (let [args (fn [seed] (scene-args seed {})) + first-load (browser/load-scene (args 7))] + ;; Microtasks drain before this macrotask: the first load parked on + ;; its first frame with a live canvas. Macrotask ordering + ;; (never wall timing) sequences the assertions. + (js/setTimeout + (fn [] + (t/is (some? (unchecked-get js/globalThis "__benchRafCb")) + "first load is waiting for a frame") + (let [second-load (browser/load-scene (args 8))] + (js/setTimeout + (fn [] + ;; The second load completed; release the parked frame so + ;; the first load observes its superseded epoch. + (if-some [parked (unchecked-get js/globalThis "__benchRafCb")] + (parked 0) + (t/is false "first load never parked on a frame")) + (-> (js/Promise.all #js [first-load second-load]) + (.then (fn [results] + (let [[stale ok] (mapv transit/decode-str (array-seq results))] + (t/is (= "stale" (:status stale)) "superseded load resolves stale") + (t/is (= "ok" (:status ok)) "newer owner completes") + (t/is (= 8 (:seed (:scene ok))) "the completer is the second load") + (let [calls (read-calls)] + (t/is (= 1 (effect-count calls "remove")) "superseded canvas is removed") + (t/is (some #{["remove" 1]} calls)) + (t/is (nil? (some #{["remove" 2]} calls)) "newer canvas survives")) + (t/is (some? @@#'browser/canvas*) "the newer owner stays live") + (t/is (true? wasm/context-initialized?))) + (cleanup) + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (cleanup) + (done))))) + 0))) + 0)))))) diff --git a/frontend/test/performance_tests/render_wasm/log_test.cljs b/frontend/test/performance_tests/render_wasm/log_test.cljs new file mode 100644 index 0000000000..d12468acc7 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/log_test.cljs @@ -0,0 +1,161 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.log-test + (:require + ["fs" :as fs] + ["node:stream" :as stream] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.node.log :as log] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "bench-log-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- capture + [] + (let [text (atom "") + out (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))})] + {:stream out :text text})) + +(t/deftest levels-filter-console-and-retain-every-line-in-the-file + (doseq [[level expected] [[:warn "! warning\n✖ failure\n"] + [:info "Phase\n attempt 1/1 completed\n! warning\n✖ failure\n"] + [:debug "Phase\n attempt 1/1 completed\n· raw child\n! warning\n✖ failure\n"]]] + (let [stdout (capture) + stderr (capture) + file (path/join @temporary-root (name level))] + (log/configure {:level level :color :never + :stdout (:stream stdout) :stderr (:stream stderr)}) + (binding [log/*log-path* file] + (log/info "Phase") + (log/info " attempt 1/1 completed") + (log/debug "raw child") + (log/warn "warning") + (log/error {:message "failure" :tail? false}) + (log/result "Run saved")) + (t/is (= "Run saved\n" @(:text stdout))) + (t/is (= (str/replace expected "✖ failure\n" (str "✖ failure\n log " file "\n")) + @(:text stderr))) + (t/is (= (str "Phase\n attempt 1/1 completed\nraw child\nwarning\n✖ failure\n log " + file "\nRun saved\n") + (fs/readFileSync file "utf8")))))) + +(t/deftest colors-apply-to-console-headings-and-preserve-indented-progress + (doseq [[mode tty no-color term colored?] + [[:always false "1" "dumb" true] [:never true "" "xterm" false] + [:auto false "" "xterm" false] [:auto true "" "xterm" true] + [:auto true "1" "xterm" false] [:auto true "" "dumb" false]]] + (let [stdout (capture) + stderr (capture) + file (path/join @temporary-root (str (random-uuid))) + env (.-env js/process) + old-no-color (.-NO_COLOR env) + old-term (.-TERM env)] + (try + (set! (.-NO_COLOR env) no-color) + (set! (.-TERM env) term) + (set! (.-isTTY ^js (:stream stderr)) tty) + (log/configure {:color mode :stdout (:stream stdout) :stderr (:stream stderr)}) + (binding [log/*log-path* file] + (log/info "Phase\n progress") + (log/result "Output /tmp/run")) + (t/is (= colored? (str/includes? @(:text stderr) "\u001b["))) + (t/is (= "Phase\n progress\n" + (str/replace @(:text stderr) #"\u001b\[[0-9;]*m" ""))) + (t/is (= "Output /tmp/run\n" @(:text stdout))) + (t/is (= "Phase\n progress\nOutput /tmp/run\n" (fs/readFileSync file "utf8"))) + (finally + (if (nil? old-no-color) (js-delete env "NO_COLOR") (set! (.-NO_COLOR env) old-no-color)) + (if (nil? old-term) (js-delete env "TERM") (set! (.-TERM env) old-term))))))) + +(t/deftest ^:async nested-tasks-route-lines-to-their-own-files-on-success-and-failure + (let [parent (path/join @temporary-root "comparison.log") + child (path/join @temporary-root "build.log") + output (capture)] + (log/configure {:level :warn :stdout (:stream output) :stderr (:stream output)}) + (binding [log/*log-path* parent] + (log/debug "comparison before") + (await ((^:async fn [] + (binding [log/*log-path* child] + (await (timers/setImmediate)) + (log/debug "child success"))))) + (log/debug "comparison between") + (try + (await ((^:async fn [] + (binding [log/*log-path* child] + (await (timers/setImmediate)) + (log/debug "child failure") + (throw (js/Error. "failed")))))) + (catch :default _ (log/debug "comparison after")))) + (t/is (= "comparison before\ncomparison between\ncomparison after\n" + (fs/readFileSync parent "utf8"))) + (t/is (= "child success\nchild failure\n" (fs/readFileSync child "utf8"))))) + +(t/deftest console-diagnostics-escape-controls-and-retain-the-log-evidence + (let [file (path/join @temporary-root "controls.log") + output (capture) + message (str "work failed\u001b\u0007" (apply str (repeat 2000 "x")))] + (log/configure {:color :never :stderr (:stream output)}) + (binding [log/*log-path* file] + (log/warn message)) + (t/is (= (str message "\n") (fs/readFileSync file "utf8"))) + (t/is (str/includes? @(:text output) "\\u001b\\u0007")) + (t/is (< (count @(:text output)) 1100)) + (doseq [separator ["" "\n"]] + (reset! (:text output) "") + (fs/writeFileSync file (str (apply str (repeat 2000 "\u0007")) separator "FATAL: compiler failed\n") "utf8") + (log/error {:message "tail evidence" :tail-log file}) + (t/is (str/includes? @(:text output) "FATAL: compiler failed")) + (t/is (every? #(let [code (.charCodeAt % 0)] + (or (= code 10) (and (>= code 32) (or (< code 127) (> code 159))))) + (seq @(:text output))))))) + +(t/deftest error-names-tail-source-and-bounds-evidence-before-appending + (let [file (path/join @temporary-root "comparison.log") + child (path/join @temporary-root "build.log") + output (capture)] + (fs/writeFileSync child (str "excluded\n" (apply str (repeat 9000 "x")) "\n" + (str/join "\n" (map #(str "line-" %) (range 30)))) "utf8") + (log/configure {:color :never :stderr (:stream output)}) + (binding [log/*log-path* file] + (log/error {:message "build failed" :fields [[:phase :renderer] [:build "id"]] + :tail-log child})) + (let [expected (str "✖ build failed\n phase :renderer\n build id\n" + " log " file "\n tail-log " child "\n tail\n" + (str/join "\n" (map #(str " line-" %) (range 10 30))) "\n")] + (t/is (= expected @(:text output))) + (t/is (= expected (fs/readFileSync file "utf8")))) + (fs/writeFileSync child (apply str (repeat 10000 "z")) "utf8") + (reset! (:text output) "") + (binding [log/*log-path* file] + (log/error {:message "bounded" :tail-log child})) + (t/is (= 8192 (count (re-find #"z+$" (str/trim @(:text output)))))) + (fs/writeFileSync file "last line\n" "utf8") + (reset! (:text output) "") + (binding [log/*log-path* file] + (log/error {:message "same source"})) + (t/is (= (str "✖ same source\n log " file "\n tail\n last line\n") + @(:text output))))) + +(t/deftest error-reporting-survives-unreadable-tail-and-file-append-failure + (let [output (capture)] + (log/configure {:color :never :stderr (:stream output)}) + (binding [log/*log-path* @temporary-root] + (log/error {:message "original failure" :tail-log (path/join @temporary-root "missing")})) + (t/is (str/starts-with? @(:text output) "✖ original failure\n")) + (t/is (str/includes? @(:text output) @temporary-root)))) diff --git a/frontend/test/performance_tests/render_wasm/merge_test.cljs b/frontend/test/performance_tests/render_wasm/merge_test.cljs new file mode 100644 index 0000000000..8e4ee8e042 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/merge_test.cljs @@ -0,0 +1,219 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.merge-test + (:require + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true])) + +(defn- metric + [value] + {:command :fixture/work :metrics {:command/elapsed [value]}}) + +(defn- case-definition + [case-id] + {:id case-id + :scene :case + :scene-description "merge scene" + :scene-seed 7 + :params {:count 10} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {}}) + +(defn- preparation + [preparation-id case-id] + {:preparation-id preparation-id + :case-id case-id :warmup? false + :fingerprint {:algorithm :sha-256 + :digest (apply str (repeat 64 "a"))} + :effective-graphics {:webgl2 true} + :diagnostics {} + :phases [{:phase :preparation :preparation-id preparation-id :outcome :completed :events []}]}) + +(defn- attempt + [case-id preparation-id warmup?] + {:case-id case-id + :preparation-id preparation-id + :warmup? warmup? + :outcome :completed + :phases [{:phase :main :preparation-id preparation-id + :outcome :completed :events [(metric 1.0)]}]}) + +(defn- make-run + [{:keys [run-id cases warmups repetitions git-sha build environment analysis + timeout-ms started-at ended-at termination preparations] + :or {git-sha "same-sha" + build {:mode :release} + environment {:node "v24"} + timeout-ms 30000 + started-at "2026-10-26T00:00:00.000Z" + ended-at "2026-10-26T00:01:00.000Z" + termination {:reason :completed}}}] + (let [run (result/create-run + {:run-id run-id + :started-at started-at + :plan {:cases (mapv case-definition cases) + :warmups warmups + :repetitions repetitions + :timeout-policy {:case-ms timeout-ms}} + :metadata {:git {:sha git-sha :dirty false} + :build {:functional build} + :environment environment + :scored? true} + :analysis analysis}) + run (reduce + (fn [run case-id] + (let [options (get preparations case-id) + warmup-id (or (:warmup options) + (str run-id "-" (name case-id))) + measured-id (or (:measured options) warmup-id)] + (-> run + (result/record-preparation (preparation warmup-id case-id)) + (cond-> (not= warmup-id measured-id) + (result/record-preparation (preparation measured-id case-id))) + (as-> run + (reduce (fn [run _] + (result/record-attempt + run (attempt case-id warmup-id true))) + run (range warmups)) + (reduce (fn [run _] + (result/record-attempt + run (attempt case-id measured-id false))) + run (range repetitions)))))) + run cases) + run (if (= :completed (:reason termination)) + (reduce #(result/record-case-outcome %1 %2 {:outcome :completed}) run cases) + run)] + (result/finish-run run ended-at termination))) + +(t/deftest merges-measured-attempts-and-updates-the-plan + (let [base (make-run {:run-id "base" + :cases [:case/load :case/pan] + :warmups 1 + :repetitions 2 + :started-at "2026-10-26T00:00:00.000Z" + :ended-at "2026-10-26T00:01:00.000Z"}) + additional (make-run {:run-id "additional" + :cases [:case/load :case/pan] + :warmups 1 + :repetitions 3 + :started-at "2026-10-26T00:02:00.000Z" + :ended-at "2026-10-26T00:03:00.000Z"}) + merged (result/merge-runs base additional {:run-id "merged"})] + (t/is (= "merged" (:run-id merged))) + (t/is (= 1 (get-in merged [:plan :warmups]))) + (t/is (= 5 (get-in merged [:plan :repetitions]))) + (t/is (= 12 (count (:attempts merged)))) + (t/is (= (range 12) (map :attempt-id (:attempts merged)))) + (t/is (= [true false false true false false false false false false false false] + (map :warmup? (:attempts merged)))) + (t/is (every? #(every? zero? (vals %)) (vals (:remaining merged)))) + (t/is (= "2026-10-26T00:03:00.000Z" (:ended-at merged))) + (t/is (= 2 (get-in merged [:metadata :diagnostics :merge + :additional-warmups-excluded]))) + (t/is (= merged (result/check-run merged))) + (t/is (= (:attempts merged) + (:attempts (result/decode (result/encode merged))))))) + +(t/deftest excludes-additional-warmup-only-preparations + (let [base (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1}) + additional (make-run {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :preparations + {:case/load {:warmup "warm-prep" + :measured "measured-prep"}}}) + merged (result/merge-runs base additional {:run-id "merged"}) + preparation-ids (map :preparation-id (:preparations merged))] + (t/is (some #{"base-load"} preparation-ids)) + (t/is (some #{"measured-prep"} preparation-ids)) + (t/is (not (some #{"warm-prep"} preparation-ids))) + (t/is (= 3 (count (:attempts merged)))) + (t/is (= "measured-prep" (:preparation-id (last (:attempts merged))))))) + +(t/deftest rejects-incompatible-runs + (let [base (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1})] + (doseq [[label options message] + [["case" {:cases [:case/pan]} #"different case definitions"] + ["timeout" {:timeout-ms 1000} #"different timeout policy"] + ["warmups" {:warmups 0} #"different warmup count"] + ["analysis" {:analysis {:draws 1000}} #"different analysis settings"] + ["git" {:git-sha "other"} #"different git provenance"] + ["build" {:build {:mode :debug}} #"different build facts"] + ["environment" {:environment {:node "v25"}} + #"different runtime environment"]]] + (t/testing label + (let [additional (make-run (merge {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1} + options))] + (t/is (thrown-with-msg? js/Error message + (result/merge-runs base additional + {:run-id "merged"})))))))) + +(t/deftest rejects-non-completed-runs + (let [completed (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1}) + interrupted (make-run {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :termination {:reason :interrupted}})] + (t/is (thrown-with-msg? js/Error #"only completed runs can be merged" + (result/merge-runs completed interrupted + {:run-id "merged"}))) + (t/is (thrown-with-msg? js/Error #"only completed runs can be merged" + (result/merge-runs interrupted completed + {:run-id "merged"}))))) + +(t/deftest rejects-duplicate-preparation-ids + (let [base (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :preparations + {:case/load {:warmup "shared" + :measured "shared"}}}) + additional (make-run {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :preparations + {:case/load {:warmup "shared" + :measured "shared"}}})] + (t/is (thrown-with-msg? js/Error #"duplicate preparation ids" + (result/merge-runs base additional + {:run-id "merged"}))))) + +(t/deftest quantity-prose-does-not-change-merge-compatibility + (let [base (make-run {:run-id "base" :cases [:case/load] :warmups 0 :repetitions 1}) + additional (-> (make-run {:run-id "additional" :cases [:case/load] :warmups 0 :repetitions 1}) + (assoc-in [:quantities :command/elapsed :description] "Elapsed command work.")) + merged (result/merge-runs base additional {:run-id "merged"})] + (t/is (= 2 (get-in merged [:plan :repetitions]))) + (t/is (= merged (result/decode (result/encode merged)))) + (let [malformed (assoc-in additional [:quantities :command/elapsed :description] 42) + report (summary/summarize malformed)] + (t/is (= :malformed-quantity + (get-in report [:cases 0 :metrics [:main :fixture/work :command/elapsed] :diagnostics 0 :reason]))) + (t/is (thrown-with-msg? js/Error #"malformed quantity descriptions" + (result/merge-runs base malformed {:run-id "invalid"})))) + (t/is (thrown-with-msg? js/Error #"different quantity units" + (result/merge-runs base + (assoc-in additional [:quantities :command/elapsed :unit] :seconds) + {:run-id "invalid"}))))) diff --git a/frontend/test/performance_tests/render_wasm/outcome_test.cljs b/frontend/test/performance_tests/render_wasm/outcome_test.cljs new file mode 100644 index 0000000000..e9ad786d34 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/outcome_test.cljs @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.outcome-test + (:require + [benches.render-wasm.node.outcome :as outcome] + [cljs.test :as t :include-macros true])) + +(t/deftest failed-reports-the-cause-with-exit-code-1-and-its-log-path + (let [cause (ex-info "boom" {:log-path "build.log"}) + result (outcome/failed cause {:home :h})] + (t/is (= [cause] (:errors result))) + (t/is (= 1 (:exit-code result))) + (t/is (= "build.log" (:log-path result))) + (t/is (= :h (:home result))))) + +(t/deftest failed-prefers-the-log-path-given-in-data + (let [result (outcome/failed (ex-info "boom" {:log-path "build.log"}) {:log-path "run.log"})] + (t/is (= "run.log" (:log-path result))))) + +(t/deftest add-error-appends-and-keeps-interruption-exit-codes + (doseq [[before after] [[0 1] [1 1] [130 130] [143 143]]] + (let [first-cause (js/Error. "first") + second-cause (js/Error. "second") + result (-> {:errors [first-cause] :exit-code before} + (outcome/add-error second-cause))] + (t/is (= [first-cause second-cause] (:errors result))) + (t/is (= after (:exit-code result)))))) + +(t/deftest publish-returns-the-outcome-when-printing-succeeds + (let [printed (atom nil) + result (outcome/publish {:exit-code 0} #(reset! printed :done))] + (t/is (= :done @printed)) + (t/is (= {:exit-code 0} result)))) + +(t/deftest publish-adds-a-failed-print-to-the-outcome + (let [cause (js/Error. "stream closed") + result (outcome/publish {:exit-code 0} #(throw cause))] + (t/is (= [cause] (:errors result))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest interruption-code-maps-signal-names + (t/is (= 143 (outcome/interruption-code "SIGTERM"))) + (t/is (= 130 (outcome/interruption-code "SIGINT")))) diff --git a/frontend/test/performance_tests/render_wasm/process_test.cljs b/frontend/test/performance_tests/render_wasm/process_test.cljs new file mode 100644 index 0000000000..65862e7bb8 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/process_test.cljs @@ -0,0 +1,615 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.process-test + "Real Node/POSIX processes, files and Linux flock ownership." + (:require + ["child_process" :as child-process] + ["fs" :as fs] + ["node:module" :as node-module] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.process :as process] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private temp-root (atom nil)) +(def ^:private test-grace-ms 100) + +(defn- file + [name] + (path/join @temp-root name)) + +(defn- pause + [ms] + (js/Promise. (fn [resolve _] (js/setTimeout resolve ms)))) + +(defn- within + "A broken lifecycle must fail the test rather than stall the test runner." + ([promise] (within promise 5000)) + ([promise timeout-ms] + (let [timer (volatile! nil)] + (-> (js/Promise.race + #js [promise + (js/Promise. + (fn [_ reject] + (vreset! timer + (js/setTimeout + #(reject (ex-info "Process test stalled" {:type :test-watchdog})) + timeout-ms))))]) + (.finally #(js/clearTimeout @timer)))))) + +(defn- ^:async with-live-group + [faults f] + (let [^js native-fs ((node-module/createRequire (path/join (.cwd js/process) "package.json")) "fs") + kill (.-kill js/process) + read-dir (.-readdirSync native-fs) + read-file (.-readFileSync native-fs) + group (atom nil) + signals (atom []) + probed (atom false) + fake-pid "2147483646"] + (set! (.-kill js/process) + (fn [pid signal] + (if (neg? pid) + (do + (reset! group (- pid)) + (when-not (:keep-child faults) + (try + (.call kill js/process pid signal) + (catch :default cause + (when-not (= "ESRCH" (.-code cause)) (throw cause))))) + (when (string? signal) + (swap! signals conj {:signal signal :time-ms (js/Date.now)})) + (when-let [cause (get faults signal)] (throw cause)) + true) + (.call kill js/process pid signal)))) + (set! (.-readdirSync native-fs) + (fn [location & args] + (if (and (= "/proc" location) @group) + (if (and (:probe faults) (compare-and-set! probed false true)) + (throw (:probe faults)) + (.concat (apply read-dir location args) #js [fake-pid])) + (apply read-dir location args)))) + (set! (.-readFileSync native-fs) + (fn [location & args] + (if (and @group (= location (str "/proc/" fake-pid "/stat"))) + (str fake-pid " (unresolved) S 0 " @group " " @group " 0") + (apply read-file location args)))) + (node-module/syncBuiltinESMExports) + (try + (await (f signals)) + (finally + (when (and (:keep-child faults) @group) + (try + (.call kill js/process (- @group) "SIGKILL") + (catch :default cause + (when-not (= "ESRCH" (.-code cause)) (throw cause))))) + (set! (.-kill js/process) kill) + (set! (.-readdirSync native-fs) read-dir) + (set! (.-readFileSync native-fs) read-file) + (node-module/syncBuiltinESMExports))))) + +(defn- ^:async wait-for + [pred] + (let [limit (process/deadline 3000)] + (loop [] + (when-not (pred) + (when (zero? (process/remaining limit)) + (throw (ex-info "Process test did not become ready" {:type :test-watchdog}))) + (await (pause 10)) + (recur))))) + +(defn- process-status + [pid] + (try + (let [stat (fs/readFileSync (str "/proc/" pid "/stat") "utf8") + fields (str/split (str/trim (subs stat (inc (.lastIndexOf stat ")")))) #"\s+")] + {:state (first fields) + :group (js/Number (nth fields 2))}) + (catch :default cause + (if (contains? #{"ENOENT" "ESRCH"} (.-code cause)) + nil + (throw cause))))) + +(defn- running? + [pid] + (when-let [status (process-status pid)] + ;; An orphan can remain as a zombie until init reaps it. It owns no pipes + ;; or locks and cannot run; kill(pid, 0) alone does not distinguish it. + (not (contains? #{"Z" "X" "x"} (:state status))))) + +(defn- read-pid + [name] + (js/Number (fs/readFileSync (file name) "utf8"))) + +(t/use-fixtures + :each + {:before + (fn [] + (reset! temp-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-wasm-process-test-")))) + :after + (fn [] + (t/async done + (let [cleanup (^:async fn [] + (try + (let [pids (->> (array-seq (fs/readdirSync @temp-root)) + (filter #(str/ends-with? % ".pid")) + (mapv read-pid))] + ;; Also clean up when an assertion or watchdog exposes a regression. + ;; Signal individual recorded PIDs so a missing detached flag cannot + ;; make test teardown kill the test runner's own process group. + (doseq [pid pids] + (when (running? pid) + (try + (.kill js/process pid "SIGKILL") + (catch :default cause + (when-not (= "ESRCH" (.-code cause)) + (throw cause)))))) + (await (wait-for #(not-any? running? pids)))) + (finally + (try + (fs/rmSync @temp-root #js {:recursive true :force true}) + (finally + (reset! temp-root nil))))))] + (.then (cleanup) + (fn [_] (done)) + (fn [cause] + (try + (t/is false (str "Process test cleanup failed: " cause)) + (finally + (done))))))))}) + +(defn- node-source + [pid-file source] + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(" (pr-str pid-file) ", String(process.pid));\n" + source)) + +(defn- node-options + [source overrides] + (merge {:command (.-execPath js/process) + :args ["-e" (node-source "root.pid" source)] + :cwd @temp-root + :deadline (process/deadline 3000) + :termination-grace-ms test-grace-ms} + overrides)) + +(t/deftest ^:async unresolved-group-termination-is-bounded-and-fatal + (await + (with-live-group + {} + (^:async fn [signals] + (let [started (js/Date.now) + cause (try + (await (within (process/run (node-options "process.exit(0);" {:termination-grace-ms 20})) 8000)) + nil + (catch :default cause cause)) + data (ex-data cause)] + (t/is (errors/fatal? cause)) + (t/is (= :process-unresolved (:type data))) + (t/is (= (.-execPath js/process) (:command data))) + (t/is (integer? (:pid data))) + (t/is (= (:pid data) (:pgid data))) + (t/is (= ["SIGTERM" "SIGKILL"] (:signals data))) + (t/is (= ["SIGTERM" "SIGKILL"] (mapv :signal @signals))) + (let [killed-at (some #(when (= "SIGKILL" (:signal %)) (:time-ms %)) @signals)] + (t/is (and (number? killed-at) (>= (- (js/Date.now) killed-at) 5000)))) + (t/is (< (- (js/Date.now) started) 8000)) + (t/is (not (running? (read-pid "root.pid"))))))))) + + +(t/deftest ^:async unresolved-owner-retains-a-live-lock-and-all-failures + (let [lock-file (file "unresolved.lock") + messages ["output handler failed" "TERM denied" "probe unavailable" "KILL denied"]] + (await + (with-live-group + {"SIGTERM" (js/Error. (nth messages 1)) + :probe (js/Error. (nth messages 2)) + "SIGKILL" (js/Error. (nth messages 3)) + :keep-child true} + (^:async fn [signals] + (let [options (node-options "" + {:command "flock" + :args ["--exclusive" lock-file (.-execPath js/process) "-e" + (node-source "root.pid" "process.stdout.write('ready'); setInterval(() => {}, 1000);")] + :deadline (process/deadline 30000) + :termination-grace-ms 20 + :on-output (fn [& _] (throw (js/Error. (first messages))))}) + cause (try + (await (within (process/run options) 8000)) + nil + (catch :default cause cause)) + entries (errors/get-errors cause) + wire (-> (errors/evidence :process cause) transit/encode-str transit/decode-str)] + (t/is (process/unresolved? cause)) + (t/is (errors/fatal? cause)) + (t/is (= [:process-output :process-signal :process-probe :process-signal :process-unresolved] + (mapv #(-> % :error ex-data :type) entries))) + (doseq [[entry message] (map vector (:errors wire) messages)] + (t/is (some #(= message (:message %)) (:cause entry)))) + (t/is (true? (:unresolved-process wire))) + (t/is (true? (:fatal wire))) + (t/is (= ["SIGTERM" "SIGKILL"] (mapv :signal @signals))) + (t/is (running? (read-pid "root.pid"))) + (t/is (= 1 (.-status (child-process/spawnSync "flock" #js ["--nonblock" lock-file "true"] + #js {:timeout 1000})))))))) + (await (wait-for #(zero? (.-status (child-process/spawnSync "flock" #js ["--nonblock" lock-file "true"] + #js {:timeout 1000}))))))) + + +(t/deftest ^:async input-stream-errors-have-one-occurrence-each-with-line-output + (let [^js native ((node-module/createRequire (path/join (.cwd js/process) "package.json")) "child_process") + spawn (.-spawn native)] + (set! (.-spawn native) + (fn [& args] + (let [^js child (apply spawn args)] + (.once (.-stdout child) "data" + (fn [_] + (.emit (.-stdout child) "error" (js/Error. "first read failed")) + (.emit (.-stdout child) "error" (js/Error. "second read failed")))) + child))) + (node-module/syncBuiltinESMExports) + (try + (let [cause (try + (await (within (process/run + (node-options "process.stdout.write('ready'); setInterval(() => {}, 1000);" + {:on-line (fn [& _] nil)})))) + nil + (catch :default cause cause)) + wire (errors/evidence :process cause)] + (t/is (= [:process-output :process-output] + (mapv #(-> % :error ex-data :type) (errors/get-errors cause)))) + (doseq [[entry message] (map vector (:errors wire) ["first read failed" "second read failed"])] + (t/is (some #(= message (:message %)) (:cause entry))))) + (finally + (set! (.-spawn native) spawn) + (node-module/syncBuiltinESMExports))))) + +(defn- ^:async rejected + [promise] + (try + (let [result (await (within promise))] + (t/is false (str "Expected a process failure, got " (pr-str result))) + nil) + (catch :default cause + (t/is (some? (ex-data cause)) "failures carry ex-info data") + (ex-data cause)))) + +(defn- stubborn-descendant + [] + (let [grandchild (node-source + "grandchild.pid" + (str "process.on('SIGTERM', () => fs.writeFileSync('grandchild.term', 'seen'));\n" + "fs.writeFileSync('grandchild.ready', 'ready');\n" + "setInterval(() => {}, 1000);\n"))] + (node-source + "descendant.pid" + (str "process.on('SIGTERM', () => fs.writeFileSync('descendant.term', 'seen'));\n" + "require('node:child_process').spawn(process.execPath,\n" + " ['-e', " (pr-str grandchild) "], {stdio: ['ignore', 'inherit', 'inherit']});\n" + "const ready = setInterval(() => {\n" + " if (fs.existsSync('grandchild.ready')) {\n" + " clearInterval(ready); fs.writeFileSync('descendant.ready', 'ready');\n" + " process.stdout.write('descendant-ready\\n');\n" + " }\n" + "}, 10);\n" + "setInterval(() => fs.appendFileSync('heartbeat', '.'), 10);\n")))) + +(defn- spawn-descendant + [] + (str "const child = require('node:child_process').spawn(process.execPath,\n" + " ['-e', " (pr-str (stubborn-descendant)) "],\n" + " {stdio: ['ignore', 'inherit', 'inherit']});\n")) + +(defn- ^:async assert-tree-stopped + [] + (t/is (not (running? (read-pid "root.pid"))) "the owner has stopped") + (t/is (not (running? (read-pid "descendant.pid"))) "the descendant has stopped at settlement") + (t/is (not (running? (read-pid "grandchild.pid"))) "the grandchild has stopped at settlement") + (t/is (= "seen" (fs/readFileSync (file "descendant.term") "utf8")) + "the owned group receives SIGTERM before SIGKILL") + (t/is (= "seen" (fs/readFileSync (file "grandchild.term") "utf8")) + "SIGTERM also reaches the grandchild") + (let [heartbeat (when (fs/existsSync (file "heartbeat")) + (fs/readFileSync (file "heartbeat") "utf8"))] + (await (pause 30)) + (t/is (= heartbeat (when (fs/existsSync (file "heartbeat")) + (fs/readFileSync (file "heartbeat") "utf8"))) + "no descendant writes after the promise settles"))) + +(t/deftest ^:async deadlines-use-date-now-and-clamp-at-zero + (let [before (js/Date.now) + limit (process/deadline 20)] + (t/is (<= (+ before 20) limit (+ (js/Date.now) 20))) + (t/is (<= 0 (process/remaining limit) 20)) + (await (pause 30)) + (t/is (zero? (process/remaining limit))) + (t/is (zero? (process/remaining (- before 1000)))))) + +(t/deftest ^:async uses-real-cwd-literal-args-env-and-closed-stdin + (let [source (str "process.stdin.resume();\n" + "process.stdin.on('end', () => {\n" + " const fact = {cwd: process.cwd(), args: process.argv.slice(1),\n" + " value: process.env.RENDERER_PROCESS_TEST, path: process.env.PATH || null};\n" + " fs.writeFileSync('fact.json', JSON.stringify(fact));\n" + " process.stdout.write(JSON.stringify(fact));\n" + " process.stderr.write('stdin-ended\\n');\n" + "});\n") + args ["space ; $(touch unexpected)" "--literal"]] + (doseq [env [{:RENDERER_PROCESS_TEST "map value"} + #js {:RENDERER_PROCESS_TEST "map value"}]] + (let [options (node-options source {:env env}) + result (await (within (process/run (update options :args into args)))) + fact (js->clj (js/JSON.parse (:stdout result)) :keywordize-keys true)] + (t/is (= 0 (:exit-code result))) + (t/is (= "stdin-ended\n" (:stderr result))) + (t/is (= {:cwd @temp-root + :args args + :value "map value" + :path (or (.-PATH (.-env js/process)) nil)} + fact)) + (t/is (= (:stdout result) (fs/readFileSync (file "fact.json") "utf8"))) + (t/is (not (fs/existsSync (file "unexpected"))) "arguments do not pass through a shell"))))) + +(t/deftest ^:async routes-both-streams-before-the-child-can-finish + (let [seen (atom {:stdout "" :stderr ""}) + source (str "process.stdout.write('ready\\n');\n" + "process.stderr.write('warning\\n');\n" + "const timer = setInterval(() => {\n" + " if (fs.existsSync('ack')) {\n" + " clearInterval(timer); fs.writeFileSync('done', 'yes');\n" + " process.stdout.write('complete\\n');\n" + " }\n" + "}, 10);\n") + result (await + (within + (process/run + (node-options + source + {:on-output + (fn [text stream] + (swap! seen update stream str text) + (when (str/includes? (:stdout @seen) "ready\n") + (fs/writeFileSync (file "ack") "yes")))}))))] + (t/is (= {:exit-code 0 :stdout "ready\ncomplete\n" :stderr "warning\n"} result)) + (t/is (= (select-keys result [:stdout :stderr]) @seen)) + (t/is (= "yes" (fs/readFileSync (file "done") "utf8"))))) + +(t/deftest ^:async lines-keep-streams-separate-and-flush-final-fragments + (let [seen (atom {:stdout [] :stderr []}) + source (str "process.stdout.write('out-'); process.stderr.write('err-');" + "setTimeout(() => { process.stdout.write('done\\r');" + "process.stderr.write('done\\nlast-err');" + "setTimeout(() => process.stdout.write('\\nlast-out'), 20); }, 20);") + result (await (process/run + (node-options source + {:capture-output? false + :on-line (fn [line stream] + (swap! seen update stream conj line))})))] + (t/is (= {:stdout ["out-done" "last-out"] :stderr ["err-done" "last-err"]} @seen)) + (t/is (= {:exit-code 0 :stdout "" :stderr ""} result)))) + +(t/deftest ^:async decodes-utf8-across-real-pipe-chunks + (let [source (str "const bytes = Buffer.from('Ω😀');\n" + "process.stdout.write(bytes.subarray(0, 1));\n" + "setTimeout(() => process.stdout.write(bytes.subarray(1)), 20);\n") + result (await (within (process/run (node-options source {}))))] + (t/is (= "Ω😀" (:stdout result))))) + +(t/deftest ^:async streams-large-output-without-retaining-it + (let [sizes (atom {:stdout 0 :stderr 0}) + source (str "process.stdout.write('x'.repeat(256 * 1024));\n" + "process.stderr.write('y'.repeat(128 * 1024));\n") + result (await + (within + (process/run + (node-options source + {:capture-output? false + :on-output (fn [text stream] + (swap! sizes update stream + (count text)))}))))] + (t/is (= {:exit-code 0 :stdout "" :stderr ""} result)) + (t/is (= {:stdout (* 256 1024) :stderr (* 128 1024)} @sizes)))) + +(t/deftest ^:async nonzero-exit-keeps-code-command-and-output + (let [options (node-options + "process.stdout.write('fact\\n'); process.stderr.write('failed\\n'); process.exitCode = 17;" + {}) + data (await (rejected (process/run options)))] + (t/is (= :process-exit (:type data))) + (t/is (= (:command options) (:command data))) + (t/is (= 17 (:exit-code data))) + (t/is (nil? (:signal data))) + (t/is (= "fact\n" (:stdout data))) + (t/is (= "failed\n" (:stderr data))))) + +(t/deftest ^:async spawn-failure-keeps-os-evidence + (let [command (file "no-such-command") + data (await (rejected (process/run (node-options "" {:command command}))))] + (t/is (= :process-spawn (:type data))) + (t/is (= command (:command data))) + (t/is (= "ENOENT" (:code data))) + (t/is (nil? (:exit-code data))) + (t/is (nil? (:signal data))))) + +(t/deftest ^:async expired-deadline-does-not-start-work + (let [data (await (rejected + (process/run + (node-options "fs.writeFileSync('started', 'yes');" + {:deadline (- (js/Date.now) 1)}))))] + (t/is (= :process-timeout (:type data))) + (t/is (= (.-execPath js/process) (:command data))) + (t/is (not (fs/existsSync (file "started")))) + (t/is (not (fs/existsSync (file "root.pid")))))) + +(t/deftest ^:async already-aborted-signal-does-not-start-work + (let [controller (js/AbortController.)] + (.abort controller "already-cancelled") + (let [data (await (rejected + (process/run + (node-options "fs.writeFileSync('started', 'yes');" + {:signal (.-signal controller)}))))] + (t/is (= :process-cancelled (:type data))) + (t/is (= "already-cancelled" (:reason data))) + (t/is (not (fs/existsSync (file "started")))) + (t/is (not (fs/existsSync (file "root.pid"))))))) + +(t/deftest ^:async timeout-kills-the-owner-and-a-term-resistant-descendant + (let [limit (process/deadline 1000) + data (await + (rejected + (process/run + (node-options (str (spawn-descendant) "setInterval(() => {}, 1000);") + {:deadline limit}))))] + (t/is (= :process-timeout (:type data))) + (t/is (= limit (:deadline data))) + (t/is (= "SIGTERM" (:signal data))) + (t/is (fs/existsSync (file "descendant.ready"))) + (await (assert-tree-stopped)))) + +(t/deftest ^:async cancellation-kills-the-whole-owned-group + (let [controller (js/AbortController.) + output (atom "") + groups (atom nil) + data (await + (rejected + (process/run + (node-options + (str (spawn-descendant) "setInterval(() => {}, 1000);") + {:signal (.-signal controller) + :on-output + (fn [text stream] + (when (= :stdout stream) + (swap! output str text) + (when (str/includes? @output "descendant-ready\n") + (reset! groups + {:root-pid (read-pid "root.pid") + :root (:group (process-status (read-pid "root.pid"))) + :descendant (:group (process-status (read-pid "descendant.pid"))) + :grandchild (:group (process-status (read-pid "grandchild.pid"))) + :runner (:group (process-status (.-pid js/process)))}) + (.abort controller "cancel-tree"))))}))))] + (t/is (= :process-cancelled (:type data))) + (t/is (= "cancel-tree" (:reason data))) + (t/is (= "SIGTERM" (:signal data))) + (t/is (= (:root-pid @groups) (:root @groups) (:descendant @groups) (:grandchild @groups)) + "the root leads the group containing all generations") + (t/is (not= (:runner @groups) (:root @groups)) "the group is separate from the runner") + (await (assert-tree-stopped)))) + + +(t/deftest ^:async default-cancellation-produces-portable-error-evidence + (let [controller (js/AbortController.) + cause (try + (await (within (process/run + (node-options "process.stdout.write('ready'); setInterval(() => {}, 1000);" + {:signal (.-signal controller) + :on-output (fn [& _] (.abort controller))})))) + nil + (catch :default cause cause)) + evidence (errors/evidence :process cause)] + (t/is (= :process-cancelled (:type (ex-data cause)))) + (t/is (= evidence (-> evidence transit/encode-str transit/decode-str))))) + +(t/deftest ^:async output-callback-failure-cleans-the-owned-group + (doseq [callback [:on-output :on-line]] + (let [seen (atom "") + data (await + (rejected + (process/run + (node-options + (str (spawn-descendant) "setInterval(() => {}, 1000);") + {callback + (fn [text stream] + (when (= :stdout stream) + (swap! seen str text) + (when (str/includes? @seen "descendant-ready") + (throw (js/Error. "Output consumer failed")))))}))))] + (t/is (= :process-output (:type data))) + (t/is (= :stdout (:stream data))) + (t/is (= "SIGTERM" (:signal data))) + (t/is (str/includes? (:stdout data) "descendant-ready\n")) + (await (assert-tree-stopped))))) + +(t/deftest ^:async successful-owner-exit-cleans-descendants-that-hold-the-pipes + (let [source (str (spawn-descendant) + "child.unref();\n" + "const timer = setInterval(() => {\n" + " if (fs.existsSync('descendant.ready')) {\n" + " clearInterval(timer); process.stdout.write('root-complete\\n');\n" + " }\n" + "}, 10);\n") + result (await (within (process/run (node-options source {}))))] + (t/is (= 0 (:exit-code result))) + (t/is (str/includes? (:stdout result) "root-complete\n")) + (t/is (str/includes? (:stdout result) "descendant-ready\n")) + (await (assert-tree-stopped)))) + +(t/deftest ^:async nonzero-owner-exit-also-cleans-descendants-that-hold-the-pipes + (let [source (str (spawn-descendant) + "child.unref();\n" + "const timer = setInterval(() => {\n" + " if (fs.existsSync('descendant.ready')) {\n" + " clearInterval(timer); process.stderr.write('root-failed\\n');\n" + " process.exitCode = 17;\n" + " }\n" + "}, 10);\n") + data (await (rejected (process/run (node-options source {}))))] + (t/is (= :process-exit (:type data))) + (t/is (= 17 (:exit-code data))) + (t/is (= "root-failed\n" (:stderr data))) + (await (assert-tree-stopped)))) + +(t/deftest ^:async lock-wait-times-out-and-cancelled-holder-releases-inherited-flock + (let [lock (file "build.lock") + controller (js/AbortController.) + holder (node-source + "holder.pid" + (str "process.on('SIGTERM', () => fs.writeFileSync('holder.term', 'seen'));\n" + "fs.writeFileSync('holder.ready', 'yes');\n" + "setInterval(() => {}, 1000);\n")) + held (rejected + (process/run + (node-options "" + {:command "flock" + :args ["--exclusive" lock (.-execPath js/process) "-e" holder] + :signal (.-signal controller)})))] + (try + (await (wait-for #(fs/existsSync (file "holder.ready")))) + (let [limit (process/deadline 150) + data (await + (rejected + (process/run + (node-options "" + {:command "flock" + :args ["--exclusive" lock (.-execPath js/process) "-e" + (node-source "waiter.pid" "fs.writeFileSync('waiter-acquired', 'yes');")] + :deadline limit}))))] + (t/is (= :process-timeout (:type data))) + (t/is (= limit (:deadline data))) + (t/is (not (fs/existsSync (file "waiter-acquired")))) + (t/is (running? (read-pid "holder.pid")) "the first owner still holds the lock")) + (.abort controller "release-lock") + (let [data (await held)] + (t/is (= :process-cancelled (:type data))) + (t/is (not (running? (read-pid "holder.pid"))))) + (let [result (await + (within + (process/run + (node-options "" + {:command "flock" + :args ["--exclusive" "--nonblock" lock (.-execPath js/process) "-e" + (node-source "probe.pid" "fs.writeFileSync('probe-acquired', 'yes');")]}))))] + (t/is (= 0 (:exit-code result))) + (t/is (= "yes" (fs/readFileSync (file "probe-acquired") "utf8"))) + (t/is (not (fs/existsSync (file "waiter-acquired")))) + (t/is (= "seen" (fs/readFileSync (file "holder.term") "utf8")))) + (finally + (.abort controller "test-cleanup") + (await held))))) diff --git a/frontend/test/performance_tests/render_wasm/protocol_test.cljs b/frontend/test/performance_tests/render_wasm/protocol_test.cljs new file mode 100644 index 0000000000..f17621553c --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/protocol_test.cljs @@ -0,0 +1,417 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.protocol-test + "Tests render drains and camera sessions with a controllable clock. + A drain renders until Full completion. A camera session moves a view through + cached previews, a settling wait and final rendering to Full." + (:require + [app.common.render-wasm.wasm :as wasm] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.runtime.protocol :as protocol] + [cljs.test :as t :include-macros true])) + +(defn- fake-clock + "Returns a controllable clock in milliseconds. + `:now` reads it; `:advance!` adds the supplied duration." + [start] + (let [t (atom start)] + {:now (fn [] @t) + :advance! (fn [ms] (swap! t + ms) @t)})) + +(defn- drain-hooks + "Returns hooks that simulate rendering and record flags in `calls`. + `:render` returns the frame types from `script` in order. Each `:frame` adds + 16 ms to the clock; `:sleep` adds the requested wait. Both return Promises." + [{:keys [clock script calls check]}] + (let [remaining (atom (vec script))] + {:now (:now clock) + :frame (fn [] + ((:advance! clock) 16) + (js/Promise.resolve ((:now clock)))) + :sleep (fn [ms] + ((:advance! clock) ms) + (js/Promise.resolve nil)) + :check (or check (fn [] nil)) + :render (fn [_timestamp flags] + (swap! calls conj flags) + (let [next (first @remaining)] + (swap! remaining rest) + next)) + :render-from-cache (fn [] 0) + :set-view (fn [_view] nil) + :set-view-start (fn [] nil) + :set-view-end (fn [] nil)})) + +(t/deftest ^:async completion-keeps-the-renderer-endpoint + (doseq [camera [false true]] + (let [clock (fake-clock 1000) + hooks (assoc (drain-hooks {:clock clock :script [] :calls (atom [])}) + :render (fn [_ _] + ((:advance! clock) 7) + (js/queueMicrotask #((:advance! clock) 500)) + wasm/FRAME_TYPE_FULL)) + result (await + (if camera + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms 0}) + (protocol/animate-view! {:to {:scale 1 :x 10 :y 5} :steps 1}) + (protocol/finish-camera!)) + (protocol/drain hooks {:flags 0 :origin 1000 :immediate true}))) + end (if camera 1023 1007)] + (t/is (= end (:end-ms result))) + (t/is (= (+ end 500) ((:now clock)))) + (if camera + (do + (t/is (= 23 (:interact-ms result))) + (t/is (= 7 (:last-input-to-full-ms result))) + (t/is (= 7 (:time-to-full-ms result)))) + (t/is (= 7 (:full-ms result))))))) + +(t/deftest immediate-full-reports-equal-boundaries + (t/async done + (let [clock (fake-clock 1000) + calls (atom []) + hooks (drain-hooks {:clock clock :script [2] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 1000 :immediate true}) + (.then (fn [{:keys [slices viewport-ready-ms full-ms]}] + (t/is (= 1 (count slices))) + (t/is (= {:timestamp 1000 + :flags 0 + :frame-type 2 + :duration-ms 0} + (first slices))) + (t/is (= full-ms viewport-ready-ms) + "immediate Full supplies both boundaries") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest progressive-frames-propagate-flags + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1 3 2] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate false}) + (.then (fn [{:keys [slices viewport-ready-ms full-ms]}] + (t/is (= [0 1 3] @calls) + "the previous frame type becomes the next flags") + (t/is (= [1 3 2] (mapv :frame-type slices))) + (t/is (< viewport-ready-ms full-ms) + "ViewportReady lands before Full") + (t/is (= (:timestamp (nth slices 1)) (+ 16 16)) + "later frames wait for rAF") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest partial-jumps-straight-to-full + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1 2] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [{:keys [slices viewport-ready-ms full-ms]}] + (t/is (= [0 1] @calls)) + (t/is (= [1 2] (mapv :frame-type slices))) + (t/is (= viewport-ready-ms full-ms) + "Full as the first completion supplies both boundaries") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest unknown-frame-types-reject + (t/async done + ;; One async context chaining the bad values: a `t/async` inside + ;; `doseq` would never run (the doseq returns nil to the runner). + (letfn [(step [remaining] + (if (empty? remaining) + (done) + (let [bad (first remaining) + clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [bad] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [_] + (t/is false (str "must reject for frame type " bad)) + (done))) + (.catch (fn [cause] + (t/is (= :benches.render-wasm.runtime.protocol/unexpected-frame-type + (:type (ex-data cause))) + (str "frame type " bad)) + (step (rest remaining))))))))] + (step [0 7 js/NaN])))) + +(t/deftest cancellation-stops-further-renders + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + fired (atom 0) + hooks (drain-hooks {:clock clock + :script [1 1 1 1] + :calls calls + :check (fn [] + (swap! fired inc) + (when (> @fired 2) + (throw (ex-info "canceled" {:phase "test"}))))})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [_] + (t/is false "must reject once check throws") + (done))) + (.catch (fn [cause] + (t/is (= "canceled" (ex-message cause))) + (t/is (= 1 (count @calls)) "cancellation stops before the next render") + (t/is (= 3 @fired) "guard fires once before and once after the render") + (done))))))) + +(t/deftest progressive-failure-retains-completed-call-evidence + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1 7] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [_] (t/is false "unknown frame must fail") (done))) + (.catch (fn [cause] + (t/is (= [1 7] (mapv :frame-type (get-in (ex-data cause) [:partial :slices])))) + (t/is (= [0 0] (mapv :duration-ms (get-in (ex-data cause) [:partial :slices])))) + (done))))))) + +(t/deftest ^:async unreadable-error-message-keeps-render-evidence + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1] :calls calls}) + render (:render hooks) + cause (ex-info "unreadable" {} (js/Error. "device lost")) + _ (js/Object.defineProperty cause "message" + #js {:get (fn [] (throw (js/Error. "message getter failed")))}) + hooks (assoc hooks :render (fn [timestamp flags] + (if (seq @calls) + (throw cause) + (render timestamp flags)))) + result (try + (await (protocol/drain hooks {:flags 0 :origin 0 :immediate true})) + nil + (catch :default error error))] + (t/is (= [1] (mapv :frame-type (get-in (ex-data result) [:partial :slices])))) + (t/is (some #(= "device lost" (:message %)) (errors/describe-cause result))))) + +(t/deftest camera-failure-retains-completed-previews-and-authored-pauses + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + previews (atom 0) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls calls}) + :render-from-cache (fn [] + (if (= 1 (swap! previews inc)) + ((:advance! clock) 3) + (throw (ex-info "preview failed" {}))))) + view {:scale 1 :x 0 :y 0}] + (-> (protocol/start-camera! {:hooks hooks :view view} {:settle-ms 0}) + (protocol/preview-view! view) + (protocol/sleep! 7) + (protocol/preview-view! (assoc view :x 1)) + (.then (fn [_] (t/is false "second preview must fail") (done))) + (.catch (fn [cause] + (t/is (= [3] (mapv :duration-ms (get-in (ex-data cause) [:partial :cached-slices])))) + (t/is (= [{:requested-ms 7 :actual-ms 7}] + (get-in (ex-data cause) [:partial :authored-sleeps]))) + (done))))))) + +(t/deftest rejected-pause-retains-earlier-camera-evidence + (t/async done + (let [clock (fake-clock 0) + sleeps (atom 0) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls (atom [])}) + :render-from-cache (fn [] ((:advance! clock) 3)) + :sleep (fn [ms] + (if (= 1 (swap! sleeps inc)) + (do ((:advance! clock) ms) (js/Promise.resolve nil)) + (js/Promise.reject (ex-info "pause failed" {}))))) + view {:scale 1 :x 0 :y 0}] + (-> (protocol/start-camera! {:hooks hooks :view view} {:settle-ms 0}) + (protocol/preview-view! view) + (protocol/sleep! 7) + (protocol/sleep! 13) + (.then (fn [_] (t/is false "second pause must fail") (done))) + (.catch (fn [cause] + (t/is (= "pause failed" (ex-message cause))) + (t/is (= [3] (mapv :duration-ms (get-in (ex-data cause) [:partial :cached-slices])))) + (t/is (= [{:requested-ms 7 :actual-ms 7}] + (get-in (ex-data cause) [:partial :authored-sleeps]))) + (done))))))) + +(t/deftest restore-drains-with-sync-tiles-immediately + (t/async done + (let [clock (fake-clock 500) + calls (atom []) + order (atom []) + hooks (assoc (drain-hooks {:clock clock :script [1 3 2] :calls calls}) + :set-view-start (fn [] (swap! order conj :start) nil) + :set-view (fn [view] + (swap! order conj [:view view]) + nil) + :set-view-end (fn [] (swap! order conj :end) nil)) + view {:scale 1 :x 10 :y 20}] + (-> (protocol/restore hooks view) + (.then (fn [{:keys [slices]}] + (t/is (= [:start [:view view] :end] @order) + "camera resets before the drain") + (t/is (= [4 1 3] @calls) + "restore drains with SyncTiles first") + (t/is (= [1 3 2] (mapv :frame-type slices))) + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest camera-session-times-gesture-settle-and-final-drain + (t/async done + (let [clock (fake-clock 0) + renders (atom []) + cached (atom []) + views (atom []) + hooks {:now (:now clock) + :frame (fn [] + ((:advance! clock) 16) + (js/Promise.resolve ((:now clock)))) + :sleep (fn [ms] + ((:advance! clock) ms) + (js/Promise.resolve nil)) + :check (fn [] nil) + :render (fn [_timestamp flags] + (swap! renders conj flags) + 2) + :render-from-cache (fn [] + (swap! cached conj true) + ((:advance! clock) 3) + nil) + :set-view (fn [view] + (swap! views conj view) + nil) + :set-view-start (fn [] nil) + :set-view-end (fn [] + ((:advance! clock) 5) + nil)} + frames [{:scale 1 :x 0 :y 0} {:scale 1 :x 100 :y 0}]] + (-> (protocol/start-camera! {:hooks hooks :view (first frames)} {:settle-ms 100}) + (protocol/preview-view! (first frames)) + (protocol/preview-view! (second frames)) + (protocol/finish-camera!) + (.then (fn [m] + (t/is (= frames @views) "every gesture frame sets the view") + (t/is (= 2 (count @cached)) "one cached preview per frame") + (t/is (= 2 (count (:cached-slices m)))) + (t/is (every? #(contains? % :timestamp) (:cached-slices m))) + (t/is (every? #(contains? % :duration-ms) (:cached-slices m)) + "cached slices keep their own shape") + (t/is (= 100 (:settling-requested-ms m))) + (t/is (= 100 (:settling-actual-ms m)) + "fake sleep advances the clock exactly") + (t/is (= 5 (:set-view-end-ms m))) + (t/is (= [4] @renders) "final drain starts with SyncTiles") + (t/is (>= (:time-to-full-ms m) (:set-view-end-ms m)) + "finalization includes _set_view_end") + (t/is (= (:time-to-viewport-ready-ms m) (:time-to-full-ms m)) + "immediate final drain gives equal boundaries") + (t/is (= 108 (:last-input-to-full-ms m)) + "last input at 35, Full at 143: settle + set_view_end + drain") + (t/is (= [wasm/FRAME_TYPE_FULL] (mapv :frame-type (:slices m))) + "frame is full") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest camera-session-chains-promises-and-records-authored-sleep + (t/async done + (let [clock (fake-clock 0) + views (atom []) + events (atom []) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls (atom [])}) + :frame (fn [] + (swap! events conj :frame) + ((:advance! clock) 16) + (js/Promise.resolve ((:now clock)))) + :set-view (fn [view] + (swap! views conj view) + ((:advance! clock) 2)) + :render-from-cache (fn [] + (swap! events conj :cached) + ((:advance! clock) 3)) + :set-view-end (fn [] + (swap! events conj :end) + ((:advance! clock) 5))) + base {:scale 1 :x 0 :y 0} + target {:scale 2 :x 100 :y 40}] + (-> (protocol/start-camera! {:hooks hooks :view base} {:settle-ms 100}) + (protocol/animate-view! {:to target :steps 3}) + (protocol/sleep! 25) + (protocol/finish-camera!) + (.then (fn [m] + (t/is (apply < (map :x @views)) + "intermediate camera views advance toward the target") + (t/is (= target (last @views))) + (t/is (= [:frame :cached :frame :cached :frame :cached :end] @events)) + (t/is (= 3 (count (:cached-slices m)))) + (t/is (= [5 5 5] (mapv :view-and-preview-ms (:cached-slices m))) + "call span also includes the view update") + (t/is (= [3 3 3] (mapv :duration-ms (:cached-slices m)))) + (t/is (= [{:requested-ms 25 :actual-ms 25}] (:authored-sleeps m))) + (t/is (= 88 (:active-ms m)) + "three rAF waits, view/cache calls and authored pause") + (t/is (= 100 (:settling-actual-ms m))) + (t/is (= 193 (:interact-ms m)) + "active, settle and camera end are one span") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest camera-session-rejects-hook-errors + (t/async done + (let [clock (fake-clock 0) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls (atom [])}) + :render-from-cache (fn [] (throw (ex-info "cache failed" {:phase "preview"}))))] + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms 100}) + (protocol/preview-view! {:scale 1 :x 10 :y 0}) + (protocol/finish-camera!) + (.then (fn [_] + (t/is false "preview error must reject") + (done))) + (.catch (fn [cause] + (t/is (= "cache failed" (ex-message cause))) + (done))))))) + +(t/deftest invalid-settle-ms-rejects + (t/async done + (let [hooks (drain-hooks {:clock (fake-clock 0) :script [2] :calls (atom [])})] + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms -1}) + (.then (fn [_] + (t/is false "invalid settle-ms must reject") + (done))) + (.catch (fn [cause] + (t/is (= ::protocol/invalid-settle-ms (:type (ex-data cause)))) + (done))))))) + +(t/deftest invalid-sleep-rejects-with-camera-evidence + (t/async done + (let [hooks (drain-hooks {:clock (fake-clock 0) :script [2] :calls (atom [])})] + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms 0}) + (protocol/sleep! -1) + (.then (fn [_] + (t/is false "invalid sleep must reject") + (done))) + (.catch (fn [cause] + (t/is (= ::protocol/invalid-sleep (:type (ex-data cause)))) + (t/is (= {:cached-slices [] :authored-sleeps []} + (:partial (ex-data cause)))) + (done))))))) diff --git a/frontend/test/performance_tests/render_wasm/report_test.cljs b/frontend/test/performance_tests/render_wasm/report_test.cljs new file mode 100644 index 0000000000..48cda88851 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/report_test.cljs @@ -0,0 +1,624 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.report-test + (:require + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.measurement :as measurement] + [benches.render-wasm.report.analysis :as method] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def saved-case + {:id :saved/pan :scene :saved :scene-seed 42 :params {:count 10} + :view {:scale 1 :x 0 :y 0} :context :reuse :preparation {}}) + +(def metadata + {:git {:sha "abc" :dirty false} + :build {:functional {:mode :release :features []}} + :environment {:browser "Chromium" :node "v24"} :scored? true}) + +(defn- event + [command values] + {:command command :metrics {:command/elapsed values}}) + +(defn- block + [phase prep events] + {:phase phase :preparation-id prep :outcome :completed :events events}) + +(defn- preparation + [id events] + {:preparation-id id :case-id :saved/pan :warmup? false + :fingerprint {:algorithm :sha-256 :digest (apply str (repeat 64 "a"))} + :effective-graphics {:renderer "hardware"} + :diagnostics {:upload-ms 123} + :phases [(block :preparation id events)]}) + +(defn- new-run + [warmups repetitions context] + (result/create-run {:run-id "run" :started-at "start" + :plan {:cases [(assoc saved-case :context context)] + :warmups warmups :repetitions repetitions} + :metadata metadata :analysis {:draws 50}})) + +(defn- add-attempt + [run prep warmup? outcome phases] + (result/record-attempt + run (measurement/attempt :saved/pan prep warmup? + (cond-> {:status (if (= :completed outcome) "ok" "failed") + :phases phases} + (= :failed outcome) (assoc :phase (name (:phase (last phases))) + :message "renderer failed"))))) + +(defn- sample-run + [values] + (reduce (fn [run value] + (add-attempt run :prep false :completed + [(block :main :prep [(event :test/pan [value])])])) + (result/record-preparation (new-run 0 (count values) :reuse) + (preparation :prep [])) values)) + +(defn- error-type + [f] + (try (f) nil (catch :default cause (:type (ex-data cause))))) + +(def main-pan [:main :test/pan :command/elapsed]) +(def main-global [:main :global :command/elapsed]) + +(defn- approximately? + [a b] + (< (js/Math.abs (- a b)) 1e-9)) + +(t/deftest mathematical-vectors + (let [settings (method/resolve-settings {:min-p95 1 :min-p99 1}) + stats (method/statistics [1 2 3 4 10] settings)] + (t/is (= 4 (get-in stats [:mean :value]))) + (t/is (= 3 (get-in stats [:median :value]))) + (t/is (= [1 10] [(get-in stats [:min :value]) (get-in stats [:max :value])])) + (t/is (approximately? (js/Math.sqrt 12.5) (get-in stats [:sample-sd :value]))) + (t/is (= 1 (get-in stats [:mad :value]))) + (t/is (approximately? 8.8 (get-in stats [:p95 :value]))) + (t/is (approximately? 9.76 (get-in stats [:p99 :value]))) + (t/is (= 1.75 (method/quantile [4 2 1 3] 0.25))) + (t/is (= 2.5 (method/quantile [1 2 3 4] 0.5))) + (t/is (= [1 4] [(method/quantile [4 1 3 2] 0) (method/quantile [4 1 3 2] 1)])))) + +(t/deftest numeric-arguments-throw-while-unavailable-results-retain-reasons + (doseq [probability [-0.1 1.1 js/NaN js/Infinity js/-Infinity nil false "0.5"]] + (t/is (= ::method/invalid-probability + (error-type #(method/quantile [1 2] probability))))) + (doseq [values [[js/NaN] [js/Infinity] [js/-Infinity] [nil] ["1"] 42 {} #{1 2}]] + (t/is (= ::method/invalid-observations (error-type #(method/quantile values 0.5))))) + (doseq [override [{:draws 0} {:seed -1} {:seed 4294967296} {:confidence 1} + {:confidence js/NaN} {:min-ci 0} {:unknown 1}]] + (t/is (= ::method/invalid-settings (error-type #(method/resolve-settings override))))) + (t/is (nil? (method/quantile [] 0.5))) + (t/is (nil? (method/quantile nil 0.5))) + (t/is (= {:reason :no-valid-observations} (:median (method/statistics [] method/defaults))))) + +(t/deftest calculated-numbers-and-overflow-have-independent-result-maps + (let [stats (method/statistics [4] method/defaults) + changes (method/changes [1e-308] [1e308] (method/resolve-settings {:draws 5 :min-ci 1}))] + (t/is (= {:value 4} (:median stats))) + (t/is (= {:value 0} (:mad stats))) + (t/is (= {:reason :insufficient-observations :minimum 2} (:sample-sd stats))) + (t/is (= {:value 1e308} (:absolute changes))) + (t/is (= {:reason :undefined-estimate} (:percentage changes))) + (t/is (= {:lower 1e308 :upper 1e308} (:absolute-ci changes))) + (t/is (= {:reason :undefined-bootstrap-estimate} (:percentage-ci changes))))) + +(t/deftest empty-singleton-and-support-thresholds + (let [settings (method/resolve-settings nil)] + (t/is (= :no-valid-observations (get-in (method/statistics [] settings) [:median :reason]))) + (t/is (= :insufficient-observations (get-in (method/statistics [7] settings) [:sample-sd :reason]))) + (t/is (= 0 (get-in (method/statistics [7] settings) [:mad :value]))) + (t/is (= :insufficient-observations (get-in (method/statistics (vec (repeat 9 7)) settings) [:median-ci :reason]))) + (t/is (= {:lower 7 :upper 7} (:median-ci (method/statistics (vec (repeat 10 7)) settings)))) + (t/is (= :insufficient-observations (get-in (method/statistics (vec (repeat 99 7)) settings) [:p95 :reason]))) + (t/is (= {:value 7} (:p95 (method/statistics (vec (repeat 100 7)) settings)))) + (t/is (= :insufficient-observations (get-in (method/statistics (vec (repeat 499 7)) settings) [:p99 :reason]))) + (t/is (= {:value 7} (:p99 (method/statistics (vec (repeat 500 7)) settings)))))) + +(t/deftest deterministic-bootstrap-and-independent-difference-vectors + (let [settings (method/resolve-settings {:draws 100 :seed 123 :confidence 0.8}) + stats (method/statistics (vec (range 1 11)) settings) + changes (method/changes (vec (range 1 11)) (vec (range 6 16)) settings)] + (t/is (= 5.5 (get-in stats [:median :value]))) + (t/is (= {:lower 3.5 :upper 7} (:median-ci stats))) + (t/is (= 5 (get-in changes [:absolute :value]))) + (t/is (approximately? 2.45 (get-in changes [:absolute-ci :lower]))) + (t/is (= 8 (get-in changes [:absolute-ci :upper]))) + (t/is (approximately? 31.201923076923077 (get-in changes [:percentage-ci :lower]))) + (t/is (approximately? 215.35714285714295 (get-in changes [:percentage-ci :upper]))) + (t/is (= stats (method/statistics (vec (range 1 11)) settings))))) + +(t/deftest zero-baseline-percentages-and-undefined-draws + (let [settings (method/resolve-settings {:draws 100 :min-ci 1 :seed 0})] + (t/is (= :zero-baseline (get-in (method/changes [0 0] [1 2] settings) [:percentage :reason]))) + (t/is (= :zero-baseline (get-in (method/changes [0 0] [1 2] settings) [:percentage-ci :reason]))) + (t/is (= :zero-bootstrap-baseline (get-in (method/changes [0 1] [2 3] settings) [:percentage-ci :reason]))) + (t/is (= :no-valid-observations (get-in (method/changes [] [1] settings) [:absolute :reason]))))) + +(t/deftest repeated-commands-keep-invocations-values-and-statistical-n-distinct + (let [phases [(block :main :prep + [(assoc-in (event :test/pan [2]) [:metrics :renderer/call-elapsed] [1 2]) + (assoc-in (event :test/zoom [4]) [:metrics :renderer/call-elapsed] [4]) + (assoc-in (event :test/pan [3]) [:metrics :renderer/call-elapsed] [3])])] + run (-> (new-run 1 2 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep true :completed phases) + (add-attempt :prep false :completed phases) + (add-attempt :prep false :completed phases) + (result/record-case-outcome :saved/pan {:outcome :completed}) + (result/finish-run "end" {:reason :completed}) + result/encode result/decode) + report (first (:cases (summary/summarize run))) + pan (get-in report [:metrics [:main :test/pan :renderer/call-elapsed]]) + global (get-in report [:metrics [:main :global :renderer/call-elapsed]])] + (t/is (= 3 (count (get-in run [:attempts 1 :phases 0 :events])))) + (t/is (= {:valid 2 :invalid 0 :absent 0 :failed 0 :warmup 1 :unattempted 0 + :invocations 4 :raw-values 6} (:counts pan))) + (t/is (= 2 (get-in pan [:statistics :count]))) + (t/is (= 6 (get-in pan [:statistics :median :value]))) + (t/is (= 10 (get-in global [:statistics :median :value]))) + (t/is (= 6 (get-in global [:counts :invocations]))) + (t/is (= 8 (get-in global [:counts :raw-values]))) + (t/is (str/includes? (format/format-run (summary/summarize run)) "invocations=4 raw-values=6 n=2")))) + +(t/deftest phase-eligibility-keeps-preparation-after-reset-failure + (let [run (-> (new-run 1 3 :reuse) + (result/record-preparation (preparation :prep [(event :test/setup [8])])) + (add-attempt :prep true :completed [(block :main :prep []) (block :reset :prep [])]) + (add-attempt :prep false :completed + [(block :main :prep [(event :test/pan [2])]) + (block :reset :prep [(event :test/restore [3])])]) + (add-attempt :prep false :failed + [(block :main :prep [(event :test/pan [100])]) + (assoc (block :reset :prep [(event :test/restore [200])]) :outcome :failed)]) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + report (first (:cases (summary/summarize run))) + prep (get-in report [:metrics [:preparation :test/setup :command/elapsed]])] + (t/is (= [0 1 2] (mapv :attempt-id (:attempts run)))) + (t/is (= {:warmups 0 :repetitions 1} (:remaining report))) + (t/is (= 8 (get-in prep [:statistics :median :value]))) + (t/is (= 1 (get-in prep [:statistics :count]))) + (t/is (= :insufficient-observations (get-in prep [:statistics :median-ci :reason]))) + (t/is (= 0 (get-in prep [:counts :warmup]))) + (t/is (= [2] (:values (summary/observations run :saved/pan main-pan)))) + (t/is (= [3] (:values (summary/observations run :saved/pan [:reset :test/restore :command/elapsed])))) + (t/is (= {:executions 3 :completed 3 :failed 0 :warmup 1} (get-in report [:phases :main]))) + (t/is (= {:executions 3 :completed 2 :failed 1 :warmup 1} (get-in report [:phases :reset]))))) + +(t/deftest fresh-preparations-follow-their-own-slot-eligibility + (let [run (-> (new-run 1 2 :fresh) + (result/record-preparation (assoc (preparation :warm [(event :test/setup [900])]) :warmup? true)) + (add-attempt :warm true :completed [(block :main :warm [])]) + (result/record-preparation (preparation :first [(event :test/setup [5])])) + (add-attempt :first false :completed [(block :main :first [])]) + (result/record-preparation (preparation :second [(event :test/setup [7])])) + (add-attempt :second false :failed [(assoc (block :main :second []) :outcome :failed)])) + obs (summary/observations run :saved/pan [:preparation :test/setup :command/elapsed])] + (t/is (= [5 7] (:values obs))) + (t/is (= 1 (get-in obs [:counts :warmup]))) + (t/is (= 2 (get-in obs [:counts :valid]))))) + +(t/deftest fresh-warmup-preparation-is-excluded-before-main-delivery + (let [run (-> (new-run 1 1 :fresh) + (result/record-preparation + (assoc (preparation :prep [(event :test/setup [9])]) :warmup? true)) + result/encode result/decode) + report (get-in (summary/summarize run) + [:cases 0 :metrics [:preparation :test/setup :command/elapsed]])] + (t/is (= 1 (get-in report [:counts :warmup]))) + (t/is (= :no-valid-observations (get-in report [:statistics :median :reason]))))) + +(t/deftest failed-preparation-excludes-a-completed-callback-from-observations + (let [run (-> (new-run 0 1 :reuse) + (result/record-preparation + (assoc (preparation :prep [(event :test/setup [9])]) + :failure {:phase "preparation" :message "initial Full failed"}))) + report (first (:cases (summary/summarize run))) + metric (get-in report [:metrics [:preparation :test/setup :command/elapsed]])] + (t/is (= {:executions 1 :completed 1 :failed 0 :warmup 0} (get-in report [:phases :preparation]))) + (t/is (= 1 (get-in metric [:counts :failed]))) + (t/is (= :no-valid-observations (get-in metric [:statistics :median :reason]))))) + +(t/deftest invalid-members-propagate-only-to-dependent-quantities + (let [bad (assoc-in (event :test/pan [0]) [:metrics :renderer/call-elapsed] [1 js/NaN]) + good (assoc-in (event :test/zoom [0]) [:metrics :renderer/call-elapsed] [2]) + run (-> (new-run 0 1 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep false :completed [(block :main :prep [bad good])])) + report (first (:cases (summary/summarize run)))] + (t/is (= [0] (:values (summary/observations run :saved/pan main-global)))) + (t/is (= [2] (:values (summary/observations run :saved/pan [:main :test/zoom :renderer/call-elapsed])))) + (doseq [scope [:test/pan :global]] + (let [metric (get-in report [:metrics [:main scope :renderer/call-elapsed]])] + (t/is (= 1 (get-in metric [:counts :invalid]))) + (t/is (= :non-finite (get-in metric [:diagnostics 0 :reason]))) + (t/is (= :no-valid-observations (get-in metric [:statistics :median :reason]))))))) + +(t/deftest raw-special-values-and-partial-failures-survive-transit + (let [values [0 -1 js/NaN js/Infinity js/-Infinity] + initial (-> (new-run 0 6 :reuse) + (result/record-preparation (preparation :prep []))) + run (-> (reduce (fn [run value] + (add-attempt run :prep false :completed + [(block :main :prep [(event :test/pan [value])])])) + initial values) + (result/record-attempt + (measurement/attempt :saved/pan :prep false + {:status "failed" :phase "main" :message "render failed" + :cause [{:message "WASM trap"}] + :partial {:clock js/NaN} + :phases [(assoc (block :main :prep [(event :test/pan [99])]) + :outcome :failed)]})) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + raw (mapv #(get-in % [:phases 0 :events 0 :metrics :command/elapsed 0]) (:attempts run)) + obs (summary/observations run :saved/pan main-pan)] + (t/is (= [0 -1] (subvec raw 0 2))) + (t/is (js/Number.isNaN (nth raw 2))) + (t/is (= [js/Infinity js/-Infinity] (subvec raw 3 5))) + (t/is (= [{:message "WASM trap"}] (get-in run [:attempts 5 :failure :cause]))) + (t/is (js/Number.isNaN (get-in run [:attempts 5 :partial :clock]))) + (t/is (= 99 (nth raw 5))) + (t/is (= [0] (:values obs))) + (t/is (= 1 (get-in obs [:counts :failed]))) + (t/is (= 4 (get-in obs [:counts :invalid]))))) + +(t/deftest empty-blocks-support-preparation-only-and-reset-only-cases + (doseq [phase [:preparation :reset]] + (let [run (-> (new-run 0 1 :reuse) + (result/record-preparation + (preparation :prep (if (= :preparation phase) [(event :test/work [2])] []))) + (add-attempt :prep false :completed + [(block :main :prep []) + (block :reset :prep (if (= :reset phase) [(event :test/work [2])] []))]) + (result/record-case-outcome :saved/pan {:outcome :completed}) + (result/finish-run "end" {:reason :completed}) + result/encode result/decode)] + (t/is (= [2] (:values (summary/observations run :saved/pan [phase :global :command/elapsed])))))) + (let [empty-run (-> (new-run 0 1 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep false :completed [(block :main :prep [])]))] + (t/is (= ::result/invalid-record + (error-type #(result/finish-run empty-run "end" {:reason :completed}))))) + (let [failed (-> (new-run 0 1 :reuse) + (result/record-preparation {:preparation-id :prep :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] + :failure {:phase "preparation" :message "setup failed"}}) + (result/finish-run "end" {:reason :preparation-error}))] + (t/is (str/includes? (format/format-run (summary/summarize (result/decode (result/encode failed)))) + "setup failed")))) + +(t/deftest comparison-reports-preparation-failures-once-per-source + (let [failure {:phase "preparation" :message "setup failed" + :errors [{:phase "upload" :operation :serialize-shapes :message "setup failed"} + {:phase "upload" :operation :end-loading :message "ending load failed"}]} + run (-> (new-run 0 2 :fresh) + (result/record-preparation (preparation :ready [])) + (add-attempt :ready false :completed [(block :main :ready [(event :test/work [2])])]) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/record-attempt (measurement/attempt :saved/pan :failed false + (assoc failure :status "failed" :phases []))) + (result/record-case-outcome :saved/pan {:outcome :failed :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + (assoc-in [:metadata :diagnostics :screenshots] + {"saved/pan" {:status :skipped :reason :no-completed-measured-attempt}}) + result/encode result/decode) + candidate (-> run + (assoc :run-id "candidate") + (assoc-in [:metadata :diagnostics :screenshots] + {"saved/pan" {:status :ok :file "candidate-pan.png"}}) + result/encode result/decode) + compared (-> (compare/compare-runs run candidate + {:allow-all-differences true}) + result/encode result/decode) + text (format/format-comparison compared)] + (t/is (= 2 (count (re-seq #"setup failed" text)))) + (t/is (= 2 (count (re-seq #"ending load failed" text)))) + (t/is (str/includes? text "median change")) + (t/is (= 2 (count (re-seq #"Preparation :failed" text)))) + (t/is (str/includes? text "Baseline screenshots\n \"saved/pan\": skipped; reason :no-completed-measured-attempt")) + (t/is (str/includes? text "Candidate screenshots\n \"saved/pan\": ok\n File \"candidate-pan.png\"")))) + +(t/deftest collected-failures-print-a-long-message-once-and-retain-enclosing-context + (let [message (str "failure-marker " (apply str (repeat 1400 "x"))) + failure {:phase "preparation" :operation :prepare-scene :path "scene.transit" :message message + :errors [{:phase "upload" :operation :serialize-shapes + :message (subs message 0 1024)}]} + run (-> (new-run 0 1 :fresh) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (= 1 (count (re-seq #"failure-marker" text)))) + (t/is (str/includes? text ":prepare-scene \"preparation\"")) + (t/is (str/includes? text "Path \"scene.transit\"")) + (t/is (str/includes? text ":serialize-shapes \"upload\"")))) + +(t/deftest unresolved-process-details-survive-the-report-entry-limit + (let [failure (errors/evidence + :cleanup + (errors/combine + (conj (mapv (fn [index] {:phase "cleanup" :error (js/Error. (str "cleanup " index))}) + (range 8)) + {:phase "cleanup" :operation :stop-process + :error (errors/fatal-error "Compiler group may still be running" + {:unresolved-process true :pid 4242 :pgid 4242 + :signals ["SIGTERM" "SIGKILL"]} nil)}))) + run (-> (new-run 0 1 :fresh) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "pgid 4242")) + (t/is (str/includes? text "SIGKILL")) + (t/is (str/includes? text "unresolved-process true")) + (t/is (str/includes? text "1 more errors recorded")))) + +(t/deftest equal-diagnostics-keep-failures-from-distinct-preparations + (let [failure {:phase "renderer" :message "shared renderer failure"}] + (doseq [phases [[] + [(block :main :ready []) + (assoc (block :reset :ready []) :outcome :failed)]]] + (t/testing (str "attempt phases " (mapv :phase phases)) + (let [run (-> (new-run 0 2 :fresh) + (result/record-preparation (preparation :ready [])) + (result/record-attempt + (measurement/attempt :saved/pan :ready false + (assoc failure :status "failed" :phases phases))) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/record-case-outcome :saved/pan {:outcome :failed :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "Preparation :failed (case :saved/pan): \"renderer\": \"shared renderer failure\"")) + (t/is (str/includes? text "Attempt 0 (preparation :ready) (case :saved/pan): \"renderer\": \"shared renderer failure\"")) + (t/is (= 2 (count (re-seq #"shared renderer failure" text))))))))) + +(t/deftest equal-diagnostics-keep-failures-from-distinct-cases + (let [failure {:phase "preparation" :message "shared setup failure"}] + (doseq [harness? [false true]] + (t/testing (if harness? "harness failure in another case" "outcome failure in another case") + (let [run (-> (result/create-run + {:run-id "run" :started-at "start" + :plan {:cases [saved-case (assoc saved-case :id :saved/other)] + :warmups 0 :repetitions 1} + :metadata (assoc-in metadata [:diagnostics :failures] + (if harness? + [(assoc failure :case-id :saved/other :stage :preparation)] + [])) + :analysis {:draws 50}}) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/record-case-outcome :saved/pan {:outcome :failed :failure failure}) + (result/record-case-outcome :saved/other {:outcome :failed :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "Preparation :failed (case :saved/pan): \"preparation\": \"shared setup failure\"")) + (t/is (str/includes? text (str (if harness? "Harness 0 :preparation" "Case outcome") + " (case :saved/other): \"preparation\": \"shared setup failure\""))) + (t/is (= 2 (count (re-seq #"shared setup failure" text))))))))) + +(t/deftest ambiguous-screenshot-matches-report-the-warning-and-each-harness-failure + (let [failure {:phase "screenshot" :message "screenshot deadline exceeded" + :cause [{:message "browser deadline"}] + :stage :case-deadline :case-id :saved/pan}] + (doseq [[label case-ids failures] + [["multiple deadlines" [:saved/pan] [failure failure]] + ["distinct IDs with the same displayed name" [:saved/pan "saved/pan"] [failure]]]] + (t/testing label + (let [run (-> (result/create-run + {:run-id "run" :started-at "start" + :plan {:cases (mapv #(assoc saved-case :id %) case-ids) + :warmups 0 :repetitions 1} + :metadata (assoc metadata :diagnostics + {:failures failures + :screenshots {"saved/pan" {:status :failed :file "saved-pan.png" + :message (:message failure) + :cause (:cause failure)}}}) + :analysis {:draws 50}}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "\"saved/pan\": warning; \"screenshot deadline exceeded\"")) + (t/is (str/includes? text "File \"saved-pan.png\"")) + (doseq [index (range (count failures))] + (t/is (str/includes? text (str "Harness " index " :case-deadline (case :saved/pan): \"screenshot\": \"screenshot deadline exceeded\"")))) + (t/is (= (inc (count failures)) (count (re-seq #"screenshot deadline exceeded" text))))))))) + +(t/deftest offline-comparison-uses-one-analysis-and-saved-quantity-units + (let [base (-> (sample-run [1 2 3]) result/encode result/decode) + candidate (-> (sample-run [3 4 5]) + (assoc-in [:analysis :seed] 99) + (assoc-in [:quantities :command/elapsed :description] "Reworded description") + result/encode result/decode) + comparison (-> (compare/compare-runs base candidate {:analysis {:draws 20}}) + result/encode result/decode)] + (t/is (= 2 (get-in comparison [:pairs 0 :metrics main-pan :absolute :value]))) + (t/is (= 20 (get-in comparison [:analysis :draws]))) + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? (format/format-comparison comparison) "median change 2")) + (let [changed (assoc-in candidate [:quantities :command/elapsed :unit] :seconds)] + (t/is (= ::compare/incompatible (error-type #(compare/compare-runs base changed)))) + (t/is (= 2 (get-in (compare/compare-runs base changed {:allow-all-differences true}) + [:pairs 0 :metrics main-pan :absolute :value])))))) + +(t/deftest unknown-and-malformed-quantities-produce-diagnostics + (doseq [[description reason] [[nil :unknown-quantity] [{:unit 2} :malformed-quantity]]] + (let [run (cond-> (sample-run [1]) + (nil? description) (update :quantities dissoc :command/elapsed) + (some? description) (assoc-in [:quantities :command/elapsed] description)) + report (get-in (summary/summarize (result/decode (result/encode run))) + [:cases 0 :metrics main-pan])] + (t/is (= reason (get-in report [:diagnostics 0 :reason]))) + (t/is (= :no-valid-observations (get-in report [:statistics :median :reason])))))) + +(defn- feature-run + [features selection] + (-> (sample-run [1]) + (assoc-in [:metadata :build :functional :features] features) + (assoc-in [:metadata :build :provenance :feature-selection] selection))) + +(t/deftest automatic-feature-permission-checks-effective-and-requested-evidence + (let [baseline (feature-run ["base" "old"] {:default-features? false :features ["base"]}) + candidate (feature-run ["base" "toggle" "transitive"] + {:default-features? false :features ["base" "toggle"]}) + expected {:feature "toggle" :baseline-features ["base" "old"] + :candidate-features ["base" "toggle" "transitive"] + :added ["toggle" "transitive"] :removed ["old"]} + edge (assoc expected :from "0" :to "1") + report (compare/compare-runs baseline candidate {:expected-feature-change expected :edge edge})] + (t/is (= :feature-toggle (get-in report [:pairs 0 :mismatches 0 :rule]))) + (t/is (= report (result/decode (result/encode report)))) + (doseq [invalid [nil (assoc expected :feature "base") (assoc expected :feature :toggle) + (assoc expected :baseline-features ["base"]) + (assoc expected :candidate-features ["base" "toggle"]) + (assoc expected :added ["toggle"]) (assoc expected :removed []) + (assoc expected :added #{"toggle" "transitive"}) + (dissoc expected :baseline-features)]] + (t/is (= ::compare/incompatible + (error-type #(compare/compare-runs baseline candidate {:expected-feature-change invalid}))))) + (doseq [selection [nil {} {:default-features? true :features ["base" "toggle"]} + {:default-features? false :features ["toggle"]} + {:default-features? false :features #{"base" "toggle"}}]] + (t/is (= ::compare/incompatible + (error-type #(compare/compare-runs + baseline (assoc-in candidate [:metadata :build :provenance :feature-selection] selection) + {:expected-feature-change expected}))))) + (doseq [bad-edge [(dissoc edge :removed) (assoc edge :added []) (assoc edge :feature :toggle)]] + (t/is (= ::compare/invalid-edge + (error-type #(compare/compare-runs baseline candidate {:edge bad-edge}))))) + (t/is (= ::compare/invalid-edge + (error-type #(compare/compare-runs baseline candidate + {:edge edge :expected-feature-change (assoc expected :added [])})))))) + +(t/deftest missing-nil-and-empty-effective-features-keep-distinct-meaning + (doseq [before [::missing nil []] after [::missing nil []]] + (let [with-features (fn [features] + (cond-> (sample-run [1]) + (= ::missing features) (update-in [:metadata :build :functional] dissoc :features) + (not= ::missing features) (assoc-in [:metadata :build :functional :features] features))) + baseline (-> (with-features before) result/encode result/decode) + candidate (-> (with-features after) result/encode result/decode) + compared (compare/compare-runs baseline candidate {:allow-build-differences true})] + (t/is (= (if (= before after) [] [:effective-features]) + (mapv :field (get-in compared [:pairs 0 :mismatches]))))))) + +(t/deftest comparison-boundaries-validate-permissions-rules-and-intervals + (let [base (sample-run [1]) + candidate (assoc-in (sample-run [2]) [:metadata :build :functional :mode] :debug) + compared (compare/compare-runs base candidate {:allow-build-differences true}) + path [:pairs 0 :metrics main-pan]] + (doseq [bad [(dissoc compared :permissions) + (assoc-in compared [:permissions :allow-build-differences] :yes) + (update compared :permissions dissoc :allow-scene-differences) + (assoc-in compared [:pairs 0 :mismatches 0 :rule] :other) + (assoc-in compared [:pairs 0 :mismatches 0 :waived?] false) + (update-in compared [:pairs 0 :mismatches 0] dissoc :rule) + (assoc-in compared (conj path :absolute :value) js/NaN) + (assoc-in compared (conj path :absolute-ci) {:lower 2 :upper 1}) + (assoc-in compared [:pairs 0 :baseline :metrics main-pan :counts :valid] -1)]] + (t/is (= ::result/invalid-record (error-type #(result/decode (transit/encode-str bad)))))))) + +(t/deftest scene-permission-requires-complete-consistent-preparation-evidence + (let [base (sample-run [1]) + changed (result/record-preparation base + (assoc-in (preparation :other []) [:fingerprint :digest] + (apply str (repeat 64 "b")))) + report (compare/compare-runs base changed {:allow-all-differences true})] + (t/is (= ::compare/incompatible + (error-type #(compare/compare-runs base changed {:allow-scene-differences true})))) + (t/is (some #(= [:fingerprint :incomplete-or-varying-evidence] (:field %)) + (get-in report [:pairs 0 :mismatches]))))) + +(t/deftest matching-missing-preparation-facts-produce-incomplete-evidence + (let [run (-> (new-run 0 1 :reuse) + (result/record-preparation + (-> (preparation :prep [(event :test/setup [9])]) + (dissoc :fingerprint :effective-graphics) + (assoc :failure {:phase "preparation" :message "setup failed"}))))] + (doseq [options [{} {:allow-scene-differences true}]] + (t/is (= ::compare/incompatible (error-type #(compare/compare-runs run run options))))) + (let [report (compare/compare-runs run run {:allow-all-differences true})] + (t/is (= #{[:fingerprint :incomplete-or-varying-evidence] + [:effective-graphics :incomplete-or-varying-evidence]} + (set (map :field (get-in report [:pairs 0 :mismatches])))))))) + +(t/deftest saved-boundaries-refuse-corrupt-identities-and-phase-history + (let [run (sample-run [1])] + (doseq [bad [(assoc-in run [:attempts 0 :attempt-id] 3) + (assoc-in run [:attempts 0 :phases] []) + (assoc-in run [:preparations 0 :phases] []) + (assoc-in run [:preparations 0 :failure] {:phase "preparation" :message "setup failed"}) + (update-in run [:attempts 0] assoc :outcome :failed + :phases [(assoc (block :main :prep []) :outcome :failed) + (block :reset :prep [])]) + (assoc-in run [:attempts 0 :preparation-id] :unknown) + (assoc-in run [:attempts 0 :phases 0 :preparation-id] :unknown) + (assoc-in run [:attempts 0 :phases 0 :events 0 :metrics :command/elapsed] []) + (assoc-in run [:attempts 0 :phases 0 :events 0 :metrics :command/elapsed] [1 2]) + (assoc-in run [:remaining :saved/pan :repetitions] 2)]] + (t/is (= ::result/invalid-record + (error-type #(result/decode (transit/encode-str bad)))))))) + +(t/deftest saved-failures-reject-malformed-diagnostics + (let [diagnostic {:phase "main" :message "renderer failed"} + failed-preparation (-> (new-run 0 1 :reuse) + (result/record-preparation (assoc (preparation :prep []) :failure diagnostic)) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + failed-attempt (-> (new-run 0 1 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep false :failed [(assoc (block :main :prep []) :outcome :failed)]) + (result/record-case-outcome :saved/pan {:outcome :failed :failure diagnostic}) + (result/finish-run "end" {:reason :failed :failure diagnostic}) + (assoc-in [:metadata :diagnostics :failures] [diagnostic]) + result/encode result/decode)] + (doseq [[run path] [[failed-preparation [:preparations 0 :failure]] + [failed-attempt [:attempts 0 :failure]] + [failed-attempt [:case-outcomes :saved/pan :failure]] + [failed-attempt [:termination :failure]] + [failed-attempt [:metadata :diagnostics :failures 0]]] + invalid [{:phase 42 :message "bad location"} + {:phase "main" :message nil} + {:phase "main" :message "bad cause" :cause [42]} + {:phase "main" :message "bad error collection" + :errors [{:phase "cleanup" :message "nested collection" :errors []}]}]] + (t/is (= ::result/invalid-record + (error-type #(result/decode (transit/encode-str (assoc-in run path invalid))))))))) + +(t/deftest case-outcomes-validate-completion-and-close-the-recording-boundary + (let [run (sample-run [1]) + completed (result/record-case-outcome run :saved/pan {:outcome :completed}) + ended (result/finish-run completed "end" {:reason :completed})] + (t/is (= ended (result/decode (result/encode ended)))) + (doseq [bad [(assoc ended :case-outcomes {}) + (-> run + (assoc-in [:plan :repetitions] 2) + (assoc-in [:remaining :saved/pan :repetitions] 1) + (assoc-in [:case-outcomes :saved/pan] {:outcome :completed})) + (assoc-in ended [:case-outcomes :unknown] {:outcome :completed})]] + (t/is (= ::result/invalid-record (error-type #(result/decode (transit/encode-str bad)))))) + (t/is (= ::result/invalid-record + (error-type #(result/record-preparation completed (preparation :other []))))) + (let [stopped (-> run + (assoc-in [:plan :repetitions] 2) + (assoc-in [:remaining :saved/pan :repetitions] 1) + (result/record-case-outcome :saved/pan {:outcome :failed}))] + (t/is (thrown-with-msg? js/Error #"case already terminated" + (add-attempt stopped :prep false :completed [(block :main :prep [])])))))) diff --git a/frontend/test/performance_tests/render_wasm/request_test.cljs b/frontend/test/performance_tests/render_wasm/request_test.cljs new file mode 100644 index 0000000000..0229b9503e --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/request_test.cljs @@ -0,0 +1,208 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.request-test + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.request :as request] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :refer [failure-data]])) + +(def ^:private console-defaults + {:level :info :color :auto}) + +(def ^:private measurement-defaults + {:seed 42 :filter nil :warmups 3 :repetitions 10 + :timeout-ms 30000 :startup-timeout-ms 30000 :output-dir nil + :screenshots false :require-gpu false}) + +(def ^:private comparison-defaults + {:allow-build-differences false :allow-scene-differences false + :allow-all-differences false}) + +(def ^:private existing-run + {:command "run" :console console-defaults + :options (assoc measurement-defaults + :build {:kind :existing :id "ready-build"} + :remove false)}) + +(def ^:private default-cases + [[{:command "build"} + {:command "build" :console console-defaults + :options {:default-features true :features [] :build-timeout-ms 300000}}] + [{:command "run" :console {} :options {:build {:kind :prepare}}} + {:command "run" :console console-defaults + :options (assoc measurement-defaults + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000} + :remove false)}] + [{:command "run" :options {:build {:kind :existing :id "ready-build"}}} + existing-run] + [{:command "ab" :options {:features ["render-cache"]}} + {:command "ab" :console console-defaults + :options (merge measurement-defaults comparison-defaults + {:features ["render-cache"] :max-features 3 + :confirm false :dry-run false :build-timeout-ms 300000})}] + [{:command "compare" + :options {:baseline-path "base.transit" :candidate-path "candidate.transit"}} + {:command "compare" :console console-defaults + :options (assoc comparison-defaults + :baseline-path "base.transit" + :candidate-path "candidate.transit" :case-pair nil)}] + [{:command "merge" + :options {:base-path "base.transit" :additional-path "extra.transit"}} + {:command "merge" :console console-defaults + :options {:base-path "base.transit" :additional-path "extra.transit" + :output-dir nil}}] + [{:command "purge" :options {:target :runs}} + {:command "purge" :console console-defaults :options {:target :runs}}] + [{:command "help"} + {:command "help" :console console-defaults :options {:topic nil}}]]) + +(t/deftest defaults-produce-complete-command-requests + (doseq [[partial expected] default-cases] + (t/testing (pr-str partial) + (t/is (= expected (request/check (request/apply-defaults partial))))))) + +(t/deftest explicit-values-and-literal-text-survive-resolution + (let [partial {:command "run" :console {:level :warn :color :never} + :options {:seed 0 :warmups 0 :filter " " :output-dir " ./runs " + :build {:kind :prepare :default-features false + :features []}}} + expected {:command "run" :console {:level :warn :color :never} + :options (assoc measurement-defaults + :seed 0 :warmups 0 :filter " " :output-dir " ./runs " + :build {:kind :prepare :default-features false + :features [] :build-timeout-ms 300000} + :remove false)}] + (t/is (= expected (request/check (request/apply-defaults partial))))) + (let [partial {:command "compare" + :options {:baseline-path " ./base " :candidate-path " ./candidate " + :case-pair [" rects/load " "other/load"]}}] + (t/is (= (assoc partial :console console-defaults + :options (merge comparison-defaults (:options partial))) + (request/check (request/apply-defaults partial))))) + (t/is (= existing-run + (request/check + (request/apply-defaults (assoc-in existing-run [:options :warmups] nil)))))) + +(t/deftest required-selections-and-paths-have-no-defaults + (doseq [partial [{} + {:command "unknown"} + {:command "run"} + {:command "run" :options {:build {}}} + {:command "run" :options {:build {:kind :unknown}}} + {:command "run" :options {:build {:kind :existing}}} + {:command "ab"} + {:command "compare"} + {:command "compare" :options {:baseline-path "base"}} + {:command "compare" :options {:candidate-path "candidate"}} + {:command "merge" :options {:base-path "base"}} + {:command "merge" :options {:additional-path "additional"}} + {:command "purge"}]] + (t/testing (pr-str partial) + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (request/apply-defaults partial))))))))) + +(t/deftest receiver-requires-complete-data-and-reports-the-field + (doseq [[_ complete] default-cases] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (dissoc complete :console))))))) + (doseq [incomplete [(dissoc existing-run :options) + (update existing-run :console dissoc :level) + (update existing-run :options dissoc :warmups) + (assoc-in existing-run [:options :warmups] nil) + (assoc-in existing-run [:options :remove] nil) + (assoc-in existing-run [:console :level] nil)]] + (let [data (failure-data #(request/check incomplete))] + (t/is (= ::request/invalid-request (:type data))) + (t/is (seq (:errors (::sm/explain data)))))) + (let [data (failure-data #(request/check (assoc-in existing-run [:options :warmups] -1)))] + (t/is (some #(= [:options :warmups] (:in %)) (:errors (::sm/explain data)))))) + +(t/deftest receiver-rejects-extra-fields-at-every-map-boundary + (doseq [[path value] [[[:extra] true] + [[:console :stream] "stdout"] + [[:options :browser] {}] + [[:options :build :features] []] + [[:options :build :default-features] false] + [[:options :build :build-timeout-ms] 1000]]] + (let [input (assoc-in existing-run path value) + resolved (request/apply-defaults input)] + (t/is (= input resolved)) + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check resolved))))))) + (doseq [partial [{:command "run" :options {:build {:kind :prepare :id "ready-build"}}} + {:command "ab" :options {:features ["cache"] :default-features false}} + {:command "ab" :options {:features ["cache"] :remove false}} + {:command "ab" :options {:features ["cache"] :build {:kind :prepare}}}]] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (request/apply-defaults partial)))))))) + +(t/deftest numeric-constraints-match-seeds-counts-and-node-timers + (doseq [[field values] [[:seed [0 4294967295]] + [:warmups [0 js/Number.MAX_SAFE_INTEGER]] + [:repetitions [1 js/Number.MAX_SAFE_INTEGER]] + [:timeout-ms [1 2147483647]] + [:startup-timeout-ms [1 2147483647]]]] + (doseq [value values] + (let [input (assoc-in existing-run [:options field] value)] + (t/is (= input (request/check input)))))) + (doseq [[field values] [[:seed [-1 4294967296 1.5 "42" js/NaN js/Infinity]] + [:warmups [-1 (inc js/Number.MAX_SAFE_INTEGER) 0.5 "3"]] + [:repetitions [0 (inc js/Number.MAX_SAFE_INTEGER) 1.5]] + [:timeout-ms [0 2147483648 1.5 "30000"]] + [:startup-timeout-ms [0 2147483648]]]] + (doseq [value values] + (t/testing (str field " " value) + (let [input (assoc-in existing-run [:options field] value) + resolved (request/apply-defaults input)] + (when (string? value) + (t/is (= value (get-in resolved [:options field])))) + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check resolved))))))))) + (doseq [command ["build" "ab"] + value [0 2147483648 1.5]] + (let [partial {:command command :options {:features ["cache"] :build-timeout-ms value}}] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (request/apply-defaults partial))))))))) + +(t/deftest command-values-use-their-domain-constraints + (doseq [partial [{:command "run" :options {:build {:kind :existing :id "../build"}}} + {:command "run" :options {:build {:kind :existing :id ""}}} + {:command "build" :options {:features ["bad feature"]}} + {:command "build" :options {:features [nil]}} + {:command "ab" :options {:features []}} + {:command "ab" :options {:features ["cache"] :max-features 0}} + {:command "ab" :options {:features ["cache"] :max-features 7}} + {:command "ab" :options {:features ["cache"] :max-features 1.5}} + {:command "compare" :options {:baseline-path " " :candidate-path "c"}} + {:command "merge" :options {:base-path "b" :additional-path ""}} + {:command "merge" :options {:base-path "b" :additional-path "a" + :output-dir " "}} + {:command "purge" :options {:target :unknown}} + {:command "help" :options {:topic "unknown"}} + {:command "help" :console {:level :error}} + {:command "help" :console {:color true}}]] + (t/testing (pr-str partial) + (let [resolved (request/apply-defaults partial)] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check resolved)))))))) + (doseq [pair [["only"] ["one" "two" "three"] ["" "two"] ["one" " "]]] + (t/is (= ::request/invalid-request + (:type (failure-data + #(request/check + (request/apply-defaults + {:command "compare" :options {:baseline-path "b" :candidate-path "c" + :case-pair pair}}))))))) + (doseq [filter-text [nil "" " " " rects "]] + (let [input (assoc-in existing-run [:options :filter] filter-text)] + (t/is (= input (request/check (request/apply-defaults input)))))) + (doseq [target [:builds :runs :comparisons :all]] + (t/is (= target + (get-in (request/check + (request/apply-defaults {:command "purge" :options {:target target}})) + [:options :target]))))) diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs new file mode 100644 index 0000000000..ee71a21f8f --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -0,0 +1,1483 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.run-test + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.cli.deferred :as deferred] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.browser :as browser] + [benches.render-wasm.node.runner.feature-comparison :as feature-comparison] + [benches.render-wasm.node.runner.run :as run] + [benches.render-wasm.node.runner.server :as server] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-run-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- output-dir + [] + (path/join @temporary-root (str "out-" (random-uuid)))) + +(defn- build-dir + [] + (let [dir (path/join @temporary-root (str "build-" (random-uuid)))] + (fs/mkdirSync dir #js {:recursive true}) + (fs/writeFileSync (path/join dir "index.html") "") + dir)) + +(defn- selected-build + [dir id] + {:id id + :dir dir + :log-path (path/join dir "build.log") + :manifest {:type :renderer-build + :id id + :status :ready + :created-at "2026-10-27T00:00:00.000Z" + :updated-at "2026-10-27T00:00:00.000Z" + :functional {:mode :release + :default-features? true + :features [] + :cljs-profile-marks false} + :provenance {:git {:sha "build-revision" :dirty false}}}}) + +(defn- collected + [id context] + {:id id + :scene :case + :scene-description "test scene" + :scene-seed 1 + :params {} + :view {:scale 1 :x 0 :y 0} + :context context + :preparation {}}) + +(defn- next-count + [state key] + (inc (get @state key 0))) + +(defn- check-page + [page] + (when-not (and (map? page) (some? (:id page))) + (throw (ex-info "browser service requires the page object" + {:phase :browser-page-startup})))) + +(def ^:private hardware-graphics + {:webgl2 true :vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan, NVIDIA)" :software false}) + +(defn- fake-browser + [{:keys [launch-failure new-page-failure close-page-failure + prepare-failure run-failure screenshot-failure + interruption controller effective-graphics events close-failure] + :or {effective-graphics (constantly {}) + events [{:command :fixture/work :metrics {:command/elapsed [1]}}]}}] + (let [state (atom {:launches 0 :pages 0 :close-pages 0 :closes 0 + :host nil :runs {} :preparations {}})] + {:state state + :service + {:launch + (fn [_] + (let [n (next-count state :launches)] + (swap! state assoc :launches n) + (when (and launch-failure (launch-failure n)) + (throw (ex-info "browser launch failed" {:phase :browser-startup}))) + {:id n :version "FakeBrowser/1" :browser #js {}})) + :new-page + (fn [owner host _] + (when-not (and (string? (:base-url host)) + (some? (get-in host [:build :id]))) + (throw (ex-info "new-page requires the opened host" + {:phase :browser-page-startup}))) + (swap! state assoc :host host) + (let [n (next-count state :pages)] + (swap! state assoc :pages n) + (when (and new-page-failure (new-page-failure n)) + (throw (ex-info "browser page failed" {:phase :browser-page-startup}))) + {:page {:id n :browser-id (:id owner)} :context {}})) + :close-page + (fn [_] + (let [n (next-count state :close-pages)] + (swap! state assoc :close-pages n) + (when (and close-page-failure (close-page-failure n)) + (throw (ex-info "page close failed" {:phase :browser-page-close}))))) + :close + (fn [_] + (let [n (:closes (swap! state update :closes inc))] + (when (and close-failure (close-failure n)) + (throw (ex-info "browser shutdown failed" {:phase :browser-close}))))) + :connected? + (fn [_] true) + :prepare + (fn [page request _] + (check-page page) + (let [{:keys [preparation-id] case-desc :case} (transit/decode-str request) + case-id (:id case-desc) + failure (when prepare-failure + (prepare-failure case-id))] + (case failure + :throw + (throw (ex-info "scene preparation failed" {:phase :scene-build})) + + :response + {:status "failed" :phase "scene-build" :message "scene rejected"} + + (let [preparation {:preparation-id preparation-id + :case-id case-id + :fingerprint {:algorithm :sha-256 + :digest (apply str (repeat 64 "a"))} + :effective-graphics (effective-graphics case-id (:id page)) + :diagnostics {:browser-id (:browser-id page)} + :phases [{:phase :preparation :preparation-id preparation-id + :outcome :completed :events []}]}] + (swap! state assoc-in [:preparations preparation-id] case-id) + {:status "ok" :preparation preparation})))) + :run + (fn [page request _] + (check-page page) + (let [{:keys [preparation-id]} (transit/decode-str request) + case-id (get-in @state [:preparations preparation-id]) + n (inc (get-in @state [:runs case-id] 0)) + _ (swap! state assoc-in [:runs case-id] n) + signal (when interruption (interruption case-id n))] + (when signal + (let [reason (ex-info "run interrupted" + {:type ::interrupted :signal signal})] + (.abort controller reason) + (throw reason))) + (case (when run-failure (run-failure case-id n)) + :throw + (throw (ex-info "attempt failed" {:phase "interact"})) + + :response + {:status "failed" :phase "interact" :message "attempt rejected"} + + {:status "ok" + :phases [{:phase :main :preparation-id preparation-id :outcome :completed + :events events}]}))) + :screenshot + (fn [page file _] + (check-page page) + (when (and screenshot-failure (screenshot-failure file)) + (throw (ex-info "screenshot failed" {:phase :screenshot}))) + (fs/writeFileSync file "png"))}})) + +(defn- options + [fake out] + {:output-dir out + :seed 42 + :filter nil + :warmups 0 + :repetitions 1 + :timeout-ms 5000 + :startup-timeout-ms 5000 + :screenshots false + :remove false + :browser (:service fake)}) + +(defn- execute + ([fake cases opts] + (execute fake cases opts (js/AbortController.))) + ([_fake cases opts controller] + (run/execute (artifacts/frontend-root) + (runs/create-home {:root (artifacts/frontend-root) :run-id "test-run" + :output-dir (:output-dir opts)}) + (selected-build (build-dir) "test-build") + cases opts (.-signal controller)))) + +(defn- attempts-for + [run case-id] + (filterv #(= case-id (:case-id %)) (:attempts run))) + +(t/deftest ^:async unmeasured-successful-work-saves-a-terminal-failure + (let [fake (fake-browser {:events []}) + response (await (execute fake [(collected :case/unmeasured :reuse)] + (options fake (path/join @temporary-root "unmeasured")))) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in saved [:termination :reason]))) + (t/is (= :no-measurements (get-in saved [:case-outcomes :case/unmeasured :failure :reason]))) + (t/is (= :completed (get-in saved [:attempts 0 :outcome]))))) + +(defn- preparations-for + [run case-id] + (filterv #(= case-id (:case-id %)) (:preparations run))) + +(defn- saved-record + [result] + (result/decode (fs/readFileSync (get-in result [:home :record-path]) "utf8"))) + +(t/deftest ^:async retained-run-keeps-all-progress-levels-in-its-log + (doseq [level [:info :warn :debug]] + (let [root @temporary-root + cases [(collected :case/a :fresh) (collected :case/b :reuse)] + build (artifacts/create-build root {:default-features? true :features []} + {:git {:sha "fixture" :dirty false}}) + stdout (atom "") + stderr (atom "") + fake (fake-browser {}) + out (output-dir)] + (doseq [file [(:index-path build) (path/join (:browser-dir build) "browser.js") + (path/join (:renderer-dir build) "render-wasm.js") + (path/join (:renderer-dir build) "render-wasm.wasm") + (path/join (:renderer-dir build) "shared.js")]] + (fs/writeFileSync file "fixture" "utf8")) + (fs/writeFileSync (:cases-path build) + (str "module.exports = {buildIdentity() {return " + (js/JSON.stringify (transit/encode-str {:build-id (:id build)})) + ";}, collectCases() {return " + (js/JSON.stringify (transit/encode-str {:build-id (:id build) :cases cases})) + ";}};") "utf8") + (artifacts/write-manifest build (assoc (:manifest build) :status :ready)) + (log/configure {:level level :color :never + :stdout #js {:write #(swap! stdout str %)} + :stderr #js {:write #(swap! stderr str %)}}) + (let [resolved (:options (request/apply-defaults + {:command "run" + :options (assoc (dissoc (options fake out) :browser) + :build {:kind :existing :id (:id build)} + :warmups 1 :repetitions 2)})) + response (await (run/run root resolved nil {:browser (:service fake)})) + saved (saved-record response) + text (fs/readFileSync (get-in response [:home :log-path]) "utf8")] + (t/is (= 0 (:exit-code response))) + (t/is (= (:id build) (get-in saved [:metadata :build :provenance :build-id]))) + (t/is (str/includes? text "▸ Case case/a (fresh)\n")) + (doseq [case-text (rest (str/split text #"▸ Case "))] + (t/is (str/includes? case-text " attempt 1/3 completed (warmup)\n")) + (t/is (str/includes? case-text " attempt 2/3 completed\n")) + (t/is (str/includes? case-text " attempt 3/3 completed\n"))) + (t/is (str/includes? text "Browser FakeBrowser/1")) + (t/is (= (not= :warn level) (str/includes? @stderr "▸ Case case/a"))) + (t/is (= (= :debug level) (str/includes? @stderr "Browser FakeBrowser/1"))))))) + +(t/deftest ^:async fresh-and-reuse-cases-follow-their-context-policies + (let [fake (fake-browser {}) + cases [(collected :case/fresh :fresh) (collected :case/reuse :reuse)] + opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2) + result (await (execute fake cases opts)) + saved (saved-record result) + run (:run result) + fresh (attempts-for run :case/fresh) + reuse (attempts-for run :case/reuse)] + (t/is (= saved run)) + (t/is (= 6 (count (:attempts run)))) + (t/is (= 3 (count (preparations-for run :case/fresh)))) + (t/is (= 1 (count (preparations-for run :case/reuse)))) + (t/is (= 3 (count (distinct (map :preparation-id fresh))))) + (t/is (= 1 (count (distinct (map :preparation-id reuse))))) + (t/is (every? #(= :completed (:outcome %)) (:attempts run))) + (t/is (every? #(every? zero? (vals %)) (vals (:remaining run)))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (zero? (:exit-code result))) + (t/is (empty? (get-in run [:metadata :diagnostics :screenshots]))) + (t/is (empty? (filter #(str/ends-with? % ".png") + (array-seq (fs/readdirSync (get-in result [:home :dir])))))))) + +(t/deftest ^:async page-startup-failure-abandons-the-case-and-keeps-its-slots + (doseq [context [:fresh :reuse]] + (let [fake (fake-browser {:new-page-failure #(= 1 %)}) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b :fresh)] + opts)) + run (:run result) + state @(:state fake)] + (t/is (empty? (attempts-for run :case/a))) + (t/is (empty? (preparations-for run :case/a))) + (t/is (= {:warmups 0 :repetitions 2} + (get-in run [:remaining :case/a]))) + (t/is (= [:completed :completed] + (mapv :outcome (attempts-for run :case/b)))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :browser-page (get-in run [:termination :failure :stage]))) + (t/is (= 1 (:exit-code result))) + (t/is (= 2 (:launches state))) + (t/is (= 2 (:closes state)))))) + +(t/deftest ^:async failed-attempts-stop-the-case-and-keep-pending-slots + (doseq [failure [:throw :response]] + (let [fake (fake-browser {:run-failure (fn [case-id n] + (when (and (= case-id :case/fresh) (= n 1)) + failure))}) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/fresh :fresh)] opts)) + run (:run result) + tries (attempts-for run :case/fresh)] + (t/is (= [:failed] (mapv :outcome tries))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/fresh]))) + (t/is (= :failed (get-in run [:case-outcomes :case/fresh :outcome]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))) + (t/is (= 1 (count (preparations-for run :case/fresh))))))) + +(t/deftest ^:async reuse-preparation-failure-leaves-slots-and-continues-to-the-next-case + (doseq [kind [:throw :response]] + (let [fake (fake-browser {:prepare-failure #(when (= % :case/reuse) kind)}) + opts (assoc (options fake (output-dir)) :screenshots true :repetitions 2) + result (await (execute fake [(collected :case/reuse :reuse) + (collected :case/fresh :fresh)] opts)) + run (:run result) + shots (get-in run [:metadata :diagnostics :screenshots])] + (t/is (empty? (attempts-for run :case/reuse))) + (t/is (= {:warmups 0 :repetitions 2} (get-in run [:remaining :case/reuse]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for run :case/fresh)))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))) + (t/is (= :skipped (get-in shots ["case/reuse" :status]))) + (t/is (= :ok (get-in shots ["case/fresh" :status]))) + (t/is (= 1 (count (re-seq (re-pattern (if (= kind :throw) + "scene preparation failed" "scene rejected")) + (:report result))))) + (let [failure (last (get-in run [:metadata :diagnostics :failures]))] + (t/is (= :preparation (:stage failure))) + (t/is (= :case/reuse (:case-id failure))) + (t/is (= "scene-build" (:phase failure))))))) + +(t/deftest ^:async fresh-preparation-failure-consumes-its-slot-and-stops-the-case + (doseq [kind [:throw :response]] + (let [preparations (atom 0) + fake (fake-browser {:prepare-failure #(when (and (= % :case/fresh) + (= 1 (swap! preparations inc))) + kind)}) + result (await (execute fake [(collected :case/fresh :fresh)] + (assoc (options fake (output-dir)) :repetitions 2 :screenshots true))) + run (:run result) + tries (attempts-for run :case/fresh)] + (t/is (= [:failed] (mapv :outcome tries))) + (t/is (= (:preparation-id (first (preparations-for run :case/fresh))) + (:preparation-id (first tries)))) + (t/is (= "scene-build" (get-in (first tries) [:failure :phase]))) + (t/is (= 1 (count (preparations-for run :case/fresh)))) + (t/is (= :skipped (get-in run [:metadata :diagnostics :screenshots "case/fresh" :status]))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/fresh]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (count (re-seq (re-pattern (if (= kind :throw) + "scene preparation failed" "scene rejected")) + (:report result))))) + (t/is (str/includes? (:report result) "no-completed-measured-attempt")) + (t/is (= (:report result) (format/format-run (summary/summarize (saved-record result))))) + (t/is (= 1 (:exit-code result)))))) + +(t/deftest ^:async reuse-attempt-failure-leaves-pending-slots-and-continues-to-the-next-case + (doseq [kind [:throw :response]] + (let [fake (fake-browser {:run-failure (fn [case-id n] + (when (and (= case-id :case/reuse) (= n 1)) kind))}) + opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2 :screenshots true) + result (await (execute fake [(collected :case/reuse :reuse) + (collected :case/fresh :fresh)] opts)) + run (saved-record result)] + (t/is (= [:failed] (mapv :outcome (attempts-for run :case/reuse)))) + (t/is (= {:warmups 0 :repetitions 2} (get-in run [:remaining :case/reuse]))) + (t/is (= [:completed :completed :completed] (mapv :outcome (attempts-for run :case/fresh)))) + (t/is (= {:status :skipped :reason :no-completed-measured-attempt} + (get-in run [:metadata :diagnostics :screenshots "case/reuse"]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result)))))) + +(t/deftest ^:async final-attempt-failure-records-why-the-screenshot-was-skipped + (doseq [context [:fresh :reuse] + failure [:preparation :attempt] + :when (or (= context :fresh) (= failure :attempt))] + (let [fake (fake-browser (if (= failure :preparation) + {:prepare-failure (constantly :response)} + {:run-failure (fn [_ _] :response)})) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a context)] opts)) + run (saved-record result)] + (t/is (= [:failed] (mapv :outcome (:attempts run)))) + (t/is (= {:status :skipped :reason :no-completed-measured-attempt} + (get-in run [:metadata :diagnostics :screenshots "case/a"]))) + (t/is (= :failed (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result)))))) + +(t/deftest ^:async browser-close-failure-abandons-the-case-and-relaunches-for-the-next + (let [fake (fake-browser {:close-page-failure #(= % 1)}) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/a :fresh) + (collected :case/b :fresh)] opts)) + run (:run result) + state @(:state fake)] + (t/is (= 2 (:launches state))) + (t/is (= 1 (count (attempts-for run :case/a)))) + (t/is (= :completed (:outcome (first (attempts-for run :case/a))))) + (t/is (= :failed (get-in run [:case-outcomes :case/a :outcome]))) + (t/is (= 1 (get-in run [:remaining :case/a :repetitions]))) + (t/is (= 2 (count (attempts-for run :case/b)))) + (t/is (every? #(= :completed (:outcome %)) (attempts-for run :case/b))) + (t/is (every? zero? (vals (get-in run [:remaining :case/b])))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async page-close-failure-after-the-final-slot-is-a-harness-failure + (let [fake (fake-browser {:close-page-failure #(= % 1)}) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + run (:run result)] + (t/is (= :failed (get-in run [:termination :reason]))) + (t/is (every? zero? (vals (get-in run [:remaining :case/a])))) + (t/is (= :browser-close + (:stage (last (get-in run [:metadata :diagnostics :failures]))))) + (t/is (= :browser-close (get-in run [:termination :failure :stage]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async initial-browser-launch-failure-aborts-before-any-case + (let [fake (fake-browser {:launch-failure #(= % 1)}) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + run (:run result) + state @(:state fake)] + (t/is (= 1 (:launches state))) + (t/is (empty? (:attempts run))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures]))))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async browser-process-details-survive-saved-run-report + (let [context {:command "chromium" :exit-code 17 :signal "SIGKILL" + :endpoint "ws://127.0.0.1:1234/devtools/browser/owner"} + fake (assoc-in (fake-browser {}) [:service :launch] + (fn [_] + (throw (ex-info "browser launch failed" context)))) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + saved (saved-record result)] + (t/is (= 1 (:exit-code result))) + (t/is (= context (select-keys (get-in saved [:termination :failure]) (keys context)))) + (t/is (= context (select-keys (first (get-in saved [:metadata :diagnostics :failures])) + (keys context)))) + (t/is (str/includes? (:report result) "chromium")) + (t/is (str/includes? (:report result) "SIGKILL")) + (t/is (str/includes? (:report result) (:endpoint context))))) + +(t/deftest ^:async hardware-preparations-complete-fresh-and-reuse-cases + (let [fake (fake-browser {:effective-graphics (constantly hardware-graphics)}) + opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1 :repetitions 2) + result (await (execute fake [(collected :case/fresh :fresh) + (collected :case/reuse :reuse)] opts)) + run (saved-record result)] + (t/is (= 0 (:exit-code result))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= 6 (count (:attempts run)))) + (t/is (every? #(= hardware-graphics (:effective-graphics %)) (:preparations run))) + (t/is (true? (get-in run [:metadata :diagnostics :require-gpu?]))))) + +(t/deftest ^:async required-gpu-failures-preserve-evidence-and-all-attempt-slots + (doseq [context [:fresh :reuse] + graphics [(assoc hardware-graphics :renderer "ANGLE (SwiftShader)" :software true) + (assoc hardware-graphics :webgl2 false) + (assoc hardware-graphics :renderer nil) + (assoc hardware-graphics :renderer " UnKnOwN ") + (assoc hardware-graphics :vendor "") + {}]] + (let [fake (fake-browser {:effective-graphics (constantly graphics)}) + opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1) + result (await (execute fake [(collected :case/a context) + (collected :case/b :reuse)] opts)) + run (saved-record result) + failure (get-in run [:termination :failure]) + state @(:state fake)] + (t/is (= 1 (:exit-code result))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= [graphics] (mapv :effective-graphics (:preparations run)))) + (t/is (= [] (:attempts run))) + (t/is (= {:case/a {:warmups 1 :repetitions 1} + :case/b {:warmups 1 :repetitions 1}} (:remaining run))) + (t/is (= :gpu (:stage failure))) + (t/is (= graphics (get-in failure [:partial :graphics]))) + (t/is (str/includes? (:message failure) (pr-str (:renderer graphics)))) + (t/is (str/includes? (:message failure) "./manage.sh run-devenv --gpu")) + (t/is (str/includes? (:message failure) "pnpm run render-bench")) + (t/is (= 1 (:closes state)))))) + +(t/deftest ^:async graphics-rejection-preserves-earlier-attempts-and-stops-the-run + (doseq [context [:fresh :reuse]] + (let [software (assoc hardware-graphics :renderer "SwiftShader" :software true) + fake (fake-browser {:effective-graphics (fn [_ page-id] + (if (= page-id 1) hardware-graphics software))}) + opts (assoc (options fake (output-dir)) :require-gpu true :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] opts)) + run (saved-record result) + failure (get-in run [:termination :failure])] + (t/is (= 1 (:exit-code result))) + (t/is (= :gpu (:stage failure))) + (t/is (= software (get-in failure [:partial :graphics]))) + (t/is (= (if (= context :fresh) 1 2) (count (:attempts run)))) + (t/is (every? #(= :completed (:outcome %)) (:attempts run))) + (t/is (= [hardware-graphics software] (mapv :effective-graphics (:preparations run)))) + (t/is (= {:case/a {:warmups 0 :repetitions (if (= context :fresh) 1 0)} + :case/b {:warmups 0 :repetitions 2}} (:remaining run))) + (t/is (= 1 (:closes @(:state fake))))))) + +(t/deftest ^:async sigint-and-sigterm-persist-partial-interrupted-records + (doseq [context [:fresh :reuse] + [signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + fake (fake-browser {:controller controller + :interruption (fn [case-id n] + (when (and (= case-id :case/a) (= n 1)) + signal))}) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] + opts controller)) + run (:run result) + saved (saved-record result) + shots (get-in run [:metadata :diagnostics :screenshots])] + (t/is (= saved run)) + (t/is (= :interrupted (get-in run [:termination :reason]))) + (t/is (= signal (get-in run [:termination :signal]))) + (t/is (= code (:exit-code result))) + (t/is (= 1 (count (attempts-for run :case/a)))) + (t/is (empty? (attempts-for run :case/b))) + (t/is (= 1 (get-in run [:remaining :case/b :repetitions]))) + (t/is (= :skipped (get-in shots ["case/b" :status]))) + (t/is (= :not-run (get-in shots ["case/b" :reason]))) + (t/is (pos? (:closes @(:state fake))))))) + +(t/deftest ^:async interruption-before-an-attempt-preserves-unspent-slots + (doseq [context [:fresh :reuse] + method [:launch :new-page :prepare] + [reason signal code] [["SIGINT" "SIGINT" 130] + ["SIGTERM" "SIGTERM" 143] + [:SIGTERM "SIGTERM" 143] + [(ex-info "stopped" {:signal "SIGTERM"}) "SIGTERM" 143] + [(js/Error. "stopped") "SIGINT" 130]]] + (let [controller (js/AbortController.) + base (fake-browser {}) + stop (^:async fn [& _] + (let [cause (ex-info "interrupted setup" {:signal signal :phase :preparation})] + (.abort controller reason) + (throw cause))) + fake (assoc base :service (assoc (:service base) method stop)) + opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] opts controller)) + run (saved-record result) + consumed (if (and (= method :prepare) (= context :fresh)) 1 0)] + (t/is (= :interrupted (get-in run [:termination :reason]))) + (t/is (= signal (get-in run [:termination :signal]))) + (t/is (= code (:exit-code result))) + (t/is (= consumed (count (:attempts run)))) + (t/is (= {:warmups (- 1 consumed) :repetitions 2} (get-in run [:remaining :case/a]))) + (t/is (= {:warmups 1 :repetitions 2} (get-in run [:remaining :case/b]))) + (t/is (= (if (and (= method :prepare) (= context :reuse)) [:preparation] []) + (mapv :stage (get-in run [:metadata :diagnostics :failures]))))))) + +(t/deftest ^:async phase-and-output-failures-retain-evidence-and-stop-the-case + (doseq [context [:fresh :reuse] + phase [:preparation :main :reset :invalid-preparation :contradictory-preparation :invalid-main :invalid-reset]] + (let [base (fake-browser {:prepare-failure (when (= :preparation phase) + #(when (= :case/a %) :response))}) + services (:service base) + fake (assoc base :service + (assoc services + :prepare (fn [page request bounds] + (let [response ((:prepare services) page request bounds)] + (if (= :case/a (get-in (transit/decode-str request) [:case :id])) + (case phase + :invalid-preparation (update response :preparation dissoc :fingerprint) + :contradictory-preparation + (-> response + (assoc-in [:preparation :failure] + {:phase "preparation" :message "callback failed"}) + (assoc-in [:preparation :phases 0 :outcome] :failed)) + response) + response))) + :run (fn [page request bounds] + (let [id (:preparation-id (transit/decode-str request)) + case-id (get-in @(:state base) [:preparations id]) + block {:phase :main :preparation-id id :outcome :completed + :events [{:command :fixture/work :metrics {:command/elapsed [1]}}]}] + (if (= :case/a case-id) + (case phase + :main {:status "failed" :phase "main" :message "main failed" + :cause [{:message "WASM trap" :code "E_RENDER"}] + :reason :renderer-rejected :deadline-exceeded false + :partial {:completed-call 7} + :phases [(assoc block :outcome :failed)]} + :reset {:status "failed" :phase "reset" :message "reset failed" + :cause [{:message "WASM trap" :code "E_RENDER"}] + :reason :renderer-rejected :deadline-exceeded false + :partial {:completed-call 7} + :phases [block (assoc block :phase :reset :outcome :failed)]} + :invalid-main {:status "ok" :phases []} + :invalid-reset {:status "ok" :phases [block {:phase :reset}]}) + ((:run services) page request bounds)))))) + response (await (execute fake [(collected :case/a context) (collected :case/b context)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response) + prep-failed? (#{:preparation :invalid-preparation :contradictory-preparation} phase) + attempts (attempts-for saved :case/a)] + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= (if (and prep-failed? (= :reuse context)) [] [:failed]) (mapv :outcome attempts))) + (t/is (= {:warmups 0 :repetitions (if (and prep-failed? (= :reuse context)) 2 1)} + (get-in saved [:remaining :case/a]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for saved :case/b)))) + (when (#{:main :reset} phase) + (t/is (= {:cause [{:message "WASM trap" :code "E_RENDER"}] + :reason :renderer-rejected :deadline-exceeded false} + (select-keys (:failure (first attempts)) [:cause :reason :deadline-exceeded]))) + (t/is (= {:completed-call 7} (:partial (first attempts))))) + (when (= :contradictory-preparation phase) + (t/is (= "invalid-output" (get-in (first (preparations-for saved :case/a)) [:failure :phase])))) + (when (= :invalid-reset phase) + (t/is (= [:main] (mapv :phase (:phases (first attempts))))) + (t/is (= {:phase :reset} (get-in (first attempts) [:partial :response :phases 1])))) + (when (#{:invalid-main :invalid-reset} phase) + (t/is (= "invalid-output" (get-in (first attempts) [:failure :phase]))))))) + +(t/deftest ^:async malformed-bridge-diagnostics-stop-the-case-with-delivered-evidence + (doseq [stage [:prepare :run] + evidence [nil "unexpected response" + {:status "unknown" :phase "main" :message "unexpected status"} + {:status "failed" :phase "main"} + {:status "failed" :phase "main" :message "bad cause" :cause [:invalid]}]] + (let [base (fake-browser {}) + service (get-in base [:service stage]) + delivered (atom nil) + fake (assoc-in base [:service stage] + (fn [page request bounds] + (let [response (service page request bounds) + params (transit/decode-str request) + case-id (if (= :prepare stage) + (get-in params [:case :id]) + (get-in @(:state base) [:preparations (:preparation-id params)]))] + (if (= :case/a case-id) + (let [phases (if (= :prepare stage) + (get-in response [:preparation :phases]) + (conj (:phases response) {:phase :reset})) + rejected (cond-> evidence + (map? evidence) (assoc :phases phases))] + (reset! delivered rejected) + (transit/decode-str (transit/encode-str rejected))) + response)))) + response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response) + failed (if (= :prepare stage) + (first (preparations-for saved :case/a)) + (first (attempts-for saved :case/a)))] + (t/is (= 1 (:exit-code response))) + (t/is (= "invalid-output" (get-in failed [:failure :phase]))) + (t/is (= "invalid bridge response" (get-in failed [:failure :message]))) + (t/is (= @delivered (if (= :prepare stage) + (get-in failed [:failure :partial :response]) + (get-in failed [:partial :response])))) + (t/is (= (if (map? evidence) [(first (:phases @delivered))] []) (:phases failed))) + (t/is (= {:warmups 0 :repetitions (if (= :prepare stage) 2 1)} + (get-in saved [:remaining :case/a]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for saved :case/b)))) + (t/is (str/includes? (:report response) "invalid bridge response"))))) + +(t/deftest ^:async stale-bridge-results-stop-the-case-with-ownership-diagnostics + (doseq [stage [:prepare :run]] + (let [base (fake-browser {}) + fake (assoc-in base [:service stage] (constantly {:status "stale"})) + response (await (execute fake [(collected :case/a :reuse)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response) + failure (get-in saved [:case-outcomes :case/a :failure])] + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= "stale" (:status failure))) + (t/is (= (if (= :prepare stage) "preparation" "attempt") (:phase failure))) + (t/is (= "browser context was superseded" (:message failure))) + (t/is (= {:warmups 0 :repetitions (if (= :prepare stage) 2 1)} + (get-in saved [:remaining :case/a])))))) + +(t/deftest ^:async runner-terminates-a-blocked-page-and-completes-the-next-case + (let [dir (build-dir) + owners (atom []) + prepared (atom {}) + attempts (atom {}) + identity-text (.stringify js/JSON (transit/encode-str {:build-id "test-build"})) + services (assoc browser/service + :launch (^:async fn [options] + (when-let [previous (last @owners)] + (t/is (false? (.isConnected ^js (:browser previous)))) + (t/is (not (fs/existsSync (:user-data-dir previous))))) + (let [owner (await (browser/launch options))] + (swap! owners conj owner) + owner)) + :prepare (fn [_ request _] + (let [{:keys [preparation-id] definition :case} (transit/decode-str request)] + (swap! prepared assoc preparation-id (:id definition)) + {:status "ok" + :preparation {:preparation-id preparation-id :case-id (:id definition) + :fingerprint {:algorithm :sha-256 :digest (apply str (repeat 64 "a"))} + :effective-graphics {} :diagnostics {} + :phases [{:phase :preparation :preparation-id preparation-id + :outcome :completed :events []}]}})) + :run (^:async fn [page request bounds] + (let [id (:preparation-id (transit/decode-str request)) + case-id (get @prepared id) + n (get (swap! attempts update case-id (fnil inc 0)) case-id)] + (await (browser/within + #(.evaluate ^js page + (if (and (= :case/a case-id) (= n 2)) + "while (true) {}" "1")) + bounds)) + {:status "ok" + :phases [{:phase :main :preparation-id id :outcome :completed + :events [{:command :fixture/work :metrics {:command/elapsed [1]}}]}]})))] + (fs/writeFileSync (path/join dir "index.html") + (str "")) + (try + (let [opts (assoc (options nil (output-dir)) :browser services :repetitions 2 + :timeout-ms 3000 :startup-timeout-ms 30000) + response (await (run/execute (artifacts/frontend-root) + (runs/create-home {:root (artifacts/frontend-root) :run-id "blocked-case" + :output-dir (:output-dir opts)}) + (selected-build dir "test-build") + [(collected :case/a :reuse) (collected :case/b :reuse)] opts nil)) + saved (saved-record response)] + (t/is (= [:completed :failed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for saved :case/b)))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome]))) + (t/is (= 2 (count @owners))) + (t/is (not= (:user-data-dir (first @owners)) (:user-data-dir (second @owners)))) + (doseq [owner @owners] + (t/is (false? (.isConnected ^js (:browser owner)))) + (t/is (not (fs/existsSync (:user-data-dir owner)))))) + (finally + (doseq [owner @owners] + (await (browser/close owner))))))) + +(t/deftest ^:async opening-and-screenshot-work-share-the-case-deadline + (doseq [stage [:page :screenshot]] + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {}) + spend (fn [page bounds] + (when (= 1 (:id page)) + (swap! clock + 300) + (when (>= @clock (:deadline bounds)) + (throw (ex-info "browser work timed out" {:phase (name stage)}))))) + fake (if (= :page stage) + (assoc-in base [:service :new-page] + (fn [owner host bounds] + (let [opened ((get-in base [:service :new-page]) owner host bounds)] + (spend (:page opened) bounds) + opened))) + (assoc-in base [:service :screenshot] + (fn [page file bounds] + (spend page bounds) + ((get-in base [:service :screenshot]) page file bounds))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 250 :screenshots true))) + saved (saved-record response)] + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome]))) + (t/is (= {:warmups 0 :repetitions (if (= :page stage) 1 0)} + (get-in saved [:remaining :case/a]))) + (when (= :screenshot stage) + (t/is (= [:completed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= 1 (count (re-seq #"browser work timed out" (:report response))))) + (t/is (str/includes? (:report response) "\"case/a\": failed; see Harness 0")) + (t/is (str/includes? (:report response) "File \"case-a.png\"")))) + (finally + (set! (.-now js/Date) system-now)))))) + +(t/deftest ^:async failed-browser-disposal-stops-later-cases + (let [fake (fake-browser {:run-failure (fn [_ _] :response) :close-failure #(= 1 %)}) + response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response)] + (t/is (= [:case/a] (mapv :case-id (:attempts saved)))) + (t/is (= {:warmups 0 :repetitions 2} (get-in saved [:remaining :case/b]))) + (t/is (= :browser-close (get-in saved [:termination :failure :stage]))))) + +(t/deftest ^:async distinct-harness-failures-with-equal-diagnostics-appear-separately + (let [fake (fake-browser {:run-failure (fn [_ _] :response) + :close-failure (constantly true)}) + response (await (execute fake [(collected :case/a :reuse)] + (options fake (output-dir)))) + saved (saved-record response) + report (:report response)] + (t/is (= [:browser-close :cleanup] (mapv :stage (get-in saved [:metadata :diagnostics :failures])))) + (t/is (str/includes? report "Harness 0")) + (t/is (str/includes? report "Harness 1")) + (t/is (str/includes? report ":browser-close")) + (t/is (str/includes? report ":cleanup")) + (t/is (= 2 (count (re-seq #"browser shutdown failed" report)))) + (t/is (= 1 (count (re-seq #"attempt rejected" report)))))) + +(t/deftest ^:async failed-startup-cleanup-aborts-even-after-the-case-deadline + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {:run-failure (fn [_ _] :response)}) + fake (assoc-in base [:service :launch] + (fn [bounds] + (let [owner ((get-in base [:service :launch]) bounds)] + (when (= 2 (:id owner)) + (reset! clock (inc (:deadline bounds))) + (throw (ex-info "browser shutdown failed after startup error" + {:type ::browser/startup-cleanup}))) + owner)))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse) + (collected :case/c :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 250))) + saved (saved-record response)] + (t/is (= :failed (get-in saved [:case-outcomes :case/b :outcome]))) + (t/is (= {:warmups 0 :repetitions 1} (get-in saved [:remaining :case/c]))) + (t/is (= :browser-launch (get-in saved [:termination :failure :stage])))) + (finally + (set! (.-now js/Date) system-now))))) + +(t/deftest ^:async cleanup-time-does-not-change-the-case-failure + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {:new-page-failure #(= 1 %)}) + fake (assoc-in base [:service :close] + (fn [owner] + (swap! clock + 1000) + ((get-in base [:service :close]) owner)))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 500))) + saved (saved-record response)] + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome])))) + (finally + (set! (.-now js/Date) system-now))))) + +(t/deftest ^:async service-cleanup-preserves-the-originating-failure-classification + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {:new-page-failure #(= 1 %)}) + fake (assoc-in base [:service :new-page] + (fn [owner host bounds] + (try + ((get-in base [:service :new-page]) owner host bounds) + (catch :default cause + (let [annotated (errors/with-context cause "page startup failed" + {::browser/deadline-exceeded false})] + (swap! clock + 1000) + (throw annotated))))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 250))) + saved (saved-record response)] + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome])))) + (finally + (set! (.-now js/Date) system-now))))) + +(t/deftest ^:async case-expiry-keeps-delivered-attempts-and-continues-in-a-new-browser + (doseq [context [:fresh :reuse]] + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {}) + active-case (atom nil) + spend (fn [duration bounds] + (when (= :case/a @active-case) + (swap! clock + duration) + (when (>= @clock (:deadline bounds)) + (throw (ex-info "browser work timed out" {:phase (:phase bounds)}))))) + services (:service base) + fake (assoc base :service + (assoc services + :prepare (fn [page request bounds] + (reset! active-case (get-in (transit/decode-str request) [:case :id])) + (spend 25 bounds) + ((:prepare services) page request bounds)) + :run (fn [page request bounds] + (spend 110 bounds) + ((:run services) page request bounds))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a context) (collected :case/b context)] + (assoc (options fake (output-dir)) + :warmups 1 :repetitions 3 :timeout-ms 350))) + saved (saved-record response) + first-prep (first (preparations-for saved :case/a)) + next-prep (first (preparations-for saved :case/b))] + (t/is (= [:completed :completed :failed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= {:warmups 0 :repetitions 1} (get-in saved [:remaining :case/a]))) + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= [:completed :completed :completed :completed] + (mapv :outcome (attempts-for saved :case/b)))) + (t/is (not= (get-in first-prep [:diagnostics :browser-id]) + (get-in next-prep [:diagnostics :browser-id])))) + (finally + (set! (.-now js/Date) system-now)))))) + +(t/deftest ^:async preparation-and-rendering-share-the-case-budget + (doseq [context [:fresh :reuse]] + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {}) + work (^:async fn [bounds delegate page request] + (await (js/Promise.resolve nil)) + (swap! clock + 150) + (when (>= @clock (:deadline bounds)) + (throw (ex-info "browser work timed out" {:phase (:phase bounds)}))) + (delegate page request bounds)) + fake (assoc base :service + (assoc (:service base) + :prepare (fn [page request bounds] + (work bounds (:prepare (:service base)) page request)) + :run (fn [page request bounds] + (work bounds (:run (:service base)) page request))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [result (await (execute fake [(collected :case/a context)] + (assoc (options fake (output-dir)) :timeout-ms 250))) + saved (saved-record result)] + (t/is (= :failed (:outcome (first (:attempts saved))))) + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= 1 (:exit-code result)))) + (finally + (set! (.-now js/Date) system-now)))))) + +(t/deftest ^:async browser-disconnection-leaves-pending-slots-and-relaunches-for-the-next-case + (let [base (fake-browser {}) + disconnected (atom nil) + new-page (:new-page (:service base)) + service (assoc (:service base) + :connected? (fn [owner] + (compare-and-set! disconnected nil (:browser owner)) + (not (identical? @disconnected (:browser owner)))) + :new-page (fn [owner host options] + (when (identical? @disconnected (:browser owner)) + (throw (js/Error. "browser disconnected"))) + (new-page owner host options))) + fake (assoc base :service service) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/a :fresh) (collected :case/b :fresh)] opts)) + run (saved-record result)] + (t/is (= [:completed] (mapv :outcome (attempts-for run :case/a)))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for run :case/b)))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async provenance-timeout-saves-the-unattempted-run + (let [bin (path/join @temporary-root "bin") + prior (.-PATH (.-env js/process)) + _ (fs/mkdirSync bin) + _ (fs/writeFileSync (path/join bin "git") + (str "#!" (.-execPath js/process) "\nsetInterval(() => {}, 1000);\n") + #js {:mode 8r755}) + fake (fake-browser {})] + (try + (set! (.-PATH (.-env js/process)) (str bin path/delimiter prior)) + (let [result (await (execute fake [(collected :case/a :fresh)] + (assoc (options fake (output-dir)) :startup-timeout-ms 50))) + run (saved-record result)] + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :runner-git (get-in run [:termination :failure :stage]))) + (t/is (= {:case/a {:warmups 0 :repetitions 1}} (:remaining run))) + (t/is (= [] (:attempts run))) + (t/is (= 1 (:exit-code result)))) + (finally + (set! (.-PATH (.-env js/process)) prior))))) + +(t/deftest ^:async unavailable-build-directory-saves-a-server-startup-failure + (let [root @temporary-root + selected (selected-build (build-dir) "unavailable-build") + home (runs/create-home {:root root :run-id "unavailable-build" :output-dir (output-dir)}) + _ (fs/renameSync (:dir selected) (str (:dir selected) ".unavailable")) + fake (fake-browser {}) + result (await (run/execute root home selected [(collected :case/a :fresh)] + (options fake (:dir home)) (.-signal (js/AbortController.)))) + run (saved-record result)] + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :server-startup (get-in run [:termination :failure :stage]))) + (t/is (= {:case/a {:warmups 0 :repetitions 1}} (:remaining run))) + (t/is (= [] (:attempts run))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async failed-browser-relaunch-aborts-the-pending-cases + (let [fake (fake-browser {:new-page-failure #(= % 1) :launch-failure #(= % 2)}) + cases [(collected :case/a :fresh) (collected :case/b :reuse) (collected :case/c :fresh)] + result (await (execute fake cases (options fake (output-dir)))) + run (saved-record result)] + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= [] (:attempts run))) + (t/is (= [:browser-page :browser-launch] (mapv :stage (get-in run [:metadata :diagnostics :failures])))) + (t/is (= {:case/a {:warmups 0 :repetitions 1} + :case/b {:warmups 0 :repetitions 1} + :case/c {:warmups 0 :repetitions 1}} (:remaining run))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async interruption-exit-status-takes-precedence-over-finalization-errors + (doseq [failure [:cleanup :record] + [signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + out (output-dir) + base (fake-browser {:controller controller :interruption (fn [_ _] signal)}) + close (:close (:service base)) + fake (assoc base :service + (assoc (:service base) :close + (fn [owner] + (close owner) + (if (= failure :cleanup) + (throw (js/Error. "cleanup rejected")) + (let [record (path/join out "run.transit")] + (fs/renameSync record (str record ".saved")) + (fs/mkdirSync record)))))) + result (await (execute fake [(collected :case/a :fresh)] (options fake out) controller))] + (t/is (= code (:exit-code result))) + (t/is (= :interrupted (get-in result [:run :termination :reason]))) + (t/is (= signal (get-in result [:run :termination :signal]))) + (t/is (seq (:errors result)))))) + +(t/deftest ^:async log-failure-still-closes-pages-browsers-and-hosts + (let [out (output-dir) + base (fake-browser {}) + prepare (:prepare (:service base)) + service (assoc (:service base) + :prepare + (fn [page request bounds] + (let [response (prepare page request bounds)] + (fs/renameSync (path/join out "run.log") + (path/join out "run.log.saved")) + (fs/mkdirSync (path/join out "run.log")) + response))) + fake (assoc base :service service) + result (await (execute fake [(collected :case/a :fresh)] + (options fake out))) + state @(:state fake) + host (:host state)] + (t/is (some errors/fatal? (:errors result))) + (t/is (= 1 (:exit-code result))) + (t/is (empty? (attempts-for (:run result) :case/a))) + (t/is (= 1 (:close-pages state))) + (t/is (= 1 (:closes state))) + (t/is (false? (.-listening ^js (:server host)))))) + +(t/deftest ^:async storage-failure-survives-recovery-and-cleanup-error-takes-precedence + (doseq [cleanup-fails [false true]] + (let [out (output-dir) + file (path/join out "run.log") + saved (str file ".saved") + base (fake-browser {}) + prepare (:prepare (:service base)) + close (:close (:service base)) + service (assoc (:service base) + :prepare (fn [page request bounds] + (let [response (prepare page request bounds)] + (fs/renameSync file saved) + (fs/mkdirSync file) + response)) + :close (fn [owner] + (close owner) + (fs/rmdirSync file) + (fs/renameSync saved file) + (when cleanup-fails + (throw (js/Error. "cleanup rejected"))))) + fake (assoc base :service service) + result (await (execute fake [(collected :case/a :fresh)] (options fake out))) + run (saved-record result)] + (t/is (= 1 (:exit-code result))) + (t/is (some errors/fatal? (:errors result))) + (t/is (str/includes? (ex-message (first (:errors result))) + (if cleanup-fails "cleanup rejected" "cannot append benchmark log"))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (str/includes? (fs/readFileSync file "utf8") "Run incomplete"))))) + +(t/deftest ^:async cleanup-retains-page-browser-and-server-failures-in-the-saved-report + (let [base (fake-browser {:close-page-failure (constantly true) :close-failure (constantly true)}) + fake (assoc-in base [:service :prepare] + (fn [& _] (throw (errors/fatal-error "preparation transport failed" {} nil)))) + close server/close] + (await (mock/with-mocks* + {server/close (fn [host] + (.then (close host) (fn [_] (throw (js/Error. "server disposal failed")))))} + (let [response (await (execute fake [(collected :case/a :fresh)] (options fake (output-dir)))) + saved (saved-record response) + failure (get-in saved [:termination :failure])] + (t/is (= 1 (:exit-code response))) + (t/is (= ["page close failed" "browser shutdown failed" "server disposal failed"] + (mapv :message (:errors failure)))) + (t/is (str/includes? (:report response) "server disposal failed")) + (t/is (str/includes? (:report response) "browser shutdown failed"))))))) + +(t/deftest ^:async cleanup-error-keeps-completed-attempt-evidence-and-exits-nonzero + (let [base (fake-browser {}) + fake (assoc base :service (assoc (:service base) :close + (fn [_] (throw (js/Error. "cleanup rejected"))))) + result (await (execute fake [(collected :case/a :fresh)] (options fake (output-dir)))) + run (saved-record result)] + (t/is (= [:completed] (mapv :outcome (:attempts run)))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= :cleanup (get-in run [:termination :failure :stage]))) + (t/is (= "cleanup rejected" (ex-message (first (:errors result))))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async interruption-during-shutdown-is-persisted + (doseq [[signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + base (fake-browser {}) + close (:close (:service base)) + service (assoc (:service base) + :close + (fn [owner] + (when-not (.-aborted (.-signal controller)) + (.abort controller signal)) + (close owner))) + fake (assoc base :service service) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)) + controller)) + run (:run result)] + (t/is (= :interrupted (get-in run [:termination :reason]))) + (t/is (= signal (get-in run [:termination :signal]))) + (t/is (= code (:exit-code result))) + (t/is (= run (saved-record result))) + (t/is (= 1 (:closes @(:state fake))))))) + +(t/deftest ^:async screenshots-record-files-and-failures-as-diagnostics + (doseq [context [:fresh :reuse]] + (let [fake (fake-browser {}) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a context)] opts)) + run (:run result) + shot (get-in run [:metadata :diagnostics :screenshots "case/a"])] + (t/is (= :ok (:status shot))) + (t/is (= "case-a.png" (:file shot))) + (t/is (fs/existsSync (path/join (get-in result [:home :dir]) "case-a.png"))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (str/includes? (:report result) "Screenshots")) + (t/is (str/includes? (:report result) "case-a.png")) + (t/is (zero? (:exit-code result))))) + (let [fake (fake-browser {:screenshot-failure (constantly true)}) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a :fresh)] opts)) + run (:run result) + shot (get-in run [:metadata :diagnostics :screenshots "case/a"])] + (t/is (= :failed (:status shot))) + (t/is (= "case-a.png" (:file shot))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (str/includes? (:report result) "\"case/a\": warning")) + (t/is (= 1 (count (re-seq #"screenshot failed" (:report result))))) + (t/is (str/includes? (:report result) "case-a.png")) + (t/is (zero? (:exit-code result))))) + +(t/deftest ^:async metadata-records-build-and-controller-provenance-and-budgets + (let [fake (fake-browser {}) + opts (assoc (options fake (output-dir)) + :timeout-ms 1234 :build-timeout-ms 5678 :startup-timeout-ms 9012) + result (await (execute fake [(collected :case/a :fresh)] opts)) + run (:run result) + diag (get-in run [:metadata :diagnostics])] + (t/is (= {:case-ms 1234} (get-in run [:plan :timeout-policy]))) + (t/is (= {:sha "build-revision" :dirty false} (get-in run [:metadata :git]))) + (t/is (= {:mode :release :default-features? true :features [] :cljs-profile-marks false} + (get-in run [:metadata :build :functional]))) + (t/is (= "test-build" (get-in run [:metadata :build :provenance :build-id]))) + (t/is (true? (get-in run [:metadata :scored?]))) + (t/is (= "FakeBrowser/1" (get-in run [:metadata :environment :browser]))) + (t/is (= 1234 (:timeout-ms diag))) + (t/is (= 5678 (:build-timeout-ms diag))) + (t/is (= 9012 (:startup-timeout-ms diag))) + (t/is (= 42 (:seed diag))) + (t/is (= 0 (:warmups diag))) + (t/is (= 1 (:repetitions diag))) + (t/is (false? (:screenshots? diag))) + (t/is (map? (:runner-git diag))) + (t/is (= :unverified (:runtime-helper-identity diag))))) + +(t/deftest ^:async run-metadata-copies-only-saved-build-environment-evidence + (doseq [environment [::missing {} {"CFLAGS" "" "NODE_ENV" "recorded-build-value"}]] + (let [fake (fake-browser {}) + root (artifacts/frontend-root) + opts (options fake (output-dir)) + build (cond-> (selected-build (build-dir) "test-build") + (not= ::missing environment) (assoc-in [:manifest :environment] environment)) + home (runs/create-home {:root root :run-id "test-run" :output-dir (:output-dir opts)}) + controller (js/AbortController.) + executed (await (run/execute root home build [(collected :case/a :fresh)] + opts (.-signal controller))) + saved (saved-record executed) + facts (get-in saved [:metadata :build])] + (t/is (zero? (:exit-code executed))) + (t/is (= saved (:run executed))) + (if (= ::missing environment) + (t/is (not (contains? facts :environment))) + (t/is (= environment (:environment facts))))))) + +(t/deftest ^:async remove-deletes-the-used-build-after-the-record-is-saved + (let [root @temporary-root + id "removable-build" + dir (path/join (artifacts/managed-root root) "builds" id) + _ (fs/mkdirSync dir #js {:recursive true}) + _ (fs/writeFileSync (path/join dir "index.html") "") + fake (fake-browser {}) + out (path/join @temporary-root "remove-run") + home (runs/create-home {:root root :run-id "remove-run" + :output-dir out :remove true}) + result (await (run/execute root home (selected-build dir id) + [(collected :case/a :fresh)] + (assoc (options fake out) :remove true) + (.-signal (js/AbortController.))))] + (t/is (fs/existsSync (get-in result [:home :record-path]))) + (t/is (fs/existsSync dir)) + (t/is (not (fs/existsSync (path/join dir "index.html")))) + (t/is (= :completed (get-in result [:run :termination :reason]))) + (t/is (zero? (:exit-code result))))) + + +(t/deftest ^:async unresolved-failure-retains-build-and-earlier-results-on-interruption + (doseq [signal [nil "SIGINT" "SIGTERM"]] + (let [root @temporary-root + id (str "unresolved-" (or signal "failure")) + dir (path/join (artifacts/managed-root root) "builds" id) + index (path/join dir "index.html") + controller (js/AbortController.) + failure (errors/fatal-error "Command group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + base (fake-browser {}) + prepare (get-in base [:service :prepare]) + fake (assoc-in base [:service :prepare] + (fn [page request bounds] + (if (= :case/b (get-in (transit/decode-str request) [:case :id])) + (do + (when signal (.abort controller signal)) + (throw failure)) + (prepare page request bounds)))) + out (path/join root id) + home (runs/create-home {:root root :run-id id :output-dir out :remove true})] + (fs/mkdirSync dir #js {:recursive true}) + (fs/writeFileSync index "retained renderer") + (let [response (await (run/execute root home (selected-build dir id) + [(collected :case/a :reuse) (collected :case/b :reuse) (collected :case/c :reuse)] + (assoc (options fake out) :remove true) (.-signal controller))) + saved (saved-record response)] + (t/is (= (case signal "SIGINT" 130 "SIGTERM" 143 1) (:exit-code response))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= [:completed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= 1 (get-in saved [:remaining :case/c :repetitions]))) + (t/is (str/includes? (:report response) "pgid 4242")) + (t/is (str/includes? (:report response) "SIGKILL")) + (t/is (= "retained renderer" (when (fs/existsSync index) (fs/readFileSync index "utf8")))) + (t/is (false? (.-listening ^js (get-in @(:state fake) [:host :server])))))))) + +(t/deftest ^:async failed-final-record-write-retains-the-selected-build + (let [root @temporary-root + id "retained-after-write-failure" + dir (path/join (artifacts/managed-root root) "builds" id) + _ (fs/mkdirSync dir #js {:recursive true}) + _ (fs/writeFileSync (path/join dir "index.html") "") + out (output-dir) + record (path/join out "run.transit") + prior (path/join out "run.before-finish.transit") + base (fake-browser {}) + close (:close (:service base)) + fake (assoc base :service + (assoc (:service base) :close + (fn [owner] + (close owner) + ;; Make the final atomic rename fail after all + ;; attempts have reached the saved record. + (fs/renameSync record prior) + (fs/mkdirSync record)))) + home (runs/create-home {:root root :run-id "failed-write" + :output-dir out :remove true}) + response (await (run/execute root home (selected-build dir id) + [(collected :case/a :fresh)] + (assoc (options fake out) :remove true) + (.-signal (js/AbortController.)))) + saved (result/decode (fs/readFileSync prior "utf8"))] + (t/is (= :running (get-in saved [:termination :reason]))) + (t/is (= 1 (count (:attempts saved)))) + (t/is (every? zero? (vals (get-in saved [:remaining :case/a])))) + (t/is (seq (:errors response))) + (t/is (= 1 (:exit-code response))) + (t/is (= "" + (fs/readFileSync (path/join dir "index.html") "utf8"))))) + +(defn- prepare-comparison-build + [{:keys [selection on-created]}] + (let [dir (build-dir) + build (-> (selected-build dir (str (random-uuid))) + (assoc :manifest-path (path/join dir "build.transit")) + (assoc-in [:manifest :functional :default-features?] false) + (assoc-in [:manifest :functional :features] (:features selection)) + (assoc-in [:manifest :provenance :feature-selection] selection))] + (on-created build) + (js/Promise.resolve build))) + +(t/deftest ^:async feature-comparison-completes-with-managed-output + (let [root @temporary-root + cases [(collected :case/a :fresh)] + fake (fake-browser {}) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir nil + :options (dissoc (options nil nil) :browser)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= 0 (:exit-code response))) + (t/is (= [:completed :completed] (mapv :status (:configurations saved)))) + (doseq [configuration (:configurations saved)] + (let [run (result/decode (fs/readFileSync (get-in configuration [:run :record-path]) "utf8"))] + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= 1 (count (:attempts run)))))))) + +(t/deftest ^:async failed-feature-configuration-prints-its-saved-cause-and-log + (let [root @temporary-root + cases [(collected :case/a :fresh)] + fake (fake-browser {:close-page-failure #(= % 1)}) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir nil + :options (assoc (dissoc (options nil nil) :browser) :filter "case/")} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal))}})) + home (:home response) + saved (result/decode (fs/readFileSync (:record-path home) "utf8")) + config (first (:configurations saved)) + request (request/apply-defaults {:command "ab" :console {:color :never} + :options {:features ["a"] :filter "case/"}})] + (t/is (= 1 (:exit-code response))) + (t/is (empty? (:errors response))) + (t/is (= [:failed :completed] (mapv :status (:configurations saved)))) + (t/is (= :browser-close (get-in config [:failure :stage]))) + (t/is (every? #(= :completed (:status %)) (:edges saved))) + (await + (mock/with-mocks* + {feature-comparison/prepare (fn [_ _ _] (js/Promise.resolve prepared)) + feature-comparison/execute (fn [_ _ _ _] (js/Promise.resolve response))} + (t/is (= 1 (await (deferred/execute (transit/encode-str request))))) + (let [log (fs/readFileSync (:log-path home) "utf8") + position (str/last-index-of log (str "Comparison " (:comparison-id home) "\n")) + report (subs log (or position 0))] + (t/is (some? position)) + (t/is (str/includes? report "Failures\n Configuration \"0\":")) + (t/is (= 1 (count (re-seq #"page close failed" report)))) + (t/is (str/includes? report (get-in config [:run :record-path]))) + (t/is (str/includes? report (get-in config [:run :log-path]))) + (t/is (str/includes? report (:record-path home)))))))) + +(t/deftest ^:async gpu-rejection-stops-later-matrix-configurations + (doseq [[hardware-first close-fails] [[false false] [true false] [true true]]] + (let [root @temporary-root + out (output-dir) + cases [(collected :case/a :fresh)] + software (assoc hardware-graphics :renderer "SwiftShader" :software true) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir out + :options (assoc (dissoc (options nil out) :browser) + :require-gpu true :repetitions 2)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (let [fake (fake-browser + {:close-failure (when close-fails (constantly true)) + :effective-graphics + (fn [_ page-id] + (if (and hardware-first (= page-id 1)) + hardware-graphics + software))})] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal)))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8")) + run (result/decode (fs/readFileSync (get-in saved [:configurations 0 :run :record-path]) "utf8")) + failure (some #(when (= :gpu (:stage %)) %) + (get-in run [:metadata :diagnostics :failures]))] + (t/is (= 1 (:exit-code response))) + (t/is (= (if hardware-first 1 0) (count (:attempts run)))) + (t/is (= software (get-in failure [:partial :graphics]))) + (t/is (= (if close-fails :cleanup :gpu) (get-in run [:termination :failure :stage]))) + (t/is (= :failed (get-in saved [:configurations 0 :status]))) + (t/is (= :gpu (get-in saved [:configurations 0 :failure :stage]))) + (t/is (= :unattempted (get-in saved [:configurations 1 :status])))))) + +(t/deftest ^:async comparison-stops-after-run-record-or-log-storage-fails + (doseq [filename ["run.transit" "run.log"]] + (let [executed (atom nil) + root @temporary-root + out (output-dir) + cases [(collected :case/a :fresh)] + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir out + :options (dissoc (options nil out) :browser)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (^:async fn [root home selected cases opts signal] + (let [base (fake-browser {}) + close (:close (:service base)) + file (path/join (:dir home) filename) + service (assoc (:service base) :close + (fn [owner] + (close owner) + (fs/renameSync file (str file ".saved")) + (fs/mkdirSync file))) + outcome (await (run/execute root home selected cases + (assoc opts :browser service) signal))] + (reset! executed outcome) + outcome))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8")) + evidence (path/join out "configs" "0" + (if (= filename "run.transit") "run.transit.saved" "run.transit"))] + (t/is (= 1 (count (:attempts (result/decode (fs/readFileSync evidence "utf8")))))) + (t/is (= 1 (:exit-code response))) + (t/is (re-find #"Case case/a" (or (:report @executed) ""))) + (t/is (= :failed (get-in saved [:configurations 0 :status]))) + (t/is (= :unattempted (get-in saved [:configurations 1 :status])))))) diff --git a/frontend/test/performance_tests/render_wasm/runner.cljs b/frontend/test/performance_tests/render_wasm/runner.cljs new file mode 100644 index 0000000000..d8ecaba150 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/runner.cljs @@ -0,0 +1,229 @@ +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.runner + (:require + [app.common.logging :as l] + [cljs.test :as t] + [clojure.string :as str] + [clojure.tools.cli :refer [parse-opts]] + [goog.object :as gobj] + [performance-tests.render-wasm.artifacts-test] + [performance-tests.render-wasm.boolean-test] + [performance-tests.render-wasm.browser-test] + [performance-tests.render-wasm.build-test] + [performance-tests.render-wasm.builder-test] + [performance-tests.render-wasm.cli-test] + [performance-tests.render-wasm.commands-test] + [performance-tests.render-wasm.compare-test] + [performance-tests.render-wasm.containers-test] + [performance-tests.render-wasm.contracts-test] + [performance-tests.render-wasm.errors-test] + [performance-tests.render-wasm.execution-test] + [performance-tests.render-wasm.feature-comparison-test] + [performance-tests.render-wasm.fingerprint-test] + [performance-tests.render-wasm.geometry-scenes-test] + [performance-tests.render-wasm.lifecycle-test] + [performance-tests.render-wasm.log-test] + [performance-tests.render-wasm.merge-test] + [performance-tests.render-wasm.outcome-test] + [performance-tests.render-wasm.process-test] + [performance-tests.render-wasm.protocol-test] + [performance-tests.render-wasm.report-test] + [performance-tests.render-wasm.request-test] + [performance-tests.render-wasm.run-test] + [performance-tests.render-wasm.runs-test] + [performance-tests.render-wasm.scene-snapshot-test] + [performance-tests.render-wasm.server-test])) + +(enable-console-print!) + +(defmethod t/report [:cljs.test/default :begin-test-var] [m] + (let [v (:var m)] + (println (str " ▸ " (:ns (meta v)) "/" (:name (meta v)))))) + +(defmethod t/report [:cljs.test/default :end-run-tests] [m] + (if (cljs.test/successful? m) + (.exit js/process 0) + (.exit js/process 1))) + +(def test-namespaces + ['performance-tests.render-wasm.artifacts-test + 'performance-tests.render-wasm.boolean-test + 'performance-tests.render-wasm.browser-test + 'performance-tests.render-wasm.build-test + 'performance-tests.render-wasm.builder-test + 'performance-tests.render-wasm.cli-test + 'performance-tests.render-wasm.commands-test + 'performance-tests.render-wasm.compare-test + 'performance-tests.render-wasm.containers-test + 'performance-tests.render-wasm.contracts-test + 'performance-tests.render-wasm.errors-test + 'performance-tests.render-wasm.execution-test + 'performance-tests.render-wasm.feature-comparison-test + 'performance-tests.render-wasm.fingerprint-test + 'performance-tests.render-wasm.geometry-scenes-test + 'performance-tests.render-wasm.lifecycle-test + 'performance-tests.render-wasm.log-test + 'performance-tests.render-wasm.merge-test + 'performance-tests.render-wasm.outcome-test + 'performance-tests.render-wasm.process-test + 'performance-tests.render-wasm.protocol-test + 'performance-tests.render-wasm.report-test + 'performance-tests.render-wasm.request-test + 'performance-tests.render-wasm.run-test + 'performance-tests.render-wasm.runs-test + 'performance-tests.render-wasm.scene-snapshot-test + 'performance-tests.render-wasm.server-test]) + +(assert (every? find-ns-obj test-namespaces) + "test-namespaces contains a namespace that isn't required in runner.cljs") + +;; This runner mirrors frontend-tests.runner for the bench-only +;; :performance-tests target. Both runners need forwarded CLI args, focused +;; namespace/var execution, fixture preservation, and app log-level +;; setup. The logic stays local to avoid cross-target test classpath +;; coupling. +(def ^:private log-levels + #{:trace :debug :info :warn :error}) + +(def cli-options + [["-f" "--focus FOCUS" "Run one test namespace or one test var, e.g. performance-tests.render-wasm.contracts-test"] + ["-l" "--log-level LEVEL" "Set app logger level: trace|debug|info|warn|error" + :parse-fn keyword + :validate [log-levels "must be one of trace, debug, info, warn, error"]] + ["-h" "--help"]]) + +(defn- argv + [] + (let [args (->> (.-argv js/process) + (array-seq) + (drop 2))] + ;; `pnpm run test:renderer-benchmarks -- --focus ...` forwards the separator + ;; to the node process, so drop one leading `--` before handing args to + ;; tools.cli. + (cond-> args + (= "--" (first args)) rest))) + +(defn- usage + [summary] + (str "Usage: pnpm run test:renderer-benchmarks -- [options]\n\n" + "Options:\n" + summary "\n\n" + "Focus examples:\n" + " pnpm run test:renderer-benchmarks -- --focus performance-tests.render-wasm.contracts-test\n" + " pnpm run test:renderer-benchmarks -- --focus performance-tests.render-wasm.contracts-test/some-test-var\n\n" + "Log level example (quiets app logging during the run):\n" + " pnpm run test:renderer-benchmarks -- --focus performance-tests.render-wasm.contracts-test --log-level warn")) + +(defn- fail! + [message] + (js/console.error message) + (.exit js/process 1)) + +(defn- parse-focus + [focus] + (let [[ns-name test-name & extra] (str/split focus #"/")] + (cond + (or (str/blank? ns-name) (seq extra)) + (fail! (str "Invalid --focus value: " focus)) + + (some? test-name) + {:ns (symbol ns-name) :test test-name} + + :else + {:ns (symbol ns-name)}))) + +(defn- fixture-value + [ns-obj fixture-name] + (let [value (gobj/get ns-obj (munge fixture-name))] + (when-not (undefined? value) + value))) + +(defn- ns-test-vars + [ns-sym] + (when-let [ns-obj (find-ns-obj ns-sym)] + (->> (js-keys ns-obj) + (keep (fn [key] + (some-> (gobj/get ns-obj key) + (.-cljs$lang$var)))) + (filter (comp :test meta)) + (sort-by (comp :line meta))))) + +(defn- ns-fixtures + [ns-sym vars] + (when-let [ns-obj (find-ns-obj ns-sym)] + (let [ns-key (or (some-> vars first meta :ns) ns-sym) + once-fixtures (fixture-value ns-obj "cljs-test-once-fixtures") + each-fixtures (fixture-value ns-obj "cljs-test-each-fixtures")] + {:once (when once-fixtures {ns-key once-fixtures}) + :each (when each-fixtures {ns-key each-fixtures})}))) + +(defn- selected-tests + [{:keys [ns test]}] + (when-not (some #{ns} test-namespaces) + (fail! (str "Unknown test namespace: " ns))) + (let [vars (vec (ns-test-vars ns))] + (when (empty? vars) + (fail! (str "No tests found in namespace: " ns))) + (if test + (let [test-sym (symbol test) + test-var (some #(when (= test-sym (:name (meta %))) %) vars)] + (if test-var + {:vars [test-var] + :fixtures (ns-fixtures ns [test-var])} + (fail! (str "Unknown test var: " ns "/" test)))) + {:vars vars + :fixtures (ns-fixtures ns vars)}))) + +(defn- merge-fixtures + [fixtures] + {:once (apply merge (keep :once fixtures)) + :each (apply merge (keep :each fixtures))}) + +(defn- run-test-vars! + [tests] + (let [vars (vec (mapcat :vars tests)) + fixtures (merge-fixtures (map :fixtures tests)) + env (assoc (t/empty-env) + :once-fixtures (:once fixtures) + :each-fixtures (:each fixtures)) + summary (volatile! {:test 0 :pass 0 :fail 0 :error 0 :type :summary})] + + (t/set-env! env) + + (t/run-block + (concat (t/test-vars-block vars) + [(fn [] + (vswap! summary + (partial merge-with +) + (:report-counters (t/get-current-env)))) + (fn [] + (t/report @summary) + (t/report (assoc @summary :type :end-run-tests)))])))) + +(defn- run-focused-test! + [focus] + (run-test-vars! [(selected-tests (parse-focus focus))])) + +(defn init + [] + (let [{:keys [options errors summary]} (parse-opts (argv) cli-options)] + (cond + (seq errors) + (fail! (str/join "\n" errors)) + + (:help options) + (do + (println (usage summary)) + (.exit js/process 0)) + + :else + (do + (l/setup! {:app (or (:log-level options) :warn)}) + + (if (:focus options) + (run-focused-test! (:focus options)) + (run-test-vars! (map #(selected-tests {:ns %}) test-namespaces))))))) diff --git a/frontend/test/performance_tests/render_wasm/runs_test.cljs b/frontend/test/performance_tests/render_wasm/runs_test.cljs new file mode 100644 index 0000000000..a0af15f0b0 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/runs_test.cljs @@ -0,0 +1,165 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.runs-test + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-runs-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- failure + [f] + (try + (f) + nil + (catch :default error error))) + +(defn- sample-run + [run-id] + (result/create-run + {:run-id run-id + :started-at "2026-10-26T00:00:00.000Z" + :plan {:cases [{:id :rects/load}] + :warmups 0 + :repetitions 1 + :timeout-policy {:case-ms 30000}} + :metadata {:git {:sha "abc" :dirty false} + :build {:functional {:mode :release}} + :environment {:node "v24.0.0"} + :scored? true}})) + +(t/deftest managed-home-lives-under-the-frontend-root + (let [home (runs/create-home {:root @temporary-root :run-id "run-1"})] + (t/is (= (path/join (artifacts/managed-root @temporary-root) "runs" "run-1") + (:dir home))) + (t/is (.isDirectory (fs/statSync (:dir home)))) + (t/is (= "" (fs/readFileSync (:log-path home) "utf8"))) + (t/is (not (fs/existsSync (:record-path home)))))) + +(t/deftest explicit-home-must-not-exist-and-resolves-from-caller-cwd + (let [previous (.cwd js/process) + relative "explicit-run"] + (try + (.chdir js/process @temporary-root) + (let [home (runs/create-home {:root @temporary-root + :run-id "run-2" + :output-dir relative})] + (t/is (= (path/join @temporary-root relative) (:dir home))) + (t/is (.isDirectory (fs/statSync (:dir home)))) + (t/is (not (fs/existsSync (path/join (artifacts/managed-root @temporary-root) + "runs" "run-2"))))) + (finally + (.chdir js/process previous)))) + (let [directory (path/join @temporary-root "already-there")] + (fs/mkdirSync directory) + (fs/writeFileSync (path/join directory "sentinel") "keep") + (let [error (failure #(runs/create-home {:root @temporary-root + :run-id "run-3" + :output-dir directory}))] + (t/is (= ::artifacts/invalid-home (:type (ex-data error)))) + (t/is (= "output directory already exists" (ex-message error))) + (t/is (= "keep" (fs/readFileSync (path/join directory "sentinel") "utf8")))))) + +(t/deftest record-writes-are-atomic-and-invalid-records-leave-the-saved-file + (let [home (runs/create-home {:root @temporary-root :run-id "run-4"}) + run (sample-run "run-4")] + (runs/write-record! home run) + (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8")))) + (t/is (empty? (filter #(str/ends-with? % ".tmp") + (array-seq (fs/readdirSync (:dir home)))))) + (let [error (failure #(runs/write-record! home (assoc-in run [:metadata :scored?] "yes")))] + (t/is (= ::result/invalid-record (:type (ex-data error)))) + (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8"))))))) + +(t/deftest record-read-failures-retain-file-context-and-cause + (let [file (path/join @temporary-root "invalid.transit")] + (fs/writeFileSync file (transit/encode-str (dissoc (sample-run "invalid") :plan))) + (let [error (failure #(runs/get-record file))] + (t/is (= {:path file :operation :read-record :type ::result/invalid-record} + (select-keys (ex-data error) [:path :operation :type]))) + (t/is (= ::result/invalid-record (:type (ex-data (ex-cause error))))) + (t/is (= (ex-message error) (ex-message (ex-cause error)))))) + (let [error (failure #(runs/get-record @temporary-root))] + (t/is (= {:path @temporary-root :operation :read-record} + (select-keys (ex-data error) [:path :operation]))) + (t/is (= "EISDIR" (.-code (ex-cause error)))))) + +(t/deftest screenshot-paths-stay-in-the-run-home + (let [home (runs/create-home {:root @temporary-root :run-id "run-5"}) + shot (runs/screenshot-path home :rects/load)] + (t/is (= "rects-load.png" (:file shot))) + (t/is (= (path/join (:dir home) "rects-load.png") (:path shot))) + (t/is (= (path/dirname (:record-path home)) (path/dirname (:path shot)))))) + +(t/deftest malformed-ids-and-empty-paths-fail-at-the-home-boundary + (doseq [id ["../escape" "" "a/b" "a\\b"]] + (t/is (thrown-with-msg? js/Error #"invalid run id" + (runs/create-home {:root @temporary-root :run-id id})))) + (t/is (thrown-with-msg? js/Error #"nonempty path" + (artifacts/check-output-dir " ")))) + +(t/deftest home-creation-reports-an-invalid-parent + (let [parent (path/join @temporary-root "file") + dir (path/join parent "run")] + (fs/writeFileSync parent "occupied") + (let [error (failure #(runs/create-home {:root @temporary-root + :run-id "run-6" :output-dir dir}))] + (t/is (= ::artifacts/invalid-home (:type (ex-data error)))) + (t/is (= (path/resolve dir) (:path (ex-data error)))) + (t/is (= "ENOTDIR" (.-code (ex-cause error)))) + (t/is (= (ex-message (ex-cause error)) (:cause (ex-data error)))) + (t/is (str/includes? (ex-message error) "cannot inspect output directory")) + (t/is (str/includes? (ex-message error) "ENOTDIR"))))) + +(t/deftest home-creation-wraps-log-failures + (let [dir (path/join @temporary-root "occupied-log")] + (fs/mkdirSync dir) + (fs/writeFileSync (path/join dir "run.log") "") + (let [error (failure #(artifacts/create-home dir "run" "cannot create run home"))] + (t/is (= ::artifacts/invalid-home (:type (ex-data error)))) + (t/is (= "cannot create run home" (ex-message error))) + (t/is (= dir (:path (ex-data error)))) + (t/is (= "EEXIST" (.-code (ex-cause error)))) + (t/is (= (ex-message (ex-cause error)) (:cause (ex-data error))))))) + +(t/deftest explicit-home-creates-missing-parents + (let [out (path/join @temporary-root "missing" "parents" "run") + home (runs/create-home {:root @temporary-root :run-id "nested" + :output-dir out})] + (t/is (.isDirectory (fs/statSync out))) + (t/is (= "" (fs/readFileSync (:log-path home) "utf8"))) + (t/is (thrown-with-msg? js/Error #"already exists" + (runs/create-home {:root @temporary-root :run-id "again" + :output-dir out}))))) + +(t/deftest remove-rejects-output-inside-builds-before-creating-it + (let [builds (path/join (artifacts/managed-root @temporary-root) "builds") + alias (path/join @temporary-root "build-alias")] + (fs/mkdirSync builds #js {:recursive true}) + (fs/symlinkSync builds alias "dir") + (doseq [out [(path/join builds "direct") + (path/join alias "missing" "nested")]] + (t/is (thrown-with-msg? js/Error #"--remove.*inside builds" + (runs/create-home {:root @temporary-root :run-id "blocked" + :output-dir out :remove true}))) + (t/is (not (fs/existsSync out)))) + (let [out (path/join @temporary-root "builds-other" "allowed")] + (runs/create-home {:root @temporary-root :run-id "allowed" + :output-dir out :remove true}) + (t/is (fs/existsSync (path/join out "run.log")))))) diff --git a/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs b/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs new file mode 100644 index 0000000000..08f6070de3 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs @@ -0,0 +1,316 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.scene-snapshot-test + (:require + [app.common.schema :as sm] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true])) + +(defn- sample-snapshot + "Simple tiny scene: root frame with one rectangle." + [] + (let [rect-id (uuid/custom 1 1) + rect (cts/setup-shape {:id rect-id :type :rect :name "Rect" + :x 32 :y 48 :width 200 :height 120}) + root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 1920 :height 1080 + :shapes [rect-id]})] + {:objects {uuid/zero root, rect-id rect} + :refs {:rect rect-id}})) + +(defn- nested-instance + "Root with two frames; frame A with two rectangles, frame B with one." + [] + (let [a-id (uuid/custom 2 1) + b-id (uuid/custom 2 2) + a1-id (uuid/custom 2 3) + a2-id (uuid/custom 2 4) + b1-id (uuid/custom 2 5) + frame-a (cts/setup-shape {:id a-id :type :frame :name "A" + :x 0 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [a1-id a2-id]}) + frame-b (cts/setup-shape {:id b-id :type :frame :name "B" + :x 200 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [b1-id]}) + rect-a1 (cts/setup-shape {:id a1-id :type :rect :name "A1" + :x 10 :y 10 :width 10 :height 10 + :parent-id a-id :frame-id a-id}) + rect-a2 (cts/setup-shape {:id a2-id :type :rect :name "A2" + :x 30 :y 10 :width 10 :height 10 + :parent-id a-id :frame-id a-id}) + rect-b1 (cts/setup-shape {:id b1-id :type :rect :name "B1" + :x 210 :y 10 :width 10 :height 10 + :parent-id b-id :frame-id b-id}) + root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 1920 :height 1080 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [a-id b-id]})] + {:objects {uuid/zero root + a-id frame-a + b-id frame-b + a1-id rect-a1 + a2-id rect-a2 + b1-id rect-b1} + :refs {:a a-id + :a1 a1-id + :b b-id}})) + +(defn- validation-error + "Returns the ex-data of the validation failure, or nil when it passes." + [instance] + (try + (snapshot/validate! instance) + nil + (catch :default cause + (ex-data cause)))) + +(t/deftest canonical-example-is-valid + (t/testing "validate! returns the same instance" + (let [instance (sample-snapshot)] + (t/is (identical? instance (snapshot/validate! instance))))) + + (t/testing "the envelope schema accepts the example" + (t/is (true? (sm/validate snapshot/schema:snapshot (sample-snapshot)))))) + +(t/deftest nested-hierarchy-is-valid + (let [instance (nested-instance)] + (t/is (identical? instance (snapshot/validate! instance))))) + +(t/deftest root-only-instance-is-valid + (let [root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :shapes []}) + instance {:objects {uuid/zero root} :refs {}}] + (t/is (identical? instance (snapshot/validate! instance))) + (t/is (= [uuid/zero] (mapv :id (snapshot/upload-order instance)))))) + +(t/deftest upload-order-starts-at-root + (let [instance (sample-snapshot) + ids (mapv :id (snapshot/upload-order instance))] + (t/is (= [uuid/zero (snapshot/ref-id instance :rect)] ids)))) + +(t/deftest upload-order-is-parent-before-child + (let [instance (nested-instance) + order (snapshot/upload-order instance) + names (mapv :name order) + position (into {} (map-indexed (fn [index shape] [(:id shape) index]) order))] + (t/is (= ["Root Frame" "A" "A1" "A2" "B" "B1"] names)) + (t/is (every? (fn [shape] + (or (= uuid/zero (:id shape)) + (< (position (:parent-id shape)) + (position (:id shape))))) + order)))) + +(t/deftest upload-order-follows-flex-reverse + (let [frame-id (uuid/custom 8 1) + a-id (uuid/custom 8 2) + b-id (uuid/custom 8 3) + frame (cts/setup-shape {:id frame-id :type :frame :name "Flex" + :x 0 :y 0 :width 200 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :layout :flex :layout-flex-dir :row-reverse + :shapes [a-id b-id]}) + shape-a (cts/setup-shape {:id a-id :type :rect :name "A" + :x 0 :y 0 :width 10 :height 10 + :parent-id frame-id :frame-id frame-id}) + shape-b (cts/setup-shape {:id b-id :type :rect :name "B" + :x 20 :y 0 :width 10 :height 10 + :parent-id frame-id :frame-id frame-id}) + root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 1920 :height 1080 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [frame-id]}) + instance {:objects {uuid/zero root + frame-id frame + a-id shape-a + b-id shape-b} + :refs {}}] + (t/is (identical? instance (snapshot/validate! instance))) + (t/is (= ["Root Frame" "Flex" "B" "A"] + (mapv :name (snapshot/upload-order instance)))))) + +(t/deftest ref-id-resolves-labels + (let [instance (sample-snapshot)] + (t/is (= (uuid/custom 1 1) (snapshot/ref-id instance :rect))) + (t/is (nil? (snapshot/ref-id instance :missing))))) + +(t/deftest missing-root-is-rejected + (let [instance (sample-snapshot) + data (validation-error (update instance :objects dissoc uuid/zero))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest non-frame-root-is-rejected + (let [instance (sample-snapshot) + data (validation-error (assoc-in instance [:objects uuid/zero :type] :rect))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest root-with-non-zero-parent-id-is-rejected + (let [instance (sample-snapshot) + data (validation-error + (assoc-in instance [:objects uuid/zero :parent-id] (uuid/custom 9 9)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest object-key-must-match-shape-id + (let [instance (sample-snapshot) + rect (get-in instance [:objects (snapshot/ref-id instance :rect)]) + data (validation-error (assoc-in instance [:objects (uuid/custom 7 7)] rect))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= (:id rect) (:shape-id data))))) + +(t/deftest missing-child-is-rejected + (let [instance (sample-snapshot) + bogus (uuid/custom 3 3) + data (validation-error + (assoc-in instance [:objects uuid/zero :shapes] [bogus]))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= bogus (:child data))))) + +(t/deftest child-not-listed-by-parent-is-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects uuid/zero :shapes] []))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "object is not reachable from the root" (:hint data))) + (t/is (= rect-id (:id data))))) + +(t/deftest mismatched-parent-id-is-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error + (assoc-in instance [:objects rect-id :parent-id] (uuid/custom 9 7)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "shape :shapes lists a child with a different :parent-id" (:hint data))) + (t/is (= rect-id (:child data))))) + +(t/deftest duplicate-child-ids-are-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error + (assoc-in instance [:objects uuid/zero :shapes] [rect-id rect-id]))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "an object is reachable more than once" (:hint data))) + (t/is (= rect-id (:id data))))) + +(t/deftest shape-listing-itself-is-rejected + (let [instance (sample-snapshot) + data (validation-error + (assoc-in instance [:objects uuid/zero :shapes] [uuid/zero]))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "root must not be a descendant of itself" (:hint data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest detached-cycle-is-rejected + (let [c1-id (uuid/custom 5 1) + c2-id (uuid/custom 5 2) + shape1 (cts/setup-shape {:id c1-id :type :rect :name "C1" + :x 0 :y 0 :width 10 :height 10 + :parent-id c2-id :frame-id c1-id + :shapes [c2-id]}) + shape2 (cts/setup-shape {:id c2-id :type :rect :name "C2" + :x 0 :y 0 :width 10 :height 10 + :parent-id c1-id :frame-id c2-id + :shapes [c1-id]}) + data (validation-error + (assoc (sample-snapshot) :objects + (assoc (get (sample-snapshot) :objects) + c1-id shape1 + c2-id shape2)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "object is not reachable from the root" (:hint data))) + (t/is (contains? #{c1-id c2-id} (:id data))))) + +(t/deftest reachable-cycle-is-rejected + (let [a-id (uuid/custom 4 1) + b-id (uuid/custom 4 2) + shape-a (cts/setup-shape {:id a-id :type :rect :name "A" + :x 0 :y 0 :width 10 :height 10 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [b-id]}) + shape-b (cts/setup-shape {:id b-id :type :rect :name "B" + :x 0 :y 0 :width 10 :height 10 + :parent-id a-id :frame-id uuid/zero + :shapes [a-id]}) + instance (-> (sample-snapshot) + (assoc-in [:objects uuid/zero :shapes] [a-id]) + (assoc-in [:objects a-id] shape-a) + (assoc-in [:objects b-id] shape-b)) + data (validation-error instance)] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "shape :shapes lists a child with a different :parent-id" (:hint data))) + (t/is (= a-id (:child data))))) + +(t/deftest non-vector-children-are-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects rect-id :shapes] 5))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "shape :shapes must be a vector" (:hint data))) + (t/is (= rect-id (:id data))))) + +(t/deftest frame-id-mismatch-is-rejected + (let [instance (nested-instance) + a1-id (snapshot/ref-id instance :a1) + data (validation-error + (assoc-in instance [:objects a1-id :frame-id] uuid/zero))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= :invalid-frame (:code data))) + (t/is (= a1-id (:shape-id data))))) + +(t/deftest missing-ref-target-is-rejected + (let [instance (sample-snapshot) + data (validation-error + (assoc-in instance [:refs :rect] (uuid/custom 6 6)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= :rect (:label data))))) + +(t/deftest non-uuid-ref-value-is-rejected + (let [data (validation-error (assoc-in (sample-snapshot) [:refs :rect] "not-a-uuid"))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + +(t/deftest missing-refs-key-is-rejected + (let [data (validation-error (dissoc (sample-snapshot) :refs))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + +(t/deftest non-canonical-shape-is-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects rect-id] {:id rect-id}))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + +(t/deftest malformed-envelopes-are-rejected + (t/testing "empty objects map" + (let [data (validation-error {:objects {} :refs {}})] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + + (t/testing "nil instance" + (let [data (validation-error nil)] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + + (t/testing "nil shape value" + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects rect-id] nil))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + + (t/testing "nil ref value" + (let [data (validation-error (assoc-in (sample-snapshot) [:refs :rect] nil))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data)))))) diff --git a/frontend/test/performance_tests/render_wasm/server_test.cljs b/frontend/test/performance_tests/render_wasm/server_test.cljs new file mode 100644 index 0000000000..8dcb7c8d79 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/server_test.cljs @@ -0,0 +1,233 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.server-test + (:require + ["fs" :as fs] + ["http" :as http] + ["net" :as net] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.runner.server :as server] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) +(def ^:private wasm-bytes [0 97 115 109 1 0 0 0]) + +(defn- write-file + [file content] + (fs/mkdirSync (path/dirname file) #js {:recursive true}) + (fs/writeFileSync file content)) + +(t/use-fixtures :each + {:before (fn [] + (let [root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-http-")) + dir (path/join root "selected")] + (reset! temporary-root root) + (write-file (path/join dir "index.html") "selected") + (write-file (path/join dir "browser" "browser.js") "export const selected = true;") + (write-file (path/join dir "cases.cjs") "module.exports = {};") + (write-file (path/join dir "renderer" "render-wasm.wasm") + (js/Buffer.from (to-array wasm-bytes))) + (write-file (path/join root "selected-outside" "secret.txt") "private sibling") + (write-file (path/join root "outside.txt") "private file"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- build + [] + {:id "selected-http-build" :dir (path/join @temporary-root "selected")}) + +(defn- request + [host request-path method] + (js/Promise. + (fn [resolve reject] + (let [address (js/URL. (:base-url host)) + req (http/request + ;; Supplying a raw path avoids URL normalization hiding traversal. + #js {:hostname (.-hostname address) + :port (.-port address) + :path request-path + :method method + :agent false} + (fn [response] + (let [chunks (atom [])] + (.on ^js response "data" #(swap! chunks conj %)) + (.once ^js response "error" reject) + (.once ^js response "end" + #(resolve {:status (.-statusCode response) + :headers (js->clj (.-headers response)) + :body (js/Buffer.concat (to-array @chunks))})))))] + (.once ^js req "error" reject) + (.setTimeout ^js req 3000 #(.destroy ^js req (js/Error. "test request timed out"))) + (.end ^js req))))) + +(defn- ^:async request-failure + [host] + (try + (await (request host "/" "GET")) + nil + (catch :default error error))) + +(t/deftest ^:async serves-only-the-selected-build-with-head-and-artifact-mime-types + (let [selected (build) + host (await (server/open selected))] + (try + (t/is (= selected (:build host))) + (t/is (re-matches #"http://127\.0\.0\.1:[0-9]+" (:base-url host))) + (let [index (await (request host "/?unused=query" "GET")) + browser (await (request host "/browser/browser.js" "GET")) + cjs (await (request host "/cases.cjs" "GET")) + wasm (await (request host "/renderer/render-wasm.wasm" "GET")) + head (await (request host "/renderer/render-wasm.wasm" "HEAD"))] + (t/is (= 200 (:status index))) + (t/is (= "selected" (.toString (:body index) "utf8"))) + (t/is (= "text/html; charset=utf-8" (get-in index [:headers "content-type"]))) + (t/is (= "export const selected = true;" (.toString (:body browser) "utf8"))) + (t/is (= "application/javascript" (get-in browser [:headers "content-type"]))) + (t/is (= "application/javascript" (get-in cjs [:headers "content-type"]))) + (t/is (= "application/wasm" (get-in wasm [:headers "content-type"]))) + (t/is (zero? (js/Buffer.compare (:body wasm) (js/Buffer.from (to-array wasm-bytes))))) + (t/is (= 200 (:status head))) + (t/is (= "8" (get-in head [:headers "content-length"]))) + (t/is (zero? (.-length (:body head))))) + (t/is (= 404 (:status (await (request host "/selected-outside/secret.txt" "GET"))))) + (t/is (= 405 (:status (await (request host "/" "POST"))))) + (finally + (await (server/close host)))))) + +(t/deftest ^:async rejects-raw-traversal-and-outside-file-and-directory-symlinks + (let [selected (build) + dir (:dir selected)] + (fs/symlinkSync (path/join @temporary-root "outside.txt") (path/join dir "escape.txt")) + (fs/symlinkSync (path/join @temporary-root "selected-outside") (path/join dir "linked") "dir") + (let [host (await (server/open selected))] + (try + (doseq [request-path ["/../selected-outside/secret.txt" + "/%2e%2e/selected-outside/secret.txt" + "/%2e%2e%2fselected-outside%2fsecret.txt" + "/%5c..%5cselected-outside%5csecret.txt" + "/escape.txt" "/linked/secret.txt"]] + (let [response (await (request host request-path "GET"))] + (t/is (= 403 (:status response)) request-path) + (t/is (= "forbidden" (.toString (:body response) "utf8")) request-path))) + (t/is (= 400 (:status (await (request host "/%zz" "GET"))))) + (t/is (= 403 (:status (await (request host "/%00" "GET"))))) + (t/is (= 404 (:status (await (request host "/missing-file" "GET"))))) + (t/is (= 404 (:status (await (request host "/browser" "GET"))))) + (finally + (await (server/close host))))))) + +(defn- connect + [host] + (js/Promise. + (fn [resolve reject] + (let [address (js/URL. (:base-url host)) + socket (net/createConnection #js {:host "127.0.0.1" :port (js/Number (.-port address))})] + (.once ^js socket "error" reject) + (.once ^js socket "connect" #(resolve socket)))))) + +(defn- bounded + [promise] + (js/Promise. + (fn [resolve reject] + (let [timer (js/setTimeout #(reject (js/Error. "test shutdown timed out")) 3000)] + (.then promise + (fn [value] (js/clearTimeout timer) (resolve value)) + (fn [error] (js/clearTimeout timer) (reject error))))))) + +(t/deftest ^:async close-destroys-owned-connections-and-keeps-other-servers-live + (let [host (await (server/open (build)))] + (try + (let [other (await (server/open (build)))] + (try + (let [socket (await (connect host)) + closed (js/Promise. (fn [resolve _] (.once ^js socket "close" resolve)))] + (try + (await (bounded (js/Promise.all (to-array [(server/close host) (server/close host)])))) + (await (bounded closed)) + (t/is (some? (await (request-failure host)))) + (let [response (await (request other "/" "GET"))] + (t/is (= 200 (:status response))) + (t/is (= "selected" (.toString (:body response) "utf8")))) + (await (server/close host)) + (finally + (.destroy ^js socket)))) + (finally + (await (server/close other))))) + (finally + (await (server/close host)))))) + +(defn- ^:async startup-failure + [opening] + (try + (let [host (await opening)] + (await (server/close host)) + nil) + (catch :default error error))) + +(t/deftest ^:async startup-abort-retains-a-server-close-failure + (let [controller (js/AbortController.) + close server/close + close-error (js/Error. "server disposal failed")] + (.abort controller) + (await (mock/with-mocks* + {server/close (fn [host] + (.then (close host) (fn [_] (throw close-error))))} + (let [cause (await (bounded (startup-failure (server/open (build) {:signal (.-signal controller)})))) + entries (errors/get-errors cause)] + (t/is (= ::server/aborted (:type (ex-data cause)))) + (t/is (= ["renderer server startup aborted" "server disposal failed"] + (mapv #(errors/message-of (:error %)) entries)))))))) + +(t/deftest ^:async startup-rejects-already-aborted-and-in-flight-signals + (let [already (js/AbortController.)] + (.abort already) + (let [error (await (startup-failure (server/open (build) {:signal (.-signal already)})))] + (t/is (= ::server/aborted (:type (ex-data error)))))) + (let [controller (js/AbortController.) + opening (server/open (build) {:signal (.-signal controller)})] + (.abort controller) + (let [error (await (startup-failure opening))] + (t/is (= ::server/aborted (:type (ex-data error)))))) + (let [host (await (server/open (build) {:startup-timeout-ms 3000}))] + (try + (t/is (= 200 (:status (await (request host "/" "GET"))))) + (finally + (await (server/close host)))))) + + +(defn- ^:async with-listen + "Calls f while http.Server#listen runs `listen` and returns the promise f returns." + [listen f] + (let [proto (.-prototype http/Server)] + (set! (.-listen proto) listen) + (try + (await (f)) + (finally + (js-delete proto "listen"))))) + +(t/deftest ^:async startup-timeout-rejects-with-the-configured-limit + (let [never-listens (fn [& _] nil) + error (await (with-listen never-listens + #(startup-failure (server/open (build) {:startup-timeout-ms 50}))))] + (t/is (= ::server/startup-timeout (:type (ex-data error)))) + (t/is (= 50 (:timeout-ms (ex-data error)))))) + +(t/deftest ^:async startup-rejects-with-the-listen-failure + (let [thrown (js/Error. "listen threw") + emitted (js/Error. "server emitted error") + throws (fn [& _] (throw thrown)) + emits (fn [& _] (this-as ^js self (js/process.nextTick #(.emit self "error" emitted))))] + (doseq [[failure listen] [[thrown throws] [emitted emits]]] + (let [error (await (with-listen listen #(startup-failure (server/open (build)))))] + (t/is (identical? failure error) (ex-message failure)))))) + +(t/deftest ^:async startup-rejects-when-the-build-directory-is-missing + (let [missing (path/join @temporary-root "missing") + error (await (startup-failure (server/open {:id "missing-build" :dir missing})))] + (t/is (= "ENOENT" (.-code ^js error))))) diff --git a/frontend/test/performance_tests/render_wasm/test_helpers.cljs b/frontend/test/performance_tests/render_wasm/test_helpers.cljs new file mode 100644 index 0000000000..4609e8f079 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/test_helpers.cljs @@ -0,0 +1,49 @@ +(ns performance-tests.render-wasm.test-helpers + "Shared helpers for renderer benchmark scene tests. + + These helpers assert nothing about shapes or the renderer. They only + capture errors, read root children in order, and run raw scopes so each + test namespace tells a complete story without copying bodies." + (:require + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as builder])) + +(defn failure-data + "Returns the ex-data of the thrown failure, or nil when `f` passes." + [f] + (try + (f) + nil + (catch :default cause + (ex-data cause)))) + +(defn child-ids + "Ids listed in the :shapes vector of `id`." + [snapshot id] + (get-in snapshot [:objects id :shapes])) + +(defn children-of + "Child shapes of `id` in order." + [snapshot id] + (mapv (:objects snapshot) (child-ids snapshot id))) + +(defn root-shapes + "Root children of a scene snapshot in creation order." + [snapshot] + (children-of snapshot uuid/zero)) + +(defn run-scope + "Runs `f` inside a raw scope and returns the scope state, so tests can + inspect partial builds without calling finish!." + [params f] + (let [state (builder/start params)] + (binding [builder/*state* state + builder/*defaults* {}] + (f)) + state)) + +(defn selrect + "Selrect of `shape` as an [x y width height] tuple." + [shape] + [(:x (:selrect shape)) (:y (:selrect shape)) + (:width (:selrect shape)) (:height (:selrect shape))]) diff --git a/frontend/text-editor/package.json b/frontend/text-editor/package.json index 34e6dcb63f..97e7113d15 100644 --- a/frontend/text-editor/package.json +++ b/frontend/text-editor/package.json @@ -28,5 +28,5 @@ "vite": "^8.2.0", "vitest": "^5.0.2" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/frontend/translations/en.po b/frontend/translations/en.po index d7c8d7716f..57da6e76b3 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -380,6 +380,10 @@ msgstr "Create new organization" msgid "dashboard.create-new-team" msgstr "Create new team" +#: src/app/main/ui/dashboard/team_form.cljs:122 +msgid "dashboard.create-team-in-organization" +msgstr "Create new team in %s" + #: src/app/main/ui/workspace/main_menu.cljs:712 msgid "dashboard.create-version-menu" msgstr "Pin this version" @@ -2427,7 +2431,10 @@ msgstr "Letter Spacing" msgid "inspect.attributes.typography.line-height" msgstr "Line Height" -#: src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:638 +msgid "inspect.attributes.typography.list-style-type.disc" +msgstr "Disc" + +#: src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:594 msgid "inspect.attributes.typography.mixed-font-family" msgstr "Mixed Font Families" @@ -5021,6 +5028,13 @@ msgstr "Start designing" msgid "options.deleted-token-with-name" msgstr "{%s} token does not exist or has been deleted." +#: src/app/main/data/plugins.cljs:129 +msgid "plugins.runtime-unavailable" +msgstr "" +"Plugins are unavailable in this browser. Something in your environment (for " +"example, an extension such as Tampermonkey) is preventing the plugin " +"runtime from starting. Disable extensions or reload the page to try again." + #: src/app/plugins/utils.cljs:579 msgid "plugins.validation.invalid-value" msgstr "expected %s, got %s (%s)" @@ -5869,14 +5883,12 @@ msgid "subscription.dashboard.banner.subscription-expired" msgstr "Your Enterprise subscription has expired. (%s)" #: src/app/main/ui/dashboard/subscription.cljs:217 -msgid "subscription.dashboard.banner.unlock-features" -msgstr "Unlock Enterprise features" +msgid "subscription.dashboard.banner.unlock-enterprise" +msgstr "Unlock Enterprise" #: src/app/main/ui/dashboard/subscription.cljs:219 -msgid "subscription.dashboard.banner.unlock-features-description-text" -msgstr "" -"Set fine-grained permissions and stay in control, without compromising the " -"open-source freedom your team loves." +msgid "subscription.dashboard.banner.unlock-features-description" +msgstr "For organizations that need advanced governance, centralized administration, and secure collaboration." #: src/app/main/ui/dashboard/subscription.cljs:224 msgid "subscription.dashboard.banner.upgrade-nitrate" @@ -8240,6 +8252,10 @@ msgstr "Add a shared library" msgid "workspace.libraries.assets-in-this-file" msgstr "ASSETS IN THIS FILE" +#: src/app/main/data/workspace/libraries.cljs:1963 +msgid "workspace.libraries.broken-library-disconnected" +msgstr "The library \"%s\" couldn't be loaded and has been disconnected from this file. Components and colors already used in the file aren't affected but will be orphan until the library is fixed and connected again." + #: src/app/main/ui/workspace/libraries.cljs:178, src/app/main/ui/workspace/libraries.cljs:207, src/app/main/ui/workspace/libraries.cljs:251 msgid "workspace.libraries.colors" msgid_plural "workspace.libraries.colors" @@ -8741,6 +8757,22 @@ msgstr[1] "Export %s elements" msgid "workspace.options.export.add-export" msgstr "Add export" +#: src/app/main/ui/exports/preview.cljs:170 +msgid "workspace.options.export.preview" +msgstr "Preview" + +#: src/app/main/ui/exports/preview.cljs:98 +msgid "workspace.options.export.preview-error" +msgstr "Could not load the preview" + +#: src/app/main/ui/exports/preview.cljs:94, src/app/main/ui/exports/preview.cljs:156 +msgid "workspace.options.export.preview-image" +msgstr "Export preview" + +#: src/app/main/ui/exports/preview.cljs:142 +msgid "workspace.options.export.preview-preset" +msgstr "Preview export preset" + #: src/app/main/ui/inspect/exports.cljs:177, src/app/main/ui/inspect/exports.cljs:211, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:228, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:263 msgid "workspace.options.export.remove-export" msgstr "Remove export" @@ -9561,6 +9593,14 @@ msgstr "Letter Spacing" msgid "workspace.options.text-options.line-height" msgstr "Line height" +#: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:251 +msgid "workspace.options.text-options.list-bullets" +msgstr "Bullet list" + +#: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:247 +msgid "workspace.options.text-options.list-none" +msgstr "No list" + #: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:212 msgid "workspace.options.text-options.strikethrough" msgstr "Strikethrough (%s)" @@ -9678,6 +9718,10 @@ msgstr "Discover [more plugins](%s)" msgid "workspace.plugins.empty-plugins" msgstr "No plugins installed yet" +#: src/app/main/data/plugins.cljs, src/app/main/ui/dashboard.cljs, src/app/main/ui/workspace/plugins.cljs +msgid "workspace.plugins.error.global-not-allowed" +msgstr "Plugins that work outside a file can only be installed from localhost for now." + #: src/app/main/ui/workspace/plugins.cljs:237 msgid "workspace.plugins.error.manifest" msgstr "The plugin manifest is incorrect." @@ -10149,6 +10193,54 @@ msgstr "**Editing is disabled** because changes can’t be saved." msgid "workspace.save-failure.stopped" msgstr "**Editing is disabled** because changes can’t be saved." +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor-point" +msgstr "Anchor point" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.bottom-center" +msgstr "Bottom center" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.bottom-left" +msgstr "Bottom left" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.bottom-right" +msgstr "Bottom right" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.center" +msgstr "Center" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.center-left" +msgstr "Center left" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.center-right" +msgstr "Center right" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.top-center" +msgstr "Top center" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.top-left" +msgstr "Top left" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.top-right" +msgstr "Top right" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.scale-and-anchor" +msgstr "Scale and anchor point" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.title" +msgstr "Scale" + #: src/app/main/ui/workspace/context_menu.cljs:667 msgid "workspace.shape.menu.add-flex" msgstr "Add flex layout" @@ -11060,6 +11152,16 @@ msgstr "" msgid "workspace.tokens.unknown-token-type-section" msgstr "Type '%s' is not supported (%s):" +#: src/app/main/data/workspace/tokens/import_export.cljs:80 +msgid "workspace.tokens.unsupported-token-data-message" +msgstr "" +"Import was successful, but some token data was skipped because Penpot " +"does not support it. Expand details to see what was skipped." + +#: src/app/main/data/workspace/tokens/import_export.cljs:83 +msgid "workspace.tokens.unsupported-token-data-section" +msgstr "'%s' is not supported (%s):" + #: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:316, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:270 msgid "workspace.tokens.use-reference" msgstr "Use a reference" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index f8143b15a5..70dd2cd4b3 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -382,6 +382,10 @@ msgstr "Crear nueva organización" msgid "dashboard.create-new-team" msgstr "Crear nuevo equipo" +#: src/app/main/ui/dashboard/team_form.cljs:122 +msgid "dashboard.create-team-in-organization" +msgstr "Crea un nuevo equipo en %s" + #: src/app/main/ui/workspace/main_menu.cljs:712 msgid "dashboard.create-version-menu" msgstr "Guardar esta versión" @@ -2458,7 +2462,10 @@ msgstr "Espaciado de letras" msgid "inspect.attributes.typography.line-height" msgstr "Interlineado" -#: src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:638 +msgid "inspect.attributes.typography.list-style-type.disc" +msgstr "Disco" + +#: src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:594 msgid "inspect.attributes.typography.mixed-font-family" msgstr "Varias Familias tipográficas" @@ -5047,6 +5054,13 @@ msgstr "Empezar a diseñar" msgid "options.deleted-token-with-name" msgstr "El token {%s} no existe o ha sido borrado." +#: src/app/main/data/plugins.cljs:129 +msgid "plugins.runtime-unavailable" +msgstr "" +"Los plugins no están disponibles en este navegador. Algo en tu entorno (por " +"ejemplo, una extensión como Tampermonkey) impide que se inicie el entorno de " +"plugins. Desactiva las extensiones o vuelve a cargar la página." + #: src/app/plugins/utils.cljs:579 msgid "plugins.validation.invalid-value" msgstr "esperado %s, obtenido %s (%s)" @@ -5891,14 +5905,14 @@ msgid "subscription.dashboard.banner.subscription-expired" msgstr "Tu suscripción Enterprise ha caducado. (%s)" #: src/app/main/ui/dashboard/subscription.cljs:217 -msgid "subscription.dashboard.banner.unlock-features" -msgstr "Desbloquea las funciones Enterprise" +msgid "subscription.dashboard.banner.unlock-enterprise" +msgstr "Desbloquea Enterprise" #: src/app/main/ui/dashboard/subscription.cljs:219 -msgid "subscription.dashboard.banner.unlock-features-description-text" +msgid "subscription.dashboard.banner.unlock-features-description" msgstr "" -"Establece permisos detallados y mantén el control, sin renunciar a la " -"libertad del código abierto que tanto valora tu equipo." +"Para organizaciones que necesitan una gobernanza avanzada, una " +"administración centralizada y una colaboración segura." #: src/app/main/ui/dashboard/subscription.cljs:224 msgid "subscription.dashboard.banner.upgrade-nitrate" @@ -7445,6 +7459,14 @@ msgstr "Añadir una biblioteca compartida" msgid "workspace.libraries.assets-in-this-file" msgstr "RECURSOS EN ESTE ARCHIVO" +#: src/app/main/data/workspace/libraries.cljs:1963 +msgid "workspace.libraries.broken-library-disconnected" +msgstr "" +"No se ha podido cargar la biblioteca \"%s\" y se ha desconectado de este " +"archivo. Los componentes y colores que ya usa el archivo no se ven " +"afectados, pero quedarán huérfanos hasta que la biblioteca se arregle y se " +"vuelva a conectar." + #: src/app/main/ui/workspace/libraries.cljs:178, src/app/main/ui/workspace/libraries.cljs:207, src/app/main/ui/workspace/libraries.cljs:251 msgid "workspace.libraries.colors" msgid_plural "workspace.libraries.colors" @@ -7949,6 +7971,22 @@ msgstr[1] "Exportar %s elementos" msgid "workspace.options.export.add-export" msgstr "Añadir exportación" +#: src/app/main/ui/exports/preview.cljs:170 +msgid "workspace.options.export.preview" +msgstr "Vista previa" + +#: src/app/main/ui/exports/preview.cljs:98 +msgid "workspace.options.export.preview-error" +msgstr "No se ha podido cargar la vista previa" + +#: src/app/main/ui/exports/preview.cljs:94, src/app/main/ui/exports/preview.cljs:156 +msgid "workspace.options.export.preview-image" +msgstr "Resultado de la exportación" + +#: src/app/main/ui/exports/preview.cljs:142 +msgid "workspace.options.export.preview-preset" +msgstr "Exportación mostrada" + #: src/app/main/ui/inspect/exports.cljs:177, src/app/main/ui/inspect/exports.cljs:211, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:228, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:263 msgid "workspace.options.export.remove-export" msgstr "Eliminar exportación" @@ -8771,6 +8809,14 @@ msgstr "Espaciado entre letras" msgid "workspace.options.text-options.line-height" msgstr "Altura de línea" +#: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:251 +msgid "workspace.options.text-options.list-bullets" +msgstr "Lista con viñetas" + +#: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:247 +msgid "workspace.options.text-options.list-none" +msgstr "Sin lista" + #: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:212 msgid "workspace.options.text-options.strikethrough" msgstr "Tachado (%s)" @@ -8888,6 +8934,12 @@ msgstr "Descubre [más extensiones](%s)" msgid "workspace.plugins.empty-plugins" msgstr "No se encuentran extensiones" +#: src/app/main/data/plugins.cljs, src/app/main/ui/dashboard.cljs, src/app/main/ui/workspace/plugins.cljs +msgid "workspace.plugins.error.global-not-allowed" +msgstr "" +"Por ahora, las extensiones que funcionan fuera de un archivo solo se pueden " +"instalar desde localhost." + #: src/app/main/ui/workspace/plugins.cljs:237 msgid "workspace.plugins.error.manifest" msgstr "El manifiesto de la expansión es incorrecto." @@ -9059,6 +9111,54 @@ msgstr "**Edición desactivada** porque no se pueden guardar los cambios." msgid "workspace.save-failure.stopped" msgstr "**Edición desactivada** porque no se pueden guardar los cambios." +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor-point" +msgstr "Punto de anclaje" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.bottom-center" +msgstr "Abajo en el centro" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.bottom-left" +msgstr "Abajo a la izquierda" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.bottom-right" +msgstr "Abajo a la derecha" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.center" +msgstr "Centro" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.center-left" +msgstr "Centro a la izquierda" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.center-right" +msgstr "Centro a la derecha" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.top-center" +msgstr "Arriba en el centro" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.top-left" +msgstr "Arriba a la izquierda" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.anchor.top-right" +msgstr "Arriba a la derecha" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.scale-and-anchor" +msgstr "Escala y punto de anclaje" + +#: src/app/main/ui/workspace/sidebar/scale.cljs +msgid "workspace.scale.title" +msgstr "Escalado" + #: src/app/main/ui/workspace/context_menu.cljs:667 msgid "workspace.shape.menu.add-flex" msgstr "Añadir flex layout" @@ -9985,6 +10085,17 @@ msgstr "" msgid "workspace.tokens.unknown-token-type-section" msgstr "El tipo '%s' no está soportado (%s):" +#: src/app/main/data/workspace/tokens/import_export.cljs:80 +msgid "workspace.tokens.unsupported-token-data-message" +msgstr "" +"La importación se ha realizado correctamente, pero algunos datos de los " +"tokens fueron omitidos porque Penpot no los soporta. Expande los detalles " +"para ver qué se omitió." + +#: src/app/main/data/workspace/tokens/import_export.cljs:83 +msgid "workspace.tokens.unsupported-token-data-section" +msgstr "'%s' no está soportado (%s):" + #: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:316, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:270 msgid "workspace.tokens.use-reference" msgstr "Usa una referencia" diff --git a/library/package.json b/library/package.json index 21ac2c9c1f..aa16f03f02 100644 --- a/library/package.json +++ b/library/package.json @@ -3,7 +3,7 @@ "version": "1.3.0", "license": "MPL-2.0", "author": "KALEIDOS SUBSIDIARY SL", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "type": "module", "repository": { "type": "git", diff --git a/library/pnpm-lock.yaml b/library/pnpm-lock.yaml index 10cbdf36b8..598b7dc50f 100644 --- a/library/pnpm-lock.yaml +++ b/library/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/mcp/package.json b/mcp/package.json index 977047cf57..d815da71ba 100644 --- a/mcp/package.json +++ b/mcp/package.json @@ -23,7 +23,7 @@ "type": "git", "url": "https://github.com/penpot/penpot.git" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "devDependencies": { "concurrently": "^10.0.6", "prettier": "^3.9.10" diff --git a/mcp/packages/browser-extension/package.json b/mcp/packages/browser-extension/package.json index 43a5c4d8fd..576897a8e7 100644 --- a/mcp/packages/browser-extension/package.json +++ b/mcp/packages/browser-extension/package.json @@ -4,7 +4,7 @@ "version": "0.1.0", "description": "Browser extension that keeps Penpot tabs connected to MCP from being discarded", "type": "module", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "scripts": { "build": "tsc && node ./scripts/build.mjs", "test": "tsx --test \"src/**/*.test.ts\"", diff --git a/mcp/packages/common/package.json b/mcp/packages/common/package.json index 267a37fef2..19d41c61b2 100644 --- a/mcp/packages/common/package.json +++ b/mcp/packages/common/package.json @@ -4,7 +4,7 @@ "description": "Shared type definitions and interfaces for Penpot MCP", "main": "dist/index.js", "types": "dist/index.d.ts", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "scripts": { "build": "tsc --build --clean && tsc --build", "watch": "tsc --watch", diff --git a/mcp/packages/plugin/package.json b/mcp/packages/plugin/package.json index 7d33e479fe..e1118ae83a 100644 --- a/mcp/packages/plugin/package.json +++ b/mcp/packages/plugin/package.json @@ -3,7 +3,7 @@ "private": true, "version": "1.0.0", "type": "module", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "scripts": { "start": "vite build --watch --config vite.config.ts", "start:multi-user": "pnpm run start", diff --git a/mcp/packages/plugin/src/PenpotUtils.ts b/mcp/packages/plugin/src/PenpotUtils.ts index f856f14cc4..762e75c3dd 100644 --- a/mcp/packages/plugin/src/PenpotUtils.ts +++ b/mcp/packages/plugin/src/PenpotUtils.ts @@ -539,7 +539,7 @@ export class PenpotUtils { * @example * // Three button variants differing in Size * // Given three main components s, m, l (here: the first three main components on the page) - * const [s, m, l] = penpot.currentPage.findAllShapes(sh => sh.isMainComponent()).slice(0, 3) as Board[]; + * const [s, m, l] = PenpotUtils.findShapes(sh => sh.isComponentMainInstance(), penpot.root).slice(0, 3) as Board[]; * const container = PenpotUtils.createVariantContainer([ * { shape: s, properties: { Size: 'Small' } }, * { shape: m, properties: { Size: 'Medium' } }, diff --git a/mcp/packages/server/data/initial_instructions.md b/mcp/packages/server/data/initial_instructions.md index fbca1c5bad..d126811742 100644 --- a/mcp/packages/server/data/initial_instructions.md +++ b/mcp/packages/server/data/initial_instructions.md @@ -352,7 +352,7 @@ Properties are often addressed positionally: `pos` parameter in various methods Use `penpotUtils.createVariantContainer(components)` — it handles the full multi-step workflow in one call: ```js // Given three main components s, m, l (here: the first three main components on the page) -const [s, m, l] = penpot.currentPage.findAllShapes(sh => sh.isMainComponent()).slice(0, 3); +const [s, m, l] = penpotUtils.findShapes(sh => sh.isComponentMainInstance(), penpot.root).slice(0, 3); // Single property: const container = penpotUtils.createVariantContainer([ { shape: s, properties: { Size: 'Small' } }, diff --git a/mcp/packages/server/package.json b/mcp/packages/server/package.json index 0bd3d8ae01..a122e28828 100644 --- a/mcp/packages/server/package.json +++ b/mcp/packages/server/package.json @@ -27,7 +27,7 @@ "engines": { "node": ">=20.20.2" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "dependencies": { "@modelcontextprotocol/node": "^2.1.1", "@modelcontextprotocol/server": "^2.3.1", diff --git a/mcp/packages/server/src/PenpotSessionSchema.ts b/mcp/packages/server/src/PenpotSessionSchema.ts new file mode 100644 index 0000000000..b70da49330 --- /dev/null +++ b/mcp/packages/server/src/PenpotSessionSchema.ts @@ -0,0 +1,14 @@ +import type { PenpotSession } from "@penpot/mcp-common"; +import { z } from "zod"; + +/** + * Schema of a Penpot session as reported by the plugin and exchanged between MCP instances. + * + * A session without an open file (e.g. a plugin connected from the dashboard) has null `fileId` and `fileName`. + */ +export const penpotSessionSchema = z.object({ + sessionId: z.string().min(1).max(128), + fileId: z.string().min(1).max(128).nullable(), + fileName: z.string().nullable(), + workspaceState: z.enum(["none", "loading", "ready"]).optional(), +}) satisfies z.ZodType; diff --git a/mcp/packages/server/src/PluginBridge.ts b/mcp/packages/server/src/PluginBridge.ts index 340016f44f..8bf426e9a8 100644 --- a/mcp/packages/server/src/PluginBridge.ts +++ b/mcp/packages/server/src/PluginBridge.ts @@ -10,6 +10,7 @@ import { UnclaimedRollbacksMessage, } from "@penpot/mcp-common"; import { createLogger } from "./logger"; +import { penpotSessionSchema } from "./PenpotSessionSchema"; import type { PenpotMcpServer } from "./PenpotMcpServer"; import type { RedisBridge } from "./RedisBridge"; import type { PenpotConnection, PluginLivenessState } from "./PenpotConnection"; @@ -39,12 +40,7 @@ export const HEARTBEAT_STALE_THRESHOLD_MS = 30000; const connectionInitSchema = z.object({ type: z.literal("initialize"), - session: z.object({ - sessionId: z.string().min(1).max(128), - fileId: z.string().min(1).max(128).nullable(), - fileName: z.string().nullable(), - workspaceState: z.enum(["none", "loading", "ready"]).optional(), - }), + session: penpotSessionSchema, }); /** @@ -191,9 +187,7 @@ export class PluginBridge implements TaskDispatchHost { } connection.frozen = false; if (message?.type === "context-update") { - const context = connectionInitSchema.shape.session - .omit({ sessionId: true }) - .parse(message.context); + const context = penpotSessionSchema.omit({ sessionId: true }).parse(message.context); Object.assign(connection.session, context); return; } diff --git a/mcp/packages/server/src/RedisBridge.test.ts b/mcp/packages/server/src/RedisBridge.test.ts index 52080fcc3c..c64650ef1e 100644 --- a/mcp/packages/server/src/RedisBridge.test.ts +++ b/mcp/packages/server/src/RedisBridge.test.ts @@ -7,6 +7,8 @@ import { RedisBridge } from "./RedisBridge"; import { PluginBridge } from "./PluginBridge"; import { PluginTask } from "./PluginTask"; import type { PenpotMcpServer } from "./PenpotMcpServer"; +import type { PenpotSession } from "@penpot/mcp-common"; +import { InvalidDiscoveryResponseError } from "./dispatch/TaskDispatcher"; // Replace only the Redis command boundary; exercise the real transport, // dispatchers, task tracking, and WebSockets above it. @@ -105,8 +107,11 @@ test("Redis discovery receives one reply per subscribed instance and returns pub second.publishDiscoveryResponse(id, { instanceId: "second", sessions: [] }); }); const responders: string[] = []; - const count = await requester.sendDiscoveryRequest("alice", "query", (response) => - responders.push(response.instanceId) + const count = await requester.sendDiscoveryRequest( + "alice", + "query", + (response) => responders.push(response.instanceId), + assert.fail ); assert.equal(count, 2); assert.deepEqual(responders.sort(), ["first", "second"]); @@ -140,12 +145,12 @@ test("a failed discovery subscription is cleaned up and can be retried", async ( owner.subscribeToDiscovery("alice", () => {}), /Subscription failed/ ); - assert.equal(await caller.sendDiscoveryRequest("alice", "after-failure", () => {}), 0); + assert.equal(await caller.sendDiscoveryRequest("alice", "after-failure", () => {}, assert.fail), 0); await caller.unsubscribeFromDiscoveryResponses("after-failure"); await owner.subscribeToDiscovery("alice", ({ id }) => { owner.publishDiscoveryResponse(id, { instanceId: "owner", sessions: [] }); }); - assert.equal(await caller.sendDiscoveryRequest("alice", "retry", () => {}), 1); + assert.equal(await caller.sendDiscoveryRequest("alice", "retry", () => {}, assert.fail), 1); await caller.unsubscribeFromDiscoveryResponses("retry"); }); @@ -172,13 +177,14 @@ function instance(t: TestContext) { const connect = async ( sessionId: string, outcome: "success" | "error" | "silent" = "success", - reply: string = sessionId + reply: string = sessionId, + file: { fileId: string | null; fileName: string | null } = { fileId: "file", fileName: "Design" } ) => { const socket = new WebSocket(`ws://127.0.0.1:${wsPort}?userToken=alice`); sockets.push(socket); await once(socket, "open"); const initialized = once(socket, "message", { signal: AbortSignal.timeout(2000) }); - socket.send(JSON.stringify({ type: "initialize", session: { sessionId, fileId: "file", fileName: "Design" } })); + socket.send(JSON.stringify({ type: "initialize", session: { sessionId, ...file } })); await initialized; socket.on("message", (raw) => { const request = JSON.parse(raw.toString()); @@ -214,6 +220,34 @@ test("full Redis discovery implicitly selects a sole remote WebSocket", async (t assert.equal((await requester.bridge.executePluginTask(new PluginTask("test", {}))).data, "only"); }); +test("full Redis discovery implicitly selects a sole remote WebSocket without an open file", async (t) => { + const owner = instance(t); + const requester = instance(t); + await owner.connect("dashboard", "success", "dashboard", { fileId: null, fileName: null }); + assert.equal((await requester.bridge.executePluginTask(new PluginTask("test", {}))).data, "dashboard"); +}); + +test("an unreadable discovery reply fails discovery at once without suggesting a retry", async (t) => { + const responder = transport(t); + const requester = instance(t); + await responder.subscribeToDiscovery("alice", ({ id }) => { + responder.publishDiscoveryResponse(id, { + instanceId: "other-version", + sessions: [{ sessionId: "tab", fileId: 42 } as unknown as PenpotSession], + }); + }); + const started = Date.now(); + await assert.rejects(requester.bridge.executePluginTask(new PluginTask("test", {})), (error: Error) => { + assert.ok(error instanceof InvalidDiscoveryResponseError); + assert.match(error.message, /cannot be read/); + assert.match(error.message, /fileId/); + assert.match(error.message, /Retrying will not help/); + assert.doesNotMatch(error.message, /temporary server load/); + return true; + }); + assert.ok(Date.now() - started < 1000); +}); + test("forwarded plugin failures retain the regular execution failure route", async (t) => { const owner = instance(t); const requester = instance(t); diff --git a/mcp/packages/server/src/RedisBridge.ts b/mcp/packages/server/src/RedisBridge.ts index b653c70faf..ca3a45364d 100644 --- a/mcp/packages/server/src/RedisBridge.ts +++ b/mcp/packages/server/src/RedisBridge.ts @@ -1,6 +1,7 @@ import Redis from "ioredis"; import { PluginTaskRequest, PluginTaskResponse, PenpotSession } from "@penpot/mcp-common"; import { createLogger } from "./logger"; +import { penpotSessionSchema } from "./PenpotSessionSchema"; import { z } from "zod"; /** Query for the user's sessions held by each subscribed MCP instance. */ @@ -30,13 +31,7 @@ const sessionClaimSchema = z.object({ const discoveryResponseSchema = z.object({ instanceId: z.string().min(1), - sessions: z.array( - z.object({ - sessionId: z.string().min(1).max(128), - fileId: z.string().min(1).max(128), - fileName: z.string(), - }) - ), + sessions: z.array(penpotSessionSchema), }); /** @@ -248,19 +243,29 @@ export class RedisBridge { return `penpot.mcp.${this.tenant}.discovery.res.${this.channelKey(requestId)}`; } - /** Subscribes to replies before publishing; returns the actual recipient count. */ + /** + * Subscribes to replies before publishing; returns the actual recipient count. + * + * A reply that cannot be parsed is passed to `onInvalidResponse` with a readable description of the problem. + */ async sendDiscoveryRequest( userToken: string, requestId: string, - onResponse: (response: SessionDiscoveryResponse) => void + onResponse: (response: SessionDiscoveryResponse) => void, + onInvalidResponse: (error: Error) => void ): Promise { const channel = this.discoveryResponseChannel(requestId); this.handlers.set(channel, (raw) => { + let response: SessionDiscoveryResponse; try { - onResponse(discoveryResponseSchema.parse(JSON.parse(raw))); + response = discoveryResponseSchema.parse(JSON.parse(raw)); } catch (error) { this.logger.error(error, "Invalid discovery response"); + const reason = error instanceof z.ZodError ? z.prettifyError(error) : String(error); + onInvalidResponse(new Error(reason)); + return; } + onResponse(response); }); try { await this.subscriber.subscribe(channel); diff --git a/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.test.ts b/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.test.ts index 4c1b153916..18f3c657bf 100644 --- a/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.test.ts +++ b/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.test.ts @@ -6,14 +6,25 @@ import { PluginTask, type AbstractPluginTask } from "../PluginTask"; import type { RedisBridge, SessionClaim, SessionDiscoveryRequest, SessionDiscoveryResponse } from "../RedisBridge"; import { UserPenpotConnections } from "../UserPenpotConnections"; import { MultiInstanceTaskDispatcher } from "./MultiInstanceTaskDispatcher"; -import { SessionDiscoveryError, type TaskDispatchHost } from "./TaskDispatcher"; +import { InvalidDiscoveryResponseError, SessionDiscoveryError, type TaskDispatchHost } from "./TaskDispatcher"; + +/** Parse failure reported for a discovery reply that the requester cannot read. */ +const INVALID_REPLY_REASON = "sessions[0].fileId: expected string"; + +/** Reply callbacks registered by one discovery request. */ +interface DiscoveryReplyHandlers { + onResponse: (response: SessionDiscoveryResponse) => void; + onInvalidResponse: (error: Error) => void; +} /** In-memory Pub/Sub transport retaining Redis's per-subscriber delivery count. */ class RedisNetwork { readonly tasks = new Map void>>(); readonly discovery = new Map void>>(); readonly taskResponses = new Map) => void>(); - readonly discoveryResponses = new Map void>(); + readonly discoveryResponses = new Map(); + /** reply callbacks of the latest discovery request, kept after its subscription is released. */ + lastDiscoveryReplyHandlers?: DiscoveryReplyHandlers; readonly claims = new Map void>>(); failDiscovery = false; dropClaims = false; @@ -25,6 +36,7 @@ class RedisNetwork { class RedisPeer { replyToDiscovery = true; repeatDiscoveryReplies = false; + sendInvalidDiscoveryReplies = false; constructor(readonly network: RedisNetwork) {} @@ -72,9 +84,16 @@ class RedisPeer { this.network.discovery.get(token)?.delete(this); } - async sendDiscoveryRequest(token: string, id: string, onResponse: (response: SessionDiscoveryResponse) => void) { + async sendDiscoveryRequest( + token: string, + id: string, + onResponse: (response: SessionDiscoveryResponse) => void, + onInvalidResponse: (error: Error) => void + ) { if (this.network.failDiscovery) throw new Error("Redis unavailable for discovery"); - this.network.discoveryResponses.set(id, onResponse); + const handlers = { onResponse, onInvalidResponse }; + this.network.discoveryResponses.set(id, handlers); + this.network.lastDiscoveryReplyHandlers = handlers; await this.network.beforeDiscoveryPublish?.(); const subscribers = this.network.discovery.get(token) ?? new Map(); for (const [peer, handler] of subscribers) { @@ -87,7 +106,12 @@ class RedisPeer { } publishDiscoveryResponse(id: string, response: SessionDiscoveryResponse) { - this.network.discoveryResponses.get(id)?.(response); + const handlers = this.network.discoveryResponses.get(id); + if (this.sendInvalidDiscoveryReplies) { + handlers?.onInvalidResponse(new Error(INVALID_REPLY_REASON)); + } else { + handlers?.onResponse(response); + } } async unsubscribeFromDiscoveryResponses(id: string) { @@ -244,6 +268,7 @@ test("incomplete discovery never dispatches even when it yields exactly one sess slow.redis.replyToDiscovery = false; await assert.rejects(new Instance(network).execute("alice"), (error: Error) => { assert.ok(error instanceof SessionDiscoveryError); + assert.match(error.message, /not every MCP server instance replied within 20ms/); assert.match(error.message, /temporary server load.*retry/); return true; }); @@ -265,7 +290,65 @@ test("duplicate replies from one instance do not substitute for a missing reply" test("discovery transport failures use the retryable discovery error", async () => { const network = new RedisNetwork(); network.failDiscovery = true; - await assert.rejects(new Instance(network).execute("alice"), SessionDiscoveryError); + await assert.rejects(new Instance(network).execute("alice"), (error: Error) => { + assert.ok(error instanceof SessionDiscoveryError); + assert.match(error.message, /the Redis request failed \(Redis unavailable for discovery\)/); + assert.match(error.message, /temporary server load.*retry/); + return true; + }); +}); + +test("an invalid reply fails discovery without dispatching even when another instance replies validly", async () => { + const network = new RedisNetwork(); + const responsive = new Instance(network); + const otherVersion = new Instance(network); + await responsive.connect("alice", "visible"); + await otherVersion.connect("alice", "unreadable"); + otherVersion.redis.sendInvalidDiscoveryReplies = true; + await assert.rejects(new Instance(network).execute("alice"), (error: Error) => { + assert.ok(error instanceof InvalidDiscoveryResponseError); + assert.ok(error.message.includes(INVALID_REPLY_REASON)); + assert.match(error.message, /Retrying will not help/); + return true; + }); + assert.deepEqual(responsive.executions, []); + assert.deepEqual(otherVersion.executions, []); + assert.equal(network.discoveryResponses.size, 0); +}); + +test("an invalid reply after discovery settled leaves its outcome unchanged", async () => { + const network = new RedisNetwork(); + const owner = new Instance(network); + await owner.connect("alice", "only"); + const requester = new Instance(network); + const sessions = requester.dispatcher.discoverSessions("alice"); + const late = network.lastDiscoveryReplyHandlers!; + assert.deepEqual( + (await sessions).map((session) => session.sessionId), + ["only"] + ); + late.onInvalidResponse(new Error(INVALID_REPLY_REASON)); + assert.equal((await requester.execute("alice")).data, "only"); +}); + +test("closing the dispatcher fails pending discoveries without blaming server load", async () => { + const network = new RedisNetwork(); + const silent = new Instance(network); + await silent.connect("alice", "tab"); + silent.redis.replyToDiscovery = false; + const requester = new Instance(network); + const rejected = assert.rejects(requester.execute("alice"), (error: Error) => { + assert.ok(error instanceof SessionDiscoveryError); + assert.match(error.message, /the MCP server is shutting down/); + assert.match(error.message, /You can retry the request/); + assert.doesNotMatch(error.message, /temporary server load/); + return true; + }); + await new Promise((resolve) => setImmediate(resolve)); + await requester.dispatcher.close(); + await rejected; + assert.deepEqual(silent.executions, []); + assert.equal(network.discoveryResponses.size, 0); }); test("disconnecting one tab retains discovery and dispatch for the remaining tab", async () => { diff --git a/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.ts b/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.ts index 10c895e659..891bb12d77 100644 --- a/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.ts +++ b/mcp/packages/server/src/dispatch/MultiInstanceTaskDispatcher.ts @@ -5,7 +5,12 @@ import type { AbstractPluginTask } from "../PluginTask"; import { RemotePluginTask } from "../RemotePluginTask"; import type { RedisBridge, SessionClaim, SessionDiscoveryResponse } from "../RedisBridge"; import { createLogger } from "../logger"; -import { SessionDiscoveryError, TaskDispatcher, type TaskDispatchHost } from "./TaskDispatcher"; +import { + InvalidDiscoveryResponseError, + SessionDiscoveryError, + TaskDispatcher, + type TaskDispatchHost, +} from "./TaskDispatcher"; /** Routes tasks over Redis and discovers live sessions across MCP instances. */ export class MultiInstanceTaskDispatcher extends TaskDispatcher { @@ -44,10 +49,10 @@ export class MultiInstanceTaskDispatcher extends TaskDispatcher { resolve = res; reject = rej; }); - const fail = () => { + const fail = (error: SessionDiscoveryError) => { if (!settled) { settled = true; - reject(new SessionDiscoveryError()); + reject(error); } }; const complete = () => { @@ -56,8 +61,18 @@ export class MultiInstanceTaskDispatcher extends TaskDispatcher { resolve(this.uniqueSessions([...responses.values()].flat())); } }; - const timeout = setTimeout(fail, this.discoveryTimeoutMs); - this.pendingDiscoveries.set(requestId, fail); + const timeout = setTimeout( + () => + fail( + new SessionDiscoveryError( + `not every MCP server instance replied within ${this.discoveryTimeoutMs}ms` + ) + ), + this.discoveryTimeoutMs + ); + this.pendingDiscoveries.set(requestId, () => + fail(new SessionDiscoveryError("the MCP server is shutting down", "You can retry the request.")) + ); const releaseResponses = () => { void this.redisBridge .unsubscribeFromDiscoveryResponses(requestId) @@ -65,18 +80,29 @@ export class MultiInstanceTaskDispatcher extends TaskDispatcher { }; // Replies can arrive before publish returns its recipient count. void this.redisBridge - .sendDiscoveryRequest(token, requestId, (response: SessionDiscoveryResponse) => { - if (settled) return; - responses.set(response.instanceId, response.sessions); - complete(); - }) + .sendDiscoveryRequest( + token, + requestId, + (response: SessionDiscoveryResponse) => { + if (settled) return; + responses.set(response.instanceId, response.sessions); + complete(); + }, + (error) => fail(new InvalidDiscoveryResponseError(error)) + ) .then((count) => { expectedResponses = count; complete(); // A slow subscribe/publish may finish after the discovery timeout. if (settled) releaseResponses(); }) - .catch(fail); + .catch((error) => + fail( + new SessionDiscoveryError( + `the Redis request failed (${error instanceof Error ? error.message : error})` + ) + ) + ); try { return await result; } finally { diff --git a/mcp/packages/server/src/dispatch/TaskDispatcher.ts b/mcp/packages/server/src/dispatch/TaskDispatcher.ts index 062af6b80c..b9ea7b617a 100644 --- a/mcp/packages/server/src/dispatch/TaskDispatcher.ts +++ b/mcp/packages/server/src/dispatch/TaskDispatcher.ts @@ -40,10 +40,33 @@ export abstract class TaskDispatcher { } } -/** Error returned when discovery cannot establish the complete set of sessions. */ +/** + * Error returned when discovery cannot establish the complete set of sessions. + * + * The default message marks the failure as transient; subclasses describe failures that a retry does not fix. + */ export class SessionDiscoveryError extends Error { - constructor() { - super("Penpot session discovery failed. This may be due to temporary server load. You can retry the request."); + /** + * @param reason - what prevented discovery from completing + * @param advice - what the caller can do about it + */ + constructor(reason: string, advice = "This may be due to temporary server load. You can retry the request.") { + super(`Penpot session discovery failed: ${reason}. ${advice}`); this.name = "SessionDiscoveryError"; } } + +/** Error returned when an MCP server instance replies to discovery with a session list that cannot be read. */ +export class InvalidDiscoveryResponseError extends SessionDiscoveryError { + /** + * @param cause - the parse failure of the reply + */ + constructor(cause: Error) { + super( + `an MCP server instance sent a session list that cannot be read (${cause.message})`, + "Retrying will not help; the MCP server instances may be running different versions. " + + "To work around it, ask the user for the session ID shown in the Penpot MCP menu and pass it as sessionId." + ); + this.name = "InvalidDiscoveryResponseError"; + } +} diff --git a/mcp/pnpm-lock.yaml b/mcp/pnpm-lock.yaml index f005fb60ac..0eccb34be8 100644 --- a/mcp/pnpm-lock.yaml +++ b/mcp/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/media-processor/package.json b/media-processor/package.json index aa7b3f7449..81e594c481 100644 --- a/media-processor/package.json +++ b/media-processor/package.json @@ -14,7 +14,7 @@ "fmt:check": "prettier --check src/ test/", "clean": "rm -rf dist/" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "dependencies": { "express": "^5.2.1", "multer": "^2.4.0", diff --git a/media-processor/pnpm-lock.yaml b/media-processor/pnpm-lock.yaml index e5c4f4debe..ea185ea811 100644 --- a/media-processor/pnpm-lock.yaml +++ b/media-processor/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/media-processor/src/services/font.ts b/media-processor/src/services/font.ts index 16e4a0a091..11965ba177 100644 --- a/media-processor/src/services/font.ts +++ b/media-processor/src/services/font.ts @@ -122,9 +122,10 @@ async function fontConvert( // FontForge's own lexer uses doubled single quotes for escaping. const escInput = inputPath.replace(/'/g, "''"); const escOutput = outputPath.replace(/'/g, "''"); - await execCommand("fontforge", ["-lang=ff", "-c", `Open('${escInput}'); Generate('${escOutput}')`], undefined, { - signal, - }); + // CID-keyed fonts (common in CJK) load as several sub-fonts and Generate + // only writes the active one unless they are merged first. + const script = `Open('${escInput}'); if ($cidfontname != ""); CIDFlatten(); endif; Generate('${escOutput}')`; + await execCommand("fontforge", ["-lang=ff", "-c", script], undefined, { signal }); return await readFile(outputPath); } catch (err: unknown) { const error = err as NodeJS.ErrnoException & { killed?: boolean; signal?: string }; diff --git a/media-processor/test/fixtures/font-cid.otf b/media-processor/test/fixtures/font-cid.otf new file mode 100644 index 0000000000..d9fc9d18ca Binary files /dev/null and b/media-processor/test/fixtures/font-cid.otf differ diff --git a/media-processor/test/font.test.ts b/media-processor/test/font.test.ts index cb0a462801..1ab1706c59 100644 --- a/media-processor/test/font.test.ts +++ b/media-processor/test/font.test.ts @@ -11,13 +11,27 @@ let ttfData: Buffer; let otfData: Buffer; let woffData: Buffer; let woff2Data: Buffer; +let cidOtfData: Buffer; + +// Returns numGlyphs from the maxp table of an sfnt font. +function sfntNumGlyphs(data: Buffer): number { + const numTables = data.readUInt16BE(4); + for (let i = 0; i < numTables; i++) { + const rec = 12 + i * 16; + if (data.toString("latin1", rec, rec + 4) === "maxp") { + return data.readUInt16BE(data.readUInt32BE(rec + 8) + 4); + } + } + throw new Error("maxp table not found"); +} beforeAll(async () => { - [ttfData, otfData, woffData, woff2Data] = await Promise.all([ + [ttfData, otfData, woffData, woff2Data, cidOtfData] = await Promise.all([ readFile(join(FIXTURES, "font-1.ttf")), readFile(join(FIXTURES, "font-1.otf")), readFile(join(FIXTURES, "font-1.woff")), readFile(join(FIXTURES, "font-1.woff2")), + readFile(join(FIXTURES, "font-cid.otf")), ]); }); @@ -58,6 +72,13 @@ describe("convertFont", () => { expect(result).toBeInstanceOf(Buffer); expect(result!.length).toBeGreaterThan(0); }); + + it("otf→ttf keeps the glyphs of every CID sub-font", async () => { + // font-cid.otf has 5 glyphs split across two CID sub-fonts + const result = await convertFont(cidOtfData, "font/otf", "font/ttf"); + expect(result!.subarray(0, 4).toString("hex")).toBe("00010000"); + expect(sfntNumGlyphs(result!)).toBeGreaterThanOrEqual(5); + }); }); describe("sourceType=woff", () => { diff --git a/operations/2.20.0-file-change-partition.md b/operations/2.20.0-file-change-partition.md new file mode 100644 index 0000000000..30d1185a92 --- /dev/null +++ b/operations/2.20.0-file-change-partition.md @@ -0,0 +1,965 @@ +# Partition `file_change` by `file_id` + +Procedure to turn `file_change` into a table partitioned by `HASH (file_id)` +with 16 partitions, aligned with the ones of `file_data`. It runs online +except the cutover (phase 4), which needs a scheduled maintenance window with +the backend and the workers stopped. + +- Migration: `backend/src/app/migrations/sql/0158-partition-file-change.sql` +- Issue: + +The result has to be **exactly** the schema of the migration, because on a big +instance the migration is marked as applied and never runs. This procedure +runs the same statements by hand. + +## When to run it, and the stop + +Run it only on the instances where the automatic migration refuses to run: +`file_change` above 10 GiB of total size or above 2 million rows. Below that +the migration does the same work on boot and this procedure has nothing to do. + +The work spans the upgrade to 2.20.0. The order is not a suggestion: the +preparation goes **before** the version update, and phases 2 and 3 need the new +code running (it is the one that moves `updated_at` on every write, the +watermark of the delta passes). + +| Phase | When | Service | +|---|---|---| +| 0 and 1 | Before the upgrade, with 2.19.0 serving | Online | +| — | The upgrade to 2.20.0 | Usual deploy (step 0.3), before phase 2 | +| 2 | After the upgrade, with 2.20.0 serving | Online | +| 3 | From phase 2 to the window, every 1-6 h | Online | +| 4 | **A scheduled maintenance window** | **Backend and workers stopped** | +| 5 | After the cutover | Online | + +Phase 2 is one pass over every row: the long job, hours or days. Phase 3 +repeats a much shorter copy, only the rows that changed since the previous +pass (seconds each), until the day of the window, so the last delta inside the +lock stays short. + +Steps 0.1 and 0.2 cannot wait for the upgrade: a 2.20.0 backend that boots +without them runs the automatic migration inside the startup transaction and, +above the guard, it fails on every boot. + +**The procedure needs one stop, and only one: phase 4, the cutover.** It is a +full stop-start of the backend and the workers, not a rolling restart (4.2 +explains why). The window covers the last delta, the renames and the start +again: seconds to minutes of writes blocked on `file_change`, and the saves +that fall inside fail and the user retries. Everything else runs with the +instance serving traffic. + +## Sizing your instance + +Every number in this procedure comes from your database: run the queries +below, the procedure does not depend on any absolute size. It is plain SQL +and runs on any supported PostgreSQL (>= 15). + +```sql +-- total size of the database and of file_change (heap + TOAST + indexes) +SELECT pg_size_pretty(pg_database_size(current_database())) AS database; +SELECT pg_size_pretty(pg_total_relation_size('file_change')) AS file_change_total, + pg_size_pretty(pg_table_size('file_change')) AS heap_and_toast, + pg_size_pretty(pg_indexes_size('file_change')) AS indexes; +SELECT count(*) AS rows FROM file_change; + +-- dead tuples waiting for vacuum: their space is not copied +SELECT n_dead_tup FROM pg_stat_user_tables WHERE relname = 'file_change'; +``` + +`pg_table_size` (heap + TOAST) is the upper bound of what the copy reads and +writes. Do not estimate it with `pg_stats.avg_width`: that measures what the +heap stores for a column, which for a TOASTed value is the pointer, not the +value. The estimate reads far below the real size (about ten times less on a +small instance, orders of magnitude on a big one). + +Peak disk during the bulk is about the old table (it does not grow) plus the +new table (at most its `pg_table_size`, less because the dead rows are not +copied) plus its six indexes (about the `pg_indexes_size` of the live rows) +plus the WAL the bulk generates (proportional to what it writes: watch the +free space of the `pg_wal` mount while the first batches run). The disk comes +back when the legacy table is dropped, days later. Time the first batches and +extrapolate: the bulk reads heap + TOAST once sequentially and writes about +the same, so its duration is proportional to `pg_table_size`, not to the heap +alone. + +## What changes + +- `file_change` is partitioned by `HASH (file_id)` with 16 partitions, the + same bounds as `file_data`, so the join between them is resolved partition by + partition. +- The primary key goes from `(id)` to `(file_id, id)`: PostgreSQL requires the + partition key in the primary key, and the application already filters by + `file_id` everywhere. +- The table has 17 columns: `data_ref_id` and `data_backend` are gone (dead + since the file storage refactor), `data` is kept (the `legacy-db` backend + reads old payloads from it) and `kind` appears (1 = snapshot, 2 = revision, + nullable until the rollback window closes). +- The old table is kept as `file_change_legacy`, a frozen copy of the + previous layout, and dropped days later. +- Almost all of the size of the table is dead space (see sizing), so the new + table holds at most its heap + TOAST, and less as the dead rows are not + copied. The disk comes back when the legacy table is dropped: the database + shrinks by the size of the old table. + +## Why this mechanism + +`ATTACH` as a `DEFAULT` partition is not possible: the 16 hash siblings cover +the whole hash space, so the constraint of a `DEFAULT` partition could never +hold, and the `ATTACH` fails when it validates every row. A `UNION` +view with triggers, dual writes in the application and logical replication +were discarded too. + +What is left is a copy in phases, driven by the `updated_at` watermark, and a +cutover of minutes: plain SQL, resumable, no new infrastructure. The heap is +small next to the TOAST, so a delta pass without an index scans the heap and +only detoastes the rows that changed. + +--- + +## Phase 0 — Before anything + +### 0.0 The role of the session + +Every session of this procedure starts with `SET ROLE` to the role that owns +`file_change` (normally `penpot`): + +```sql +SET ROLE penpot; -- replace with the owner of file_change (see 0.6) +``` + +The new table, its 16 partitions, its indexes and its TOAST are created with +the role of the session, and `ALTER TABLE ... OWNER TO` does not recurse to the +partitions. Without this, a procedure run as a superuser leaves the table +owned by that superuser and the backend, which connects as `penpot`, gets +`permission denied` after the cutover. + +### 0.1 Mark the migration as applied + +The runner applies at startup every migration that is not in the `migrations` +table. Without this, it would copy the whole table inside the transaction of the +startup. + +```sql +INSERT INTO migrations (module, step) +VALUES ('main', '0158-partition-file-change'); +``` + +The module is `main` and the step has no `.sql` (see +`backend/src/app/util/migrations.clj`). + +### 0.2 Add the `kind` column to the current table + +Until the cutover, `file_change` is the current table, and the new code writes +`kind` on every insert: without the column the backend does not start. + +```sql +ALTER TABLE file_change ADD COLUMN IF NOT EXISTS kind smallint; +``` + +Nullable and without a default on purpose: it is a metadata-only operation +(it does not rewrite a single row of the table), the old code keeps inserting +rows, and the values of the rows that are already there are computed by the +copy from their `label`. + +### 0.3 Deploy the code + +The `updated_at` watermark has to be alive **before** the bulk copy: the new +code updates it on every mutation (locks, soft deletes), the old one does +not. Rows changed between the bulk and a delta pass would be lost. + +```sql +SELECT max(updated_at) FROM file_change; +``` + +### 0.4 Index for the delta passes + +The delta passes filter by `updated_at`. Without an index on that column every +pass scans the whole heap; with it, it is an index range that only +touches the rows that changed. + +```sql +-- CONCURRENTLY: it does not block the writes. It cannot run inside a +-- transaction: run it alone. +CREATE INDEX CONCURRENTLY IF NOT EXISTS file_change__updated_at__idx + ON file_change (updated_at); +``` + +- The build reads the heap once sequentially and writes one narrow entry per + row (check its size with `pg_relation_size` after the build): it does not + block the writes either way. Time the build and extrapolate the bulk from + it (an index on a column without TOAST does not touch the + TOAST). +- If it is interrupted, the index is left `INVALID` and has to be dropped and + built again: + +```sql +SELECT i.indisvalid + FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid + WHERE c.relname = 'file_change__updated_at__idx'; +``` + +- The index goes with the old table: at the cutover it is renamed to + `file_change__updated_at__idx_legacy` and disappears with it. The new table + does not carry it (nothing queries by `updated_at`; the GC uses + `deleted_at`). + +### 0.5 Check the environment + +```sql +-- Space: peak about the old table (it does not grow) + the new table +-- (at most its `pg_table_size`, see sizing) + the WAL the bulk generates +SELECT pg_size_pretty(pg_database_size(current_database())); + +-- The WAL must not run out of room (note the free space of the mount point): +-- df -h + +-- Nothing must reference file_change (0 rows expected): +SELECT conrelid::regclass FROM pg_constraint + WHERE confrelid = 'file_change'::regclass; + +-- There must be no triggers on file_change: +SELECT tgname FROM pg_trigger + WHERE tgrelid = 'file_change'::regclass AND NOT tgisinternal; + +-- Replication slots: the bulk generates WAL proportional to what it writes, +-- watch the retained WAL while the first batches run. +SELECT slot_name, active, pg_size_pretty(pg_wal_lsn_diff( + pg_current_wal_lsn(), restart_lsn)) AS retained_wal + FROM pg_replication_slots; + +-- The legacy blobs of the snapshots were already moved to `file_data` by +-- migration 0157-migrate-legacy-file-data (on the big instances, by hand, +-- with its runbook 2.19.0-migrate-legacy-file-data.md), so there is nothing +-- to copy here: the `data` column travels with the row to the new table. +-- These two numbers are informational guards. The first one has to be 0 +-- after 0157 ran; if it is not, run 0157 (or its runbook) before starting. +SELECT count(*) AS snapshots_without_data + FROM file_change AS fc + WHERE fc.data IS NOT NULL + AND fc.label IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM file_data AS fd + WHERE fd.file_id = fc.file_id + AND fd.id = fc.id + AND fd.type = 'snapshot'); + +-- The revisions with legacy data are not migrated (they are the ephemeral +-- changes, out of scope). This number is informative: their data travels +-- with the row to the new table. +SELECT count(*) AS revisions_with_data + FROM file_change + WHERE data IS NOT NULL + AND label IS NULL; +``` + +- Check that the backup / PITR is working before starting. +- Pick a low activity window for the cutover: a freeze of the writes to + `file_change` of a couple of minutes (the final delta inside the lock + copies only the rows changed since the last delta pass, so it is quick; the + saves of files during the freeze fail and the user retries, and the rest of + the database keeps working). The window has to cover the whole cutover, with + the backend and the workers stopped and started again (see 4.2). + +### 0.6 Owner and grants + +```sql +-- the role that has to own every object at the end (see 0.0) +SELECT tableowner FROM pg_tables + WHERE schemaname = current_schema() AND tablename = 'file_change'; + +-- grants added by the operator (monitoring, reporting). This procedure +-- creates a new table: they do not travel with it, so save them to re-apply +-- them after the cutover. +SELECT grantee, privilege_type + FROM information_schema.role_table_grants + WHERE table_schema = current_schema() AND table_name = 'file_change'; +``` + +--- + +## Phase 1 — Create the partitioned table + +The new indexes carry a `_new` suffix because index names are unique per +schema and the old table still holds the canonical ones until the cutover. +The same goes for the primary key of the new table: its index is named after +the constraint. The constraints that have no index (the `CHECK` and the +foreign keys) are named canonically from the start, because a constraint name +is unique per table, not per schema, so it can live with the one of the old +table; the partitions inherit it when they are created. The partitions are +created with their final name, because their indexes are named after the +partition and not after the table. + +```sql +CREATE TABLE file_change_new ( + file_id uuid NOT NULL, + id uuid NOT NULL DEFAULT gen_random_uuid(), + created_at timestamptz NOT NULL DEFAULT now(), + session_id uuid, + revn bigint NOT NULL DEFAULT 0, + data bytea, + changes bytea, + profile_id uuid, + features text[], + label text, + version integer, + created_by text NOT NULL DEFAULT 'system', + updated_at timestamptz DEFAULT now(), + deleted_at timestamptz, + migrations text[], + locked_by uuid, + -- Nullable on purpose, like the migration: the previous version inserts + -- without `kind`, and a rollback to it has to keep writing. The `NOT NULL` + -- comes in a later migration, once the rollback window is closed. + kind smallint, + CONSTRAINT file_change_kind_check CHECK (kind IN (1, 2)) +) PARTITION BY HASH (file_id); + +-- the columns keep the storage of PostgreSQL by default (migration +-- 0156-reset-pg-storage-defaults dropped the explicit EXTERNAL overrides) + +COMMENT ON COLUMN file_change_new.kind IS + 'The kind of the row: 1 is a snapshot (it has a label and its data lives in file_data) and 2 is a revision (it has no label and it holds the changes).'; +COMMENT ON COLUMN file_change_new.locked_by IS + 'Profile ID of user who has locked this version. Only the creator can lock/unlock their own versions. Locked versions cannot be deleted by others.'; + +-- 16 partitions, the same bounds as file_data. The name of the table is +-- temporary (the old one holds it until the cutover) but the name of the +-- partitions is not: it is the final one, so their indexes are born with +-- the canonical names (renaming a partition does not rename its indexes) +-- and the result is exactly the schema of the migration. +CREATE TABLE file_change_00 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 0); +CREATE TABLE file_change_01 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 1); +CREATE TABLE file_change_02 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 2); +CREATE TABLE file_change_03 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 3); +CREATE TABLE file_change_04 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 4); +CREATE TABLE file_change_05 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 5); +CREATE TABLE file_change_06 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 6); +CREATE TABLE file_change_07 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 7); +CREATE TABLE file_change_08 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 8); +CREATE TABLE file_change_09 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 9); +CREATE TABLE file_change_10 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 10); +CREATE TABLE file_change_11 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 11); +CREATE TABLE file_change_12 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 12); +CREATE TABLE file_change_13 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 13); +CREATE TABLE file_change_14 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 14); +CREATE TABLE file_change_15 PARTITION OF file_change_new + FOR VALUES WITH (MODULUS 16, REMAINDER 15); + +-- The primary key includes the partition key (required on a partitioned table) +ALTER TABLE file_change_new ADD PRIMARY KEY (file_id, id); + +-- Foreign keys (no CREATE TABLE copies them). They take the canonical name +-- already: they have no index, so they do not collide with the old table and +-- the partitions inherit it. +ALTER TABLE file_change_new + ADD CONSTRAINT file_change_file_id_fkey FOREIGN KEY (file_id) + REFERENCES file(id) DEFERRABLE; +ALTER TABLE file_change_new + ADD CONSTRAINT file_change_locked_by_fkey FOREIGN KEY (locked_by) + REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE; +ALTER TABLE file_change_new + ADD CONSTRAINT file_change_profile_id_fkey FOREIGN KEY (profile_id) + REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE; +``` + +```sql +SELECT count(*) AS partitions FROM pg_partition_tree('file_change_new') t + WHERE t.isleaf; +-- 16 +``` + +--- + +## Phase 2 — Bulk copy + +Copies every row (including the soft deleted ones: the GC manages them the +same way in the new table). The `kind` is computed from the `label`: a row +with a label is a snapshot, a row without one is a revision. + +The pointer of the copy lives in the database, not in the terminal: a batch +can be interrupted and resumed without losing anything. + +```sql +CREATE TABLE IF NOT EXISTS file_change_copy_state ( + name text PRIMARY KEY, + last_id uuid, + last_at timestamptz +); + +-- for a procedure that already created the table without the mark +ALTER TABLE file_change_copy_state + ADD COLUMN IF NOT EXISTS last_at timestamptz; + +INSERT INTO file_change_copy_state (name, last_id) +VALUES ('bulk', '00000000-0000-0000-0000-000000000000') +ON CONFLICT (name) DO NOTHING; + +-- The mark of the delta passes starts at the bulk: the first pass of phase 3 +-- covers the rows changed since here, and `DO NOTHING` keeps a re-run from +-- resetting a mark that the passes already moved. +INSERT INTO file_change_copy_state (name, last_at) VALUES ('delta', now()) +ON CONFLICT (name) DO NOTHING; + +-- The mark is mandatory: a pass without it copies nothing, in silence. The +-- `DROP ... IF EXISTS` keeps this block re-runnable. +ALTER TABLE file_change_copy_state + DROP CONSTRAINT IF EXISTS file_change_copy_state_delta_mark, + ADD CONSTRAINT file_change_copy_state_delta_mark + CHECK (name <> 'delta' OR last_at IS NOT NULL); +``` + +One batch (idempotent). It **always returns one row**: the number of rows it +copied, `0` when there is nothing left to copy. + +```sql +WITH batch AS MATERIALIZED ( + SELECT id FROM file_change + WHERE id > (SELECT last_id FROM file_change_copy_state WHERE name = 'bulk') + ORDER BY id + LIMIT 10000 +), +copied AS ( + INSERT INTO file_change_new (id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, kind) + SELECT id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, + CASE WHEN label IS NOT NULL THEN 1 ELSE 2 END + FROM file_change + WHERE id IN (SELECT id FROM batch) + ON CONFLICT (file_id, id) DO UPDATE + SET created_at = EXCLUDED.created_at, + session_id = EXCLUDED.session_id, + revn = EXCLUDED.revn, + data = EXCLUDED.data, + changes = EXCLUDED.changes, + profile_id = EXCLUDED.profile_id, + features = EXCLUDED.features, + label = EXCLUDED.label, + version = EXCLUDED.version, + created_by = EXCLUDED.created_by, + updated_at = EXCLUDED.updated_at, + deleted_at = EXCLUDED.deleted_at, + migrations = EXCLUDED.migrations, + locked_by = EXCLUDED.locked_by, + kind = EXCLUDED.kind + WHERE file_change_new.updated_at IS DISTINCT FROM EXCLUDED.updated_at + RETURNING 1 +), +-- `moved` advances the pointer only when the batch copied something. +-- PostgreSQL executes a data-modifying CTE exactly once even when the final +-- query does not read its output. +moved AS ( + UPDATE file_change_copy_state + SET last_id = (SELECT id FROM batch ORDER BY id DESC LIMIT 1) + WHERE name = 'bulk' + AND EXISTS (SELECT 1 FROM copied) + RETURNING 1 +) +SELECT count(*) AS copied FROM copied; +``` + +Repeat this statement until it returns `0`. The `0` comes from the final +`SELECT`, not from a missing row: a batch that finds nothing to copy still +returns it, so it is the signal to stop. + +- The pointer is a `uuid` (random order): the pagination is stable but it does + not group by physical position. The total read is the heap, in + a scattered order. +- **Estimated duration**: time the first batches and extrapolate. The bulk + reads the heap plus the live TOAST once (see sizing) and writes about the + live data. Batches are rows divided by batch size: count your rows in + sizing and divide. +- Pace the batches yourself (a pause between statements) if the streaming + replica falls behind or the backend sees latency. Every batch detoastes its + share of the TOAST: the cost is I/O, not CPU. +- Watch: replica lag, `pg_stat_activity` (locks between the bulk and + autovacuum), the space of `pg_wal` and of the data dir. + +```sql +SELECT count(*) AS missing + FROM file_change o + WHERE NOT EXISTS (SELECT 1 FROM file_change_new n + WHERE n.file_id = o.file_id + AND n.id = o.id); +``` + +`missing` is zero when no row of the old table is absent from the new one; it +is the check that matters, not the total count. The new table can have **more** +rows than the old one: the GC may have hard deleted a row from the source +after the bulk copied it, and it cleans that row up after the cutover. Rows +written during the bulk are missing until the delta passes pick them up. + +--- + +## Phase 3 — Delta passes + +Every pass copies the rows modified since the **previous pass**. The watermark +lives in `file_change_copy_state.last_at` (the start of the last pass; the +setup block of phase 2 seeds it with the moment of the bulk, so the first pass +covers what changed since then), and the SQL subtracts one hour of margin, +which absorbs the clock drift between the application (which writes +`updated_at`) and the database (which writes `now()`) and the drift between +batches. A row that a pass already copied but did not change again is not +rewritten: the `WHERE` of the `ON CONFLICT` skips it. + +```sql +BEGIN; + +WITH mark AS ( + SELECT last_at - INTERVAL '1 hour' AS since + FROM file_change_copy_state + WHERE name = 'delta' +) +INSERT INTO file_change_new (id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, kind) +SELECT id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, + CASE WHEN label IS NOT NULL THEN 1 ELSE 2 END + FROM file_change, mark + WHERE updated_at > mark.since +ON CONFLICT (file_id, id) DO UPDATE + SET created_at = EXCLUDED.created_at, + session_id = EXCLUDED.session_id, + revn = EXCLUDED.revn, + data = EXCLUDED.data, + changes = EXCLUDED.changes, + profile_id = EXCLUDED.profile_id, + features = EXCLUDED.features, + label = EXCLUDED.label, + version = EXCLUDED.version, + created_by = EXCLUDED.created_by, + updated_at = EXCLUDED.updated_at, + deleted_at = EXCLUDED.deleted_at, + migrations = EXCLUDED.migrations, + locked_by = EXCLUDED.locked_by, + kind = EXCLUDED.kind + WHERE file_change_new.updated_at IS DISTINCT FROM EXCLUDED.updated_at; + +-- The mark is the start of this transaction (`now()`): the rows changed +-- after it are the window of the next pass. Both statements go in the same +-- transaction on purpose; a mark written later would leave a gap. +UPDATE file_change_copy_state SET last_at = now() WHERE name = 'delta'; + +COMMIT; +``` + +- Repeat the transaction every 1-6 h (depending on the activity) until the + pass copies very few rows. With the index of phase 0.4 the pass is an index + range over the window, so it takes seconds on a quiet instance; without it, + it scans the heap. To see how many rows are left in the window: + +```sql +SELECT count(*) FROM file_change + WHERE updated_at > (SELECT last_at FROM file_change_copy_state WHERE name = 'delta'); +``` + +- The objects GC can keep running. Its soft deletes are covered by the + watermark, and its hard deletes leave a row with `deleted_at` in the new + table that the GC itself cleans up later. The hard delete of a **file** is + blocked while the old table (or the new one) has rows for it, because the + foreign key on `file_id` is `NO ACTION`. The GC deletes the pending files in + chunks of 100, and one blocked file aborts the transaction of the whole + chunk: the chunk is logged as `error on chunk deletion` and skipped, its + files wait in the trash, and the next daily run retries the same chunk and + fails again until the cutover. The other chunks keep making progress. **Do + not run the cutover at the same time as a big batch of the GC.** + +--- + +## Phase 4 — Cutover (a freeze of minutes) + +### 4.1 Before the lock + +- One last delta pass. +- Build the secondary indexes of the new table. The bulk and the delta passes + only need the primary key (`ON CONFLICT (file_id, id)`), so building them + here keeps five indexes and their WAL out of every batch. The `deleted_at` + index carries `file_id` (the primary key is `(file_id, id)`): the objects GC + needs the `file_id` to lock by pair, and with the index it reads the rows it + has to delete without touching the table. + +```sql +-- Secondary indexes: the canonical names, with the _new suffix. There is no +-- index for the system snapshots: the old one filtered them by +-- `data IS NOT NULL`, and with the payload living in `file_data` the column +-- no longer tells snapshots apart. The new one could only filter +-- them by `deleted_at IS NULL`, and the system snapshots always carry a +-- deletion date in the future, so it would never match a row. +CREATE INDEX file_change_new__deleted_at__idx + ON file_change_new (deleted_at, file_id, id) WHERE deleted_at IS NOT NULL; +CREATE INDEX file_change_new__file_id__revn__idx + ON file_change_new (file_id, revn); +CREATE INDEX file_change_new__locked_by__idx + ON file_change_new (locked_by) WHERE locked_by IS NOT NULL; +CREATE INDEX file_change_new__profile_id__idx + ON file_change_new (profile_id) WHERE profile_id IS NOT NULL; +CREATE INDEX file_change_new__user_snapshots__idx + ON file_change_new (file_id) + WHERE label IS NOT NULL AND created_by = 'user' AND deleted_at IS NULL; +``` + +- Prepare and rehearse the cutover script in staging. + +### 4.2 The cutover (one transaction, a brief exclusive lock) + +- **Stop the backend and the workers before the transaction, and start them + again after it.** The pool keeps server-side prepared statements, and a + prepared statement holds the OID of the table it was prepared with: + renaming the table does not change that OID. A connection that had already + run `insert into file_change` five times (the default threshold of the + driver) would keep writing to `file_change_legacy` after the cutover, and + the pool recycles its connections every 30 minutes, so the window of + silent writes is long. +- Run the transaction, the `ANALYZE`, and the statement that records the + instant of the cutover (all of it in the block below). +- Start the backend and the workers again. + +```sql +SET ROLE penpot; -- the owner of file_change (see 0.0 and 0.6) + +BEGIN; + +LOCK TABLE file_change IN ACCESS EXCLUSIVE MODE; + +-- The final delta: the rows modified since the last pass (its mark, minus +-- the hour of margin). The mark is the one the last pass left. +WITH mark AS ( + SELECT last_at - INTERVAL '1 hour' AS since + FROM file_change_copy_state + WHERE name = 'delta' +) +INSERT INTO file_change_new (id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, kind) +SELECT id, file_id, created_at, session_id, revn, data, changes, + profile_id, features, label, version, created_by, + updated_at, deleted_at, migrations, locked_by, + CASE WHEN label IS NOT NULL THEN 1 ELSE 2 END + FROM file_change, mark + WHERE updated_at > mark.since +ON CONFLICT (file_id, id) DO UPDATE + SET created_at = EXCLUDED.created_at, + session_id = EXCLUDED.session_id, + revn = EXCLUDED.revn, + data = EXCLUDED.data, + changes = EXCLUDED.changes, + profile_id = EXCLUDED.profile_id, + features = EXCLUDED.features, + label = EXCLUDED.label, + version = EXCLUDED.version, + created_by = EXCLUDED.created_by, + updated_at = EXCLUDED.updated_at, + deleted_at = EXCLUDED.deleted_at, + migrations = EXCLUDED.migrations, + locked_by = EXCLUDED.locked_by, + kind = EXCLUDED.kind + WHERE file_change_new.updated_at IS DISTINCT FROM EXCLUDED.updated_at; + +-- Check inside the lock, and abort if it fails: no row of the old table may +-- be missing in the new one. The new table can carry extra rows that the GC +-- hard deleted from the source; it cleans them up after the cutover. +DO $$ +DECLARE missing bigint; +BEGIN + SELECT count(*) INTO missing + FROM file_change o + WHERE NOT EXISTS (SELECT 1 FROM file_change_new n + WHERE n.file_id = o.file_id + AND n.id = o.id); + + IF missing > 0 THEN + RAISE EXCEPTION 'file_change: % rows of the old table are missing in file_change_new; cutover aborted', missing; + END IF; +END $$; + +-- The old table frees the canonical names (the same names the migration +-- 0158 uses, with the _legacy suffix) +ALTER TABLE file_change RENAME CONSTRAINT file_change_pkey + TO file_change_pkey_legacy; +ALTER TABLE file_change RENAME CONSTRAINT file_change_file_id_fkey + TO file_change_file_id_fkey_legacy; +ALTER TABLE file_change RENAME CONSTRAINT file_change_locked_by_fkey + TO file_change_locked_by_fkey_legacy; +ALTER TABLE file_change RENAME CONSTRAINT file_change_profile_id_fkey + TO file_change_profile_id_fkey_legacy; + +ALTER INDEX file_change__data_ref_id__idx + RENAME TO file_change__data_ref_id__idx_legacy; +ALTER INDEX file_change__deleted_at__idx + RENAME TO file_change__deleted_at__idx_legacy; +ALTER INDEX file_change__file_id__revn__idx + RENAME TO file_change__file_id__revn__idx_legacy; +ALTER INDEX file_change__locked_by__idx + RENAME TO file_change__locked_by__idx_legacy; +ALTER INDEX file_change__profile_id__idx + RENAME TO file_change__profile_id__idx_legacy; +ALTER INDEX file_change__system_snapshots__idx + RENAME TO file_change__system_snapshots__idx_legacy; +ALTER INDEX IF EXISTS file_change__user_snapshots__idx + RENAME TO file_change__user_snapshots__idx_legacy; +ALTER INDEX IF EXISTS file_change__updated_at__idx + RENAME TO file_change__updated_at__idx_legacy; + +-- And the new table takes them. Only the primary key needs the rename: the +-- `CHECK` and the foreign keys already carry the canonical name (they have no +-- index, so they could take it from the start), and the partitions inherited +-- it. +ALTER TABLE file_change_new RENAME CONSTRAINT file_change_new_pkey + TO file_change_pkey; + +ALTER INDEX file_change_new__deleted_at__idx + RENAME TO file_change__deleted_at__idx; +ALTER INDEX file_change_new__file_id__revn__idx + RENAME TO file_change__file_id__revn__idx; +ALTER INDEX file_change_new__locked_by__idx + RENAME TO file_change__locked_by__idx; +ALTER INDEX file_change_new__profile_id__idx + RENAME TO file_change__profile_id__idx; +ALTER INDEX file_change_new__user_snapshots__idx + RENAME TO file_change__user_snapshots__idx; + +-- And the tables are swapped. The partitions and their indexes keep the +-- names they were born with (they are named after the partition, not after +-- the table), so there is nothing else to rename. +ALTER TABLE file_change RENAME TO file_change_legacy; +ALTER TABLE file_change_new RENAME TO file_change; + +-- The foreign key of the legacy table goes away: it is `NO ACTION`, so +-- while the legacy table has rows for a file, the objects GC cannot delete +-- that file (it deletes the rows of the file first, and the legacy ones are +-- not its business). The legacy table is frozen: nothing writes to it again. +ALTER TABLE file_change_legacy + DROP CONSTRAINT file_change_file_id_fkey_legacy; + +COMMIT; + +ANALYZE file_change; + +-- The instant of the cutover, for the checks of phase 5. It goes after the +-- `COMMIT`: inside the transaction `now()` is the start of it (the moment of +-- the `LOCK`), and the rows the final delta copies could pass for writes +-- happened after the cutover. +INSERT INTO file_change_copy_state (name, last_at) VALUES ('cutover', now()) +ON CONFLICT (name) DO UPDATE SET last_at = excluded.last_at; +``` + +- `file_change_legacy` keeps its data and its indexes (with the `_legacy` + suffix): a frozen copy of the previous layout. Its foreign key on `file_id` + is dropped, so the objects GC can delete the files again. +- The writes to `file_change` during the lock fail: they are seconds-minutes, + the affected saves are retried. +- The objects GC and the four RPC commands keep working: same table name and + same index names after the rename. +- Once the backend is up again, check that the saves go to the partitioned + table and that nothing went to the table that is going away: + +```sql +-- the rows written after the cutover: this one grows with every save +SELECT count(*) FROM file_change + WHERE updated_at > (SELECT last_at FROM file_change_copy_state + WHERE name = 'cutover'); + +-- the rows that went to the legacy table: this one has to stay at 0 +SELECT count(*) FROM file_change_legacy + WHERE updated_at > (SELECT last_at FROM file_change_copy_state + WHERE name = 'cutover'); +``` + +A number above zero in the second query means that some connection was +still writing with a prepared statement of the old table: those rows are +lost when the legacy table is dropped, so they have to be copied over +before that. + +--- + +## Phase 5 — After the cutover + +1. Check that the access paths prune to one partition (when the planner + chooses an index scan): + +```sql +EXPLAIN SELECT * FROM file_change + WHERE file_id = '00000000-0000-0000-0000-000000000000' + AND id = '00000000-0000-0000-0000-000000000000'; +``` + +It has to show a single partition (`file_change_NN`), not an `Append` of 16. + +2. Check that the schema is the one of the migration `0158`. Every row of this + query has to show `got = expected`: + +```sql +SELECT 'columns' AS what, count(*) AS got, 17 AS expected + FROM pg_attribute + WHERE attrelid = 'file_change'::regclass AND attnum > 0 AND NOT attisdropped +UNION ALL +SELECT 'partitions', count(*), 16 + FROM pg_partition_tree('file_change') t WHERE t.isleaf +UNION ALL +SELECT 'primary key', count(*), 1 + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'p' +UNION ALL +SELECT 'foreign keys', count(*), 3 + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'f' +UNION ALL +SELECT 'check constraints', count(*), 1 + FROM pg_constraint + WHERE conrelid = 'file_change'::regclass AND contype = 'c' +UNION ALL +SELECT 'indexes', count(*), 6 + FROM pg_indexes + WHERE tablename = 'file_change' AND schemaname = current_schema() +UNION ALL +SELECT 'partitions out of name', count(*), 0 + FROM pg_class + WHERE relkind = 'r' AND relname ~ '^file_change_new[0-9]' +UNION ALL +SELECT 'child indexes out of name', count(*), 0 + FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + JOIN pg_class p ON p.oid = i.indrelid + WHERE p.relname ~ '^file_change_[0-9]{2}$' + AND c.relname NOT LIKE p.relname || '_%' +UNION ALL +SELECT 'partition constraints out of name', count(*), 0 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + WHERE t.relname ~ '^file_change_[0-9]{2}$' + AND c.contype <> 'n' + AND c.conname LIKE 'file_change_new%'; +``` + + The `NOT NULL` constraints (`contype = 'n'`, tracked in `pg_constraint` from + PostgreSQL 17) are excluded from the last check: their name is generated by + the column and changes with the server version, so they are not part of the + parity. The `CHECK`, the primary key and the foreign keys are. + + And the names, to compare them with the migration: + +```sql +SELECT a.attname AS column + FROM pg_attribute a + WHERE a.attrelid = 'file_change'::regclass AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum; + +SELECT indexname FROM pg_indexes + WHERE tablename = 'file_change' AND schemaname = current_schema() + ORDER BY indexname; + +-- the partitions, with their name +SELECT c.relname AS partition + FROM pg_class c + JOIN pg_partition_tree('file_change') t ON t.relid = c.oid + WHERE t.isleaf + ORDER BY c.relname; + +-- the indexes of the partitions: their name starts with the partition +SELECT c.relname AS child_index + FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + JOIN pg_class p ON p.oid = i.indrelid + WHERE p.relname ~ '^file_change_[0-9]{2}$' + ORDER BY c.relname; + +-- every table, partition and index of the family shares the owner of the +-- table: a session without `SET ROLE` (0.0) shows here +SELECT count(*) AS out_of_owner + FROM pg_class c + WHERE c.relname LIKE 'file_change%' + AND c.relkind IN ('r', 'p', 'i') + AND pg_get_userbyid(c.relowner) <> + (SELECT tableowner FROM pg_tables + WHERE schemaname = current_schema() AND tablename = 'file_change'); +``` + + The owner check has to return `0`. If it does not, the session that + created the table did not use `SET ROLE`: fix it before opening the + service (re-own the table, the 16 partitions and their indexes) and + re-apply the grants saved in 0.6. + +3. The partition-wise join is already on: the pool of the backend sets + `enable_partitionwise_join = on` on every connection (it is in the code, + there is nothing to change in the database role). + +4. Autovacuum: each of the 16 partitions has its own TOAST (check their + sizes with `pg_total_relation_size` per partition); with + `autovacuum_vacuum_scale_factor = 0.2` the threshold is reached earlier + and in steps. Nothing to change, just watch `last_autovacuum` of the + partitions the first week. + +5. The aggregations that do not filter by `file_id` (the team snapshot + quota and the telemetry counts) read all 16 partitions. To know their + cost on your instance, time them before and after with + `EXPLAIN (ANALYZE, BUFFERS)` and compare: + + - The team quota is already fixed in the code: it resolves the `file_id` of + the team first, which lets the planner prune the partitions when it + chooses an index scan (with a sequential scan it reads all of them). + - The telemetry scans the whole heap, the same before and after: the + partitioned run pays a small cost per partition on top. To remove that + scan, the index `file_change__created_at__idx` turns both metrics into an + index scan over the rows of the day, at the cost of one more index on the + biggest table. Decide it before phase 1, the index has to exist in both + tables. + +6. Drop the helper table of the copy and, days later, the old table: + +```sql +DROP TABLE IF EXISTS file_change_copy_state; +DROP TABLE file_change_legacy; -- out of hours: it generates WAL by + -- dropping the pages, let it roll +``` + +That is where the dead space comes back: the database shrinks by about the +size of the old table (see sizing). + +--- + +## Risks + +| Risk | Mitigation | +|---|---| +| Massive WAL during the bulk (proportional to the copy; see sizing) | Throttle between batches; watch `pg_wal`, the replication slots and the replica lag | +| Latency seen by the application during the bulk | Throttle; low traffic hours; the bulk is I/O and does not block writes | +| `updated_at` not reliable in the rows changed before the deploy | The deploy goes before the bulk (0.3) and the delta passes cover from the start of the bulk | +| Lock/unlock/soft delete during the passes | Covered by the watermark; the cutover includes the final delta inside the lock | +| Hard delete of the GC between passes | The row is left with `deleted_at` in the new table; the GC cleans it up later | +| Orphan rows if a `file` is deleted halfway | The foreign key on `file_id` is `NO ACTION`: the GC deletes the rows of the file before the file, so a chunk that fails leaves the file in the trash and the next run deletes it | +| The files in the trash cannot be deleted while the legacy table has rows for them | The foreign key of the legacy table is dropped at the cutover; until then the files wait in the trash | +| Saves failing during the lock of the cutover | Freeze of minutes in a low traffic window; manual retries | +| Planner regression after the cutover | Immediate `ANALYZE` + the pruning `EXPLAIN`; the pool already sets the GUC | +| Clock drift between the application and the database | The delta window starts an hour before the bulk; the margin absorbs the drift | + +## Success metrics + +- After the cutover: `file_change` (partitioned) holds at most the + `pg_table_size` of the old table (see sizing). +- After the `DROP` of the old table: the database shrinks by about the size + that table had. +- The access paths that filter by `file_id` (the lookup of a snapshot, the + quota of a file and the lock of the GC by pair) prune to one partition when + the planner chooses an index scan (`EXPLAIN`). The quota of the team and + the discovery of the GC do not filter by a single file: they read all the + 16 partitions. +- No visible change for the user. diff --git a/operations/2.20.0-file-data-storage-object-id.md b/operations/2.20.0-file-data-storage-object-id.md new file mode 100644 index 0000000000..5355c3a909 --- /dev/null +++ b/operations/2.20.0-file-data-storage-object-id.md @@ -0,0 +1,285 @@ +# 2.20.0 — File data storage object foreign key + +Runbook for the release that moves the file data storage pointer out of +`file_data.metadata->>'storage-ref-id'` and into a column, +`file_data.storage_object_id`, with a foreign key to `storage_object(id)`. + +## When to run this + +The migration `0159-add-file-data-storage-object-id` does the whole job on a +fresh database and on small instances: it adds the column, backfills the +pointer, creates the index and the foreign key. + +A large instance builds everything online and keeps the stopped window as +short as possible: + +- the column, the index and the foreign key are built online, without stopping + the service; +- the backfill runs in the stopped window **only if** the metadata pointer + exists; when the storage backend was never enabled there is nothing to + backfill and the new binaries roll out like any other deploy; +- the step is then marked as applied in the `migrations` table, so the + migration body does not run at startup. + +The coordinated restart is required only when the instance uses the storage +backend (`PENPOT_FILE_DATA_BACKEND=storage`) or still has rows that carry the +metadata pointer. The previous code writes and reads `metadata.storage-ref-id`, +the new code reads the column, and a rolling deploy would mix both. With the +default `db` backend and no row carrying the pointer (see the pre-flight +query), the previous and the new code read the same `file_data.data` column, so +a rolling deploy is safe. + +**Do not run the backfill while the previous version is serving** when +`PENPOT_FILE_DATA_BACKEND` is configured to `storage`, or when the table still +has rows with the metadata pointer. The previous code writes the pointer only to +`metadata.storage-ref-id`, and its storage GC reads the reference from there. +Removing the key while that code runs leaves the rows it writes without a +pointer (unreadable after the deploy), and it can make the GC delete an object +that a live row still references. When the pre-flight query returns 0 there is +nothing to backfill and this risk does not apply. + +## 0. Pre-flight + +```sql +SELECT version(); -- 16.x: `NOT VALID` foreign keys on a partitioned table need 18 + +SELECT count(*) FROM pg_inherits + WHERE inhparent = 'file_data'::regclass; -- 16 + +-- Whether the storage backend was ever enabled. When this is 0 there is no +-- pointer in the metadata and step 6 is skipped. +SELECT count(*) FROM file_data + WHERE backend = 'storage' AND jsonb_exists(metadata, 'storage-ref-id'); + +-- Every online ALTER uses a short lock timeout and is retried, so a long +-- transaction cannot make it queue and block the table behind it. +SELECT pid, state, now() - xact_start AS age, left(query, 60) + FROM pg_stat_activity + WHERE xact_start IS NOT NULL + AND now() - xact_start > interval '1 minute'; +``` + +## 1. Column + +```sql +BEGIN; +SET LOCAL lock_timeout = '3s'; +ALTER TABLE file_data + ADD COLUMN storage_object_id uuid NULL; +COMMIT; +``` + +Skip this step if the column already exists. The change is catalog-only (a +nullable column without a default), so it takes `ACCESS EXCLUSIVE` for an +instant. If it fails with `lock timeout`, wait for a quieter moment and retry: +it did not block anything. + +## 2. One index per partition, concurrently + +```sql +CREATE INDEX CONCURRENTLY file_data_00__storage_object_id__idx + ON file_data_00 (storage_object_id); +-- ... up to file_data_15 +``` + +Run each statement in autocommit: `CREATE INDEX CONCURRENTLY` cannot run inside +a transaction block. An interrupted build leaves the index `INVALID`: drop it +and retry. + +```sql +SELECT c.relname, i.indisvalid + FROM pg_index AS i + JOIN pg_class AS c ON c.oid = i.indexrelid + WHERE c.relname LIKE 'file_data_%storage_object_id%' + ORDER BY c.relname; +``` + +## 3. Parent index and partition attachment + +```sql +CREATE INDEX file_data__storage_object_id__idx + ON ONLY file_data (storage_object_id); + +ALTER INDEX file_data__storage_object_id__idx + ATTACH PARTITION file_data_00__storage_object_id__idx; +-- ... up to file_data_15 +``` + +The step creates the index unconditionally: if an index with that name already +exists, drop it and rebuild it. Verify the result before continuing: + +```sql +SELECT i.indisvalid, i.indpred IS NULL AS plain, i.indexprs IS NULL AS no_expression + FROM pg_index AS i + WHERE i.indexrelid = 'file_data__storage_object_id__idx'::regclass; +-- t | t | t + +SELECT count(*) FROM pg_inherits + WHERE inhparent = 'file_data__storage_object_id__idx'::regclass; +-- 16 +``` + +## 4. Foreign key, online + +```sql +BEGIN; +SET LOCAL lock_timeout = '3s'; +ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE; +COMMIT; +``` + +The column is NULL until step 6, so the validation finds nothing to check and +the `SHARE ROW EXCLUSIVE` lock (it blocks writes, not reads) is brief. Retry on +`lock timeout`. If the constraint already exists, do not add it again: verify +its shape with the query in step 7. Drop it first only if the shape is wrong. + +Creating it here, before the backfill, keeps the full validation scan out of +the stopped window: the backfill validates each row it writes. + +`INITIALLY IMMEDIATE` is required. The storage GC catches the violation inside +the chunk transaction and retries the chunk; an `INITIALLY DEFERRED` check +fires at commit, outside that transaction, and the GC would never move past a +blocked chunk. + +## 5. Stop every instance + +Coordinated restart when the pre-flight count is not 0: stop all of them before +the next step. No rolling deploy. When the count is 0 there is nothing to +backfill: run steps 1-4 online, run the verification (step 7) and mark the step +as applied (step 8) without stopping the service. + +## 6. Backfill, only if the metadata pointer exists + +Skip this step when the pre-flight count is 0: the storage backend was never +enabled, no row carries the pointer, and the new code reads all file data from +the `data` column. + +```sql +WITH candidates AS MATERIALIZED ( + SELECT file_id, + id, + (metadata->>'storage-ref-id')::uuid AS soid + FROM file_data + WHERE backend = 'storage' + AND storage_object_id IS NULL + AND (metadata->>'storage-ref-id') ~* + '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$' +) +UPDATE file_data AS fd + SET storage_object_id = c.soid, + metadata = fd.metadata - 'storage-ref-id' + FROM candidates AS c + WHERE fd.file_id = c.file_id + AND fd.id = c.id + AND EXISTS (SELECT 1 FROM storage_object AS so WHERE so.id = c.soid); +``` + +The `MATERIALIZED` CTE keeps the uuid cast away from values that did not pass +the regex: without it PostgreSQL can inline the subquery and cast a corrupt +value, aborting the statement. The foreign key already exists, so every written +row is validated here instead of in a separate full scan. + +## 7. Verify + +Do not mark the migration as applied until every check passes. + +```sql +-- Column. +SELECT data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'file_data' + AND column_name = 'storage_object_id'; +-- uuid | YES + +-- Rows that will be unreadable. Must be 0. The backfill only fills pointers it +-- can resolve, so a non-zero count means pointers that were already corrupt or +-- orphan: inspect them and accept them before continuing. With the backfill +-- skipped this must still be 0, or the storage backend was enabled after all. +-- A row pointing at a storage_object with deleted_at set is legitimate and +-- restorable, not unreadable: the backfill fills it on purpose and the +-- restore flow revives the object, so it is not counted here. +SELECT count(*) FROM file_data + WHERE backend = 'storage' AND storage_object_id IS NULL; + +-- Index: valid, plain, with every partition attached. +SELECT i.indisvalid, i.indpred IS NULL AS plain, i.indexprs IS NULL AS no_expression + FROM pg_index AS i + WHERE i.indexrelid = 'file_data__storage_object_id__idx'::regclass; +SELECT count(*) FROM pg_inherits + WHERE inhparent = 'file_data__storage_object_id__idx'::regclass; -- 16 + +-- Foreign key: deferrable, checked immediately, no action, validated. +SELECT condeferrable, condeferred, confdeltype, convalidated + FROM pg_constraint + WHERE conname = 'file_data_storage_object_id_fkey' + AND conrelid = 'file_data'::regclass; +-- t | f | a | t +``` + +## 8. Mark the migration as applied + +Only after step 7 passes: + +```sql +INSERT INTO migrations (module, step) +VALUES ('main', '0159-add-file-data-storage-object-id') +ON CONFLICT (module, step) DO NOTHING; +``` + +Check it: + +```sql +SELECT module, step, created_at FROM migrations + WHERE module = 'main' AND step = '0159-add-file-data-storage-object-id'; +``` + +The row is the only thing that skips the migration at startup. Without it, the +migration runs at the next start and repeats the backfill and the foreign key. + +## 9. Start the new instances + +Start every instance you stopped in step 5. When the pre-flight count was 0 +nothing was stopped, and the rolling deploy simply finishes. + +## Post-deploy checks + +```sql +SELECT conname, condeferrable, condeferred, confdeltype, convalidated + FROM pg_constraint + WHERE conname = 'file_data_storage_object_id_fkey'; + +SELECT count(*) FROM file_data + WHERE backend = 'storage' AND storage_object_id IS NULL; + +-- The delete check must use the index, not a sequential scan. +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) + DELETE FROM storage_object WHERE id = ''; +ROLLBACK; +``` + +## Rollback + +Before step 8: + +```sql +ALTER TABLE file_data + DROP CONSTRAINT IF EXISTS file_data_storage_object_id_fkey; + +-- A partitioned index cannot be dropped with CONCURRENTLY; the plain drop +-- takes a brief ACCESS EXCLUSIVE. +DROP INDEX file_data__storage_object_id__idx; + +ALTER TABLE file_data DROP COLUMN storage_object_id; +``` + +After step 8 the backfill has already removed the metadata key that the +previous code reads, so a rollback to the previous binaries is **not +supported** when the instance uses the storage backend: the files offloaded by +the new code would be unreadable, and the recovery is a data restore, not a +binary switch. With the default `db` backend and no storage rows, the previous +binaries read the same `data` column and the rollback is a plain binary switch. diff --git a/operations/README.md b/operations/README.md new file mode 100644 index 0000000000..9590e8e865 --- /dev/null +++ b/operations/README.md @@ -0,0 +1,16 @@ +# Operations + +Procedures that are executed by hand on an instance, not by the application. + +They are not part of the product: they describe how to do a one-off operation +on a running database, with the exact statements to run and what to check +between them. + +Each procedure lives here while the operation is alive. When the operation is +done, the file is either deleted or marked as historical, so the folder does +not grow with procedures that nobody will run again. + +| Procedure | Version | +|---|---| +| [Migrate legacy file payloads to `file_data`](2.19.0-migrate-legacy-file-data.md) | 2.19.0 | +| [Partition `file_change` by `file_id`](2.20.0-file-change-partition.md) | 2.20.0 | diff --git a/package.json b/package.json index a27aad2cc9..ebe144f38a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en España SL", "private": true, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "repository": { "type": "git", "url": "https://github.com/penpot/penpot" diff --git a/plugins/CHANGELOG.md b/plugins/CHANGELOG.md index 5a6b8cd350..b86307e031 100644 --- a/plugins/CHANGELOG.md +++ b/plugins/CHANGELOG.md @@ -1,17 +1,10 @@ -## 1.6.0 (2026-11-10) +## 1.7.0 (Unreleased) ### 🚀 Features - **plugin-types, plugins-runtime:** `penpotMgmt` returns `Team`, `Project` and `ProjectFile` objects. Assigning `name` renames teams, projects and files, `project.pinned` pins a project and `file.shared` shares a file as a team library; `createProject`, `createFile`, `moveTo`, `duplicate`, `remove` and `restore` live on the objects. Added `penpotMgmt.listTeams`, `penpotMgmt.createTeam`, `penpotMgmt.getFile`, `listDeletedProjects` and `listDeletedFiles`, and the `manage:projects`, `manage:teams` and `manage:delete` permissions. Restoration uses the dashboard's Trash operation; projects need recoverable files and empty projects cannot be restored; the API exposes no permanent deletion method. Team deletion is not exposed. - **plugins-runtime, plugin-types**: Added global plugins (`scope: "global"`) that remain active across dashboard and workspace navigation until closed or logged out, with `penpotMgmt` for project and file discovery, file navigation and workspace events. Global scope adds the `allow:global` permission, which users must accept when installing the plugin. -- **plugin-types:** Added `LibraryComponent.restore()` to restore a deleted component and its main instance without changing the current page, selection, zoom, or viewport. Requires `library:write` permission and only applies to the current file's library. -- **plugin-types:** Added `Shape.updateMain()`, `LibraryComponent.duplicate()`, writable component `annotation`, and `Library.deletedComponents`. Declared the existing `LibraryComponent.remove()` method. -- **plugin-types:** Added `LibraryContext.tokensSource` to read or set the current file's token source to a connected library, or restore local tokens. -- **plugin-types:** Added `paddingType` (`'simple' | 'multiple'`) to flex and grid layouts and `marginType` (`'simple' | 'multiple'`) to layout children, exposing whether the four padding/margin sides are mirrored or honoured independently. -- **plugin-types**: Added `waitForLayoutUpdate` to wait until pending layout updates have finished. It rejects when the optional timeout elapses, defaulting to 30 seconds so a wait never hangs. -- **plugin-types**: Added `waitForLayoutUpdate` to the `Shape` interface to wait until the pending layout updates of a shape and its children have finished - **plugin-types:** Added the `contentrollback` event and the `ContentRollback` type. The event fires after the backend rejected some changes and the workspace undid them, with the ids of the files, pages and shapes affected. It fires once the undone data is in the workspace, also when the workspace has to reload the file to undo the changes. -- **plugin-types:** `Stroke` exposes per-side widths for boards and rectangles, also supported by `borderWidth` and `dimension` tokens. Read-only `strokeWidthType` reports `simple` when all four effective widths match, otherwise `multiple`. ### 🩹 Fixes @@ -21,6 +14,26 @@ - **plugins-runtime**: `library.connectLibrary()` now resolves once the library's assets are loaded; it used to resolve with an empty library the first time. `library.connected` and `library.availableLibraries()` ignore files opened earlier in the same tab, which used to show up as connected and be left out of the available libraries. - **plugins-runtime:** Writes now fail with an error while the workspace cannot take them (saves failing, a user who cannot edit, a version preview, a canvas that stopped rendering), whatever the `throwValidationErrors` flag. They used to change the workspace with edits that would never be saved. The error says why. The Inspect tab does not block writes, and undo blocks still open and close. - **plugins-runtime:** Writes that wait for the server (`createShapeFromSvgWithImages`, `uploadMediaData`, `uploadMediaUrl`, `connectLibrary`) check again when the server answers. If the workspace turned read-only meanwhile (a save failed), the promise rejects with the read-only error: no shape is added and no library is linked. An image the server already stored stays in the file's media, unused. +- **plugins-runtime:** `Shape.component()` returns `null` for a copy whose library is not available, instead of throwing `Assert failed: (uuid? id)`. +- **plugins-runtime:** `Shape.componentRoot()` returns the root of the shape's own component tree. It used to return a shape that pointed at nothing when the copy was on another page or file than its main, and threw when the copy's library was not available. +- **plugins-runtime:** `LibraryComponent.mainInstance()` returns `null` for a deleted component, instead of a shape with an empty `id` whose `parent` throws. +- **plugins-runtime**: `TokenFontFamilies.resolvedValue` and the `fontFamilies` member of `TokenTypographyValue` now keep a family whose name has several words (`Hanken Grotesk`) as one entry, instead of splitting it into an array of its words, and keep names with words the resolver would otherwise read as colors or numbers (`Red Hat Display`, `Press Start 2P`) unchanged. +- **plugins-runtime:** A SES `repairIntrinsics` failure at startup (for example, when a browser extension wraps the native `Temporal` global) no longer leaves the whole app blank. Plugins are skipped for that session, the error is logged, and a telemetry event reports the unavailable runtime; the editor, login and sign-up pages load as usual. Plugins work as before in every other environment. When startup fails, SES has already replaced `Function.prototype.toString` in the host realm before throwing, but no intrinsics are hardened and no sandbox is created. Closes #12924. + +## 1.6.0 (2026-11-10) + +### 🚀 Features + +- **plugin-types:** Added `LibraryComponent.restore()` to restore a deleted component and its main instance without changing the current page, selection, zoom, or viewport. Requires `library:write` permission and only applies to the current file's library. +- **plugin-types:** Added `Shape.updateMain()`, `LibraryComponent.duplicate()`, writable component `annotation`, and `Library.deletedComponents`. Declared the existing `LibraryComponent.remove()` method. +- **plugin-types:** Added `LibraryContext.tokensSource` to read or set the current file's token source to a connected library, or restore local tokens. +- **plugin-types:** Added `paddingType` (`'simple' | 'multiple'`) to flex and grid layouts and `marginType` (`'simple' | 'multiple'`) to layout children, exposing whether the four padding/margin sides are mirrored or honoured independently. +- **plugin-types**: Added `waitForLayoutUpdate` to wait until pending layout updates have finished. It rejects when the optional timeout elapses, defaulting to 30 seconds so a wait never hangs. +- **plugin-types**: Added `waitForLayoutUpdate` to the `Shape` interface to wait until the pending layout updates of a shape and its children have finished +- **plugin-types:** `Stroke` exposes per-side widths for boards and rectangles, also supported by `borderWidth` and `dimension` tokens. Read-only `strokeWidthType` reports `simple` when all four effective widths match, otherwise `multiple`. + +### 🩹 Fixes + - **plugins-runtime:** `LibraryComponent.remove()` now rejects linked-library and missing component handles, and does nothing for an already deleted component. - **plugins-runtime**: An interaction obtained from `Shape.interactions` now keeps addressing that interaction instead of the position it held when the array was read. Removing every interaction of a shape from a single read removes all of them rather than leaving some behind, and writing through a held interaction after an earlier one is removed no longer lands on a different interaction. - **plugins-runtime**: `Shape.removeInteraction()` now rejects an interaction belonging to a different shape with a validation error, instead of removing whichever interaction sat at the same position on the target shape. @@ -30,7 +43,6 @@ - **plugins-runtime**: Removed the premature deep-hardening of the host plugin context, which froze shared host functions (including `Function.prototype`) before SES override taming, causing `TypeError: Cannot assign to read only property 'toString'` on later host-side function extension. Related to #11001. - **plugins-runtime**: The plugin modal no longer shows two resize grips in Firefox. Firefox now shows only its native grip. Closes #11795. - **plugins-runtime**: Fixed the `fontFamilies` token property mapping so `Shape.applyToken(token, ["fontFamilies"])` resolves to the canonical `:font-family` attribute and applied-token readback exposes the documented `fontFamilies` key instead of the undocumented singular `fontFamily`. Closes #11405. -- **plugins-runtime**: `TokenFontFamilies.resolvedValue` and the `fontFamilies` member of `TokenTypographyValue` now keep a family whose name has several words (`Hanken Grotesk`) as one entry, instead of splitting it into an array of its words, and keep names with words the resolver would otherwise read as colors or numbers (`Red Hat Display`, `Press Start 2P`) unchanged. - **plugins-runtime**: Opening a plugin no longer drops running background plugins (`allowBackground`, such as the MCP plugin) from the runtime registry. They kept running, but their UI messages were no longer delivered to the plugin and they could no longer be unloaded, so MCP tasks timed out after another plugin was opened. - **plugins-runtime**: `Shape.applyToken()`, `Token.applyToShapes()` and `Token.applyToSelected()` now reject an explicit per-side stroke width property (`strokeWidthTop`, `strokeWidthRight`, `strokeWidthBottom`, `strokeWidthLeft`) on shapes other than boards and rectangles, matching the `strokes` setter. The uniform `strokeWidth` property keeps working on every shape. Closes #12319. diff --git a/plugins/apps/colors-to-tokens-plugin/package.json b/plugins/apps/colors-to-tokens-plugin/package.json index cfed5c5f6b..4b3815450e 100644 --- a/plugins/apps/colors-to-tokens-plugin/package.json +++ b/plugins/apps/colors-to-tokens-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/colors-to-tokens-plugin/src/app/app.component.ts b/plugins/apps/colors-to-tokens-plugin/src/app/app.component.ts index 1e1e204541..b3a4412a00 100644 --- a/plugins/apps/colors-to-tokens-plugin/src/app/app.component.ts +++ b/plugins/apps/colors-to-tokens-plugin/src/app/app.component.ts @@ -72,7 +72,9 @@ export class AppComponent { messages$ = fromEvent>(window, 'message'); initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); theme = toSignal( diff --git a/plugins/apps/composable-test-suite/package.json b/plugins/apps/composable-test-suite/package.json index 81261c19e9..3ed82a417d 100644 --- a/plugins/apps/composable-test-suite/package.json +++ b/plugins/apps/composable-test-suite/package.json @@ -27,5 +27,5 @@ "vite": "^8.3.2", "vite-live-preview": "^0.4.0" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/contrast-plugin/package.json b/plugins/apps/contrast-plugin/package.json index d773fe0297..c691b94f0c 100644 --- a/plugins/apps/contrast-plugin/package.json +++ b/plugins/apps/contrast-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/contrast-plugin/src/app/app.component.ts b/plugins/apps/contrast-plugin/src/app/app.component.ts index 6c4e9263bf..5866d08f2d 100644 --- a/plugins/apps/contrast-plugin/src/app/app.component.ts +++ b/plugins/apps/contrast-plugin/src/app/app.component.ts @@ -115,7 +115,9 @@ export class AppComponent { #messages$ = fromEvent>(window, 'message'); #initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); selection = toSignal( diff --git a/plugins/apps/create-palette-plugin/package.json b/plugins/apps/create-palette-plugin/package.json index 016063ea11..2a8aa79e1d 100644 --- a/plugins/apps/create-palette-plugin/package.json +++ b/plugins/apps/create-palette-plugin/package.json @@ -12,5 +12,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/e2e/package.json b/plugins/apps/e2e/package.json index f432a1e113..072ee17a99 100644 --- a/plugins/apps/e2e/package.json +++ b/plugins/apps/e2e/package.json @@ -7,5 +7,5 @@ "test": "vitest", "lint": "eslint ." }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/example-styles/package.json b/plugins/apps/example-styles/package.json index a30835a099..f4a2cc3632 100644 --- a/plugins/apps/example-styles/package.json +++ b/plugins/apps/example-styles/package.json @@ -11,5 +11,5 @@ "serve": "vite preview", "lint": "eslint ." }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/icons-plugin/package.json b/plugins/apps/icons-plugin/package.json index 936de38fe3..ece117134c 100644 --- a/plugins/apps/icons-plugin/package.json +++ b/plugins/apps/icons-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/icons-plugin/src/app/app.component.ts b/plugins/apps/icons-plugin/src/app/app.component.ts index ee24b3e356..6ed0f1ce8f 100644 --- a/plugins/apps/icons-plugin/src/app/app.component.ts +++ b/plugins/apps/icons-plugin/src/app/app.component.ts @@ -44,7 +44,9 @@ export class AppComponent { ); public initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); public theme = toSignal( diff --git a/plugins/apps/lorem-ipsum-plugin/package.json b/plugins/apps/lorem-ipsum-plugin/package.json index 34eab294e3..bfd30bb446 100644 --- a/plugins/apps/lorem-ipsum-plugin/package.json +++ b/plugins/apps/lorem-ipsum-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/lorem-ipsum-plugin/src/app/app.component.ts b/plugins/apps/lorem-ipsum-plugin/src/app/app.component.ts index 1016cc4a4e..775ffc4986 100644 --- a/plugins/apps/lorem-ipsum-plugin/src/app/app.component.ts +++ b/plugins/apps/lorem-ipsum-plugin/src/app/app.component.ts @@ -69,7 +69,9 @@ export class AppComponent { messages$ = fromEvent>(window, 'message'); initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); theme = toSignal( diff --git a/plugins/apps/plugin-api-test-suite/package.json b/plugins/apps/plugin-api-test-suite/package.json index 03a09c7c00..bec6a176cf 100644 --- a/plugins/apps/plugin-api-test-suite/package.json +++ b/plugins/apps/plugin-api-test-suite/package.json @@ -20,5 +20,5 @@ "devDependencies": { "playwright": "^1.63.0" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/poc-state-plugin/package.json b/plugins/apps/poc-state-plugin/package.json index f2138cd8c0..5969a002e5 100644 --- a/plugins/apps/poc-state-plugin/package.json +++ b/plugins/apps/poc-state-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/poc-tokens-plugin/package.json b/plugins/apps/poc-tokens-plugin/package.json index a152f95df1..de2a45b045 100644 --- a/plugins/apps/poc-tokens-plugin/package.json +++ b/plugins/apps/poc-tokens-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "exit 0" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/poc-tokens-plugin/src/app/app.component.ts b/plugins/apps/poc-tokens-plugin/src/app/app.component.ts index 849e890e16..feef780661 100644 --- a/plugins/apps/poc-tokens-plugin/src/app/app.component.ts +++ b/plugins/apps/poc-tokens-plugin/src/app/app.component.ts @@ -44,7 +44,9 @@ export class AppComponent { ); public initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); public theme = toSignal( diff --git a/plugins/apps/rename-layers-plugin/package.json b/plugins/apps/rename-layers-plugin/package.json index a8a0ce18d3..9cc0224d95 100644 --- a/plugins/apps/rename-layers-plugin/package.json +++ b/plugins/apps/rename-layers-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/rename-layers-plugin/src/app/app.component.ts b/plugins/apps/rename-layers-plugin/src/app/app.component.ts index 70055739cc..8f39f284c9 100644 --- a/plugins/apps/rename-layers-plugin/src/app/app.component.ts +++ b/plugins/apps/rename-layers-plugin/src/app/app.component.ts @@ -38,7 +38,9 @@ export class AppComponent { } initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); theme = toSignal( diff --git a/plugins/apps/table-plugin/package.json b/plugins/apps/table-plugin/package.json index 3ae9e4bd23..c00abae34a 100644 --- a/plugins/apps/table-plugin/package.json +++ b/plugins/apps/table-plugin/package.json @@ -13,5 +13,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/apps/table-plugin/src/app/app.component.ts b/plugins/apps/table-plugin/src/app/app.component.ts index 4cce7de045..ad1e492806 100644 --- a/plugins/apps/table-plugin/src/app/app.component.ts +++ b/plugins/apps/table-plugin/src/app/app.component.ts @@ -40,7 +40,9 @@ export class AppComponent { messages$ = fromEvent>(window, 'message'); initialTheme$ = of( - new URLSearchParams(window.location.search).get('theme'), + // The theme arrives in the query string or in the query part of the hash. + new URLSearchParams(window.location.search).get('theme') ?? + new URLSearchParams(window.location.hash.split('?')[1]).get('theme'), ).pipe(filter((theme) => !!theme)); theme = toSignal( diff --git a/plugins/libs/plugin-types/package.json b/plugins/libs/plugin-types/package.json index cd738a4fdc..cce1fd85b5 100644 --- a/plugins/libs/plugin-types/package.json +++ b/plugins/libs/plugin-types/package.json @@ -7,5 +7,5 @@ "build": "node ../../tools/scripts/build-types.mjs", "lint": "tsc -p . --noEmit" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/libs/plugins-runtime/package.json b/plugins/libs/plugins-runtime/package.json index 337f6cd9af..05e1b061e1 100644 --- a/plugins/libs/plugins-runtime/package.json +++ b/plugins/libs/plugins-runtime/package.json @@ -16,5 +16,5 @@ "lint": "eslint .", "test": "vitest" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/libs/plugins-runtime/src/index.ts b/plugins/libs/plugins-runtime/src/index.ts index a38203471d..ecb69143b9 100644 --- a/plugins/libs/plugins-runtime/src/index.ts +++ b/plugins/libs/plugins-runtime/src/index.ts @@ -1,6 +1,7 @@ import 'ses'; import './lib/modal/plugin-modal'; +import { bootstrapSES } from './lib/bootstrap.js'; import { ɵloadPlugin, setContextBuilder, @@ -15,19 +16,27 @@ import type { Context } from '@penpot/plugin-types'; console.log('%c[PLUGINS] Loading plugin system', 'color: #008d7c'); -repairIntrinsics({ - evalTaming: 'unsafeEval', - stackFiltering: 'verbose', - errorTaming: 'unsafe', - consoleTaming: 'unsafe', - errorTrapping: 'none', - unhandledRejectionTrapping: 'none', -}); +const sesError = bootstrapSES(); + +if (sesError) { + console.error( + '[PLUGINS] SES bootstrap failed; plugins disabled in this environment.', + sesError, + ); +} + +/** Returns the SES bootstrap error, or null when the runtime can be used. */ +export const getPluginsRuntimeError = (): Error | null => sesError; // eslint-disable-next-line @typescript-eslint/no-explicit-any const globalThisAny$ = globalThis as any; export const initPluginsRuntime = (contextBuilder: (id: string) => Context) => { + if (sesError) { + console.error('[PLUGINS] Skipping plugin runtime initialization.'); + return; + } + try { console.log('%c[PLUGINS] Initialize runtime', 'color: #008d7c'); setContextBuilder(contextBuilder); diff --git a/plugins/libs/plugins-runtime/src/lib/bootstrap.spec.ts b/plugins/libs/plugins-runtime/src/lib/bootstrap.spec.ts new file mode 100644 index 0000000000..d77b3e0803 --- /dev/null +++ b/plugins/libs/plugins-runtime/src/lib/bootstrap.spec.ts @@ -0,0 +1,131 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { bootstrapSES } from './bootstrap.js'; + +// Intentional snapshot of the production options in bootstrap.ts: if they +// change, this test must fail. Do not replace this literal with an import. +const REPAIR_OPTIONS = { + evalTaming: 'unsafeEval', + stackFiltering: 'verbose', + errorTaming: 'unsafe', + consoleTaming: 'unsafe', + errorTrapping: 'none', + unhandledRejectionTrapping: 'none', +}; + +const temporalError = () => + new TypeError('unexpected Temporal __proto__: [object Object]'); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + vi.doUnmock('ses'); + vi.doUnmock('./modal/plugin-modal'); + vi.resetModules(); +}); + +describe('bootstrapSES', () => { + it('repairs intrinsics with the production options and reports success', () => { + const repair = vi.fn(); + vi.stubGlobal('repairIntrinsics', repair); + + expect(bootstrapSES()).toBeNull(); + expect(repair).toHaveBeenCalledWith(REPAIR_OPTIONS); + }); + + it('returns the SES error instead of throwing it', () => { + const error = temporalError(); + vi.stubGlobal('repairIntrinsics', () => { + throw error; + }); + + expect(bootstrapSES()).toBe(error); + }); +}); + +describe('plugins runtime entry point', () => { + it('installs the runtime globals when SES bootstrap succeeds', async () => { + const globalAny = globalThis as unknown as Record; + const context = { theme: 'dark' }; + const runtimeKeys = [ + 'ɵcontext', + 'ɵloadPlugin', + 'ɵloadPluginByUrl', + 'ɵunloadPlugin', + ]; + const previous = runtimeKeys.map((key) => globalAny[key]); + + vi.stubGlobal('repairIntrinsics', vi.fn()); + vi.doMock('ses', () => ({})); + // The custom element is registered at module load; mock it so repeated + // entry-point imports do not redefine it in the same jsdom registry. + vi.doMock('./modal/plugin-modal', () => ({})); + vi.resetModules(); + + try { + const runtime = await import('../index.js'); + runtime.initPluginsRuntime(() => context as never); + + expect(runtime.getPluginsRuntimeError()).toBeNull(); + expect(globalAny['ɵcontext']).toBe(context); + expect(typeof globalAny['ɵloadPlugin']).toBe('function'); + expect(typeof globalAny['ɵloadPluginByUrl']).toBe('function'); + expect(typeof globalAny['ɵunloadPlugin']).toBe('function'); + } finally { + for (const [index, key] of runtimeKeys.entries()) { + if (previous[index] === undefined) { + delete globalAny[key]; + } else { + globalAny[key] = previous[index]; + } + } + } + }); + + it('loads with a broken SES bootstrap without installing runtime globals', async () => { + const error = temporalError(); + const globalAny = globalThis as unknown as Record; + const runtimeKeys = [ + 'ɵcontext', + 'ɵloadPlugin', + 'ɵloadPluginByUrl', + 'ɵunloadPlugin', + ]; + const previous = runtimeKeys.map((key) => globalAny[key]); + + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.stubGlobal('repairIntrinsics', () => { + throw error; + }); + // Keep the stub in place: `import 'ses'` in index.ts would otherwise + // install a new repairIntrinsics global before bootstrap runs. + vi.doMock('ses', () => ({})); + // The custom element is registered at module load; mock it so repeated + // entry-point imports do not redefine it in the same jsdom registry. + vi.doMock('./modal/plugin-modal', () => ({})); + vi.resetModules(); + + try { + for (const key of runtimeKeys) { + delete globalAny[key]; + } + + const runtime = await import('../index.js'); + + expect(runtime.getPluginsRuntimeError()).toBe(error); + expect(() => + runtime.initPluginsRuntime(() => ({}) as never), + ).not.toThrow(); + for (const key of runtimeKeys) { + expect(globalAny[key]).toBeUndefined(); + } + } finally { + for (const [index, key] of runtimeKeys.entries()) { + if (previous[index] === undefined) { + delete globalAny[key]; + } else { + globalAny[key] = previous[index]; + } + } + } + }); +}); diff --git a/plugins/libs/plugins-runtime/src/lib/bootstrap.ts b/plugins/libs/plugins-runtime/src/lib/bootstrap.ts new file mode 100644 index 0000000000..0057f0f413 --- /dev/null +++ b/plugins/libs/plugins-runtime/src/lib/bootstrap.ts @@ -0,0 +1,27 @@ +/** + * Repairs the JavaScript intrinsics needed by the SES sandbox. + * + * SES can throw during this step when the host page already contains a + * `Temporal` global it does not recognize (for example, one wrapped by a + * browser extension). The exception happens at module load time in + * `index.ts`, so an uncaught failure here takes down the whole app. + * + * Returns the caught error so callers can report it and disable plugins, + * instead of propagating it to the bundle loader. + */ +export function bootstrapSES(): Error | null { + try { + repairIntrinsics({ + evalTaming: 'unsafeEval', + stackFiltering: 'verbose', + errorTaming: 'unsafe', + consoleTaming: 'unsafe', + errorTrapping: 'none', + unhandledRejectionTrapping: 'none', + }); + + return null; + } catch (err) { + return err instanceof Error ? err : new Error(String(err)); + } +} diff --git a/plugins/libs/plugins-styles/package.json b/plugins/libs/plugins-styles/package.json index c70d423063..8c3192f869 100644 --- a/plugins/libs/plugins-styles/package.json +++ b/plugins/libs/plugins-styles/package.json @@ -6,5 +6,5 @@ "build": "node ../../tools/scripts/build-css.mjs", "lint": "echo 0" }, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3" + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0" } diff --git a/plugins/package.json b/plugins/package.json index 4b04171c49..f44b073df0 100644 --- a/plugins/package.json +++ b/plugins/package.json @@ -3,7 +3,7 @@ "version": "0.6.0", "type": "module", "license": "MIT", - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "scripts": { "start": "pnpm run start:app:runtime", "start:app:runtime": "concurrently --kill-others --names build,server \"pnpm --filter @penpot/plugins-runtime run build:watch\" \"pnpm --filter @penpot/plugins-runtime run preview\"", diff --git a/plugins/pnpm-lock.yaml b/plugins/pnpm-lock.yaml index 94c234158b..98926f1f94 100644 --- a/plugins/pnpm-lock.yaml +++ b/plugins/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bbe85fa79b..6d453d4eeb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/render-wasm/_build_env b/render-wasm/_build_env index 5c804469d0..83759da674 100644 --- a/render-wasm/_build_env +++ b/render-wasm/_build_env @@ -67,7 +67,7 @@ export EMCC_CFLAGS="--no-entry \ -sERROR_ON_UNDEFINED_SYMBOLS=0 \ -sMAX_WEBGL_VERSION=2 \ -sEXPORT_NAME=createRustSkiaModule \ - -sEXPORTED_RUNTIME_METHODS=GL,UTF8ToString,stringToUTF8,HEAPU8,HEAP32,HEAPU32,HEAPF32 \ + -sEXPORTED_RUNTIME_METHODS=GL,UTF8ToString,stringToUTF8,HEAPU8,HEAP32,HEAPU32,HEAPF32,stackSave,stackRestore \ -sENVIRONMENT=web,worker,node \ -sMODULARIZE=1 \ -sFILESYSTEM=0 \ diff --git a/render-wasm/package.json b/render-wasm/package.json index a7a15fb5f0..050ecf2bff 100644 --- a/render-wasm/package.json +++ b/render-wasm/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en España SL", "private": true, - "packageManager": "pnpm@12.18.0+sha512.9d7882d5dc82147ecfc4e062aa34834e0ad0296710a1837a1115ae012127d36b7109e321ac1598a1dd0a65597681b1e99917e12ed0d749eae91187943363beb3", + "packageManager": "pnpm@12.21.0+sha512.66fd8278ee28935ce951792d34367ff3af5c432a2756445823029533835b8da529bbe0be5130afd56ed0713fb8789e47bad82d93bd5ae2fe54b15562d6276ef0", "repository": { "type": "git", "url": "https://github.com/penpot/penpot" diff --git a/render-wasm/pnpm-lock.yaml b/render-wasm/pnpm-lock.yaml index 452e972384..5508a208e9 100644 --- a/render-wasm/pnpm-lock.yaml +++ b/render-wasm/pnpm-lock.yaml @@ -7,153 +7,162 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.18.0 - version: 12.18.0 + specifier: 12.21.0 + version: 12.21.0 packages: - '@pnpm/exe.android-arm64@12.18.0': - resolution: {integrity: sha512-fOF6IwgvnUKEmVedGqiAsNPfN2Ry8GvaqzSQOiF3RJFbpBfPKO/6YT54s1GjToyYinRxw9preHHPe4y2N3SzhA==} + '@pnpm/exe.android-arm64@12.21.0': + resolution: {integrity: sha512-iaWidxR5B9NUBpwMznCQu1yUpKB6BjTY5BYX2lORj76VsfINdj8Z8XWSzSNrGI8qlxa28Jzdp5fiaspzOaJZlA==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.18.0': - resolution: {integrity: sha512-6OiolnaCsB3/GcJ9XNrenbUH3zFHRJ+5QynrMUUEyThbgnnCzWxa199hpQlOmxN4rAXuYQ0IYgWFNUZxyzg31Q==} + '@pnpm/exe.android-x64@12.21.0': + resolution: {integrity: sha512-35e5v3wlwjIYxDywq+R6/z8CE4gLOEgRxt18yqz1tMRLmx2mnW+CFJI2Q8+srTNjn7gykXUC0x1k195JNikC5A==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.18.0': - resolution: {integrity: sha512-zkq+4YzVIX39ETYLyC3W7LY7MpHEU/J3nRhHl4i2yPld2Cf2+rEKRh3ejJ4AmaQ77HHJLCktY4Z3jq0fpcxFXg==} + '@pnpm/exe.darwin-arm64@12.21.0': + resolution: {integrity: sha512-F5OyMX/6zLSowpTZEQRojbCmPS1Pq+fyRaIinslsIe6Glh3qbyQ52u+Fe0eEzSaRuIbWZkXmAAXSWh6Q5Wrlgg==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.18.0': - resolution: {integrity: sha512-qMsL9qx92unOTa6hgkfiTJv8ceZaTx0IQg7InhYDVX2ZrTECujaVEEVypnJK5m3eGUsP1KD7Y4qIgleCwfRsYw==} + '@pnpm/exe.darwin-x64@12.21.0': + resolution: {integrity: sha512-EzpvcsaE6hrrgYeEiAFYd2rSgFybYyxXhkAY7K3y6Hg3loVfcyJ81hHczpFzi8lW5RXtUlFECRPZ8ZAMRFQYYg==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.18.0': - resolution: {integrity: sha512-/9vy/4ms52cU4MjZ5NhHd6N2d9L9W0kARf5/ff2tweAHu0VeUzs/0qp/7ZHFuddm2Wa/m0AKQoPum9AVUCqKsw==} + '@pnpm/exe.freebsd-x64@12.21.0': + resolution: {integrity: sha512-wNjuEnmdLks4aayPu/QZWQ5tQM9Jn+odUp/vwQM8BXRYoulkTZhtZTHoD60N6mUdLWef1hgUeUh8llNGEUsimw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.18.0': - resolution: {integrity: sha512-Vp+TIMPPyackVCXkOdEKH/RmAn/r07EcIyrgZYQK7VrkksqZOie0GnWIx+85mPqeFEYCRtXIZAFJ9L4ceTndDg==} + '@pnpm/exe.linux-arm64-musl@12.21.0': + resolution: {integrity: sha512-Ceit9zn4g5ZqCJtx1fV8eJE7p8aeXMrGZ+Dc01vRqZLIUNgNeI+Ur0ruyqxtsnJx6rGADyXzohumxPLHjaJ5AQ==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.18.0': - resolution: {integrity: sha512-2dpyVBCHT+FfpuQh9i+7MXAqNWYLCvD5vYr5p6Eimxoga8USLnjbUgyj72AOrO0hjc5ri3TEFP3jiyBHjkSZpQ==} + '@pnpm/exe.linux-arm64@12.21.0': + resolution: {integrity: sha512-4Oj/iF2TDCe2lcH0n4TupPR5sSdrNLclJB6xBElLMPCUOAvBhdf2rRlSaO5M4erJ6g4Fyy7YTXTpcmfFsKyxEA==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.18.0': - resolution: {integrity: sha512-BBOOOfNJ2QwIIya3eKqOjiN4WNTszfRz7HCvw9XggcT7+ftMjNP8XJ/GfW+ltCBeUFR9d8WBVsr2m/NLjX2CZQ==} + '@pnpm/exe.linux-ppc64@12.21.0': + resolution: {integrity: sha512-s2o7rdSDeUTBFsNqxptpZXlTgT6cojmMVUOvTQvFNSv6Easian02ha4tR32fwwD97Au4YzIIjMCwywCF4D82Sw==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.18.0': - resolution: {integrity: sha512-7kMN77xSOYJX8GPSH5RbNAc5oM61LMmcSUukDDUwllnZPXm8GRPskCNsTWy8SCw01Kr1FB+5SwGNMiTdiziLNQ==} + '@pnpm/exe.linux-riscv64@12.21.0': + resolution: {integrity: sha512-EYv3GzxeBTITCuM3yGHHbpRFpUvyulkXpQ6LsjKAQNL3iaba1E6h9iDmlouLb3Io4AMMsfdk0/uG6poQ41uewA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.18.0': - resolution: {integrity: sha512-htuSCpEcmDzhKoJqXbPkvfr176dBHU0QIF5mWcRiqxRXAB/1JoYiMYotfjWpk+ztgpB9qlx4XnWlLFAf7gzc6g==} + '@pnpm/exe.linux-s390x@12.21.0': + resolution: {integrity: sha512-8VXGNGVNEfs7pL9GTZOzFxjFEnMt4WL3rLOADR5/kPQ7ROTV9GfpVI2DPO/zArXFSkvfc9Zad/WypHLbvzoaTQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.18.0': - resolution: {integrity: sha512-Uj+vzfIQ62kDP10x36yHZIXVs5ohdNtiPYwn5FhCLlNScHPo0OAI+ShGDtLofiBXKE7umu71Jmjm7N8a8zVcDw==} + '@pnpm/exe.linux-x64-musl@12.21.0': + resolution: {integrity: sha512-MnC5vpIXYubhDZCgCwlfJMg7Cg6HupN2IuouT9gD/tUmOfVdQYqeyYccWbthyo/Jeto8de1oQlxeRN1+1Hl2DQ==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.18.0': - resolution: {integrity: sha512-OJ+X2wcraldCwj75pCIerWARQSRpdfj0eBTs/2ODyhfeW46ylOnjTIE/GKra6Mn6E72/gBmXS8igrUioa+F9fQ==} + '@pnpm/exe.linux-x64@12.21.0': + resolution: {integrity: sha512-8eufCjHXspKbNS/kh8q4Q0gqJoOuvC4Q5pgos6XS8By/nAMtP+jf8F1Cw9er840kfAX+PJ/ts+A9qnLE91rvXQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.18.0': - resolution: {integrity: sha512-wEY6jiACGcbhhi5yD59doHqNTndS7AaujaPmtWAu1DKkHru2hy5fJ9yUGecW72Ypaw2/I3BEprfMOopap2NQtw==} + '@pnpm/exe.netbsd-x64@12.21.0': + resolution: {integrity: sha512-eyh1M8hQZCkLWxuva7S/fc1SbFbgqg0/5di0dgWmyiuc2Io3dbWVihDun8X+XM8zsR5uhIDvvZpbS9aZQbCovg==} + cpu: [x64] + os: [netbsd] + + '@pnpm/exe.win32-arm64@12.21.0': + resolution: {integrity: sha512-7xTxxUXo/grRLivSeIl6B21gnsz7ooe5EMPcJmQqwxc8WLJVReN8MzrlHru8bqXfVr4X3+6ZGZscntKenlCBqQ==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.18.0': - resolution: {integrity: sha512-5FJkghO7RLRkQpWxeVEygW9XsmnNuxzPfXbgwFWCHsFyFhpDn/vOdL88ToQEM0Kt5igdW5s6U0/IDH8Dvkt3pg==} + '@pnpm/exe.win32-x64@12.21.0': + resolution: {integrity: sha512-9TpkJemVApxJ1Z1w0ZFGN+AuSx4nJOqXSJUqMqDvaXTYwGOzvBiiWdFP9GN0ketiWCQS1Q0/pza2E7I3pP0OBQ==} cpu: [x64] os: [win32] - pnpm@12.18.0: - resolution: {integrity: sha512-nXiC1dyCFH7PxOBiqjSDTgrQKWcQoYN6ERWuASEn02txCeMhrBWYod0KZVl2gbHpmRfhLtDXSerpEYeUM2O+sw==} + pnpm@12.21.0: + resolution: {integrity: sha512-Zv2CeO4ok1zpUXktNDZ/869cQyonVkRYIwKVM4NbjaUpu+C+UTCv1W7QcT+4eJ5Hutgtk71a4v5UsVVi1idu8A==} engines: {node: '>=18'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.18.0': + '@pnpm/exe.android-arm64@12.21.0': optional: true - '@pnpm/exe.android-x64@12.18.0': + '@pnpm/exe.android-x64@12.21.0': optional: true - '@pnpm/exe.darwin-arm64@12.18.0': + '@pnpm/exe.darwin-arm64@12.21.0': optional: true - '@pnpm/exe.darwin-x64@12.18.0': + '@pnpm/exe.darwin-x64@12.21.0': optional: true - '@pnpm/exe.freebsd-x64@12.18.0': + '@pnpm/exe.freebsd-x64@12.21.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.18.0': + '@pnpm/exe.linux-arm64-musl@12.21.0': optional: true - '@pnpm/exe.linux-arm64@12.18.0': + '@pnpm/exe.linux-arm64@12.21.0': optional: true - '@pnpm/exe.linux-ppc64@12.18.0': + '@pnpm/exe.linux-ppc64@12.21.0': optional: true - '@pnpm/exe.linux-riscv64@12.18.0': + '@pnpm/exe.linux-riscv64@12.21.0': optional: true - '@pnpm/exe.linux-s390x@12.18.0': + '@pnpm/exe.linux-s390x@12.21.0': optional: true - '@pnpm/exe.linux-x64-musl@12.18.0': + '@pnpm/exe.linux-x64-musl@12.21.0': optional: true - '@pnpm/exe.linux-x64@12.18.0': + '@pnpm/exe.linux-x64@12.21.0': optional: true - '@pnpm/exe.win32-arm64@12.18.0': + '@pnpm/exe.netbsd-x64@12.21.0': optional: true - '@pnpm/exe.win32-x64@12.18.0': + '@pnpm/exe.win32-arm64@12.21.0': optional: true - pnpm@12.18.0: + '@pnpm/exe.win32-x64@12.21.0': + optional: true + + pnpm@12.21.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.18.0 - '@pnpm/exe.android-x64': 12.18.0 - '@pnpm/exe.darwin-arm64': 12.18.0 - '@pnpm/exe.darwin-x64': 12.18.0 - '@pnpm/exe.freebsd-x64': 12.18.0 - '@pnpm/exe.linux-arm64': 12.18.0 - '@pnpm/exe.linux-arm64-musl': 12.18.0 - '@pnpm/exe.linux-ppc64': 12.18.0 - '@pnpm/exe.linux-riscv64': 12.18.0 - '@pnpm/exe.linux-s390x': 12.18.0 - '@pnpm/exe.linux-x64': 12.18.0 - '@pnpm/exe.linux-x64-musl': 12.18.0 - '@pnpm/exe.win32-arm64': 12.18.0 - '@pnpm/exe.win32-x64': 12.18.0 + '@pnpm/exe.android-arm64': 12.21.0 + '@pnpm/exe.android-x64': 12.21.0 + '@pnpm/exe.darwin-arm64': 12.21.0 + '@pnpm/exe.darwin-x64': 12.21.0 + '@pnpm/exe.freebsd-x64': 12.21.0 + '@pnpm/exe.linux-arm64': 12.21.0 + '@pnpm/exe.linux-arm64-musl': 12.21.0 + '@pnpm/exe.linux-ppc64': 12.21.0 + '@pnpm/exe.linux-riscv64': 12.21.0 + '@pnpm/exe.linux-s390x': 12.21.0 + '@pnpm/exe.linux-x64': 12.21.0 + '@pnpm/exe.linux-x64-musl': 12.21.0 + '@pnpm/exe.netbsd-x64': 12.21.0 + '@pnpm/exe.win32-arm64': 12.21.0 + '@pnpm/exe.win32-x64': 12.21.0 --- lockfileVersion: '9.0' diff --git a/render-wasm/scripts/cargo-build.cjs b/render-wasm/scripts/cargo-build.cjs index 035703a03c..248c9ae2cf 100644 --- a/render-wasm/scripts/cargo-build.cjs +++ b/render-wasm/scripts/cargo-build.cjs @@ -15,6 +15,38 @@ async function main() { // A failed invocation must not leave facts from an earlier build. fs.rmSync(infoFile, { force: true }); + // This helper inherits the same shell setup as Cargo. Keep only build inputs, + // excluding credentials, invocation identities and artifact destinations. + const environmentKeys = new Set([ + "RUSTFLAGS", + "RUSTC_WRAPPER", + "CARGO_TARGET_DIR", + "CARGO_HOME", + "CC", + "CFLAGS", + "CXXFLAGS", + "LDFLAGS", + "PENPOT_WASM_FUNCTION_NAMES", + "CARGO_ENCODED_RUSTFLAGS", + "CARGO_BUILD_TARGET", + "BUILD_MODE", + "EMCC_CFLAGS", + "EM_INITIAL_HEAP", + "EM_MEMORY_GROWTH_GEOMETRIC_STEP", + "EM_MALLOC", + "EM_CACHE", + "NODE_ENV", + "CARGO_NET_OFFLINE", + ]); + const environment = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => + environmentKeys.has(key) || + key.startsWith("CARGO_PROFILE_") || + key.startsWith("EMSDK"), + ), + ); + const artifacts = []; const buildScripts = new Map(); let succeeded = false; @@ -99,6 +131,7 @@ async function main() { package_id: artifact.package_id, features: artifact.features, profile: artifact.profile, + environment, out_dir: outDir, artifact_dir: path.dirname(path.resolve(jsFiles[0])), shared_file: sharedFile, diff --git a/render-wasm/scripts/cargo-build.test.cjs b/render-wasm/scripts/cargo-build.test.cjs index 01a080aa42..f36acb088e 100644 --- a/render-wasm/scripts/cargo-build.test.cjs +++ b/render-wasm/scripts/cargo-build.test.cjs @@ -97,11 +97,12 @@ function cargoOutput(records) { return records.map((record) => JSON.stringify(record)).join("\n") + "\n"; } -function run(f, records) { +function run(f, records, env = {}) { return spawnSync(process.execPath, [helper, f.info, f.manifest], { cwd: f.root, input: cargoOutput(records), encoding: "utf8", + env, }); } @@ -126,6 +127,7 @@ for (const fresh of [false, true]) { package_id: f.artifact.package_id, features: f.artifact.features, profile: f.artifact.profile, + environment: {}, out_dir: f.outDir, artifact_dir: f.artifactDir, shared_file: path.join(f.outDir, "render_wasm_shared.js"), @@ -137,6 +139,127 @@ for (const fresh of [false, true]) { }); } +test("records only allowlisted values from the helper process environment", (t) => { + const f = fixture(t); + const environment = { + RUSTFLAGS: "-C target-feature=+simd128", + RUSTC_WRAPPER: "", + CARGO_TARGET_DIR: "/tmp/target with spaces", + CARGO_HOME: "/opt/cargo", + CC: "emcc", + CFLAGS: "-O3\n-g0", + CXXFLAGS: "-std=c++17", + LDFLAGS: "-sALLOW_MEMORY_GROWTH=1", + PENPOT_WASM_FUNCTION_NAMES: "1", + CARGO_ENCODED_RUSTFLAGS: "-C\u001fopt-level=3", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + BUILD_MODE: "release", + EMCC_CFLAGS: "-O3 -sASSERTIONS=0", + CARGO_PROFILE_RELEASE_LTO: "true", + CARGO_PROFILE_CUSTOM_OPT_LEVEL: "z", + EMSDK: "/opt/emsdk", + EMSDK_NODE: "/opt/emsdk/node/bin/node", + EM_INITIAL_HEAP: "268435456", + EM_MEMORY_GROWTH_GEOMETRIC_STEP: "0.8", + EM_MALLOC: "dlmalloc", + EM_CACHE: "/tmp/emsdk_cache", + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + }; + const result = run(f, [f.artifact, f.buildScript, finished], { + ...environment, + SECRET_TOKEN: "must not be saved", + PATH: "/private/path", + HOME: "/private/home", + CARGO_REGISTRIES_PRIVATE_TOKEN: "must not be saved", + CARGO_PROFILE: "not the prefix", + RUSTFLAGS_EXTRA: "not an exact key", + EM_UNRELATED: "not allowlisted", + VERSION_TAG: "build-identity", + RENDER_BENCH_BUILD_ID: "build-identity", + FRONTEND_DEST: "/per-build/renderer", + EXPORT_DEST: "/per-build/export", + FRONTEND_SHARED_DEST: "/per-build/shared.js", + EXPORT_SHARED_DEST: "/per-build/export-shared.js", + RENDER_WASM_SHARED_ARTIFACT: "/per-build/shared.js", + RENDER_WASM_BUILD_INFO: f.info, + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(f.info, "utf8")).environment, + environment, + ); +}); + +test("preserves unset versus empty environment values", (t) => { + const f = fixture(t); + for (const environment of [{}, { CFLAGS: "" }, { CFLAGS: "-O3" }]) { + const result = run(f, [f.artifact, f.buildScript, finished], environment); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(f.info, "utf8")).environment, + environment, + ); + } +}); + +test("captures exported setup values after sourcing the actual build environment", (t) => { + const f = fixture(t); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + 'source "$1" frontend --offline; export EMSDK_NODE="/setup/node"; CFLAGS="not exported"; shift; exec "$@"', + "cargo-build-test", + path.resolve(__dirname, "..", "_build_env"), + process.execPath, + helper, + f.info, + f.manifest, + ], + { + cwd: f.root, + input: cargoOutput([f.artifact, f.buildScript, finished]), + encoding: "utf8", + env: { + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + RUSTFLAGS: "-C debuginfo=0", + RUSTC_WRAPPER: "", + EMCC_CFLAGS: "must be replaced by setup", + EM_INITIAL_HEAP: "1", + SECRET_TOKEN: "must not be saved", + VERSION_TAG: "per-build-identity", + FRONTEND_DEST: "/per-build/renderer", + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + const environment = JSON.parse(fs.readFileSync(f.info, "utf8")).environment; + assert.deepEqual(environment, { + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + RUSTFLAGS: "-C debuginfo=0 -C target-feature=+simd128", + RUSTC_WRAPPER: "", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + CARGO_TARGET_DIR: "target/frontend", + BUILD_MODE: "release", + CARGO_PROFILE_DIR: "release", + EMSDK_NODE: "/setup/node", + EM_INITIAL_HEAP: "268435456", + EM_MEMORY_GROWTH_GEOMETRIC_STEP: "0.8", + EM_MALLOC: "dlmalloc", + EM_CACHE: "/tmp/emsdk_cache", + EMCC_CFLAGS: environment.EMCC_CFLAGS, + }); + assert.match(environment.EMCC_CFLAGS, /^-O3 -sASSERTIONS=0 /); + assert.match(environment.EMCC_CFLAGS, /-sINITIAL_HEAP=268435456/); + assert.match(environment.EMCC_CFLAGS, /-sMEMORY_GROWTH_GEOMETRIC_STEP=0\.8/); + assert.match(environment.EMCC_CFLAGS, /-sMALLOC=dlmalloc/); +}); + test("records an empty actual feature set", (t) => { const f = fixture(t); const result = run(f, [ diff --git a/render-wasm/src/main.rs b/render-wasm/src/main.rs index fe38624844..239594020e 100644 --- a/render-wasm/src/main.rs +++ b/render-wasm/src/main.rs @@ -588,6 +588,15 @@ pub extern "C" fn set_shape_hidden(hidden: bool) -> Result<()> { Ok(()) } +#[no_mangle] +#[wasm_error] +pub extern "C" fn set_shape_hide_fill_on_export(hide: bool) -> Result<()> { + with_current_shape_mut!(state, |shape: &mut Shape| { + shape.set_hide_fill_on_export(hide); + }); + Ok(()) +} + #[no_mangle] #[wasm_error] pub extern "C" fn set_shape_corners(r1: f32, r2: f32, r3: f32, r4: f32) -> Result<()> { @@ -775,7 +784,8 @@ pub extern "C" fn get_shape_extrect(a: u32, b: u32, c: u32, d: u32) -> Result<*m /// Raster image via the GPU surface. Returns `[len][width][height][bytes]` /// (LE). `format` selects the encoder: 0 = PNG, 1 = JPEG, 2 = WEBP (see -/// `RasterFormat`). +/// `RasterFormat`). `export` drops the root's fills when it hides them on +/// export; thumbnails keep them. #[no_mangle] #[wasm_error] pub extern "C" fn render_shape_pixels( @@ -785,6 +795,7 @@ pub extern "C" fn render_shape_pixels( d: u32, scale: f32, format: u32, + export: bool, ) -> Result<*mut u8> { let id = uuid_from_u32_quartet(a, b, c, d); @@ -796,7 +807,7 @@ pub extern "C" fn render_shape_pixels( with_state!(state, { let (data, width, height) = - state.render_shape_pixels(&id, scale, performance::get_time(), format)?; + state.render_shape_pixels(&id, scale, performance::get_time(), format, export)?; let len = data.len() as u32; let mut buf = Vec::with_capacity(4 + data.len()); diff --git a/render-wasm/src/render.rs b/render-wasm/src/render.rs index 9072be3f08..14da5d4de2 100644 --- a/render-wasm/src/render.rs +++ b/render-wasm/src/render.rs @@ -2634,10 +2634,8 @@ impl RenderState { let frame_type = self.render_shape_tree_partial(base_object, tree, timestamp, allow_stop)?; - // `draw_atlas` needs a snapshot of the tile atlas. Partial frames are not - // presented (only flushed), so defer composition until the viewport is - // ready and avoid re-snapshotting up to 4096² on every rAF during async - // tile work. + // Partial frames are not presented (only flushed), so defer + // composition until the viewport is ready. let should_compose = !self.options.is_interactive_transform() && matches!(frame_type, FrameType::Full | FrameType::ViewportReady) && !self.viewport_presented; diff --git a/render-wasm/src/render/pdf.rs b/render-wasm/src/render/pdf.rs index 746b26242d..29ddd4f863 100644 --- a/render-wasm/src/render/pdf.rs +++ b/render-wasm/src/render/pdf.rs @@ -3,7 +3,7 @@ use std::mem::ManuallyDrop; use skia_safe as skia; use crate::error::{Error, Result}; -use crate::state::ShapesPoolRef; +use crate::state::{ShapesPoolMutRef, ShapesPoolRef}; use crate::uuid::Uuid; use super::vector; @@ -143,7 +143,7 @@ pub fn export_abort(slot: &mut Option) { pub fn render_to_pdf( shared: &mut RenderResources, id: &Uuid, - tree: ShapesPoolRef, + tree: ShapesPoolMutRef, ) -> Result> { render_to_pdf_multi(shared, &[*id], tree) } @@ -154,7 +154,7 @@ pub fn render_to_pdf( pub fn render_to_pdf_multi( shared: &mut RenderResources, ids: &[Uuid], - tree: ShapesPoolRef, + tree: ShapesPoolMutRef, ) -> Result> { if ids.is_empty() { return Err(Error::CriticalError( @@ -164,7 +164,9 @@ pub fn render_to_pdf_multi( let mut session = None; export_begin(&mut session)?; for id in ids { - if let Err(err) = export_add_page(&mut session, shared, id, tree) { + let page = + tree.with_export_root(id, |tree| export_add_page(&mut session, shared, id, tree)); + if let Err(err) = page { export_abort(&mut session); return Err(err); } @@ -234,7 +236,7 @@ mod tests { let mut resources = RenderResources::try_new_headless().expect("headless resources"); let _guard = crate::globals::TestRenderResourcesGuard::install(&mut resources); - let pdf = render_to_pdf_multi(&mut resources, &[a, b], &pool).expect("pdf"); + let pdf = render_to_pdf_multi(&mut resources, &[a, b], &mut pool).expect("pdf"); assert!(pdf.starts_with(b"%PDF"), "missing PDF header"); assert_eq!(page_count(&pdf), 2, "expected two pages"); } @@ -270,11 +272,11 @@ mod tests { #[test] fn empty_id_list_is_an_error() { - let pool = ShapesPool::new(); + let mut pool = ShapesPool::new(); let mut resources = RenderResources::try_new_headless().expect("headless resources"); let _guard = crate::globals::TestRenderResourcesGuard::install(&mut resources); - let err = render_to_pdf_multi(&mut resources, &[], &pool).expect_err("empty"); + let err = render_to_pdf_multi(&mut resources, &[], &mut pool).expect_err("empty"); assert!( matches!(err, Error::CriticalError(_)), "empty ids must fail critically: {err:?}" @@ -295,7 +297,7 @@ mod tests { let mut resources = RenderResources::try_new_headless().expect("headless resources"); let _guard = crate::globals::TestRenderResourcesGuard::install(&mut resources); - let pdf = render_to_pdf(&mut resources, &id, &pool).expect("pdf"); + let pdf = render_to_pdf(&mut resources, &id, &mut pool).expect("pdf"); assert!(pdf.starts_with(b"%PDF")); assert_eq!(page_count(&pdf), 1); } diff --git a/render-wasm/src/render/pdf/to_unicode.rs b/render-wasm/src/render/pdf/to_unicode.rs index ffb36f233e..412e6aea85 100644 --- a/render-wasm/src/render/pdf/to_unicode.rs +++ b/render-wasm/src/render/pdf/to_unicode.rs @@ -781,7 +781,7 @@ mod tests { .add(family, LIGATURE_FONT, false, false) .expect("ligature font"); let _guard = crate::globals::TestRenderResourcesGuard::install(&mut resources); - let pdf = super::super::render_to_pdf(&mut resources, &id, &pool).expect("pdf"); + let pdf = super::super::render_to_pdf(&mut resources, &id, &mut pool).expect("pdf"); let trailer = parse_xref(&pdf).expect("xref").trailer; assert!( diff --git a/render-wasm/src/render/surfaces.rs b/render-wasm/src/render/surfaces.rs index 47a89c8521..92d6da5948 100644 --- a/render-wasm/src/render/surfaces.rs +++ b/render-wasm/src/render/surfaces.rs @@ -504,7 +504,6 @@ pub struct Surfaces { backbuffer: skia::Surface, // Atlas used to keep tiles. tile_atlas: skia::Surface, - tile_atlas_image: Option, tiles: TileTextureCache, pub atlas: DocAtlas, @@ -570,7 +569,6 @@ impl Surfaces { export, backbuffer, tile_atlas, - tile_atlas_image: None, tiles, atlas, sampling_options, @@ -596,7 +594,6 @@ impl Surfaces { self.tiles .repack(self.tile_atlas.width(), self.tile_atlas.height(), slot); self.tile_atlas.canvas().clear(skia::Color::TRANSPARENT); - self.tile_atlas_image = None; } pub fn set_dpr(&mut self, dpr: f32) { @@ -622,18 +619,11 @@ impl Surfaces { background: skia::Color, ) { self.tiles.update(viewbox, tile_viewbox); - if self.tiles.needs_snapshot() || self.tile_atlas_image.is_none() { - self.tile_atlas_image = Some(self.tile_atlas.image_snapshot()); - self.tiles.snapshot(); - } - let Some(atlas_image) = self.tile_atlas_image.as_ref() else { - return; - }; let sampling = self.tile_atlas_sampling(); let canvas = self.backbuffer.canvas(); canvas.clear(background); canvas.draw_atlas( - atlas_image, + &self.tile_atlas.image_snapshot(), &self.tiles.transforms, &self.tiles.textures, None, @@ -706,13 +696,6 @@ impl Surfaces { return; } - if self.tiles.needs_snapshot() || self.tile_atlas_image.is_none() { - self.tile_atlas_image = Some(self.tile_atlas.image_snapshot()); - self.tiles.snapshot(); - } - let Some(atlas_image) = self.tile_atlas_image.as_ref() else { - return; - }; let sampling = self.tile_atlas_sampling(); let canvas = self.backbuffer.canvas(); @@ -720,7 +703,7 @@ impl Surfaces { canvas.reset_matrix(); canvas.draw_atlas( - atlas_image, + &self.tile_atlas.image_snapshot(), &batch.transforms, &batch.textures, None, @@ -1516,27 +1499,30 @@ impl Surfaces { // paint.set_color(color); // backbuffer_canvas.draw_rect(tile_rect, &paint); + let dst = skia::Rect::from_xywh( + tile_rect.left, + tile_rect.top, + src_rect_f.width(), + src_rect_f.height(), + ); + // Draw current surface directly to target (no snapshot) - self.current.draw( + draw_surface_src_rect_to_dst( + &mut self.current, backbuffer_canvas, - ( - tile_rect.left - src_rect_f.left, - tile_rect.top - src_rect_f.top, - ), + src_rect_f, + dst, sampling_options, - None, ); // Also draw to cache for render_from_cache if draw_on_cache == DrawOnCache::Yes { - self.current.draw( + draw_surface_src_rect_to_dst( + &mut self.current, self.cache.canvas(), - ( - tile_rect.left - src_rect_f.left, - tile_rect.top - src_rect_f.top, - ), + src_rect_f, + dst, sampling_options, - None, ); } } @@ -1557,7 +1543,6 @@ impl Surfaces { pub fn invalidate_tile_cache(&mut self) { self.tiles.clear(); self.atlas.tile_doc_rects.clear(); - self.tile_atlas_image = None; } pub fn gc(&mut self) { @@ -1691,7 +1676,6 @@ impl TileAtlasTextureProvider { pub struct TileTextureCache { tile_size: f32, slot_size: i32, - is_updated: bool, provider: TileAtlasTextureProvider, transforms: Vec, textures: Vec, @@ -1716,7 +1700,6 @@ impl TileTextureCache { Self { tile_size: tiles::TILE_SIZE, slot_size: TILE_SIZE, - is_updated: false, provider: TileAtlasTextureProvider::new(texture_width, texture_height, TILE_SIZE), transforms: Vec::with_capacity(capacity), textures: Vec::with_capacity(capacity), @@ -1736,7 +1719,6 @@ impl TileTextureCache { pub fn repack(&mut self, texture_width: i32, texture_height: i32, slot_size: i32) { let capacity = ((texture_width / slot_size) * (texture_height / slot_size)) as usize; self.slot_size = slot_size; - self.is_updated = true; self.provider = TileAtlasTextureProvider::new(texture_width, texture_height, slot_size); self.transforms = Vec::with_capacity(capacity); self.textures = Vec::with_capacity(capacity); @@ -1754,14 +1736,6 @@ impl TileTextureCache { } } - pub fn needs_snapshot(&self) -> bool { - self.is_updated - } - - pub fn snapshot(&mut self) { - self.is_updated = false; - } - fn gc_non_visible( &mut self, tile_viewbox: &TileViewbox, @@ -1969,7 +1943,6 @@ impl TileTextureCache { self.removed.remove(&key); } - self.is_updated = true; tile_ref } @@ -1999,7 +1972,6 @@ impl TileTextureCache { } self.removed.insert(key); } - self.is_updated = true; } pub fn remove_at(&mut self, tile: Tile, scale: f32) { @@ -2011,13 +1983,11 @@ impl TileTextureCache { self.textures[tile_ref.index].set_empty(); } self.removed.insert(key); - self.is_updated = true; } pub fn clear(&mut self) { for k in self.grid.keys() { self.removed.insert(*k); } - self.is_updated = true; } } diff --git a/render-wasm/src/render/svg/fixtures.rs b/render-wasm/src/render/svg/fixtures.rs index a112752b6b..b8dfa4b785 100644 --- a/render-wasm/src/render/svg/fixtures.rs +++ b/render-wasm/src/render/svg/fixtures.rs @@ -7,9 +7,9 @@ use skia_safe as skia; use crate::globals::TestRenderResourcesGuard; use crate::render::{FontStore, RenderResources}; use crate::shapes::{ - make_corners, Fill, FontFamily, FontStyle, Frame, Group, GrowType, ImageFill, Paragraph, Path, - Rect, Segment, SolidColor, Stroke, StrokeKind, StrokeStyle, TextAlign, TextContent, - TextDirection, TextSpan, Type, + make_corners, Fill, FontFamily, FontStyle, Frame, Group, GrowType, ImageFill, ListStyleType, + Paragraph, Path, Rect, Segment, SolidColor, Stroke, StrokeKind, StrokeStyle, TextAlign, + TextContent, TextDirection, TextSpan, Type, }; use crate::state::ShapesPool; use crate::utils::uuid_from_u32_quartet; @@ -384,6 +384,52 @@ pub(super) fn add_text_with_spans( shape.set_shape_type(Type::Text(content)); } +/// Fixed text with one solid-filled paragraph per `(text, list style)` entry. +pub(super) fn add_text_with_list_paragraphs( + pool: &mut ShapesPool, + id: Uuid, + (l, t, r, b): (f32, f32, f32, f32), + font_size: f32, + fill: skia::Color, + paragraphs: Vec<(&str, ListStyleType)>, +) { + let bounds = skia::Rect::from_ltrb(l, t, r, b); + let mut content = TextContent::new(bounds, GrowType::Fixed); + let line_height = 1.2; + for (text, list_style_type) in paragraphs { + let span = TextSpan::new( + text.to_string(), + FontFamily::new(Uuid::nil(), 400, FontStyle::Normal), + font_size, + line_height, + 0.0, + None, + None, + TextDirection::LTR, + 400, + Uuid::nil(), + vec![Fill::Solid(SolidColor(fill))], + ); + content.add_paragraph( + Paragraph::new( + TextAlign::Left, + TextDirection::LTR, + None, + None, + line_height, + 0.0, + vec![span], + ) + .with_list_style_type(list_style_type), + ); + } + + let shape = pool.add_shape(id); + shape.set_parent(Uuid::nil()); + shape.set_selrect(l, t, r, b); + shape.set_shape_type(Type::Text(content)); +} + /// Adds a rectangle with a single stroke and no fill. pub(super) fn add_stroked_rect( pool: &mut ShapesPool, diff --git a/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_list_bullets_before_list_paragraphs_only.snap b/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_list_bullets_before_list_paragraphs_only.snap new file mode 100644 index 0000000000..d76f3044a5 --- /dev/null +++ b/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_list_bullets_before_list_paragraphs_only.snap @@ -0,0 +1,24 @@ +--- +source: src/render/svg/tests.rs +expression: svg +--- + + + + Bullet + + + + Plain + + + + Bullet + + + + + + Plain + + diff --git a/render-wasm/src/render/svg/tests.rs b/render-wasm/src/render/svg/tests.rs index 2212371f1e..96178a7c0a 100644 --- a/render-wasm/src/render/svg/tests.rs +++ b/render-wasm/src/render/svg/tests.rs @@ -2,8 +2,8 @@ use super::fixtures::*; use crate::shapes::{ radius_to_sigma, AlignContent, AlignItems, BlendMode, Blur, BlurType, Fill, FlexDirection, - ImageFill, ImageFillTransform, JustifyContent, JustifyItems, Shadow, ShadowStyle, SolidColor, - StrokeCap, StrokeKind, WrapType, + ImageFill, ImageFillTransform, JustifyContent, JustifyItems, ListStyleType, Shadow, + ShadowStyle, SolidColor, StrokeCap, StrokeKind, WrapType, }; use crate::state::ShapesPool; use crate::uuid::Uuid; @@ -2063,6 +2063,33 @@ fn exports_text_with_multiple_solid_fills() { insta::assert_snapshot!(svg); } +#[test] +fn exports_list_bullets_before_list_paragraphs_only() { + let mut pool = ShapesPool::new(); + let id = uid(1); + add_text_with_list_paragraphs( + &mut pool, + id, + (40.0, 0.0, 300.0, 120.0), + 30.0, + skia::Color::RED, + vec![ + ("Bullet", ListStyleType::Disc), + ("Plain", ListStyleType::None), + ], + ); + pool.get_mut(&id) + .unwrap() + .add_stroke(solid_stroke(StrokeKind::Center, 2.0, skia::Color::BLUE)); + + let svg = render(&pool, id); + // One bullet, before "Bullet" only: a red fill and a blue stroke. + assert_eq!(svg.matches("( + skia_paragraphs: impl IntoIterator>, + text_paragraphs: &[TextParagraph], + x: f32, + top: f32, + layout_width: f32, +) -> Vec { + let mut markers: Vec = Vec::new(); + let mut y = top; + for (skia_paragraph, text_paragraph) in skia_paragraphs.into_iter().zip(text_paragraphs) { + let Some(skia_paragraph) = skia_paragraph else { + continue; + }; + if let Some(marker) = list_marker( + skia_paragraph, + text_paragraph, + x, + y, + layout_width, + markers.last(), + ) { + markers.push(marker); + } + y += skia_paragraph.height(); + } + markers +} + +/// The bullet of a list paragraph, in shape coordinates. It hangs before the +/// text box's start edge, level with the first line, a gap away that grows +/// with the font size. +/// +/// An empty paragraph has no style run, so it takes the paint of `previous` +/// (the last list item painted in the same pass) and its own span's size. +pub fn list_marker( + skia_paragraph: &skia::textlayout::Paragraph, + text_paragraph: &TextParagraph, + x: f32, + y_accum: f32, + layout_width: f32, + previous: Option<&ListMarker>, +) -> Option { + if text_paragraph.list_style_type() == ListStyleType::None { + return None; + } + + let line_metrics = skia_paragraph.get_line_metrics(); + let line = line_metrics.first()?; + let style_metrics = line.get_style_metrics(line.start_index..line.end_index); + let (paint, font_size) = match style_metrics.first() { + Some((_, metrics)) => { + let mut paint = metrics.text_style.foreground(); + paint.set_anti_alias(true); + (paint, metrics.text_style.font_size()) + } + None => { + let font_size = text_paragraph.children().first()?.font_size; + (previous?.paint.clone(), font_size) + } + }; + + let radius = (font_size * LIST_MARKER_RADIUS_FACTOR).max(LIST_MARKER_MIN_RADIUS); + let offset = radius + font_size * LIST_MARKER_GAP_FACTOR; + let center_x = if text_paragraph.text_direction() == TextDirection::RTL { + x + layout_width + offset + } else { + x - offset + }; + let baseline = y_accum + line.baseline as f32; + + Some(ListMarker { + paint, + center: Point::new(center_x, baseline - font_size * LIST_MARKER_CENTER_FACTOR), + radius, + }) +} + fn draw_decoration_segment(canvas: &Canvas, deco: &TextDecorationSegment) { let mut decoration_paint = deco.text_style.foreground(); decoration_paint.set_anti_alias(true); @@ -1235,7 +1337,12 @@ fn calculate_decoration_metrics( #[cfg(test)] mod tests { use super::*; - use crate::shapes::{FontFamily, FontStyle, TextAlign, TextDirection, TextSpan, TextTransform}; + use crate::globals::TestRenderResourcesGuard; + use crate::render::RenderResources; + use crate::shapes::{ + Fill, FontFamily, FontStyle, GrowType, ListStyleType, SolidColor, TextAlign, TextDirection, + TextSpan, TextTransform, Type, + }; use crate::uuid::Uuid; fn span( @@ -1319,4 +1426,320 @@ mod tests { assert!(decorated_span_ranges(¶).is_empty()); } + + fn sized_span(text: &str, font_size: f32, fills: Vec) -> TextSpan { + TextSpan::new( + text.to_string(), + FontFamily::new(Uuid::nil(), 400, FontStyle::Normal), + font_size, + 1.2, + 0.0, + None, + None, + TextDirection::LTR, + 400, + Uuid::nil(), + fills, + ) + } + + fn list_paragraph( + text: &str, + font_size: f32, + direction: TextDirection, + list_style_type: ListStyleType, + ) -> TextParagraph { + TextParagraph::new( + TextAlign::Left, + direction, + None, + None, + 1.2, + 0.0, + vec![sized_span(text, font_size, vec![])], + ) + .with_list_style_type(list_style_type) + } + + fn filled_list_item(text: &str, font_size: f32, color: skia::Color) -> TextParagraph { + TextParagraph::new( + TextAlign::Left, + TextDirection::LTR, + None, + None, + 1.2, + 0.0, + vec![sized_span( + text, + font_size, + vec![Fill::Solid(SolidColor(color))], + )], + ) + .with_list_style_type(ListStyleType::Disc) + } + + /// A fixed text at (10, 20), `width` wide, laid out into paragraph layouts. + fn layout_paragraphs(paragraphs: Vec, width: f32) -> Vec { + let selrect = crate::math::Rect::from_xywh(10.0, 20.0, width, 200.0); + let mut content = TextContent::new(selrect, GrowType::Fixed); + for paragraph in paragraphs { + content.add_paragraph(paragraph); + } + let mut shape = Shape::new(Uuid::nil()); + shape.set_shape_type(Type::Text(content.clone())); + shape.set_selrect(10.0, 20.0, 10.0 + width, 220.0); + + let mut builders = content.paragraph_builder_group_from_text(None); + calculate_text_layout_data(&shape, &content, &mut builders, true).paragraphs + } + + fn first_baseline(layout: &ParagraphLayout) -> f32 { + layout.y + layout.paragraph.get_line_metrics()[0].baseline as f32 + } + + #[test] + fn list_marker_hangs_before_the_first_line() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![list_paragraph( + "item", + 20.0, + TextDirection::LTR, + ListStyleType::Disc, + )], + 300.0, + ); + + let marker = layouts[0] + .marker + .as_ref() + .expect("disc paragraph has a marker"); + // Font size 20: radius 3.2, gap 6.6, center 6.4 above the baseline. + assert!((marker.center.x - (10.0 - 9.8)).abs() < 0.01, "{marker:?}"); + assert!( + (marker.center.y - (first_baseline(&layouts[0]) - 6.4)).abs() < 0.01, + "{marker:?}" + ); + assert!((marker.radius - 3.2).abs() < 0.01, "{marker:?}"); + } + + #[test] + fn rtl_list_marker_hangs_after_the_text_box() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![list_paragraph( + "item", + 20.0, + TextDirection::RTL, + ListStyleType::Disc, + )], + 300.0, + ); + + let marker = layouts[0] + .marker + .as_ref() + .expect("disc paragraph has a marker"); + assert!( + (marker.center.x - (10.0 + 300.0 + 9.8)).abs() < 0.01, + "{marker:?}" + ); + } + + #[test] + fn wrapped_list_paragraph_has_one_marker_on_its_first_line() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![list_paragraph( + "one two three four five six", + 20.0, + TextDirection::LTR, + ListStyleType::Disc, + )], + 60.0, + ); + + assert!(layouts[0].paragraph.get_line_metrics().len() > 1); + let marker = layouts[0] + .marker + .as_ref() + .expect("disc paragraph has a marker"); + assert!( + (marker.center.y - (first_baseline(&layouts[0]) - 6.4)).abs() < 0.01, + "{marker:?}" + ); + } + + #[test] + fn list_marker_keeps_a_gap_before_the_text_at_any_size() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + for font_size in [4.0, 14.0, 72.0, 300.0] { + let layouts = layout_paragraphs( + vec![list_paragraph( + "item", + font_size, + TextDirection::LTR, + ListStyleType::Disc, + )], + 2000.0, + ); + + let marker = layouts[0] + .marker + .as_ref() + .expect("disc paragraph has a marker"); + // The text starts at x = 10; the gap grows with the font size. + assert!( + marker.rect().right() <= 10.0 - font_size * 0.32, + "font size {font_size}: {marker:?}" + ); + } + } + + /// The color painted at `point` when `paragraph` is painted through the + /// cached layout of a 200x100 text at (40, 40). + fn cached_paint_color_at( + paragraph: TextParagraph, + point: impl Fn(&ParagraphLayout) -> Point, + ) -> skia::Color { + let selrect = crate::math::Rect::from_xywh(40.0, 40.0, 200.0, 100.0); + let mut content = TextContent::new(selrect, GrowType::Fixed); + content.add_paragraph(paragraph); + let mut shape = Shape::new(Uuid::nil()); + shape.set_shape_type(Type::Text(content)); + shape.set_selrect(40.0, 40.0, 240.0, 140.0); + + let content = shape.get_text_content().clone(); + let mut builders = content.paragraph_builder_group_from_text(None); + let layouts = calculate_text_layout_data(&shape, &content, &mut builders, true); + let at = point(&layouts.paragraphs[0]); + + let mut surface = skia::surfaces::raster_n32_premul((300, 200)).expect("raster surface"); + surface.canvas().clear(skia::Color::TRANSPARENT); + let painted = + try_paint_from_layout_cache(None, Some(surface.canvas()), &shape, None, false) + .expect("cached paint"); + assert!(painted, "the layout cache should be usable"); + + let info = skia::ImageInfo::new( + (1, 1), + skia::ColorType::RGBA8888, + skia::AlphaType::Unpremul, + None, + ); + let mut px = [0u8; 4]; + assert!(surface.image_snapshot().read_pixels( + &info, + &mut px, + 4, + (at.x as i32, at.y as i32), + skia::image::CachingHint::Disallow + )); + skia::Color::from_argb(px[3], px[0], px[1], px[2]) + } + + #[test] + fn cached_paint_draws_the_list_marker() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let color = + cached_paint_color_at(filled_list_item("item", 30.0, skia::Color::RED), |layout| { + layout + .marker + .as_ref() + .expect("disc paragraph has a marker") + .center + }); + assert_eq!(color, skia::Color::RED); + } + + #[test] + fn cached_paint_draws_no_marker_for_plain_text() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + // Where a 30px bullet would be: radius 4.8 plus a 9.9 gap before x = 40. + let paragraph = filled_list_item("item", 30.0, skia::Color::RED) + .with_list_style_type(ListStyleType::None); + let color = cached_paint_color_at(paragraph, |layout| { + Point::new(40.0 - 14.7, first_baseline(layout) - 9.6) + }); + assert_eq!(color.a(), 0, "{color:?}"); + } + + #[test] + fn plain_paragraph_has_no_marker() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![list_paragraph( + "item", + 20.0, + TextDirection::LTR, + ListStyleType::None, + )], + 300.0, + ); + + assert!(layouts[0].marker.is_none()); + } + + #[test] + fn small_text_marker_keeps_a_minimum_radius() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![list_paragraph( + "item", + 4.0, + TextDirection::LTR, + ListStyleType::Disc, + )], + 300.0, + ); + + let marker = layouts[0] + .marker + .as_ref() + .expect("disc paragraph has a marker"); + assert!((marker.radius - 1.5).abs() < 0.01, "{marker:?}"); + } + + #[test] + fn empty_list_paragraph_borrows_the_previous_item_paint() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![ + filled_list_item("item", 20.0, skia::Color::RED), + filled_list_item("", 10.0, skia::Color::BLUE), + ], + 300.0, + ); + + let empty = layouts[1].marker.as_ref().expect("empty item has a marker"); + // Its own size, the previous item's paint. + assert!((empty.radius - 1.6).abs() < 0.01, "{empty:?}"); + assert_eq!(empty.paint.color(), skia::Color::RED); + } + + #[test] + fn empty_first_list_paragraph_has_no_marker() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let layouts = layout_paragraphs( + vec![list_paragraph( + "", + 20.0, + TextDirection::LTR, + ListStyleType::Disc, + )], + 300.0, + ); + + assert!(layouts[0].marker.is_none()); + } } diff --git a/render-wasm/src/shapes.rs b/render-wasm/src/shapes.rs index 5802f9f6fb..111745cbae 100644 --- a/render-wasm/src/shapes.rs +++ b/render-wasm/src/shapes.rs @@ -191,6 +191,7 @@ pub struct Shape { pub background_blur: Option, pub opacity: f32, pub hidden: bool, + pub hide_fill_on_export: bool, pub svg: Option, pub svg_attrs: Option, pub shadows: Vec, @@ -295,6 +296,7 @@ impl Shape { vertical_align: VerticalAlign::Top, opacity: 1., hidden: false, + hide_fill_on_export: false, blur: None, background_blur: None, svg: None, @@ -466,6 +468,10 @@ impl Shape { self.hidden = value; } + pub fn set_hide_fill_on_export(&mut self, value: bool) { + self.hide_fill_on_export = value; + } + pub fn svg_transform(&self) -> Option { self.svg_transform } @@ -1741,12 +1747,15 @@ impl Shape { for st in structure { match st.entry_type { StructureEntryType::AddChild => { - if result.is_empty() { - result.insert(st.id); - } else { - let index = usize::min(result.len() - 1, st.index as usize); - result.shift_insert(index, st.id); - } + // Same as `d/insert-at-index`: the index counts the child + // at its old position. + let index = st.index as usize; + let index = match result.get_index_of(&st.id) { + Some(current) if current < index => usize::min(index - 1, result.len() - 1), + Some(_) => index, + None => usize::min(index, result.len()), + }; + result.shift_insert(index, st.id); } StructureEntryType::RemoveChild => { to_remove.insert(&st.id); @@ -2225,6 +2234,48 @@ mod tests { Shape::new(Uuid::nil()) } + fn add_child(shape: &mut Shape, id: Uuid, index: u32) { + shape.apply_structure(&vec![StructureEntry::new( + StructureEntryType::AddChild, + index, + Uuid::nil(), + id, + 0.0, + )]); + } + + #[test] + fn add_child_index_counts_the_child_at_its_old_position() { + let ids: Vec = (1..=4).map(|i| Uuid::from_u64_pair(0, i)).collect(); + let [a, b, c, d] = [ids[0], ids[1], ids[2], ids[3]]; + + let cases = [ + (b, 4, vec![a, c, d, b]), + (b, 3, vec![a, c, b, d]), + (b, 2, vec![a, b, c, d]), + (b, 1, vec![a, b, c, d]), + (d, 0, vec![d, a, b, c]), + ]; + + for (id, index, expected) in cases { + let mut shape = any_shape(); + shape.children = ids.clone(); + add_child(&mut shape, id, index); + assert_eq!(shape.children, expected, "move {id} to {index}"); + } + } + + #[test] + fn add_child_inserts_a_new_child_at_the_end() { + let ids: Vec = (1..=3).map(|i| Uuid::from_u64_pair(0, i)).collect(); + let new_id = Uuid::from_u64_pair(0, 9); + + let mut shape = any_shape(); + shape.children = ids.clone(); + add_child(&mut shape, new_id, 3); + assert_eq!(shape.children, [ids, vec![new_id]].concat()); + } + #[test] fn add_fill_pushes_a_new_fill() { let mut shape = any_shape(); diff --git a/render-wasm/src/shapes/text.rs b/render-wasm/src/shapes/text.rs index a8328be0d1..d3a15eb34c 100644 --- a/render-wasm/src/shapes/text.rs +++ b/render-wasm/src/shapes/text.rs @@ -1,4 +1,4 @@ -use crate::render::text::decoration_segments; +use crate::render::text::{decoration_segments, list_marker, list_markers}; use crate::{ math::{Bounds, Matrix, Rect}, render::{default_font, DEFAULT_EMOJI_FONT}, @@ -309,6 +309,34 @@ impl TextContentLayout { } } +// List bullet geometry, in font sizes, as Chrome draws a CSS disc marker +// (measured with Source Sans Pro). +pub const LIST_MARKER_RADIUS_FACTOR: f32 = 0.16; +pub const LIST_MARKER_MIN_RADIUS: f32 = 1.5; +/// Space between the bullet and the text box's start edge. +pub const LIST_MARKER_GAP_FACTOR: f32 = 0.33; +/// How far above the baseline the bullet's center sits. +pub const LIST_MARKER_CENTER_FACTOR: f32 = 0.32; + +/// A list bullet, painted with the style of the first run on its line. +#[derive(Debug, Clone)] +pub struct ListMarker { + pub paint: skia::Paint, + pub center: Point, + pub radius: f32, +} + +impl ListMarker { + pub fn rect(&self) -> Rect { + Rect::new( + self.center.x - self.radius, + self.center.y - self.radius, + self.center.x + self.radius, + self.center.y + self.radius, + ) + } +} + #[derive(Debug, Clone)] pub struct TextDecorationSegment { #[allow(dead_code)] @@ -626,11 +654,40 @@ impl TextContent { } } - if has_lines { - Some(Rect::from_ltrb(min_x, min_y, max_x, max_y)) - } else { - None + if !has_lines { + return None; } + + let mut rect = Rect::from_ltrb(min_x, min_y, max_x, max_y); + if let Some(markers) = self.list_markers_rect(selrect, valign) { + rect.join(markers); + } + Some(rect) + } + + /// Union of the list bullets, which hang outside the text box. + fn list_markers_rect(&self, selrect: &Rect, valign: VerticalAlign) -> Option { + let paragraphs = &self.layout.paragraphs; + let total_height: f32 = paragraphs + .iter() + .filter_map(|group| group.first()) + .map(|p| p.height()) + .sum(); + let top = selrect.y() + vertical_align_offset(selrect.height(), total_height, valign); + + list_markers( + paragraphs.iter().map(|group| group.first()), + self.paragraphs(), + selrect.x(), + top, + self.get_width(selrect.width()), + ) + .iter() + .map(ListMarker::rect) + .reduce(|mut rect, marker_rect| { + rect.join(marker_rect); + rect + }) } fn compute_and_cache_extrect( @@ -643,7 +700,11 @@ impl TextContent { // (line.left) reflect alignment within that huge width and are // unusable for tight bounds. Fall back to content_rect. if self.grow_type() == GrowType::AutoWidth { - return self.content_rect(selrect, valign); + let mut rect = self.content_rect(selrect, valign); + if let Some(markers) = self.list_markers_rect(selrect, valign) { + rect.join(markers); + } + return rect; } let layout_matches_container = self @@ -1396,6 +1457,13 @@ pub enum TextTransform { Capitalize, } +#[derive(Debug, PartialEq, Clone, Copy, Default)] +pub enum ListStyleType { + #[default] + None, + Disc, +} + // FIXME: Rethink this type. We'll probably need to move the serialization to the // wasm module and store here meaningful model values (and/or skia type aliases) #[derive(Debug, PartialEq, Clone)] @@ -1406,6 +1474,7 @@ pub struct Paragraph { text_transform: Option, line_height: f32, letter_spacing: f32, + list_style_type: ListStyleType, children: Vec, } @@ -1418,6 +1487,7 @@ impl Default for Paragraph { text_transform: None, line_height: 1.0, letter_spacing: 0.0, + list_style_type: ListStyleType::None, children: vec![], } } @@ -1441,10 +1511,16 @@ impl Paragraph { text_transform, line_height, letter_spacing, + list_style_type: ListStyleType::None, children, } } + pub fn with_list_style_type(mut self, list_style_type: ListStyleType) -> Self { + self.list_style_type = list_style_type; + self + } + pub fn children(&self) -> &[TextSpan] { &self.children } @@ -1557,6 +1633,10 @@ impl Paragraph { self.text_transform } + pub fn list_style_type(&self) -> ListStyleType { + self.list_style_type + } + pub fn paragraph_to_style(&self) -> ParagraphStyle { let mut style = ParagraphStyle::default(); @@ -1857,6 +1937,7 @@ pub struct ParagraphLayout { pub x: f32, pub y: f32, pub decorations: Vec, + pub marker: Option, } #[derive(Debug)] @@ -1920,20 +2001,39 @@ pub fn calculate_text_layout_data( vertical_align_offset(selrect_height, total_text_height, shape.vertical_align()); let mut paragraph_layouts: Vec = Vec::new(); let mut y_accum = base_y + vertical_offset; + // Last list marker of each builder, so an empty item borrows its pass' paint. + let mut previous_markers: Vec> = Vec::new(); for (i, group_paragraphs) in built_groups.into_iter().enumerate() { // For each paragraph in the group (e.g., fill, stroke, etc.) - for skia_paragraph in group_paragraphs.into_iter() { + for (builder_index, skia_paragraph) in group_paragraphs.into_iter().enumerate() { + if previous_markers.len() <= builder_index { + previous_markers.resize(builder_index + 1, None); + } let decorations = text_paragraphs .get(i) .map(|text_paragraph| { decoration_segments(&skia_paragraph, text_paragraph, x, y_accum) }) .unwrap_or_default(); + let marker = text_paragraphs.get(i).and_then(|text_paragraph| { + list_marker( + &skia_paragraph, + text_paragraph, + x, + y_accum, + text_width, + previous_markers[builder_index].as_ref(), + ) + }); + if marker.is_some() { + previous_markers[builder_index] = marker.clone(); + } paragraph_layouts.push(ParagraphLayout { paragraph: skia_paragraph, x, y: y_accum, decorations, + marker, }); } y_accum += paragraph_heights[i]; @@ -2404,6 +2504,119 @@ mod tests { shape } + /// A laid-out 200x100 text at the origin with one "hello" paragraph. + fn list_text_shape( + grow_type: GrowType, + direction: TextDirection, + list_style_type: ListStyleType, + ) -> Shape { + let mut paragraph = test_paragraph(&["hello"]).with_list_style_type(list_style_type); + paragraph.text_direction = direction; + let mut content = TextContent::new(Rect::from_xywh(0.0, 0.0, 200.0, 100.0), grow_type); + content.add_paragraph(paragraph); + + let mut shape = Shape::new(Uuid::nil()); + shape.set_shape_type(shapes::Type::Text(content)); + shape.set_selrect(0.0, 0.0, 200.0, 100.0); + shape + } + + #[test] + fn extrect_holds_the_painted_marker_when_vertically_aligned() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + for valign in [VerticalAlign::Center, VerticalAlign::Bottom] { + let mut shape = + list_text_shape(GrowType::Fixed, TextDirection::LTR, ListStyleType::Disc); + shape.set_vertical_align(valign); + + let content = shape.get_text_content().clone(); + let mut builders = content.paragraph_builder_group_from_text(None); + let layout = calculate_text_layout_data(&shape, &content, &mut builders, true); + let marker = layout.paragraphs[0] + .marker + .as_ref() + .expect("disc paragraph has a marker") + .rect(); + + let extrect = text_extrect(&shape); + assert!( + extrect.left() <= marker.left() + && extrect.top() <= marker.top() + && extrect.right() >= marker.right() + && extrect.bottom() >= marker.bottom(), + "{valign:?}: {marker:?} outside {extrect:?}" + ); + // With top alignment the marker would sit near y = 0. + assert!(marker.top() > 10.0, "{valign:?}: {marker:?}"); + } + } + + fn text_extrect(shape: &Shape) -> Rect { + shape + .get_text_content() + .calculate_bounds(shape, false) + .to_rect() + } + + // Font size 14: a gap of 4.62 and a radius of 2.24. + const MARKER_REACH: f32 = 4.62 + 2.0 * 2.24; + + #[test] + fn extrect_includes_a_list_marker_before_the_text() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + for grow_type in [GrowType::Fixed, GrowType::AutoHeight, GrowType::AutoWidth] { + let shape = list_text_shape(grow_type, TextDirection::LTR, ListStyleType::Disc); + let extrect = text_extrect(&shape); + assert!( + extrect.left() <= -MARKER_REACH + 0.01, + "{grow_type:?}: {extrect:?}" + ); + assert_eq!(shape.selrect().left(), 0.0); + } + } + + #[test] + fn extrect_includes_an_rtl_list_marker_after_the_text() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let shape = list_text_shape(GrowType::Fixed, TextDirection::RTL, ListStyleType::Disc); + let extrect = text_extrect(&shape); + assert!( + extrect.right() >= 200.0 + MARKER_REACH - 0.01, + "{extrect:?}" + ); + } + + #[test] + fn extrect_of_plain_text_stays_inside_the_text_box() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let shape = list_text_shape(GrowType::Fixed, TextDirection::LTR, ListStyleType::None); + let extrect = text_extrect(&shape); + assert!(extrect.left() >= 0.0, "{extrect:?}"); + } + + #[test] + fn extrect_grows_when_bullets_are_turned_on() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let mut shape = list_text_shape(GrowType::Fixed, TextDirection::LTR, ListStyleType::None); + assert!(text_extrect(&shape).left() >= 0.0); + + let selrect = shape.selrect; + if let shapes::Type::Text(content) = &mut shape.shape_type { + let paragraph = content.paragraphs_mut()[0] + .clone() + .with_list_style_type(ListStyleType::Disc); + content.paragraphs_mut()[0] = paragraph; + content.update_layout(selrect); + } + + assert!(text_extrect(&shape).left() <= -MARKER_REACH + 0.01); + } + /// Whether resizing `shape` to `width`x`height`, with no content change, /// rebuilt its cached Skia paragraphs. fn resize_rebuilds_paragraphs(shape: &mut Shape, width: f32, height: f32) -> bool { diff --git a/render-wasm/src/shapes/text_paths.rs b/render-wasm/src/shapes/text_paths.rs index 44cc21a115..344906971a 100644 --- a/render-wasm/src/shapes/text_paths.rs +++ b/render-wasm/src/shapes/text_paths.rs @@ -1,4 +1,4 @@ -use crate::render::text::decoration_segments; +use crate::render::text::{decoration_segments, list_markers}; use crate::shapes::text::{vertical_align_offset, Paragraph, TextContent}; use crate::shapes::VerticalAlign; use skia_safe::{ @@ -29,10 +29,25 @@ impl TextPaths { .collect(); let total_height: f32 = paragraphs.iter().map(|p| p.height()).sum(); - let mut offset_y = self.bounds.y() + let top = self.bounds.y() + vertical_align_offset(self.bounds.height(), total_height, vertical_align); - let mut paths = Vec::new(); + let mut paths: Vec = list_markers( + paragraphs.iter().map(Some), + self.0.paragraphs(), + self.bounds.x(), + top, + layout_width, + ) + .into_iter() + .map(|marker| { + let mut builder = skia::PathBuilder::new(); + builder.add_circle(marker.center, marker.radius, None); + builder.detach() + }) + .collect(); + + let mut offset_y = top; for (paragraph, text_paragraph) in paragraphs.iter_mut().zip(self.0.paragraphs()) { let origin = Point::new(self.bounds.x(), offset_y); Self::collect_paragraph_paths(paragraph, text_paragraph, origin, &mut paths); @@ -86,3 +101,107 @@ impl Deref for TextPaths { &self.0 } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::globals::TestRenderResourcesGuard; + use crate::math::Rect; + use crate::render::RenderResources; + use crate::shapes::{ + FontFamily, FontStyle, GrowType, ListStyleType, TextAlign, TextDirection, TextSpan, + }; + use crate::uuid::Uuid; + + /// Paths of a 200x100 text at the origin, one 14px paragraph per entry. + fn paths_of_paragraphs( + paragraphs: Vec<(&str, TextDirection, ListStyleType)>, + ) -> Vec { + let mut content = + TextContent::new(Rect::from_xywh(0.0, 0.0, 200.0, 100.0), GrowType::Fixed); + for (text, direction, list_style_type) in paragraphs { + let span = TextSpan::new( + text.to_string(), + FontFamily::new(Uuid::nil(), 400, FontStyle::Normal), + 14.0, + 1.2, + 0.0, + None, + None, + direction, + 400, + Uuid::nil(), + vec![], + ); + content.add_paragraph( + Paragraph::new(TextAlign::Left, direction, None, None, 1.2, 0.0, vec![span]) + .with_list_style_type(list_style_type), + ); + } + TextPaths::new(content).get_paths(VerticalAlign::Top) + } + + fn paths_of(list_style_type: ListStyleType) -> Vec { + paths_of_paragraphs(vec![("hello", TextDirection::LTR, list_style_type)]) + } + + /// Bounds of the paths that lie wholly outside the 200px-wide text box. + fn bullet_bounds(paths: Vec) -> Vec { + paths + .into_iter() + .map(|path| *path.bounds()) + .filter(|bounds| bounds.right() < 0.0 || bounds.left() > 200.0) + .collect() + } + + #[test] + fn list_paragraph_paths_include_the_bullet() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + // Font size 14: the bullet is centered at x = -6.86 with radius 2.24. + let bullet = paths_of(ListStyleType::Disc) + .into_iter() + .map(|path| *path.bounds()) + .find(|bounds| bounds.right() < 0.0) + .expect("a path before the text box"); + assert!((bullet.center_x() + 6.86).abs() < 0.01, "{bullet:?}"); + assert!((bullet.width() - 4.48).abs() < 0.01, "{bullet:?}"); + } + + #[test] + fn empty_list_item_paths_include_its_bullet() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let bullets = bullet_bounds(paths_of_paragraphs(vec![ + ("hello", TextDirection::LTR, ListStyleType::Disc), + ("", TextDirection::LTR, ListStyleType::Disc), + ])); + assert_eq!(bullets.len(), 2, "{bullets:?}"); + } + + #[test] + fn rtl_list_paragraph_paths_put_the_bullet_after_the_text_box() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + let bullets = bullet_bounds(paths_of_paragraphs(vec![( + "hello", + TextDirection::RTL, + ListStyleType::Disc, + )])); + assert_eq!(bullets.len(), 1, "{bullets:?}"); + // Font size 14: radius 2.24 plus a 4.62 gap past the right edge. + assert!( + (bullets[0].center_x() - (200.0 + 6.86)).abs() < 0.01, + "{bullets:?}" + ); + } + + #[test] + fn plain_paragraph_paths_have_no_bullet() { + let mut resources = RenderResources::try_new_headless().expect("headless resources"); + let _guard = TestRenderResourcesGuard::install(&mut resources); + assert!(paths_of(ListStyleType::None) + .iter() + .all(|path| path.bounds().left() >= 0.0)); + } +} diff --git a/render-wasm/src/state.rs b/render-wasm/src/state.rs index 97296058b0..eadfd7616f 100644 --- a/render-wasm/src/state.rs +++ b/render-wasm/src/state.rs @@ -98,16 +98,24 @@ impl State { scale: f32, timestamp: i32, format: RasterFormat, + export: bool, ) -> Result<(Vec, i32, i32)> { - get_render_state().render_shape_pixels(id, &self.shapes, scale, timestamp, format) + let render = |tree: &ShapesPool| { + get_render_state().render_shape_pixels(id, tree, scale, timestamp, format) + }; + if export { + self.shapes.with_export_root(id, render) + } else { + render(&self.shapes) + } } pub fn render_shape_pdf(&mut self, id: &Uuid) -> Result> { - crate::render::pdf::render_to_pdf(get_resources(), id, &self.shapes) + crate::render::pdf::render_to_pdf(get_resources(), id, &mut self.shapes) } pub fn render_shapes_pdf(&mut self, ids: &[Uuid]) -> Result> { - crate::render::pdf::render_to_pdf_multi(get_resources(), ids, &self.shapes) + crate::render::pdf::render_to_pdf_multi(get_resources(), ids, &mut self.shapes) } pub fn pdf_export_begin(&mut self) -> Result<()> { @@ -115,7 +123,10 @@ impl State { } pub fn pdf_export_add_page(&mut self, id: &Uuid) -> Result<()> { - crate::render::pdf::export_add_page(&mut self.pdf_export, get_resources(), id, &self.shapes) + let session = &mut self.pdf_export; + self.shapes.with_export_root(id, |tree| { + crate::render::pdf::export_add_page(session, get_resources(), id, tree) + }) } pub fn pdf_export_finish(&mut self) -> Result> { @@ -127,7 +138,9 @@ impl State { } pub fn render_shape_svg(&mut self, id: &Uuid, scale: f32) -> Result> { - crate::render::svg::render_to_svg(get_resources(), id, &self.shapes, scale) + self.shapes.with_export_root(id, |tree| { + crate::render::svg::render_to_svg(get_resources(), id, tree, scale) + }) } pub fn render_shapes_svg(&mut self, ids: &[Uuid], scale: f32) -> Result> { @@ -142,7 +155,9 @@ impl State { scale: f32, format: RasterFormat, ) -> Result<(Vec, i32, i32)> { - crate::render::raster::render_to_raster(get_resources(), id, &self.shapes, scale, format) + self.shapes.with_export_root(id, |tree| { + crate::render::raster::render_to_raster(get_resources(), id, tree, scale, format) + }) } /// Distinct font families used by the (visible) subtree rooted at `id`, in diff --git a/render-wasm/src/state/shapes_pool.rs b/render-wasm/src/state/shapes_pool.rs index cd9542068a..e4ee3d22c3 100644 --- a/render-wasm/src/state/shapes_pool.rs +++ b/render-wasm/src/state/shapes_pool.rs @@ -142,6 +142,24 @@ impl ShapesPoolImpl { self.uuid_to_idx.contains_key(id) } + /// Runs `render` with the fills of `id` removed when it hides them on + /// export. Only the exported board applies the flag; nested boards keep + /// their fills. + pub fn with_export_root(&mut self, id: &Uuid, render: impl FnOnce(&Self) -> T) -> T { + let hide = self.get(id).is_some_and(|shape| shape.hide_fill_on_export); + let fills = if hide { + self.get_mut(id) + .map(|shape| std::mem::take(&mut shape.fills)) + } else { + None + }; + let result = render(self); + if let (Some(fills), Some(shape)) = (fills, self.get_mut(id)) { + shape.fills = fills; + } + result + } + pub fn get_mut(&mut self, id: &Uuid) -> Option<&mut Shape> { let idx = *self.uuid_to_idx.get(id)?; // The caller may change the shape, so its modified copy goes stale. @@ -611,6 +629,24 @@ mod tests { (pool, parent, child) } + #[test] + fn with_export_root_hides_only_the_root_fills() { + let (mut pool, parent, child) = pool_with_parent_and_child(); + let fills = vec![shapes::Fill::Solid(shapes::SolidColor(skia::Color::RED))]; + for id in [parent, child] { + let shape = pool.get_mut(&id).expect("shape"); + shape.set_fills(fills.clone()); + shape.set_hide_fill_on_export(true); + } + + pool.with_export_root(&parent, |tree| { + assert!(tree.get(&parent).expect("parent").fills.is_empty()); + assert_eq!(tree.get(&child).expect("child").fills, fills); + }); + + assert_eq!(pool.get(&parent).expect("parent").fills, fills); + } + #[test] fn set_modifiers_passes_a_move_down_to_children() { let (mut pool, parent, child) = pool_with_parent_and_child(); diff --git a/render-wasm/src/state/text_editor.rs b/render-wasm/src/state/text_editor.rs index aaec3e90fd..9857cb134f 100644 --- a/render-wasm/src/state/text_editor.rs +++ b/render-wasm/src/state/text_editor.rs @@ -3,7 +3,7 @@ use macros::ToJs; use crate::shapes::{ - Fill, FontStyle, TextAlign, TextContent, TextDecoration, TextDirection, + Fill, FontStyle, ListStyleType, TextAlign, TextContent, TextDecoration, TextDirection, TextPositionWithAffinity, TextTransform, VerticalAlign, }; use crate::uuid::Uuid; @@ -109,6 +109,7 @@ pub struct TextEditorStyles { pub text_direction: Multiple, // Multiple pub text_decoration: Multiple, pub text_transform: Multiple, + pub list_style_type: Multiple, // The font family is decomposed into its independent parts so the family // dropdown shows a single family even when the selection mixes variants of // the same family (e.g. regular + bold italic). The id identifies the @@ -232,6 +233,7 @@ impl TextEditorStyles { text_direction: Multiple::empty(), text_decoration: Multiple::empty(), text_transform: Multiple::empty(), + list_style_type: Multiple::empty(), font_family_id: Multiple::empty(), font_style: Multiple::empty(), font_size: Multiple::empty(), @@ -249,6 +251,7 @@ impl TextEditorStyles { self.text_direction.reset(); self.text_decoration.reset(); self.text_transform.reset(); + self.list_style_type.reset(); self.font_family_id.reset(); self.font_style.reset(); self.font_size.reset(); @@ -638,6 +641,9 @@ impl TextEditorState { self.current_styles .text_align .merge(Some(paragraph.text_align())); + self.current_styles + .list_style_type + .merge(Some(paragraph.list_style_type())); let mut char_pos = 0; for span in paragraph.children() { @@ -706,6 +712,9 @@ impl TextEditorState { self.current_styles .text_align .set_single(Some(current_paragraph.text_align())); + self.current_styles + .list_style_type + .set_single(Some(current_paragraph.list_style_type())); if let Some((text_span_index, _)) = current_text_span { if let Some(text_span) = current_paragraph.children().get(text_span_index) { self.current_styles @@ -1030,6 +1039,34 @@ mod tests { content } + fn list_content() -> TextContent { + let black = Fill::Solid(SolidColor(Color::BLACK)); + let mut content = + TextContent::new(Rect::from_xywh(0.0, 0.0, 200.0, 100.0), GrowType::Fixed); + content.add_paragraph( + Paragraph::new( + TextAlign::Left, + TextDirection::LTR, + None, + None, + 1.2, + 0.0, + vec![span("Bullet", 14.0, black.clone())], + ) + .with_list_style_type(ListStyleType::Disc), + ); + content.add_paragraph(Paragraph::new( + TextAlign::Left, + TextDirection::LTR, + None, + None, + 1.2, + 0.0, + vec![span("Plain", 14.0, black)], + )); + content + } + fn drain_events(editor: &mut TextEditorState) -> Vec { let mut events = Vec::new(); loop { @@ -1046,6 +1083,38 @@ mod tests { TextPositionWithAffinity::new_downstream_affinity(0, offset) } + #[test] + fn caret_styles_report_the_paragraph_list_style_type() { + let content = list_content(); + let mut editor = TextEditorState::new(); + + editor.place_caret_from_position( + &content, + &TextPositionWithAffinity::new_downstream_affinity(0, 2), + ); + assert_eq!( + editor.current_styles.list_style_type.value(), + &Some(ListStyleType::Disc) + ); + + editor.place_caret_from_position( + &content, + &TextPositionWithAffinity::new_downstream_affinity(1, 2), + ); + assert_eq!( + editor.current_styles.list_style_type.value(), + &Some(ListStyleType::None) + ); + } + + #[test] + fn selection_styles_mark_mixed_list_style_types_as_multiple() { + let content = list_content(); + let mut editor = TextEditorState::new(); + editor.select_all(&content); + assert!(editor.current_styles.list_style_type.is_multiple()); + } + #[test] fn set_caret_from_position_alone_does_not_emit_styles_changed() { let content = hello_world_content(); diff --git a/render-wasm/src/wasm/shapes/base_props.rs b/render-wasm/src/wasm/shapes/base_props.rs index 9ab65f78fe..826876ec75 100644 --- a/render-wasm/src/wasm/shapes/base_props.rs +++ b/render-wasm/src/wasm/shapes/base_props.rs @@ -11,6 +11,7 @@ use super::RawShapeType; const FLAG_CLIP_CONTENT: u8 = 0b0000_0001; const FLAG_HIDDEN: u8 = 0b0000_0010; +const FLAG_HIDE_FILL_ON_EXPORT: u8 = 0b0000_0100; pub(crate) const RAW_BASE_PROPS_SIZE: usize = std::mem::size_of::(); @@ -78,6 +79,10 @@ impl RawBasePropsData { (self.flags & FLAG_HIDDEN) != 0 } + fn hide_fill_on_export(&self) -> bool { + (self.flags & FLAG_HIDE_FILL_ON_EXPORT) != 0 + } + fn blend_mode(&self) -> BlendMode { RawBlendMode::from(self.blend_mode).into() } @@ -112,6 +117,7 @@ pub(crate) fn apply_base_props(raw: &RawBasePropsData) -> Result<()> { shape.set_shape_type(shape_type.into()); shape.set_clip(raw.clip_content()); shape.set_hidden(raw.hidden()); + shape.set_hide_fill_on_export(raw.hide_fill_on_export()); shape.set_blend_mode(raw.blend_mode()); shape.set_opacity(raw.opacity); shape.set_constraint_h(raw.constraint_h()); diff --git a/render-wasm/src/wasm/text.rs b/render-wasm/src/wasm/text.rs index c154afbf89..54e47defc4 100644 --- a/render-wasm/src/wasm/text.rs +++ b/render-wasm/src/wasm/text.rs @@ -4,7 +4,8 @@ use super::{fills::RawFillData, fonts::RawFontStyle}; use crate::mem::{self, SerializableResult}; use crate::shapes::{ - self, GrowType, Shape, TextAlign, TextDecoration, TextDirection, TextTransform, Type, + self, GrowType, ListStyleType, Shape, TextAlign, TextDecoration, TextDirection, TextTransform, + Type, }; use crate::utils::{uuid_from_u32, uuid_from_u32_quartet}; use crate::{with_current_shape, with_current_shape_mut, with_state}; @@ -94,6 +95,31 @@ impl From for Option { } } +#[derive(Debug, PartialEq, Clone, Copy, ToJs)] +#[repr(u8)] +pub enum RawListStyleType { + None = 0, + Disc = 1, +} + +impl From for ListStyleType { + fn from(value: RawListStyleType) -> Self { + match value { + RawListStyleType::None => ListStyleType::None, + RawListStyleType::Disc => ListStyleType::Disc, + } + } +} + +impl From for RawListStyleType { + fn from(value: ListStyleType) -> Self { + match value { + ListStyleType::None => RawListStyleType::None, + ListStyleType::Disc => RawListStyleType::Disc, + } + } +} + #[repr(C)] #[repr(align(4))] #[derive(Debug, Clone, Copy)] @@ -105,6 +131,8 @@ pub struct RawParagraphData { text_transform: RawTextTransform, line_height: f32, letter_spacing: f32, + list_style_type: RawListStyleType, + _padding: [u8; 3], } impl From<[u8; RAW_PARAGRAPH_DATA_SIZE]> for RawParagraphData { @@ -253,6 +281,7 @@ impl From for shapes::Paragraph { value.attrs.letter_spacing, spans, ) + .with_list_style_type(value.attrs.list_style_type.into()) } } diff --git a/render-wasm/src/wasm/text/helpers.rs b/render-wasm/src/wasm/text/helpers.rs index be51aadcb2..adab557a50 100644 --- a/render-wasm/src/wasm/text/helpers.rs +++ b/render-wasm/src/wasm/text/helpers.rs @@ -815,6 +815,7 @@ pub fn split_paragraph_at_cursor( let text_transform = para.text_transform(); let line_height = para.line_height(); let letter_spacing = para.letter_spacing(); + let list_style_type = para.list_style_type(); let para = &mut paragraphs[cursor.paragraph]; let children = para.children_mut(); @@ -836,7 +837,8 @@ pub fn split_paragraph_at_cursor( line_height, letter_spacing, new_para_children, - ); + ) + .with_list_style_type(list_style_type); paragraphs.insert(cursor.paragraph + 1, new_para); @@ -847,7 +849,7 @@ pub fn split_paragraph_at_cursor( mod tests { use super::*; use crate::math::Rect; - use crate::shapes::{FontFamily, FontStyle, GrowType, TextAlign, TextSpan}; + use crate::shapes::{FontFamily, FontStyle, GrowType, ListStyleType, TextAlign, TextSpan}; use crate::uuid::Uuid; fn span(text: &str) -> TextSpan { @@ -941,6 +943,19 @@ mod tests { TextPositionWithAffinity::new_downstream_affinity(paragraph, offset) } + #[test] + fn split_paragraph_keeps_the_list_style_type() { + let mut content = content(vec![ + paragraph(&["Hello world"]).with_list_style_type(ListStyleType::Disc) + ]); + assert!(split_paragraph_at_cursor(&mut content, &caret(0, 5))); + + let paragraphs = content.paragraphs(); + assert_eq!(paragraphs.len(), 2); + assert_eq!(paragraphs[0].list_style_type(), ListStyleType::Disc); + assert_eq!(paragraphs[1].list_style_type(), ListStyleType::Disc); + } + #[test] fn delete_char_before_deletes_a_whole_emoji() { for emoji in ["😀", "👍🏽", "👨\u{200d}👩\u{200d}👧", "🇪🇸"] { diff --git a/render-wasm/src/wasm/text_editor.rs b/render-wasm/src/wasm/text_editor.rs index 9853876249..71d552f2bf 100644 --- a/render-wasm/src/wasm/text_editor.rs +++ b/render-wasm/src/wasm/text_editor.rs @@ -11,7 +11,8 @@ use crate::utils::uuid_to_u32_quartet; use crate::uuid::Uuid; use crate::wasm::fills::RawFillData; use crate::wasm::text::{ - helpers as text_helpers, RawTextAlign, RawTextDecoration, RawTextDirection, RawTextTransform, + helpers as text_helpers, RawListStyleType, RawTextAlign, RawTextDecoration, RawTextDirection, + RawTextTransform, }; use crate::with_state; use skia_safe::Color; @@ -741,6 +742,13 @@ pub extern "C" fn text_editor_get_current_styles() -> *mut u8 { }) .unwrap_or(RawTextTransform::None as u32); + let list_style_type = styles + .list_style_type + .value() + .as_ref() + .map(|value| RawListStyleType::from(*value) as u32) + .unwrap_or(RawListStyleType::None as u32); + let font_family_id = styles .font_family_id .value() @@ -817,7 +825,9 @@ pub extern "C" fn text_editor_get_current_styles() -> *mut u8 { bytes.extend_from_slice(&font_variant_id[3].to_le_bytes()); // 112 // 28 bytes.extend_from_slice(&line_height.to_le_bytes()); // 116 // 29 bytes.extend_from_slice(&letter_spacing.to_le_bytes()); // 120 // 30 - bytes.extend_from_slice(&fill_bytes); // 124 + bytes.extend_from_slice(&(*styles.list_style_type.state() as u32).to_le_bytes()); // 124 // 31 + bytes.extend_from_slice(&list_style_type.to_le_bytes()); // 128 // 32 + bytes.extend_from_slice(&fill_bytes); // 132 mem::write_bytes(bytes) }) diff --git a/renovate.json b/renovate.json index 1d7acb30de..6f1e06fb35 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,8 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["github>penpot/renovate-config"], - "labels": ["dependencies", "team infra", "no issue required"], + "labels": ["dependencies", "team infra", "renovate"], + "prBodyNotes": ["Related to #12126"], "semanticCommits": "disabled", "commitMessagePrefix": ":arrow_up:", "commitMessageAction": "Update", diff --git a/scripts/changelog.py b/scripts/changelog.py index 280ffa1f85..a33d3a95bf 100644 --- a/scripts/changelog.py +++ b/scripts/changelog.py @@ -10,6 +10,7 @@ Subcommands: boom Generate the :boom: section from `breaking change` issues security Generate the :lock: section from advisories fixed in a version cross-ref Compare merged milestone PRs against the changelog section + dev-only-bugs List milestone bugs that never reached production report Generate the anomaly report (CHANGES-ISSUES.md) Usage: @@ -20,6 +21,7 @@ Usage: python3 scripts/changelog.py security "2.19.1" python3 scripts/changelog.py security "2.19.1" --write python3 scripts/changelog.py cross-ref "2.16.0" + python3 scripts/changelog.py dev-only-bugs "2.16.0" [--apply] python3 scripts/changelog.py report "2.16.0" python3 scripts/changelog.py report "2.16.0" --changes CHANGES.md --output CHANGES-ISSUES.md @@ -43,7 +45,7 @@ GH_PY = Path(__file__).resolve().parent / "gh.py" BOOM_HEADING = "### :boom: Breaking changes & Deprecations" SECURITY_HEADING = "### :lock: Security fixes" BREAKING_LABEL = "breaking change" -EXCLUDED_LABELS = {"release blocker", "no changelog"} +EXCLUDED_LABELS = {"no changelog"} # PR label carried by the security fixes that reference their advisory in the # PR body, and the GHSA id pattern used to find that reference. @@ -615,6 +617,113 @@ def cmd_cross_ref(args: argparse.Namespace) -> None: print(f" #{pr['number']} {pr['title'][:80]}") +# ───────────────────────────────────────────── +# Subcommand: dev-only-bugs +# ───────────────────────────────────────────── + +# Bugs found and fixed while building a feature of the same, unreleased +# milestone never reached end users and stay out of the changelog. +AUTO_PARENT_TYPES = {"Feature", "Enhancement"} + +VERSION_FIELD_RE = re.compile( + r"^#{2,4}\s*(?:affected versions?|versions?|environment)[^\n]*\n(.*?)(?=^#{2,4}\s|\Z)", + re.IGNORECASE | re.MULTILINE | re.DOTALL, +) +DEV_ENV_RE = re.compile(r"\b(?:hourly|develop|staging|pre|unreleased)\b", re.IGNORECASE) +PROD_ENV_RE = re.compile( + r"\b(?:production|pro|main|saas|design\.penpot\.app|self[- ]?hosted?)\b", + re.IGNORECASE, +) +VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(?:\.(\d+))?") + + +def version_field(body: str | None) -> str: + """Return the "Affected versions" / "Environment" text of an issue body.""" + match = VERSION_FIELD_RE.search(body or "") + return " ".join(match.group(1).split()) if match else "" + + +def full_version(match: tuple[str, ...]) -> tuple[int, int, int]: + """Normalize a version regex match to (major, minor, patch).""" + major, minor, patch = match + return (int(major), int(minor), int(patch) if patch else 0) + + +def only_unreleased_envs(text: str, milestone: str) -> bool: + """True when the version text names unreleased environments only: + hourly, develop, staging, PRE or exactly this milestone. Any + production hint or other version number keeps the bug in the changelog.""" + current_match = VERSION_RE.search(milestone) + if current_match is None: + raise ValueError(f"milestone {milestone!r} is not a semver version") + current = full_version(current_match.groups()) + versions = [full_version(m) for m in VERSION_RE.findall(text)] + dev = bool(DEV_ENV_RE.search(text)) or current in versions + prod = bool(PROD_ENV_RE.search(text)) or any(v != current for v in versions) + return dev and not prod + + +def find_dev_only_bugs(milestone: str, issues: list[dict]) -> tuple[list[dict], list[dict]]: + """Split the milestone's changelog bugs that never reached production. + + ``issues`` come from ``gh.py issue`` (they carry ``body`` and + ``parent``). Bugs labelled ``release blocker`` are analyzed like any + other bug. Returns ``(auto, suggested)``: ``auto`` bugs are + sub-issues of a Feature or Enhancement of this milestone and get the + `no changelog` label; ``suggested`` ones need a human to confirm.""" + auto, suggested = [], [] + for issue in issues: + if issue.get("issue_type") != "Bug" or issue.get("state") != "CLOSED": + continue + if "no changelog" in set(issue.get("labels", [])): + continue + parent = issue.get("parent") or {} + entry = {"issue": issue["number"], "issue_title": issue.get("title", "")} + if parent.get("milestone") == milestone: + entry["reason"] = f"sub-issue of {parent.get('issue_type')} #{parent['number']} in {milestone}" + (auto if parent.get("issue_type") in AUTO_PARENT_TYPES else suggested).append(entry) + continue + # `regression` says something that worked broke, not where: it + # also marks regressions users hit in production. + if "regression" in issue.get("labels", []): + entry["reason"] = "labelled `regression`" + suggested.append(entry) + continue + text = version_field(issue.get("body")) + if only_unreleased_envs(text, milestone): + entry["reason"] = f'only seen on "{text[:80]}"' + suggested.append(entry) + return auto, suggested + + +def fetch_milestone_bugs(milestone: str) -> list[dict]: + """Closed milestone bugs with body and parent (two batched lookups).""" + issues = json.loads(run_gh_py("issues", milestone)) + numbers = [str(i["number"]) for i in issues if i.get("issue_type") == "Bug"] + return json.loads(run_gh_py("issue", *numbers)) if numbers else [] + + +def cmd_dev_only_bugs(args: argparse.Namespace) -> None: + """List bugs that never reached production; --apply labels the sure ones.""" + auto, suggested = find_dev_only_bugs(args.milestone, fetch_milestone_bugs(args.milestone)) + + print(f"Label `no changelog` ({len(auto)}):") + for e in auto: + print(f" #{e['issue']} {e['issue_title'][:70]} ({e['reason']})") + print(f"Candidates to confirm ({len(suggested)}):") + for e in suggested: + print(f" #{e['issue']} {e['issue_title'][:70]} ({e['reason']})") + + if args.apply: + for e in auto: + subprocess.run( + ["gh", "issue", "edit", str(e["issue"]), "--repo", REPO, + "--add-label", "no changelog"], + check=True, capture_output=True, + ) + print(f"Labelled {len(auto)} issues `no changelog`", file=sys.stderr) + + # ───────────────────────────────────────────── # Subcommand: report # ───────────────────────────────────────────── @@ -872,6 +981,9 @@ def cmd_report(args: argparse.Namespace) -> None: ): anomalies_d.append({"version": ver, "line": line[:100]}) + # Bugs that may never have reached production: warnings, not anomalies. + dev_auto, dev_suggested = find_dev_only_bugs(milestone, fetch_milestone_bugs(milestone)) + def fmt_ms(ms: str | None) -> str: return ms if ms else "_none_" @@ -1033,6 +1145,26 @@ def cmd_report(args: argparse.Namespace) -> None: "✅ No highlight gaps found. All released X.Y.0 versions have properly referenced :rocket: entries.\n\n" ) + if dev_auto or dev_suggested: + f.write("## Bugs that may never have reached production\n\n") + f.write( + "These are warnings, not anomalies. Bugs found while building " + "a feature of this release never affected end users. Add the " + "`no changelog` label to the ones you confirm and re-run.\n\n" + ) + if dev_auto: + f.write( + "### Sub-issues of a feature of this milestone (run `dev-only-bugs --apply`)\n\n" + ) + for e in dev_auto: + f.write(f"- {issue_link_title(e['issue'], e['issue_title'][:80])}: {e['reason']}\n") + f.write("\n") + if dev_suggested: + f.write("### Candidates to confirm\n\n") + for e in dev_suggested: + f.write(f"- {issue_link_title(e['issue'], e['issue_title'][:80])}: {e['reason']}\n") + f.write("\n") + f.write("---\n\n") f.write("## Context\n\n") f.write(f"- Milestone: **{milestone}**\n") @@ -1100,6 +1232,15 @@ def main() -> None: help="Path to the changelog file (default: CHANGES.md)") p_cross.set_defaults(func=cmd_cross_ref) + p_dev = sub.add_parser( + "dev-only-bugs", + help="List milestone bugs that never reached production", + ) + p_dev.add_argument("milestone", help="Milestone title (e.g. '2.16.0')") + p_dev.add_argument("--apply", action="store_true", + help="Label the sub-issues of a milestone feature `no changelog`") + p_dev.set_defaults(func=cmd_dev_only_bugs) + p_report = sub.add_parser( "report", help="Generate the anomaly report (CHANGES-ISSUES.md)" ) diff --git a/scripts/check-commit b/scripts/check-commit index df4672ea59..bc2fa0b6fc 100755 --- a/scripts/check-commit +++ b/scripts/check-commit @@ -13,6 +13,7 @@ Usage: ./scripts/check-commit ./scripts/check-commit --commit HEAD~1 ./scripts/check-commit -c abc1234 + ./scripts/check-commit --message-file .git/COMMIT_EDITMSG """ import argparse @@ -60,15 +61,23 @@ FORBIDDEN_TRAILER_PATTERN = re.compile(r"^\s*Co-authored-by\s*:", re.IGNORECASE) URL_PATTERN = re.compile(r"https?://\S+") +# Git appends the staged diff after this marker for `git commit -v` and +# discards everything after it during cleanup. +SCISSORS_PATTERN = re.compile( + r"^.\s------------------------ >8 ------------------------$", + re.MULTILINE, +) + # ═══════════════════════════════════════════════════════════════════════════════ # Helpers # ═══════════════════════════════════════════════════════════════════════════════ -def run_git(args): +def run_git(args, input_text=None): """Run a git command and return (returncode, stdout, stderr).""" try: result = subprocess.run( ["git"] + args, + input=input_text, capture_output=True, text=True, check=False, @@ -91,6 +100,39 @@ def get_commit_message(commit_ref): return out.rstrip("\n") +def get_message_from_file(path): + """Return the commit message stored in *path*. + + The commit-msg hook receives the message before git removes comment + lines, so strip them the same way git does before validating. + """ + try: + with open(path, encoding="utf-8") as fh: + raw = fh.read() + except (OSError, UnicodeDecodeError) as exc: + print(f"ERROR: could not read message file {path}: {exc}", file=sys.stderr) + sys.exit(1) + + scissors = SCISSORS_PATTERN.search(raw) + if scissors: + raw = raw[:scissors.start()] + + rc, out, err = run_git(["stripspace", "--strip-comments"], input_text=raw) + if rc != 0: + print( + f"ERROR: could not clean message file {path}: {err.strip()}", + file=sys.stderr, + ) + sys.exit(1) + + message = out.rstrip("\n") + if not message.strip(): + print(f"ERROR: message file {path} has no commit message", file=sys.stderr) + sys.exit(1) + + return message + + # ═══════════════════════════════════════════════════════════════════════════════ # Validators # ═══════════════════════════════════════════════════════════════════════════════ @@ -243,17 +285,26 @@ def main(): parser = argparse.ArgumentParser( description="Check a commit message against Penpot commit guidelines." ) - parser.add_argument( + source = parser.add_mutually_exclusive_group() + source.add_argument( "-c", "--commit", - default="HEAD", + metavar="REF", help="Commit to check (default: HEAD)", ) + source.add_argument( + "-m", "--message-file", + metavar="FILE", + help="Read the commit message from FILE instead of a commit", + ) args = parser.parse_args() - commit_ref = args.commit - message = get_commit_message(commit_ref) - - print(f"Checking commit {commit_ref} ...\n") + if args.message_file is not None: + message = get_message_from_file(args.message_file) + print(f"Checking commit message in {args.message_file} ...\n") + else: + commit_ref = args.commit if args.commit is not None else "HEAD" + message = get_commit_message(commit_ref) + print(f"Checking commit {commit_ref} ...\n") validators = [ ("Regex pattern", check_regex), diff --git a/scripts/gh.py b/scripts/gh.py index eebd5155fd..a94c5be8fb 100755 --- a/scripts/gh.py +++ b/scripts/gh.py @@ -15,7 +15,7 @@ Subcommands: Usage: python3 scripts/gh.py issues (default: state=closed) python3 scripts/gh.py issues "2.16.0" --state all - python3 scripts/gh.py issues "2.16.0" --exclude "release blocker,no changelog" + python3 scripts/gh.py issues "2.16.0" --exclude "no changelog" python3 scripts/gh.py issues "2.16.0" --label "bug" (include only issues with label) python3 scripts/gh.py issues "2.16.0" --label "bug,regression" --exclude "no changelog" python3 scripts/gh.py issues "2.16.0" --compare CHANGES.md @@ -752,8 +752,10 @@ GQL_ISSUE_BY_NUMBER_QUERY_ITEM = """\ number title state + body milestone {{ title }} issueType {{ name }} + parent {{ number issueType {{ name }} milestone {{ title }} }} labels(first: 20) {{ nodes {{ name }} }} assignees(first: 10) {{ nodes {{ login }} }} closedByPullRequestsReferences(first: 5) {{ nodes {{ number }} }} @@ -775,12 +777,19 @@ def issue_node_to_dict(node: dict) -> dict: """Convert a GraphQL issue node to the shared issue dict shape.""" issue_type = node.get("issueType") ms = node.get("milestone") + parent = node.get("parent") return { "number": node["number"], "title": node["title"], "state": node["state"], + "body": node.get("body"), "milestone": ms["title"] if ms else None, "issue_type": issue_type["name"] if issue_type else None, + "parent": { + "number": parent["number"], + "issue_type": (parent.get("issueType") or {}).get("name"), + "milestone": (parent.get("milestone") or {}).get("title"), + } if parent else None, "labels": [lbl["name"] for lbl in node["labels"]["nodes"]], "assignees": node_assignees(node), "closing_prs": [pr["number"] for pr in node["closedByPullRequestsReferences"]["nodes"]], @@ -1083,7 +1092,7 @@ def main() -> None: ) p_issues.add_argument( "--exclude", "--exclude-labels", - help="Comma-separated labels to exclude, e.g. 'release blocker,no changelog'" + help="Comma-separated labels to exclude, e.g. 'no changelog'" ) p_issues.add_argument( "--label", "--labels", diff --git a/scripts/test_changelog.py b/scripts/test_changelog.py index e27d3de1d2..626332a9fa 100644 --- a/scripts/test_changelog.py +++ b/scripts/test_changelog.py @@ -131,9 +131,14 @@ class IssueExcludedTests(unittest.TestCase): self.assertTrue(ch.issue_excluded(self.issue(state="OPEN"))) self.assertTrue(ch.issue_excluded(self.issue(issue_type="Task"))) self.assertTrue(ch.issue_excluded(self.issue(project_status="Rejected"))) - self.assertTrue(ch.issue_excluded(self.issue(labels=["release blocker"]))) self.assertTrue(ch.issue_excluded(self.issue(labels=["no changelog"]))) + def test_keeps_a_release_blocker_issue(self): + # `release blocker` marks work that blocks the release; it is not a + # changelog exclusion. Bugs go through dev-only-bugs, anything else is + # a human call in the pre-flight. + self.assertFalse(ch.issue_excluded(self.issue(labels=["release blocker"]))) + class RenderBoomEntryTests(unittest.TestCase): def test_entry_is_a_heading_plus_note_plus_references(self): @@ -657,5 +662,76 @@ class CmdSecurityTests(unittest.TestCase): self.assertIn("will not be listed", err) +def bug(number, body="", parent=None, labels=()): + return { + "number": number, + "title": f"Bug {number}", + "state": "CLOSED", + "issue_type": "Bug", + "labels": list(labels), + "body": body, + "parent": parent, + } + + +class OnlyUnreleasedEnvsTests(unittest.TestCase): + def check(self, text): + return ch.only_unreleased_envs(text, "2.19.0") + + def test_dev_environments(self): + for text in ["HOURLY", "https://hourly.penpot.dev/", "PRE & hourly", + "staging (current)", "develop (2.19.0 unreleased)", + "2.19.0-RC1 (develop)", "2.19.0"]: + with self.subTest(text=text): + self.assertTrue(self.check(text)) + + def test_production_hints(self): + for text in ["PRO", "pre/pro/hourly", "develop / staging / main", + "2.18.0-RC5", "2.18.1, 2.19.0-RC1", "self-hosted: 2.17.2", + "design.penpot.app and self-host"]: + with self.subTest(text=text): + self.assertFalse(self.check(text)) + + def test_no_signal(self): + for text in ["", "_No response_", "-", "development build"]: + with self.subTest(text=text): + self.assertFalse(self.check(text)) + + def test_released_patch_of_the_same_minor_is_production(self): + # 2.19.0/2.19.1 are released versions, not this milestone (2.19.2): + # any version other than the milestone itself keeps the bug in. + for text in ["2.19.0 and later", "2.19.1", "2.19.0-RC1", "since 2.16"]: + with self.subTest(text=text): + self.assertFalse(ch.only_unreleased_envs(text, "2.19.2")) + + def test_milestone_version_itself_is_unreleased(self): + for text in ["2.19.2", "2.19.2-RC1 (develop)"]: + with self.subTest(text=text): + self.assertTrue(ch.only_unreleased_envs(text, "2.19.2")) + + +class FindDevOnlyBugsTests(unittest.TestCase): + def test_splits_auto_and_suggested(self): + issues = [ + bug(1, parent={"number": 10, "issue_type": "Feature", "milestone": "2.19.0"}), + bug(2, parent={"number": 11, "issue_type": "Enhancement", "milestone": "2.19.0"}), + bug(3, parent={"number": 12, "issue_type": "EPIC", "milestone": "2.19.0"}), + bug(4, parent={"number": 13, "issue_type": "Feature", "milestone": "2.18.0"}), + bug(5, body="### Affected versions\n\nhourly\n"), + bug(6, body="### Affected versions\n\n2.18.2\n"), + bug(7, parent={"number": 10, "issue_type": "Feature", "milestone": "2.19.0"}, + labels=["no changelog"]), + {**bug(8, parent={"number": 10, "issue_type": "Feature", "milestone": "2.19.0"}), + "issue_type": "Enhancement"}, + bug(9, body="### Affected versions\n\n2.18.2\n", labels=["regression"]), + bug(10, parent={"number": 10, "issue_type": "Feature", "milestone": "2.19.0"}, + labels=["release blocker"]), + bug(11, body="### Affected versions\n\nhourly\n", labels=["release blocker"]), + ] + auto, suggested = ch.find_dev_only_bugs("2.19.0", issues) + self.assertEqual([e["issue"] for e in auto], [1, 2, 10]) + self.assertEqual([e["issue"] for e in suggested], [3, 5, 9, 11]) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/test_check_commit.py b/scripts/test_check_commit.py index 037bc8fd56..fca40ccb93 100644 --- a/scripts/test_check_commit.py +++ b/scripts/test_check_commit.py @@ -5,15 +5,20 @@ Run with: python3 scripts/test_check_commit.py -Covers the body line-wrapping validator added to enforce the commit body -wrap rule documented in .serena/memories/workflow/creating-commits.md. +Covers the commit message rules enforced by scripts/check-commit and documented +in .serena/memories/workflow/creating-commits.md. """ +import contextlib import importlib.machinery import importlib.util +import io +import os import pathlib import sys +import tempfile import unittest +import unittest.mock # Loading scripts/check-commit would otherwise emit scripts/__pycache__/. sys.dont_write_bytecode = True @@ -180,5 +185,71 @@ class SubjectRulesRegressionTests(unittest.TestCase): self.assertTrue(ok, error) +class MessageFileTests(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmpdir.cleanup) + env = unittest.mock.patch.dict( + os.environ, + { + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "core.commentChar", + "GIT_CONFIG_VALUE_0": "#", + }, + ) + env.start() + self.addCleanup(env.stop) + + def write_message(self, text): + path = pathlib.Path(self.tmpdir.name) / "COMMIT_EDITMSG" + path.write_text(text, encoding="utf-8") + return str(path) + + def test_reads_message_from_file(self): + path = self.write_message(":bug: Fix crash\n\nBody.\n") + self.assertEqual( + check_commit.get_message_from_file(path), + ":bug: Fix crash\n\nBody.", + ) + + def test_ignores_comment_lines(self): + path = self.write_message( + ":bug: Fix crash\n\nBody.\n\n" + "# Please enter the commit message for your changes.\n" + "# " + "long comment " * 20 + "\n" + ) + message = check_commit.get_message_from_file(path) + self.assertEqual(message, ":bug: Fix crash\n\nBody.") + ok, error = check_commit.check_body_line_length(message) + self.assertTrue(ok, error) + + def test_comment_only_file_exits(self): + path = self.write_message("# Nothing but a comment\n") + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + check_commit.get_message_from_file(path) + + def test_ignores_diff_after_scissors_line(self): + path = self.write_message( + ":bug: Fix crash\n\nBody.\n\n" + "# ------------------------ >8 ------------------------\n" + "diff --git a/f b/f\n" + "+" + "long diff line " * 10 + "\n" + ) + message = check_commit.get_message_from_file(path) + self.assertEqual(message, ":bug: Fix crash\n\nBody.") + + +class CommandLineTests(unittest.TestCase): + def test_empty_message_file_path_is_rejected(self): + argv = ["check-commit", "--message-file", ""] + with unittest.mock.patch.object(sys, "argv", argv): + with contextlib.redirect_stdout(io.StringIO()): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + check_commit.main() + self.assertEqual(raised.exception.code, 1) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tracebitmap.txt b/tracebitmap.txt deleted file mode 100644 index a14ad92425..0000000000 --- a/tracebitmap.txt +++ /dev/null @@ -1,28 +0,0 @@ -How to trace a bitmap to svg in multiple colors: - - -Dependencies: imagemagick, netpbm, potrace - -Generate the PPM format from PNG: - - convert download.png download.ppm - -Generate a mask file for each color of the file: - - ppmcolormask "#184d47" download.ppm > download-mask1.pbm - ppmcolormask "#66462C" download.ppm > download-mask2.pbm - -Then trace the bitmap (bitmap -> svg): - - potrace --flat -b svg download-mask1.pbm -o download1.svg - potrace --flat -b svg download-mask2.pbm -o download2.svg - -Extract a list of colors from an image: - - convert download.png +dither -colors 10 -unique-colors txt: - - -Articles: - -- http://confluence.concord.org/display/CCTR/Removing%20Antialiasing.html -- http://confluence.concord.org/display/CCTR/Tracing%20Color%20Raster%20Images.html