From 7322cd8ec5aefc406cd9604113c548b9aed8d97d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 16 Oct 2026 11:43:42 +0200 Subject: [PATCH 01/30] :books: Update changelog --- .serena/memories/scripts/gh.md | 1 + CHANGES.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.serena/memories/scripts/gh.md b/.serena/memories/scripts/gh.md index 7182120524..363dc612ea 100644 --- a/.serena/memories/scripts/gh.md +++ b/.serena/memories/scripts/gh.md @@ -119,6 +119,7 @@ python3 scripts/gh.py advisories GHSA-xvj6-fh9w-gjw7 ## Key principles +- `issues --compare` collects every `#NNNN` token in the file regardless of its link target, so a same-numbered reference to another tracker (e.g. `[Taiga #12312](...)`) or an inline prose number falsely marks that GitHub issue as already listed; verify against `changelog.py cross-ref` when a number collides. - Changelog-specific checks (merge-status verification, milestone-vs-changelog cross-reference, anomaly report) live in `scripts/changelog.py`, which calls this script via subprocess — see `mem:scripts/changelog`. - All output is JSON — pipe into `jq` or other tools for further processing. diff --git a/CHANGES.md b/CHANGES.md index c59713b1db..3a2b8dab1c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ - Add a graceful instance drain through the /readyz endpoint [#12281](https://github.com/penpot/penpot/issues/12281) (PR: [#12269](https://github.com/penpot/penpot/pull/12269)) - Add an accumulated queue gauge to the CLIMIT metrics [#12282](https://github.com/penpot/penpot/issues/12282) (PR: [#12269](https://github.com/penpot/penpot/pull/12269)) +- Expose the backend log events as a metric by logger and level [#12312](https://github.com/penpot/penpot/issues/12312) (PR: [#12313](https://github.com/penpot/penpot/pull/12313)) ## 2.18.3 From 4b668acf96acd030979f3848f8c77f7ebfbd976f Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Fri, 16 Oct 2026 13:07:34 +0200 Subject: [PATCH 02/30] :bug: Retry saves that a busy backend refused or rolled back A save waits for the lock that another save holds on the same file. A slow save, such as one on a large file, blocks the other saves to that file. The waiting request also holds the user's only save slot on the backend, so the user's saves to other files wait behind it too. Those saves ended in answers that stopped autosave for good: - a 429 after 60s, when no save slot freed up in time; - a 504 written by the backend after 300s, with :statement-timeout, when Postgres cancelled the wait for the lock. In both cases the backend never applied the save, so sending it again is safe. transient-error? now treats them as temporary, and the save takes the retry path. A statement timeout takes 300s per attempt, so the slow failure limit still bounds it. GET requests and error reports are unchanged: the check lives in the save pipeline, not in retryable-types. Both answers were reproduced in the devenv by holding the file lock from a separate database session. --- .../workspace-state-persistence-subtleties.md | 2 +- frontend/src/app/main/data/persistence.cljs | 22 +++++-- .../data/persistence_retry_test.cljs | 62 +++++++++++++++++++ .../frontend_tests/data/persistence_test.cljs | 14 +++++ 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/.serena/memories/frontend/workspace-state-persistence-subtleties.md b/.serena/memories/frontend/workspace-state-persistence-subtleties.md index 57094e1994..2ca4855b58 100644 --- a/.serena/memories/frontend/workspace-state-persistence-subtleties.md +++ b/.serena/memories/frontend/workspace-state-persistence-subtleties.md @@ -26,7 +26,7 @@ - A terminal failure halts the queue on purpose: the edits stay queued so `wait-persisted-or-error` keeps rejecting, and `workspace/versions.cljs` relies on that to avoid snapshotting a file whose edits were never saved. `recover-persistence` lifts the halt when a file is opened again or the role changes. A transport failure still inside its retry window (`slow-retry-pending?`) is resent once (`resume-persistence true`), never released; any other halt emits `release-failed-file` (drops every queued commit of the file that failed) then `resume-persistence`. Opening a file also replays the queued edits of its version onto the loaded data (`pending-file-changes` → `restore-pending-changes` in `bundle-fetched`); a replay that throws keeps the server data and the queue. That is what unblocks the editor without a reload; because the queue is shared, dropping the failed file is also what lets other files' pending edits drain. `release-failed-file` runs at recovery time, never at failure time — clearing the error eagerly would make waiters resolve. - The split between `initialize-persistence-state` and `recover-persistence` is deliberate: initialization deliberately does NOT lift a halt, so undeliverable edits are released when the user acts, not when the view mounts. - **Every non-transport terminal save failure is permanent.** There is no "stale but valid" category: permission lost, file gone, and server-rejected changes all fail again on a resend. Losing edit permission is not temporary — the backend raises `:not-found` for it (`check-edition-permissions!` → `perms/make-check-fn`), and `:not-found` routes to the exception page. So `:save-permission-denied` (a client-side `can-edit` check) must never be resumed: replaying the save to discover that routes the user to the exception page. Release it instead. -- Save failures split transient vs terminal (`transient-error?`: the repo retryable types `:network`/`:offline`/`:bad-gateway`/`:service-unavailable`/`:gateway-error` plus `:invalid-save-response`; everything else is terminal). Terminal keeps the `:error` halt + `flash-persistence` path; transient enters a `:retrying` episode: the head commit stays queued and resends with backoff 2s/8s/20s (3 retries, then the `:error` path). A transient failure that outlasts the backoff arms `slow-retry-cycle`: one silent resend (`:recovering`) every `slow-retry-delay-ms` (30s) until a save lands, `retry-give-up-ms` (24h since `:failing-since`) passes, or the head commit collects `max-slow-failures` (3) attempts that failed after `slow-failure-ms` (90s) or more (`:slow-failures`, counted in `persistence-transient-failure` from the attempt's elapsed time): a gateway timeout may leave the backend applying the save, so a save that keeps failing slowly must not be replayed for a day, while fast failures (offline) never count; the cycle stops on `::resume-persistence`, `::initialize-persistence` and `finalize-workspace`. Leaving the workspace also emits `suspend-retries` (`:suspended` in `:persistence`, drops `:retry-token` so a pending backoff timer goes stale): no backoff, slow cycle or `online` resend runs outside the workspace; only `workspace-initialized` (not a role change) lifts it via `recover-persistence` (`opened-file?`), which also resends a `:retrying` episode whose backoff was cut short. Its toast is `errors.save-still-retrying` without a reload link; a landing save hides the failure toast (`errors/persistence-failed-tag`). Replaying a save is safe because changes are idempotent, so there is no unknown-outcome guard: a commit whose earlier request left `active-requests` is simply sent again under the same `:commit-id`; only a request still in flight blocks a resend (`:in-flight` stays silent). Tests hold the slow cycle by stubbing `rx/timer` to never fire for `slow-retry-delay-ms`, or instant timers loop forever. Retry timers carry the episode token; a superseded token stays silent. Status stays `:retrying` through re-entries (`next-status` refuses `:pending`/`:saving` from it); waiters (`wait-persisted-or-error`) wait through it and reject only on `:error`. +- Save failures split transient vs terminal (`transient-error?`: the repo retryable types `:network`/`:offline`/`:bad-gateway`/`:service-unavailable`/`:gateway-error`, plus `:invalid-save-response`, plus save-only `busy-server-error?`: `:rate-limit` (429 from rlimit/climit — a save parked on another save's file lock holds the profile's single `:update-file/by-profile` permit, so the user's saves to other files 429 after 60 s) and backend-written 504 `:server-error` with `:statement-timeout`/`:idle-in-transaction-timeout` (Postgres cancels the lock wait at 300 s, before nginx; rolled back, so resend is safe). Kept out of `rp/retryable-types` on purpose so GETs do not retry them; everything else is terminal). Terminal keeps the `:error` halt + `flash-persistence` path; transient enters a `:retrying` episode: the head commit stays queued and resends with backoff 2s/8s/20s (3 retries, then the `:error` path). A transient failure that outlasts the backoff arms `slow-retry-cycle`: one silent resend (`:recovering`) every `slow-retry-delay-ms` (30s) until a save lands, `retry-give-up-ms` (24h since `:failing-since`) passes, or the head commit collects `max-slow-failures` (3) attempts that failed after `slow-failure-ms` (90s) or more (`:slow-failures`, counted in `persistence-transient-failure` from the attempt's elapsed time): a gateway timeout may leave the backend applying the save, so a save that keeps failing slowly must not be replayed for a day, while fast failures (offline) never count; the cycle stops on `::resume-persistence`, `::initialize-persistence` and `finalize-workspace`. Leaving the workspace also emits `suspend-retries` (`:suspended` in `:persistence`, drops `:retry-token` so a pending backoff timer goes stale): no backoff, slow cycle or `online` resend runs outside the workspace; only `workspace-initialized` (not a role change) lifts it via `recover-persistence` (`opened-file?`), which also resends a `:retrying` episode whose backoff was cut short. Its toast is `errors.save-still-retrying` without a reload link; a landing save hides the failure toast (`errors/persistence-failed-tag`). Replaying a save is safe because changes are idempotent, so there is no unknown-outcome guard: a commit whose earlier request left `active-requests` is simply sent again under the same `:commit-id`; only a request still in flight blocks a resend (`:in-flight` stays silent). Tests hold the slow cycle by stubbing `rx/timer` to never fire for `slow-retry-delay-ms`, or instant timers loop forever. Retry timers carry the episode token; a superseded token stays silent. Status stays `:retrying` through re-entries (`next-status` refuses `:pending`/`:saving` from it); waiters (`wait-persisted-or-error`) wait through it and reject only on `:error`. - One reconnect notice per episode: sticky toast tagged `:persistence-reconnecting` (single-toast store, re-show replaces), hidden by tag on drain (`:saved`) and on terminal failure; recovery is silent. Header indicator has a `:retrying` state (`workspace.header.retrying`). - Resume triggers: backoff timer and the browser `online` event (guarded by `exists? js/window`; re-enters the runner only for a live `:retrying` episode). New local edits during `:retrying` only join the queue: the episode keeps its `:run-id`, so `append-commit` does not re-enter, and the live runner (still waiting on the head's `commit-persisted`) sends them after the head. Re-entering on edits would bypass the backoff and spend an attempt per batched commit. - Tests instant-trigger retries by stubbing `rx/timer` (recording delays to assert the schedule); dynamic bindings do not survive `await` continuations, so no dynamic var for delays. diff --git a/frontend/src/app/main/data/persistence.cljs b/frontend/src/app/main/data/persistence.cljs index 76031e7c85..0a857da12c 100644 --- a/frontend/src/app/main/data/persistence.cljs +++ b/frontend/src/app/main/data/persistence.cljs @@ -138,17 +138,31 @@ (#{:error :saved} status) (dissoc :run-id :last-progress-at :stall-reported)))))))) +(def ^:private busy-server-codes + "Server error codes for a database timeout that rolled the save back." + #{:statement-timeout :idle-in-transaction-timeout}) + +(defn- busy-server-error? + "True when the backend was too busy to apply the save: it refused it + (429) or a database timeout rolled it back. Resending it is safe." + [{:keys [type cause-type code]}] + (let [type (or cause-type type)] + (or (= :rate-limit type) + (and (= :server-error type) + (contains? busy-server-codes code))))) + (defn transient-error? "True when a save failure is worth retrying with backoff: a transient transport failure (from the shared `repo/retryable-types` set, which says - nothing about the edits themselves) or an unusable save response. - Everything else (auth, validation, state) is terminal and keeps the - `:error` path. Checked against both `:type` and `:cause-type` because + nothing about the edits themselves), a busy backend, or an unusable save + response. Everything else (auth, validation, state) is terminal and keeps + the `:error` path. Checked against both `:type` and `:cause-type` because `persistence-failed` wraps the original cause under `:type :persistence`." - [{:keys [type cause-type code]}] + [{:keys [type cause-type code] :as error}] (boolean (or (contains? rp/retryable-types type) (contains? rp/retryable-types cause-type) + (busy-server-error? error) (= :invalid-save-response code)))) (def ^:private retry-delays-ms diff --git a/frontend/test/frontend_tests/data/persistence_retry_test.cljs b/frontend/test/frontend_tests/data/persistence_retry_test.cljs index 6bbfbedf47..1eba1bd1b5 100644 --- a/frontend/test/frontend_tests/data/persistence_retry_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_retry_test.cljs @@ -199,6 +199,68 @@ (t/deftest ^:async cloudflare-origin-timeout-save-is-retried (await (check-gateway-response-retries 524))) +(defn- check-busy-server-response-retries + "Answers the first send with `response`, a refusal the backend wrote + because it was busy, built by the real `rp/handle-response`, and the retry + with a valid revision. The save was never applied, so it must take the + retry path and land, never halt in `:error`." + [response] + (let [calls (atom 0)] + (with-persistence + (^:async fn [{:keys [file-id failures requests store]}] + (await + (mock/with-mocks* + {rx/timer (mock/stub instant-backoff)} + (ptk/emit! store (local-commit file-id) ::dps/force-persist) + (await (async/wait-for #(and (= :saved (get-in @store [:persistence :status])) + (empty? (get-in @store [:persistence :queue]))) + (str "a busy " (:status response) " is retried and the save lands"))) + (t/is (= 2 (count @requests)) "the busy answer plus one retry") + (t/is (empty? @failures) "a retried busy answer never reaches the failure toast")))) + (fn [_ _] + (if (= 1 (swap! calls inc)) + (rp/handle-response (assoc response :uri "https://penpot.example.com/api/main/methods/update-file")) + (rx/of {:revn 1})))))) + +;; Variant: the save waited too long for a concurrency slot (no body). +(t/deftest ^:async concurrency-limit-save-is-retried + (await (check-busy-server-response-retries {:status 429 :body nil}))) + +;; Variant: the database cancelled the save while it waited for the file lock. +(t/deftest ^:async statement-timeout-save-is-retried + (await (check-busy-server-response-retries + {:status 504 + :body {:type :server-error + :code :statement-timeout + :hint "The operation took too long and was cancelled"}}))) + +;; Variant: the database ended the save's transaction after it sat idle. +(t/deftest ^:async idle-transaction-timeout-save-is-retried + (await (check-busy-server-response-retries + {:status 504 + :body {:type :server-error + :code :idle-in-transaction-timeout + :hint "The transaction was idle too long and was cancelled"}}))) + +;; Scenario: the backend fails the save for any other reason. Proves: only +;; the busy-server codes retry; a server defect stops saving at once. +(t/deftest ^:async other-server-errors-stay-terminal + (await + (with-persistence + (^:async fn [{:keys [file-id requests store]}] + (await + (mock/with-mocks* + {rx/timer (mock/stub instant-backoff)} + (ptk/emit! store (local-commit file-id) ::dps/force-persist) + (await (async/wait-for #(= :error (get-in @store [:persistence :status])) + "the save fails")) + (t/is (= :server-error (get-in @store [:persistence :error :cause-type]))) + (t/is (= 1 (count @requests)) "no retry")))) + (fn [_ _] + (rp/handle-response {:status 500 + :body {:type :server-error :code :unhandled} + :uri "https://penpot.example.com/api/main/methods/update-file"}))))) + ;; Scenario: every send fails transiently. The 2s / 8s / 20s retries fire ;; and then the failure falls through to the exact terminal path: `:error` ;; carrying the cause, queue intact, one flash. Proves: the budget bounds diff --git a/frontend/test/frontend_tests/data/persistence_test.cljs b/frontend/test/frontend_tests/data/persistence_test.cljs index 9735771305..f6555fa165 100644 --- a/frontend/test/frontend_tests/data/persistence_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_test.cljs @@ -693,6 +693,19 @@ (t/is (not (dps/transient-error? {:type :persistence :code :save-permission-denied}))) (t/is (not (dps/transient-error? {})))) +;; A busy backend refused or rolled back the save, so it is safe to resend: +;; a rate or concurrency limit, or a database timeout. Any other server +;; error stays terminal. +(t/deftest busy-server-classification + (t/is (dps/transient-error? {:type :rate-limit})) + (t/is (dps/transient-error? {:type :persistence :cause-type :rate-limit :code :rate-limited})) + (t/is (dps/transient-error? {:type :server-error :code :statement-timeout})) + (t/is (dps/transient-error? {:type :persistence :cause-type :server-error :code :statement-timeout})) + (t/is (dps/transient-error? {:type :persistence :cause-type :server-error :code :idle-in-transaction-timeout})) + (t/is (not (dps/transient-error? {:type :server-error}))) + (t/is (not (dps/transient-error? {:type :persistence :cause-type :server-error :code :unhandled}))) + (t/is (not (dps/transient-error? {:type :persistence :cause-type :validation :code :statement-timeout})))) + ;; The transient set is owned by `repo/retryable-types`: every transport ;; type retryable at the HTTP layer must also be transient for the save ;; pipeline, in both the direct and the wrapped (`persistence-failed`) @@ -988,3 +1001,4 @@ (await (async/wait-for #(not (contains? @tasks :persistence)) "the release turns the guard off")) (t/is (= :saved (get-in @store [:persistence :status]))))))) + From 781ac534e08e164afa8e26511e15cc94ea3e9e85 Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Fri, 16 Oct 2026 13:07:34 +0200 Subject: [PATCH 03/30] :sparkles: Count save failures with a save-failed audit event We only learn how often users lose work from support tickets. Each time the user is told a save failed, persistence now emits a save-failed audit event with the cause type, the code, whether the queue is still retrying and how many changes are waiting. It follows the same rule as the failure toast: a repeated failure of the slow retry cycle is not counted again. Failures handed to another handler, such as an expired session, are counted too, since they also stop the save. --- .../workspace-state-persistence-subtleties.md | 1 + frontend/src/app/main/data/persistence.cljs | 45 +++++++++++++------ .../frontend_tests/data/persistence_test.cljs | 33 ++++++++++++++ 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/.serena/memories/frontend/workspace-state-persistence-subtleties.md b/.serena/memories/frontend/workspace-state-persistence-subtleties.md index 2ca4855b58..66c19f551d 100644 --- a/.serena/memories/frontend/workspace-state-persistence-subtleties.md +++ b/.serena/memories/frontend/workspace-state-persistence-subtleties.md @@ -32,6 +32,7 @@ - Tests instant-trigger retries by stubbing `rx/timer` (recording delays to assert the schedule); dynamic bindings do not survive `await` continuations, so no dynamic var for delays. - `save-request-timeout-ms` is 7 minutes and wraps the `update-file` request in `rx/timeout`, failing it as a transient `:network` error so the retry path resends instead of blocking the queue forever. It is deliberately longer than `saving-stall-timeout-ms` (5 min) so the stalled report fires before the retry. - `sync-unload-guard` keeps `:persistence` in `st/ongoing-tasks` while edits are buffered or queued, which is what holds the browser leave-page warning on. It syncs on `::append-commit`, `::discard-commit` and `::release-failed-file` (every queue change). It is the one effect that must stay in `store.cljs`'s `omitset`, and `store.cljs`'s `ongoing-tasks` watcher is guarded by `exists? js/window` because tests run without a window. +- `persistence-failed` emits a `save-failed` audit event (`:cause-type :code :retrying :queued-commits`) whenever the user is notified (`notify-failure?`, same rule as the toast), including delegated auth/not-found failures. - Undo transactions can stay open only temporarily; timed-out pending transactions are force-committed after about 20s. Undo entries are capped at 50. - Undo/redo are ignored while a normal editor/drawing interaction is active, except grid-layout edition handles undo through this path. - After local commits and when render-wasm is active, text shapes get derived `:position-data` recomputed in a separate commit tagged `#{:position-data}`; that tag is excluded from the position-data watcher to avoid loops. diff --git a/frontend/src/app/main/data/persistence.cljs b/frontend/src/app/main/data/persistence.cljs index 0a857da12c..f7f7779478 100644 --- a/frontend/src/app/main/data/persistence.cljs +++ b/frontend/src/app/main/data/persistence.cljs @@ -14,6 +14,7 @@ [app.common.uuid :as uuid] [app.main.data.changes :as dch] [app.main.data.common :as-alias dc] + [app.main.data.event :as ev] [app.main.data.helpers :as dsh] [app.main.data.notifications :as ntf] [app.main.data.workspace :as-alias dw] @@ -313,6 +314,23 @@ (update :queue #(into (empty %) (remove dropped) %)) (update :index #(apply dissoc % dropped))))))) +(defn- notify-failure? + "False for a failed attempt of the slow cycle: it repeats a failure the + user was already told about." + [pstate] + (not (and (:recovering pstate) + (slow-retry-pending? pstate)))) + +(defn- save-failed-event + "Audit event for a save failure the user is told about, so failed saves + can be counted." + [{:keys [error queue] :as pstate}] + (ev/event {::ev/name "save-failed" + :cause-type (:cause-type error) + :code (:code error) + :retrying (slow-retry-pending? pstate) + :queued-commits (count queue)})) + (defn- persistence-failed [commit-id cause] (ptk/reify ::persistence-failed @@ -336,14 +354,16 @@ ptk/WatchEvent (watch [_ state stream] - (rx/merge - ;; The terminal toast supersedes the reconnect notice; hide it - ;; explicitly instead of relying on the single-toast replacement. - (rx/of (ptk/data-event ::error cause) - (ntf/hide :tag reconnecting-tag)) - ;; A transport failure that outlasts the backoff can still pass, so - ;; the queue keeps trying at a slow pace. - (let [pstate (:persistence state)] + (let [pstate (:persistence state)] + (rx/merge + ;; The terminal toast replaces the reconnect notice, hidden by tag. + (rx/of (ptk/data-event ::error cause) + (ntf/hide :tag reconnecting-tag)) + (if (notify-failure? pstate) + (rx/of (save-failed-event pstate)) + (rx/empty)) + ;; A transport failure that outlasts the backoff can still pass, so + ;; the queue keeps trying at a slow pace. (if (and (not (:suspended pstate)) (slow-retry-pending? pstate)) (slow-retry-cycle stream) @@ -351,13 +371,10 @@ ptk/EffectEvent (effect [_ state _] - ;; A failed slow attempt repeats a failure the user was already warned - ;; about. Report without invoking global handlers that may reload the - ;; file or navigate away before the user can recover the retained - ;; changes. + ;; Notifies without the global handlers, which may reload the file or + ;; navigate away and lose the retained changes. (let [pstate (:persistence state)] - (when-not (and (:recovering pstate) - (slow-retry-pending? pstate)) + (when (notify-failure? pstate) (errors/flash-persistence cause :retrying (slow-retry-pending? pstate))))))) (defn- persistence-transient-failure diff --git a/frontend/test/frontend_tests/data/persistence_test.cljs b/frontend/test/frontend_tests/data/persistence_test.cljs index f6555fa165..80ba7608bd 100644 --- a/frontend/test/frontend_tests/data/persistence_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_test.cljs @@ -21,6 +21,7 @@ [app.common.types.shape :as cts] [app.common.uuid :as uuid] [app.main.data.changes :as dch] + [app.main.data.event :as ev] [app.main.data.persistence :as dps] [app.main.data.render-wasm :as drw] [app.main.data.workspace :as dw] @@ -1002,3 +1003,35 @@ "the release turns the guard off")) (t/is (= :saved (get-in @store [:persistence :status]))))))) +(defn- save-failed-events + "Starts collecting the `save-failed` audit events that `store` processes." + [store] + (let [events (atom [])] + (->> (ptk/input-stream store) + (rx/filter #(= ::ev/event (ptk/type %))) + (rx/map deref) + (rx/filter #(= "save-failed" (::ev/name %))) + (rx/subs! #(swap! events conj %))) + events)) + +;; Scenario: the backend rejects a save as invalid data. Proves: the user's +;; failure toast comes with one `save-failed` audit event that records why. +(t/deftest ^:async a-failed-save-emits-one-save-failed-event + (await + (with-persistence + (^:async fn [{:keys [file-id store]}] + (let [events (save-failed-events store)] + (ptk/emit! store (local-commit file-id) ::dps/force-persist) + (await (async/wait-for #(= :error (get-in @store [:persistence :status])) + "the save fails")) + (await (async/wait-for #(seq @events) "the event is emitted")) + (await (async/settle)) + (t/is (= 1 (count @events))) + (t/is (= {:cause-type :validation + :code :referential-integrity + :retrying false + :queued-commits 1} + (select-keys (first @events) + [:cause-type :code :retrying :queued-commits]))))) + (fn [_ _] + (rx/throw (ex-info "invalid" {:type :validation :code :referential-integrity})))))) From a1e51aacba8c572894ef0b421a314eb92cdfe825 Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Fri, 16 Oct 2026 13:07:36 +0200 Subject: [PATCH 04/30] :bug: Keep the workspace open when an export fails A failed export raised the exporter error to the global handler, which replaced the workspace with the error page. Users then reloaded and lost the changes that were not saved yet. - A failed single or multiple export now shows an error toast and keeps the workspace. Exporter defects still report as unhandled. - A failed multiple export request ends its progress listeners, so the leave-page warning it turned on is released. - Before exporting, the pending changes of the exported files are sent and the export waits up to 10s for them, instead of 400ms. Exports are built from the server copy, so if some changes are still unsaved a toast warns that the export may not include them. force-persist-files-and-wait waits only for the given files, so a stuck save of another file does not delay the export. --- .../src/app/main/data/exports/assets.cljs | 106 +++++++++++------ frontend/src/app/main/data/persistence.cljs | 24 ++++ .../data/exports_assets_test.cljs | 108 +++++++++++++++++- .../frontend_tests/data/persistence_test.cljs | 67 +++++++++++ frontend/translations/en.po | 8 ++ frontend/translations/es.po | 8 ++ 6 files changed, 286 insertions(+), 35 deletions(-) diff --git a/frontend/src/app/main/data/exports/assets.cljs b/frontend/src/app/main/data/exports/assets.cljs index b1ced71ad6..d0cf4597e0 100644 --- a/frontend/src/app/main/data/exports/assets.cljs +++ b/frontend/src/app/main/data/exports/assets.cljs @@ -13,11 +13,14 @@ [app.main.data.exports.wasm :as wasm.exports] [app.main.data.helpers :as dsh] [app.main.data.modal :as modal] + [app.main.data.notifications :as ntf] [app.main.data.persistence :as dwp] + [app.main.errors :as errors] [app.main.features :as features] [app.main.repo :as rp] [app.main.store :as st] [app.util.dom :as dom] + [app.util.i18n :refer [tr]] [app.util.websocket :as ws] [beicon.v2.core :as rx] [cuerdas.core :as str] @@ -25,6 +28,37 @@ (def default-timeout 5000) +;; Longest wait for the pending changes of the exported files to be saved. +(def save-wait-timeout-ms 10000) + +(defn- export-file-ids + [exports] + (into #{} (keep :file-id) exports)) + +(defn- warn-unsaved-changes + "Warns that the export may miss the changes of `file-ids` not saved yet." + [file-ids] + (ptk/reify ::warn-unsaved-changes + ptk/WatchEvent + (watch [_ state _] + (when (dwp/unsaved-changes? state file-ids) + (rx/of (ntf/show {:content (tr "workspace.export.unsaved-changes") + :type :toast + :level :warning + :timeout 8000})))))) + +(defn- export-failed + "Shows an export failure as a toast, keeping the workspace open." + [cause] + (ptk/reify ::export-failed + ptk/EffectEvent + (effect [_ _ _] + (errors/flash :cause cause + :type (if (#{:server-error :internal} (:type (ex-data cause))) + :unhandled + :handled) + :hint (tr "errors.export-failed"))))) + (defn normalize-export [{:keys [object-id name] :as export}] (assoc export :name (if (str/blank? name) @@ -226,16 +260,16 @@ :wait true :is-wasm (wasm-export-enabled? state)})] (rx/concat - (dwp/force-persist-and-wait 400) + (dwp/force-persist-files-and-wait (export-file-ids [export]) save-wait-timeout-ms) + (rx/of (warn-unsaved-changes (export-file-ids [export]))) (->> (rp/cmd! :export params) (rx/map (fn [{:keys [filename mtype uri]}] (dom/trigger-download-uri filename mtype uri) (clear-export-state uuid/zero))) (rx/catch (fn [cause] - (rx/concat - (rx/of (clear-export-state uuid/zero)) - (rx/throw cause)))))))))))) + (rx/of (clear-export-state uuid/zero) + (export-failed cause)))))))))))) (defn request-multiple-export [{:keys [exports cmd name] @@ -246,6 +280,8 @@ ptk/WatchEvent (watch [_ state _] (let [resource-id (volatile! nil) + ;; Emits when the export request fails. + failed-s (rx/subject) profile-id (:profile-id state) ws-conn (:ws-conn state) params (cond-> @@ -263,46 +299,48 @@ (rx/map :payload) (rx/filter #(= :export-update (:type %))) (rx/filter #(= @resource-id (:resource-id %))) + (rx/take-until failed-s) (rx/share)) stopper (rx/filter #(or (= "ended" (:status %)) (= "error" (:status %))) - progress-stream)] + progress-stream) + + file-ids + (export-file-ids exports)] (swap! st/ongoing-tasks conj :export) - (rx/merge - ;; Force that all data is persisted; best effort. - (rx/of ::dwp/force-persist) + (rx/concat + (dwp/force-persist-files-and-wait file-ids save-wait-timeout-ms) + (rx/of (warn-unsaved-changes file-ids)) + (rx/merge + ;; Updates the export state with each progress message, until + ;; shortly after the export ends or fails. + (->> progress-stream + (rx/map update-export-status) + (rx/take-until (rx/delay 500 stopper)) + (rx/finalize (fn [] + (swap! st/ongoing-tasks disj :export)))) - ;; Launch the exportation process and stores the resource id - ;; locally. - (->> (rp/cmd! :export params) - (rx/map (fn [{:keys [id] :as resource}] - (vreset! resource-id id) - (initialize-export-status exports cmd resource)))) + ;; Clears the export state a while after the export ends. + (->> progress-stream + (rx/filter #(= "ended" (:status %))) + (rx/take 1) + (rx/delay default-timeout) + (rx/map #(clear-export-state @resource-id)) + (rx/take-until (rx/delay 6000 stopper))) - ;; We proceed to update the export state with incoming - ;; progress updates. We delay the stopper for give some time - ;; to update the status with ended or errored status before - ;; close the stream. - (->> progress-stream - (rx/map update-export-status) - (rx/take-until (rx/delay 500 stopper)) - (rx/finalize (fn [] - (swap! st/ongoing-tasks disj :export)))) - - ;; We hide need to hide the ui elements of the export after - ;; some interval. We also delay a little bit more the stopper - ;; for ensure that after some security time, the stream is - ;; completely closed. - (->> progress-stream - (rx/filter #(= "ended" (:status %))) - (rx/take 1) - (rx/delay default-timeout) - (rx/map #(clear-export-state @resource-id)) - (rx/take-until (rx/delay 6000 stopper))))))))) + ;; Sends the export request and keeps its resource id. It goes + ;; last so a failure reaches the progress listeners above. + (->> (rp/cmd! :export params) + (rx/map (fn [{:keys [id] :as resource}] + (vreset! resource-id id) + (initialize-export-status exports cmd resource))) + (rx/catch (fn [cause] + (rx/push! failed-s cause) + (rx/of (export-failed cause)))))))))))) (defn request-export [{:keys [exports] :as params}] diff --git a/frontend/src/app/main/data/persistence.cljs b/frontend/src/app/main/data/persistence.cljs index f7f7779478..82e9013cb5 100644 --- a/frontend/src/app/main/data/persistence.cljs +++ b/frontend/src/app/main/data/persistence.cljs @@ -109,6 +109,30 @@ ([timeout-ms] (rx/concat (rx/of ::force-persist) (wait-persisted timeout-ms)))) +(defn- files-queued? + "True when the queue in `pstate` still holds changes of any of `file-ids`." + [{:keys [queue index]} file-ids] + (boolean (some #(contains? file-ids (get-in index [% :file-id])) queue))) + +(defn unsaved-changes? + "True when any of `file-ids` has changes the server does not have yet." + [state file-ids] + (files-queued? (:persistence state) file-ids)) + +(defn force-persist-files-and-wait + "Sends the pending changes and waits until no change of `file-ids` is + queued, saving fails, or `timeout-ms` passes. Never raises; check + `unsaved-changes?` afterwards." + [file-ids timeout-ms] + (rx/concat + (rx/of ::force-persist) + (->> (rx/from-atom refs/persistence {:emit-current-value? true}) + (rx/filter #(or (= :error (:status %)) + (not (files-queued? % file-ids)))) + (rx/take 1) + (rx/ignore) + (rx/timeout timeout-ms (rx/empty))))) + (defn- next-status "Refuses downgrades: a save in progress stays :saving, a failed save stays :error until persistence is resumed, and a retrying episode stays diff --git a/frontend/test/frontend_tests/data/exports_assets_test.cljs b/frontend/test/frontend_tests/data/exports_assets_test.cljs index d4ae9edea8..0caae3d10e 100644 --- a/frontend/test/frontend_tests/data/exports_assets_test.cljs +++ b/frontend/test/frontend_tests/data/exports_assets_test.cljs @@ -9,12 +9,15 @@ [app.common.uuid :as uuid] [app.main.data.exports.assets :as de] [app.main.data.persistence :as dwp] + [app.main.errors :as errors] [app.main.repo :as repo] [app.main.store :as st] [app.util.dom :as dom] + [app.util.i18n :as i18n] [app.util.websocket :as ws] [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] + [frontend-tests.helpers.async :as async] [frontend-tests.helpers.events :as the] [frontend-tests.helpers.mock :as mock] [potok.v2.core :as ptk])) @@ -55,7 +58,7 @@ (rx/of {:filename "export.png" :mtype "image/png" :uri "blob:export"}))) - dwp/force-persist-and-wait (mock/stub (fn [_] (rx/of ::force-persisted))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/of ::force-persisted))) dom/trigger-download-uri (mock/stub (fn [& _] nil))} (fn [done'] (let [completed (fn [_state] @@ -97,3 +100,106 @@ (de/request-multiple-export {:exports exports}) :the/end))) done)))) + +(defn- export-store + "A store for the export tests that records store errors in `store-errors`." + [state store-errors] + (ptk/store {:state state :on-error #(swap! store-errors conj %)})) + +(defn- unsaved-state + "Test state whose save queue still holds a change of `file-id`." + [file-id] + (let [commit-id (uuid/next)] + (assoc (test-state) + :persistence {:queue (conj #queue [] commit-id) + :index {commit-id {:id commit-id :file-id file-id}}}))) + +;; Scenario: the exporter fails a single export. Proves: the user gets a +;; toast and stays in the workspace; the failure is not raised to the global +;; error handler, which would replace the workspace with an error page. +(t/deftest ^:async failed-simple-export-shows-a-toast-instead-of-raising + (let [store-errors (atom []) + flashed (atom []) + store (export-store (test-state) store-errors)] + (await + (mock/with-mocks* + {repo/cmd! (mock/stub (fn [_ _] + (rx/throw (ex-info "exporter failed" + {:type :server-error + :code :unhandled})))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) + errors/flash (fn [& {:keys [hint type]}] + (swap! flashed conj [hint type])) + i18n/tr (mock/stub #(str "translated:" %))} + (ptk/emit! store (de/request-simple-export {:export export})) + (await (async/wait-for #(seq @flashed) "the failure is shown")) + (await (async/settle)) + (t/is (= [["translated:errors.export-failed" :unhandled]] @flashed) + "an exporter defect reports as unhandled") + (t/is (empty? @store-errors) "nothing reaches the global error handler") + (t/is (nil? (:export @store)) "the export state is cleared"))))) + +;; Scenario: the export request of a multiple export fails. Proves: the user +;; gets a toast, and the leave-page warning the export turned on is released +;; even though no progress message will ever arrive. +(t/deftest ^:async failed-multiple-export-shows-a-toast-and-releases-the-leave-warning + (let [store-errors (atom []) + flashed (atom []) + tasks (atom #{}) + store (export-store (test-state) store-errors)] + (await + (mock/with-mocks* + {repo/cmd! (mock/stub (fn [_ _] + (rx/throw (ex-info "exporter failed" + {:type :server-error})))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) + ws/get-rcv-stream (mock/stub (fn [_] (rx/subject))) + st/ongoing-tasks tasks + errors/flash (fn [& {:keys [hint]}] (swap! flashed conj hint)) + i18n/tr (mock/stub #(str "translated:" %))} + (ptk/emit! store (de/request-multiple-export {:exports [(assoc export :enabled true)]})) + (await (async/wait-for #(seq @flashed) "the failure is shown")) + (await (async/wait-for #(not (contains? @tasks :export)) + "the leave-page warning is released")) + (t/is (= ["translated:errors.export-failed"] @flashed)) + (t/is (empty? @store-errors) "nothing reaches the global error handler"))))) + +;; Scenario: an export starts while its file has a change not saved yet. +;; Proves: the user is warned that the export may miss it, and the export +;; runs anyway. +(t/deftest ^:async export-warns-when-the-file-has-unsaved-changes + (let [file-id (uuid/next) + downloaded (atom false) + store (export-store (unsaved-state file-id) (atom []))] + (await + (mock/with-mocks* + {repo/cmd! (mock/stub (fn [_ _] + (rx/of {:filename "export.png" + :mtype "image/png" + :uri "blob:export"}))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) + dom/trigger-download-uri (mock/stub (fn [& _] (reset! downloaded true))) + i18n/tr (mock/stub #(str "translated:" %))} + (ptk/emit! store (de/request-simple-export {:export (assoc export :file-id file-id)})) + (await (async/wait-for #(true? @downloaded) "the export runs anyway")) + (t/is (= "translated:workspace.export.unsaved-changes" + (get-in @store [:notification :content]))) + (t/is (= :warning (get-in @store [:notification :level]))))))) + +;; Scenario: an export starts with every change of its file saved. Proves: +;; no warning is shown. +(t/deftest ^:async export-shows-no-warning-when-the-file-is-saved + (let [downloaded (atom false) + store (export-store (test-state) (atom []))] + (await + (mock/with-mocks* + {repo/cmd! (mock/stub (fn [_ _] + (rx/of {:filename "export.png" + :mtype "image/png" + :uri "blob:export"}))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) + dom/trigger-download-uri (mock/stub (fn [& _] (reset! downloaded true)))} + (ptk/emit! store (de/request-simple-export {:export (assoc export :file-id (uuid/next))})) + (await (async/wait-for #(true? @downloaded) "the export runs")) + (await (async/settle)) + (t/is (nil? (:notification @store))))))) diff --git a/frontend/test/frontend_tests/data/persistence_test.cljs b/frontend/test/frontend_tests/data/persistence_test.cljs index 80ba7608bd..32150ad8dd 100644 --- a/frontend/test/frontend_tests/data/persistence_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_test.cljs @@ -26,6 +26,7 @@ [app.main.data.render-wasm :as drw] [app.main.data.workspace :as dw] [app.main.errors :as errors] + [app.main.refs :as refs] [app.main.repo :as rp] [app.main.store :as st] [app.util.i18n :as i18n] @@ -33,6 +34,7 @@ [cljs.test :as t :include-macros true] [frontend-tests.helpers.async :as async] [frontend-tests.helpers.mock :as mock] + [okulary.core :as l] [potok.v2.core :as ptk])) (t/deftest reopening-restores-only-the-retained-edits-of-the-loaded-file @@ -1003,6 +1005,71 @@ "the release turns the guard off")) (t/is (= :saved (get-in @store [:persistence :status]))))))) +(defn- wait-files-saved + "Emits into `store` an event that runs `force-persist-files-and-wait` for + `file-ids` and then flips `done`." + [store file-ids timeout-ms done] + (ptk/emit! store + (ptk/reify ::wait-files-saved + ptk/WatchEvent + (watch [_ _ _] + (rx/concat + (dps/force-persist-files-and-wait file-ids timeout-ms) + (rx/of (ptk/reify ::files-saved + ptk/EffectEvent + (effect [_ _ _] (reset! done true))))))))) + +;; Scenario: an edit waits in the buffer when an export asks for its file +;; to be saved. Proves: the wait sends the edit and only ends once the save +;; lands. +(t/deftest ^:async waiting-for-a-file-sends-and-waits-for-its-edits + (let [done (atom false)] + (await + (with-persistence + (^:async fn [{:keys [file-id response requests store]}] + (await + (mock/with-mocks* + {refs/persistence (l/derived :persistence store)} + (ptk/emit! store (local-commit file-id)) + (wait-files-saved store #{file-id} 60000 done) + (await (async/wait-for #(= 1 (count @requests)) "the edit is sent at once")) + (await (async/settle)) + (t/is (false? @done) "the wait holds while the save is in flight") + (rx/push! response {:revn 1}) + (await (async/wait-for #(true? @done) "the landed save ends the wait")) + (t/is (not (dps/unsaved-changes? @store #{file-id})))))))))) + +;; Scenario: a save of one file hangs while another file is exported. +;; Proves: the wait only covers the exported files. +(t/deftest ^:async waiting-for-a-file-ignores-other-files + (let [done (atom false)] + (await + (with-persistence + (^:async fn [{:keys [file-id requests store]}] + (await + (mock/with-mocks* + {refs/persistence (l/derived :persistence store)} + (ptk/emit! store (local-commit file-id) ::dps/force-persist) + (await (async/wait-for #(= 1 (count @requests)) "the other file's save hangs")) + (wait-files-saved store #{(uuid/next)} 60000 done) + (await (async/wait-for #(true? @done) "the wait ends at once"))))))))) + +;; Scenario: the file's save hangs past the wait. Proves: the wait gives up +;; without raising, and the file still reports unsaved changes. +(t/deftest ^:async waiting-for-a-file-gives-up-after-the-timeout + (let [done (atom false)] + (await + (with-persistence + (^:async fn [{:keys [file-id requests store]}] + (await + (mock/with-mocks* + {refs/persistence (l/derived :persistence store)} + (ptk/emit! store (local-commit file-id)) + (wait-files-saved store #{file-id} 1 done) + (await (async/wait-for #(true? @done) "the wait times out")) + (t/is (= 1 (count @requests))) + (t/is (dps/unsaved-changes? @store #{file-id}))))))))) + (defn- save-failed-events "Starts collecting the `save-failed` audit events that `store` processes." [store] diff --git a/frontend/translations/en.po b/frontend/translations/en.po index e08161f63b..4269991302 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -1589,6 +1589,10 @@ msgstr "The email «%s» has many permanent bounce reports." msgid "errors.email-spam-or-permanent-bounces" msgstr "The email «%s» has been reported as spam or permanently bounce." +#: src/app/main/data/exports/assets.cljs +msgid "errors.export-failed" +msgstr "The export failed. Try again in a moment." + #: src/app/main/errors.cljs:391 msgid "errors.feature-mismatch" msgstr "" @@ -6919,6 +6923,10 @@ msgstr "Remove guide" msgid "workspace.debug.title" msgstr "Debugging tools" +#: src/app/main/data/exports/assets.cljs +msgid "workspace.export.unsaved-changes" +msgstr "Some of your latest changes are not saved yet, so this export may not include them." + #: src/app/main/ui/workspace/sidebar/layers.cljs:841 msgid "workspace.focus.focus-mode" msgstr "Focus mode" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index 3807fd81d0..2c807ed06f 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -1573,6 +1573,10 @@ msgstr "El correo electrónico «%s» tiene varios reportes de rebote permanente msgid "errors.email-spam-or-permanent-bounces" msgstr "El email «%s» tiene reportes de spam o de rebote permanente." +#: src/app/main/data/exports/assets.cljs +msgid "errors.export-failed" +msgstr "La exportación ha fallado. Inténtalo de nuevo en un momento." + #: src/app/main/errors.cljs:391 msgid "errors.feature-mismatch" msgstr "" @@ -6757,6 +6761,10 @@ msgstr "Duplicar fila" msgid "workspace.debug.title" msgstr "Herramientas de depuración" +#: src/app/main/data/exports/assets.cljs +msgid "workspace.export.unsaved-changes" +msgstr "Algunos de tus últimos cambios aún no se han guardado, así que puede que esta exportación no los incluya." + #: src/app/main/ui/workspace/sidebar/layers.cljs:841 msgid "workspace.focus.focus-mode" msgstr "Modo foco" From a3fc7c84652fc03517a576b8dc5c40d9fd28994c Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Fri, 16 Oct 2026 13:07:37 +0200 Subject: [PATCH 05/30] :sparkles: Save before downloading a .penpot file and warn if behind The .penpot download is built from the server copy. Users who downloaded a file to keep their work while saves were failing got an old or empty file, with no warning. Opening the download dialog now sends the pending changes of the files and waits up to 10s for them. If some are still not saved, the dialog says the download will not include them. --- .../workspace-state-persistence-subtleties.md | 1 + frontend/src/app/main/data/exports/files.cljs | 40 ++++++++--- frontend/src/app/main/ui/exports/files.cljs | 9 ++- frontend/src/app/main/ui/exports/files.scss | 4 ++ .../data/exports_files_test.cljs | 72 +++++++++++++++++++ frontend/test/frontend_tests/runner.cljs | 2 + frontend/translations/en.po | 4 ++ frontend/translations/es.po | 4 ++ 8 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 frontend/test/frontend_tests/data/exports_files_test.cljs diff --git a/.serena/memories/frontend/workspace-state-persistence-subtleties.md b/.serena/memories/frontend/workspace-state-persistence-subtleties.md index 66c19f551d..33e977bee9 100644 --- a/.serena/memories/frontend/workspace-state-persistence-subtleties.md +++ b/.serena/memories/frontend/workspace-state-persistence-subtleties.md @@ -32,6 +32,7 @@ - Tests instant-trigger retries by stubbing `rx/timer` (recording delays to assert the schedule); dynamic bindings do not survive `await` continuations, so no dynamic var for delays. - `save-request-timeout-ms` is 7 minutes and wraps the `update-file` request in `rx/timeout`, failing it as a transient `:network` error so the retry path resends instead of blocking the queue forever. It is deliberately longer than `saving-stall-timeout-ms` (5 min) so the stalled report fires before the retry. - `sync-unload-guard` keeps `:persistence` in `st/ongoing-tasks` while edits are buffered or queued, which is what holds the browser leave-page warning on. It syncs on `::append-commit`, `::discard-commit` and `::release-failed-file` (every queue change). It is the one effect that must stay in `store.cljs`'s `omitset`, and `store.cljs`'s `ongoing-tasks` watcher is guarded by `exists? js/window` because tests run without a window. +- Exports and the `.penpot` download call `force-persist-files-and-wait file-ids timeout` (waits only for those files, ends on `:error` or timeout, never raises) and then `unsaved-changes? state file-ids` to warn; it reads `refs/persistence`, so tests mock it with `(l/derived :persistence store)`. - `persistence-failed` emits a `save-failed` audit event (`:cause-type :code :retrying :queued-commits`) whenever the user is notified (`notify-failure?`, same rule as the toast), including delegated auth/not-found failures. - Undo transactions can stay open only temporarily; timed-out pending transactions are force-committed after about 20s. Undo entries are capped at 50. - Undo/redo are ignored while a normal editor/drawing interaction is active, except grid-layout edition handles undo through this path. diff --git a/frontend/src/app/main/data/exports/files.cljs b/frontend/src/app/main/data/exports/files.cljs index 73917747be..fb12f6c5d3 100644 --- a/frontend/src/app/main/data/exports/files.cljs +++ b/frontend/src/app/main/data/exports/files.cljs @@ -11,6 +11,7 @@ [app.common.schema :as sm] [app.main.data.event :as ev] [app.main.data.modal :as modal] + [app.main.data.persistence :as dps] [app.main.repo :as rp] [app.util.sse :as sse] [beicon.v2.core :as rx] @@ -33,9 +34,25 @@ (def check-export-files (sm/check-fn schema:export-files)) +;; Longest wait for the pending changes of the downloaded files to be saved. +(def save-wait-timeout-ms 10000) + +(defn- show-export-dialog + "Opens the download dialog, flagging whether the files have changes not + saved yet." + [team-id files] + (ptk/reify ::show-export-dialog + ptk/WatchEvent + (watch [_ state _] + (rx/of (modal/show {:type ::export-files + :team-id team-id + :files files + :unsaved (dps/unsaved-changes? state (into #{} (map :id) files))}))))) + (defn open-export-dialog [files] - (let [files (check-export-files files)] + (let [files (check-export-files files) + file-ids (into #{} (map :id) files)] (ptk/reify ::export-files ptk/WatchEvent (watch [_ state _] @@ -45,16 +62,17 @@ ::ev/origin "dashboard" :format "binfile-v3" :num-files (count files)})) - (->> (rx/from files) - (rx/mapcat - (fn [file] - (->> (rp/cmd! :has-file-libraries {:file-id (:id file)}) - (rx/map #(assoc file :has-libraries %))))) - (rx/reduce conj []) - (rx/map (fn [files] - (modal/show {:type ::export-files - :team-id team-id - :files files})))))))))) + ;; The download is built from the server copy: send the pending + ;; changes first. + (rx/concat + (dps/force-persist-files-and-wait file-ids save-wait-timeout-ms) + (->> (rx/from files) + (rx/mapcat + (fn [file] + (->> (rp/cmd! :has-file-libraries {:file-id (:id file)}) + (rx/map #(assoc file :has-libraries %))))) + (rx/reduce conj []) + (rx/map (partial show-export-dialog team-id)))))))))) (defn export-files [& {:keys [type files]}] diff --git a/frontend/src/app/main/ui/exports/files.cljs b/frontend/src/app/main/ui/exports/files.cljs index 526e05cfef..121f91e1fd 100644 --- a/frontend/src/app/main/ui/exports/files.cljs +++ b/frontend/src/app/main/ui/exports/files.cljs @@ -13,6 +13,7 @@ [app.main.data.exports.files :as fexp] [app.main.data.modal :as modal] [app.main.store :as st] + [app.main.ui.ds.notifications.context-notification :refer [context-notification*]] [app.main.ui.ds.product.loader :refer [loader*]] [app.main.ui.icons :as deprecated-icon] [app.util.dom :as dom] @@ -68,7 +69,7 @@ {::mf/register modal/components ::mf/register-as ::fexp/export-files ::mf/props :obj} - [{:keys [team-id files]}] + [{:keys [team-id files unsaved]}] (let [state* (mf/use-state (partial initialize-state files)) has-libs? (some :has-libraries files) @@ -124,6 +125,12 @@ [:button {:class (stl/css :modal-close-btn) :on-click on-cancel} deprecated-icon/close]] + (when unsaved + [:> context-notification* {:level :warning + :appearance :ghost + :class (stl/css :unsaved-warning)} + (tr "files-download-modal.unsaved-changes")]) + (cond (= status :prepare) [:* diff --git a/frontend/src/app/main/ui/exports/files.scss b/frontend/src/app/main/ui/exports/files.scss index 8959807951..83a206f288 100644 --- a/frontend/src/app/main/ui/exports/files.scss +++ b/frontend/src/app/main/ui/exports/files.scss @@ -35,6 +35,10 @@ @extend %modal-close-btn-base; } +.unsaved-warning { + margin-block-end: var(--sp-xxl); +} + .modal-content { @include deprecated.body-small-typography; diff --git a/frontend/test/frontend_tests/data/exports_files_test.cljs b/frontend/test/frontend_tests/data/exports_files_test.cljs new file mode 100644 index 0000000000..fa545900d6 --- /dev/null +++ b/frontend/test/frontend_tests/data/exports_files_test.cljs @@ -0,0 +1,72 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns frontend-tests.data.exports-files-test + "Tests for opening the `.penpot` download dialog: the pending changes are + sent first, and the dialog is told when some are still not saved." + (:require + [app.common.uuid :as uuid] + [app.main.data.exports.files :as fexp] + [app.main.data.modal :as modal] + [app.main.data.persistence :as dps] + [app.main.repo :as rp] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock] + [potok.v2.core :as ptk])) + +(defn- file-param + [file-id] + {:id file-id :name "File" :project-id (uuid/next) :is-shared false}) + +(defn- with-queued-change + "Adds to `state` a save queue that still holds a change of `file-id`." + [state file-id] + (let [commit-id (uuid/next)] + (assoc state :persistence {:queue (conj #queue [] commit-id) + :index {commit-id {:id commit-id :file-id file-id}}}))) + +(defn- ^:async dialog-props + "Opens the download dialog for `file-id` from `state` and returns the + props the dialog is shown with, plus the file ids the save wait got." + [state file-id] + (let [store (ptk/store {:state state :on-error #(t/is false (str %))}) + waited (atom nil)] + (await + (mock/with-mocks* + {rp/cmd! (mock/stub (fn [_ _] (rx/of false))) + dps/force-persist-files-and-wait (mock/stub (fn [file-ids _] + (reset! waited file-ids) + (rx/empty)))} + (ptk/emit! store (fexp/open-export-dialog [(file-param file-id)])) + (await (async/wait-for #(some? (get-in @store [::modal/modal :props])) + "the dialog opens")))) + (assoc (get-in @store [::modal/modal :props]) ::waited @waited))) + +;; Scenario: the user downloads a file whose changes are all saved. Proves: +;; the pending changes of that file are sent first, and the dialog shows no +;; warning. +(t/deftest ^:async download-of-a-saved-file-has-no-warning + (let [file-id (uuid/next) + props (await (dialog-props {} file-id))] + (t/is (= #{file-id} (::waited props)) "the save wait covers the downloaded file") + (t/is (false? (:unsaved props))))) + +;; Scenario: the file still has a change the server does not have after the +;; wait. Proves: the dialog is told, so it can warn that the download will +;; not include it. +(t/deftest ^:async download-of-a-file-with-unsaved-changes-warns + (let [file-id (uuid/next) + props (await (dialog-props (with-queued-change {} file-id) file-id))] + (t/is (true? (:unsaved props))))) + +;; Scenario: another file has unsaved changes. Proves: only the downloaded +;; file's changes count. +(t/deftest ^:async unsaved-changes-of-another-file-do-not-warn + (let [file-id (uuid/next) + props (await (dialog-props (with-queued-change {} (uuid/next)) file-id))] + (t/is (false? (:unsaved props))))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index e7a5859879..961e68b601 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -10,6 +10,7 @@ [frontend-tests.copy-as-svg-test] [frontend-tests.data.dashboard-test] [frontend-tests.data.exports-assets-test] + [frontend-tests.data.exports-files-test] [frontend-tests.data.fix-deleted-fonts-test] [frontend-tests.data.modal-test] [frontend-tests.data.nitrate-test] @@ -126,6 +127,7 @@ 'frontend-tests.data.repo-test 'frontend-tests.data.store-test 'frontend-tests.data.exports-assets-test + 'frontend-tests.data.exports-files-test 'frontend-tests.data.uploads-test 'frontend-tests.data.viewer-test 'frontend-tests.data.workspace-colors-test diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 4269991302..3e9bdb1ecc 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -2177,6 +2177,10 @@ msgstr "Include shared library assets in file libraries" msgid "files-download-modal.title" msgstr "Download files" +#: src/app/main/ui/exports/files.cljs +msgid "files-download-modal.unsaved-changes" +msgstr "Some of your latest changes are not saved yet, so this download will not include them." + #: src/app/main/ui/settings/password.cljs:31 msgid "generic.error" msgstr "An error has occurred" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index 2c807ed06f..4062753ac2 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -2113,6 +2113,10 @@ msgstr "Incluir librerias compartidas dentro de las librerias del fichero" msgid "files-download-modal.title" msgstr "Descargar ficheros" +#: src/app/main/ui/exports/files.cljs +msgid "files-download-modal.unsaved-changes" +msgstr "Algunos de tus últimos cambios aún no se han guardado, así que esta descarga no los incluirá." + #: src/app/main/ui/settings/password.cljs:31 msgid "generic.error" msgstr "Ha ocurrido un error" From f23eb31bd71eba191c061326360942859dc3f277 Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Fri, 16 Oct 2026 13:07:44 +0200 Subject: [PATCH 06/30] :sparkles: Send pending edits when the user leaves the tab or window Edits wait up to 3s in a buffer before they are sent. Closing the laptop or switching away right after an edit could leave that batch unsent, and a crash or a discarded tab then lost it. The batch is now sent at once when the tab is hidden (another tab, a minimized window, a closed tab or laptop) or the window loses focus (another app or a plugin iframe). The 3s window itself is unchanged. pagehide and beforeunload are left out: they come too late for an async save, and the tab is already hidden by then. --- .../workspace-state-persistence-subtleties.md | 1 + frontend/src/app/main/data/persistence.cljs | 17 ++++++++++ .../frontend_tests/data/persistence_test.cljs | 34 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/.serena/memories/frontend/workspace-state-persistence-subtleties.md b/.serena/memories/frontend/workspace-state-persistence-subtleties.md index 33e977bee9..eee6de48b8 100644 --- a/.serena/memories/frontend/workspace-state-persistence-subtleties.md +++ b/.serena/memories/frontend/workspace-state-persistence-subtleties.md @@ -32,6 +32,7 @@ - Tests instant-trigger retries by stubbing `rx/timer` (recording delays to assert the schedule); dynamic bindings do not survive `await` continuations, so no dynamic var for delays. - `save-request-timeout-ms` is 7 minutes and wraps the `update-file` request in `rx/timeout`, failing it as a transient `:network` error so the retry path resends instead of blocking the queue forever. It is deliberately longer than `saving-stall-timeout-ms` (5 min) so the stalled report fires before the retry. - `sync-unload-guard` keeps `:persistence` in `st/ongoing-tasks` while edits are buffered or queued, which is what holds the browser leave-page warning on. It syncs on `::append-commit`, `::discard-commit` and `::release-failed-file` (every queue change). It is the one effect that must stay in `store.cljs`'s `omitset`, and `store.cljs`'s `ongoing-tasks` watcher is guarded by `exists? js/window` because tests run without a window. +- `leave-signals` (document `visibilitychange`→hidden, window `blur`) emits `::force-persist` when edits are buffered, so leaving the tab/window sends the 3 s batch at once; tests mock `dps/leave-signals` BEFORE `initialize-persistence` runs. pagehide/beforeunload are too late for an async save. - Exports and the `.penpot` download call `force-persist-files-and-wait file-ids timeout` (waits only for those files, ends on `:error` or timeout, never raises) and then `unsaved-changes? state file-ids` to warn; it reads `refs/persistence`, so tests mock it with `(l/derived :persistence store)`. - `persistence-failed` emits a `save-failed` audit event (`:cause-type :code :retrying :queued-commits`) whenever the user is notified (`notify-failure?`, same rule as the toast), including delegated auth/not-found failures. - Undo transactions can stay open only temporarily; timed-out pending transactions are force-committed after about 20s. Undo entries are capped at 50. diff --git a/frontend/src/app/main/data/persistence.cljs b/frontend/src/app/main/data/persistence.cljs index 82e9013cb5..475bf9b45c 100644 --- a/frontend/src/app/main/data/persistence.cljs +++ b/frontend/src/app/main/data/persistence.cljs @@ -838,6 +838,17 @@ (swap! st/ongoing-tasks conj :persistence) (swap! st/ongoing-tasks disj :persistence))))) +(defn leave-signals + "Emits when the tab is hidden or the window loses focus: another tab or + app, a plugin iframe, a closed laptop. Empty outside the browser." + [] + (if (and (exists? js/window) (exists? js/document)) + (rx/merge + (->> (rx/from-event js/document "visibilitychange") + (rx/filter #(= "hidden" (.-visibilityState js/document)))) + (rx/from-event js/window "blur")) + (rx/empty))) + (defn initialize-persistence [] (ptk/reify ::initialize-persistence @@ -896,6 +907,12 @@ (rx/map #(ptk/data-event ::persistence-notification)) (rx/take-until stoper-s)) + ;; Sends the buffered edits when the user leaves the tab or window. + (->> (leave-signals) + (rx/filter (fn [_] @buffered?)) + (rx/map (fn [_] ::force-persist)) + (rx/take-until stoper-s)) + ;; Syncs the leave-page warning on every local commit and on every ;; queue change (append after a flush, discard after a save, ;; release of a failed file). diff --git a/frontend/test/frontend_tests/data/persistence_test.cljs b/frontend/test/frontend_tests/data/persistence_test.cljs index 32150ad8dd..44307d42c9 100644 --- a/frontend/test/frontend_tests/data/persistence_test.cljs +++ b/frontend/test/frontend_tests/data/persistence_test.cljs @@ -1005,6 +1005,40 @@ "the release turns the guard off")) (t/is (= :saved (get-in @store [:persistence :status]))))))) +;; Scenario: an edit waits in the 3 s buffer and the user switches tabs or +;; windows. Proves: leaving sends the buffered edit at once, well before +;; the buffer would (the bounded `wait-for` gives up long before 3 s). +(t/deftest ^:async leaving-the-tab-sends-buffered-edits-at-once + (let [leave-s (rx/subject)] + (await + (mock/with-mocks* + {dps/leave-signals (fn [] leave-s)} + (await + (with-persistence + (^:async fn [{:keys [file-id requests store]}] + (ptk/emit! store (local-commit file-id)) + (await (async/settle)) + (t/is (empty? @requests) "the edit waits in the buffer") + (rx/push! leave-s :hidden) + (await (async/wait-for #(= 1 (count @requests)) + "leaving sends the edit at once")) + (t/is (= 1 (count @requests)))))))))) + +;; Scenario: nothing is buffered when the user leaves. Proves: a leave +;; signal alone sends nothing. +(t/deftest ^:async leaving-with-nothing-buffered-sends-nothing + (let [leave-s (rx/subject)] + (await + (mock/with-mocks* + {dps/leave-signals (fn [] leave-s)} + (await + (with-persistence + (^:async fn [{:keys [requests store]}] + (rx/push! leave-s :hidden) + (await (async/settle)) + (t/is (empty? @requests)) + (t/is (nil? (get-in @store [:persistence :status])))))))))) + (defn- wait-files-saved "Emits into `store` an event that runs `force-persist-files-and-wait` for `file-ids` and then flips `done`." From d0f455d5218bb44ec9e0666796bdbf46c5d299e2 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 16 Oct 2026 10:31:10 +0000 Subject: [PATCH 07/30] :zap: Chunk storage-gc-deleted reads with a keyset cursor The storage-gc-deleted task used to stream all the pending rows with db/plan inside one long read-only transaction, blocking the table vacuum for the whole run and generating bloat on long deletions. Now every chunk of 25 objects is one short read-write transaction: the SELECT reads and locks the next objects after a (deleted_at, id) keyset cursor with FOR UPDATE SKIP LOCKED, deletes the database rows grouped by backend and removes their backend content. Concurrent runs never take the same chunk, and the double SELECT of the old lock-ids step is gone. A chunk that fails aborts its transaction: nothing of the chunk is deleted, its locks are released, and the run skips it; the cursor still advances to the last row read, so a failed chunk is not visited again in the same run. AI-assisted-by: glm-5.3-flash --- .serena/memories/backend/storage.md | 2 +- backend/src/app/main.clj | 9 +- backend/src/app/storage/gc_deleted.clj | 208 +++++++++++------- .../backend_tests/storage_gc_deleted_test.clj | 64 ++++++ 4 files changed, 204 insertions(+), 79 deletions(-) diff --git a/.serena/memories/backend/storage.md b/.serena/memories/backend/storage.md index 88a0c30042..0e2abd696b 100644 --- a/.serena/memories/backend/storage.md +++ b/.serena/memories/backend/storage.md @@ -25,7 +25,7 @@ - Existing object values can remain readable until physical deletion. - `:expired-at` blocks reads after the expiration time. - `del-object!` sets `deleted_at`. It does not remove backend content. -- `storage-gc-deleted` removes the database row and backend content after the deletion delay, streaming the pending rows with `db/plan` inside a read-only transaction and deleting them in chunks of 25, each chunk in its own transaction: a chunk that fails (for example, an object a foreign key still references) is logged and skipped instead of aborting the run. +- `storage-gc-deleted` removes the database row and backend content after the deletion delay, processing the pending objects in chunks of 25 with keyset pagination (`(deleted_at, id)` cursor, matching the `storage_object(deleted_at, id)` index). Each chunk is one short read-write transaction: `SELECT ... FOR UPDATE SKIP LOCKED` (so two concurrent runs never take the same chunk; no separate read transaction exists and the run does not block the table vacuum) followed by the database delete and the backend delete grouped by backend. The cursor advances to the `(deleted_at, id)` of the last row read regardless of the deletion result, so a chunk that fails is skipped and not visited again in the same run: the error aborts the transaction (nothing of the chunk is deleted, locks are released), the handler catches it inside the transaction function and rolls it back explicitly — the commit that `tx-run!` performs afterwards is a JDBC-driver no-op on the already-rolled-back connection — and the loop continues with the returned rows. Accepted quirks, both benign and self-healing: (1) with rows of several backends in one chunk, a failure after one group's physical delete leaves those rows pending with already-removed content — pending rows are excluded from reads and the next run finishes them, since the physical delete is idempotent (S3 ignores missing keys, FS uses `deleteIfExists`); (2) rows locked by a concurrent run are skipped via `SKIP LOCKED` and deferred to the next scheduled run, and the `deleted_at <= now` cutoff is sampled per chunk. It emits the `penpot_storage_gc_deleted_objects_total{backend}` counter (one increment per deleted backend group, reported only after the chunk commits, so rolled-back chunks are never counted twice); run duration is covered by the generic `penpot_tasks_timing{name="storage-gc-deleted"}` histogram and chunk failures by the log-derived `penpot_log_events_total`. Every 5000 deleted objects the run pauses 5 seconds (throttle between chunk transactions, never holding one open). - `storage-gc-touched` finds references before it sets `deleted_at`. - `objects-gc` removes deleted domain rows and touches their storage object IDs. Each proc streams its pending keys with `db/plan` inside a read-only transaction and deletes them in chunks, each chunk in its own transaction with `FOR UPDATE SKIP LOCKED`: a chunk that fails is logged and skipped instead of aborting the run. - Use `::db/reuse-conn true` with `sto/resolve` inside a database transaction. diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 221c20af3a..65e55047fe 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -105,6 +105,12 @@ ::mdef/labels ["name"] ::mdef/type :histogram} + :storage-gc-deleted-objects + {::mdef/name "penpot_storage_gc_deleted_objects_total" + ::mdef/help "Storage objects permanently deleted by storage-gc-deleted, by backend." + ::mdef/labels ["backend"] + ::mdef/type :counter} + :redis-eval-timing {::mdef/name "penpot_redis_eval_timing" ::mdef/help "Redis EVAL commands execution timings (ms)" @@ -230,7 +236,8 @@ ::sto.gc-deleted/handler {::db/pool (ig/ref ::db/pool) - ::sto/storage (ig/ref ::sto/storage)} + ::sto/storage (ig/ref ::sto/storage) + ::mtx/metrics (ig/ref ::mtx/metrics)} ::sto.gc-touched/handler {::db/pool (ig/ref ::db/pool)} diff --git a/backend/src/app/storage/gc_deleted.clj b/backend/src/app/storage/gc_deleted.clj index 8356e35b0e..197889a3fb 100644 --- a/backend/src/app/storage/gc_deleted.clj +++ b/backend/src/app/storage/gc_deleted.clj @@ -2,7 +2,7 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) KALEIDOS INC Sucursal en España SL +;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.storage.gc-deleted "A task responsible to permanently delete already marked as deleted @@ -17,110 +17,164 @@ [app.common.logging :as l] [app.common.time :as ct] [app.db :as db] + [app.metrics :as mtx] [app.storage :as sto] [app.storage.impl :as impl] [integrant.core :as ig])) -(def ^:private sql:lock-sobjects - "SELECT id FROM storage_object - WHERE id = ANY(?::uuid[]) - FOR UPDATE - SKIP LOCKED") - -(defn- lock-ids - "Perform a select before delete for proper object locking and - prevent concurrent operations and we proceed only with successfully - locked objects." - [conn ids] - (let [ids (db/create-array conn "uuid" ids)] - (->> (db/exec! conn [sql:lock-sobjects ids]) - (into #{} (map :id)) - (not-empty)))) - (def ^:private sql:delete-sobjects "DELETE FROM storage_object WHERE id = ANY(?::uuid[])") -(defn- delete-sobjects - [conn ids] - (let [ids (db/create-array conn "uuid" ids)] - (-> (db/exec-one! conn [sql:delete-sobjects ids]) - (db/get-update-count)))) - -(defn- delete-in-bulk - [cfg backend-id ids] - ;; Each chunk runs in its own transaction, on its own connection: a chunk - ;; that fails (for example, an object that a foreign key still references) is - ;; logged and skipped, and it cannot poison the cursor that feeds the rest. - (try - (db/tx-run! cfg - (fn [{:keys [::db/conn ::sto/storage]}] - (when-let [ids (lock-ids conn ids)] - (let [total (delete-sobjects conn ids)] - (-> (impl/resolve-backend storage backend-id) - (impl/del-objects-in-bulk ids)) - - (doseq [id ids] - (l/dbg :hint "permanently delete storage object" - :id (str id) - :backend (name backend-id))) - total)))) - (catch Throwable cause - (l/err :hint "unexpected error on bulk deletion" - :ids ids - :cause cause)))) +(defn- delete-by-backend + "Deletes the locked objects of one backend group: the database rows and + their backend content. Runs inside the caller's transaction. Returns the + number of deleted rows; reporting it (metrics) is the caller's job, once + the transaction has committed." + [{:keys [::db/conn ::sto/storage]} backend-id ids] + (let [ids-array (db/create-array conn "uuid" ids) + total (-> (db/exec-one! conn [sql:delete-sobjects ids-array]) + (db/get-update-count))] + (-> (impl/resolve-backend storage backend-id) + (impl/del-objects-in-bulk ids)) + (doseq [id ids] + (l/dbg :hint "permanently delete storage object" + :id (str id) + :backend (name backend-id))) + total)) (defn- group-by-backend - [items] - (d/group-by (comp keyword :backend) :id #{} items)) + [rows] + (d/group-by (comp keyword :backend) :id #{} rows)) + +(defn- process-chunk-deletion + "Deletes the locked rows of one chunk, grouped by backend. Runs inside the + caller's transaction: any failure here aborts it, so the whole chunk is + rolled back (nothing is deleted and the locks are released) and skipped by + the caller. Returns a map of backend id to deleted row count." + [tx rows] + (->> (group-by-backend rows) + (map (fn [[backend-id ids]] + [backend-id (delete-by-backend tx backend-id ids)])) + (into {}))) (def ^:private sql:get-deleted-sobjects - "SELECT s.* + "SELECT s.id, s.backend, s.deleted_at FROM storage_object AS s WHERE s.deleted_at IS NOT NULL AND s.deleted_at <= ? - ORDER BY s.deleted_at ASC") + AND (s.deleted_at, s.id) > (?, ?) + ORDER BY s.deleted_at ASC, s.id ASC + LIMIT ? + FOR UPDATE SKIP LOCKED") + +(def ^:private sql:get-first-deleted-sobjects + "SELECT s.id, s.backend, s.deleted_at + FROM storage_object AS s + WHERE s.deleted_at IS NOT NULL + AND s.deleted_at <= ? + ORDER BY s.deleted_at ASC, s.id ASC + LIMIT ? + FOR UPDATE SKIP LOCKED") (def ^:private chunk-size - "Pending objects deleted in each transaction." + "Pending objects read, locked and deleted in each transaction." 25) -(defn- pending-row - "Takes the columns the GC needs from a `db/plan` row. +(def ^:private throttle-every + "Deleted objects between throttle pauses." + 5000) - A plan row is a view over the open result set, valid only while the reduction - is running, so it cannot be stored for later: this has to happen inside the - reducing function." - [row] - {:id (:id row) - :backend (:backend row)}) +(def ^:private throttle-ms + "Pause applied every `throttle-every` deleted objects, letting the + database (and its replicas) breathe on huge backlogs." + 5000) + +(defn- throttle + "Sleeps `throttle-ms` when the deleted count crosses a `throttle-every` + boundary. Runs between chunk transactions, never holding one open." + [prev-total new-total] + (when (> (quot new-total throttle-every) + (quot prev-total throttle-every)) + (l/inf :hint "throttling storage gc" + :deleted new-total + :pause-ms throttle-ms) + (Thread/sleep (long throttle-ms)))) + +(defn- fetch-and-lock-chunk + "Reads and locks one chunk of pending objects in `(deleted_at, id)` order. + Objects already locked by another process are skipped, so two concurrent + runs never take the same chunk. Runs inside the caller's transaction." + [conn last-deleted-at last-id] + (let [[sql params] (if (some? last-deleted-at) + [sql:get-deleted-sobjects + [(ct/now) last-deleted-at last-id chunk-size]] + [sql:get-first-deleted-sobjects + [(ct/now) chunk-size]])] + (db/exec! conn (into [sql] params)))) (defn- clean-deleted - "Walks the pending objects with `db/plan`, a server-side cursor that streams - the rows instead of materialising them, and deletes them chunk by chunk. + "Processes the pending objects chunk by chunk with keyset pagination: each + chunk is one short transaction that reads and locks the next `chunk-size` + objects after the `(deleted_at, id)` cursor and deletes them (database rows + and backend content). - The rows are read in their own read-only transaction and every chunk of - `chunk-size` objects is deleted in its own one (see `delete-in-bulk`), so a - chunk that fails is logged and skipped instead of aborting the whole run, and - the work of the previous chunks is already committed." - [cfg] - (let [xform (comp (map pending-row) - (partition-all chunk-size) - (mapcat group-by-backend)) - rf (fn - ([total] total) - ([total [backend-id ids]] - (+ total (or (delete-in-bulk cfg backend-id ids) 0))))] - (db/tx-run! (assoc cfg ::db/read-only true) - (fn [{:keys [::db/conn]}] - (transduce xform rf 0 - (db/plan conn [sql:get-deleted-sobjects (ct/now)])))))) + A chunk that fails aborts its transaction: nothing of the chunk is deleted, + its locks are released, and it is logged and skipped. The cursor advances + to the `(deleted_at, id)` of the last row of each read chunk regardless of + the deletion result, so a failed chunk is not visited again in the same + run. There is no long-lived read transaction: each transaction lives only + as long as one chunk, letting the vacuum proceed between chunks. Rows + locked by a concurrent run are skipped and deferred to the next run; a + chunk that fails after a partial backend delete self-heals on the next + run (physical deletes are idempotent). Every `throttle-every` deleted + objects the run pauses `throttle-ms` (see `throttle`)." + [{:keys [::mtx/metrics] :as cfg}] + (loop [total 0 + last-deleted-at nil + last-id nil] + + (let [[rows counts] + (db/tx-run! cfg + (fn [{:keys [::db/conn] :as tx}] + (let [rows (fetch-and-lock-chunk conn last-deleted-at last-id)] + (try + [rows (process-chunk-deletion tx rows)] + (catch Throwable cause + ;; Explicit rollback, then return normally (do + ;; not rethrow): the caller needs `rows` to + ;; advance the keyset cursor past the failed + ;; chunk. The commit that `tx-run!` performs + ;; afterwards is a JDBC no-op on the already + ;; rolled-back connection. + (db/rollback! conn) + (l/err :hint "unexpected error on chunk deletion" + :ids (map :id rows) + :cause cause) + [rows {}])))))] + + ;; Reported only after the chunk commits: a rolled-back chunk leaves + ;; an empty count map, so retried rows are never counted twice. + (doseq [[backend-id n] counts] + (mtx/run! metrics + :id :storage-gc-deleted-objects + :inc n + :labels [(name backend-id)])) + + (if-let [cursor (peek rows)] + (let [new-total (+ total (reduce + 0 (vals counts)))] + (throttle total new-total) + (recur new-total + (:deleted-at cursor) + (:id cursor))) + total)))) (defmethod ig/assert-key ::handler [_ params] (assert (sto/valid-storage? (::sto/storage params)) "expect valid storage") - (assert (db/pool? (::db/pool params)) "expect valid storage")) + (assert (db/pool? (::db/pool params)) "expect valid storage") + (assert (mtx/metrics? (::mtx/metrics params)) "expect valid metrics instance")) (defmethod ig/init-key ::handler [_ cfg] diff --git a/backend/test/backend_tests/storage_gc_deleted_test.clj b/backend/test/backend_tests/storage_gc_deleted_test.clj index 9e1f87f723..ecd805af73 100644 --- a/backend/test/backend_tests/storage_gc_deleted_test.clj +++ b/backend/test/backend_tests/storage_gc_deleted_test.clj @@ -10,6 +10,7 @@ [app.common.uuid :as uuid] [app.db :as db] [app.storage :as sto] + [app.storage.impl :as impl] [backend-tests.helpers :as th] [clojure.test :as t])) @@ -91,3 +92,66 @@ (t/testing "the object of the next chunk is gone" (t/is (nil? (find-object last-id)))))) + +(t/deftest failed-batch-is-not-visited-again-in-the-same-run + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + + ;; A full first batch of deletable objects. + batch1 (mapv (fn [_] (put-object! (ct/in-past "30m"))) (range chunk-size)) + + ;; The blocked object leads a second full batch, in `deleted_at` order + ;; after the first one: the whole batch fails and must be skipped, and + ;; the cursor must not come back to any of its objects. + blocked (blocked-object! (:id file)) + fillers (mapv (fn [_] (put-object! (ct/in-past "20m"))) (range (dec chunk-size))) + + ;; Objects after the failed batch: the next read batch starts right + ;; after the `(deleted_at, id)` of the last row of the failed one. + batch3 (mapv (fn [_] (put-object! (ct/in-past "5m"))) (range 5)) + + result (th/run-task! :storage-gc-deleted {})] + + (t/testing "the first and the last batches are fully deleted" + (t/is (= (+ (count batch1) (count batch3)) (:deleted result)))) + + (t/testing "the objects of the first batch are gone" + (t/is (every? #(nil? (find-object (:id %))) batch1))) + + (t/testing "the objects of the failed batch are still pending" + (t/is (some? (find-object (:id blocked)))) + (t/is (every? #(some? (find-object (:id %))) fillers))) + + (t/testing "the objects after the failed batch are gone" + (t/is (every? #(nil? (find-object (:id %))) batch3))))) + +(t/deftest physical-failure-rolls-back-the-chunk-and-continues + "Pins the chunk rollback semantics: the physical delete of the first + chunk fails (mocked) after its rows were already deleted inside the + transaction, so the transaction must roll back (the rows stay pending) + while the run skips the failed chunk and deletes the rest." + (let [;; A full first chunk of deletable objects, older in + ;; `deleted_at` order than the rest, so the failing physical + ;; delete hits exactly this chunk. + old (mapv (fn [_] (put-object! (ct/in-past "30m"))) (range chunk-size)) + recent (mapv (fn [_] (put-object! (ct/in-past "5m"))) (range 5)) + + failing (atom true) + orig impl/del-objects-in-bulk + result (with-redefs [impl/del-objects-in-bulk + (fn [backend ids] + (if @failing + (do (reset! failing false) + (throw (ex-info "simulated backend failure" {}))) + (orig backend ids)))] + (th/run-task! :storage-gc-deleted {}))] + + (t/testing "the run finishes and only the objects after the failed chunk are deleted" + (t/is (= (count recent) (:deleted result)))) + + (t/testing "the objects of the failed chunk are still pending" + (t/is (every? #(some? (find-object (:id %))) old))) + + (t/testing "the objects after the failed chunk are gone" + (t/is (every? #(nil? (find-object (:id %))) recent))))) From da1791bf007ed84f0c75e62c7511da8afe8c8c31 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 16 Oct 2026 15:46:17 +0200 Subject: [PATCH 08/30] :paperclip: Disable temporary the storage gc job from cron --- backend/src/app/main.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 65e55047fe..be1ccfafb3 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -590,8 +590,9 @@ {:cron #penpot/cron "0 0 0 * * ?" ;; daily :task :objects-gc} - {:cron #penpot/cron "0 0 0 * * ?" ;; daily - :task :storage-gc-deleted} + ;; FIXME: temporary commented + ;; {:cron #penpot/cron "0 0 0 * * ?" ;; daily + ;; :task :storage-gc-deleted} {:cron #penpot/cron "0 0 0 * * ?" ;; daily :task :storage-gc-touched} From 16ea1ab3cd6fad142fde02e6f2bf8c1b4d02186c Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 18 Oct 2026 18:01:22 +0000 Subject: [PATCH 09/30] :wrench: Ignore generated artifacts and local root files Local scratch files and autogenerated build output were showing up in git status: - /*.html and /*.json cover local reports and token dumps at the repo root; !/package.json keeps the tracked root manifest visible to git. - /media-processor/target/ and /scripts/__pycache__/ cover build and Python cache output. - render-wasm artifacts copied into the exporter go in their own commented block. Record the exporter and render-wasm generated paths in the module memories so nobody edits or commits them by hand. AI-assisted-by: deepseek-v4.1-flash --- .gitignore | 11 +++++++++++ .serena/memories/exporter/core.md | 1 + .serena/memories/render-wasm/core.md | 2 ++ 3 files changed, 14 insertions(+) diff --git a/.gitignore b/.gitignore index 707d430b13..4ad6684de2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,10 @@ .repl opencode.json .opencode/package-lock.json +/*.html /*.jpg +/*.json +!/package.json /*.md !CHANGES.md !CONTRIBUTING.md @@ -91,7 +94,14 @@ opencode.json /playwright/.cache/ /render-wasm/target/ /render-wasm/.render-wasm-build.lock + +# render-wasm build artifacts copied into the exporter (autogenerated) +/exporter/resources/wasm/render-wasm.js +/exporter/resources/wasm/render-wasm.wasm +/exporter/src/app/wasm/shared.js + /media-processor/dist/ +/media-processor/target/ /**/node_modules /**/.yarn/* /.pnpm-store @@ -112,4 +122,5 @@ opencode.json /.ci-logs /tmp/ /.codex/ +/scripts/__pycache__/ /tools/__pycache__ diff --git a/.serena/memories/exporter/core.md b/.serena/memories/exporter/core.md index 9b7078045b..f48948d8d6 100644 --- a/.serena/memories/exporter/core.md +++ b/.serena/memories/exporter/core.md @@ -5,6 +5,7 @@ ## Layout and commands - Source: `exporter/src/`; config: `deps.edn`, `shadow-cljs.edn`, `package.json`; runtime helpers/assets: `vendor/`, `scripts/`. +- `exporter/resources/wasm/render-wasm.{js,wasm}` and `exporter/src/app/wasm/shared.js` are generated render-wasm build output, not source: they are gitignored. Never hand-edit or commit them; regenerate by rebuilding render-wasm. - From `exporter/`: setup `./scripts/setup`; watch `pnpm run watch` or `pnpm run watch:app`; production build `pnpm run build`; test bundle `pnpm run build:test`; tests `pnpm run test` or `pnpm run test:quiet`; lint `pnpm run lint:clj`; format check/fix `pnpm run check-fmt:clj` / `pnpm run fmt:clj`. - Because exporter consumes `common/`, shared file/shape/model changes may need exporter verification even when the immediate change is not under `exporter/`. - Cross-cutting testing principles and anti-patterns: `mem:testing`. diff --git a/.serena/memories/render-wasm/core.md b/.serena/memories/render-wasm/core.md index 17345b7a7d..88cf0e560e 100644 --- a/.serena/memories/render-wasm/core.md +++ b/.serena/memories/render-wasm/core.md @@ -20,6 +20,8 @@ `./build` sources `_build_env`, which sets the Emscripten paths and `EMCC_CFLAGS`. The WASM heap starts at 256 MB and uses geometric growth. +Build output is gitignored and stays untracked everywhere it lands: `frontend/resources/public/js/render-wasm.{js,wasm}` and `frontend/src/app/render_wasm/api/shared.js` from `./build`, plus `exporter/resources/wasm/render-wasm.{js,wasm}` and `exporter/src/app/wasm/shared.js`. Never commit generated JS/WASM; rebuild instead. + ## Commands From `render-wasm/`: From 49f746ddd8bf5bebcca8902b6e41f3982bebd255 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 18 Oct 2026 18:39:50 +0000 Subject: [PATCH 10/30] :books: Never commit the plan file with the implementation The implement-plan flow asked to commit the closed plan together with the code, so the resulting commit looked like it was missing a piece. State the rule in the three places that describe it: the implement-plan skill, the commit workflow memory and the agents README. A plan is session bookkeeping, not part of the change, and its absence from the commit is correct rather than a missing piece. AI-assisted-by: deepseek-v4.1-flash --- .agents/README.md | 4 ++++ .agents/skills/implement-plan/SKILL.md | 19 +++++++++++-------- .serena/memories/workflow/creating-commits.md | 6 ++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.agents/README.md b/.agents/README.md index 657ab6eac3..371444a89d 100644 --- a/.agents/README.md +++ b/.agents/README.md @@ -410,6 +410,10 @@ append-only `Review Log` (UTC ISO 8601, one line per entry): A plan approved with no changes goes `draft` → `done` directly. +A plan is never staged or committed: the `done` status and its +`Review Log` are session bookkeeping, not part of the change. Their +absence from the commit is expected. + ## 7. Connecting `gh` CLI with a token The `create-issue` and `create-pr` flows need an authenticated `gh` diff --git a/.agents/skills/implement-plan/SKILL.md b/.agents/skills/implement-plan/SKILL.md index 9d99e48a6d..e3c4b3d6cd 100644 --- a/.agents/skills/implement-plan/SKILL.md +++ b/.agents/skills/implement-plan/SKILL.md @@ -107,12 +107,15 @@ When the implementation is complete, close the plan file first: flip its with the issue URL when one exists (standalone mode, e.g. `https://github.com/penpot/penpot/issues/NNNN`); in continue/direct mode with no issue, just `done` with no invented identifier. Never -record commit hashes. Then load the **`create-commit`** skill -and follow its workflow to commit the changes together with the -closed plan file, so plan and code land in the same commit. Provide a brief summary -of what was implemented and why, the issue reference (`issue-NNNN`) when -there is one, and the model name you are running as so the -`AI-assisted-by` trailer is set correctly. +record commit hashes. + +**The plan file is never staged or committed.** It is session +bookkeeping, not part of the change: the commit carries the code and +the tests. Then load the **`create-commit`** skill and follow its +workflow to commit them. Provide a brief summary of what was +implemented and why, the issue reference (`issue-NNNN`) when there is +one, and the model name you are running as so the `AI-assisted-by` +trailer is set correctly. ### Step-by-step mode (on request) @@ -124,8 +127,8 @@ per task" — loop one task at a time: one commit per task, never two tasks in one commit. Same inputs as always: what and why, the issue reference, your model name. - After the final task, close the plan file (`Status: done`, one - `Review Log` line with the issue URL when one exists) and include - it in that last commit. + `Review Log` line with the issue URL when one exists). It is never + staged: session bookkeeping does not go into the commit. - Show the user the result (what changed, files touched, how it was verified). - WAIT for the user's confirmation before starting the next task. diff --git a/.serena/memories/workflow/creating-commits.md b/.serena/memories/workflow/creating-commits.md index a60234819f..1fd50b88de 100644 --- a/.serena/memories/workflow/creating-commits.md +++ b/.serena/memories/workflow/creating-commits.md @@ -2,6 +2,12 @@ Commit only on explicit request. Before commit: `git status`; exclude unrelated user changes. +**Never stage a plan file.** A plan is session bookkeeping, not part of the +change: the commit carries the code and the tests. The `implement-plan` flow +still closes the plan (`Status: done` plus one `Review Log` line with the issue +URL); that is not a file to ship, and its absence from the commit is correct, +not a missing piece. + Do not guess or hallucinate git author information (Name or Email). Never include the `--author` flag in git commands unless specifically instructed by the user for a unique case; assume the local environment is already configured. Allow git commit to From 91a5b3a9fb74246ee3c55d51f735ed5d71b5155d Mon Sep 17 00:00:00 2001 From: makesomethingshit Date: Mon, 19 Oct 2026 16:44:32 +0900 Subject: [PATCH 11/30] :bug: Fix token tree path handling for repeated and nested segments (#11591) * :bug: Fix token tree collapsing repeated path segments Tokens whose path contains the same segment name twice in a row (e.g. test.pointer.pointer) collapsed to a single level in the token tree. build-tree-node marked a node as a leaf when the current segment name matched the first segment of every remaining path. Because group-by-first-segment rewrites nested paths into the segment name, a still-nested remaining path was indistinguishable from a token ending at the current node, so test.pointer.pointer merged into one leaf. Mark rewritten segments with an internal ::remaining? marker and strip it when the rest-path is fully consumed; is-leaf? only accepts unmarked remaining segments as leaves. Closes #11588 AI-assisted-by: glm-5.3-flash Signed-off-by: Junsoo Choi * :bug: Preserve ancestor tokens in path trees Keep a node's token and descendants independently when imported paths contain both an ancestor and a child. Render both in the token tree and include the ancestor token in existing path-based group operations. Cover accepted same-set imports, leaf IDs, repeated segments, and internal marker removal without changing import or collision validation. AI-assisted-by: GPT-6 Signed-off-by: makesomethingshit * :recycle: Rework token tree path building Carry the segments left to descend as a ::remaining-path vector on each segment and advance it by subvec, instead of rewriting the segment :name and re-splitting it later. With that, build-tree-node finds the leaf token with a predicate over ::remaining-path and takes children from the segments that still have one, so the pre-grouping children-first sort no longer changes the result. Drop sort-children-first and its segment-has-children? helper, and document inside-path?. AI-assisted-by: deepseek-v4.1-flash * :bug: Fix and relocate the token tree tests The ancestor-and-descendant test called a function that does not exist in app.common.types.tokens-lib, so the namespace failed to load and the whole common suite errored. Use make-tokens-status-from-lib and the two-argument get-tokens-in-active-sets instead. Move the build-tree-root tests to path-names-tree-test so path-names-test stays a pure split/join unit test. AI-assisted-by: deepseek-v4.1-flash * :recycle: Refactor the token tree rendering and share path helpers Extract the token pill into a leaf-node* component and treat a node as a leaf only when it holds a token and has no children, so a node can carry both a token and its children. Make token-separator public and reuse it from the token tree and the management panel instead of local copies. Both management path filters inlined the same ancestor-or-descendant rule; extract it as token-in-path? and cover it with a unit test. AI-assisted-by: deepseek-v4.1-flash --------- Signed-off-by: Junsoo Choi Signed-off-by: makesomethingshit Co-authored-by: luisddm --- common/src/app/common/path_names.cljc | 147 +++++++------- common/src/app/common/types/tokens_lib.cljc | 8 +- .../common_tests/path_names_tree_test.cljc | 97 +++++++++ common/test/common_tests/runner.cljc | 2 + .../main/ui/workspace/tokens/management.cljs | 14 +- .../tokens/management/token_tree.cljs | 185 +++++++++++------- frontend/test/frontend_tests/runner.cljs | 2 + .../tokens/token_path_test.cljs | 39 ++++ 8 files changed, 347 insertions(+), 147 deletions(-) create mode 100644 common/test/common_tests/path_names_tree_test.cljc create mode 100644 frontend/test/frontend_tests/tokens/token_path_test.cljs diff --git a/common/src/app/common/path_names.cljc b/common/src/app/common/path_names.cljc index de96093155..cc1526eae0 100644 --- a/common/src/app/common/path_names.cljc +++ b/common/src/app/common/path_names.cljc @@ -118,7 +118,10 @@ Some naming conventions: [path] (last (split-path path))) -(defn inside-path? [child parent] +(defn inside-path? + "Returns true when `child` equals `parent` or is nested inside it, + comparing both as path segments (e.g. 'a / b / c' is inside 'a / b')." + [child parent] (let [child-path (split-path child) parent-path (split-path parent)] (and (<= (count parent-path) (count child-path)) @@ -135,90 +138,94 @@ Some naming conventions: ;; Tree building functions -------------------------------------------------- -"Build tree structure from flat list of paths" - -"`build-tree-root` is the main function to build the tree." - -"Receives a list of segments with 'name' properties representing paths, - and a separator string." -"E.g segments = [{... :name 'one/two/three'} {... :name 'one/two/four'} {... :name 'one/five'}]" - -"Transforms into a tree structure like: - [{:name 'one' - :path 'one' - :depth 0 - :leaf nil - :children [{:name 'two' - :path 'one.two' - :depth 1 - :leaf nil - :children [{... :name 'three'} {... :name 'four'}]} - {:name 'five' - :path 'one.five' - :depth 1 - :leaf {... :name 'five'} - :children nil}]}]" - -(defn- sort-by-children - "Sorts segments so that those with children come first." - [segments separator] - (sort-by (fn [segment] - (let [path (split-path (:name segment) :separator separator) - path-length (count path)] - (if (= path-length 1) - 1 - 0))) - segments)) +(defn- attach-remaining-path + "Annotates a segment with ::remaining-path: the vector of path segments + still to descend after the current level. Computed once, at the root, + from the segment :name." + [segment separator] + (assoc segment ::remaining-path (split-path (:name segment) :separator separator))) (defn- group-by-first-segment - "Groups segments by their first path segment and update segment name." - [segments separator] + "Groups segments by the first element of their ::remaining-path and + advances that path by one element. + + A segment whose ::remaining-path becomes empty is a token that ends at + the current node; a non-empty one still has descendants below it." + [segments] (reduce (fn [acc segment] - (let [[first-segment & remaining-segments] (split-path (:name segment) :separator separator) - rest-path (when (seq remaining-segments) (join-path remaining-segments :separator separator :with-spaces? false))] - (update acc first-segment (fnil conj []) - (if rest-path - (assoc segment :name rest-path) - segment)))) + (let [path (::remaining-path segment)] + (update acc (first path) (fnil conj []) + (update segment ::remaining-path subvec 1)))) {} segments)) (defn- sort-and-group-segments - "Sorts elements and groups them by their first path segment." - [segments separator] - (let [sorted (sort-by-children segments separator) - grouped (group-by-first-segment sorted separator)] - (into (sorted-map) grouped))) + "Groups elements by their first path segment, keyed in sorted order." + [segments] + (into (sorted-map) (group-by-first-segment segments))) (defn- build-tree-node "Builds a single tree node with computed children." - [segment-name remaining-segments separator parent-path depth] + [segment-name remaining-segments parent-path depth] (let [current-path (if parent-path (str parent-path "." segment-name) segment-name) - is-leaf? (and (seq remaining-segments) - (every? (fn [segment] - (let [remaining-segment-name (first (split-path (:name segment) :separator separator))] - (= segment-name remaining-segment-name))) - remaining-segments)) + leaf-segment (some (fn [segment] + (when (empty? (::remaining-path segment)) + (-> segment + (assoc :name segment-name) + (dissoc ::remaining-path)))) + remaining-segments) - leaf-segment (when is-leaf? (first remaining-segments)) - node {:name segment-name - :path current-path - :depth depth - :leaf leaf-segment - :children (when-not is-leaf? - (let [grouped-elements (sort-and-group-segments remaining-segments separator)] - (mapv (fn [[child-segment-name remaining-child-segments]] - (build-tree-node child-segment-name remaining-child-segments separator current-path (inc depth))) - grouped-elements)))}] - node)) + child-segments (filter #(seq (::remaining-path %)) remaining-segments) + + children (when (seq child-segments) + (let [grouped-elements (sort-and-group-segments child-segments)] + (mapv (fn [[child-segment-name remaining-child-segments]] + (build-tree-node child-segment-name remaining-child-segments current-path (inc depth))) + grouped-elements)))] + + {:name segment-name + :path current-path + :depth depth + :leaf leaf-segment + :children children})) + +(defn- attach-remaining-paths + "Annotates every segment with its ::remaining-path vector." + [segments separator] + (map #(attach-remaining-path % separator) segments)) + +(defn- build-root-node + "Builds a root-level tree node from a grouped element." + [[segment-name remaining-segments]] + (build-tree-node segment-name remaining-segments nil 0)) (defn build-tree-root - "Builds the root level of the tree." + "Builds the root level of the tree from a flat list of path segments. + + Receives a list of segments with a :name that represents a path, and + the separator used to split each name, e.g.: + + [{... :name 'one/two/three'} {... :name 'one/two/four'} {... :name 'one/five'}] + + Transforms them into a tree of nodes shaped like: + + [{:name 'one' + :path 'one' + :depth 0 + :leaf nil + :children [{:name 'two' + :path 'one.two' + :depth 1 + :leaf nil + :children [{... :name 'three'} {... :name 'four'}]} + {:name 'five' + :path 'one.five' + :depth 1 + :leaf {... :name 'five'} + :children nil}]}]" [segments separator] - (let [grouped-elements (sort-and-group-segments segments separator)] - (mapv (fn [[segment-name remaining-segments]] - (build-tree-node segment-name remaining-segments separator nil 0)) - grouped-elements))) + (let [grouped-elements (sort-and-group-segments (attach-remaining-paths segments separator))] + (mapv build-root-node grouped-elements))) diff --git a/common/src/app/common/types/tokens_lib.cljc b/common/src/app/common/types/tokens_lib.cljc index a99a9502d2..082da2d43e 100644 --- a/common/src/app/common/types/tokens_lib.cljc +++ b/common/src/app/common/types/tokens_lib.cljc @@ -147,7 +147,9 @@ (check-token-attrs) (map->Token))) -(def ^:private token-separator ".") +(def token-separator + "Separator used to split a token name into its path segments." + ".") (defn get-token-path [token] @@ -167,9 +169,9 @@ (rename-path node nil new-name)) ([node token new-name] (let [element (if token (:name token) (:path node)) - split-path (cpn/split-path element :separator ".") + split-path (cpn/split-path element :separator token-separator) updated-split-element-name (assoc split-path (:depth node) new-name) - new-element-path (cpn/join-path updated-split-element-name :separator "." :with-spaces? false)] + new-element-path (cpn/join-path updated-split-element-name :separator token-separator :with-spaces? false)] new-element-path))) ;; === Token Set diff --git a/common/test/common_tests/path_names_tree_test.cljc b/common/test/common_tests/path_names_tree_test.cljc new file mode 100644 index 0000000000..b39b5fc0b2 --- /dev/null +++ b/common/test/common_tests/path_names_tree_test.cljc @@ -0,0 +1,97 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.path-names-tree-test + (:require + [app.common.files.tokens :as cfo] + [app.common.path-names :as cpn] + [app.common.types.tokens-lib :as ctob] + [clojure.test :as t])) + +(defn- single-chain-terminal + "Follows the first child down a single-token tree to its terminal node." + [root] + (loop [node root] + (if (:children node) + (recur (first (:children node))) + node))) + +(defn- tree-payloads + "Collects every node and :leaf map in a tree for marker-leak checks." + [node] + (cons node + (concat (when (:leaf node) [(:leaf node)]) + (mapcat tree-payloads (:children node))))) + +(defn- tree-leaves + "Collects every token leaf from a tree." + [nodes] + (mapcat (fn [node] + (concat (when (:leaf node) [(:leaf node)]) + (tree-leaves (:children node)))) + nodes)) + +(t/deftest build-tree-root-preserves-repeated-segments + (t/testing "#11588: adjacent repeated path segments keep their depth" + (let [tree (cpn/build-tree-root [{:name "test.pointer.pointer" :id "t1"}] ".") + terminal (single-chain-terminal (first tree))] + (t/is (= "test" (-> tree first :name))) + (t/is (= "pointer" (-> tree first :children first :name))) + (t/is (nil? (-> tree first :children first :leaf))) + (t/is (= "pointer" (:name terminal))) + (t/is (= "test.pointer.pointer" (:path terminal))) + (t/is (= {:name "pointer" :id "t1"} (:leaf terminal))) + (t/is (nil? (:children terminal))) + (t/is (every? #(not (contains? % :app.common.path-names/remaining-path)) + (tree-payloads (first tree)))))) + (t/testing "control case without repeated segments is unchanged" + (let [tree (cpn/build-tree-root [{:name "test.thing.other-thing" :id "t2"}] ".") + terminal (single-chain-terminal (first tree))] + (t/is (= "test" (-> tree first :name))) + (t/is (= "thing" (-> tree first :children first :name))) + (t/is (= "other-thing" (:name terminal))) + (t/is (= {:name "other-thing" :id "t2"} (:leaf terminal))))) + (t/testing "repeated and non-repeated siblings coexist" + (let [tree (cpn/build-tree-root [{:name "test.pointer.pointer" :id "t1"} + {:name "test.thing.other-thing" :id "t2"}] ".") + children (-> tree first :children)] + (t/is (= ["pointer" "thing"] (mapv :name children))) + (t/is (= {:name "pointer" :id "t1"} (-> children first :children first :leaf))))) + (t/testing "longer repeated runs keep every depth" + (doseq [[path id] [["test.test.pointer" "t3"] ["test.test.test" "t4"] + ["a.a" "t5"] ["a.b.b" "t6"] ["a.a.a" "t8"]]] + (let [segments (cpn/split-path path :separator ".") + root (first (cpn/build-tree-root [{:name path :id id}] ".")) + terminal (single-chain-terminal root)] + (t/is (= (last segments) (:name terminal)) path) + (t/is (= (inc (count segments)) (count (tree-payloads root))) path) + (t/is (= {:name (last segments) :id id} (:leaf terminal)) path))))) + +(t/deftest build-tree-root-preserves-active-set-ancestor-and-descendant + (t/testing "#11588: an ancestor token and its descendant in the active set are both kept" + (let [tokens-lib (ctob/read-multi-set-dtcg + {"Global" {"a.a" {"$value" "#aaa" "$type" "color"} + "a.a.a" {"$value" "#bbb" "$type" "color"}} + "$themes" [] + "$metadata" {"tokenSetOrder" ["Global"] + "activeSets" ["Global"]}}) + tokens-status (cfo/make-tokens-status-from-lib tokens-lib) + active-tokens (cfo/get-tokens-in-active-sets tokens-status tokens-lib) + tree (cpn/build-tree-root (vec (vals active-tokens)) ".") + collision (-> tree first :children first) + leaves (tree-leaves tree)] + (t/is (= #{"a.a" "a.a.a"} (set (keys active-tokens)))) + (t/is (= 2 (count leaves))) + (t/is (= #{(get-in active-tokens ["a.a" :id]) + (get-in active-tokens ["a.a.a" :id])} + (set (map :id leaves)))) + (t/is (= "a.a" (:path collision))) + (t/is (= (get-in active-tokens ["a.a" :id]) + (get-in collision [:leaf :id]))) + (t/is (= (get-in active-tokens ["a.a.a" :id]) + (get-in collision [:children 0 :leaf :id]))) + (t/is (every? #(not (contains? % :app.common.path-names/remaining-path)) + (mapcat tree-payloads tree)))))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index 88e10df432..7a4c4243f4 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -80,6 +80,7 @@ [common-tests.math-test] [common-tests.media-test] [common-tests.path-names-test] + [common-tests.path-names-tree-test] [common-tests.record-test] [common-tests.schema-test] [common-tests.spec-test] @@ -184,6 +185,7 @@ 'common-tests.math-test 'common-tests.media-test 'common-tests.path-names-test + 'common-tests.path-names-tree-test 'common-tests.record-test 'common-tests.schema-test 'common-tests.spec-test diff --git a/frontend/src/app/main/ui/workspace/tokens/management.cljs b/frontend/src/app/main/ui/workspace/tokens/management.cljs index c50c8e0d58..dbf6e42980 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management.cljs @@ -26,6 +26,13 @@ [cuerdas.core :as str] [rumext.v2 :as mf])) +(defn token-in-path? + "True when `token-name` is exactly `path` or nested below it. This keeps an + ancestor token inside the operations of its own path-based group." + [path token-name] + (or (= token-name path) + (str/starts-with? token-name (str path ctob/token-separator)))) + (defn- get-sorted-token-groups "Separate token-types into groups of `empty` or `filled` depending if tokens exist for that type. Sort each group alphabetically (by their type)." @@ -130,9 +137,9 @@ (fn [tokens-filtered-by-type node] (->> tokens-filtered-by-type (filter (fn [token] - (let [token-path (cpn/split-path (:name token) :separator ".")] + (let [token-path (cpn/split-path (:name token) :separator ctob/token-separator)] (and (> (count token-path) 0) - (str/starts-with? (:name token) (str (:path node) "."))))))))) + (token-in-path? (:path node) (:name token))))))))) ;; Filter tokens by their path and return their ids filter-tokens-by-path-ids @@ -142,7 +149,8 @@ (->> selected-token-set-tokens (filter (fn [token] (let [[_ token-value] token] - (and (= (:type token-value) type) (str/starts-with? (:name token-value) (str path ".")))))) + (and (= (:type token-value) type) + (token-in-path? path (:name token-value)))))) (mapv (fn [token] (let [[_ token-value] token] (:id token-value))))))) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/token_tree.cljs b/frontend/src/app/main/ui/workspace/tokens/management/token_tree.cljs index 3a5973209a..55bbffd676 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/token_tree.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/token_tree.cljs @@ -31,8 +31,56 @@ [:on-node-context-menu {:optional true} fn?] [:can-edit {:optional true} :boolean]]) +(defn- leaf-node? + "A tree node is a leaf when it holds a token and has no children." + [node] + (and (:leaf node) (not (seq (:children node))))) + +(defn- sort-nodes + "Sorts tree nodes so that leaf tokens come before groups, sorting each + set naturally by name." + [nodes] + (let [[leafs groups] + (reduce (fn [[leafs groups] node] + (if (leaf-node? node) + [(conj leafs node) groups] + [leafs (conj groups node)])) + [[] []] + nodes)] + (concat (d/natural-sort-by :name leafs) + (d/natural-sort-by :name groups)))) + +(def ^:private schema:leaf-node + [:map + [:leaf :any] + [:tokens-lib {:optional true} :any] + [:selected-token-set-id {:optional true} :any] + [:selected-shapes :any] + [:is-selected-inside-layout {:optional true} :boolean] + [:active-theme-tokens {:optional true} :any] + [:can-edit {:optional true} :boolean] + [:on-token-pill-click {:optional true} fn?] + [:on-pill-context-menu {:optional true} fn?]]) + +(mf/defc leaf-node* + {::mf/private true + ::mf/schema schema:leaf-node} + [{:keys [leaf tokens-lib selected-token-set-id selected-shapes + is-selected-inside-layout active-theme-tokens can-edit + on-token-pill-click on-pill-context-menu]}] + (let [id (:id leaf) + token (ctob/get-token tokens-lib selected-token-set-id id)] + [:> token-pill* {:token token + :selected-shapes selected-shapes + :is-selected-inside-layout is-selected-inside-layout + :active-theme-tokens active-theme-tokens + :on-click on-token-pill-click + :can-edit can-edit + :on-context-menu on-pill-context-menu}])) + (mf/defc folder-node* - {::mf/schema schema:folder-node} + {::mf/private true + ::mf/schema schema:folder-node} [{:keys [node type folded-token-paths @@ -47,20 +95,12 @@ can-edit]}] (let [full-path (str (name type) "." (:path node)) is-folder-expanded (not (contains? (set (or folded-token-paths [])) full-path)) + children (:children node) sorted-children (mf/with-memo [children] - (let [[leafs groups] - (reduce (fn [[l g] item] - (if (:leaf item) - [(conj l item) g] - [l (conj g item)])) - [[] []] - children) - sorted-leafs (d/natural-sort-by :name leafs) - sorted-groups (d/natural-sort-by :name groups)] - (concat sorted-leafs sorted-groups))) + (sort-nodes children)) swap-folder-expanded (mf/use-fn @@ -80,40 +120,50 @@ :expanded is-folder-expanded :aria-expanded is-folder-expanded :aria-controls (str "folder-children-" (:path node)) - :is-expandable (not (:leaf node)) + :is-expandable (boolean (seq children)) :on-toggle-expand swap-folder-expanded :on-context-menu node-context-menu-prep}] (when is-folder-expanded [:div {:class (stl/css :folder-children-wrapper) :id (str "folder-children-" (:path node))} - (when (seq children) - (for [child sorted-children] - (if (not (:leaf child)) - [:ul {:class (stl/css :node-parent) - :key (:path child)} - [:> folder-node* {:type type - :node child - :folded-token-paths folded-token-paths - :selected-shapes selected-shapes - :is-selected-inside-layout is-selected-inside-layout - :active-theme-tokens active-theme-tokens - :on-token-pill-click on-token-pill-click - :on-pill-context-menu on-pill-context-menu - :on-node-context-menu on-node-context-menu - :tokens-lib tokens-lib - :can-edit can-edit - :selected-token-set-id selected-token-set-id}]] - (let [id (:id (:leaf child)) - token (ctob/get-token tokens-lib selected-token-set-id id)] - [:> token-pill* - {:key id - :token token - :selected-shapes selected-shapes - :is-selected-inside-layout is-selected-inside-layout - :active-theme-tokens active-theme-tokens - :on-click on-token-pill-click - :can-edit can-edit - :on-context-menu on-pill-context-menu}]))))])])) + (when-let [leaf (:leaf node)] + [:> leaf-node* {:key (:id leaf) + :leaf leaf + :tokens-lib tokens-lib + :selected-token-set-id selected-token-set-id + :selected-shapes selected-shapes + :is-selected-inside-layout is-selected-inside-layout + :active-theme-tokens active-theme-tokens + :can-edit can-edit + :on-token-pill-click on-token-pill-click + :on-pill-context-menu on-pill-context-menu}]) + + (for [child sorted-children] + (if (leaf-node? child) + [:> leaf-node* {:key (:id (:leaf child)) + :leaf (:leaf child) + :tokens-lib tokens-lib + :selected-token-set-id selected-token-set-id + :selected-shapes selected-shapes + :is-selected-inside-layout is-selected-inside-layout + :active-theme-tokens active-theme-tokens + :can-edit can-edit + :on-token-pill-click on-token-pill-click + :on-pill-context-menu on-pill-context-menu}] + [:ul {:class (stl/css :node-parent) + :key (:path child)} + [:> folder-node* {:type type + :node child + :folded-token-paths folded-token-paths + :selected-shapes selected-shapes + :is-selected-inside-layout is-selected-inside-layout + :active-theme-tokens active-theme-tokens + :on-token-pill-click on-token-pill-click + :on-pill-context-menu on-pill-context-menu + :on-node-context-menu on-node-context-menu + :tokens-lib tokens-lib + :can-edit can-edit + :selected-token-set-id selected-token-set-id}]]))])])) (def ^:private schema:token-tree [:map @@ -146,42 +196,35 @@ on-node-context-menu can-edit can-edit-tokens]}] - (let [separator "." - raw-tree + (let [raw-tree (mf/with-memo [tokens] - (cpn/build-tree-root tokens separator)) + (cpn/build-tree-root tokens ctob/token-separator)) - on-node-context-menu (mf/use-fn - (mf/deps can-edit-tokens on-node-context-menu) - (fn [event node] - (when can-edit-tokens - (on-node-context-menu event node)))) + ordered-nodes + (mf/with-memo [raw-tree] + (sort-nodes raw-tree)) + + on-node-context-menu + (mf/use-fn + (mf/deps can-edit-tokens on-node-context-menu) + (fn [event node] + (when can-edit-tokens + (on-node-context-menu event node))))] - ordered-nodes (mf/with-memo [raw-tree] - (let [[leafs groups] - (reduce (fn [[l g] item] - (if (:leaf item) - [(conj l item) g] - [l (conj g item)])) - [[] []] - raw-tree) - sorted-leafs (d/natural-sort-by :name leafs) - sorted-groups (d/natural-sort-by :name groups)] - (concat sorted-leafs sorted-groups)))] [:div {:class (stl/css :token-tree-wrapper)} (for [node ordered-nodes] - (if (:leaf node) - (let [token (ctob/get-token tokens-lib selected-token-set-id (get-in node [:leaf :id]))] - [:> token-pill* - {:token token - :key (:id (:leaf node)) - :selected-shapes selected-shapes - :is-selected-inside-layout is-selected-inside-layout - :active-theme-tokens active-theme-tokens - :on-click on-token-pill-click - :can-edit can-edit - :on-context-menu on-pill-context-menu}]) - ;; Render segment folder + (if (leaf-node? node) + [:> leaf-node* {:key (:id (:leaf node)) + :leaf (:leaf node) + :tokens-lib tokens-lib + :selected-token-set-id selected-token-set-id + :selected-shapes selected-shapes + :is-selected-inside-layout is-selected-inside-layout + :active-theme-tokens active-theme-tokens + :can-edit can-edit + :on-token-pill-click on-token-pill-click + :on-pill-context-menu on-pill-context-menu}] + [:ul {:class (stl/css :node-parent) :key (:path node)} [:> folder-node* {:node node diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index 7db391183b..e33ae65f8f 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -99,6 +99,7 @@ [frontend-tests.tokens.logic.tokens-status-test] [frontend-tests.tokens.style-dictionary-test] [frontend-tests.tokens.token-errors-test] + [frontend-tests.tokens.token-path-test] [frontend-tests.tokens.workspace-tokens-remap-test] [frontend-tests.ui.check-updates-test] [frontend-tests.ui.color-token-position-test] @@ -242,6 +243,7 @@ 'frontend-tests.tokens.logic.token-remapping-test 'frontend-tests.tokens.style-dictionary-test 'frontend-tests.tokens.token-errors-test + 'frontend-tests.tokens.token-path-test 'frontend-tests.tokens.logic.tokens-status-test 'frontend-tests.tokens.workspace-tokens-remap-test 'frontend-tests.ui.check-updates-test diff --git a/frontend/test/frontend_tests/tokens/token_path_test.cljs b/frontend/test/frontend_tests/tokens/token_path_test.cljs new file mode 100644 index 0000000000..06033e0882 --- /dev/null +++ b/frontend/test/frontend_tests/tokens/token_path_test.cljs @@ -0,0 +1,39 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.tokens.token-path-test + (:require + [app.main.ui.workspace.tokens.management :as wtm] + [cljs.test :as t :include-macros true])) + +;; https://github.com/penpot/penpot/issues/11588 +;; A token whose name equals a tree node path must stay in the operations of +;; that node, alongside the tokens nested below it. + +(t/deftest token-in-path-matches-the-path-itself + (t/testing "an ancestor token is in its own path" + (t/is (true? (wtm/token-in-path? "a.a" "a.a"))) + (t/is (true? (wtm/token-in-path? "a" "a"))))) + +(t/deftest token-in-path-matches-nested-tokens + (t/testing "a token nested below the path is in the path" + (t/is (true? (wtm/token-in-path? "a.a" "a.a.a"))) + (t/is (true? (wtm/token-in-path? "a" "a.b.c"))) + (t/is (true? (wtm/token-in-path? "a.a" "a.a.b.c"))))) + +(t/deftest token-in-path-rejects-non-descendants + (t/testing "a sibling is not in the path" + (t/is (false? (wtm/token-in-path? "a.a" "a.b")))) + + (t/testing "a deeper path does not match a shorter token" + (t/is (false? (wtm/token-in-path? "a.a.a" "a.a")))) + + (t/testing "a name sharing the prefix without a separator boundary is not in the path" + (t/is (false? (wtm/token-in-path? "a.a" "a.ab"))) + (t/is (false? (wtm/token-in-path? "a" "ab")))) + + (t/testing "an unrelated path is not in the path" + (t/is (false? (wtm/token-in-path? "x.y" "z"))))) From a5678657f583dcd3a3707a5ffe34753796fc27c9 Mon Sep 17 00:00:00 2001 From: Eva Marco Date: Mon, 19 Oct 2026 10:48:39 +0200 Subject: [PATCH 12/30] :bug: Fix padding on sets section (#12422) --- frontend/src/app/main/ui/workspace/tokens/sidebar.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/main/ui/workspace/tokens/sidebar.scss b/frontend/src/app/main/ui/workspace/tokens/sidebar.scss index 24e7c22f87..023fc72182 100644 --- a/frontend/src/app/main/ui/workspace/tokens/sidebar.scss +++ b/frontend/src/app/main/ui/workspace/tokens/sidebar.scss @@ -28,7 +28,7 @@ overflow-y: auto; scrollbar-gutter: stable; position: relative; - padding-block: var(--sp-xs) var(--sp-l); + padding-block-start: var(--sp-xs); } .tokens-section-wrapper { From f4f581ab74f8a4615f5ff45307ebadad9a1fbbf8 Mon Sep 17 00:00:00 2001 From: Eva Marco Date: Mon, 19 Oct 2026 10:50:32 +0200 Subject: [PATCH 13/30] :bug: Fix export scale dropdown clipped in Inspect (#12351) * :bug: Fix export scale dropdown clipped in Inspect The export selects in the Inspect tab used a dropdown class that always opened them upwards, overriding the select's own up/down choice. On elements with few properties the export section sits near the top of the panel, so the scale list (now eight options) went past the top of the scroll area and hid 0.5x. Keep only the dropdown width and let the select pick the direction. Closes #12346 AI-assisted-by: claude-opus-5-5 * :recycle: Move Inspect exports styles to DS tokens Replace the deprecated spacing and size variables, the flex mixins, the color aliases and the typography mixins with DS tokens, and use logical properties. The typographies switch to the closest DS ones: body-small for the suffix input (line-height 1.3 instead of 1.4) and headline-small for the export button (12px instead of 11px). The @extend placeholders still come from refactor/ and are left for a later migration. AI-assisted-by: claude-opus-5-5 * :recycle: Use DS buttons in Inspect exports Replace the add, remove and export buttons in the Inspect exports section with the DS icon-button and button, as the workspace export section does. Delete their now unused styles and the sizes import; the export button class only stretches the button and centers its label. AI-assisted-by: claude-opus-5-5 * :bug: Fix add export button lookup in the Inspect exports e2e test The add export button in Inspect is a DS icon button now and lost its .add-export class, so the test timed out waiting for it. Find it by its role and accessible name instead. AI-assisted-by: claude-opus-5-5 --- .../ui/specs/viewer-inspect-exports.spec.js | 5 +- frontend/src/app/main/ui/inspect/exports.cljs | 37 +++++----- frontend/src/app/main/ui/inspect/exports.scss | 73 ++++++------------- 3 files changed, 45 insertions(+), 70 deletions(-) diff --git a/frontend/playwright/ui/specs/viewer-inspect-exports.spec.js b/frontend/playwright/ui/specs/viewer-inspect-exports.spec.js index 3e99476fe5..b1da71a01f 100644 --- a/frontend/playwright/ui/specs/viewer-inspect-exports.spec.js +++ b/frontend/playwright/ui/specs/viewer-inspect-exports.spec.js @@ -23,11 +23,8 @@ test("[View mode] Export presets are preserved when navigating between boards in // Enter inspect (code) mode await viewer.showCode(); - // Wait for the inspect panel to load - await page.waitForSelector(".main_ui_inspect_exports__add-export"); - // Add an export preset via the "+" button in the Export section - const addExportButton = page.locator(".main_ui_inspect_exports__add-export"); + const addExportButton = page.getByRole("button", { name: "Add export" }); await addExportButton.click(); // Verify the "Export 1 element" button appears, confirming the preset was added diff --git a/frontend/src/app/main/ui/inspect/exports.cljs b/frontend/src/app/main/ui/inspect/exports.cljs index dab7869948..c70e37ff05 100644 --- a/frontend/src/app/main/ui/inspect/exports.cljs +++ b/frontend/src/app/main/ui/inspect/exports.cljs @@ -14,7 +14,9 @@ [app.main.store :as st] [app.main.ui.components.select :refer [select]] [app.main.ui.components.title-bar :refer [title-bar*]] - [app.main.ui.icons :as deprecated-icon] + [app.main.ui.ds.buttons.button :refer [button*]] + [app.main.ui.ds.buttons.icon-button :refer [icon-button*]] + [app.main.ui.ds.foundations.assets.icon :as i] [app.util.dom :as dom] [app.util.i18n :refer [tr c]] [app.util.keyboard :as kbd] @@ -161,17 +163,19 @@ [:> title-bar* {:collapsable false :title (tr "workspace.options.export") :class (stl/css :title-spacing-export-viewer)} - [:button {:class (stl/css :add-export) - :on-click add-export} deprecated-icon/add]]] + [:> icon-button* {:variant "ghost" + :aria-label (tr "workspace.options.export.add-export") + :on-click add-export + :icon i/add}]]] (cond (= :multiple exports) [:div {:class (stl/css :multiple-exports)} [:div {:class (stl/css :label)} (tr "settings.multiple")] [:div {:class (stl/css :actions)} - [:button {:class (stl/css :action-btn) - :on-click ()} - deprecated-icon/remove-icon]]] + [:> icon-button* {:variant "ghost" + :aria-label (tr "workspace.options.export.remove-export") + :icon i/remove}]]] (seq @exports) [:div {:class (stl/css :element-set-content)} @@ -183,14 +187,14 @@ [:& select {:default-value (d/name (:type export)) :options format-options - :dropdown-class (stl/css :dropdown-upwards) + :dropdown-class (stl/css :dropdown) :on-change (partial on-type-change index)}]] (when (scale-enabled? export) [:div {:class (stl/css :size-select)} [:& select {:default-value (str (:scale export)) :options size-options - :dropdown-class (stl/css :dropdown-upwards) + :dropdown-class (stl/css :dropdown) :on-change (partial on-scale-change index)}]]) [:label {:class (stl/css :suffix-input) :for "suffix-export-input"} @@ -203,16 +207,15 @@ :on-change on-suffix-change :on-key-down manage-key-down}]]] - [:button {:class (stl/css :action-btn) - :on-click (partial delete-export index)} - deprecated-icon/remove-icon]])]) + [:> icon-button* {:variant "ghost" + :aria-label (tr "workspace.options.export.remove-export") + :on-click (partial delete-export index) + :icon i/remove}]])]) (when (or (= :multiple exports) (seq @exports)) - [:button - {:on-click (when-not in-progress? on-download) - :class (stl/css-case - :export-btn true - :btn-disabled in-progress?) - :disabled in-progress?} + [:> button* {:variant "secondary" + :class (stl/css :export-btn) + :on-click (when-not in-progress? on-download) + :disabled in-progress?} (if in-progress? (tr "workspace.options.exporting-object") (tr "workspace.options.export-object" (c (count shapes))))])])) diff --git a/frontend/src/app/main/ui/inspect/exports.scss b/frontend/src/app/main/ui/inspect/exports.scss index 1feb282b4c..67345837ac 100644 --- a/frontend/src/app/main/ui/inspect/exports.scss +++ b/frontend/src/app/main/ui/inspect/exports.scss @@ -5,9 +5,11 @@ // Copyright (c) KALEIDOS SUBSIDIARY SL @use "refactor/common-refactor.scss" as deprecated; +@use "ds/typography.scss" as *; +@use "ds/_utils.scss" as *; .element-set { - padding-bottom: deprecated.$s-16; + padding-block-end: var(--sp-l); margin: 0; } @@ -17,33 +19,22 @@ .title-spacing-export-viewer { margin: 0; - color: var(--entry-foreground-color-hover); - margin-inline-start: calc(-1 * deprecated.$s-8); - width: calc(100% + deprecated.$s-8); -} - -.add-export { - @extend %button-tertiary; - - height: deprecated.$s-32; - width: deprecated.$s-28; - - svg { - @extend %button-icon; - - stroke: var(--icon-foreground); - } + color: var(--color-foreground-primary); + margin-inline-start: calc(-1 * var(--sp-s)); + inline-size: calc(100% + var(--sp-s)); } .element-set-content { - @include deprecated.flex-column; - - margin-bottom: deprecated.$s-4; + display: flex; + flex-direction: column; + gap: var(--sp-xs); + margin-block-end: var(--sp-xs); } .multiple-exports { - @include deprecated.flex-row; - + display: flex; + align-items: center; + gap: var(--sp-xs); grid-column: 1 / span 9; } @@ -52,24 +43,15 @@ } .actions { - @include deprecated.flex-row; + display: flex; + align-items: center; + gap: var(--sp-xs); } .element-group { display: grid; grid-template-columns: repeat(9, 1fr); - column-gap: deprecated.$s-4; - - .action-btn { - @extend %button-tertiary; - - height: deprecated.$s-32; - width: deprecated.$s-28; - - svg { - @extend %button-icon; - } - } + column-gap: var(--sp-xs); } .input-wrapper { @@ -82,10 +64,8 @@ grid-column: span 3; padding: 0; - .dropdown-upwards { - bottom: deprecated.$s-36; - width: deprecated.$s-80; - top: unset; + .dropdown { + inline-size: px2rem(80); } } @@ -93,24 +73,19 @@ grid-column: span 2; padding: 0; - .dropdown-upwards { - bottom: deprecated.$s-36; - top: unset; - width: deprecated.$s-80; + .dropdown { + inline-size: px2rem(80); } } .suffix-input { @extend %input-element; - @include deprecated.body-small-typography; + @include use-typography("body-small"); grid-column: span 3; } .export-btn { - @extend %button-secondary; - @include deprecated.uppercase-title-typography; - - height: deprecated.$s-32; - width: 100%; + justify-content: center; + inline-size: 100%; } From 5262aab3ffe60ec6bc3b6eaaed28d834f45b1fab Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 11:15:51 +0200 Subject: [PATCH 14/30] :bug: Count saved versions in the version quota (#12405) * :bug: Count saved versions in the version quota The snapshots-per-file and snapshots-per-team quotes counted the saved versions by file_change.data, a column that only the legacy-db file data backend writes: versions keep their payload in file_data, so the count returned 0 and a configured quota never applied. Count by label instead, which tells a version apart with every file data backend, and add a partial index on file_change with the same predicate so both counts are answered by an index only scan instead of walking every row of every file of the team (10.3 seconds in production for a team with 829 versions). Closes #12404 AI-assisted-by: deepseek-v4.1-flash * :recycle: Drop the ! suffix from the snapshot quota test helpers The check helpers assert the outcome of a command instead of mutating anything, so they do not take the ! that this codebase reserves for functions whose contract changes state or runs a command (the th/command! call inside them already carries it). AI-assisted-by: deepseek-v4.1-flash --- .../http-storage-filedata-subtleties.md | 3 + .serena/memories/prod-infra/core.md | 1 + backend/src/app/migrations.clj | 5 +- .../sql/0155-add-idx-to-file-change.sql | 38 ++++++++ backend/src/app/rpc/quotes.clj | 4 +- .../test/backend_tests/rpc_quotes_test.clj | 94 +++++++++++++++++++ 6 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 backend/src/app/migrations/sql/0155-add-idx-to-file-change.sql diff --git a/.serena/memories/backend/http-storage-filedata-subtleties.md b/.serena/memories/backend/http-storage-filedata-subtleties.md index 188ece7277..fb735b6c56 100644 --- a/.serena/memories/backend/http-storage-filedata-subtleties.md +++ b/.serena/memories/backend/http-storage-filedata-subtleties.md @@ -25,4 +25,7 @@ - File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data. - `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob. - Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. +- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only written by the `legacy-db` backend, plus the rows written before the 2025-07 file storage refactor. +- Tell a snapshot apart by `label IS NOT NULL` (with `created_by` / `deleted_at` as needed), never by `file_change.data IS NOT NULL`: the label holds with every file data backend, and the payload column stopped meaning "this is a snapshot" when the payload moved out. Queries that identify versions must use the label. +- The version quota counts of `app.rpc.quotes` (`sql:get-snapshots-per-file`, `sql:get-snapshots-per-team`) are served by the partial index `file_change__user_snapshots__idx`, whose predicate mirrors their where clause; changing one without the other drops the index only scan and the count walks the whole table again. - `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. diff --git a/.serena/memories/prod-infra/core.md b/.serena/memories/prod-infra/core.md index e90f353b7e..5dfaeebed7 100644 --- a/.serena/memories/prod-infra/core.md +++ b/.serena/memories/prod-infra/core.md @@ -39,6 +39,7 @@ Production runs the deployed toolbox, not the devenv one: - **Start from the ids, not from an inventory.** The ids come from the error or the logs. Unbounded queries over tables with tens of millions of rows (`storage_object`, `file_media_object`) do not finish in a reasonable time; bounded to an id list they are index lookups. - **One query per reference column.** Two columns in one `IN (col_a, col_b)` lose the index and scan the whole table; split them. - The read-only replica cuts long queries: see `mem:prod-infra/grafana`. +- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table (file saves write to `file_change` on every save) build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block); the shipped migration uses `IF NOT EXISTS`, finds it and does nothing. A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_indexes` and `pg_index.indisvalid`). ## See also diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index a3c4647a33..861629bb54 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -505,7 +505,10 @@ :fn (mg/resource "app/migrations/sql/0153-del-orphan-profile-rels-after-team-leave.sql")} {:name "0154-del-legacy-file-object-thumbnail-table" - :fn (mg/resource "app/migrations/sql/0154-del-legacy-file-object-thumbnail-table.sql")}]) + :fn (mg/resource "app/migrations/sql/0154-del-legacy-file-object-thumbnail-table.sql")} + + {:name "0155-add-idx-to-file-change" + :fn (mg/resource "app/migrations/sql/0155-add-idx-to-file-change.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0155-add-idx-to-file-change.sql b/backend/src/app/migrations/sql/0155-add-idx-to-file-change.sql new file mode 100644 index 0000000000..28357cc75c --- /dev/null +++ b/backend/src/app/migrations/sql/0155-add-idx-to-file-change.sql @@ -0,0 +1,38 @@ +-- Index for the version (snapshot) quota checks of app.rpc.quotes: +-- sql:get-snapshots-per-file and sql:get-snapshots-per-team count the live +-- versions created by a user, and this partial index answers both counts +-- with an index only scan, without walking the whole file_change table. +-- +-- The predicate must stay identical to the queries: the planner only uses a +-- partial index when the query where clause implies the index predicate. +-- Versions are told apart by label (every snapshot has one); file_change.data +-- is a legacy column, only written by the legacy-db file data backend. + +CREATE INDEX IF NOT EXISTS file_change__user_snapshots__idx + ON file_change (file_id) + WHERE label IS NOT NULL + AND created_by = 'user' + AND deleted_at IS NULL; + +-- NOTE: on instances where file_change is large, run the statement above +-- out of band, before deploying, with CONCURRENTLY instead: this migration +-- runs inside a transaction, and a plain CREATE INDEX locks the table +-- against writes for the whole build (in Penpot every file save writes a row +-- here), while CONCURRENTLY does not. Run it on its own, not inside a +-- transaction block: +-- +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS file_change__user_snapshots__idx +-- ON file_change (file_id) +-- WHERE label IS NOT NULL +-- AND created_by = 'user' +-- AND deleted_at IS NULL; +-- +-- A CONCURRENTLY build interrupted midway leaves the index INVALID: drop it +-- and retry. Verify with: +-- +-- SELECT indexdef FROM pg_indexes +-- WHERE indexname = 'file_change__user_snapshots__idx'; +-- SELECT indisvalid FROM pg_index +-- WHERE indexrelid = 'file_change__user_snapshots__idx'::regclass; +-- +-- Once the index exists, this migration finds it and does nothing. diff --git a/backend/src/app/rpc/quotes.clj b/backend/src/app/rpc/quotes.clj index e391f0b0ac..9f2e67c8ec 100644 --- a/backend/src/app/rpc/quotes.clj +++ b/backend/src/app/rpc/quotes.clj @@ -427,7 +427,7 @@ WHERE fc.file_id = ? AND fc.created_by = 'user' AND fc.deleted_at IS NULL - AND fc.data IS NOT NULL") + AND fc.label IS NOT NULL") (defmethod check-quote ::snapshots-per-file [{:keys [::profile-id ::file-id ::team-id ::project-id ::target] :as quote}] @@ -460,7 +460,7 @@ WHERE p.team_id = ? AND fc.created_by = 'user' AND fc.deleted_at IS NULL - AND fc.data IS NOT NULL") + AND fc.label IS NOT NULL") (defmethod check-quote ::snapshots-per-team [{:keys [::profile-id ::team-id ::target] :as quote}] diff --git a/backend/test/backend_tests/rpc_quotes_test.clj b/backend/test/backend_tests/rpc_quotes_test.clj index b279ff96d3..b0740f1b22 100644 --- a/backend/test/backend_tests/rpc_quotes_test.clj +++ b/backend/test/backend_tests/rpc_quotes_test.clj @@ -472,3 +472,97 @@ (t/is (= :restriction (th/ex-type error))) (t/is (= :max-quote-reached (th/ex-code error))) (t/is (= "media-storage-bytes-per-team" (:target (ex-data error)))))))) + +;; --- SNAPSHOTS (VERSIONS) QUOTES + +(defn- check-snapshot-ok + "Save a version of the file and check that it was created." + [profile file label] + (let [out (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label label})] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (t/is (= label (get-in out [:result :label]))) + out)) + +(defn- check-snapshot-ko + "Try to save a version of the file and check that the quota refused it." + [profile file label target] + (let [out (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label label}) + error (:error out)] + ;; (th/print-result! out) + (t/is (not (th/success? out))) + (t/is (= :restriction (th/ex-type error))) + (t/is (= :max-quote-reached (th/ex-code error))) + (t/is (= target (:target (ex-data error)))) + out)) + +(t/deftest snapshots-per-file-quote + (with-mocks [mock {:target 'app.config/get + :return (th/config-get-mock + {:quotes-snapshots-per-file 2 + :quotes-snapshots-per-team 1000})}] + + (let [profile (th/create-profile* 1) + project (th/create-project* 1 {:profile-id (:id profile) + :team-id (:default-team-id profile)}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project) + :is-shared false})] + + (check-snapshot-ok profile file "version1") + (check-snapshot-ok profile file "version2") + (check-snapshot-ko profile file "version3" "snapshots-per-file")))) + +(t/deftest snapshots-per-team-quote + (with-mocks [mock {:target 'app.config/get + :return (th/config-get-mock + {:quotes-snapshots-per-file 1000 + :quotes-snapshots-per-team 3})}] + + (let [profile (th/create-profile* 1) + project (th/create-project* 1 {:profile-id (:id profile) + :team-id (:default-team-id profile)}) + file-1 (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project) + :is-shared false}) + file-2 (th/create-file* 2 {:profile-id (:id profile) + :project-id (:id project) + :is-shared false})] + + ;; the quota adds up the versions of every file of the team + (check-snapshot-ok profile file-1 "version1") + (check-snapshot-ok profile file-2 "version2") + (check-snapshot-ok profile file-1 "version3") + (check-snapshot-ko profile file-2 "version4" "snapshots-per-team")))) + +(t/deftest snapshots-quote-freed-on-delete + (with-mocks [mock {:target 'app.config/get + :return (th/config-get-mock + {:quotes-snapshots-per-file 2 + :quotes-snapshots-per-team 1000})}] + + (let [profile (th/create-profile* 1) + project (th/create-project* 1 {:profile-id (:id profile) + :team-id (:default-team-id profile)}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project) + :is-shared false}) + snapshot (:result (check-snapshot-ok profile file "version1"))] + + (check-snapshot-ok profile file "version2") + (check-snapshot-ko profile file "version3" "snapshots-per-file") + + ;; deleting a version takes it out of the count right away + (let [out (th/command! {::th/type :delete-file-snapshot + ::rpc/profile-id (:id profile) + :id (:id snapshot)})] + (t/is (nil? (:error out))) + (t/is (true? (:result out))) + + (check-snapshot-ok profile file "version4"))))) From c401151541c3265395ee158b665940c51471bb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elena=20Torr=C3=B3?= Date: Mon, 19 Oct 2026 12:07:51 +0200 Subject: [PATCH 15/30] :bug: Fix mask group drop shadow in PDF export (#12362) --- render-wasm/src/render/pdf.rs | 1 + render-wasm/src/render/vector.rs | 73 +++++++++++++++++++------------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/render-wasm/src/render/pdf.rs b/render-wasm/src/render/pdf.rs index c61fef322b..b5b7366509 100644 --- a/render-wasm/src/render/pdf.rs +++ b/render-wasm/src/render/pdf.rs @@ -42,6 +42,7 @@ pub fn render_to_pdf( let metadata = skia::pdf::Metadata { creator: "Penpot".to_string(), producer: "Penpot (Skia PDF)".to_string(), + raster_dpi: Some(300.0), ..Default::default() }; diff --git a/render-wasm/src/render/vector.rs b/render-wasm/src/render/vector.rs index 15a8dc0c32..7eddda2b8b 100644 --- a/render-wasm/src/render/vector.rs +++ b/render-wasm/src/render/vector.rs @@ -822,34 +822,7 @@ fn render_group( let child_opts = opts.for_children_of(element); if masked { - // Mirror the GPU mask: render all children (including the mask shape) - // as content, then re-draw the mask silhouette (the group's first child) - // with DstIn to clip everything to it. - let paint = Paint::default(); - let subtree_bounds = element.extrect(tree, scale); - canvas.save_layer( - &skia::canvas::SaveLayerRec::default() - .bounds(&subtree_bounds) - .paint(&paint), - ); - - for child_id in &children { - render_tree_inner(shared, canvas, child_id, tree, scale, &child_opts)?; - } - - if let Some(mask_id) = element.mask_id() { - let mut mask_paint = Paint::default(); - mask_paint.set_blend_mode(skia::BlendMode::DstIn); - canvas.save_layer( - &skia::canvas::SaveLayerRec::default() - .bounds(&subtree_bounds) - .paint(&mask_paint), - ); - render_tree_inner(shared, canvas, mask_id, tree, scale, &child_opts)?; - canvas.restore(); // mask layer - } - - canvas.restore(); // composition layer + render_masked_children(shared, canvas, element, &children, tree, scale, &child_opts)?; } else { for child_id in &children { render_tree_inner(shared, canvas, child_id, tree, scale, &child_opts)?; @@ -863,6 +836,42 @@ fn render_group( Ok(()) } +fn render_masked_children( + shared: &mut RenderResources, + canvas: &Canvas, + element: &Shape, + children: &[Uuid], + tree: ShapesPoolRef, + scale: f32, + child_opts: &TreeOpts, +) -> Result<()> { + let subtree_bounds = element.extrect(tree, scale); + canvas.save_layer( + &skia::canvas::SaveLayerRec::default() + .bounds(&subtree_bounds) + .paint(&Paint::default()), + ); + + for child_id in children { + render_tree_inner(shared, canvas, child_id, tree, scale, child_opts)?; + } + + if let Some(mask_id) = element.mask_id() { + let mut mask_paint = Paint::default(); + mask_paint.set_blend_mode(skia::BlendMode::DstIn); + canvas.save_layer( + &skia::canvas::SaveLayerRec::default() + .bounds(&subtree_bounds) + .paint(&mask_paint), + ); + render_tree_inner(shared, canvas, mask_id, tree, scale, child_opts)?; + canvas.restore(); // mask layer + } + + canvas.restore(); // composition layer + Ok(()) +} + // --------------------------------------------------------------------------- // Frames // --------------------------------------------------------------------------- @@ -991,8 +1000,12 @@ fn render_container_drop_shadows( let children = children_paint_order(tree, element); let child_opts = opts.for_children_of(element); - for child_id in &children { - render_tree_inner(shared, canvas, child_id, tree, scale, &child_opts)?; + if element.is_masked_group() { + render_masked_children(shared, canvas, element, &children, tree, scale, &child_opts)?; + } else { + for child_id in &children { + render_tree_inner(shared, canvas, child_id, tree, scale, &child_opts)?; + } } canvas.restore(); From 84d92cfd5a5bf0b05ab104ca923d5ac2b372d8c7 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 13:01:35 +0200 Subject: [PATCH 16/30] :sparkles: Add tail buckets and update-file validation timings (#12409) * :sparkles: Extend backend histogram buckets beyond 7.5s Give every backend timing histogram a tail past the shared 7500ms cap so slow RPC calls, CLIMIT waits and background tasks report real quantiles instead of clipping at the top bucket. Request and wait metrics grow to 60s, background tasks to 300s and Redis eval to 10s; the sub-7.5s buckets stay untouched to keep the history comparable. Closes #12408 AI-assisted-by: muse-spark-1.3-contributor * :bug: Fix flaky assertion in climit republish test Read the last published waiting value instead of the exact publication list. The sampler thread started with the component runs its first tick asynchronously, and when it lands inside the recording window it publishes the same count twice, which made the full-suite CI run fail while focused runs stayed green. AI-assisted-by: muse-spark-1.3-contributor * :sparkles: Add file_data blob to JSON decode helper Allow decoding file_data blobs (app.util.blob) to JSON for investigations with jq, offline from a blob file or from nREPL with the backend running. AI-assisted-by: muse-spark-1.3-contributor * :sparkles: Add validation step timings to update-file logs and metrics Measure the six phases inside process-changes-and-validate (resolve-libs, process-changes, soft and hard affected and schema validations) with tpoint and try/finally, so timings are recorded even when a hard validation throws. Publish each phase to a new histogram penpot_rpc_update_file_validation_timing with a fixed step label using the request buckets (up to 60000 ms), and extend the update-file debug log with the same *-ms fields plus file, project and team ids. The deferred log only runs on success, so failures get an inline error log with partial timings. AI-assisted-by: muse-spark-1.3-contributor * :lipstick: Translate decode-file-data comments to English Translate the Spanish docstrings, section headers and test messages of the file_data decode helper to English, keeping code untouched. The repo convention is English in code and comments; the helper was first written in Spanish. AI-assisted-by: muse-spark-1.3-contributor --- .serena/memories/clojure/design-rules.md | 7 +- .serena/memories/critical-info.md | 2 +- .serena/memories/prod-infra/grafana.md | 2 +- backend/scripts/decode-file-data | 19 ++ backend/src/app/main.clj | 16 +- backend/src/app/metrics.clj | 19 ++ backend/src/app/rpc/commands/files_update.clj | 119 +++++-- backend/src/app/scripts/decode_file_data.clj | 300 ++++++++++++++++++ backend/test/backend_tests/climit_test.clj | 6 +- .../files_update_validation_timing_test.clj | 128 ++++++++ backend/test/backend_tests/metrics_test.clj | 72 +++++ .../scripts_decode_file_data_test.clj | 82 +++++ 12 files changed, 743 insertions(+), 29 deletions(-) create mode 100755 backend/scripts/decode-file-data create mode 100644 backend/src/app/scripts/decode_file_data.clj create mode 100644 backend/test/backend_tests/files_update_validation_timing_test.clj create mode 100644 backend/test/backend_tests/scripts_decode_file_data_test.clj diff --git a/.serena/memories/clojure/design-rules.md b/.serena/memories/clojure/design-rules.md index 48c7f40193..2b439ee038 100644 --- a/.serena/memories/clojure/design-rules.md +++ b/.serena/memories/clojure/design-rules.md @@ -2,10 +2,11 @@ How to shape a function in this codebase. Each rule came from a review round; do not re-derive from taste. -- **Preconditions belong to the boundary, not to the core.** A function documents what it needs (hints, docstring) and assumes it. Absence checks live where the value enters the system: the caller that resolves optional objects, `ig/init-key`, the request handler. A `(when (and (some? a) (some? b)) ...)` inside a leaf function means the guard is in the wrong place — if you cannot get `a` or `b`, you should not be calling it. -- **Optional-by-design data is guarded where the optionality is born.** Example: `ConnectorStatistics` is nil when the server option is off, so the guard is `when-let [cs (some-> server ...)]` at the entry point and the consumer assumes `cs`. +- **Preconditions belong to the boundary, not to the core.** A function documents what it needs (hints, docstring) and assumes it. Absence checks live where the value enters the system: the caller that resolves optional objects, `ig/init-key`, the request handler — including `nil` that is optional-by-design (`when-let [cs (some-> server ...)]` at the entry point, as with `ConnectorStatistics` when the server option is off). A `(when (and (some? a) (some? b)) ...)` inside a leaf function means the guard is in the wrong place — if you cannot get `a` or `b`, you should not be calling it. - **A static set of operations is written statically.** With a fixed, small set (four metrics) write the four calls. Do not build a collection and iterate it (`doseq` over a literal vector): it allocates, adds indirection and hides each operation. Use a collection when the set is dynamic (config, registry, input). -- **The name says what the function is; `!` says it mutates.** `!` marks functions whose contract is to change state or run a command (`run!`, `submit!`, `swap!`, `shutdown-now`). Reporting helpers (log, sample-and-publish) do not take it: the mutation happens in the `!` API they call. Keep a family consistent. +- **Default to no `!`; existing `!` is legacy being removed gradually.** Do not introduce new `!` names except the allowed cases documented in `mem:clojure/idioms` (add and justify the special case there first). Allowed: mutating public fns in `app.db` (`insert!`, `update!`, `delete!`, `run!`, `tx-run!`, ...) and fns whose contract is to work over transients (stdlib `conj!`/`assoc!`/`dissoc!`/`persistent!` and custom helpers over them). Existing `!` elsewhere (e.g. `emit!`, `send!`, `submit!`, `validate-file!`) stays until its refactor; keep a family consistent in the meantime. Reporting helpers (log, sample-and-publish) never take `!`: the mutation happens in the `!` API they call. +- **Functions that obtain data start with `get-` or `fetch-`.** A reader must see at a glance that the function reads (`get-file`, `get-snapshot`, `get-file-json`); a bare noun hides it (`file-json` says nothing about reading). New helpers that fetch by id follow this, never a `current-` prefix: the id parameter already says which one it is. +- **`?` marks a predicate, not a data prop.** A var or function that answers a question takes `?` (`export-error?`, `(defn editable? …)`); a map key that names a property does not (`:export-error`, never `:export-error?`), even when its value is a boolean. Existing `?` keys are legacy to remove gradually; do not add a new one. - **Do not shape production code for tests.** No return values added just to assert them; assert on the observable effect. When preconditions move, move the tests with them. Reference implementation: the http metrics samplers in `app.http` (`sample-worker-metrics` / `sample-connector-metrics` are guard-free; `sample-http-metrics` guards at the boundary). diff --git a/.serena/memories/critical-info.md b/.serena/memories/critical-info.md index 5e84c9834e..63bbce922a 100644 --- a/.serena/memories/critical-info.md +++ b/.serena/memories/critical-info.md @@ -7,7 +7,7 @@ You are working on the GitHub project `penpot/penpot`, a monorepo. - Edits/stale refs/duplication cleanup: `mem:memory-maintenance`. - Cross-cutting testing principles, TDD workflow, and anti-patterns: `mem:testing`. - Verified Clojure language behaviors that contradict common assumptions (e.g. `int?` covers `Long`; `integer?` is the general predicate): `mem:clojure/idioms` — read before assuming stdlib predicate semantics. -- When designing a solution or an API, read `mem:clojure/design-rules` (function shape, boundary checks, naming). +- Read `mem:clojure/design-rules` (function shape, boundary checks, naming) before writing or modifying Clojure/ClojureScript code — not only when designing a solution or an API. # Development workflow diff --git a/.serena/memories/prod-infra/grafana.md b/.serena/memories/prod-infra/grafana.md index 10af277db2..085308da1b 100644 --- a/.serena/memories/prod-infra/grafana.md +++ b/.serena/memories/prod-infra/grafana.md @@ -6,7 +6,7 @@ Keep this memory free of internal names: no host names, no deployed versions, no ## Datasources -- Metrics — **Prometheus**, uid `xkRmyFBGk`. PromQL. Metric families: `penpot_*` (RPC, task timings, CLIMIT, HTTP, websocket, storage, and the application log events of the backend), `hikaricp_*`, `jvm_*` and `process_*` on the backend (`job="app"`); `pg_*` from the Postgres exporter (`job="postgres"`); `redis_*` from the Redis exporter (`job="redis"`); `node_*` and `container_*` from node-exporter and cAdvisor. Backend histograms are in **milliseconds**. +- Metrics — **Prometheus**, uid `xkRmyFBGk`. PromQL. Metric families: `penpot_*` (RPC, task timings, CLIMIT, HTTP, websocket, storage, and the application log events of the backend), `hikaricp_*`, `jvm_*` and `process_*` on the backend (`job="app"`); `pg_*` from the Postgres exporter (`job="postgres"`); `redis_*` from the Redis exporter (`job="redis"`); `node_*` and `container_*` from node-exporter and cAdvisor. Backend histograms are in **milliseconds**. Tails (ms): requests and waits to 60000, background tasks to 300000, Redis eval to 10000; the sub-7500 buckets are identical everywhere, so old quantiles stay comparable. - Logs — **Loki**, uid `h3CGKhEMk`. LogQL, and filter by the structured metadata `logger`/`level` (`{job="penpot"} | logger="..." | level="ERROR"`). Two things to remember when counting from it: the CLIMIT rejections have their own counters (`penpot_rpc_climit_rejected_total`, by `name` and `reason`) and their `hint="rejected"` line goes out at WARN only when something is left queued behind the call (a lone caller that times out is written at TRACE), so counting lines is not counting rejections -- the same goes for `hint="enqueued"` and `hint="acquired"`; and the only signal that exists nowhere else is the profile behind a queue (`logger="app.rpc.climit"`, whose `id` carries the key), chased there on demand. - SQL — **Postgres**, uid `y8ea9tfMk`. Panels need `rawQuery: true`, `editorMode: "code"` and `format: "table"`. The `task` table is the only truth about the queue depth. It is a **read-only replica**: it rejects any write, `CREATE TEMP TABLE` included, so queries through it must be plain `SELECT`s. It also cuts long-running queries (the MCP path has a short timeout), so bound them (e.g. a list of ids); an empty answer there is not proof that there is no data. - **Postgres (telemetry)**, uid `9YAUuM_Gz`. diff --git a/backend/scripts/decode-file-data b/backend/scripts/decode-file-data new file mode 100755 index 0000000000..04c84ccd73 --- /dev/null +++ b/backend/scripts/decode-file-data @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Decode a file_data blob (app.util.blob) to JSON for inspection with jq. +# +# Offline use (no DB): +# backend/scripts/decode-file-data --input /tmp/blob.bin --format summary | jq . +# +# DB use (backend running, via nREPL): +# ./scripts/nrepl-eval.mjs "(require '[app.scripts.decode-file-data :as dfd] :reload)" +# ./scripts/nrepl-eval.mjs '(dfd/get-file-json sys/system #uuid "..." :format :summary)' + +SCRIPT_DIR=$(dirname "$0") + +source "$SCRIPT_DIR/_env" + +if [ -f "$SCRIPT_DIR/_env.local" ]; then + source "$SCRIPT_DIR/_env.local" +fi + +exec clojure -M:dev -m app.scripts.decode-file-data "$@" diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index be1ccfafb3..9b03601b0d 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -25,7 +25,7 @@ [app.http.session.tasks :as-alias session.tasks] [app.http.websocket :as http.ws] [app.loggers.webhooks :as-alias webhooks] - [app.metrics :as-alias mtx] + [app.metrics :as mtx] [app.metrics.definition :as-alias mdef] [app.msgbus :as-alias mbus] [app.redis :as-alias rds] @@ -66,16 +66,25 @@ ::mdef/help "A total number of bytes processed by update-file." ::mdef/type :counter} + :update-file-validation-timing + {::mdef/name "penpot_rpc_update_file_validation_timing" + ::mdef/help "update-file validation step timings (milliseconds)." + ::mdef/labels ["step"] + ::mdef/buckets mtx/default-request-histogram-buckets + ::mdef/type :histogram} + :rpc-main-timing {::mdef/name "penpot_rpc_main_timing" ::mdef/help "RPC command method call timing for main" ::mdef/labels ["name"] + ::mdef/buckets mtx/default-request-histogram-buckets ::mdef/type :histogram} :rpc-management-timing {::mdef/name "penpot_rpc_management_timing" ::mdef/help "RPC command method call timing for management." ::mdef/labels ["name"] + ::mdef/buckets mtx/default-request-histogram-buckets ::mdef/type :histogram} :websocket-active-connections @@ -103,6 +112,7 @@ {::mdef/name "penpot_tasks_timing" ::mdef/help "Background tasks timing (milliseconds)." ::mdef/labels ["name"] + ::mdef/buckets mtx/default-tasks-histogram-buckets ::mdef/type :histogram} :storage-gc-deleted-objects @@ -115,6 +125,7 @@ {::mdef/name "penpot_redis_eval_timing" ::mdef/help "Redis EVAL commands execution timings (ms)" ::mdef/labels ["name"] + ::mdef/buckets mtx/default-redis-histogram-buckets ::mdef/type :histogram} :rpc-climit-waiting @@ -133,6 +144,7 @@ {::mdef/name "penpot_rpc_climit_timing" ::mdef/help "Summary of the time between queuing and executing on the CLIMIT" ::mdef/labels ["name"] + ::mdef/buckets mtx/default-request-histogram-buckets ::mdef/type :histogram} :audit-http-handler-queue-size @@ -151,12 +163,14 @@ {::mdef/name "penpot_audit_http_handler_timing" ::mdef/help "Summary of the time between queuing and executing on the audit log http handler" ::mdef/labels [] + ::mdef/buckets mtx/default-request-histogram-buckets ::mdef/type :histogram} :http-server-dispatch-timing {::mdef/name "penpot_http_server_dispatch_timing" ::mdef/help "Histogram of dispatch handler" ::mdef/labels [] + ::mdef/buckets mtx/default-request-histogram-buckets ::mdef/type :histogram} :http-worker-queue-size diff --git a/backend/src/app/metrics.clj b/backend/src/app/metrics.clj index c02e570e8b..96b55c7271 100644 --- a/backend/src/app/metrics.clj +++ b/backend/src/app/metrics.clj @@ -132,6 +132,25 @@ (def default-histogram-buckets [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500]) +;; Tail for request and wait timings (RPC, CLIMIT waits, audit handler, +;; HTTP dispatch): covers the 30s/60s CLIMIT timeouts while keeping the +;; sub-7.5s resolution of the default buckets untouched. +(def default-request-histogram-buckets + [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500 + 10000 15000 30000 60000]) + +;; Tail for background tasks (GC sweeps, file workers): they run until +;; done with no CLIMIT timeout, bounded in practice by the 300s postgres +;; statement timeout. +(def default-tasks-histogram-buckets + [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500 + 10000 15000 30000 60000 120000 300000]) + +;; Tail for Redis eval: sub-second in practice, one extra bucket covers +;; the pathological (timeout) case without adding idle series. +(def default-redis-histogram-buckets + [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500 10000]) + (defmulti run-collector! (fn [mdef _] (::mdef/type mdef))) (defmulti create-collector ::mdef/type) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 0c19c1c315..12d7aa63d6 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -169,7 +169,8 @@ (assoc :file file) (assoc :changes changes)) - cfg (assoc cfg ::timestamp (ct/now)) + cfg (assoc cfg ::timestamp (ct/now) + ::validation-timings (atom {})) tpoint (ct/tpoint)] @@ -210,9 +211,39 @@ (binding [l/*context* (some-> (meta params) (get :app.http/request) (errors/request->context))] - (-> (update-file* cfg params) - (rph/with-defer #(let [elapsed (tpoint)] - (l/trace :hint "update-file" :time (ct/format-duration elapsed)))))))) + (try + (-> (update-file* cfg params) + (rph/with-defer #(let [elapsed (tpoint) + timings @(::validation-timings cfg)] + (l/dbg :hint "update-file" + :file-id (str (:id file)) + :project-id (str (:project-id file)) + :team-id (str (:team-id file)) + :status "success" + :time (ct/format-duration elapsed) + :resolve-libs-ms (get timings :resolve-libs-ms) + :process-changes-ms (get timings :process-changes-ms) + :soft-validate-affected-ms (get timings :soft-validate-affected-ms) + :soft-validate-schema-ms (get timings :soft-validate-schema-ms) + :validate-affected-ms (get timings :validate-affected-ms) + :validate-schema-ms (get timings :validate-schema-ms))))) + (catch Throwable cause + ;; NOTE: `with-defer` hooks only run on the success path + ;; (`handle-response`), so timings collected before a hard + ;; validation failure would be lost without this inline log. + (let [timings @(::validation-timings cfg)] + (l/dbg :hint "update-file" + :file-id (str (:id file)) + :project-id (str (:project-id file)) + :team-id (str (:team-id file)) + :status "error" + :resolve-libs-ms (get timings :resolve-libs-ms) + :process-changes-ms (get timings :process-changes-ms) + :soft-validate-affected-ms (get timings :soft-validate-affected-ms) + :soft-validate-schema-ms (get timings :soft-validate-schema-ms) + :validate-affected-ms (get timings :validate-affected-ms) + :validate-schema-ms (get timings :validate-schema-ms))) + (throw cause)))))) (defn- update-file* "Internal function, part of the update-file process, that encapsulates @@ -379,6 +410,16 @@ (catch Throwable cause (l/error :hint "file schema validation error" :cause cause)))) +(defn- record-validation-timing + [cfg step-key step-label elapsed-ms] + (when-let [metrics (::mtx/metrics cfg)] + (when (mtx/metrics? metrics) + (mtx/run! metrics {:id :update-file-validation-timing + :labels [step-label] + :val (double elapsed-ms)}))) + (when-let [timings (::validation-timings cfg)] + (swap! timings assoc step-key elapsed-ms))) + (defn- soft-validate-file! [file libs changes] (try @@ -389,11 +430,21 @@ (defn- process-changes-and-validate [cfg file changes skip-validate] - (let [libs - (when (and (or (contains? cf/flags :file-validation) - (contains? cf/flags :soft-file-validation)) - (not skip-validate)) - (bfc/get-resolved-file-libraries cfg file)) + (let [should-resolve-libs? + (and (or (contains? cf/flags :file-validation) + (contains? cf/flags :soft-file-validation)) + (not skip-validate)) + + tlibs + (ct/tpoint) + + libs + (try + (when should-resolve-libs? + (bfc/get-resolved-file-libraries cfg file)) + (finally + (when should-resolve-libs? + (record-validation-timing cfg :resolve-libs-ms "resolve-libs" (inst-ms (tlibs)))))) ;; The main purpose of this atom is provide a contextual state ;; for the changes subsystem where optionally some hints can @@ -403,32 +454,56 @@ state (atom {}) - file - (binding [cpc/*state* state] - (-> (files/check-version! file) - (update :revn inc) - (update :data cpc/process-changes changes) - (update :data d/without-nils))) + tproc + (ct/tpoint) file - (if-let [media-refs (-> @state :media-refs not-empty)] - (bfc/update-media-references! cfg file media-refs) - file)] + (try + (let [file + (binding [cpc/*state* state] + (-> (files/check-version! file) + (update :revn inc) + (update :data cpc/process-changes changes) + (update :data d/without-nils))) + + file + (if-let [media-refs (-> @state :media-refs not-empty)] + (bfc/update-media-references! cfg file media-refs) + file)] + file) + (finally + (record-validation-timing cfg :process-changes-ms "process-changes" (inst-ms (tproc)))))] (binding [pmap/*tracked* nil] (when (contains? cf/flags :soft-file-validation) - (soft-validate-file! file libs changes)) + (let [t (ct/tpoint)] + (try + (soft-validate-file! file libs changes) + (finally + (record-validation-timing cfg :soft-validate-affected-ms "soft-validate-affected" (inst-ms (t))))))) (when (contains? cf/flags :soft-file-schema-validation) - (soft-validate-file-schema! file)) + (let [t (ct/tpoint)] + (try + (soft-validate-file-schema! file) + (finally + (record-validation-timing cfg :soft-validate-schema-ms "soft-validate-schema" (inst-ms (t))))))) (when (and (contains? cf/flags :file-validation) (not skip-validate)) - (val/validate-file-affected! file libs changes)) + (let [t (ct/tpoint)] + (try + (val/validate-file-affected! file libs changes) + (finally + (record-validation-timing cfg :validate-affected-ms "validate-affected" (inst-ms (t))))))) (when (and (contains? cf/flags :file-schema-validation) (not skip-validate)) - (val/validate-file-schema! file))) + (let [t (ct/tpoint)] + (try + (val/validate-file-schema! file) + (finally + (record-validation-timing cfg :validate-schema-ms "validate-schema" (inst-ms (t)))))))) file)) diff --git a/backend/src/app/scripts/decode_file_data.clj b/backend/src/app/scripts/decode_file_data.clj new file mode 100644 index 0000000000..b7d775eafc --- /dev/null +++ b/backend/src/app/scripts/decode_file_data.clj @@ -0,0 +1,300 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.scripts.decode-file-data + "Decode a file's `file_data` binary blob to `jq`-readable JSON. + + The content of `file_data.data` (and `file.data` on the `legacy-db` + backend, or the `file-data` object on storage) is not text: it is a blob + encoded with `app.util.blob` (versions 1, 3, 4 and 5; transit+json or + fressian plus lz4/zstd). That is why it cannot be inspected with SQL + directly. + + Two uses: + + 1. Offline (no DB, just the blob): dump the `bytea` to a file with + `psql` (`\\copy ...` or `encode(data, 'base64')`) and pass it with `--input`. + Needs no system and no `file_data` read permissions. + + backend/scripts/decode-file-data --input /tmp/blob.bin --format summary | jq . + backend/scripts/decode-file-data --input /tmp/blob.b64 --from-base64 --format pages | jq . + + 2. Inside the penpot context (nREPL with the backend running, where + `sys/system` already exists): call `get-file-json` or `get-snapshot-json`. + They resolve the backend (`legacy-db`, `db`, `storage`), realize the + pointer-maps and objects-maps and return JSON. + + (require '[app.scripts.decode-file-data :as dfd] :reload) + (spit \"/tmp/maquette.json\" + (dfd/get-file-json sys/system #uuid \"9e056cfe-f83f-81f8-8008-cc885732f00d\" + :format :full)) + (spit \"/tmp/snap-518.json\" + (dfd/get-snapshot-json sys/system #uuid \"9e056cfe-f83f-81f8-8008-cc885732f00d\" + #uuid \"\" :format :summary)) + + Then filter with `jq`: `.pages`, `.data[\"pages-index\"] | keys`, + `[.data[\"pages-index\"][] | {id, name}]`, etc." + (:require + [app.binfile.common :as bfc] + [app.common.json :as json] + [app.db :as db] + [app.features.file-snapshots :as fsnap] + [app.util.blob :as blob] + [app.util.objects-map :as omap-legacy] + [app.util.pointer-map :as pmap] + [clojure.java.io :as io] + [clojure.string :as str]) + (:import + java.util.Base64)) + +(set! *warn-on-reflection* true) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; JSON sanitization +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- sanitize-key + [k] + (cond + (keyword? k) (subs (str k) 1) + (symbol? k) (str k) + (uuid? k) (str k) + :else (str k))) + +(declare sanitize) + +(defn- sanitize-map + [m] + (persistent! + (reduce-kv (fn [acc k v] + (assoc! acc (sanitize-key k) (sanitize v))) + (transient {}) + m))) + +(defn sanitize + "Convert a blob-decoded value to plain JSON-ready data. + + - Unrealized pointers become `{\"$pointer\": \"\"}`. + - `objects-map` instances are materialized to a plain map. + - Bytes become `{\"$bytes\": \"\"}`. + - UUIDs, instants, keywords and symbols become strings." + [x] + (cond + (nil? x) nil + (string? x) x + (boolean? x) x + + (number? x) + (if (ratio? x) (double x) x) + + (keyword? x) (subs (str x) 1) + (symbol? x) (str x) + (uuid? x) (str x) + (inst? x) (str x) + + (bytes? x) + {"$bytes" (.encodeToString (Base64/getEncoder) ^bytes x)} + + (pmap/pointer-map? x) + {"$pointer" (str (pmap/get-id x))} + + (omap-legacy/objects-map? x) + (sanitize (into {} x)) + + (map? x) + (sanitize-map x) + + (set? x) + (mapv sanitize x) + + (coll? x) + (mapv sanitize x) + + :else + (str x))) + +(defn data->json + "Encode sanitizable data to a JSON string." + [data] + (json/encode (sanitize data))) + +(defn blob->json + "Decode a `file_data` blob (`app.util.blob`) and return it as JSON." + [^bytes blob-bytes] + (data->json (blob/decode blob-bytes))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; jq-oriented investigation summary +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn summarize-data + "Small, stable summary for `jq`: pages (in `:pages` order) with object + counts, plus component, color and typography counters. Takes the already + decoded `:data` (not the JSON)." + [data] + (let [pages (:pages data) + pages-index (:pages-index data) + page-entry (fn [pid] + {:id (str pid) + :name (get-in pages-index [pid :name]) + :objects (count (get-in pages-index [pid :objects]))})] + {:pages (mapv page-entry (or pages [])) + :stats {:pages (count (or pages [])) + :components (count (:components data)) + :colors (count (:colors data)) + :typographies (count (:typographies data))}})) + +(defn file-data-summary-json + [data] + (data->json (summarize-data data))) + +(defn file-data-pages-json + [data] + (data->json (:pages (summarize-data data)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; DB access (for use from nREPL with the backend running) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- write-output + [^String payload output] + (if output + (do (spit output payload) output) + (print payload)) + payload) + +(defn get-file-json + "Return a file as JSON (realized). For nREPL. + + `format` can be `:full` (whole file), `:data` (`:data` only), + `:summary` or `:pages`. With `:output` it also writes it to a file." + [system file-id & {:keys [format output] :or {format :full}}] + (let [file (db/run! system bfc/get-file file-id :realize? true) + data (case format + :full file + :data (:data file) + :summary (summarize-data (:data file)) + :pages (:pages (summarize-data (:data file))))] + (write-output (data->json data) output))) + +(defn get-snapshot-json + "Return a snapshot as JSON (realized). For nREPL. + + Takes the same `format` and `output` as `get-file-json`." + [system file-id snapshot-id & {:keys [format output] :or {format :full}}] + (let [snapshot (db/run! system + (fn [cfg] + (fsnap/get-snapshot cfg file-id snapshot-id))) + data (case format + :full snapshot + :data (:data snapshot) + :summary (summarize-data (:data snapshot)) + :pages (:pages (summarize-data (:data snapshot))))] + (write-output (data->json data) output))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Offline CLI: decode a file holding the blob +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:private cli-usage + (str "Usage: decode-file-data --input PATH [--output PATH] [--format full|data|summary|pages]\n" + " [--from-base64|--from-hex]\n" + "\n" + "Decode a file_data blob (app.util.blob) to JSON for inspection with jq.\n" + "\n" + "Examples:\n" + " psql -c \"\\\\copy (select data from file_data where file_id='...' and id='...') to '/tmp/blob.bin' with binary\"\n" + " decode-file-data --input /tmp/blob.bin --format summary | jq .\n" + " decode-file-data --input /tmp/blob.b64 --from-base64 --format pages | jq .\n")) + +(defn- parse-args + [args] + (loop [args args + opts {:format "full"}] + (if (empty? args) + opts + (let [[a b & rest-args] args] + (cond + (contains? #{"-h" "--help"} a) + (assoc opts :help true) + + (= a "--input") + (recur rest-args (assoc opts :input b)) + + (= a "--output") + (recur rest-args (assoc opts :output b)) + + (= a "--format") + (recur rest-args (assoc opts :format b)) + + (= a "--from-base64") + (recur (cons b rest-args) (assoc opts :from :base64)) + + (= a "--from-hex") + (recur (cons b rest-args) (assoc opts :from :hex)) + + :else + (assoc opts :error (str "unknown argument: " a))))))) + +(defn- hex->bytes + [^String s] + (let [s (str/trim s) + s (cond + (str/starts-with? s "\\x") (subs s 2) + (str/starts-with? s "0x") (subs s 2) + :else s) + s (str/replace s #"\s+" "") + n (/ (count s) 2) + out (byte-array n)] + (dotimes [i n] + (aset-byte out i (unchecked-byte (Integer/parseInt (subs s (* i 2) (+ (* i 2) 2)) 16)))) + out)) + +(defn- read-input-bytes + [{:keys [input from]}] + (cond + (nil? input) + (throw (ex-info "missing --input" {})) + + (= from :base64) + (.decode (Base64/getDecoder) ^String (str/trim (slurp (io/file input)))) + + (= from :hex) + (hex->bytes (slurp (io/file input))) + + :else + (java.nio.file.Files/readAllBytes (.toPath (io/file input))))) + +(defn -main + [& args] + (let [{:keys [help error input format output] :as opts} (parse-args (vec args))] + (cond + help + (do (print cli-usage) nil) + + error + (do (binding [*out* *err*] (println error) (println cli-usage)) (System/exit 2)) + + (nil? input) + (do (binding [*out* *err*] (println "missing --input") (println cli-usage)) (System/exit 2)) + + (not (contains? #{"full" "data" "summary" "pages"} (or format "full"))) + (do (binding [*out* *err*] (println (str "unknown format: " format))) (System/exit 2)) + + :else + (let [blob-bytes (read-input-bytes opts) + decoded (blob/decode blob-bytes) + payload (case format + "full" decoded + "data" (:data decoded (:data decoded decoded)) + "summary" (summarize-data (:data decoded decoded)) + "pages" (:pages (summarize-data (:data decoded decoded)))) + ;; NOTE: the file_data blob already IS the file's :data; but if + ;; someone dumps the whole serialized row, tolerate both. + out (data->json payload)] + (if output + (spit output out) + (println out)))))) diff --git a/backend/test/backend_tests/climit_test.clj b/backend/test/backend_tests/climit_test.clj index c0d82344ef..470f591e5a 100644 --- a/backend/test/backend_tests/climit_test.clj +++ b/backend/test/backend_tests/climit_test.clj @@ -661,7 +661,11 @@ (record-metrics (fn [] (#'climit/republish-waiting (th/metrics-stub) (:counters climit)) - (t/is (= [3] (waiting-value-of name)))))))) + ;; The value is the contract, not the publication count: the + ;; sampler thread started with the component runs its first + ;; tick asynchronously, and when it lands inside this window + ;; it publishes the same count a second time. + (t/is (= 3 (waiting-last name)))))))) (t/deftest halting-the-component-stops-the-republisher ;; The republisher is a daemon thread started with the component: halting it diff --git a/backend/test/backend_tests/files_update_validation_timing_test.clj b/backend/test/backend_tests/files_update_validation_timing_test.clj new file mode 100644 index 0000000000..d34ee359fa --- /dev/null +++ b/backend/test/backend_tests/files_update_validation_timing_test.clj @@ -0,0 +1,128 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.files-update-validation-timing-test + (:require + [app.binfile.common :as bfc] + [app.common.files.validate :as val] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.main :as main] + [app.metrics :as mtx] + [app.metrics.definition :as-alias mdef] + [app.rpc.commands.files-update :as files-update] + [clojure.test :as t] + [integrant.core :as ig]) + (:import + io.prometheus.client.CollectorRegistry)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Helpers +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- bucket-value + [metrics mname label-names label-values le] + (.getSampleValue ^CollectorRegistry (mtx/get-registry metrics) + (str mname "_bucket") + (into-array String (conj (vec label-names) "le")) + (into-array String (conj (vec label-values) le)))) + +(defn- test-file + [] + {:id (uuid/next) + :revn 0 + :version 0 + :data {}}) + +(defn- test-cfg + [metrics] + {::mtx/metrics metrics + ::files-update/validation-timings (atom {})}) + +(def ^:private all-flags + #{:file-validation + :file-schema-validation + :soft-file-validation + :soft-file-schema-validation}) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Tests +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest validation-timing-metric-uses-request-buckets + (t/is (= mtx/default-request-histogram-buckets + (get-in main/default-metrics [:update-file-validation-timing ::mdef/buckets]))) + (t/is (= "penpot_rpc_update_file_validation_timing" + (get-in main/default-metrics [:update-file-validation-timing ::mdef/name])))) + +(t/deftest process-changes-and-validate-records-all-steps + (let [metrics (ig/init-key :app.metrics/metrics {:default main/default-metrics}) + cfg (test-cfg metrics)] + (binding [cf/flags all-flags] + (with-redefs [bfc/get-resolved-file-libraries (fn [_ _] {}) + val/validate-file-affected! (fn [_ _ _] nil) + val/validate-file-schema! (fn [_] nil)] + (let [file (@#'files-update/process-changes-and-validate cfg (test-file) [] nil) + timings @(::files-update/validation-timings cfg)] + (t/is (map? file)) + (t/is (number? (:resolve-libs-ms timings))) + (t/is (number? (:process-changes-ms timings))) + (t/is (number? (:soft-validate-affected-ms timings))) + (t/is (number? (:soft-validate-schema-ms timings))) + (t/is (number? (:validate-affected-ms timings))) + (t/is (number? (:validate-schema-ms timings))) + (doseq [step ["resolve-libs" "process-changes" + "soft-validate-affected" "soft-validate-schema" + "validate-affected" "validate-schema"]] + (t/is (= 1.0 (bucket-value metrics "penpot_rpc_update_file_validation_timing" + ["step"] [step] "60000.0")) + (str "expected one sample for step " step)))))))) + +(t/deftest process-changes-and-validate-skips-when-skip-validate + (let [metrics (ig/init-key :app.metrics/metrics {:default main/default-metrics}) + cfg (test-cfg metrics)] + (binding [cf/flags all-flags] + (with-redefs [bfc/get-resolved-file-libraries (fn [_ _] + (throw (ex-info "must not resolve libs" {}))) + val/validate-file-affected! (fn [_ _ _] + (throw (ex-info "must not validate" {}))) + val/validate-file-schema! (fn [_] + (throw (ex-info "must not validate schema" {})))] + ;; soft validations still run: they only check flags, not skip-validate + (let [file (@#'files-update/process-changes-and-validate cfg (test-file) [] true) + timings @(::files-update/validation-timings cfg)] + (t/is (map? file)) + (t/is (nil? (:resolve-libs-ms timings))) + (t/is (number? (:process-changes-ms timings))) + (t/is (number? (:soft-validate-affected-ms timings))) + (t/is (number? (:soft-validate-schema-ms timings))) + (t/is (nil? (:validate-affected-ms timings))) + (t/is (nil? (:validate-schema-ms timings))) + (t/is (= 1.0 (bucket-value metrics "penpot_rpc_update_file_validation_timing" + ["step"] ["process-changes"] "60000.0")))))))) + +(t/deftest process-changes-and-validate-records-timing-on-error + (let [metrics (ig/init-key :app.metrics/metrics {:default main/default-metrics}) + cfg (test-cfg metrics)] + (binding [cf/flags all-flags] + (with-redefs [bfc/get-resolved-file-libraries (fn [_ _] {}) + val/validate-file-affected! (fn [_ _ _] + (throw (ex-info "bad file" {:code :referential-integrity}))) + val/validate-file-schema! (fn [_] nil)] + (let [timings-atom (::files-update/validation-timings cfg)] + (t/is (thrown? clojure.lang.ExceptionInfo + (@#'files-update/process-changes-and-validate cfg (test-file) [] nil))) + ;; the failing step still recorded its timing via finally + (t/is (number? (:validate-affected-ms @timings-atom))) + (t/is (number? (:resolve-libs-ms @timings-atom))) + (t/is (number? (:process-changes-ms @timings-atom)))))))) + +(t/deftest record-validation-timing-tolerates-missing-collaborators + ;; production code must not blow up when called without metrics + ;; (e.g. scripts) or without the timings collector. + (let [record @#'files-update/record-validation-timing] + (t/is (nil? (record {} :resolve-libs-ms "resolve-libs" 12))) + (t/is (nil? (record {::mtx/metrics nil} :resolve-libs-ms "resolve-libs" 12))))) diff --git a/backend/test/backend_tests/metrics_test.clj b/backend/test/backend_tests/metrics_test.clj index f55e751d5e..15f5fde49d 100644 --- a/backend/test/backend_tests/metrics_test.clj +++ b/backend/test/backend_tests/metrics_test.clj @@ -6,13 +6,16 @@ (ns backend-tests.metrics-test (:require + [app.main :as main] [app.metrics :as mtx] + [app.metrics.definition :as-alias mdef] [clojure.test :as t] [integrant.core :as ig] [promesa.exec :as px]) (:import io.prometheus.client.Collector$MetricFamilySamples io.prometheus.client.Collector$MetricFamilySamples$Sample + io.prometheus.client.CollectorRegistry java.util.concurrent.ScheduledThreadPoolExecutor)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -74,3 +77,72 @@ (finally (px/shutdown-now sampler))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; HISTOGRAM BUCKETS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest histogram-bucket-tails-extend-past-7-5s + ;; The sub-7.5s prefix stays identical, so p50/p90 resolution and the + ;; history remain comparable; only the tail grows, per latency profile. + (t/is (= mtx/default-histogram-buckets + (subvec mtx/default-request-histogram-buckets 0 14))) + (t/is (= mtx/default-histogram-buckets + (subvec mtx/default-tasks-histogram-buckets 0 14))) + (t/is (= mtx/default-histogram-buckets + (subvec mtx/default-redis-histogram-buckets 0 14))) + + (t/is (= [10000 15000 30000 60000] + (subvec mtx/default-request-histogram-buckets 14))) + (t/is (= [10000 15000 30000 60000 120000 300000] + (subvec mtx/default-tasks-histogram-buckets 14))) + (t/is (= [10000] + (subvec mtx/default-redis-histogram-buckets 14)))) + +(t/deftest default-metrics-wire-bucket-tails-per-profile + ;; Requests and waits cover the 30s/60s CLIMIT timeouts, tasks run + ;; until done (300s postgres statement timeout), Redis only needs the + ;; pathological case covered. + (doseq [id [:rpc-main-timing :rpc-management-timing :rpc-climit-timing + :audit-http-handler-timing :http-server-dispatch-timing + :update-file-validation-timing]] + (t/is (= mtx/default-request-histogram-buckets + (get-in main/default-metrics [id ::mdef/buckets])) + (str "request buckets on " id))) + (t/is (= mtx/default-tasks-histogram-buckets + (get-in main/default-metrics [:tasks-timing ::mdef/buckets]))) + (t/is (= mtx/default-redis-histogram-buckets + (get-in main/default-metrics [:redis-eval-timing ::mdef/buckets])))) + +(defn- bucket-value + "Cumulative count of the `le` bucket of a histogram, read from the + registry. Nil when that boundary does not exist." + [metrics mname label-names label-values le] + (.getSampleValue ^CollectorRegistry (mtx/get-registry metrics) + (str mname "_bucket") + (into-array String (conj (vec label-names) "le")) + (into-array String (conj (vec label-values) le)))) + +(t/deftest histogram-tails-are-registered + ;; A 9s observation must land in the new tail instead of piling up in + ;; +Inf: every histogram used to stop at 7500. + (let [metrics (ig/init-key :app.metrics/metrics {:default main/default-metrics})] + (mtx/run! metrics :id :rpc-main-timing :val 9000 :labels (into-array String ["update-file"])) + (t/is (= 0.0 (bucket-value metrics "penpot_rpc_main_timing" ["name"] ["update-file"] "7500.0"))) + (t/is (= 1.0 (bucket-value metrics "penpot_rpc_main_timing" ["name"] ["update-file"] "10000.0"))) + (t/is (nil? (bucket-value metrics "penpot_rpc_main_timing" ["name"] ["update-file"] "120000.0"))) + + (mtx/run! metrics :id :tasks-timing :val 200000 :labels (into-array String ["file-gc"])) + (t/is (= 0.0 (bucket-value metrics "penpot_tasks_timing" ["name"] ["file-gc"] "60000.0"))) + (t/is (= 1.0 (bucket-value metrics "penpot_tasks_timing" ["name"] ["file-gc"] "300000.0"))) + + (mtx/run! metrics :id :redis-eval-timing :val 9000 :labels (into-array String ["claim"])) + (t/is (= 1.0 (bucket-value metrics "penpot_redis_eval_timing" ["name"] ["claim"] "10000.0"))) + (t/is (nil? (bucket-value metrics "penpot_redis_eval_timing" ["name"] ["claim"] "15000.0"))) + + (mtx/run! metrics :id :http-server-dispatch-timing :val 45000) + (t/is (= 1.0 (bucket-value metrics "penpot_http_server_dispatch_timing" [] [] "60000.0"))) + + (mtx/run! metrics :id :update-file-validation-timing :val 9000 :labels (into-array String ["validate-affected"])) + (t/is (= 0.0 (bucket-value metrics "penpot_rpc_update_file_validation_timing" ["step"] ["validate-affected"] "7500.0"))) + (t/is (= 1.0 (bucket-value metrics "penpot_rpc_update_file_validation_timing" ["step"] ["validate-affected"] "10000.0"))))) diff --git a/backend/test/backend_tests/scripts_decode_file_data_test.clj b/backend/test/backend_tests/scripts_decode_file_data_test.clj new file mode 100644 index 0000000000..e14618542f --- /dev/null +++ b/backend/test/backend_tests/scripts_decode_file_data_test.clj @@ -0,0 +1,82 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns backend-tests.scripts-decode-file-data-test + (:require + [app.common.json :as json] + [app.common.uuid :as uuid] + [app.scripts.decode-file-data :as dfd] + [app.util.blob :as blob] + [clojure.java.io :as io] + [clojure.test :as t]) + (:import + java.util.Base64)) + +(defn- sample-file-data + [] + (let [p1 (uuid/next) + p2 (uuid/next)] + {:pages [p1 p2] + :pages-index + {p1 {:id p1 :name "Accueil1" :objects {:a 1 :b 2}} + p2 {:id p2 :name "Contact" :objects {:c 3}}} + :components {(uuid/next) {:name "comp"}} + :colors {(uuid/next) {:name "red"}} + :typographies {}})) + +(t/deftest sanitize-converts-special-types + (let [id (uuid/next) + out (dfd/sanitize {:id id + :kw :pages-index + :when (java.util.Date. 0) + :tags #{:a :b} + :raw (.getBytes "hi" "UTF-8")})] + (t/is (= (str id) (get out "id"))) + (t/is (= "pages-index" (get out "kw"))) + (t/is (string? (get out "when"))) + (t/is (= ["a" "b"] (sort (get out "tags")))) + (t/is (= {"$bytes" (.encodeToString (Base64/getEncoder) (.getBytes "hi" "UTF-8"))} + (get out "raw"))))) + +(t/deftest sanitize-keeps-integers-as-integers + (t/is (= 2 (dfd/sanitize 2))) + (t/is (= 2.5 (dfd/sanitize 2.5)))) + +(t/deftest blob-roundtrip-all-versions-to-json + (doseq [version [1 3 4 5]] + (let [data (sample-file-data) + encoded (blob/encode data {:version version}) + payload (dfd/blob->json encoded) + parsed (json/decode payload)] + (t/is (map? parsed) (str "version " version " produces a JSON map")) + (t/is (= 2 (count (get parsed "pages"))) (str "version " version " keeps pages"))))) + +(t/deftest summary-lists-pages-in-order-with-object-counts + (let [data (sample-file-data) + summary (dfd/summarize-data data) + pages (:pages summary)] + (t/is (= 2 (count pages))) + (t/is (= ["Accueil1" "Contact"] (mapv :name pages))) + (t/is (= [2 1] (mapv :objects pages))) + (t/is (= {:pages 2 :components 1 :colors 1 :typographies 0} + (:stats summary))) + (let [pages-json (json/decode (dfd/file-data-pages-json data))] + (t/is (= ["Accueil1" "Contact"] (mapv #(get % "name") pages-json)))))) + +(t/deftest cli-decodes-blob-file-to-json + (let [data (sample-file-data) + tmp-in (java.io.File/createTempFile "blob" ".bin") + tmp-out (java.io.File/createTempFile "out" ".json") + _ (.deleteOnExit tmp-in) + _ (.deleteOnExit tmp-out) + _ (with-open [os (io/output-stream tmp-in)] + (.write os ^bytes (blob/encode data {:version 5}))) + _ (dfd/-main "--input" (.getAbsolutePath tmp-in) + "--output" (.getAbsolutePath tmp-out) + "--format" "summary") + parsed (json/decode (slurp tmp-out))] + (t/is (== 2 (get-in parsed ["stats" "pages"]))) + (t/is (= ["Accueil1" "Contact"] (mapv #(get % "name") (get parsed "pages")))))) From 4c655e6e9e75f8b5499308ca7b6e9f2ba5eb48eb Mon Sep 17 00:00:00 2001 From: Eva Marco Date: Mon, 19 Oct 2026 13:40:26 +0200 Subject: [PATCH 17/30] :bug: Keep the picture in place in the SVG preview of an image crop (#12387) With the SVG renderer, resizing a shape with an image fill and the mod key showed the picture scaled during the drag, as in a normal resize. The crop only showed after releasing the mouse. The SVG preview applies the resize matrix to the shape node and ignores the fill transform change carried by the modifiers. Now the pictures of the shape fill patterns get a matrix that takes them from the old fill transform to the new one, which undoes that resize and keeps them in place. This works for shapes and boards. Closes #12349 AI-assisted-by: claude-opus-5-5 --- common/src/app/common/geom/image_crop.cljc | 16 +++++ .../common_tests/geom_image_crop_test.cljc | 28 ++++++++ .../shapes/frame/dynamic_modifiers.cljs | 64 ++++++++++++++++--- 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/common/src/app/common/geom/image_crop.cljc b/common/src/app/common/geom/image_crop.cljc index fdd9f89527..15ad24b9bd 100644 --- a/common/src/app/common/geom/image_crop.cljc +++ b/common/src/app/common/geom/image_crop.cljc @@ -9,6 +9,7 @@ image fill). The shape is a window over a picture that keeps its size and position; the window never flips." (:require + [app.common.geom.matrix :as gmt] [app.common.math :as mth])) (defn crop-axis @@ -36,3 +37,18 @@ :y (/ (- (* y height) dy) height') :width (/ (* w width) width') :height (/ (* h height) height')})) + +(defn preview-image-matrix + "Return the matrix that takes the picture of a `width` x `height` window + from its box under `transform` to its box under `transform'`, both in + the coordinates of the window before the resize. While dragging, the + SVG preview resizes the whole shape with one matrix; this one, applied + to the picture first, undoes that resize so the picture stays in place." + [transform transform' width height] + (let [x (* (get transform :x 0.0) width) + y (* (get transform :y 0.0) height) + x' (* (get transform' :x 0.0) width) + y' (* (get transform' :y 0.0) height) + sx (/ (get transform' :width 1.0) (get transform :width 1.0)) + sy (/ (get transform' :height 1.0) (get transform :height 1.0))] + (gmt/matrix sx 0 0 sy (- x' (* x sx)) (- y' (* y sy))))) diff --git a/common/test/common_tests/geom_image_crop_test.cljc b/common/test/common_tests/geom_image_crop_test.cljc index cdd30e444f..31c686fbab 100644 --- a/common/test/common_tests/geom_image_crop_test.cljc +++ b/common/test/common_tests/geom_image_crop_test.cljc @@ -7,6 +7,8 @@ (ns common-tests.geom-image-crop-test (:require [app.common.geom.image-crop :as gic] + [app.common.geom.matrix :as gmt] + [app.common.geom.point :as gpt] [app.common.math :as mth] [clojure.test :as t])) @@ -65,3 +67,29 @@ (t/is (close-vals? [(- dy) height] [dy2 h2]) "window back at its original place") (t/is (close-vals? ((juxt :x :y :width :height) tf) ((juxt :x :y :width :height) back))))))) + +(t/deftest preview-image-matrix-test + (let [width 400 + height 300 + tf {:x 0 :y -0.2 :width 1 :height 1.4}] + + (t/testing "the picture keeps its place once the window is resized" + (doseq [[dx dy w' h'] [[0 60 400 240] + [0 300 400 100] + [-50 75 500 150]]] + (let [tf' (gic/update-image-transform tf width height dx dy w' h') + matrix (gic/preview-image-matrix tf tf' width height) + ;; Resize the preview applies to the whole shape: the old + ;; window goes to the new one. + window (-> (gmt/matrix) + (gmt/translate (gpt/point dx dy)) + (gmt/scale (gpt/point (/ w' width) (/ h' height)))) + [x y w h] (picture-box tf [0 0] [width height]) + p1 (gpt/transform (gpt/point x y) (gmt/multiply window matrix)) + p2 (gpt/transform (gpt/point (+ x w) (+ y h)) (gmt/multiply window matrix))] + (t/is (close-vals? [x y (+ x w) (+ y h)] [(:x p1) (:y p1) (:x p2) (:y p2)]) + (str "window " [dx dy w' h']))))) + + (t/testing "no change gives the identity" + (t/is (gmt/close? (gmt/matrix) (gic/preview-image-matrix tf tf width height))) + (t/is (gmt/close? (gmt/matrix) (gic/preview-image-matrix nil nil width height)))))) diff --git a/frontend/src/app/main/ui/workspace/shapes/frame/dynamic_modifiers.cljs b/frontend/src/app/main/ui/workspace/shapes/frame/dynamic_modifiers.cljs index 237ce41434..ace4f726a7 100644 --- a/frontend/src/app/main/ui/workspace/shapes/frame/dynamic_modifiers.cljs +++ b/frontend/src/app/main/ui/workspace/shapes/frame/dynamic_modifiers.cljs @@ -9,6 +9,7 @@ [app.common.data :as d] [app.common.data.macros :as dm] [app.common.files.helpers :as cfh] + [app.common.geom.image-crop :as gic] [app.common.geom.matrix :as gmt] [app.common.geom.point :as gpt] [app.common.geom.rect :as grc] @@ -20,6 +21,7 @@ [app.util.debug :as dbg] [app.util.dom :as dom] [app.util.timers :as ts] + [cuerdas.core :as str] [rumext.v2 :as mf])) (defn get-shape-node @@ -31,6 +33,14 @@ base-node (dom/query base-node (dm/str "#shape-" id))))) +(defn- get-fill-image-nodes + "Retrieve the pictures of the shape image fills, inside its own fill + patterns" + [shape shape-node] + (when (or (some :fill-image (:fills shape)) + (some? (:fill-image shape))) + (dom/query-all shape-node ":scope > defs > pattern image"))) + (defn get-nodes "Retrieve the DOM nodes to apply the matrix transformation" [base-node {:keys [id parent-id] :as shape}] @@ -43,11 +53,13 @@ masking-child? (:masking-child? (meta shape))] (cond frame? - [shape-node - (dom/query shape-node ".frame-children") - (dom/query (dm/str "#thumbnail-container-" id)) - (dom/query (dm/str "#thumbnail-" id)) - (dom/query (dm/str "#frame-title-" id))] + (d/concat-vec + [shape-node + (dom/query shape-node ".frame-children") + (dom/query (dm/str "#thumbnail-container-" id)) + (dom/query (dm/str "#thumbnail-" id)) + (dom/query (dm/str "#frame-title-" id))] + (get-fill-image-nodes shape shape-node)) ;; For groups we don't want to transform the whole group but only ;; its filters/masks @@ -72,9 +84,38 @@ (dom/query (dm/str "#frame-title-" id)))] :else - [shape-node - (when (dbg/enabled? :shape-titles) - (dom/query (dm/str "#frame-title-" id)))])))) + (d/concat-vec + [shape-node + (when (dbg/enabled? :shape-titles) + (dom/query (dm/str "#frame-title-" id)))] + (get-fill-image-nodes shape shape-node)))))) + +(defn- fill-image-matrix + "Return the matrix that keeps the picture of an image fill `node` in + place while `modifiers` change the fill transform (resize with the mod + key). Return nil when the modifiers do not change the fill." + [shape modifiers node] + (let [changes (->> (:structure-parent modifiers) + (filter #(= :change-property (:type %))) + (d/index-by :property :value)) + selrect (dm/get-prop shape :selrect) + id (dom/get-attribute node "id") + + [transform transform' :as change] + (if (some? id) + ;; Fill images have an id ending with the fill index + (when-let [fills' (get changes :fills)] + (let [index (d/parse-integer (last (str/split id "-")))] + [(dm/get-in (nth (:fills shape) index nil) [:fill-image :transform]) + (dm/get-in (nth fills' index nil) [:fill-image :transform])])) + (when-let [fill-image' (get changes :fill-image)] + [(dm/get-in shape [:fill-image :transform]) + (get fill-image' :transform)]))] + + (when (some? change) + (gic/preview-image-matrix transform transform' + (dm/get-prop selrect :width) + (dm/get-prop selrect :height))))) (defn transform-region! [node modifiers] @@ -167,6 +208,13 @@ (= (dom/get-tag-name node) "pattern") (set-transform-att! node "patternTransform" transform) + ;; Picture of an image fill: it gets the shape transform + ;; through the shape node, and only needs its own when the + ;; fill transform changes + (= (dom/get-tag-name node) "image") + (when-let [matrix (fill-image-matrix shape modifiers node)] + (set-transform-att! node "transform" matrix)) + (and (some? transform) (some? node)) (set-transform-att! node "transform" transform))))))) From 688c0fc398c3c35e35991607a5ba1269f297f6d3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 14:00:13 +0200 Subject: [PATCH 18/30] :bug: Keep file_change revisions on their own deletion schedule (#12419) * :bug: Keep file_change revisions on their own deletion schedule Restoring a file from the trash cleared the deletion mark on every file_change row of that file, revisions included. A revision is born already scheduled for deletion one hour later, and objects-gc only looks at rows that carry a date, so unmarking one left it in the table for ever: production had 358,517 of them across 3,614 files. The file deletion cascade had the mirror problem: it rewrote the date of rows that already had one, pushing a revision out to the trash lifetime. Restrict the restore to durable history and date only the rows that have no date yet, so a revision keeps the schedule it was born with. A pinning test covers the snapshot update path, which cannot reach a revision because it resolves its row through a read filter. The backend memories now carry the model of the three file_data classes and this invariant, in place of the two superseded stubs. Closes #12418 AI-assisted-by: deepseek-v4.1-flash * :books: Add autovacuum and Postgres exporter notes to Grafana memory Record what this Grafana instance can and cannot show about autovacuum so the next session does not have to re-derive it: - The Postgres exporter scrapes the primary as well as the read-only replica, so the primary's pg_stat_activity and pg_settings_* are in Prometheus even though the SQL datasource cannot reach them. - It publishes no pg_stat_progress_* collector, and pg_stat_activity_count has no backend_type label: autovacuum workers are the usename/application_name-less bucket with state=active. The launcher shows up as the AutoVacuumMain wait event, and it is absent on the replica. - A metric the exporter does not publish needs a custom query block in its YAML queries file and a restart of the exporter. - The Postgres server ships its own log to Loki (the job is postgresql, primary only), whose streams carry only filename, instance, job and level. The completed vacuum lines live there: a per-table history, not a view of what runs now. - The PostgreSQL section now ends with the vacuum panels, with the Vacuum TXID age table last; an in-flight per-table view would need a pg_stat_progress_vacuum collector in the exporter itself. AI-assisted-by: deepseek-v4.1-flash --- .../auth-permissions-product-domains.md | 4 +- .serena/memories/backend/core.md | 3 +- .../http-storage-filedata-subtleties.md | 31 --- .../backend/rpc-db-worker-subtleties.md | 5 - .serena/memories/backend/subtleties.md | 9 +- .serena/memories/prod-infra/grafana.md | 8 +- backend/src/app/rpc/commands/files.clj | 10 +- backend/src/app/tasks/delete_object.clj | 8 +- .../file_change_retention_test.clj | 209 ++++++++++++++++++ 9 files changed, 242 insertions(+), 45 deletions(-) delete mode 100644 .serena/memories/backend/http-storage-filedata-subtleties.md delete mode 100644 .serena/memories/backend/rpc-db-worker-subtleties.md create mode 100644 backend/test/backend_tests/file_change_retention_test.clj diff --git a/.serena/memories/backend/auth-permissions-product-domains.md b/.serena/memories/backend/auth-permissions-product-domains.md index 130cd62609..7a18794d60 100644 --- a/.serena/memories/backend/auth-permissions-product-domains.md +++ b/.serena/memories/backend/auth-permissions-product-domains.md @@ -9,7 +9,7 @@ - LDAP login validates credentials against the external directory, fetches identity data, then logs in or registers a matching Penpot profile. LDAP registration is not a separate Penpot signup flow. - Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI. - Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation. -- HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/http-storage-filedata-subtleties`. +- HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/subtleties`. ## Permission model @@ -38,4 +38,4 @@ - Enable LDAP login locally with frontend flag `enable-login-with-ldap`; the devenv includes a configured test LDAP service. - OIDC testing requires external provider app credentials plus matching backend/frontend config. - Backend domain tests usually live under `backend/test/backend_tests/rpc/commands/*_test.clj` or nearby backend test namespaces. Use focused `clojure -M:dev:test --focus ...` from `backend/` when possible. -- For auth/session or HTTP behavior, combine backend tests with the HTTP/session notes in `mem:backend/http-storage-filedata-subtleties` because RPC-level tests may not exercise cookie/header transforms. +- For auth/session or HTTP behavior, combine backend tests with the HTTP/session notes in `mem:backend/subtleties` because RPC-level tests may not exercise cookie/header transforms. diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index a8756775a8..42e7c7b756 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -4,9 +4,8 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m ## Focused memories -- RPC, DB helpers, workers, cron, CLIMIT: `mem:backend/subtleties` +- RPC, DB helpers, workers, cron, CLIMIT, config and HTTP/session middleware, http self-metrics, media processing, and file data persistence: `mem:backend/subtleties` - Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`. -- HTTP sessions, config, media processing, and file data persistence: `mem:backend/http-storage-filedata-subtleties`. - Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains` - Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:backend/audit-log` - Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`. diff --git a/.serena/memories/backend/http-storage-filedata-subtleties.md b/.serena/memories/backend/http-storage-filedata-subtleties.md deleted file mode 100644 index fb735b6c56..0000000000 --- a/.serena/memories/backend/http-storage-filedata-subtleties.md +++ /dev/null @@ -1,31 +0,0 @@ -# Backend HTTP, Storage, Media, and File Data Subtleties - -## Config and HTTP/session middleware - -- `app.config/config` and `flags` are dynamic `defonce` vars populated from `PENPOT_*` env vars through the shared schema string transformer. Tests and tooling can bind them. -- `parse-flags` automatically adds `:disable-secure-session-cookies` when `public-uri` is plain HTTP and not localhost. This changes cookie defaults without an explicit env flag. -- The backend sets Clojure `*assert*` globally from the `:backend-asserts` feature flag. Assertion-dependent checks can therefore differ by runtime flags. -- Request body parsing is mostly POST-oriented and supports Transit JSON plus plain JSON. Plain JSON request keys are kebab-decoded before being merged into `:params`. -- Response formatting negotiates with `Accept` or `_fmt=json`. Transit is the default for collection/boolean bodies; JSON encoding has special pointer-map handling. -- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`; JWTs with `kid=1` and `ver=1` are decoded as v1 session tokens, otherwise they are treated as legacy tokens. -- Shared-key auth requires `x-shared-key` as ` ` and stores the lowercased key id on the request. If no shared keys are configured it always rejects. -- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. DB sessions support both legacy string ids and v2 UUID session ids. -- Session cookies are renewed when using a legacy string id or when `modified-at` is older than the renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. - -## Storage and media - -- Storage abstraction, backend configuration, logical buckets, object lifecycle, deduplication, access rules, and garbage collection: `mem:backend/storage`. -- SVG validation strips DOCTYPE and uses secure SAX parsing. Basic SVG info falls back to 100x100 dimensions when width/height/viewBox are missing. -- Raster metadata is shell-derived with ImageMagick `identify`, verifies detected MIME against the supplied MIME, and swaps dimensions for EXIF orientations 6/8. -- Remote image download requires 2xx status, `content-length`, a known MIME, and size under the configured maximum before writing the temp file; mismatched byte count is an internal error. -- Font processing shells out to FontForge and WOFF conversion tools and can derive TTF/OTF/WOFF variants from uploaded fonts. - -## File data persistence - -- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data. -- `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob. -- Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. -- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only written by the `legacy-db` backend, plus the rows written before the 2025-07 file storage refactor. -- Tell a snapshot apart by `label IS NOT NULL` (with `created_by` / `deleted_at` as needed), never by `file_change.data IS NOT NULL`: the label holds with every file data backend, and the payload column stopped meaning "this is a snapshot" when the payload moved out. Queries that identify versions must use the label. -- The version quota counts of `app.rpc.quotes` (`sql:get-snapshots-per-file`, `sql:get-snapshots-per-team`) are served by the partial index `file_change__user_snapshots__idx`, whose predicate mirrors their where clause; changing one without the other drops the index only scan and the count walks the whole table again. -- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. diff --git a/.serena/memories/backend/rpc-db-worker-subtleties.md b/.serena/memories/backend/rpc-db-worker-subtleties.md deleted file mode 100644 index 9ee1d462c1..0000000000 --- a/.serena/memories/backend/rpc-db-worker-subtleties.md +++ /dev/null @@ -1,5 +0,0 @@ -# Backend RPC/DB/Worker Subtleties (superseded) - -This memory was superseded by `mem:backend/subtleties`, which holds everything it had (RPC wrappers, DB helpers, workers and cron) plus the CLIMIT notes, config/HTTP middleware, http self-metrics, storage and file data. Read that one. - -Nothing should point here any more. diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md index ed7f390af6..cee764853a 100644 --- a/.serena/memories/backend/subtleties.md +++ b/.serena/memories/backend/subtleties.md @@ -74,4 +74,11 @@ - File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data. - `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob. - Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. -- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. \ No newline at end of file +- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. +- A `file_data` row is one of three classes, told apart by `type`: `main` is the file's current state and carries `id = file_id`; `snapshot` is a point-in-time copy whose `id` is the one its `file_change` row shares; `fragment` is a piece of the pointer-map, referenced from the file's data. +- Their lifecycles differ: `main` and `snapshot` live with the file (dated when it is deleted, undated when it is restored), while a `fragment` carries no date and is collected by `file_gc` once the file stops referencing it. Neither is a session buffer, so do not read a `fragment` as an ephemeral revision. +- File snapshots (versions) keep their metadata row in `file_change` and their payload in `file_data` (`type = 'snapshot'`). With the `db` backend `file_change.data` stays NULL: that column is only written by the `legacy-db` backend, plus the rows written before the 2025-07 file storage refactor. +- Tell a snapshot apart by `label IS NOT NULL` (with `created_by` / `deleted_at` as needed), never by `file_change.data IS NOT NULL`: the label holds with every file data backend, and the payload column stopped meaning "this is a snapshot" when the payload moved out. Queries that identify versions must use the label. +- The version quota counts of `app.rpc.quotes` (`sql:get-snapshots-per-file`, `sql:get-snapshots-per-team`) are served by the partial index `file_change__user_snapshots__idx`, whose predicate mirrors their where clause; changing one without the other drops the index only scan and the count walks the whole table again. +- Revisions live only in `file_change` and have no `file_data` counterpart. They are born already scheduled for deletion one hour later (`app.rpc.commands.files-update`) and the snapshot paths cannot reach one, because they resolve their row through a `label IS NOT NULL` filter. +- A deletion date is assigned once and never moved: `restore-files` (`app.rpc.commands.files`) unmarks only durable history, the `:file` method of `app.tasks.delete-object` dates only rows that have no date yet, and `objects-gc` collects only rows that carry one. Unmarking a revision leaves it invisible to `objects-gc` for ever, because nothing sets that date again. \ No newline at end of file diff --git a/.serena/memories/prod-infra/grafana.md b/.serena/memories/prod-infra/grafana.md index 085308da1b..533b09516f 100644 --- a/.serena/memories/prod-infra/grafana.md +++ b/.serena/memories/prod-infra/grafana.md @@ -7,8 +7,13 @@ Keep this memory free of internal names: no host names, no deployed versions, no ## Datasources - Metrics — **Prometheus**, uid `xkRmyFBGk`. PromQL. Metric families: `penpot_*` (RPC, task timings, CLIMIT, HTTP, websocket, storage, and the application log events of the backend), `hikaricp_*`, `jvm_*` and `process_*` on the backend (`job="app"`); `pg_*` from the Postgres exporter (`job="postgres"`); `redis_*` from the Redis exporter (`job="redis"`); `node_*` and `container_*` from node-exporter and cAdvisor. Backend histograms are in **milliseconds**. Tails (ms): requests and waits to 60000, background tasks to 300000, Redis eval to 10000; the sub-7500 buckets are identical everywhere, so old quantiles stay comparable. + - The Postgres exporter scrapes **the primary as well as the read-only replica**, so the primary's `pg_stat_activity`, `pg_settings_*` and `pg_stat_bgwriter_*` are in Prometheus even though the SQL datasource cannot reach them (the replica's views describe the replica, not the primary). + - Per-database data from the exporter is **aggregate only**: `pg_settings_*` (the whole server configuration, so autovacuum, checkpoint and memory parameters are readable from Prometheus), `pg_stat_database_*`, `pg_stat_bgwriter_*`, `pg_stat_activity_*`, and replication/lock counts. It publishes **no per-table series** (`pg_stat_user_tables_*` resolves empty), so dead tuples, per-table reloptions and relation sizes are SQL-only questions. + - It publishes **no `pg_stat_progress_*` collector** and `pg_stat_activity_count` carries no `backend_type` label, so a session is only identifiable by its remaining labels: autovacuum workers are the bucket with `usename` and `application_name` empty plus `state="active"`, and `pg_wait_events_connections{wait_event="AutoVacuumMain"}` is the autovacuum launcher alive (absent on the replica). + - A metric the exporter does not publish itself has to be added as a custom query block in its YAML queries file on the exporter host (that is where `pg_wait_events_connections` comes from), and in this build the file is read at startup: restart the exporter after editing. There is no per-metric flag to turn on. - Logs — **Loki**, uid `h3CGKhEMk`. LogQL, and filter by the structured metadata `logger`/`level` (`{job="penpot"} | logger="..." | level="ERROR"`). Two things to remember when counting from it: the CLIMIT rejections have their own counters (`penpot_rpc_climit_rejected_total`, by `name` and `reason`) and their `hint="rejected"` line goes out at WARN only when something is left queued behind the call (a lone caller that times out is written at TRACE), so counting lines is not counting rejections -- the same goes for `hint="enqueued"` and `hint="acquired"`; and the only signal that exists nowhere else is the profile behind a queue (`logger="app.rpc.climit"`, whose `id` carries the key), chased there on demand. -- SQL — **Postgres**, uid `y8ea9tfMk`. Panels need `rawQuery: true`, `editorMode: "code"` and `format: "table"`. The `task` table is the only truth about the queue depth. It is a **read-only replica**: it rejects any write, `CREATE TEMP TABLE` included, so queries through it must be plain `SELECT`s. It also cuts long-running queries (the MCP path has a short timeout), so bound them (e.g. a list of ids); an empty answer there is not proof that there is no data. + - The Postgres server also ships its own log (`job="postgresql"`, primary only). Those streams carry just `filename`, `instance`, `job` and `level` — no `environment`/`hostname` — so selectors need `instance=~"$host.$env.*"`. `log_autovacuum_min_duration` is low enough that every autovacuum leaves an `automatic vacuum of table "db.schema.table"` line with pages, tuples, index scans and `elapsed`: a per-table history of **finished** vacuums (retrospective, not a view of what runs now). A day of vacuums is only a couple of hundred lines, so pulling them raw over `query_range` and parsing the table, the pages scanned, `tuples: N removed` and the `removable cutoff` XID age in the client works well — less fighting than making LogQL's `pattern`/`regexp` chew a multi-line message. +- SQL — **Postgres**, uid `y8ea9tfMk`. Panels need `rawQuery: true`, `editorMode: "code"` and `format: "table"`. The `task` table is the only truth about the queue depth. It is a **read-only replica**: it rejects any write, `CREATE TEMP TABLE` included, so queries through it must be plain `SELECT`s. Catalog data (`pg_class`, `pg_partition_tree`, `pg_stats`, and `reloptions`) describes the primary faithfully, because catalogs replicate; the statistics views (`pg_stat_user_tables`, `pg_stat_all_tables`) read as zero there, since statistics are per-instance and are not replicated. It also cuts long-running queries (the MCP path has a short timeout), so bound them (e.g. a list of ids); an empty answer there is not proof that there is no data. - **Postgres (telemetry)**, uid `9YAUuM_Gz`. - `$__rate_interval` and `$__interval` are substituted by Grafana; use them instead of fixed windows in graphs. @@ -16,6 +21,7 @@ Keep this memory free of internal names: no host names, no deployed versions, no - `Production status`, uid `penpot-production-status`. Sections: general state and deploy, `RPC y CLIMIT`, PostgreSQL and pool, Redis/msgbus, tasks, JVM/process. Variables `$env` and `$host` (custom), no auto refresh. Every section opens collapsed except `RPC y CLIMIT`, which is the one read on a normal day. - Per-host investigation dashboards are separate, and the alert rules deeplink into them. +- The PostgreSQL section ends with a block of vacuum panels: two stats (autovacuum workers active on the primary, and the launcher alive), a Loki timeseries of finished vacuums per interval and table, and then the `Vacuum TXID age` table, which is the last panel of the section. The worker stat is an approximation — it counts the `pg_stat_activity` bucket described above, because the exporter publishes no `backend_type` — and the Loki panel is retrospective by construction. A per-table, in-flight view (table, phase, progress) needs a `pg_stat_progress_vacuum` collector added to the exporter itself. ## Conventions diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index c5169643bc..584ba73e35 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -1305,8 +1305,16 @@ WHERE file_id = ANY(?::uuid[])") (def ^:private sql:restore-file-changes + ;; Only durable history comes back: a snapshot (carries `label`) or a legacy + ;; payload row (carries `data`, the column the new model replaces with + ;; `file_data`). Revisions have neither and are deliberately left alone: they + ;; are born already scheduled for deletion one hour later, and the collector + ;; only looks at rows that carry a date, so unmarking one here would keep it + ;; in the table for ever. The `data` half is a guard for the rows still + ;; pending migration, not a dependency: drop it with the last one. "UPDATE file_change SET deleted_at = null - WHERE file_id = ANY(?::uuid[])") + WHERE file_id = ANY(?::uuid[]) + AND (label IS NOT NULL OR data IS NOT NULL)") (def ^:private sql:restore-file-data "UPDATE file_data SET deleted_at = null diff --git a/backend/src/app/tasks/delete_object.clj b/backend/src/app/tasks/delete_object.clj index 00d0046f27..b6a30e2092 100644 --- a/backend/src/app/tasks/delete_object.clj +++ b/backend/src/app/tasks/delete_object.clj @@ -60,10 +60,14 @@ :file-id id :cause cause)))) - ;; Mark file change to be deleted + ;; Mark file change to be deleted, but only the rows that do not carry a + ;; date yet. A revision is born already scheduled for deletion one hour + ;; later, and rewriting its date here would push it out to the trash + ;; lifetime; the collector would still take it, just much later than it + ;; should. (db/update! conn :file-change {:deleted-at deleted-at} - {:file-id id} + {:file-id id :deleted-at nil} {::db/return-keys false}) ;; Mark file data fragment to be deleted diff --git a/backend/test/backend_tests/file_change_retention_test.clj b/backend/test/backend_tests/file_change_retention_test.clj new file mode 100644 index 0000000000..7cede5a164 --- /dev/null +++ b/backend/test/backend_tests/file_change_retention_test.clj @@ -0,0 +1,209 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.file-change-retention-test + "Retention of `file_change` rows through the file lifecycle. + + A file has two kinds of row. Durable history: the snapshots, which a restore + brings back. And ephemeral revisions: the xlog of an edit, born already + scheduled for deletion one hour later, which nobody needs once the sessions + that were going to read them are gone. + + The invariant under test is that a deletion date is assigned once and never + moved: restoring a file must not unmark a revision, and deleting a file must + not push a revision's date out to the trash lifetime. When either happens the + revision becomes invisible to `objects-gc`, which only looks at rows that + carry a date, and it stays in the table for ever." + (:require + [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.db :as db] + [app.rpc :as-alias rpc] + [app.rpc.commands.files :as files] + [app.util.blob :as blob] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(def ^:private test-now + "The instant every test pins the clock to. Whole seconds on purpose: a value + with a sub-second part does not survive the round trip through the database, + and the test could not compare it for equality." + (ct/inst "2026-10-01T00:00:00Z")) + +(def ^:private revision-lifetime + "The lifetime the `update-file` flow gives a revision when it inserts it." + (ct/duration {:hours 1})) + +(defn- create-file + [i] + (let [profile (th/create-profile* i {:is-active true})] + [profile (th/create-file* i {:profile-id (:id profile) + :project-id (:default-project-id profile)})])) + +(defn- insert-change + "Inserts a `file_change` row directly, so the test builds the exact shape it + needs. The row class is decided by what the caller passes: a revision carries + only `changes`, a snapshot carries `label`, and a legacy payload row carries + `data` without a label. Returns the new row id." + [file-id {:keys [label data changes created-at deleted-at created-by]}] + (let [created-at (or created-at (ct/now)) + id (uuid/next)] + (th/db-insert! :file-change + {:id id + :file-id file-id + :created-at created-at + :updated-at created-at + :deleted-at deleted-at + :revn 1 + :version 0 + :created-by (or created-by "system") + :label label + :data data + :changes changes} + {::db/return-keys false}) + id)) + +(defn- insert-revision + "A revision exactly as the `update-file` flow writes it: no label, no data and + `changes` filled, already scheduled for deletion one hour after creation." + [file-id created-at] + (insert-change file-id {:created-at created-at + :changes (blob/encode {:foo "bar"}) + :deleted-at (ct/plus created-at revision-lifetime)})) + +(defn- find-change + [id] + (th/db-get :file-change {:id id} {::db/remove-deleted false})) + +(defn- find-file + [id] + (th/db-get :file {:id id} {::db/remove-deleted false})) + +(defn- restore-file + [file-id] + (db/tx-run! th/*system* + (fn [{:keys [::db/conn]}] + (#'files/restore-files conn [file-id])))) + +(t/deftest restoring-a-file-leaves-the-revision-schedule-alone + (binding [ct/*clock* (ct/fixed-clock test-now)] + (let [[_ file] (create-file 1) + file-id (:id file) + trash-at (ct/in-future {:days 7}) + old-at (ct/in-past {:hours 2}) + + revision-id (insert-revision file-id old-at) + revision-at (:deleted-at (find-change revision-id)) + + snapshot-id (insert-change file-id {:created-at old-at + :created-by "user" + :label "a snapshot" + :deleted-at trash-at}) + + legacy-id (insert-change file-id {:created-at old-at + :data (blob/encode {:legacy true}) + :deleted-at trash-at})] + + (restore-file file-id) + + (t/testing "the revision keeps the schedule it was born with" + (t/is (= revision-at (:deleted-at (find-change revision-id))))) + + (t/testing "the snapshot comes back" + (t/is (nil? (:deleted-at (find-change snapshot-id))))) + + (t/testing "the legacy payload row comes back" + (t/is (nil? (:deleted-at (find-change legacy-id)))))))) + +(t/deftest deleting-a-file-does-not-push-the-revision-schedule-out + (binding [ct/*clock* (ct/fixed-clock test-now)] + (let [[_ file] (create-file 1) + file-id (:id file) + trash-at (ct/in-future {:days 7}) + old-at (ct/in-past {:hours 2}) + system-at (ct/in-future {:days 2}) + + revision-id (insert-revision file-id old-at) + revision-at (:deleted-at (find-change revision-id)) + + snapshot-id (insert-change file-id {:created-at old-at + :created-by "user" + :label "a snapshot"}) + + system-id (insert-change file-id {:created-at old-at + :label "internal/snapshot/1" + :deleted-at system-at})] + + (th/run-task! :delete-object {:object :file + :deleted-at trash-at + :id file-id}) + + (t/testing "the revision keeps the schedule it was born with" + (t/is (= revision-at (:deleted-at (find-change revision-id))))) + + (t/testing "the snapshot takes the file's deletion date" + (t/is (= (:deleted-at (find-file file-id)) + (:deleted-at (find-change snapshot-id))))) + + (t/testing "the system snapshot keeps the date it already had" + (t/is (= system-at (:deleted-at (find-change system-id)))))))) + +(t/deftest updating-a-snapshot-cannot-reach-a-revision + ;; The snapshot paths clear the deletion mark of the row they touch, which is + ;; only legitimate for durable history. They reach a row through + ;; `sql:snapshots`, which filters `label IS NOT NULL`, and a revision never + ;; carries one. This test pins that: the row has to keep the date it was born + ;; with, and the RPC has to reject the id. If a future change loosens that + ;; read filter, this goes red instead of the revision losing its date in + ;; silence. + (binding [ct/*clock* (ct/fixed-clock test-now)] + (let [[profile file] (create-file 1) + file-id (:id file) + revision-id (insert-revision file-id (ct/in-past {:hours 2})) + revision-at (:deleted-at (find-change revision-id)) + + out (th/command! {::th/type :update-file-snapshot + ::rpc/profile-id (:id profile) + :id revision-id + :label "renamed"})] + + (t/testing "the RPC rejects the id" + (t/is (th/ex-info? (:error out))) + (t/is (= :not-found (:type (ex-data (:error out)))))) + + (t/testing "and the revision keeps the date it was born with" + (t/is (= revision-at (:deleted-at (find-change revision-id)))))))) + +(t/deftest a-deleted-then-restored-file-does-not-leak-its-revisions + ;; The lifecycle that leaked 358,517 rows in production: a file is edited, it + ;; ends up in the trash, it comes back, and the revisions that were scheduled + ;; to die an hour after they were written have to die anyway. + (binding [ct/*clock* (ct/fixed-clock test-now)] + (let [[_ file] (create-file 1) + file-id (:id file) + + revision-id (insert-revision file-id (ct/in-past {:hours 2})) + revision-at (:deleted-at (find-change revision-id))] + + (th/run-task! :delete-object {:object :file + :deleted-at (ct/in-future {:days 7}) + :id file-id}) + + (t/testing "deleting the file leaves the revision schedule alone" + (t/is (= revision-at (:deleted-at (find-change revision-id))))) + + (restore-file file-id) + + (t/testing "restoring the file leaves it alone too" + (t/is (= revision-at (:deleted-at (find-change revision-id))))) + + (th/run-task! :objects-gc {}) + + (t/testing "so the collector takes it" + (t/is (nil? (find-change revision-id))))))) From c99980327fd7630a143b82006f9f5736e5d9ceb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bel=C3=A9n=20Albeza?= Date: Mon, 19 Oct 2026 15:06:56 +0200 Subject: [PATCH 19/30] :bug: Resize fit-content flex boards while typing in v3 text editor (#12330) * :bug: Resize fit-content flex boards while typing in v3 text editor Typing in an auto-width text did not resize its fit-content layout parent until the editor closed. The v3 editor now previews the live text size in the layout on each keystroke, also when the text sits in a group. * :bug: Preview text layout when typing with a pending caret style Both v3 editor commit paths share v3-update-editor-content. * :bug: Skip the layout preview for absolute texts Absolute layout items do not affect their board. * :zap: Preview text layout only when the text size changes Each preview propagates modifiers, so skip unchanged sizes. * :bug: Leave interactive render mode when the v3 editor closes Finalize only ends it when it applies geometry. --- .../data/text-editor/get-file-12325.json | 727 ++++++++++++++++++ .../ui/specs/text-editor-v3.spec.js | 74 +- .../src/app/main/data/workspace/texts.cljs | 73 +- .../ui/workspace/shapes/text/v3_editor.cljs | 9 +- 4 files changed, 866 insertions(+), 17 deletions(-) create mode 100644 frontend/playwright/data/text-editor/get-file-12325.json diff --git a/frontend/playwright/data/text-editor/get-file-12325.json b/frontend/playwright/data/text-editor/get-file-12325.json new file mode 100644 index 0000000000..2c039f5bb8 --- /dev/null +++ b/frontend/playwright/data/text-editor/get-file-12325.json @@ -0,0 +1,727 @@ +{ + "~:features": { + "~#set": [ + "fdata/path-data", + "plugins/runtime", + "design-tokens/v1", + "layout/grid", + "styles/v2", + "fdata/pointer-map", + "fdata/objects-map", + "components/v2", + "fdata/shape-data-type", + "text-editor/v2" + ] + }, + "~:team-id": "~u9e6e22b2-db76-81d6-8006-75d7cdbb8bad", + "~:permissions": { + "~:type": "~:membership", + "~:is-owner": true, + "~:is-admin": true, + "~:can-edit": true, + "~:can-read": true, + "~:is-logged": true + }, + "~:has-media-trimmed": false, + "~:comment-thread-seqn": 0, + "~:name": "Bug 12325", + "~:revn": 3, + "~:modified-at": "~m1753957736516", + "~:vern": 0, + "~:id": "~u238a17e0-75ff-8075-8006-934586ea2230", + "~:is-shared": false, + "~:migrations": { + "~#ordered-set": [ + "legacy-2", + "legacy-3", + "legacy-5", + "legacy-6", + "legacy-7", + "legacy-8", + "legacy-9", + "legacy-10", + "legacy-11", + "legacy-12", + "legacy-13", + "legacy-14", + "legacy-16", + "legacy-17", + "legacy-18", + "legacy-19", + "legacy-25", + "legacy-26", + "legacy-27", + "legacy-28", + "legacy-29", + "legacy-31", + "legacy-32", + "legacy-33", + "legacy-34", + "legacy-36", + "legacy-37", + "legacy-38", + "legacy-39", + "legacy-40", + "legacy-41", + "legacy-42", + "legacy-43", + "legacy-44", + "legacy-45", + "legacy-46", + "legacy-47", + "legacy-48", + "legacy-49", + "legacy-50", + "legacy-51", + "legacy-52", + "legacy-53", + "legacy-54", + "legacy-55", + "legacy-56", + "legacy-57", + "legacy-59", + "legacy-62", + "legacy-65", + "legacy-66", + "legacy-67", + "0001-remove-tokens-from-groups", + "0002-normalize-bool-content-v2", + "0002-clean-shape-interactions", + "0003-fix-root-shape", + "0003-convert-path-content-v2", + "0004-clean-shadow-color", + "0005-deprecate-image-type", + "0006-fix-old-texts-fills", + "0007-clear-invalid-strokes-and-fills-v2", + "0008-fix-library-colors-v4", + "0009-clean-library-colors", + "0009-add-partial-text-touched-flags" + ] + }, + "~:version": 67, + "~:project-id": "~u9e6e22b2-db76-81d6-8006-75d7cdc30669", + "~:created-at": "~m1753957644225", + "~:data": { + "~:pages": ["~u238a17e0-75ff-8075-8006-934586ea2231"], + "~:pages-index": { + "~u238a17e0-75ff-8075-8006-934586ea2231": { + "~:objects": { + "~u00000000-0000-0000-0000-000000000000": { + "~#shape": { + "~:y": 0, + "~:hide-fill-on-export": false, + "~:transform": { + "~#matrix": { + "~:a": 1.0, + "~:b": 0.0, + "~:c": 0.0, + "~:d": 1.0, + "~:e": 0.0, + "~:f": 0.0 + } + }, + "~:rotation": 0, + "~:name": "Root Frame", + "~:width": 0.01, + "~:type": "~:frame", + "~:points": [ + { + "~#point": { + "~:x": 0.0, + "~:y": 0.0 + } + }, + { + "~#point": { + "~:x": 0.01, + "~:y": 0.0 + } + }, + { + "~#point": { + "~:x": 0.01, + "~:y": 0.01 + } + }, + { + "~#point": { + "~:x": 0.0, + "~:y": 0.01 + } + } + ], + "~:r2": 0, + "~:proportion-lock": false, + "~:transform-inverse": { + "~#matrix": { + "~:a": 1.0, + "~:b": 0.0, + "~:c": 0.0, + "~:d": 1.0, + "~:e": 0.0, + "~:f": 0.0 + } + }, + "~:r3": 0, + "~:r1": 0, + "~:id": "~u00000000-0000-0000-0000-000000000000", + "~:parent-id": "~u00000000-0000-0000-0000-000000000000", + "~:frame-id": "~u00000000-0000-0000-0000-000000000000", + "~:strokes": [], + "~:x": 0, + "~:proportion": 1.0, + "~:r4": 0, + "~:selrect": { + "~#rect": { + "~:x": 0, + "~:y": 0, + "~:width": 0.01, + "~:height": 0.01, + "~:x1": 0, + "~:y1": 0, + "~:x2": 0.01, + "~:y2": 0.01 + } + }, + "~:fills": [ + { + "~:fill-color": "#FFFFFF", + "~:fill-opacity": 1 + } + ], + "~:flip-x": null, + "~:height": 0.01, + "~:flip-y": null, + "~:shapes": [ + "~uc0a701d6-7df9-80ac-8008-c9e6c5b7f7f4", + "~uc0a701d6-7df9-80ac-8008-c9e6c8b2a9e2" + ] + } + }, + "~uc0a701d6-7df9-80ac-8008-c9e6c5b7f7f4": { + "~#shape": { + "~:id": "~uc0a701d6-7df9-80ac-8008-c9e6c5b7f7f4", + "~:name": "Text board", + "~:type": "~:frame", + "~:parent-id": "~u00000000-0000-0000-0000-000000000000", + "~:frame-id": "~u00000000-0000-0000-0000-000000000000", + "~:shapes": ["~uc0a701d6-7df9-80ac-8008-c9e6c47fbce4"], + "~:fills": [ + { + "~:fill-color": "#B1B2B5", + "~:fill-opacity": 1 + } + ], + "~:strokes": [], + "~:hide-fill-on-export": false, + "~:show-content": true, + "~:proportion-lock": false, + "~:proportion": 1, + "~:r1": 0, + "~:r2": 0, + "~:r3": 0, + "~:r4": 0, + "~:layout": "~:flex", + "~:layout-flex-dir": "~:row", + "~:layout-wrap-type": "~:nowrap", + "~:layout-gap-type": "~:multiple", + "~:layout-gap": { + "~:row-gap": 0, + "~:column-gap": 0 + }, + "~:layout-padding-type": "~:simple", + "~:layout-padding": { + "~:p1": 0, + "~:p2": 0, + "~:p3": 0, + "~:p4": 0 + }, + "~:layout-align-items": "~:start", + "~:layout-justify-content": "~:start", + "~:layout-align-content": "~:stretch", + "~:layout-item-h-sizing": "~:auto", + "~:layout-item-v-sizing": "~:auto", + "~:x": 0, + "~:y": 0, + "~:width": 99.99999904632568, + "~:height": 17, + "~:transform": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:transform-inverse": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:rotation": 0, + "~:points": [ + { + "~#point": { + "~:x": 0, + "~:y": 0 + } + }, + { + "~#point": { + "~:x": 99.99999904632568, + "~:y": 0 + } + }, + { + "~#point": { + "~:x": 99.99999904632568, + "~:y": 17 + } + }, + { + "~#point": { + "~:x": 0, + "~:y": 17 + } + } + ], + "~:selrect": { + "~#rect": { + "~:x": 0, + "~:y": 0, + "~:width": 99.99999904632568, + "~:height": 17, + "~:x1": 0, + "~:y1": 0, + "~:x2": 99.99999904632568, + "~:y2": 17 + } + } + } + }, + "~uc0a701d6-7df9-80ac-8008-c9e6c47fbce4": { + "~#shape": { + "~:id": "~uc0a701d6-7df9-80ac-8008-c9e6c47fbce4", + "~:name": "Hello wide world", + "~:type": "~:text", + "~:grow-type": "~:auto-width", + "~:parent-id": "~uc0a701d6-7df9-80ac-8008-c9e6c5b7f7f4", + "~:frame-id": "~uc0a701d6-7df9-80ac-8008-c9e6c5b7f7f4", + "~:content": { + "~:type": "root", + "~:children": [ + { + "~:type": "paragraph-set", + "~:children": [ + { + "~:type": "paragraph", + "~:children": [ + { + "~:line-height": "1.2", + "~:font-style": "normal", + "~:typography-ref-id": null, + "~:text-transform": "none", + "~:text-align": "left", + "~:font-id": "sourcesanspro", + "~:font-size": "14", + "~:font-weight": "400", + "~:typography-ref-file": null, + "~:text-direction": "ltr", + "~:font-variant-id": "regular", + "~:text-decoration": "none", + "~:letter-spacing": "0", + "~:fills": [ + { + "~:fill-color": "#000000", + "~:fill-opacity": 1 + } + ], + "~:font-family": "sourcesanspro", + "~:text": "Hello wide world" + } + ] + } + ] + } + ] + }, + "~:x": 0, + "~:y": 0, + "~:width": 99.99999904632568, + "~:height": 17, + "~:transform": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:transform-inverse": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:rotation": 0, + "~:points": [ + { + "~#point": { + "~:x": 0, + "~:y": 0 + } + }, + { + "~#point": { + "~:x": 99.99999904632568, + "~:y": 0 + } + }, + { + "~#point": { + "~:x": 99.99999904632568, + "~:y": 17 + } + }, + { + "~#point": { + "~:x": 0, + "~:y": 17 + } + } + ], + "~:selrect": { + "~#rect": { + "~:x": 0, + "~:y": 0, + "~:width": 99.99999904632568, + "~:height": 17, + "~:x1": 0, + "~:y1": 0, + "~:x2": 99.99999904632568, + "~:y2": 17 + } + } + } + }, + "~uc0a701d6-7df9-80ac-8008-c9e6c8b2a9e2": { + "~#shape": { + "~:id": "~uc0a701d6-7df9-80ac-8008-c9e6c8b2a9e2", + "~:name": "Group board", + "~:type": "~:frame", + "~:parent-id": "~u00000000-0000-0000-0000-000000000000", + "~:frame-id": "~u00000000-0000-0000-0000-000000000000", + "~:shapes": ["~uc0a701d6-7df9-80ac-8008-c9e6ca7a304a"], + "~:fills": [ + { + "~:fill-color": "#B1B2B5", + "~:fill-opacity": 1 + } + ], + "~:strokes": [], + "~:hide-fill-on-export": false, + "~:show-content": true, + "~:proportion-lock": false, + "~:proportion": 1, + "~:r1": 0, + "~:r2": 0, + "~:r3": 0, + "~:r4": 0, + "~:layout": "~:flex", + "~:layout-flex-dir": "~:row", + "~:layout-wrap-type": "~:nowrap", + "~:layout-gap-type": "~:multiple", + "~:layout-gap": { + "~:row-gap": 0, + "~:column-gap": 0 + }, + "~:layout-padding-type": "~:simple", + "~:layout-padding": { + "~:p1": 0, + "~:p2": 0, + "~:p3": 0, + "~:p4": 0 + }, + "~:layout-align-items": "~:start", + "~:layout-justify-content": "~:start", + "~:layout-align-content": "~:stretch", + "~:layout-item-h-sizing": "~:auto", + "~:layout-item-v-sizing": "~:auto", + "~:x": 0, + "~:y": 100, + "~:width": 120.99999821186066, + "~:height": 17, + "~:transform": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:transform-inverse": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:rotation": 0, + "~:points": [ + { + "~#point": { + "~:x": 0, + "~:y": 100 + } + }, + { + "~#point": { + "~:x": 120.99999821186066, + "~:y": 100 + } + }, + { + "~#point": { + "~:x": 120.99999821186066, + "~:y": 117 + } + }, + { + "~#point": { + "~:x": 0, + "~:y": 117 + } + } + ], + "~:selrect": { + "~#rect": { + "~:x": 0, + "~:y": 100, + "~:width": 120.99999821186066, + "~:height": 17, + "~:x1": 0, + "~:y1": 100, + "~:x2": 120.99999821186066, + "~:y2": 117 + } + } + } + }, + "~uc0a701d6-7df9-80ac-8008-c9e6ca7a304a": { + "~#shape": { + "~:id": "~uc0a701d6-7df9-80ac-8008-c9e6ca7a304a", + "~:name": "Group", + "~:type": "~:group", + "~:parent-id": "~uc0a701d6-7df9-80ac-8008-c9e6c8b2a9e2", + "~:frame-id": "~uc0a701d6-7df9-80ac-8008-c9e6c8b2a9e2", + "~:shapes": ["~uc0a701d6-7df9-80ac-8008-c9e6c781a50f"], + "~:fills": [], + "~:strokes": [], + "~:proportion-lock": false, + "~:proportion": 1, + "~:x": 0, + "~:y": 100, + "~:width": 120.99999821186066, + "~:height": 17, + "~:transform": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:transform-inverse": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:rotation": 0, + "~:points": [ + { + "~#point": { + "~:x": 0, + "~:y": 100 + } + }, + { + "~#point": { + "~:x": 120.99999821186066, + "~:y": 100 + } + }, + { + "~#point": { + "~:x": 120.99999821186066, + "~:y": 117 + } + }, + { + "~#point": { + "~:x": 0, + "~:y": 117 + } + } + ], + "~:selrect": { + "~#rect": { + "~:x": 0, + "~:y": 100, + "~:width": 120.99999821186066, + "~:height": 17, + "~:x1": 0, + "~:y1": 100, + "~:x2": 120.99999821186066, + "~:y2": 117 + } + } + } + }, + "~uc0a701d6-7df9-80ac-8008-c9e6c781a50f": { + "~#shape": { + "~:id": "~uc0a701d6-7df9-80ac-8008-c9e6c781a50f", + "~:name": "Hello grouped world", + "~:type": "~:text", + "~:grow-type": "~:auto-width", + "~:parent-id": "~uc0a701d6-7df9-80ac-8008-c9e6ca7a304a", + "~:frame-id": "~uc0a701d6-7df9-80ac-8008-c9e6c8b2a9e2", + "~:content": { + "~:type": "root", + "~:children": [ + { + "~:type": "paragraph-set", + "~:children": [ + { + "~:type": "paragraph", + "~:children": [ + { + "~:line-height": "1.2", + "~:font-style": "normal", + "~:typography-ref-id": null, + "~:text-transform": "none", + "~:text-align": "left", + "~:font-id": "sourcesanspro", + "~:font-size": "14", + "~:font-weight": "400", + "~:typography-ref-file": null, + "~:text-direction": "ltr", + "~:font-variant-id": "regular", + "~:text-decoration": "none", + "~:letter-spacing": "0", + "~:fills": [ + { + "~:fill-color": "#000000", + "~:fill-opacity": 1 + } + ], + "~:font-family": "sourcesanspro", + "~:text": "Hello grouped world" + } + ] + } + ] + } + ] + }, + "~:x": 0, + "~:y": 100, + "~:width": 120.99999821186066, + "~:height": 17, + "~:transform": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:transform-inverse": { + "~#matrix": { + "~:a": 1, + "~:b": 0, + "~:c": 0, + "~:d": 1, + "~:e": 0, + "~:f": 0 + } + }, + "~:rotation": 0, + "~:points": [ + { + "~#point": { + "~:x": 0, + "~:y": 100 + } + }, + { + "~#point": { + "~:x": 120.99999821186066, + "~:y": 100 + } + }, + { + "~#point": { + "~:x": 120.99999821186066, + "~:y": 117 + } + }, + { + "~#point": { + "~:x": 0, + "~:y": 117 + } + } + ], + "~:selrect": { + "~#rect": { + "~:x": 0, + "~:y": 100, + "~:width": 120.99999821186066, + "~:height": 17, + "~:x1": 0, + "~:y1": 100, + "~:x2": 120.99999821186066, + "~:y2": 117 + } + } + } + } + }, + "~:id": "~u238a17e0-75ff-8075-8006-934586ea2231", + "~:name": "Page 1" + } + }, + "~:id": "~u238a17e0-75ff-8075-8006-934586ea2230", + "~:options": { + "~:components-v2": true, + "~:base-font-size": "16px" + } + } +} diff --git a/frontend/playwright/ui/specs/text-editor-v3.spec.js b/frontend/playwright/ui/specs/text-editor-v3.spec.js index fa49a0704a..894640a028 100644 --- a/frontend/playwright/ui/specs/text-editor-v3.spec.js +++ b/frontend/playwright/ui/specs/text-editor-v3.spec.js @@ -23,7 +23,6 @@ async function openEditorAndSelectAll(workspace) { await workspace.page.keyboard.press("ControlOrMeta+a"); } - test("Typography at a collapsed caret only styles newly typed text", async ({ page, }) => { @@ -218,8 +217,6 @@ test.describe("BUG 11083 - Changing typography must not quit the editor", () => }); }); - - test("BUG 10467 - Auto-width text captures every typed character", async ({ page, }) => { @@ -681,3 +678,74 @@ test.describe("Text Editor context menu", () => { await expect(menuItem(page, "Select all")).not.toHaveAttribute("disabled"); }); }); + +test.describe("BUG 12325 - Fit-content flex board follows an auto-width text while typing", () => { + // Two grey fit-content flex boards, one per test. The content + // (0, 0)-(121, 117) loads centred in the viewport at zoom 1. + const CONTENT_CENTER = { x: 60.5, y: 58.5 }; + + // Screenshot of a small canvas patch centred on (x, y), in canvas + // coordinates. + async function patchAt(workspace, x, y) { + const viewport = await workspace.viewport.boundingBox(); + const px = viewport.x + viewport.width / 2 + x - CONTENT_CENTER.x; + const py = viewport.y + viewport.height / 2 + y - CONTENT_CENTER.y; + return workspace.page.screenshot({ + clip: { x: px - 2, y: py - 2, width: 4, height: 4 }, + }); + } + + async function openFile(page) { + const workspace = new WasmWorkspacePage(page, { textEditor: true }); + await workspace.setupEmptyFile(); + await workspace.mockGetFile("text-editor/get-file-12325.json"); + await workspace.goToWorkspace(); + await workspace.waitForFirstRender(); + return workspace; + } + + // Shrinks `text` to its first character while editing and checks that a spot + // in the right part of its board, at `boardY`, shows the empty canvas. + async function expectBoardShrinksWhileTyping(workspace, text, boardY) { + const x = 80; + const inside = boardY + 8; + const outside = boardY + 50; + expect(await patchAt(workspace, x, inside)).not.toEqual( + await patchAt(workspace, x, outside), + ); + + await workspace.textEditor.startEditing(); + await workspace.page.keyboard.press("End"); + for (let i = 0; i < text.length - 1; i++) { + await workspace.page.keyboard.press("Backspace"); + } + + // Without leaving the editor, the board already shrank around the text, so + // the spot shows the empty canvas. + await expect + .poll(async () => + (await patchAt(workspace, x, inside)).equals( + await patchAt(workspace, x, outside), + ), + ) + .toBe(true); + } + + test("The board shrinks with the text", async ({ page }) => { + const workspace = await openFile(page); + await workspace.clickToggableLayer("Text board"); + await workspace.clickLeafLayer("Hello wide world"); + + await expectBoardShrinksWhileTyping(workspace, "Hello wide world", 0); + }); + + test("The board shrinks with a text inside a group", async ({ page }) => { + const workspace = await openFile(page); + await workspace.clickToggableLayer("Group board"); + // The first "Group" match is the board row. + await workspace.clickToggableLayer("Group", {}, 1); + await workspace.clickLeafLayer("Hello grouped world"); + + await expectBoardShrinksWhileTyping(workspace, "Hello grouped world", 100); + }); +}); diff --git a/frontend/src/app/main/data/workspace/texts.cljs b/frontend/src/app/main/data/workspace/texts.cljs index 1eb94ca1c0..f737947e3a 100644 --- a/frontend/src/app/main/data/workspace/texts.cljs +++ b/frontend/src/app/main/data/workspace/texts.cljs @@ -1453,18 +1453,77 @@ (gsh/transform-shape (ctm/change-size shape width height)))))) {:undo-group (when new-shape? id)})))))))) +(defn- layout-item + "The ancestor of `shape` (or `shape` itself) that is a direct child of its board." + [objects shape] + (loop [item shape] + (let [parent-id (:parent-id item)] + (if (or (= parent-id (:frame-id shape)) + (nil? (get objects parent-id))) + item + (recur (get objects parent-id)))))) + +(defn- v3-sync-text-layout + "Previews an auto-grow text's live size in its layout, so a fit-content parent + follows it while typing. Geometry is committed on finalize." + [id] + (ptk/reify ::v3-sync-text-layout + ptk/WatchEvent + (watch [_ state _] + (let [objects (dsh/lookup-page-objects state) + shape (get objects id) + preview? (and (cfh/text-shape? shape) + (not= :fixed (:grow-type shape)) + (ctl/any-layout-descent? objects shape) + (not (ctl/position-absolute? (layout-item objects shape)))) + size (when preview? + (dwwt/get-wasm-text-new-size shape)) + previous (or (get-in state [::layout-preview id]) + (select-keys (:selrect shape) [:width :height]))] + ;; Modifiers are costly, so skip keystrokes that keep the size. + (if (and (some? size) + (not (and (mth/close? (:width size) (:width previous)) + (mth/close? (:height size) (:height previous))))) + (rx/of #(assoc-in % [::layout-preview id] size) + (dwm/set-wasm-modifiers (dwwt/resize-wasm-text-modifiers shape))) + (rx/empty)))))) + +(defn v3-stop-text-layout-preview + "Drops the layout preview of the text `id` and leaves interactive render mode." + [id] + (ptk/reify ::v3-stop-text-layout-preview + ptk/WatchEvent + (watch [_ state _] + (if (contains? (get state ::layout-preview) id) + (rx/of #(update % ::layout-preview dissoc id) + (dwm/clear-local-transform)) + (rx/empty))))) + +(defn v3-update-editor-content + "Commits `content` from the WASM editor to the text `shape-id` and, while + editing, previews its new size in the layout." + [shape-id content & {:keys [finalize?]}] + (let [text (txt/content->text content) + name (when (not= text "") + (txt/generate-shape-name text)) + update-event (v2-update-text-shape-content shape-id content + :update-name? true + :name name + :finalize? finalize?)] + (if finalize? + update-event + (ptk/reify ::v3-update-editor-content + ptk/WatchEvent + (watch [_ _ _] + (rx/of update-event + (v3-sync-text-layout shape-id))))))) + (defn v3-sync-editor-content "Event pushing the WASM editor content back into the shape, or nil when there is nothing to sync. Every text edit commits through it, menu or keystroke alike." [& {:keys [finalize?]}] (when-let [{:keys [shape-id content]} (wasm.text-editor/text-editor-sync-content)] - (let [text (txt/content->text content) - name (when (not= text "") - (txt/generate-shape-name text))] - (v2-update-text-shape-content shape-id content - :update-name? true - :name name - :finalize? finalize?)))) + (v3-update-editor-content shape-id content :finalize? finalize?))) (defn- sync-editor-content-stream "Stream of the sync event for `reason`, after asking WASM to repaint." diff --git a/frontend/src/app/main/ui/workspace/shapes/text/v3_editor.cljs b/frontend/src/app/main/ui/workspace/shapes/text/v3_editor.cljs index ca486c0fe1..74f95eea13 100644 --- a/frontend/src/app/main/ui/workspace/shapes/text/v3_editor.cljs +++ b/frontend/src/app/main/ui/workspace/shapes/text/v3_editor.cljs @@ -9,7 +9,6 @@ (:require-macros [app.main.style :as stl]) (:require [app.common.data.macros :as dm] - [app.common.types.text :as txt] [app.config :as cf] [app.main.data.helpers :as dsh] [app.main.data.workspace :as dw] @@ -87,12 +86,7 @@ ;; Sync first so the cached content stays index-aligned with WASM. (text-editor/text-editor-sync-content) (if-let [{:keys [content]} (wasm.api/apply-pending-caret-styles! shape-id range)] - (let [text (txt/content->text content) - name (when (not= text "") (txt/generate-shape-name text))] - (st/emit! (dwt/v2-update-text-shape-content - shape-id content - :update-name? true - :name name))) + (st/emit! (dwt/v3-update-editor-content shape-id content)) (sync-wasm-text-editor-content!)))) (defn- collapse-input-caret @@ -663,6 +657,7 @@ (fn [] (on-blur) (st/emit! dw/hide-context-menu + (dwt/v3-stop-text-layout-preview shape-id) (dwu/commit-undo-transaction shape-id)) (text-editor/text-editor-dispose) (wasm.api/request-render-preserving-target "text-editor-dispose")))) From 655300e4d19688cf1b2c85e3b6dd5715e1b41856 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Mon, 19 Oct 2026 15:29:40 +0200 Subject: [PATCH 20/30] :bug: Include board drop shadow in multi-element raster export (#12440) Headless raster/PDF used the board selrect so overflowing children stayed cropped, but that also clipped the board's own drop shadow. Use export_root_bounds: own strokes/shadows/blur without joining children. Closes #12388 --- render-wasm/src/render/pdf.rs | 12 +--- render-wasm/src/render/raster.rs | 12 +--- render-wasm/src/render/svg/tests.rs | 87 ++++++++++++++++++++++++++++- render-wasm/src/shapes.rs | 20 ++++++- 4 files changed, 109 insertions(+), 22 deletions(-) diff --git a/render-wasm/src/render/pdf.rs b/render-wasm/src/render/pdf.rs index b5b7366509..cf53731c30 100644 --- a/render-wasm/src/render/pdf.rs +++ b/render-wasm/src/render/pdf.rs @@ -24,15 +24,9 @@ pub fn render_to_pdf( let shape = tree .get(id) .ok_or_else(|| crate::error::Error::CriticalError("Shape not found for PDF".to_string()))?; - // Boards export at their own bounds (like the classic exporter, which - // screenshots exactly the board box): content overflowing a board lands - // outside the page/surface and is cropped. Other shapes keep the extended - // rect so their own shadows/blur are included. - let bounds = if matches!(shape.shape_type, crate::shapes::Type::Frame(_)) { - shape.selrect() - } else { - shape.extrect(tree, scale) - }; + // Boards crop overflowing children (classic board box) but keep their own + // strokes/shadows/blur. Other shapes use full extrect. + let bounds = shape.export_root_bounds(tree, scale); let page_w = bounds.width() * scale; let page_h = bounds.height() * scale; diff --git a/render-wasm/src/render/raster.rs b/render-wasm/src/render/raster.rs index 740d20b8d9..52dd798006 100644 --- a/render-wasm/src/render/raster.rs +++ b/render-wasm/src/render/raster.rs @@ -79,15 +79,9 @@ pub fn render_to_raster( let Some(shape) = tree.get(id) else { return Ok((Vec::new(), 0, 0)); }; - // Boards export at their own bounds (like the classic exporter, which - // screenshots exactly the board box): content overflowing a board lands - // outside the page/surface and is cropped. Other shapes keep the extended - // rect so their own shadows/blur are included. - let bounds = if matches!(shape.shape_type, crate::shapes::Type::Frame(_)) { - shape.selrect() - } else { - shape.extrect(tree, scale) - }; + // Boards crop overflowing children (classic board box) but keep their own + // strokes/shadows/blur. Other shapes use full extrect. + let bounds = shape.export_root_bounds(tree, scale); let width = (bounds.width() * scale).ceil() as i32; let height = (bounds.height() * scale).ceil() as i32; diff --git a/render-wasm/src/render/svg/tests.rs b/render-wasm/src/render/svg/tests.rs index 7557eebb7c..0a047e8d07 100644 --- a/render-wasm/src/render/svg/tests.rs +++ b/render-wasm/src/render/svg/tests.rs @@ -3639,17 +3639,98 @@ fn group_with_empty_path(group_fills: Vec) -> ShapesPool { } fn raster_png(pool: &ShapesPool, id: Uuid) -> Vec { + let (bytes, _, _) = raster_png_size(pool, id); + bytes +} + +fn raster_png_size(pool: &ShapesPool, id: Uuid) -> (Vec, i32, i32) { let mut resources = crate::render::RenderResources::try_new_headless().expect("headless resources"); - let (bytes, _, _) = crate::render::raster::render_to_raster( + crate::render::raster::render_to_raster( &mut resources, &id, pool, 1.0, crate::render::raster::RasterFormat::Png, ) - .expect("raster export"); - bytes + .expect("raster export") +} + +/// #12388: multi-element (headless) raster must include a board's own drop +/// shadow, matching single-export sizing (spread 4 → +4px each side). +#[test] +fn raster_export_of_board_includes_own_drop_shadow_spread() { + let mut pool = ShapesPool::new(); + let frame_id = uid(1); + add_frame( + &mut pool, + frame_id, + Uuid::nil(), + (0.0, 0.0, 200.0, 120.0), + skia::Color::from_rgb(240, 240, 240), + true, + ); + { + let frame = pool.get_mut(&frame_id).unwrap(); + frame.add_shadow(Shadow::new( + skia::Color::from_rgb(0, 200, 0), + 0.0, + 4.0, + (0.0, 0.0), + ShadowStyle::Drop, + false, + )); + } + + let (_, width, height) = raster_png_size(&pool, frame_id); + assert_eq!( + (width, height), + (208, 128), + "board raster must grow by drop-shadow spread, not stay at selrect" + ); +} + +/// Overflowing children stay cropped; only the board's own effects expand +/// the headless raster page (classic exporter board-box rule). +#[test] +fn raster_export_of_board_crops_overflowing_child_but_keeps_own_shadow() { + let mut pool = ShapesPool::new(); + let frame_id = uid(1); + let child = uid(2); + add_frame( + &mut pool, + frame_id, + Uuid::nil(), + (0.0, 0.0, 200.0, 120.0), + skia::Color::from_rgb(240, 240, 240), + true, + ); + { + let frame = pool.get_mut(&frame_id).unwrap(); + frame.add_shadow(Shadow::new( + skia::Color::from_rgb(0, 200, 0), + 0.0, + 4.0, + (0.0, 0.0), + ShadowStyle::Drop, + false, + )); + frame.add_child(child); + } + add_solid_rect( + &mut pool, + child, + frame_id, + (150.0, 80.0, 280.0, 200.0), + skia::Color::from_rgb(255, 0, 0), + ); + + let (_, width, height) = raster_png_size(&pool, frame_id); + assert_eq!( + (width, height), + (208, 128), + "overflowing child must not grow board raster; own shadow still must" + ); } #[test] diff --git a/render-wasm/src/shapes.rs b/render-wasm/src/shapes.rs index 3f75443a16..e766b723d3 100644 --- a/render-wasm/src/shapes.rs +++ b/render-wasm/src/shapes.rs @@ -1148,10 +1148,28 @@ impl Shape { extrect } - fn own_extrect_bounds(&self) -> Bounds { + pub(crate) fn own_extrect_bounds(&self) -> Bounds { self.expand_own_bounds(self.own_base_bounds(), true) } + /// Bounds for a headless export rooted at this shape. + /// + /// Frames keep the classic board-box crop of overflowing children, but + /// still expand by their own strokes / drop shadows / layer blur so those + /// effects are not clipped (parity with single-export `extrect` for the + /// board itself). Other shapes use full [`Self::extrect`]. + pub fn export_root_bounds(&self, tree: ShapesPoolRef, scale: f32) -> math::Rect { + if matches!(self.shape_type, Type::Frame(_)) { + let mut bounds = self.own_extrect_bounds(); + if !self.transform.is_identity() { + bounds.transform_mut(&self.centered_transform()); + } + bounds.to_rect() + } else { + self.extrect(tree, scale) + } + } + /// The shape's own geometry bounds, before stroke/shadow/blur margins. fn own_base_bounds(&self) -> Bounds { let shape = self; From c7f16d3236940bc3276f87bd6767d0f43bdee840 Mon Sep 17 00:00:00 2001 From: Pablo Alba Date: Mon, 19 Oct 2026 15:56:05 +0200 Subject: [PATCH 21/30] :bug: Keep accented letters in organization avatar initials (#12328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get-initials used an ASCII-only [a-zA-Z] match, so names like Ábaco and Órbita skipped the leading accented letter. Match Unicode letters with \p{L} (CLJS RegExp + u/g flags) so org default avatars match the expected first letter. Closes #12324. AI-assisted-by: Cursor --- common/src/app/common/data.cljc | 12 ++++++++++-- common/test/common_tests/data_test.cljc | 13 ++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index 8a1efbe5d3..9a4aaa3020 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -1202,12 +1202,20 @@ (str/replace (str s) #"([*_~`\[\]()>#+=\-|{}.!@\\])" (fn [[_ c]] (str "\\" c))) "")) +;; \p{L} (Unicode letter) works in Java regex natively, but in JavaScript it +;; requires the "u" flag which ClojureScript regex literals don't support. +;; Do not use the "g" flag: a shared global RegExp mutates lastIndex and +;; corrupts later get-initials calls (org avatar lists in the team switcher). +(def ^:private letter-re + #?(:clj #"\p{L}" + :cljs (js/RegExp. "\\p{L}" "u"))) + (defn get-initials "Returns up to two uppercase initials extracted from a string. - Non-letter prefixes in each token are ignored." + Non-letter prefixes in each token are ignored (Unicode letters)." [name] (->> (str/split (str/trim (or name "")) #"\s+") - (keep #(first (re-seq #"[a-zA-Z]" %))) + (keep #(re-find letter-re %)) (take 2) (map str/upper) (apply str))) diff --git a/common/test/common_tests/data_test.cljc b/common/test/common_tests/data_test.cljc index 353435bab8..fa56367800 100644 --- a/common/test/common_tests/data_test.cljc +++ b/common/test/common_tests/data_test.cljc @@ -34,7 +34,18 @@ (t/is (= "AB" (d/get-initials "123 Alpha ## beta"))) (t/is (= "PD" (d/get-initials " penpot design tool "))) (t/is (= "" (d/get-initials nil))) - (t/is (= "" (d/get-initials "!!! ???")))) + (t/is (= "" (d/get-initials "!!! ???"))) + ;; Accented / Unicode letters (org default avatars) + (t/is (= "Á" (d/get-initials "Ábaco"))) + (t/is (= "Ó" (d/get-initials "Órbita"))) + (t/is (= "ÁN" (d/get-initials "Álvaro Núñez"))) + ;; Sequential calls must stay stable (CLJS global RegExp lastIndex) + (t/is (= ["Á" "BB" "JG" "L" "Á"] + (mapv d/get-initials ["Ábaco" + "Black Bird" + "Jean Grey Academy" + "Library" + "Ábaco"])))) (t/deftest normalize-string-test ;; nil input returns empty string From 38a6961ff1ddde8538895e63f17ebbcd4d40ed5b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 16:08:47 +0200 Subject: [PATCH 22/30] :sparkles: Make keyboard shortcuts layout-independent (#12326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert ":bug: Fix shortcuts with modifiers on letters and digits (#12292)" This reverts commit 21ecfb67b70ac2d6f4defb05a543f0fc18d692a2. * :bug: Migrate mousetrap to key with sane modifiers Finish the half-done mousetrap migration from code/keyCode to key: canonical lowercase tokens on both bind and event sides so modifiers match again, US-only shift remap and keypress legacy removed, and AltGr phantom ctrl filtered on Windows. Record punctuation positionally through getLayoutMap (American fallback) so Shift+Period stores shift+. on every layout, while letters and digits stay glyph-based for AZERTY. Drop dead 219 aliases and revert the stray CamelCase escape command. AI-assisted-by: muse-spark-1.3-contributor * :recycle: Rewrite mousetrap as ESM class without behavior changes Turn the 2012 constructor-plus-prototype shape into a class with underscore-private methods, const tables and arrow listeners. Drop dead _belongsTo, fold one-line prevent/stop helpers into _fireCallback, and let handleKey absorb the old _handleKey indirection. Public API and matching logic stay identical; verified by existing tests plus a runtime smoke. AI-assisted-by: muse-spark-1.3-contributor * :tada: Add node:test suite for mousetrap shortcuts Cover canonical key normalization, modifiers with AltGr, bind arrays plus aliases keyup reset overwrite, sequences with fake timers, and the full stopCallback matrix, all through isolated instances and synthetic events. Hook the zero-dependency suite into frontend/scripts/test for CI. AI-assisted-by: muse-spark-1.3-contributor * :bug: Record shortcuts by glyph instead of position Drop the getLayoutMap positional layer and the American fallback: punctuation now records event.key lowercased, the same token Mousetrap dispatches, so Shift combos match with and without a layout map. Keeps the Numpad numN branch both sides share. Updates the pinned positional tests and adds the Spanish Shift-plus regression. AI-assisted-by: muse-spark-1.3-contributor * :bug: Resolve shifted digits to the digit itself Shift+DigitN now resolves to the digit on every layout, so shift+0 fires on QWERTY ()) as well as AZERTY (0), like the old keyCode behavior. Applies in the recorder and Mousetrap; unshifted digits and numpad are untouched. Also stages the layout-map suite left untracked. AI-assisted-by: muse-spark-1.3-contributor * :bug: Cover punctuation defaults with layout twins Some defaults only fired where their glyph is unshifted (+ dead on US layouts, shift++ dead outside ES/DE, \ dead where it needs AltGr). Bind the twin token per layout family and freeze the display with :show-command. No tooltip or translation changes. AI-assisted-by: muse-spark-1.3-contributor * :sparkles: Warn when the keyboard layout API is missing Shows a warning on the shortcuts settings page in browsers without navigator.keyboard.getLayoutMap, where Shift with punctuation may not fire. Uses a synchronous probe, English and Spanish strings, and documents the layout limitations in the user guide. AI-assisted-by: muse-spark-1.3-contributor * :bug: Fix hide-ui twins and qualify layout docs The ctrl+alt+\ twin could never fire (the AltGr filter drops Ctrl), so it goes away; alt+º covers Chrome on Spanish layouts where AltGr+º resolves to º. The user guide now states which claims need the layout map and which hold everywhere. AI-assisted-by: muse-spark-1.3-contributor --------- Signed-off-by: Eva Marco Co-authored-by: Eva Marco --- docs/user-guide/first-steps/shortcuts.njk | 4 +- frontend/packages/mousetrap/README.md | 19 + frontend/packages/mousetrap/index.js | 1362 +++++++---------- frontend/packages/mousetrap/package.json | 3 + .../mousetrap/test/bind-dispatch.test.js | 183 +++ frontend/packages/mousetrap/test/helpers.js | 89 ++ .../mousetrap/test/layout-map.test.js | 95 ++ .../packages/mousetrap/test/modifiers.test.js | 63 + .../packages/mousetrap/test/normalize.test.js | 109 ++ .../packages/mousetrap/test/sequences.test.js | 108 ++ .../packages/mousetrap/test/smoke.test.js | 31 + .../mousetrap/test/stop-callback.test.js | 102 ++ .../scripts/check-translations/words.es.txt | 1 + frontend/scripts/test | 5 + .../app/main/data/dashboard/shortcuts.cljs | 3 +- frontend/src/app/main/data/shortcuts.cljs | 31 + .../src/app/main/data/viewer/shortcuts.cljs | 3 +- .../data/workspace/grid_layout/shortcuts.cljs | 3 +- .../main/data/workspace/path/shortcuts.cljs | 9 +- .../app/main/data/workspace/shortcuts.cljs | 12 +- .../src/app/main/ui/settings/shortcuts.cljs | 5 + .../src/app/main/ui/settings/shortcuts.scss | 4 + frontend/src/app/main/ui/shortcuts.cljs | 155 +- frontend/test/frontend_tests/runner.cljs | 2 - .../ui/settings_shortcuts_test.cljs | 169 +- .../ui/shortcuts_recording_test.cljs | 60 - frontend/translations/en.po | 7 + frontend/translations/es.po | 15 +- 28 files changed, 1714 insertions(+), 938 deletions(-) create mode 100644 frontend/packages/mousetrap/README.md create mode 100644 frontend/packages/mousetrap/test/bind-dispatch.test.js create mode 100644 frontend/packages/mousetrap/test/helpers.js create mode 100644 frontend/packages/mousetrap/test/layout-map.test.js create mode 100644 frontend/packages/mousetrap/test/modifiers.test.js create mode 100644 frontend/packages/mousetrap/test/normalize.test.js create mode 100644 frontend/packages/mousetrap/test/sequences.test.js create mode 100644 frontend/packages/mousetrap/test/smoke.test.js create mode 100644 frontend/packages/mousetrap/test/stop-callback.test.js delete mode 100644 frontend/test/frontend_tests/ui/shortcuts_recording_test.cljs diff --git a/docs/user-guide/first-steps/shortcuts.njk b/docs/user-guide/first-steps/shortcuts.njk index e4708c9800..ab8279486a 100644 --- a/docs/user-guide/first-steps/shortcuts.njk +++ b/docs/user-guide/first-steps/shortcuts.njk @@ -35,7 +35,9 @@ desc: Get quickstart tips, shortcuts, and tutorials for Penpot! Learn interface

Paste cannot be customized.

Keyboard layouts

-

Punctuation keys (such as -, [, / or ') are recorded by their physical position on the keyboard, so they work the same on AZERTY, QWERTZ and other layouts. Letters and numbers are recorded by the character they type. With Shift, the base key is recorded, not the shifted character.

+

Letters are recorded by the character they type, so they follow your layout (on AZERTY, the A key records a). With Shift, digit keys record the digit itself (Shift0, not the shifted symbol), on every layout and in every browser.

+

Punctuation keys (such as -, [, / or ') are recorded by their physical position on the keyboard, so the same key works on AZERTY, QWERTZ and other layouts — but only in browsers that expose the keyboard layout (Chromium). There, Shift with the + key records Shift+. In browsers without that support, such as Firefox or Safari, punctuation falls back to the typed character (Shift* instead): those shortcuts still work in the browser where you recorded them, but Shift combinations with punctuation may stop working if you switch browsers. If that affects you, record those shortcuts in a Chromium browser.

+

Two known limitations: number keys set opacity (0–9) only where digits are typed without Shift, so AZERTY layouts need the numeric keypad for them; and \ needs AltGr on some layouts (Spanish), where it is recorded with Alt and its exact form depends on the browser.

Reset to default

When you have custom shortcuts, a Reset all button appears next to the search field. It lists every shortcut that will go back to its default before you confirm.

diff --git a/frontend/packages/mousetrap/README.md b/frontend/packages/mousetrap/README.md new file mode 100644 index 0000000000..848c8d8bfc --- /dev/null +++ b/frontend/packages/mousetrap/README.md @@ -0,0 +1,19 @@ +# @penpot/mousetrap + +Vendored keyboard-shortcut library (based on Mousetrap 1.6.5), migrated +from `code`/`keyCode` to `event.key` with lowercase canonical tokens so +shortcuts behave the same across keyboard layouts. + +## Tests + +Unit suite with the Node standard runner, no browser and no extra +dependencies: + +```bash +pnpm test # node --test "test/*.test.js" +``` + +The suite builds real `Mousetrap` instances over fake targets and +synthetic events, with fake timers for sequences. It runs in the frontend +CI through `frontend/scripts/test` and must stay deterministic (no +network, no real time, no platform reads). diff --git a/frontend/packages/mousetrap/index.js b/frontend/packages/mousetrap/index.js index 4558a758b0..e3864396a4 100644 --- a/frontend/packages/mousetrap/index.js +++ b/frontend/packages/mousetrap/index.js @@ -21,233 +21,103 @@ */ const globalDocument = globalThis?.document; /** - * mapping of special keycodes to their corresponding keys + * Canonical key names, all lowercase. * - * everything in this dictionary cannot use keypress events - * so it has to be here to map to the correct keycodes for - * keyup/keydown events + * `event.key` (final character for the active layout) is the source of + * truth. Single printable characters are lowercased, `" "` becomes + * `"space"`, and named keys map through this table.modifiers pressed + * alone (`Shift`, `Control`, ...) normalize to `shift`, `ctrl`, `alt`, + * `meta` so bind-time and event-time tokens always match. * * @type {Object} */ -var _MAP = { - 8: "backspace", - 9: "tab", - 13: "enter", - 16: "shift", - 17: "ctrl", - 18: "alt", - 20: "capslock", - 27: "esc", - 32: "space", - 33: "pageup", - 34: "pagedown", - 35: "end", - 36: "home", - 37: "left", - 38: "up", - 39: "right", - 40: "down", - 45: "ins", - 46: "del", - 91: "meta", - 93: "meta", - 224: "meta", - 219: "219", +const _KEY_NAMES = { + " ": "space", + Spacebar: "space", + ArrowUp: "up", + Up: "up", + ArrowDown: "down", + Down: "down", + ArrowLeft: "left", + Left: "left", + ArrowRight: "right", + Right: "right", + Escape: "escape", + Esc: "escape", + Enter: "enter", + Backspace: "backspace", + Delete: "del", + Del: "del", + Insert: "ins", + Ins: "ins", + Tab: "tab", + Home: "home", + End: "end", + PageUp: "pageup", + PageDown: "pagedown", + CapsLock: "capslock", + Shift: "shift", + Control: "ctrl", + Alt: "alt", + Meta: "meta", }; /** - * mapping for special characters so they can support - * - * this dictionary is only used incase you want to bind a - * keyup or keydown event to one of these keys + * Aliases accepted in bind strings, all resolving to lowercase canonical + * tokens. `mod` follows the platform (Meta on Apple, Ctrl elsewhere). * * @type {Object} */ -var _KEYCODE_MAP = { - 106: "*", - 107: "+", - 109: "-", - 110: ".", - 111: "/", - 186: ";", - 187: "=", - 188: ",", - 189: "-", - 190: ".", - 191: "/", - 192: "`", - 219: "[", - 220: "\\", - 221: "]", - 222: "'", -}; +const globalNavigator = globalThis.navigator; -/** - * this is a mapping of keys that require shift on a US keypad - * back to the non shift equivelents - * - * this is so you can use keyup events with these keys - * - * note that this will only work reliably on US keyboards - * - * @type {Object} - */ -var _SHIFT_MAP = { - "~": "`", - "!": "1", - "@": "2", - "#": "3", - $: "4", - "%": "5", - "^": "6", - "&": "7", - "*": "8", - "(": "9", - ")": "0", - _: "-", - "+": "=", - ":": ";", - '\"': "'", - "<": ",", - ">": ".", - "?": "/", - "|": "\\", -}; - -/** - * this is a list of special strings you can use to map - * to modifier keys when you specify your keyboard shortcuts - * - * @type {Object} - */ - -var globalNavigator = globalThis.navigator; - -var _SPECIAL_ALIASES = { +const _SPECIAL_ALIASES = { + ctrl: "ctrl", + control: "ctrl", + alt: "alt", option: "alt", + shift: "shift", + meta: "meta", command: "meta", - return: "enter", - escape: "esc", - plus: "+", mod: /Mac|iPod|iPhone|iPad/.test(globalNavigator?.platform) ? "meta" : "ctrl", + return: "enter", + enter: "enter", + esc: "escape", + escape: "escape", + del: "del", + delete: "del", + ins: "ins", + insert: "ins", + space: "space", + plus: "+", }; /** - * variable to store the flipped version of _MAP from above - * needed to check if we should use keypress or not when no action - * is specified + * Physical codes treated positionally. Letters and digits resolve by + * glyph (`event.key`), but punctuation resolves by position (`event.code`) + * through the layout map, so the same physical key matches on every + * layout with and without `Shift`. Without a map the typed glyph applies. * - * @type {Object|undefined} + * @type {Set} */ -var _REVERSE_MAP; +const _PUNCTUATION_CODES = new Set([ + "Minus", + "Equal", + "BracketLeft", + "BracketRight", + "Backslash", + "Semicolon", + "Quote", + "Backquote", + "Comma", + "Period", + "Slash", +]); /** - * loop through the f keys, f1 to f19 and add them to the map - * programatically + * Resolves an event to its canonical key. Implemented as + * `Mousetrap#_characterFromEvent` so punctuation can consult the + * instance layout map (see `setLayoutMap`). */ -for (var i = 1; i < 20; ++i) { - _MAP[111 + i] = "f" + i; -} - -/** - * loop through to map numbers on the numeric keypad - */ -for (i = 0; i <= 9; ++i) { - // This needs to use a string cause otherwise since 0 is falsey - // mousetrap will never fire for numpad 0 pressed as part of a keydown - // event. - // - // @see https://github.com/ccampbell/mousetrap/pull/258 - _MAP[i + 96] = i.toString(); -} - -/** - * cross browser add event method - * - * @param {Element|HTMLDocument} object - * @param {string} type - * @param {Function} callback - * @returns void - */ -function _addEvent(object, type, callback) { - if (object.addEventListener) { - object.addEventListener(type, callback, false); - return; - } - - object.attachEvent("on" + type, callback); -} - -/** - * true for the letter (A-Z) and main-row digit (0-9) keycodes - * - * @param {number} which - * @return {boolean} - */ -function _isLetterOrDigit(which) { - return (which >= 48 && which <= 57) || (which >= 65 && which <= 90); -} - -/** - * takes the event and returns the key character - * - * @param {Event} e - * @return {string} - */ -function _characterFromEvent(e) { - // Numpad digits as "num0".."num9" — keeps them separate from main-row bindings across NumLock states and event types. - if (e.code && e.code.indexOf("Numpad") === 0) { - var suffix = e.code.substring(6); - if (suffix.length === 1 && suffix >= "0" && suffix <= "9") { - return "num" + suffix; - } - } - - // for keypress events we should return the character as is - if (e.type == "keypress") { - var character = String.fromCharCode(e.which); - - // if the shift key is not pressed then it is safe to assume - // that we want the character to be lowercase. this means if - // you accidentally have caps lock on then your key bindings - // will continue to work - // - // the only side effect that might not be desired is if you - // bind something like 'A' cause you want to trigger an - // event when capital A is pressed caps lock will no longer - // trigger the event. shift+a will though. - if (!e.shiftKey) { - character = character.toLowerCase(); - } - - return character; - } - - // for non keypress events the special maps are needed - if (_MAP[e.which]) { - return _MAP[e.which]; - } - - if (_KEYCODE_MAP[e.which]) { - return _KEYCODE_MAP[e.which]; - } - - // keydown bindings name the key, not the glyph it types under modifiers - // ("fi" for Option+Shift+5, ")" for Shift+0) - if (_isLetterOrDigit(e.which)) { - return String.fromCharCode(e.which).toLowerCase(); - } - - // if it is not in the special map - if (typeof e.key === "string") { - return e.key.toLowerCase(); - } - - // with keydown and keyup events the character seems to always - // come in as an uppercase character whether you are pressing shift - // or not. we should make sure it is always lowercase for comparisons - return String.fromCharCode(e.which).toLowerCase(); -} /** * checks if two arrays are equal @@ -263,11 +133,18 @@ function _modifiersMatch(modifiers1, modifiers2) { /** * takes a key event and figures out what the modifiers are * + * On Windows `AltGr` reports `ctrlKey + altKey` as well. When the + * `AltGraph` modifier is active the phantom `ctrl` is dropped so typing + * (e.g. `]` on Spanish keyboards) does not fire `ctrl+alt` shortcuts. + * * @param {Event} e * @returns {Array} */ function _eventModifiers(e) { - var modifiers = []; + const modifiers = []; + const altGraph = + typeof e.getModifierState === "function" && + e.getModifierState("AltGraph"); if (e.shiftKey) { modifiers.push("shift"); @@ -277,7 +154,7 @@ function _eventModifiers(e) { modifiers.push("alt"); } - if (e.ctrlKey) { + if (e.ctrlKey && !altGraph) { modifiers.push("ctrl"); } @@ -289,92 +166,15 @@ function _eventModifiers(e) { } /** - * prevents default for this event + * determines if the key is a modifier key or not * - * @param {Event} e - * @returns void - */ -function _preventDefault(e) { - if (e.preventDefault) { - e.preventDefault(); - return; - } - - e.returnValue = false; -} - -/** - * stops propogation for this event - * - * @param {Event} e - * @returns void - */ -function _stopPropagation(e) { - if (e.stopPropagation) { - e.stopPropagation(); - return; - } - - e.cancelBubble = true; -} - -/** - * determines if the keycode specified is a modifier key or not - * - * @param {string} key + * @param {string} key canonical lowercase key * @returns {boolean} */ function _isModifier(key) { return key == "shift" || key == "ctrl" || key == "alt" || key == "meta"; } -/** - * reverses the map lookup so that we can look for specific keys - * to see what can and can't use keypress - * - * @return {Object} - */ -function _getReverseMap() { - if (!_REVERSE_MAP) { - _REVERSE_MAP = {}; - for (var key in _MAP) { - // pull out the numeric keypad from here cause keypress should - // be able to detect the keys from the character - if (key > 95 && key < 112) { - continue; - } - - if (_MAP.hasOwnProperty(key)) { - _REVERSE_MAP[_MAP[key]] = key; - } - } - } - return _REVERSE_MAP; -} - -/** - * picks the best action based on the key combination - * - * @param {string} key - character for key - * @param {Array} modifiers - * @param {string=} action passed in - */ -function _pickBestAction(key, modifiers, action) { - // if no action was picked in we should try to pick the one - // that we think would work best for this key - if (!action) { - action = _getReverseMap()[key] ? "keydown" : "keypress"; - } - - // modifier keys don't work as expected with keypress, - // switch to keydown - if (action == "keypress" && modifiers.length) { - action = "keydown"; - } - - return action; -} - /** * Converts from a string key combination to an array * @@ -398,29 +198,24 @@ function _keysFromString(combination) { * @returns {Object} */ function _getKeyInfo(combination, action) { - var keys; - var key; - var i; - var modifiers = []; + let keys; + let key; + const modifiers = []; // take the keys from this pattern and figure out what the actual // pattern is all about keys = _keysFromString(combination); - for (i = 0; i < keys.length; ++i) { + for (let i = 0; i < keys.length; ++i) { key = keys[i]; - // normalize key names + // normalize key names to lowercase canonical tokens if (_SPECIAL_ALIASES[key]) { key = _SPECIAL_ALIASES[key]; - } - - // if this is not a keypress event then we should - // be smart about using shift keys - // this will only work for US keyboards however - if (action && action != "keypress" && _SHIFT_MAP[key]) { - key = _SHIFT_MAP[key]; - modifiers.push("shift"); + } else if (key.length === 1) { + key = key.toLowerCase(); + } else { + key = (_KEY_NAMES[key] || key).toLowerCase(); } // if this key is a modifier then add it to the list of modifiers @@ -429,9 +224,10 @@ function _getKeyInfo(combination, action) { } } - // depending on what the key combination is - // we will try to pick the best event for it - action = _pickBestAction(key, modifiers, action); + // everything runs on keydown unless the caller asked for keyup + if (!action) { + action = "keydown"; + } return { key: key, @@ -440,240 +236,297 @@ function _getKeyInfo(combination, action) { }; } -function _belongsTo(element, ancestor) { - if (element === null || element === globalDocument) { - return false; - } +class Mousetrap { + constructor(targetElement = globalDocument) { + /** + * a list of all the callbacks setup via Mousetrap.bind() + * + * @type {Object} + */ + this._callbacks = {}; - if (element === ancestor) { - return true; - } + /** + * direct map of string combinations to callbacks used for trigger() + * + * @type {Object} + */ + this._directMap = {}; - return _belongsTo(element.parentNode, ancestor); -} + /** + * keeps track of what level each sequence is at since multiple + * sequences can start out with the same sequence + * + * @type {Object} + */ + this._sequenceLevels = {}; -function Mousetrap(targetElement) { - var self = this; + /** + * variable to store the setTimeout call + * + * @type {null|number} + */ + this._resetTimer = undefined; - targetElement = targetElement || globalDocument; + /** + * temporary state where we will ignore the next keyup + * + * @type {boolean|string} + */ + this._ignoreNextKeyup = false; - /** - * element to attach key events to - * - * @type {Element} - */ - self.target = targetElement; + /** + * are we currently inside of a sequence? + * type of action ("keyup" or "keydown") or false + * + * @type {boolean|string} + */ + this._nextExpectedAction = false; - /** - * a list of all the callbacks setup via Mousetrap.bind() - * - * @type {Object} - */ - self._callbacks = {}; + /** + * layout map for positional punctuation resolution, as returned by + * `navigator.keyboard.getLayoutMap()` (an object with `.get(code)`). + * `null` means glyph fallback. Set via `setLayoutMap`. + * + * @type {Object|null} + */ + this._layoutMap = null; - /** - * direct map of string combinations to callbacks used for trigger() - * - * @type {Object} - */ - self._directMap = {}; - - /** - * keeps track of what level each sequence is at since multiple - * sequences can start out with the same sequence - * - * @type {Object} - */ - var _sequenceLevels = {}; - - /** - * variable to store the setTimeout call - * - * @type {null|number} - */ - var _resetTimer; - - /** - * temporary state where we will ignore the next keyup - * - * @type {boolean|string} - */ - var _ignoreNextKeyup = false; - - /** - * temporary state where we will ignore the next keypress - * - * @type {boolean} - */ - var _ignoreNextKeypress = false; - - /** - * are we currently inside of a sequence? - * type of action ("keyup" or "keydown" or "keypress") or false - * - * @type {boolean|string} - */ - var _nextExpectedAction = false; - - /** - * resets all sequence counters except for the ones passed in - * - * @param {Object} doNotReset - * @returns void - */ - function _resetSequences(doNotReset) { - doNotReset = doNotReset || {}; - - var activeSequences = false, - key; - - for (key in _sequenceLevels) { - if (doNotReset[key]) { - activeSequences = true; - continue; - } - _sequenceLevels[key] = 0; - } - - if (!activeSequences) { - _nextExpectedAction = false; + if (targetElement) { + targetElement.addEventListener("keydown", (e) => this._handleKeyEvent(e)); + targetElement.addEventListener("keyup", (e) => this._handleKeyEvent(e)); } } /** - * finds all callbacks that match based on the keycode, modifiers, - * and action + * binds an event to mousetrap * - * @param {string} character - * @param {Array} modifiers - * @param {Event|Object} e - * @param {string=} sequenceName - name of the sequence we are looking for - * @param {string=} combination - * @param {number=} level - * @returns {Array} - */ - function _getMatches( - character, - modifiers, - e, - sequenceName, - combination, - level, - ) { - var i; - var callback; - var matches = []; - var action = e.type; - - // if there are no events related to this keycode - if (!self._callbacks[character]) { - return []; - } - - // if a modifier key is coming up on its own we should allow it - if (action == "keyup" && _isModifier(character)) { - modifiers = [character]; - } - - // loop through all callbacks for the key that was pressed - // and see if any of them match - for (i = 0; i < self._callbacks[character].length; ++i) { - callback = self._callbacks[character][i]; - - // if a sequence name is not specified, but this is a sequence at - // the wrong level then move onto the next match - if ( - !sequenceName && - callback.seq && - _sequenceLevels[callback.seq] != callback.level - ) { - continue; - } - - // if the action we are looking for doesn't match the action we got - // then we should keep going - if (action != callback.action) { - continue; - } - - // if this is a keypress event and the meta key and control key - // are not pressed that means that we need to only look at the - // character, otherwise check the modifiers as well - // - // chrome will not fire a keypress if meta or control is down - // safari will fire a keypress if meta or meta+shift is down - // firefox will fire a keypress if meta or control is down - if ( - (action == "keypress" && !e.metaKey && !e.ctrlKey) || - _modifiersMatch(modifiers, callback.modifiers) - ) { - // when you bind a combination or sequence a second time it - // should overwrite the first one. if a sequenceName or - // combination is specified in this call it does just that - // - // @todo make deleting its own method? - var deleteCombo = !sequenceName && callback.combo == combination; - var deleteSequence = - sequenceName && - callback.seq == sequenceName && - callback.level == level; - if (deleteCombo || deleteSequence) { - self._callbacks[character].splice(i, 1); - } - - matches.push(callback); - } - } - - return matches; - } - - /** - * actually calls the callback function + * can be a single key, a combination of keys separated with +, + * an array of keys, or a sequence of keys separated by spaces * - * if your callback function returns false this will use the jquery - * convention - prevent default and stop propogation on the event + * be sure to list the modifier keys first to make sure that the + * correct key ends up getting bound (the last key in the pattern) * + * @param {string|Array} keys * @param {Function} callback - * @param {Event} e + * @param {string=} action - 'keydown' or 'keyup' (defaults to 'keydown') * @returns void */ - function _fireCallback(callback, e, combo, sequence) { - // if this event should not happen stop here - if (self.stopCallback(e, e.target || e.srcElement, combo, sequence)) { - return; + bind(keys, callback, action, overwrite) { + keys = keys instanceof Array ? keys : [keys]; + this._bindMultiple(keys, callback, action, overwrite); + return this; + } + + /** + * unbinds an event to mousetrap + * + * the unbinding sets the callback function of the specified key combo + * to an empty function and deletes the corresponding key in the + * _directMap dict. + * + * TODO: actually remove this from the _callbacks dictionary instead + * of binding an empty function + * + * the keycombo+action has to be exactly the same as + * it was defined in the bind method + * + * @param {string|Array} keys + * @param {string} action + * @returns void + */ + unbind(keys, action) { + return this.bind(keys, function () {}, action); + } + + /** + * triggers an event that has already been bound + * + * @param {string} keys + * @param {string=} action + * @returns void + */ + trigger(keys, action) { + if (this._directMap[keys + ":" + action]) { + this._directMap[keys + ":" + action]({}, keys); + } + return this; + } + + /** + * resets the library back to its initial state. this is useful + * if you want to clear out the current keyboard shortcuts and bind + * new ones - for example if you switch to another page + * + * @returns void + */ + reset() { + this._callbacks = {}; + this._directMap = {}; + return this; + } + + /** + * sets the keyboard layout map used for positional punctuation + * resolution (the same map the shortcut recorder uses). + * + * Pass the `KeyboardLayoutMap` from + * `navigator.keyboard.getLayoutMap()`, or `null` to go back to the + * glyph fallback. Read live at dispatch time, so call it whenever + * the map resolves without rebinding. + * + * @param {Object|null} map - object with `.get(code)`, or null + * @returns void + */ + setLayoutMap(map) { + this._layoutMap = map; + return this; + } + + /** + * takes the event and returns the canonical key + * + * Letters, digits and named keys resolve by glyph: single characters + * lowercase so `Shift+Z` matches a `ctrl+shift+z` bind, `" "` maps to + * `space`, and named keys map through `_KEY_NAMES`. The exception is a + * digit row key held with `Shift`, which resolves to the digit itself + * (`Digit0` types ")" on US layouts but "0" on AZERTY, so `shift+0` + * must mean the physical digit key everywhere). Punctuation + * resolves positionally via the layout map (see `setLayoutMap`), + * falling back to the glyph when no map is set. Numpad digits stay + * positional (`num0`..`num9`) so they keep their own bindings. + * + * @param {Event} e + * @return {string|null} + */ + _characterFromEvent(e) { + // Numpad digits as "num0".."num9" — keeps them separate from main-row bindings across NumLock states and event types. + if (e.code && e.code.indexOf("Numpad") === 0) { + const suffix = e.code.substring(6); + if (suffix.length === 1 && suffix >= "0" && suffix <= "9") { + return "num" + suffix; + } } - if (callback(e, combo) === false) { - _preventDefault(e); - _stopPropagation(e); + // Punctuation is positional: with a layout map the primary + // (unshifted) character of the physical key wins, so `Shift+*` on a + // Spanish layout (code `BracketRight`) resolves to `+` like the + // recorder stores. Without a map the typed glyph applies. + if (e.code && _PUNCTUATION_CODES.has(e.code) && this._layoutMap) { + let mapped = null; + try { + mapped = this._layoutMap.get(e.code); + } catch { + mapped = null; + } + if (typeof mapped === "string" && mapped !== "") { + return mapped.toLowerCase(); + } } + + const key = e.key; + if (typeof key !== "string" || key === "") { + return null; + } + + // Shifted digits resolve to the digit itself: with `Shift` held, + // `Digit0` types ")" on US layouts but "0" on AZERTY, so the glyph + // can't be the token. `shift+0` means the physical digit key + // everywhere, like `shift+z` means `z` regardless of `Shift`. + if (e.shiftKey && e.code && /^Digit[0-9]$/.test(e.code) && key.length === 1) { + return e.code.charAt(5); + } + + if (_KEY_NAMES[key]) { + return _KEY_NAMES[key]; + } + + if (key.length === 1) { + return key.toLowerCase(); + } + + if (/^F\d{1,2}$/.test(key)) { + return key.toLowerCase(); + } + + if (key === "Dead") { + return null; + } + + return key.toLowerCase(); + } + + /** + * should we stop this event before firing off callbacks + * + * @param {Event} e + * @param {Element} element + * @return {boolean} + */ + stopCallback(e, element, combo) { + // if the element has the data attribute "mousetrap-dont-stop" then no need + // to stop. It should be used like
...
+ // or :div {:data-mousetrap-dont-stop true} + if ("mousetrapDontStop" in element.dataset) { + return false; + } + + if ((" " + element.className + " ").indexOf(" mousetrap ") > -1) { + return false; + } + + // Keyup events need to be dispatched always. Otherwise some events can be stuck + if (e.type == "keyup") { + return false; + } + + if ("composedPath" in e && typeof e.composedPath === "function") { + // For open shadow trees, update `element` so that the following check works. + const initialEventTarget = e.composedPath()[0]; + if (initialEventTarget !== e.target) { + element = initialEventTarget; + } + } + + // stop for input, select, textarea and button + const shouldStop = + element.tagName == "INPUT" || + element.tagName == "SELECT" || + element.tagName == "TEXTAREA" || + (element.tagName == "BUTTON" && + typeof combo === "string" && + combo.includes("tab")) || + (element.contentEditable && + (element.contentEditable == "true" || + element.contentEditable === "plaintext-only")); + + return shouldStop; } /** * handles a character key event * + * exposed publicly so it can be overwritten by extensions + * * @param {string} character * @param {Array} modifiers * @param {Event} e * @returns void */ - self._handleKey = function (character, modifiers, e) { - var callbacks = _getMatches(character, modifiers, e); - var i; - var doNotReset = {}; - var maxLevel = 0; - var processedSequenceCallback = false; + handleKey(character, modifiers, e) { + const callbacks = this._getMatches(character, modifiers, e); + const doNotReset = {}; + let maxLevel = 0; + let processedSequenceCallback = false; // Calculate the maxLevel for sequences so we can only execute the longest callback sequence - for (i = 0; i < callbacks.length; ++i) { + for (let i = 0; i < callbacks.length; ++i) { if (callbacks[i].seq) { maxLevel = Math.max(maxLevel, callbacks[i].level); } } // loop through matching callbacks for this key event - for (i = 0; i < callbacks.length; ++i) { + for (let i = 0; i < callbacks.length; ++i) { // fire for all sequence callbacks // this is because if for example you have multiple sequences // bound such as "g i" and "g t" they both need to fire the @@ -696,7 +549,7 @@ function Mousetrap(targetElement) { // keep a list of which sequences were matches for later doNotReset[callbacks[i].seq] = 1; - _fireCallback( + this._fireCallback( callbacks[i].callback, e, callbacks[i].combo, @@ -708,12 +561,12 @@ function Mousetrap(targetElement) { // if there were no sequence matches but we are still here // that means this is a regular match so we should fire that if (!processedSequenceCallback) { - _fireCallback(callbacks[i].callback, e, callbacks[i].combo); + this._fireCallback(callbacks[i].callback, e, callbacks[i].combo); } } // if the key you pressed matches the type of sequence without - // being a modifier (ie "keyup" or "keypress") then we should + // being a modifier (ie "keyup" or "keydown") then we should // reset all sequences that were not matched by this event // // this is so, for example, if you have the sequence "h a t" and you @@ -723,136 +576,29 @@ function Mousetrap(targetElement) { // modifier keys are ignored because you can have a sequence // that contains modifiers such as "enter ctrl+space" and in most // cases the modifier key will be pressed before the next key - // - // also if you have a sequence such as "ctrl+b a" then pressing the - // "b" key will trigger a "keypress" and a "keydown" - // - // the "keydown" is expected when there is a modifier, but the - // "keypress" ends up matching the _nextExpectedAction since it occurs - // after and that causes the sequence to reset - // - // we ignore keypresses in a sequence that directly follow a keydown - // for the same character - var ignoreThisKeypress = e.type == "keypress" && _ignoreNextKeypress; - if ( - e.type == _nextExpectedAction && - !_isModifier(character) && - !ignoreThisKeypress - ) { - _resetSequences(doNotReset); + if (e.type == this._nextExpectedAction && !_isModifier(character)) { + this._resetSequences(doNotReset); } - - _ignoreNextKeypress = processedSequenceCallback && e.type == "keydown"; - }; - - /** - * handles a keydown event - * - * @param {Event} e - * @returns void - */ - function _handleKeyEvent(e) { - // normalize e.which for key events - // @see http://stackoverflow.com/questions/4285627/javascript-keycode-vs-charcode-utter-confusion - if (typeof e.which !== "number") { - e.which = e.keyCode; - } - - var character = _characterFromEvent(e); - - // no character found then stop - if (!character) { - return; - } - - // need to use === for the character check because the character can be 0 - if (e.type == "keyup" && _ignoreNextKeyup === character) { - _ignoreNextKeyup = false; - return; - } - - self.handleKey(character, _eventModifiers(e), e); } /** - * called to set a 1 second timeout on the specified sequence + * binds multiple combinations to the same callback * - * this is so after each key press in the sequence you have 1 second - * to press the next key before you have to start over - * - * @returns void - */ - function _resetSequenceTimer() { - clearTimeout(_resetTimer); - _resetTimer = setTimeout(_resetSequences, 1000); - } - - /** - * binds a key sequence to an event - * - * @param {string} combo - combo specified in bind call - * @param {Array} keys + * @param {Array} combinations * @param {Function} callback - * @param {string=} action + * @param {string|undefined} action * @returns void */ - function _bindSequence(combo, keys, callback, action) { - // start off by adding a sequence level record for this combination - // and setting the level to 0 - _sequenceLevels[combo] = 0; - - /** - * callback to increase the sequence level for this sequence and reset - * all other sequences that were active - * - * @param {string} nextAction - * @returns {Function} - */ - function _increaseSequence(nextAction) { - return function () { - _nextExpectedAction = nextAction; - ++_sequenceLevels[combo]; - _resetSequenceTimer(); - }; - } - - /** - * wraps the specified callback inside of another function in order - * to reset all sequence counters as soon as this sequence is done - * - * @param {Event} e - * @returns void - */ - function _callbackAndReset(e) { - _fireCallback(callback, e, combo); - - // we should ignore the next key up if the action is key down - // or keypress. this is so if you finish a sequence and - // release the key the final key will not trigger a keyup - if (action !== "keyup") { - _ignoreNextKeyup = _characterFromEvent(e); - } - - // weird race condition if a sequence ends with the key - // another sequence begins with - setTimeout(_resetSequences, 10); - } - - // loop through keys one at a time and bind the appropriate callback - // function. for any key leading up to the final one it should - // increase the sequence. after the final, it should reset all sequences - // - // if an action is specified in the original bind call then that will - // be used throughout. otherwise we will pass the action that the - // next key in the sequence should match. this allows a sequence - // to mix and match keypress and keydown events depending on which - // ones are better suited to the key provided - for (var i = 0; i < keys.length; ++i) { - var isFinal = i + 1 === keys.length; - var wrappedCallback = isFinal - ? _callbackAndReset - : _increaseSequence(action || _getKeyInfo(keys[i + 1]).action); - _bindSingle(keys[i], wrappedCallback, action, combo, i); + _bindMultiple(combinations, callback, action, overwrite) { + for (let i = 0; i < combinations.length; ++i) { + this._bindSingle( + combinations[i], + callback, + action, + undefined, + undefined, + overwrite, + ); } } @@ -866,39 +612,31 @@ function Mousetrap(targetElement) { * @param {number=} level - what part of the sequence the command is * @returns void */ - function _bindSingle( - combination, - callback, - action, - sequenceName, - level, - overwrite, - ) { + _bindSingle(combination, callback, action, sequenceName, level, overwrite) { // store a direct mapped reference for use with Mousetrap.trigger - self._directMap[combination + ":" + action] = callback; + this._directMap[combination + ":" + action] = callback; // make sure multiple spaces in a row become a single space combination = combination.replace(/\s+/g, " "); - var sequence = combination.split(" "); - var info; + const sequence = combination.split(" "); // if this pattern is a sequence of keys then run through this method // to reprocess each pattern one key at a time if (sequence.length > 1) { - _bindSequence(combination, sequence, callback, action); + this._bindSequence(combination, sequence, callback, action); return; } - info = _getKeyInfo(combination, action); + const info = _getKeyInfo(combination, action); // make sure to initialize array if this is the first time // a callback is added for this key - self._callbacks[info.key] = self._callbacks[info.key] || []; + this._callbacks[info.key] = this._callbacks[info.key] || []; // remove an existing match if there is one if (overwrite) { - _getMatches( + this._getMatches( info.key, info.modifiers, { type: info.action }, @@ -914,7 +652,7 @@ function Mousetrap(targetElement) { // // this is important because the way these are processed expects // the sequence ones to come first - self._callbacks[info.key][sequenceName ? "unshift" : "push"]({ + this._callbacks[info.key][sequenceName ? "unshift" : "push"]({ callback: callback, modifiers: info.modifiers, action: info.action, @@ -925,168 +663,218 @@ function Mousetrap(targetElement) { } /** - * binds multiple combinations to the same callback + * binds a key sequence to an event * - * @param {Array} combinations + * @param {string} combo - combo specified in bind call + * @param {Array} keys * @param {Function} callback - * @param {string|undefined} action + * @param {string=} action * @returns void */ - self._bindMultiple = function (combinations, callback, action, overwrite) { - for (var i = 0; i < combinations.length; ++i) { - _bindSingle( - combinations[i], - callback, - action, - undefined, - undefined, - overwrite, - ); - } - }; + _bindSequence(combo, keys, callback, action) { + // start off by adding a sequence level record for this combination + // and setting the level to 0 + this._sequenceLevels[combo] = 0; - if (targetElement) { - _addEvent(targetElement, "keypress", _handleKeyEvent); - _addEvent(targetElement, "keydown", _handleKeyEvent); - _addEvent(targetElement, "keyup", _handleKeyEvent); - } -} + /** + * callback to increase the sequence level for this sequence and reset + * all other sequences that were active + * + * @param {string} nextAction + * @returns {Function} + */ + const _increaseSequence = (nextAction) => () => { + this._nextExpectedAction = nextAction; + ++this._sequenceLevels[combo]; + this._resetSequenceTimer(); + }; -/** - * binds an event to mousetrap - * - * can be a single key, a combination of keys separated with +, - * an array of keys, or a sequence of keys separated by spaces - * - * be sure to list the modifier keys first to make sure that the - * correct key ends up getting bound (the last key in the pattern) - * - * @param {string|Array} keys - * @param {Function} callback - * @param {string=} action - 'keypress', 'keydown', or 'keyup' - * @returns void - */ -Mousetrap.prototype.bind = function (keys, callback, action, overwrite) { - var self = this; - keys = keys instanceof Array ? keys : [keys]; - self._bindMultiple.call(self, keys, callback, action, overwrite); - return self; -}; + /** + * wraps the specified callback inside of another function in order + * to reset all sequence counters as soon as this sequence is done + * + * @param {Event} e + * @returns void + */ + const _callbackAndReset = (e) => { + this._fireCallback(callback, e, combo); -/** - * unbinds an event to mousetrap - * - * the unbinding sets the callback function of the specified key combo - * to an empty function and deletes the corresponding key in the - * _directMap dict. - * - * TODO: actually remove this from the _callbacks dictionary instead - * of binding an empty function - * - * the keycombo+action has to be exactly the same as - * it was defined in the bind method - * - * @param {string|Array} keys - * @param {string} action - * @returns void - */ -Mousetrap.prototype.unbind = function (keys, action) { - var self = this; - return self.bind.call(self, keys, function () {}, action); -}; + // we should ignore the next key up if the action is keydown. + // this is so if you finish a sequence and + // release the key the final key will not trigger a keyup + if (action !== "keyup") { + this._ignoreNextKeyup = this._characterFromEvent(e); + } -/** - * triggers an event that has already been bound - * - * @param {string} keys - * @param {string=} action - * @returns void - */ -Mousetrap.prototype.trigger = function (keys, action) { - var self = this; - if (self._directMap[keys + ":" + action]) { - self._directMap[keys + ":" + action]({}, keys); - } - return self; -}; + // weird race condition if a sequence ends with the key + // another sequence begins with + setTimeout(() => this._resetSequences(), 10); + }; -/** - * resets the library back to its initial state. this is useful - * if you want to clear out the current keyboard shortcuts and bind - * new ones - for example if you switch to another page - * - * @returns void - */ -Mousetrap.prototype.reset = function () { - var self = this; - self._callbacks = {}; - self._directMap = {}; - return self; -}; - -/** - * should we stop this event before firing off callbacks - * - * @param {Event} e - * @param {Element} element - * @return {boolean} - */ -Mousetrap.prototype.stopCallback = function (e, element, combo) { - // if the element has the data attribute "mousetrap-dont-stop" then no need - // to stop. It should be used like
...
- // or :div {:data-mousetrap-dont-stop true} - if ("mousetrapDontStop" in element.dataset) { - return false; - } - - if ((" " + element.className + " ").indexOf(" mousetrap ") > -1) { - return false; - } - - // Keyup events need to be dispatched always. Otherwise some events can be stuck - if (e.type == "keyup") { - return false; - } - - if ("composedPath" in e && typeof e.composedPath === "function") { - // For open shadow trees, update `element` so that the following check works. - const initialEventTarget = e.composedPath()[0]; - if (initialEventTarget !== e.target) { - element = initialEventTarget; + // loop through keys one at a time and bind the appropriate callback + // function. for any key leading up to the final one it should + // increase the sequence. after the final, it should reset all sequences + // + // if an action is specified in the original bind call then that will + // be used throughout. otherwise we will pass the action that the + // next key in the sequence should match. + for (let i = 0; i < keys.length; ++i) { + const isFinal = i + 1 === keys.length; + const wrappedCallback = isFinal + ? _callbackAndReset + : _increaseSequence(action || _getKeyInfo(keys[i + 1]).action); + this._bindSingle(keys[i], wrappedCallback, action, combo, i); } } - // stop for input, select, textarea and button - const shouldStop = - element.tagName == "INPUT" || - element.tagName == "SELECT" || - element.tagName == "TEXTAREA" || - (element.tagName == "BUTTON" && combo.includes("tab")) || - (element.contentEditable && - (element.contentEditable == "true" || - element.contentEditable === "plaintext-only")); + /** + * resets all sequence counters except for the ones passed in + * + * @param {Object} doNotReset + * @returns void + */ + _resetSequences(doNotReset) { + doNotReset = doNotReset || {}; - return shouldStop; -}; + let activeSequences = false; -/** - * exposes _handleKey publicly so it can be overwritten by extensions - */ -Mousetrap.prototype.handleKey = function () { - var self = this; - return self._handleKey.apply(self, arguments); -}; + for (const key in this._sequenceLevels) { + if (doNotReset[key]) { + activeSequences = true; + continue; + } + this._sequenceLevels[key] = 0; + } -/** - * allow custom key mappings - */ -export function addKeycodes(object) { - for (var key in object) { - if (object.hasOwnProperty(key)) { - _MAP[key] = object[key]; + if (!activeSequences) { + this._nextExpectedAction = false; } } - _REVERSE_MAP = null; + + /** + * called to set a 1 second timeout on the specified sequence + * + * this is so after each key press in the sequence you have 1 second + * to press the next key before you have to start over + * + * @returns void + */ + _resetSequenceTimer() { + clearTimeout(this._resetTimer); + this._resetTimer = setTimeout(() => this._resetSequences(), 1000); + } + + /** + * finds all callbacks that match based on the key, modifiers, + * and action + * + * @param {string} character canonical lowercase key + * @param {Array} modifiers + * @param {Event|Object} e + * @param {string=} sequenceName - name of the sequence we are looking for + * @param {string=} combination + * @param {number=} level + * @returns {Array} + */ + _getMatches(character, modifiers, e, sequenceName, combination, level) { + const matches = []; + const action = e.type; + + // if there are no events related to this key + if (!this._callbacks[character]) { + return []; + } + + // if a modifier key is coming up on its own we should allow it + if (action == "keyup" && _isModifier(character)) { + modifiers = [character]; + } + + // loop through all callbacks for the key that was pressed + // and see if any of them match + for (let i = 0; i < this._callbacks[character].length; ++i) { + const callback = this._callbacks[character][i]; + + // if a sequence name is not specified, but this is a sequence at + // the wrong level then move onto the next match + if ( + !sequenceName && + callback.seq && + this._sequenceLevels[callback.seq] != callback.level + ) { + continue; + } + + // if the action we are looking for doesn't match the action we got + // then we should keep going + if (action != callback.action) { + continue; + } + + if (_modifiersMatch(modifiers, callback.modifiers)) { + // when you bind a combination or sequence a second time it + // should overwrite the first one. if a sequenceName or + // combination is specified in this call it does just that + // + // @todo make deleting its own method? + const deleteCombo = !sequenceName && callback.combo == combination; + const deleteSequence = + sequenceName && + callback.seq == sequenceName && + callback.level == level; + if (deleteCombo || deleteSequence) { + this._callbacks[character].splice(i, 1); + } + + matches.push(callback); + } + } + + return matches; + } + + /** + * actually calls the callback function + * + * if your callback function returns false this will use the jquery + * convention - prevent default and stop propogation on the event + * + * @param {Function} callback + * @param {Event} e + * @returns void + */ + _fireCallback(callback, e, combo, sequence) { + // if this event should not happen stop here + if (this.stopCallback(e, e.target || e.srcElement, combo, sequence)) { + return; + } + + if (callback(e, combo) === false) { + e.preventDefault(); + e.stopPropagation(); + } + } + + /** + * handles a key event + * + * @param {Event} e + * @returns void + */ + _handleKeyEvent(e) { + const character = this._characterFromEvent(e); + // no character found then stop + if (!character) { + return; + } + + if (e.type == "keyup" && this._ignoreNextKeyup === character) { + this._ignoreNextKeyup = false; + return; + } + + this.handleKey(character, _eventModifiers(e), e); + } } /** @@ -1095,6 +883,8 @@ export function addKeycodes(object) { * This method is needed to allow the global mousetrap functions to work * now that mousetrap is a constructor function. */ -var instance = new Mousetrap(globalDocument); +const instance = new Mousetrap(globalDocument); + +export { Mousetrap }; export default instance; diff --git a/frontend/packages/mousetrap/package.json b/frontend/packages/mousetrap/package.json index d5c2a52548..103baefbe2 100644 --- a/frontend/packages/mousetrap/package.json +++ b/frontend/packages/mousetrap/package.json @@ -4,6 +4,9 @@ "description": "Simple library for handling keyboard shortcuts", "main": "index.js", "type": "module", + "scripts": { + "test": "node --test \"test/*.test.js\"" + }, "packageManager": "pnpm@12.14.0+sha512.c18ba6d87993d3d164cb5e8948f32b2c06ee854487e06cd8d86781342fb3e5bb4634367034711e86802baa9c37bea906ce1929ad08d70884c97d801df13389a6", "author": "Craig Campbell", "license": "Apache-2.0 WITH LLVM-exception" diff --git a/frontend/packages/mousetrap/test/bind-dispatch.test.js b/frontend/packages/mousetrap/test/bind-dispatch.test.js new file mode 100644 index 0000000000..7512151126 --- /dev/null +++ b/frontend/packages/mousetrap/test/bind-dispatch.test.js @@ -0,0 +1,183 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { freshInstance, fire, makeKeyEvent, makeSpiedEvent } from "./helpers.js"; + +describe("bind and dispatch", () => { + it("binds a single key", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("m", () => { + calls++; + }); + fire(target, makeKeyEvent({ key: "m", code: "KeyM" })); + assert.equal(calls, 1); + }); + + it("binds an array of combos to the same callback", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + const cb = () => { + calls++; + }; + mousetrap.bind(["del", "backspace"], cb); + fire(target, makeKeyEvent({ key: "Delete", code: "Delete" })); + fire(target, makeKeyEvent({ key: "Backspace", code: "Backspace" })); + assert.equal(calls, 2); + }); + + it("supports the plus forms", () => { + const { mousetrap, target } = freshInstance(); + let plus = 0; + let shiftPlus = 0; + mousetrap.bind("+", () => { + plus++; + }); + mousetrap.bind("shift++", () => { + shiftPlus++; + }); + fire(target, makeKeyEvent({ key: "+", code: "Equal" })); + fire(target, makeKeyEvent({ key: "*", code: "BracketRight", shiftKey: true })); + // "*" normalizes by glyph, so "shift++" (shift + "+" key) does not fire; + // this pins the layout-independent behavior for the "+" physical key. + assert.equal(plus, 1); + assert.equal(shiftPlus, 0); + }); + + it("resolves bind-string aliases to canonical tokens", () => { + const cases = [ + ["control+z", { key: "z", code: "KeyZ", ctrlKey: true }], + ["command+z", { key: "z", code: "KeyZ", metaKey: true }], + ["option+q", { key: "q", code: "KeyQ", altKey: true }], + ["mod+z", { key: "z", code: "KeyZ", ctrlKey: true }], + ["return", { key: "Enter", code: "Enter" }], + ["esc", { key: "Escape", code: "Escape" }], + ["plus", { key: "+", code: "Equal" }], + ]; + for (const [combo, eventOver] of cases) { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind(combo, () => { + calls++; + }); + fire(target, makeKeyEvent(eventOver)); + assert.equal(calls, 1, combo); + } + }); + + it("defaults to keydown and honors explicit keyup", () => { + const { mousetrap, target } = freshInstance(); + let downs = 0; + let ups = 0; + mousetrap.bind("x", () => { + downs++; + }); + mousetrap.bind( + "y", + () => { + ups++; + }, + "keyup", + ); + fire(target, makeKeyEvent({ key: "x", code: "KeyX", type: "keyup" })); + fire(target, makeKeyEvent({ key: "y", code: "KeyY" })); + fire(target, makeKeyEvent({ key: "y", code: "KeyY", type: "keyup" })); + assert.equal(downs, 0); + assert.equal(ups, 1); + }); + + it("fires a bare modifier on keyup", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind( + "shift", + () => { + calls++; + }, + "keyup", + ); + fire(target, makeKeyEvent({ key: "Shift", code: "ShiftLeft", type: "keyup", shiftKey: true })); + assert.equal(calls, 1); + }); + + it("calls preventDefault and stopPropagation when the callback returns false", () => { + const { mousetrap, target } = freshInstance(); + mousetrap.bind("ctrl+s", () => false); + const { event, calls } = makeSpiedEvent({ key: "s", code: "KeyS", ctrlKey: true }); + fire(target, event); + assert.equal(calls.preventDefault, 1); + assert.equal(calls.stopPropagation, 1); + }); + + it("reset() clears every binding", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("a", () => { + calls++; + }); + mousetrap.reset(); + fire(target, makeKeyEvent({ key: "a", code: "KeyA" })); + assert.equal(calls, 0); + }); + + it("overwrite replaces the previous callback for the combo", () => { + const { mousetrap, target } = freshInstance(); + const seen = []; + mousetrap.bind("ctrl+z", () => { + seen.push("old"); + }); + mousetrap.bind( + "ctrl+z", + () => { + seen.push("new"); + }, + undefined, + true, + ); + fire(target, makeKeyEvent({ key: "z", code: "KeyZ", ctrlKey: true })); + assert.deepEqual(seen, ["new"]); + }); + + it("unbind pins the current noop behavior (TODO: really remove)", () => { + // NOTE: unbind() binds an empty function on top instead of removing + // the original, so the original still fires. Pinned as-is; fixing it + // is a behavior change out of scope. + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("a", () => { + calls++; + }); + mousetrap.unbind("a"); + fire(target, makeKeyEvent({ key: "a", code: "KeyA" })); + assert.equal(calls, 1); + }); + + it("trigger dispatches a bind registered with an explicit action", () => { + const { mousetrap } = freshInstance(); + let calls = 0; + let seenCombo = null; + mousetrap.bind( + "ctrl+q", + (e, combo) => { + calls++; + seenCombo = combo; + }, + "keydown", + ); + mousetrap.trigger("ctrl+q", "keydown"); + assert.equal(calls, 1); + assert.equal(seenCombo, "ctrl+q"); + }); + + it("trigger documents the default-action quirk (TODO)", () => { + // NOTE: binds without an explicit action are stored under + // "combo:undefined", so trigger(combo, "keydown") misses them. + // Pinned as-is; fixing it is a behavior change out of scope. + const { mousetrap } = freshInstance(); + let calls = 0; + mousetrap.bind("ctrl+q", () => { + calls++; + }); + mousetrap.trigger("ctrl+q", "keydown"); + assert.equal(calls, 0); + }); +}); diff --git a/frontend/packages/mousetrap/test/helpers.js b/frontend/packages/mousetrap/test/helpers.js new file mode 100644 index 0000000000..d6061984a1 --- /dev/null +++ b/frontend/packages/mousetrap/test/helpers.js @@ -0,0 +1,89 @@ +import { Mousetrap } from "../index.js"; + +/** + * Fake event target capturing listeners. The Mousetrap class only needs + * `addEventListener`, so tests run in plain Node without a DOM. + */ +export function makeTarget() { + const listeners = {}; + const target = { + listeners, + addEventListener(type, callback) { + listeners[type] = listeners[type] || []; + listeners[type].push(callback); + }, + }; + return target; +} + +/** + * Fake focused element for `stopCallback`. Covers every property the + * implementation reads: dataset, className, tagName, contentEditable. + */ +export function makeElement(over = {}) { + return { + dataset: {}, + className: "", + tagName: "DIV", + contentEditable: "false", + ...over, + }; +} + +/** + * Synthetic keyboard event with sane defaults. Override any field per + * case (`key`, `code`, `shiftKey`, ...). The `target` defaults to a plain + * DIV so `stopCallback` lets the event through. + */ +export function makeKeyEvent(over = {}) { + return { + type: "keydown", + key: "a", + code: "KeyA", + shiftKey: false, + ctrlKey: false, + altKey: false, + metaKey: false, + target: makeElement(), + ...over, + }; +} + +/** + * Dispatches a synthetic event through every listener registered for + * its type on a fake target. + */ +export function fire(target, event) { + for (const callback of target.listeners[event.type] || []) { + callback(event); + } +} + +/** + * Event spies recording `preventDefault`/`stopPropagation` calls for the + * `callback → false` convention. + */ +export function makeSpiedEvent(over = {}) { + const calls = { preventDefault: 0, stopPropagation: 0 }; + const event = makeKeyEvent({ + preventDefault() { + calls.preventDefault++; + }, + stopPropagation() { + calls.stopPropagation++; + }, + ...over, + }); + return { event, calls }; +} + +/** + * Fresh isolated instance plus its fake target. Timers used by sequences + * are real by default; tests needing determinism enable `mock.timers` + * themselves and restore it afterwards. + */ +export function freshInstance() { + const target = makeTarget(); + const mousetrap = new Mousetrap(target); + return { mousetrap, target }; +} diff --git a/frontend/packages/mousetrap/test/layout-map.test.js b/frontend/packages/mousetrap/test/layout-map.test.js new file mode 100644 index 0000000000..3cb34eb722 --- /dev/null +++ b/frontend/packages/mousetrap/test/layout-map.test.js @@ -0,0 +1,95 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { freshInstance, fire, makeKeyEvent } from "./helpers.js"; + +function firedWithMap(map, bindStr, eventOver) { + const { mousetrap, target } = freshInstance(); + mousetrap.setLayoutMap(map); + let calls = 0; + mousetrap.bind(bindStr, () => { + calls++; + }); + fire(target, makeKeyEvent(eventOver)); + return calls; +} + +const spanishMap = { + get(code) { + return code === "BracketRight" ? "+" : undefined; + }, +}; + +describe("layout map", () => { + it("resolves punctuation by position, so Shift+* fires a shift++ bind", () => { + const event = { key: "*", code: "BracketRight", shiftKey: true }; + assert.equal(firedWithMap(spanishMap, "shift++", event), 1); + assert.equal(firedWithMap(spanishMap, "shift+*", event), 0); + }); + + it("matches the unshifted primary on the same position", () => { + const event = { key: "+", code: "BracketRight" }; + assert.equal(firedWithMap(spanishMap, "+", event), 1); + }); + + it("falls back to the glyph when the map misses the code", () => { + const miss = { get: () => undefined }; + const event = { key: "*", code: "BracketRight", shiftKey: true }; + assert.equal(firedWithMap(miss, "shift+*", event), 1); + assert.equal(firedWithMap(miss, "shift++", event), 0); + }); + + it("falls back to the glyph when the map throws", () => { + const broken = { + get() { + throw new Error("no map"); + }, + }; + assert.equal( + firedWithMap(broken, "shift+*", { key: "*", code: "BracketRight", shiftKey: true }), + 1, + ); + }); + + it("setLayoutMap(null) restores the glyph fallback", () => { + const { mousetrap, target } = freshInstance(); + mousetrap.setLayoutMap(spanishMap); + mousetrap.setLayoutMap(null); + let calls = 0; + mousetrap.bind("shift+*", () => { + calls++; + }); + fire(target, makeKeyEvent({ key: "*", code: "BracketRight", shiftKey: true })); + assert.equal(calls, 1); + }); + + it("maps stay per instance", () => { + const a = freshInstance(); + const b = freshInstance(); + a.mousetrap.setLayoutMap(spanishMap); + let callsA = 0; + let callsB = 0; + a.mousetrap.bind("shift++", () => { + callsA++; + }); + b.mousetrap.bind("shift+*", () => { + callsB++; + }); + const event = { key: "*", code: "BracketRight", shiftKey: true }; + fire(a.target, makeKeyEvent(event)); + fire(b.target, makeKeyEvent(event)); + assert.equal(callsA, 1); + assert.equal(callsB, 1); + }); + + it("letters still resolve by glyph with a map set", () => { + const { mousetrap, target } = freshInstance(); + mousetrap.setLayoutMap(spanishMap); + let calls = 0; + mousetrap.bind("a", () => { + calls++; + }); + // AZERTY: the "A" label sits on the KeyQ position. + fire(target, makeKeyEvent({ key: "a", code: "KeyQ" })); + assert.equal(calls, 1); + }); +}); diff --git a/frontend/packages/mousetrap/test/modifiers.test.js b/frontend/packages/mousetrap/test/modifiers.test.js new file mode 100644 index 0000000000..748b71feeb --- /dev/null +++ b/frontend/packages/mousetrap/test/modifiers.test.js @@ -0,0 +1,63 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { freshInstance, fire, makeKeyEvent } from "./helpers.js"; + +function fired(bindStr, eventOver) { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind(bindStr, () => { + calls++; + }); + fire(target, makeKeyEvent(eventOver)); + return calls; +} + +describe("modifiers", () => { + it("matches each flag on its own", () => { + assert.equal(fired("shift+x", { key: "x", code: "KeyX", shiftKey: true }), 1); + assert.equal(fired("alt+x", { key: "x", code: "KeyX", altKey: true }), 1); + assert.equal(fired("ctrl+x", { key: "x", code: "KeyX", ctrlKey: true }), 1); + assert.equal(fired("meta+x", { key: "x", code: "KeyX", metaKey: true }), 1); + }); + + it("does not fire without the bound modifier", () => { + assert.equal(fired("shift+x", { key: "x", code: "KeyX" }), 0); + assert.equal(fired("ctrl+x", { key: "x", code: "KeyX", altKey: true }), 0); + }); + + it("combines modifiers regardless of bind order", () => { + const event = { key: "q", code: "KeyQ", ctrlKey: true, altKey: true, shiftKey: true }; + assert.equal(fired("ctrl+alt+shift+q", event), 1); + assert.equal(fired("shift+ctrl+alt+q", event), 1); + }); + + it("drops the phantom ctrl while AltGraph is active", () => { + const altgr = { + key: "]", + code: "BracketRight", + ctrlKey: true, + altKey: true, + getModifierState: (mod) => mod === "AltGraph", + }; + assert.equal(fired("ctrl+alt+]", altgr), 0); + assert.equal(fired("alt+]", altgr), 1); + }); + + it("keeps a real ctrl+alt without AltGraph", () => { + const event = { + key: "]", + code: "BracketRight", + ctrlKey: true, + altKey: true, + getModifierState: () => false, + }; + assert.equal(fired("ctrl+alt+]", event), 1); + }); + + it("works when the event has no getModifierState at all", () => { + assert.equal( + fired("ctrl+x", { key: "x", code: "KeyX", ctrlKey: true, getModifierState: undefined }), + 1, + ); + }); +}); diff --git a/frontend/packages/mousetrap/test/normalize.test.js b/frontend/packages/mousetrap/test/normalize.test.js new file mode 100644 index 0000000000..b91b5ff40d --- /dev/null +++ b/frontend/packages/mousetrap/test/normalize.test.js @@ -0,0 +1,109 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { freshInstance, fire, makeKeyEvent } from "./helpers.js"; + +function fired(bindStr, eventOver) { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind(bindStr, () => { + calls++; + }); + fire(target, makeKeyEvent(eventOver)); + return calls; +} + +describe("canonical key normalization", () => { + // [native event.key, code, canonical bind] — every _KEY_NAMES entry. + const table = [ + [" ", "Space", "space"], + ["Spacebar", "Space", "space"], + ["ArrowUp", "ArrowUp", "up"], + ["Up", "ArrowUp", "up"], + ["ArrowDown", "ArrowDown", "down"], + ["Down", "ArrowDown", "down"], + ["ArrowLeft", "ArrowLeft", "left"], + ["Left", "ArrowLeft", "left"], + ["ArrowRight", "ArrowRight", "right"], + ["Right", "ArrowRight", "right"], + ["Escape", "Escape", "escape"], + ["Esc", "Escape", "escape"], + ["Enter", "Enter", "enter"], + ["Backspace", "Backspace", "backspace"], + ["Delete", "Delete", "del"], + ["Del", "Delete", "del"], + ["Insert", "Insert", "ins"], + ["Ins", "Insert", "ins"], + ["Tab", "Tab", "tab"], + ["Home", "Home", "home"], + ["End", "End", "end"], + ["PageUp", "PageUp", "pageup"], + ["PageDown", "PageDown", "pagedown"], + ["CapsLock", "CapsLock", "capslock"], + ]; + + for (const [nativeKey, code, canonical] of table) { + it(`maps ${nativeKey} to ${canonical}`, () => { + assert.equal(fired(canonical, { key: nativeKey, code }), 1); + }); + } + + it("lowercases single characters so Shift+Z matches shift+z", () => { + assert.equal(fired("shift+z", { key: "Z", code: "KeyZ", shiftKey: true }), 1); + assert.equal(fired("a", { key: "A", code: "KeyA" }), 1); + }); + + it("resolves a shifted digit row key to the digit itself", () => { + // US Shift+0 types ")", AZERTY Shift+0 types "0": shift+0 must mean + // the physical digit key on every layout (production keyCode behavior). + assert.equal(fired("shift+0", { key: ")", code: "Digit0", shiftKey: true }), 1); + assert.equal(fired("shift+1", { key: "!", code: "Digit1", shiftKey: true }), 1); + assert.equal(fired("shift+2", { key: "@", code: "Digit2", shiftKey: true }), 1); + assert.equal(fired("shift+8", { key: "*", code: "Digit8", shiftKey: true }), 1); + // ...so the shifted glyphs are not bindable tokens for digit codes. + assert.equal(fired("shift+)", { key: ")", code: "Digit0", shiftKey: true }), 0); + assert.equal(fired("shift+*", { key: "*", code: "Digit8", shiftKey: true }), 0); + // AZERTY shifted digits already type the digit: unchanged. + assert.equal(fired("shift+0", { key: "0", code: "Digit0", shiftKey: true }), 1); + // Unshifted digits are untouched. + assert.equal(fired("0", { key: "0", code: "Digit0" }), 1); + assert.equal(fired("&", { key: "&", code: "Digit1" }), 1); + }); + + it("without a layout map, punctuation resolves by glyph", () => { + // Spanish layout: Shift + the physical "+" key (BracketRight) types "*". + assert.equal(fired("shift+*", { key: "*", code: "BracketRight", shiftKey: true }), 1); + assert.equal(fired("shift+]", { key: "*", code: "BracketRight", shiftKey: true }), 0); + assert.equal(fired("+", { key: "+", code: "BracketRight" }), 1); + // US layout: Shift + BracketRight types "}". + assert.equal(fired("shift+}", { key: "}", code: "BracketRight", shiftKey: true }), 1); + assert.equal(fired("shift+]", { key: "}", code: "BracketRight", shiftKey: true }), 0); + assert.equal(fired("?", { key: "?", code: "Slash" }), 1); + }); + + it("lowercases function keys", () => { + assert.equal(fired("f1", { key: "F1", code: "F1" }), 1); + assert.equal(fired("f13", { key: "F13", code: "F13" }), 1); + }); + + it("ignores Dead keys and missing/empty keys without throwing", () => { + assert.equal(fired("dead", { key: "Dead", code: "Quote" }), 0); + assert.equal(fired("a", { key: undefined, code: "KeyA" }), 0); + assert.equal(fired("a", { key: "", code: "KeyA" }), 0); + }); + + it("keeps numpad digits positional as num0..num9", () => { + assert.equal(fired("num0", { key: "0", code: "Numpad0" }), 1); + assert.equal(fired("num9", { key: "9", code: "Numpad9" }), 1); + // NumLock off: the key says Insert but the position wins. + assert.equal(fired("num0", { key: "Insert", code: "Numpad0" }), 1); + }); + + it("falls back to the glyph for non-digit numpad codes", () => { + assert.equal(fired("+", { key: "+", code: "NumpadAdd" }), 1); + }); + + it("lowercases unknown long keys", () => { + assert.equal(fired("foo", { key: "Foo", code: "Foo" }), 1); + assert.equal(fired("ñ", { key: "Ñ", code: "Semicolon" }), 1); + }); +}); diff --git a/frontend/packages/mousetrap/test/sequences.test.js b/frontend/packages/mousetrap/test/sequences.test.js new file mode 100644 index 0000000000..69098258d9 --- /dev/null +++ b/frontend/packages/mousetrap/test/sequences.test.js @@ -0,0 +1,108 @@ +import { describe, it, beforeEach, afterEach, mock } from "node:test"; +import assert from "node:assert/strict"; +import { freshInstance, fire, makeKeyEvent } from "./helpers.js"; + +function down(over) { + return makeKeyEvent({ type: "keydown", ...over }); +} + +function up(over) { + return makeKeyEvent({ type: "keyup", ...over }); +} + +describe("sequences", () => { + beforeEach(() => { + mock.timers.enable({ apis: ["setTimeout"] }); + }); + + afterEach(() => { + mock.timers.reset(); + }); + + it("fires after the full sequence, not on the prefix alone", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("g v", () => { + calls++; + }); + fire(target, down({ key: "g", code: "KeyG" })); + assert.equal(calls, 0); + fire(target, down({ key: "v", code: "KeyV" })); + assert.equal(calls, 1); + }); + + it("advances every sequence sharing a prefix", () => { + const { mousetrap, target } = freshInstance(); + const seen = []; + mousetrap.bind("g i", () => { + seen.push("i"); + }); + mousetrap.bind("g t", () => { + seen.push("t"); + }); + fire(target, down({ key: "g", code: "KeyG" })); + fire(target, down({ key: "t", code: "KeyT" })); + assert.deepEqual(seen, ["t"]); + }); + + it("resets on a wrong key mid-sequence", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("g v", () => { + calls++; + }); + fire(target, down({ key: "g", code: "KeyG" })); + fire(target, down({ key: "e", code: "KeyE" })); + fire(target, down({ key: "v", code: "KeyV" })); + assert.equal(calls, 0); + // and the sequence works again afterwards + fire(target, down({ key: "g", code: "KeyG" })); + fire(target, down({ key: "v", code: "KeyV" })); + assert.equal(calls, 1); + }); + + it("expires after one second without the next key", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("g v", () => { + calls++; + }); + fire(target, down({ key: "g", code: "KeyG" })); + mock.timers.tick(1001); + fire(target, down({ key: "v", code: "KeyV" })); + assert.equal(calls, 0); + }); + + it("supports modifiers inside sequences", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("ctrl+k u", () => { + calls++; + }); + fire(target, down({ key: "k", code: "KeyK", ctrlKey: true })); + fire(target, down({ key: "u", code: "KeyU" })); + assert.equal(calls, 1); + }); + + it("supports keyup sequences and ignores the final keyup", () => { + const { mousetrap, target } = freshInstance(); + const seen = []; + mousetrap.bind( + "g v", + () => { + seen.push("seq"); + }, + "keyup", + ); + mousetrap.bind( + "v", + () => { + seen.push("keyup-v"); + }, + "keyup", + ); + fire(target, up({ key: "g", code: "KeyG" })); + fire(target, up({ key: "v", code: "KeyV" })); + assert.deepEqual(seen, ["seq"]); + }); +}); diff --git a/frontend/packages/mousetrap/test/smoke.test.js b/frontend/packages/mousetrap/test/smoke.test.js new file mode 100644 index 0000000000..ae3498f046 --- /dev/null +++ b/frontend/packages/mousetrap/test/smoke.test.js @@ -0,0 +1,31 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { freshInstance, fire, makeKeyEvent } from "./helpers.js"; + +describe("smoke", () => { + it("fires a bound combo and ignores it without the modifier", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("ctrl+z", () => { + calls++; + }); + + fire(target, makeKeyEvent({ key: "z", code: "KeyZ", ctrlKey: true })); + fire(target, makeKeyEvent({ key: "z", code: "KeyZ" })); + + assert.equal(calls, 1); + }); + + it("reset() leaves the instance deaf", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("ctrl+z", () => { + calls++; + }); + mousetrap.reset(); + + fire(target, makeKeyEvent({ key: "z", code: "KeyZ", ctrlKey: true })); + + assert.equal(calls, 0); + }); +}); diff --git a/frontend/packages/mousetrap/test/stop-callback.test.js b/frontend/packages/mousetrap/test/stop-callback.test.js new file mode 100644 index 0000000000..b02303dcec --- /dev/null +++ b/frontend/packages/mousetrap/test/stop-callback.test.js @@ -0,0 +1,102 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { Mousetrap } from "../index.js"; +import { + freshInstance, + fire, + makeKeyEvent, + makeElement, +} from "./helpers.js"; + +describe("stopCallback", () => { + function stopped(elementOver, eventOver = {}, combo = "ctrl+z") { + const { mousetrap } = freshInstance(); + const event = makeKeyEvent({ target: makeElement(elementOver), ...eventOver }); + return mousetrap.stopCallback(event, event.target, combo); + } + + it("stops inside form fields", () => { + for (const tagName of ["INPUT", "SELECT", "TEXTAREA"]) { + assert.equal(stopped({ tagName }), true, tagName); + } + }); + + it("stops buttons only for tab combos, tolerating a missing combo", () => { + assert.equal(stopped({ tagName: "BUTTON" }, {}, "shift+tab"), true); + assert.equal(stopped({ tagName: "BUTTON" }, {}, "ctrl+z"), false); + assert.equal( + stopped({ tagName: "BUTTON" }, {}, undefined), + false, + ); + }); + + it("stops editable content", () => { + assert.equal(stopped({ contentEditable: "true" }), true); + assert.equal(stopped({ contentEditable: "plaintext-only" }), true); + assert.equal(stopped({ contentEditable: "false" }), false); + assert.equal(stopped({ contentEditable: "inherit" }), false); + }); + + it("lets events through on mousetrap-marked elements", () => { + assert.equal(stopped({ className: "mousetrap" }), false); + assert.equal(stopped({ className: "foo mousetrap bar" }), false); + assert.equal(stopped({ dataset: { mousetrapDontStop: "" } }), false); + }); + + it("always lets keyup through", () => { + assert.equal(stopped({ tagName: "INPUT" }, { type: "keyup" }), false); + }); + + it("retargets through composedPath in open shadow trees", () => { + const inner = makeElement({ tagName: "DIV" }); + const event = makeKeyEvent({ + target: makeElement({ tagName: "INPUT" }), + composedPath: () => [inner], + }); + const { mousetrap } = freshInstance(); + assert.equal(mousetrap.stopCallback(event, event.target, "ctrl+z"), false); + }); + + it("gates dispatch end to end: typing in an input fires nothing", () => { + const { mousetrap, target } = freshInstance(); + let calls = 0; + mousetrap.bind("ctrl+z", () => { + calls++; + }); + fire( + target, + makeKeyEvent({ + key: "z", + code: "KeyZ", + ctrlKey: true, + target: makeElement({ tagName: "INPUT" }), + }), + ); + assert.equal(calls, 0); + }); +}); + +describe("exports", () => { + it("exposes the shared default instance", async () => { + const mod = await import("../index.js"); + assert.equal(typeof mod.default.bind, "function"); + assert.equal(typeof mod.default.reset, "function"); + }); + + it("constructs without a target in Node: no listeners, no throw", () => { + const mousetrap = new Mousetrap(); + assert.deepEqual(mousetrap._callbacks, {}); + }); + + it("wires keydown and keyup on the given target", async () => { + const { Mousetrap: Klass } = await import("../index.js"); + const seen = []; + const target = { + addEventListener(type) { + seen.push(type); + }, + }; + new Klass(target); + assert.deepEqual(seen, ["keydown", "keyup"]); + }); +}); diff --git a/frontend/scripts/check-translations/words.es.txt b/frontend/scripts/check-translations/words.es.txt index 009f4c1c4e..ef5db91657 100644 --- a/frontend/scripts/check-translations/words.es.txt +++ b/frontend/scripts/check-translations/words.es.txt @@ -280,6 +280,7 @@ lectores letras manifiesto menos +navegadores nosotros otras pero diff --git a/frontend/scripts/test b/frontend/scripts/test index 6d822b9542..c61c28b543 100755 --- a/frontend/scripts/test +++ b/frontend/scripts/test @@ -10,3 +10,8 @@ popd pnpm run lint:scss; pnpm run test; + +# Mousetrap JS unit tests (node:test, zero deps) +pushd packages/mousetrap; +pnpm test; +popd diff --git a/frontend/src/app/main/data/dashboard/shortcuts.cljs b/frontend/src/app/main/data/dashboard/shortcuts.cljs index b225a50060..f406151f9b 100644 --- a/frontend/src/app/main/data/dashboard/shortcuts.cljs +++ b/frontend/src/app/main/data/dashboard/shortcuts.cljs @@ -50,7 +50,8 @@ (def shortcut-create-new-project {:create-new-project {:tooltip "+" :label (fn [] (tr "shortcuts.create-new-project")) - :command "+" + :command ["+" "="] + :show-command "+" :section [:dashboard] :subsections [:generic] :fn #(st/emit! (dd/create-element))}}) diff --git a/frontend/src/app/main/data/shortcuts.cljs b/frontend/src/app/main/data/shortcuts.cljs index 6ed22e5fb1..7181f5ebb3 100644 --- a/frontend/src/app/main/data/shortcuts.cljs +++ b/frontend/src/app/main/data/shortcuts.cljs @@ -17,6 +17,37 @@ (log/set-level! :warn) +;; Keyboard layout map shared by the shortcut recorder +;; (`app.main.ui.shortcuts`) and the Mousetrap dispatcher: punctuation +;; resolves by physical position through this map, so recording and +;; dispatch use the same token. `nil` until `getLayoutMap` resolves (or +;; when the browser has no keyboard API); both sides then fall back to +;; the typed glyph, which keeps them consistent with and without a map. +(defonce layout-map (atom nil)) + +(defn layout-map-supported? + "Synchronous keyboard-layout API probe: true when the browser exposes + navigator.keyboard.getLayoutMap (Chromium). Punctuation then resolves + by physical position; without it both sides fall back to the typed + glyph, so Shift+punctuation shortcuts may not fire." + ([] (layout-map-supported? + (when (exists? js/navigator) js/navigator))) + ([navigator] + (let [keyboard (when (some? navigator) (.-keyboard ^js navigator))] + (boolean + (and (some? keyboard) + (exists? (.-getLayoutMap ^js keyboard))))))) + +(let [navigator (when (exists? js/navigator) js/navigator)] + (when (layout-map-supported? navigator) + (-> (.getLayoutMap ^js (.-keyboard ^js navigator)) + (.then (fn [m] + (cljs.core/reset! layout-map m) + (.setLayoutMap mousetrap m)) + (fn [_] + (cljs.core/reset! layout-map nil) + (.setLayoutMap mousetrap nil)))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/frontend/src/app/main/data/viewer/shortcuts.cljs b/frontend/src/app/main/data/viewer/shortcuts.cljs index 23f71e9b57..83878c9682 100644 --- a/frontend/src/app/main/data/viewer/shortcuts.cljs +++ b/frontend/src/app/main/data/viewer/shortcuts.cljs @@ -17,7 +17,8 @@ (def shortcuts {:increase-zoom {:tooltip "+" :label (fn [] (tr "shortcuts.increase-zoom")) - :command "+" + :command ["+" "="] + :show-command "+" :subsections [:zoom-viewer] :section [:viewer] :fn #(st/emit! dv/increase-zoom)} diff --git a/frontend/src/app/main/data/workspace/grid_layout/shortcuts.cljs b/frontend/src/app/main/data/workspace/grid_layout/shortcuts.cljs index 1e234dc62d..de5349fca3 100644 --- a/frontend/src/app/main/data/workspace/grid_layout/shortcuts.cljs +++ b/frontend/src/app/main/data/workspace/grid_layout/shortcuts.cljs @@ -55,7 +55,8 @@ :increase-zoom {:tooltip "+" :label (fn [] (tr "shortcuts.increase-zoom")) - :command "+" + :command ["+" "="] + :show-command "+" :section [:workspace] :fn #(st/emit! (dw/increase-zoom nil))} diff --git a/frontend/src/app/main/data/workspace/path/shortcuts.cljs b/frontend/src/app/main/data/workspace/path/shortcuts.cljs index c1d2269383..bd924df5fe 100644 --- a/frontend/src/app/main/data/workspace/path/shortcuts.cljs +++ b/frontend/src/app/main/data/workspace/path/shortcuts.cljs @@ -59,7 +59,8 @@ :add-node {:tooltip (ds/shift "+") :label (fn [] (tr "shortcuts.add-node")) - :command "shift++" + :command ["shift++" "shift+="] + :show-command "shift++" :subsections [:path-editor] :section [:workspace] :fn #(st/emit! (drp/add-node))} @@ -117,8 +118,7 @@ :snap-nodes {:tooltip (ds/meta "'") :label (fn [] (tr "shortcuts.snap-nodes")) - ;;https://github.com/ccampbell/mousetrap/issues/85 - :command [(ds/c-mod "'") (ds/c-mod "219")] + :command (ds/c-mod "'") :subsections [:path-editor] :section [:workspace] :fn #(st/emit! (drp/toggle-snap))} @@ -207,7 +207,8 @@ :increase-zoom {:tooltip "+" :label (fn [] (tr "shortcuts.increase-zoom")) - :command "+" + :command ["+" "="] + :show-command "+" :section [:workspace] :fn #(st/emit! (dw/increase-zoom nil))} diff --git a/frontend/src/app/main/data/workspace/shortcuts.cljs b/frontend/src/app/main/data/workspace/shortcuts.cljs index f91e51a5ba..bdbd3cedaf 100644 --- a/frontend/src/app/main/data/workspace/shortcuts.cljs +++ b/frontend/src/app/main/data/workspace/shortcuts.cljs @@ -567,8 +567,7 @@ :toggle-guides {:tooltip (ds/meta "'") :label (fn [] (tr "shortcuts.toggle-guides")) - ;;https://github.com/ccampbell/mousetrap/issues/85 - :command [(ds/c-mod "'") (ds/c-mod "219")] + :command (ds/c-mod "'") :show-command (ds/c-mod "'") :subsections [:main-menu] :section [:workspace] @@ -619,8 +618,7 @@ :toggle-snap-guides {:tooltip (ds/meta-shift "'") :label (fn [] (tr "shortcuts.toggle-snap-guides")) - ;;https://github.com/ccampbell/mousetrap/issues/85 - :command [(ds/c-mod "shift+'") (ds/c-mod "shift+219")] + :command (ds/c-mod "shift+'") :show-command (ds/c-mod "shift+'") :subsections [:main-menu] :section [:workspace] @@ -679,7 +677,8 @@ :hide-ui {:tooltip "\\" :label (fn [] (tr "shortcuts.hide-ui")) - :command "\\" + :command ["\\" "alt+\\" "alt+º"] + :show-command "\\" :subsections [:panels :basics] :section [:workspace :basics] :fn #(st/emit! (toggle-layout-flag :hide-ui))} @@ -716,7 +715,8 @@ :zoom-selected {:tooltip (ds/shift "2") :label (fn [] (tr "shortcuts.zoom-selected")) - :command ["shift+2" "shift+num2" "@" "\""] + :command ["shift+2" "shift+num2" "shift+'" "shift+\""] + :show-command "shift+2" :subsections [:zoom-workspace] :section [:workspace] :fn #(st/emit! dw/zoom-to-selected-shape)} diff --git a/frontend/src/app/main/ui/settings/shortcuts.cljs b/frontend/src/app/main/ui/settings/shortcuts.cljs index 7df63314d0..2b246339b2 100644 --- a/frontend/src/app/main/ui/settings/shortcuts.cljs +++ b/frontend/src/app/main/ui/settings/shortcuts.cljs @@ -23,6 +23,7 @@ [app.main.ui.ds.foundations.typography.heading :refer [heading*]] [app.main.ui.ds.foundations.typography.text :refer [text*]] [app.main.ui.ds.layout.tab-switcher :refer [tab-switcher*]] + [app.main.ui.ds.notifications.context-notification :refer [context-notification*]] [app.main.ui.ds.product.empty-state :refer [empty-state*]] [app.main.ui.settings.import-shortcuts-diff-modal] [app.main.ui.settings.restore-shortcuts-modal] @@ -477,6 +478,10 @@ [:> icon* {:icon-id i/info :class (stl/css :shortcuts-description-icon)}] (tr "shortcuts.reload-hint")] + (when-not (ds/layout-map-supported?) + [:> context-notification* {:level :warning + :class (stl/css :layout-map-warning)} + (tr "shortcuts.layout-map-missing")]) [:> (mf/provider ctx/shortcuts-ctx) {:value shortcuts-ctx-value} [:div {:class (stl/css :shortcuts-content)} diff --git a/frontend/src/app/main/ui/settings/shortcuts.scss b/frontend/src/app/main/ui/settings/shortcuts.scss index 522147f61e..c86ca879b5 100644 --- a/frontend/src/app/main/ui/settings/shortcuts.scss +++ b/frontend/src/app/main/ui/settings/shortcuts.scss @@ -314,3 +314,7 @@ padding: var(--sp-s) 0; margin-block-end: var(--sp-l); } + +.layout-map-warning { + margin-block-end: var(--sp-m); +} diff --git a/frontend/src/app/main/ui/shortcuts.cljs b/frontend/src/app/main/ui/shortcuts.cljs index 74035a119f..1520f3258b 100644 --- a/frontend/src/app/main/ui/shortcuts.cljs +++ b/frontend/src/app/main/ui/shortcuts.cljs @@ -34,41 +34,40 @@ (def ^:private modifier-keys #{"Control" "Shift" "Alt" "Meta"}) +(def ^:private punctuation-codes + #{"Minus" "Equal" "BracketLeft" "BracketRight" "Backslash" + "Semicolon" "Quote" "Backquote" "Comma" "Period" "Slash"}) + (def ^:private key-name-map {"ArrowUp" "up" + "Up" "up" "ArrowDown" "down" + "Down" "down" "ArrowLeft" "left" + "Left" "left" "ArrowRight" "right" + "Right" "right" "Escape" "escape" + "Esc" "escape" "Enter" "enter" "Backspace" "backspace" "Delete" "del" + "Del" "del" + "Insert" "ins" + "Ins" "ins" "Tab" "tab" + "Home" "home" + "End" "end" + "PageUp" "pageup" + "PageDown" "pagedown" + "CapsLock" "capslock" + "Spacebar" "space" + "Control" "ctrl" + "Shift" "shift" + "Alt" "alt" + "Meta" "meta" " " "space"}) -;; Mousetrap (the runtime shortcut matcher, vendored at -;; frontend/packages/mousetrap) resolves a keydown positionally - via keyCode -;; - ONLY for this punctuation set (its _KEYCODE_MAP); it falls back to -;; `event.key.toLowerCase()` (glyph-based) for letters and digits, which -;; aren't in that map. Recording punctuation from `code` therefore matches -;; what Mousetrap triggers regardless of layout (e.g. the key next to Right -;; Shift on a US layout is always "Period", whatever glyph it types on a -;; given ISO layout). With modifiers, letters and digits come from keyCode -;; like Mousetrap's keydown match (see `letter-or-digit`); otherwise from the -;; glyph, like its keypress match. -(def ^:private code-name-map - {"Minus" "-" - "Equal" "=" - "BracketLeft" "[" - "BracketRight" "]" - "Backslash" "\\" - "Semicolon" ";" - "Quote" "'" - "Backquote" "`" - "Comma" "," - "Period" "." - "Slash" "/"}) - (defn- event->native "Unwraps a goog.events.BrowserEvent to the native event, since the wrapper copies `key` but not `code`." @@ -77,42 +76,100 @@ (or (.getBrowserEvent event) event) event)) -(defn- letter-or-digit - "Letter or digit named by keyCode when a modifier is held, ignoring the glyph - it makes the key type (Option+Shift+5 types \"fi\" on macOS)." +(defn- alt-graph? [^js event] - (let [key-code (.-keyCode event)] - (when (and (or (.-ctrlKey event) (.-metaKey event) - (.-altKey event) (.-shiftKey event)) - (or (<= 48 key-code 57) - (<= 65 key-code 90))) - (.toLowerCase (js/String.fromCharCode key-code))))) + (boolean + (or (when (fn? (.-getModifierState event)) + (.getModifierState event "AltGraph")) + (let [native (event->native event)] + (when (and (not (identical? native event)) + (fn? (.-getModifierState native))) + (.getModifierState native "AltGraph")))))) + +(defn- numpad-code? + [code] + (and (string? code) + (str/starts-with? code "Numpad") + (= 1 (count (subs code 6))))) + +(defn- numpad-alias + [code] + (let [suffix (subs code 6)] + (when (re-matches #"[0-9]" suffix) + (str "num" suffix)))) + +(defn- punctuation-alias + "Resolves punctuation by physical position through the shared layout + map (`app.main.data.shortcuts/layout-map`, also pushed into Mousetrap), + so recording and dispatch use the same token. Without a map the typed + glyph applies, which keeps both sides consistent." + [code key] + (let [mapped (when (some? @ds/layout-map) + (.get ^js @ds/layout-map code))] + (if (string? mapped) + (str/lower mapped) + (str/lower key)))) + +(defn- digit-code? + [code] + (and (string? code) + (boolean (re-matches #"Digit[0-9]" code)))) + +(defn- digit-alias + "Shifted digits resolve to the digit itself: with Shift held, Digit0 + types ) on US layouts but 0 on AZERTY, so shift+0 must mean the + physical digit key everywhere, like shift+z means z." + [^js event code key] + (when (and (.-shiftKey ^js event) + (digit-code? code) + (= 1 (count key))) + (subs code 5))) (defn- normalize-key + "Normalizes `event.key` to the lowercase canonical token mousetrap + matches on. Letters and unshifted digits resolve by glyph (`a`, not + `q` on AZERTY); a digit row key held with Shift resolves to the digit + itself (`shift+0`, never `shift+)`); punctuation resolves by physical + position through the layout map, so `Shift+*` on a Spanish layout + records `shift++`, the primary key the user sees." [^js event] - (let [key (.-key event)] - (when (and key (not (contains? modifier-keys key))) - (or (get key-name-map key) - (get code-name-map (.-code (event->native event))) - (letter-or-digit event) - (some-> key .toLowerCase))))) + (let [native (event->native event) + key (.-key ^js event) + code (.-code ^js native)] + (when (and (string? key) (not (contains? modifier-keys key))) + (cond + ;; Numpad digits keep their own `numN` bindings, like mousetrap. + (numpad-code? code) + (numpad-alias code) + + ;; Punctuation: primary key of the physical position. + (contains? punctuation-codes code) + (punctuation-alias code key) + + :else + (or (get key-name-map key) + ;; Shifted digit row: the digit itself, on every layout. + (digit-alias event code key) + (str/lower key)))))) (defn- event->modifier-parts [^js event] - (cond-> [] - (and (.-ctrlKey event) - (not (cf/check-platform? :macos))) - (conj "ctrl") + (let [alt-graph? (alt-graph? event)] + (cond-> [] + (and (.-ctrlKey event) + (not (cf/check-platform? :macos)) + (not alt-graph?)) + (conj "ctrl") - (and (.-metaKey event) - (cf/check-platform? :macos)) - (conj "command") + (and (.-metaKey event) + (cf/check-platform? :macos)) + (conj "command") - (.-altKey event) - (conj "alt") + (.-altKey event) + (conj "alt") - (.-shiftKey event) - (conj "shift"))) + (.-shiftKey event) + (conj "shift")))) (defn- keyboard-event->mousetrap [^js event] diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index e33ae65f8f..3a3639433f 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -123,7 +123,6 @@ [frontend-tests.ui.settings-shortcuts-test] [frontend-tests.ui.shape-filters-test] [frontend-tests.ui.shortcuts-labels-test] - [frontend-tests.ui.shortcuts-recording-test] [frontend-tests.ui.sidebar-scroll-test] [frontend-tests.ui.stroke-menu-test] [frontend-tests.ui.workspace-libraries-test] @@ -270,7 +269,6 @@ 'frontend-tests.ui.settings-shortcuts-test 'frontend-tests.ui.shape-filters-test 'frontend-tests.ui.shortcuts-labels-test - 'frontend-tests.ui.shortcuts-recording-test 'frontend-tests.ui.sidebar-scroll-test 'frontend-tests.ui.stroke-menu-test 'frontend-tests.ui.workspace-libraries-test diff --git a/frontend/test/frontend_tests/ui/settings_shortcuts_test.cljs b/frontend/test/frontend_tests/ui/settings_shortcuts_test.cljs index e9913c9b26..fd82bfe17d 100644 --- a/frontend/test/frontend_tests/ui/settings_shortcuts_test.cljs +++ b/frontend/test/frontend_tests/ui/settings_shortcuts_test.cljs @@ -2,6 +2,7 @@ (:require [app.config :as cf] [app.main.data.profile :as du] + [app.main.data.shortcuts :as ds] [app.main.ui.settings.import-shortcuts-diff-modal :as diff-modal] [app.main.ui.settings.restore-shortcuts-modal :as restore-modal] [app.main.ui.settings.shortcuts :as sut] @@ -293,9 +294,8 @@ ;; --- keyboard-event->mousetrap across keyboard layouts ------------------- ;; `keyboard-event->mousetrap` (private, exercised via #') turns a recorded -;; keydown into a Mousetrap-style combo string, reading `event.code` -;; (layout-independent) rather than `event.key` (layout- and Shift-dependent) -;; so the recorded combo matches what Mousetrap resolves positionally. Real +;; keydown into a Mousetrap-style combo string from `event.key` (the typed +;; glyph), so recording and dispatch always use the same token. Real ;; keydown events are wrapped by goog.events in a goog.events.BrowserEvent, ;; which forwards `key` but not `code`; the fixtures below simulate that ;; wrapper via `getBrowserEvent`. @@ -326,35 +326,47 @@ :metaKey (boolean meta)}) (t/deftest mousetrap-shift-period-on-us-layout - (t/testing "US layout: Shift + Period key types \">\"" + (t/testing "US layout without a layout map: Shift + Period types \">\" and + records the glyph, the same token Mousetrap dispatches" (with-redefs [cf/check-platform? (constantly false)] - (t/is (= "shift+." + (t/is (= "shift+>" (#'ui-shortcuts/keyboard-event->mousetrap (mock-wrapped-keydown {:key ">" :code "Period" :shift true}))))))) -(t/deftest mousetrap-shift-period-on-spanish-iso-layout - (t/testing "Spanish (ISO) layout: Shift + Period key types \":\" instead of - \">\", but the physical key (code \"Period\") is the same as on - a US layout, so the recorded combo must match" +(t/deftest mousetrap-shift-period-with-layout-map-records-primary + (t/testing "with a layout map, Shift + Period records the primary \".\" + even when the typed glyph differs" (with-redefs [cf/check-platform? (constantly false)] - (t/is (= "shift+." + (try + (reset! ds/layout-map #js {:get (fn [code] (when (= code "Period") "."))}) + (t/is (= "shift+." + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key ">" :code "Period" :shift true})))) + (t/is (= "shift+." + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key ":" :code "Period" :shift true})))) + (finally + (reset! ds/layout-map nil)))))) + +(t/deftest mousetrap-shift-period-on-spanish-iso-layout + (t/testing "Spanish (ISO) layout without a layout map: Shift + Period + types \":\" and records the glyph" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "shift+:" (#'ui-shortcuts/keyboard-event->mousetrap (mock-wrapped-keydown {:key ":" :code "Period" :shift true}))))))) (t/deftest mousetrap-shift-period-on-arbitrary-layout - (t/testing "an arbitrary layout where Shift + Period produces a character - unrelated to \".\" or \":\" still normalizes to \"shift+.\", - since matching is driven by the physical key, not the glyph" + (t/testing "without a layout map the typed glyph wins over the physical + position: \"?\" on the Period position records \"shift+?\"" (with-redefs [cf/check-platform? (constantly false)] - (t/is (= "shift+." + (t/is (= "shift+?" (#'ui-shortcuts/keyboard-event->mousetrap (mock-wrapped-keydown {:key "?" :code "Period" :shift true}))))))) (t/deftest mousetrap-letter-key-on-azerty-layout (t/testing "French AZERTY: the key printed \"A\" sits where \"Q\" is on a US - layout (code \"KeyQ\") and types \"a\". Mousetrap doesn't - resolve letters positionally (absent from its - _MAP/_KEYCODE_MAP), so recording must stay glyph-based + layout (code \"KeyQ\") and types \"a\". Letters resolve by glyph (\"a\", not \"q\") to match what that physical key triggers" (with-redefs [cf/check-platform? (constantly false)] (t/is (= "a" @@ -364,19 +376,17 @@ (t/deftest mousetrap-digit-key-on-azerty-layout (t/testing "French AZERTY: the unshifted top-row key at the US \"1\" position (code \"Digit1\") types \"&\". Digits, like letters, - are absent from Mousetrap's _MAP/_KEYCODE_MAP, so recording - must stay glyph-based (\"&\") rather than resolve positionally - to \"1\"" + resolve by glyph (\"&\") rather than positionally to \"1\"" (with-redefs [cf/check-platform? (constantly false)] (t/is (= "&" (#'ui-shortcuts/keyboard-event->mousetrap (mock-wrapped-keydown {:key "&" :code "Digit1"}))))))) (t/deftest mousetrap-reads-code-from-unwrapped-native-event - (t/testing "falls back to reading `code` straight off the event when it is - not a goog.events.BrowserEvent wrapper (no getBrowserEvent)" + (t/testing "recording works on a plain native event too (glyph wins + without a layout map)" (with-redefs [cf/check-platform? (constantly false)] - (t/is (= "shift+." + (t/is (= "shift+:" (#'ui-shortcuts/keyboard-event->mousetrap (mock-native-keydown {:key ":" :code "Period" :shift true}))))))) @@ -417,3 +427,116 @@ (#'ui-shortcuts/keyboard-event->mousetrap (mock-wrapped-keydown {:key "q" :code "KeyQ" :ctrl true :alt true :shift true}))))))) + +(t/deftest mousetrap-shifted-letter-lowercases + (t/testing "Ctrl+Shift+Z types \"Z\" but records \"ctrl+shift+z\" so the + bind and the event use the same token" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "ctrl+shift+z" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "Z" :code "KeyZ" + :ctrl true :shift true}))))))) + +(t/deftest mousetrap-shifted-digit-resolves-to-digit + (t/testing "a digit row key held with Shift records the digit itself: US + Shift+0 types \")\" but records \"shift+0\", so the physical + digit key means the same on every layout (production keyCode + behavior)" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "shift+0" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key ")" :code "Digit0" :shift true})))) + (t/is (= "shift+1" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "!" :code "Digit1" :shift true})))) + (t/is (= "shift+2" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "@" :code "Digit2" :shift true})))) + (t/is (= "shift+8" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "*" :code "Digit8" :shift true}))))))) + +(t/deftest mousetrap-shifted-digit-already-digit-unchanged + (t/testing "AZERTY Shift+Digit0 already types \"0\": unchanged" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "shift+0" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "0" :code "Digit0" :shift true}))))))) + +(t/deftest mousetrap-space-and-named-keys + (t/testing "\" \" records as space and named keys stay lowercase" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "space" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key " " :code "Space"})))) + (t/is (= "escape" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "Escape" :code "Escape"})))) + (t/is (= "shift+tab" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "Tab" :code "Tab" :shift true})))) + (t/is (= "del" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "Delete" :code "Delete"}))))))) + +(t/deftest mousetrap-altgr-drops-phantom-ctrl + (t/testing "Spanish AltGr++] types \"]\" with ctrl+alt flags, but the + phantom ctrl is dropped so no ctrl+alt shortcut fires" + (with-redefs [cf/check-platform? (constantly false)] + (let [event #js {:key "]" + :shiftKey false :ctrlKey true :altKey true :metaKey false + :getModifierState (fn [m] (= m "AltGraph")) + :getBrowserEvent (fn [] nil)}] + (t/is (= "alt+]" + (#'ui-shortcuts/keyboard-event->mousetrap event))))))) + +(t/deftest mousetrap-real-ctrl-alt-keeps-both + (t/testing "a real Ctrl+Alt without AltGraph still records both modifiers" + (with-redefs [cf/check-platform? (constantly false)] + (let [event #js {:key "]" + :shiftKey false :ctrlKey true :altKey true :metaKey false + :getModifierState (fn [_] false) + :getBrowserEvent (fn [] nil)}] + (t/is (= "ctrl+alt+]" + (#'ui-shortcuts/keyboard-event->mousetrap event))))))) + +(t/deftest mousetrap-spanish-shift-plus-with-layout-map-records-primary + (t/testing "Spanish layout with a layout map: Shift + the physical \"+\" + key types \"*\" but records the primary \"shift++\", which is + what the user sees and what Mousetrap matches" + (with-redefs [cf/check-platform? (constantly false)] + (try + (reset! ds/layout-map #js {:get (fn [code] (when (= code "BracketRight") "+"))}) + (t/is (= "shift++" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "*" :code "BracketRight" :shift true})))) + (t/is (= "+" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "+" :code "BracketRight"})))) + (finally + (reset! ds/layout-map nil)))))) + +(t/deftest mousetrap-spanish-shift-plus-without-layout-map-records-glyph + (t/testing "Spanish layout without a layout map: Shift + the physical + \"+\" key records the typed glyph \"shift+*\", the same token + Mousetrap dispatches" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "shift+*" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "*" :code "BracketRight" :shift true}))))))) + +(t/deftest mousetrap-spanish-unshifted-plus-records-plus + (t/testing "Spanish layout: the unshifted physical \"+\" key records \"+\"" + (with-redefs [cf/check-platform? (constantly false)] + (t/is (= "+" + (#'ui-shortcuts/keyboard-event->mousetrap + (mock-wrapped-keydown {:key "+" :code "BracketRight"}))))))) + +(t/deftest layout-map-supported-matches-keyboard-api + (t/testing "false without a navigator or without a keyboard API" + (t/is (false? (ds/layout-map-supported? nil))) + (t/is (false? (ds/layout-map-supported? #js {}))) + (t/is (false? (ds/layout-map-supported? #js {:keyboard #js {}})))) + (t/testing "true when getLayoutMap exists (Chromium)" + (t/is (true? (ds/layout-map-supported? + #js {:keyboard #js {:getLayoutMap (fn [] nil)}}))))) diff --git a/frontend/test/frontend_tests/ui/shortcuts_recording_test.cljs b/frontend/test/frontend_tests/ui/shortcuts_recording_test.cljs deleted file mode 100644 index 9bb4479fbe..0000000000 --- a/frontend/test/frontend_tests/ui/shortcuts_recording_test.cljs +++ /dev/null @@ -1,60 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns frontend-tests.ui.shortcuts-recording-test - (:require - [app.main.ui.shortcuts :as ss] - [cljs.test :as t :include-macros true])) - -(def ^:private keyboard-event->mousetrap - @#'ss/keyboard-event->mousetrap) - -(defn- key-event - [props] - (clj->js (merge {:altKey false :shiftKey false :ctrlKey false :metaKey false} - props))) - -(t/deftest records-alt-digit-from-key-code - ;; macOS Option+Shift+5 types "fi"; Mousetrap matches it as "5" - (t/is (= "alt+shift+5" - (keyboard-event->mousetrap - (key-event {:key "fi" :code "Digit5" :keyCode 53 :altKey true :shiftKey true}))))) - -(t/deftest records-alt-letter-from-key-code - ;; macOS Option+A types "å"; Mousetrap matches it as "a" - (t/is (= "alt+a" - (keyboard-event->mousetrap - (key-event {:key "å" :code "KeyA" :keyCode 65 :altKey true}))))) - -(t/deftest records-shift-digit-from-key-code - ;; US Shift+0 types ")"; Mousetrap matches it as "0" - (t/is (= "shift+0" - (keyboard-event->mousetrap - (key-event {:key ")" :code "Digit0" :keyCode 48 :shiftKey true}))))) - -(t/deftest records-modified-letter-from-key-code - ;; Russian layout V types "м"; Mousetrap matches Alt+V as "v" - (t/is (= "alt+v" - (keyboard-event->mousetrap - (key-event {:key "м" :code "KeyV" :keyCode 86 :altKey true}))))) - -(t/deftest records-unmodified-key-from-glyph - ;; without modifiers Mousetrap matches the typed glyph on keypress - (t/is (= "&" - (keyboard-event->mousetrap - (key-event {:key "&" :code "Digit1" :keyCode 49})))) - (t/is (= "ñ" - (keyboard-event->mousetrap - (key-event {:key "ñ" :code "" :keyCode 0}))))) - -(t/deftest records-shift-punctuation-from-code - ;; Shift+. types ":" and Shift+= types "+" (US); the key is recorded - (t/is (= "shift+." - (keyboard-event->mousetrap - (key-event {:key ":" :code "Period" :keyCode 190 :shiftKey true})))) - (t/is (= "shift+=" - (keyboard-event->mousetrap - (key-event {:key "+" :code "Equal" :keyCode 187 :shiftKey true}))))) diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 2c7108dbe4..525bb3589d 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -5021,6 +5021,13 @@ msgstr "Join nodes" msgid "shortcuts.key-combo" msgstr "Press the new keys" +#: src/app/main/ui/settings/shortcuts.cljs:484 +msgid "shortcuts.layout-map-missing" +msgstr "" +"This browser does not expose the keyboard layout, so shortcuts that " +"combine Shift with punctuation (for example Shift+,) may not work here. " +"They keep working in Chromium browsers." + #: src/app/main/data/workspace/text/shortcuts.cljs:249 msgid "shortcuts.line-through" msgstr "Toggle line through" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index a069f620eb..82081caf82 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -5067,6 +5067,13 @@ msgstr "Unir nodos" msgid "shortcuts.key-combo" msgstr "Pulsa las nuevas teclas" +#: src/app/main/ui/settings/shortcuts.cljs:484 +msgid "shortcuts.layout-map-missing" +msgstr "" +"Este navegador no expone el mapa de teclado, así que los atajos que " +"combinan Shift con puntuación (por ejemplo Shift+,) pueden no funcionar " +"aquí. Siguen funcionando en navegadores Chromium." + #: src/app/main/data/workspace/text/shortcuts.cljs:249 msgid "shortcuts.line-through" msgstr "Alternar tachado" @@ -6344,10 +6351,6 @@ msgstr "Tipografías" msgid "workspace.assets.typography.add-typography" msgstr "Añadir tipografía" -#: src/app/main/ui/workspace/sidebar/assets/typographies.cljs:461 -msgid "workspace.assets.typography.multiple-texts-selected" -msgstr "Selecciona un solo texto para crear un estilo de tipografía" - #: src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:855 msgid "workspace.assets.typography.font-size" msgstr "Tamaño" @@ -6368,6 +6371,10 @@ msgstr "Interletrado" msgid "workspace.assets.typography.line-height" msgstr "Interlineado" +#: src/app/main/ui/workspace/sidebar/assets/typographies.cljs:461 +msgid "workspace.assets.typography.multiple-texts-selected" +msgstr "Selecciona un solo texto para crear un estilo de tipografía" + #: src/app/main/ui/dashboard/grid.cljs:226, src/app/main/ui/workspace/libraries.cljs:1070, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:802, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:838, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:957, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:976 msgid "workspace.assets.typography.sample" msgstr "Ag" From 5abf66dbf0cf0f887cd6041b68a250222c8000a1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 14:00:25 +0000 Subject: [PATCH 23/30] :bug: Set a finite idle timeout on binfile transactions Binfile v1/v3 import and the v3 export get-file step hold a single transaction while the ZIP is decoded and the file data is realized. Disabling idle_in_transaction_session_timeout left a stuck connection holding locks (and a pool slot) forever. Raise it to 30 minutes instead. The longest import/export observed in production is under 8 minutes, so the limit still bounds a stuck session. statement_timeout stays at 300 s. Add set-transaction-idle-timeout and configure-import-transaction in app.binfile.common, use them from both paths, and cover the helper with a test. AI-assisted-by: deepseek-v4.1-flash --- .serena/memories/backend/subtleties.md | 3 ++- backend/src/app/binfile/common.clj | 22 +++++++++++++++++++-- backend/src/app/binfile/v1.clj | 2 +- backend/src/app/binfile/v3.clj | 5 +++-- backend/test/backend_tests/binfile_test.clj | 17 ++++++++++++++++ 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md index cee764853a..b6db941801 100644 --- a/.serena/memories/backend/subtleties.md +++ b/.serena/memories/backend/subtleties.md @@ -81,4 +81,5 @@ - Tell a snapshot apart by `label IS NOT NULL` (with `created_by` / `deleted_at` as needed), never by `file_change.data IS NOT NULL`: the label holds with every file data backend, and the payload column stopped meaning "this is a snapshot" when the payload moved out. Queries that identify versions must use the label. - The version quota counts of `app.rpc.quotes` (`sql:get-snapshots-per-file`, `sql:get-snapshots-per-team`) are served by the partial index `file_change__user_snapshots__idx`, whose predicate mirrors their where clause; changing one without the other drops the index only scan and the count walks the whole table again. - Revisions live only in `file_change` and have no `file_data` counterpart. They are born already scheduled for deletion one hour later (`app.rpc.commands.files-update`) and the snapshot paths cannot reach one, because they resolve their row through a `label IS NOT NULL` filter. -- A deletion date is assigned once and never moved: `restore-files` (`app.rpc.commands.files`) unmarks only durable history, the `:file` method of `app.tasks.delete-object` dates only rows that have no date yet, and `objects-gc` collects only rows that carry one. Unmarking a revision leaves it invisible to `objects-gc` for ever, because nothing sets that date again. \ No newline at end of file +- A deletion date is assigned once and never moved: `restore-files` (`app.rpc.commands.files`) unmarks only durable history, the `:file` method of `app.tasks.delete-object` dates only rows that have no date yet, and `objects-gc` collects only rows that carry one. Unmarking a revision leaves it invisible to `objects-gc` for ever, because nothing sets that date again. +- Binfile v1/v3 import and the v3 export `get-file` step run inside one long transaction. They raise `idle_in_transaction_session_timeout` to 30 min (`app.binfile.common/transaction-idle-timeout-ms`) instead of disabling it, so a connection left idle while the ZIP is decoded or the file data is realized is bounded; `statement_timeout` stays at 300 s. diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 7fe8e6a532..34e93ae686 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -660,10 +660,28 @@ data library-ids))) -(defn disable-database-timeouts! +(def transaction-idle-timeout-ms + "Idle-in-transaction guard for the long binfile transactions: 30 + minutes. The import and the export of a file run inside one + transaction that can go quiet while the ZIP is decoded and the file + data is realized. The longest operation observed in production is + under 8 minutes, so this leaves room for bigger files and still + aborts a connection that is left idle for half an hour." + 1800000) + +(defn set-transaction-idle-timeout + "Set `idle_in_transaction_session_timeout` for the current + transaction. Must be called inside a transaction: `SET LOCAL` is + scoped to it." [cfg] (let [conn (db/get-connection cfg)] - (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"]) + (db/exec-one! conn [(str "SET LOCAL idle_in_transaction_session_timeout = " + transaction-idle-timeout-ms)]))) + +(defn configure-import-transaction + [cfg] + (set-transaction-idle-timeout cfg) + (let [conn (db/get-connection cfg)] (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]))) (defn invalidate-thumbnails diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index fd340a487d..a8b1ea32c3 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -455,7 +455,7 @@ (defn- read-import-v1 [{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}] - (bfc/disable-database-timeouts! cfg) + (bfc/configure-import-transaction cfg) (pu/with-open [input (zstd-input-stream input) input (io/data-input-stream input)] diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index cdc3dd66bb..69c3e4781e 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -228,6 +228,7 @@ (let [detach? (and (not embed-assets) (not include-libraries))] (db/tx-run! cfg (fn [cfg] + (bfc/set-transaction-idle-timeout cfg) (cond-> (bfc/get-file cfg file-id {:realize? true :include-deleted? true @@ -934,7 +935,7 @@ (defn- import-files* [{:keys [::manifest] :as cfg}] - (bfc/disable-database-timeouts! cfg) + (bfc/configure-import-transaction cfg) (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id) @@ -962,7 +963,7 @@ :hint "unable to perform in-place update with binfile containing more than 1 file" :manifest manifest)) - (bfc/disable-database-timeouts! cfg) + (bfc/configure-import-transaction cfg) (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true) file (first (get manifest :files)) diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 42edaaf4b5..61cf253c8e 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -390,3 +390,20 @@ d)))] (t/is (= :validation (:type out))) (t/is (= :max-file-size-reached (:code out)))))) + +(def ^:private sql:current-idle-timeout + "SELECT current_setting('idle_in_transaction_session_timeout')::interval + = interval '30 minutes' AS ok") + +(t/deftest binfile-transactions-use-a-finite-idle-timeout + (db/tx-run! th/*system* + (fn [{::db/keys [conn] :as cfg}] + (t/testing "export transaction" + (bfc/set-transaction-idle-timeout cfg) + (t/is (true? (-> (db/exec-one! conn [sql:current-idle-timeout]) + :ok)))) + + (t/testing "import transaction" + (bfc/configure-import-transaction cfg) + (t/is (true? (-> (db/exec-one! conn [sql:current-idle-timeout]) + :ok))))))) From 7f53483a32332292c0ea91bead81bef1ce75aa25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elena=20Torr=C3=B3?= Date: Mon, 19 Oct 2026 17:17:34 +0200 Subject: [PATCH 24/30] :bug: Keep full quality rendering during the v3 text layout preview (#12360) --- frontend/src/app/main/data/workspace/modifiers.cljs | 11 +++++++---- frontend/src/app/main/data/workspace/texts.cljs | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/main/data/workspace/modifiers.cljs b/frontend/src/app/main/data/workspace/modifiers.cljs index 47d97b4249..e2cfc06963 100644 --- a/frontend/src/app/main/data/workspace/modifiers.cljs +++ b/frontend/src/app/main/data/workspace/modifiers.cljs @@ -706,8 +706,9 @@ #_:clj-kondo/ignore (defn set-wasm-modifiers [modif-tree & {:keys [ignore-constraints ignore-snap-pixel snap-ignore-axis - subtree-ids-by-id selection-rect-cache] - :or {ignore-constraints false ignore-snap-pixel false snap-ignore-axis nil} + subtree-ids-by-id selection-rect-cache interactive?] + :or {ignore-constraints false ignore-snap-pixel false snap-ignore-axis nil + interactive? true} :as params}] (let [modif-tree (without-nil-ids modif-tree)] (ptk/reify ::set-wasm-modifiers @@ -726,8 +727,10 @@ ;; the renderer into fast + atlas-backdrop mode so the live ;; preview is cheap, tiles never appear sequentially and the main ;; thread is not blocked. The pair is closed in - ;; `clear-local-transform`. - (ensure-interactive-transform-start!) + ;; `clear-local-transform`. Previews outside a gesture pass + ;; `:interactive? false` to keep full quality rendering. + (when interactive? + (ensure-interactive-transform-start!)) (let [snap-pixel? (and (not ignore-snap-pixel) (contains? (:workspace-layout state) :snap-pixel-grid)) translation? (every? #(ctm/only-move? (:modifiers %)) (vals modif-tree))] diff --git a/frontend/src/app/main/data/workspace/texts.cljs b/frontend/src/app/main/data/workspace/texts.cljs index f737947e3a..e66b378f54 100644 --- a/frontend/src/app/main/data/workspace/texts.cljs +++ b/frontend/src/app/main/data/workspace/texts.cljs @@ -1485,7 +1485,8 @@ (not (and (mth/close? (:width size) (:width previous)) (mth/close? (:height size) (:height previous))))) (rx/of #(assoc-in % [::layout-preview id] size) - (dwm/set-wasm-modifiers (dwwt/resize-wasm-text-modifiers shape))) + (dwm/set-wasm-modifiers (dwwt/resize-wasm-text-modifiers shape) + :interactive? false)) (rx/empty)))))) (defn v3-stop-text-layout-preview From fd03ce2a639d879d033ed85699c07c88808366b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bel=C3=A9n=20Albeza?= Date: Mon, 19 Oct 2026 17:17:35 +0200 Subject: [PATCH 25/30] :bug: Show text typed on a new line during the v3 layout preview (#12384) A modified shape is a cached copy, and edits through get_mut did not reset it. After Enter the preview stops sending modifiers while the size holds, so the canvas kept painting the old text. AI-assisted-by: claude-opus-5-5 --- render-wasm/src/state/shapes_pool.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/render-wasm/src/state/shapes_pool.rs b/render-wasm/src/state/shapes_pool.rs index f39825ebaf..60824bd42f 100644 --- a/render-wasm/src/state/shapes_pool.rs +++ b/render-wasm/src/state/shapes_pool.rs @@ -144,6 +144,10 @@ impl ShapesPoolImpl { pub fn get_mut(&mut self, id: &Uuid) -> Option<&mut Shape> { let idx = *self.uuid_to_idx.get(id)?; + // The caller may change the shape, so its modified copy goes stale. + if let Some(cell) = self.modified_shape_cache.get_mut(&idx) { + *cell = OnceCell::new(); + } Some(&mut self.shapes[idx]) } @@ -555,3 +559,22 @@ impl Clone for ShapesPoolImpl { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_mut_refreshes_the_modified_shape() { + let mut pool = ShapesPool::new(); + pool.initialize(1); + let id = Uuid::new_v4(); + pool.add_shape(id).set_opacity(1.0); + + pool.set_modifiers(HashMap::from([(id, skia::Matrix::translate((10.0, 0.0)))])); + assert_eq!(pool.get(&id).expect("shape").opacity, 1.0); + + pool.get_mut(&id).expect("shape").set_opacity(0.5); + assert_eq!(pool.get(&id).expect("shape").opacity, 0.5); + } +} From 61a66f31ab1be963e90970dd9ae8c5894aa7d5c7 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 15:22:10 +0000 Subject: [PATCH 26/30] :ambulance: Disable media storage quota check on upload Comment out the media-storage-bytes-per-team check in upload-file-media-object. The SUM over storage_object takes 3-12s per upload on prod (3.5TB DB, ~34 slow runs/hour) and blocks uploads. Keep the SQL and quote definition for re-enable with a cheaper implementation. Update the upload quota test to document the disabled state until the new implementation lands. AI-assisted-by: muse-spark-1.3-contributor --- backend/src/app/rpc/commands/media.clj | 27 ++++++++++++------- .../test/backend_tests/rpc_quotes_test.clj | 11 ++++---- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/backend/src/app/rpc/commands/media.clj b/backend/src/app/rpc/commands/media.clj index 99d5daea5a..ce75f53994 100644 --- a/backend/src/app/rpc/commands/media.clj +++ b/backend/src/app/rpc/commands/media.clj @@ -40,11 +40,13 @@ (declare create-file-media-object) -(def ^:private sql:get-team-id-for-file - "SELECT p.team_id - FROM file AS f - JOIN project AS p ON (p.id = f.project_id) - WHERE f.id = ?") +;; TEMPORARILY DISABLED (2026-10-19): only used by the disabled +;; media-storage-bytes-per-team quote check below. Kept for re-enable. +;; (def ^:private sql:get-team-id-for-file +;; "SELECT p.team_id +;; FROM file AS f +;; JOIN project AS p ON (p.id = f.project_id) +;; WHERE f.id = ?") (def ^:private schema:upload-file-media-object [:map {:title "upload-file-media-object"} @@ -64,11 +66,16 @@ (media.v/validate-media-type! content) (media.v/validate-media-size! content) - (let [team-id (:team-id (db/exec-one! pool [sql:get-team-id-for-file file-id]))] - (quotes/check! cfg {::quotes/id ::quotes/media-storage-bytes-per-team - ::quotes/profile-id profile-id - ::quotes/team-id team-id - ::quotes/incr (:size content)})) + ;; TEMPORARILY DISABLED (2026-10-19): media-storage-bytes-per-team + ;; quote performs a full SUM over storage_object per upload (3-12s in + ;; prod on a 3.5TB DB, ~34 slow executions/hour). Disabled until we + ;; land a cheaper approximation (cached counter / rewritten query). + ;; TODO: re-enable with the new implementation. + ;; (let [team-id (:team-id (db/exec-one! pool [sql:get-team-id-for-file file-id]))] + ;; (quotes/check! cfg {::quotes/id ::quotes/media-storage-bytes-per-team + ;; ::quotes/profile-id profile-id + ;; ::quotes/team-id team-id + ;; ::quotes/incr (:size content)})) (db/run! cfg (fn [{:keys [::db/conn] :as cfg}] ;; We get the minimal file for proper checking if diff --git a/backend/test/backend_tests/rpc_quotes_test.clj b/backend/test/backend_tests/rpc_quotes_test.clj index b0740f1b22..ab6b8ede9f 100644 --- a/backend/test/backend_tests/rpc_quotes_test.clj +++ b/backend/test/backend_tests/rpc_quotes_test.clj @@ -442,6 +442,7 @@ (t/is (true? true) "deduped storage counted once, under quota")))) (t/deftest media-upload-enforces-storage-quote + ;; TEMPORARILY DISABLED (2026-10-19), see body. (with-mocks [mock {:target 'app.config/get :return (th/config-get-mock {:quotes-media-storage-bytes-per-team 100})}] @@ -466,12 +467,10 @@ out (th/command! params)] - ;; 312043 bytes > 100 byte limit → should be rejected - (t/is (not (th/success? out))) - (let [error (:error out)] - (t/is (= :restriction (th/ex-type error))) - (t/is (= :max-quote-reached (th/ex-code error))) - (t/is (= "media-storage-bytes-per-team" (:target (ex-data error)))))))) + ;; TEMPORARILY DISABLED (2026-10-19): upload path no longer checks + ;; media-storage-bytes-per-team, so even over quota the upload + ;; succeeds. TODO: restore rejection with cheaper implementation. + (t/is (th/success? out))))) ;; --- SNAPSHOTS (VERSIONS) QUOTES From 29cd50621dd2c014da8be9af752cee0af2c8a78a Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Mon, 19 Oct 2026 17:23:26 +0200 Subject: [PATCH 27/30] :bug: Exclude inner shadows from WebGL export bounds (#12427) Inner shadows were joined into extrect via the drop-shadow filter, so a Y offset grew the PNG/SVG export surface. Only drop shadows expand bounds, matching classic export. Closes #12386. --- ...ports_leaf_inner_shadow_as_svg_filter.snap | 4 +- render-wasm/src/shapes.rs | 49 ++++++++++++++----- render-wasm/src/shapes/shadows.rs | 4 ++ 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_leaf_inner_shadow_as_svg_filter.snap b/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_leaf_inner_shadow_as_svg_filter.snap index feb3c531dd..1127c671f0 100644 --- a/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_leaf_inner_shadow_as_svg_filter.snap +++ b/render-wasm/src/render/svg/snapshots/render_wasm__render__svg__tests__exports_leaf_inner_shadow_as_svg_filter.snap @@ -3,6 +3,6 @@ source: src/render/svg/tests.rs expression: svg --- - - + + diff --git a/render-wasm/src/shapes.rs b/render-wasm/src/shapes.rs index e766b723d3..aaf4e55a49 100644 --- a/render-wasm/src/shapes.rs +++ b/render-wasm/src/shapes.rs @@ -1005,16 +1005,14 @@ impl Shape { let max_stroke = Stroke::max_bounds_width(self.strokes.iter(), self.is_open()); let mut rect = bounds.to_rect(); - for shadow in self.shadows_visible() { - if !shadow.hidden() { - if let Some(filter) = shadow.get_drop_shadow_filter() { - let mut source = rect; - if mitered && shadow.spread > 0.0 { - let tip = (MITER_LIMIT - 1.0) * (max_stroke + shadow.spread); - source.outset((tip, tip)); - } - rect.join(filter.compute_fast_bounds(source)); + for shadow in self.drop_shadows_visible() { + if let Some(filter) = shadow.get_drop_shadow_filter() { + let mut source = rect; + if mitered && shadow.spread > 0.0 { + let tip = (MITER_LIMIT - 1.0) * (max_stroke + shadow.spread); + source.outset((tip, tip)); } + rect.join(filter.compute_fast_bounds(source)); } } Bounds::from_rect(&rect) @@ -1514,10 +1512,6 @@ impl Shape { .filter(|shadow| shadow.style() == ShadowStyle::Inner && !shadow.hidden()) } - pub fn shadows_visible(&self) -> impl DoubleEndedIterator { - self.shadows.iter().rev().filter(|shadow| !shadow.hidden()) - } - pub fn to_path_transform(&self) -> Option { match self.shape_type { Type::Path(_) | Type::Bool(_) => { @@ -2571,6 +2565,35 @@ mod tests { assert!(extrect.bottom > 50.0 + 20.0); } + /// Inner shadows are drawn inside the shape; they must not inflate export + /// size (WebGL PNG surface uses `extrect`). Regression for #12386. + #[test] + fn leaf_extrect_ignores_inner_shadow_offset() { + let mut pool = ShapesPool::new(); + pool.initialize(1); + + let shape_id = Uuid::new_v4(); + { + let shape = pool.add_shape(shape_id); + shape.set_shape_type(Type::Rect(Rect::default())); + shape.set_selrect(0.0, 0.0, 200.0, 200.0); + shape.add_shadow(inner_shadow(0.0, (0.0, 1.0))); + } + + let shape = pool.get(&shape_id).expect("shape should exist"); + let baseline = { + let mut no_shadow = shape.clone(); + no_shadow.clear_shadows(); + no_shadow.calculate_extrect(&pool, 1.0) + }; + let extrect = shape.calculate_extrect(&pool, 1.0); + + assert_eq!(extrect.height(), baseline.height()); + assert_eq!(extrect.width(), baseline.width()); + assert_eq!(extrect.height(), 200.0); + assert_eq!(extrect.width(), 200.0); + } + #[test] fn test_apply_transform() { let mut shape = Shape::new(Uuid::new_v4()); diff --git a/render-wasm/src/shapes/shadows.rs b/render-wasm/src/shapes/shadows.rs index 11a0e13295..b13c6d76bf 100644 --- a/render-wasm/src/shapes/shadows.rs +++ b/render-wasm/src/shapes/shadows.rs @@ -87,6 +87,10 @@ impl Shadow { } pub fn get_drop_shadow_filter(&self) -> Option { + if self.style == ShadowStyle::Inner { + return None; + } + let sigma = radius_to_sigma(self.blur); let mut filter = image_filters::drop_shadow_only( (self.offset.0, self.offset.1), From 2d335e9e83403f19d67a37e37853b85a7579f567 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 15:37:13 +0000 Subject: [PATCH 28/30] :wrench: Reset explicit EXTERNAL storage to PG defaults Drop the legacy TOAST STORAGE EXTERNAL overrides set when PostgreSQL ran on ZFS with transparent compression. Migration 0156 restores the type default (SET STORAGE DEFAULT) on 62 columns across 21 tables; existing TOAST data keeps its strategy until rewritten. Raise the documented PostgreSQL floor to >=16, required by the new SET STORAGE DEFAULT syntax. AI-assisted-by: muse-spark-1.3-contributor --- backend/src/app/migrations.clj | 5 +- .../sql/0156-reset-pg-storage-defaults.sql | 129 ++++++++++++++++++ docs/technical-guide/configuration.md | 2 +- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 861629bb54..9ccedd96e1 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -508,7 +508,10 @@ :fn (mg/resource "app/migrations/sql/0154-del-legacy-file-object-thumbnail-table.sql")} {:name "0155-add-idx-to-file-change" - :fn (mg/resource "app/migrations/sql/0155-add-idx-to-file-change.sql")}]) + :fn (mg/resource "app/migrations/sql/0155-add-idx-to-file-change.sql")} + + {:name "0156-reset-pg-storage-defaults" + :fn (mg/resource "app/migrations/sql/0156-reset-pg-storage-defaults.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql b/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql new file mode 100644 index 0000000000..54bd98ace5 --- /dev/null +++ b/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql @@ -0,0 +1,129 @@ +-- Reset explicit `STORAGE EXTERNAL` settings back to the PostgreSQL defaults. +-- +-- Those were set when PostgreSQL ran on top of ZFS with transparent +-- compression (see 0041-mod-pg-storage-options.sql and the per-table +-- migrations that followed it): skipping the TOAST compression done by +-- PostgreSQL made sense there. On regular filesystems it only wastes +-- space and I/O bandwidth, so restore the default storage strategy of +-- each column type (EXTENDED for every column below: text, text[], +-- jsonb and bytea). +-- +-- This only touches the catalog, it does not rewrite the tables: +-- already stored TOAST values keep their strategy until the row is +-- rewritten, new writes use the default. +-- +-- NOTE: DEFAULT restores the storage mode of the column's data type +-- (EXTENDED for every column below: text, text[], jsonb and bytea). It +-- requires PostgreSQL 16 or newer, which is the minimum version the +-- project supports (see the database section of the technical guide). +-- RESET STORAGE is not valid syntax for a column. + +ALTER TABLE access_token + ALTER COLUMN name SET STORAGE DEFAULT, + ALTER COLUMN perms SET STORAGE DEFAULT, + ALTER COLUMN token SET STORAGE DEFAULT; + +ALTER TABLE audit_log + ALTER COLUMN context SET STORAGE DEFAULT, + ALTER COLUMN name SET STORAGE DEFAULT, + ALTER COLUMN props SET STORAGE DEFAULT, + ALTER COLUMN source SET STORAGE DEFAULT, + ALTER COLUMN type SET STORAGE DEFAULT; + +ALTER TABLE comment + ALTER COLUMN content SET STORAGE DEFAULT; + +ALTER TABLE comment_thread + ALTER COLUMN page_name SET STORAGE DEFAULT, + ALTER COLUMN participants SET STORAGE DEFAULT; + +ALTER TABLE file + ALTER COLUMN data SET STORAGE DEFAULT, + ALTER COLUMN data_backend SET STORAGE DEFAULT, + ALTER COLUMN features SET STORAGE DEFAULT, + ALTER COLUMN name SET STORAGE DEFAULT; + +ALTER TABLE file_change + ALTER COLUMN changes SET STORAGE DEFAULT, + ALTER COLUMN data SET STORAGE DEFAULT, + ALTER COLUMN features SET STORAGE DEFAULT, + ALTER COLUMN label SET STORAGE DEFAULT; + +ALTER TABLE file_data_fragment + ALTER COLUMN metadata SET STORAGE DEFAULT; + +ALTER TABLE file_media_object + ALTER COLUMN mtype SET STORAGE DEFAULT, + ALTER COLUMN name SET STORAGE DEFAULT; + +ALTER TABLE file_thumbnail + ALTER COLUMN data SET STORAGE DEFAULT, + ALTER COLUMN props SET STORAGE DEFAULT; + +ALTER TABLE global_complaint_report + ALTER COLUMN content SET STORAGE DEFAULT, + ALTER COLUMN type SET STORAGE DEFAULT; + +ALTER TABLE profile + ALTER COLUMN auth_backend SET STORAGE DEFAULT, + ALTER COLUMN email SET STORAGE DEFAULT, + ALTER COLUMN fullname SET STORAGE DEFAULT, + ALTER COLUMN lang SET STORAGE DEFAULT, + ALTER COLUMN password SET STORAGE DEFAULT, + ALTER COLUMN props SET STORAGE DEFAULT, + ALTER COLUMN theme SET STORAGE DEFAULT; + +ALTER TABLE profile_complaint_report + ALTER COLUMN content SET STORAGE DEFAULT, + ALTER COLUMN type SET STORAGE DEFAULT; + +ALTER TABLE project + ALTER COLUMN name SET STORAGE DEFAULT; + +ALTER TABLE scheduled_task + ALTER COLUMN cron_expr SET STORAGE DEFAULT, + ALTER COLUMN id SET STORAGE DEFAULT; + +ALTER TABLE server_error_report + ALTER COLUMN content SET STORAGE DEFAULT; + +ALTER TABLE server_prop + ALTER COLUMN content SET STORAGE DEFAULT, + ALTER COLUMN id SET STORAGE DEFAULT; + +ALTER TABLE storage_object + ALTER COLUMN backend SET STORAGE DEFAULT, + ALTER COLUMN metadata SET STORAGE DEFAULT; + +ALTER TABLE task + ALTER COLUMN error SET STORAGE DEFAULT, + ALTER COLUMN label SET STORAGE DEFAULT, + ALTER COLUMN name SET STORAGE DEFAULT, + ALTER COLUMN props SET STORAGE DEFAULT, + ALTER COLUMN queue SET STORAGE DEFAULT, + ALTER COLUMN status SET STORAGE DEFAULT; + +ALTER TABLE team + ALTER COLUMN name SET STORAGE DEFAULT; + +ALTER TABLE team_font_variant + ALTER COLUMN font_family SET STORAGE DEFAULT, + ALTER COLUMN font_style SET STORAGE DEFAULT; + +ALTER TABLE team_invitation + ALTER COLUMN email_to SET STORAGE DEFAULT, + ALTER COLUMN role SET STORAGE DEFAULT; + +ALTER TABLE usage_quote + ALTER COLUMN target SET STORAGE DEFAULT; + +ALTER TABLE webhook + ALTER COLUMN error_code SET STORAGE DEFAULT, + ALTER COLUMN mtype SET STORAGE DEFAULT, + ALTER COLUMN secret_key SET STORAGE DEFAULT, + ALTER COLUMN uri SET STORAGE DEFAULT; + +ALTER TABLE webhook_delivery + ALTER COLUMN error_code SET STORAGE DEFAULT, + ALTER COLUMN req_data SET STORAGE DEFAULT, + ALTER COLUMN rsp_data SET STORAGE DEFAULT; diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 294885c897..59939a522b 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -484,7 +484,7 @@ PENPOT_SECRET_KEY: my-super-secure-key ### Database -Penpot only supports PostgreSQL and we highly recommend >=13 version. If you are using official +Penpot only supports PostgreSQL and we highly recommend >=16 version. If you are using official docker images this is already solved for you. Essential database configuration: From 534d46c7b3e40816ec4f2d6c2744724022521cf3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 19 Oct 2026 15:59:16 +0000 Subject: [PATCH 29/30] :books: Update changelog Add the missing 2.18.4 entries for the closed storage GC, version quota and file_change revision issues, with their fix PRs referenced inline. AI-assisted-by: muse-spark-1.3-contributor --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3a2b8dab1c..d0ea0e6aee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,9 @@ - Fix file restore leaving its storage objects marked for deletion [#12278](https://github.com/penpot/penpot/issues/12278) (PR: [#12269](https://github.com/penpot/penpot/pull/12269)) - Fix backend workers staying parked forever when nested calls take the same concurrency limits [#12279](https://github.com/penpot/penpot/issues/12279) (PR: [#12269](https://github.com/penpot/penpot/pull/12269)) - Fix one profile's activity blocking the audit events of every other profile [#12280](https://github.com/penpot/penpot/issues/12280) (PR: [#12269](https://github.com/penpot/penpot/pull/12269)) +- Fix storage garbage collection blocking database vacuum during long deletions [#12396](https://github.com/penpot/penpot/issues/12396) (PR: [#12397](https://github.com/penpot/penpot/pull/12397)) +- Fix saving a version taking seconds and the version quota never counting saved versions [#12404](https://github.com/penpot/penpot/issues/12404) (PR: [#12405](https://github.com/penpot/penpot/pull/12405)) +- Fix file_change accumulating revisions that the objects GC never collects [#12418](https://github.com/penpot/penpot/issues/12418) (PR: [#12419](https://github.com/penpot/penpot/pull/12419)) ### :sparkles: New features & Enhancements From 0801674af2b88f4520cd47e403a6331f67bd430c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elena=20Torr=C3=B3?= Date: Mon, 19 Oct 2026 18:21:02 +0200 Subject: [PATCH 30/30] :bug: Ignore export scale in render-wasm PDF export (#12447) --- exporter/src/app/wasm.cljs | 5 ++--- exporter/src/app/wasm/render.cljs | 2 +- frontend/src/app/main/data/exports/wasm.cljs | 4 ++-- frontend/src/app/render_wasm/api.cljs | 5 ++--- render-wasm/src/main.rs | 4 ++-- render-wasm/src/render/pdf.rs | 11 +++-------- render-wasm/src/render/vector.rs | 3 +-- render-wasm/src/state.rs | 4 ++-- 8 files changed, 15 insertions(+), 23 deletions(-) diff --git a/exporter/src/app/wasm.cljs b/exporter/src/app/wasm.cljs index 0160a89a4a..88ae874009 100644 --- a/exporter/src/app/wasm.cljs +++ b/exporter/src/app/wasm.cljs @@ -287,11 +287,10 @@ (defn render-shape-pdf "Renders the shape subtree to PDF bytes (Uint8Array)." - [shape-id scale] + [shape-id] (let [buf (uuid/get-u32 shape-id)] (-> (h/call wasm/internal-module "_render_shape_pdf" - (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3) - scale) + (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3)) (read-render-result LEN-HEADER-BYTES)))) (defn render-shape-svg diff --git a/exporter/src/app/wasm/render.cljs b/exporter/src/app/wasm/render.cljs index 77e53a9e6b..7889a6e3c5 100644 --- a/exporter/src/app/wasm/render.cljs +++ b/exporter/src/app/wasm/render.cljs @@ -418,7 +418,7 @@ (defn- render-object-bytes [type id scale] (case type - :pdf (let [bytes (wasm/render-shape-pdf id scale)] + :pdf (let [bytes (wasm/render-shape-pdf id)] (l/dbg :hint "PDF generated via Skia (render-wasm headless)" :object-id (str id) :backend "skia-wasm" diff --git a/frontend/src/app/main/data/exports/wasm.cljs b/frontend/src/app/main/data/exports/wasm.cljs index ff56fd5796..696b8ed716 100644 --- a/frontend/src/app/main/data/exports/wasm.cljs +++ b/frontend/src/app/main/data/exports/wasm.cljs @@ -30,8 +30,8 @@ nil)) (defn export-pdf-uri - [{:keys [scale object-id]}] - (let [bytes (wasm.api/render-shape-pdf object-id (or scale 1)) + [{:keys [object-id]}] + (let [bytes (wasm.api/render-shape-pdf object-id) blob (wapi/create-blob bytes "application/pdf")] (wapi/create-uri blob))) diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index 78623aa43e..2227b4155e 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -3010,7 +3010,7 @@ {:x x :y y :width w :height h}))))) (defn render-shape-pdf - [shape-id scale] + [shape-id] (when (initialized?) (let [buffer (uuid/get-u32 shape-id) @@ -3019,8 +3019,7 @@ (aget buffer 0) (aget buffer 1) (aget buffer 2) - (aget buffer 3) - scale) + (aget buffer 3)) heap (mem/get-heap-u8) heapu32 (mem/get-heap-u32) diff --git a/render-wasm/src/main.rs b/render-wasm/src/main.rs index 125f553a0b..11a8256842 100644 --- a/render-wasm/src/main.rs +++ b/render-wasm/src/main.rs @@ -818,11 +818,11 @@ pub fn free_gpu_resources() { #[no_mangle] #[wasm_error] -pub extern "C" fn render_shape_pdf(a: u32, b: u32, c: u32, d: u32, scale: f32) -> Result<*mut u8> { +pub extern "C" fn render_shape_pdf(a: u32, b: u32, c: u32, d: u32) -> Result<*mut u8> { let id = uuid_from_u32_quartet(a, b, c, d); with_state!(state, { - let data = state.render_shape_pdf(&id, scale)?; + let data = state.render_shape_pdf(&id)?; let len = data.len() as u32; let mut buf = Vec::with_capacity(4 + data.len()); diff --git a/render-wasm/src/render/pdf.rs b/render-wasm/src/render/pdf.rs index cf53731c30..b8e204dcb6 100644 --- a/render-wasm/src/render/pdf.rs +++ b/render-wasm/src/render/pdf.rs @@ -19,17 +19,13 @@ pub fn render_to_pdf( shared: &mut RenderResources, id: &Uuid, tree: ShapesPoolRef, - scale: f32, ) -> Result> { let shape = tree .get(id) .ok_or_else(|| crate::error::Error::CriticalError("Shape not found for PDF".to_string()))?; // Boards crop overflowing children (classic board box) but keep their own // strokes/shadows/blur. Other shapes use full extrect. - let bounds = shape.export_root_bounds(tree, scale); - - let page_w = bounds.width() * scale; - let page_h = bounds.height() * scale; + let bounds = shape.export_root_bounds(tree, 1.0); let mut pdf_bytes: Vec = Vec::new(); @@ -42,13 +38,12 @@ pub fn render_to_pdf( let document = skia::pdf::new_document(&mut pdf_bytes, Some(&metadata)); - let mut on_page = document.begin_page((page_w, page_h), None); + let mut on_page = document.begin_page((bounds.width(), bounds.height()), None); { let page_canvas = on_page.canvas(); - page_canvas.scale((scale, scale)); page_canvas.translate((-bounds.left(), -bounds.top())); - vector::render_tree_pdf(shared, page_canvas, id, tree, scale, bounds)?; + vector::render_tree_pdf(shared, page_canvas, id, tree, bounds)?; } let document = on_page.end_page(); diff --git a/render-wasm/src/render/vector.rs b/render-wasm/src/render/vector.rs index 7eddda2b8b..e9d873f353 100644 --- a/render-wasm/src/render/vector.rs +++ b/render-wasm/src/render/vector.rs @@ -474,10 +474,9 @@ pub(super) fn render_tree_pdf( canvas: &Canvas, id: &Uuid, tree: ShapesPoolRef, - scale: f32, page: skia::Rect, ) -> Result<()> { - render_tree_dispatch(shared, canvas, id, tree, scale, page, true) + render_tree_dispatch(shared, canvas, id, tree, 1.0, page, true) } #[allow(clippy::too_many_arguments)] diff --git a/render-wasm/src/state.rs b/render-wasm/src/state.rs index 5b2684ecb4..9efce2b024 100644 --- a/render-wasm/src/state.rs +++ b/render-wasm/src/state.rs @@ -101,8 +101,8 @@ impl State { get_render_state().render_shape_pixels(id, &self.shapes, scale, timestamp, format) } - pub fn render_shape_pdf(&mut self, id: &Uuid, scale: f32) -> Result> { - crate::render::pdf::render_to_pdf(get_resources(), id, &self.shapes, scale) + pub fn render_shape_pdf(&mut self, id: &Uuid) -> Result> { + crate::render::pdf::render_to_pdf(get_resources(), id, &self.shapes) } pub fn render_shape_svg(&mut self, id: &Uuid, scale: f32) -> Result> {