diff --git a/.serena/memories/backend/testing.md b/.serena/memories/backend/testing.md index 62bd36173d..671ebc1b50 100644 --- a/.serena/memories/backend/testing.md +++ b/.serena/memories/backend/testing.md @@ -32,4 +32,4 @@ JVM `clojure.test` (kaocha runner) under `backend/test/backend_tests/`. `worker-config` can talk to a different Redis key prefix than a cfg the test builds itself. The symptom is a component that silently does nothing locally and races in CI (which sets no `PENPOT_TENANT`). -- `backend-tests.helpers` has `metrics-stub` (a metrics instance with no collectors, where every `run!` is a no-op), `init-climit` (a concurrency limiter built from an inline config map, with an optional second argument for the metrics instance to publish to) and `with-climit`, the macro that builds one, runs the body and halts it. `init-climit` starts the republisher daemon thread, so a test that builds a limiter without halting it leaks that thread for the rest of the JVM run: use `with-climit`. Log lines are capturable with `add-watch` on `app.common.logging/log-record` (it holds the last record; the watcher sees every one, but only of the levels the logger has enabled: a line the app writes at TRACE never gets there). +- `backend-tests.helpers` has `metrics-stub` (a metrics instance with no collectors, where every `run!` is a no-op), `init-climit` (a concurrency limiter built from an inline config map, with an optional second argument for the metrics instance to publish to) and `with-climit`, the macro that builds one, runs the body and halts it. `init-climit` starts the republisher daemon thread, so a test that builds a limiter without halting it leaks that thread for the rest of the JVM run: use `with-climit`. Log lines are capturable with `add-watch` on `app.common.logging/log-record` (it holds the last record; the watcher sees every one, but only of the levels the logger has enabled: a line the app writes at TRACE never gets there). The logger swaps the record on an executor thread, after the call that logged it returns: keep the watch installed until the record arrives (poll for it) and only then remove it, or the record lands on nobody under load. diff --git a/.serena/memories/frontend/testing.md b/.serena/memories/frontend/testing.md index 37f5304395..3bfbe08542 100644 --- a/.serena/memories/frontend/testing.md +++ b/.serena/memories/frontend/testing.md @@ -40,6 +40,7 @@ To assert over emitted event sequences, observe termination: subscribe through ` - Teardown belongs to the terminal continuation, never to `finally`-around-triggers (it would dispose in-flight flows). - A body that awaits must be `(^:async fn …)` even if the rest is sync; sync sequences are atomic vs the event loop. - The `:esm` test build cannot load the CommonJS `sax` module behind `tubax`, so SVG parsing fails in tests (`dwm/valid-svg-string?` answers false, `dwm/svg->clj` errors): mock both with the parsed data. +- `set!` mocks are global and outlive the fixture, so a flow another module left running (a store's pending timer) can still reach the mocked transport mid-test; `with-persistence` records every command for that reason. Filter to the commands your layer issues — persistence only sends `update-file` — before counting or asserting emptiness, and never let a stray call release a `wait-for` on one. From `frontend/`: - Full unit test run (always builds, suppressed output): `pnpm run test:quiet`. diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index e45794a739..6019839a59 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -13,6 +13,7 @@ [app.common.pprint :as pp] [app.common.thumbnails :as thc] [app.common.time :as ct] + [app.common.types.page :as ctp] [app.common.types.shape :as cts] [app.common.types.tokens-lib :as ctob] [app.common.types.tokens-status :as ctos] @@ -3433,3 +3434,35 @@ (t/is (= (into #{} (map :id) (:result miss)) (into #{} (map :id) (:result hit)))) (t/is (every? #(not (contains? % :migrations)) (:result hit)))))))) + +(t/deftest add-page-rejects-an-invalid-shape + ;; Duplicate page and the undo of a page deletion send the whole page as + ;; the client holds it. A tab that loaded the file before a migration + ;; ran holds the old format, here camelCase svg attrs (#13107), and the + ;; page must not reach the file: the next migrating read would reject + ;; the whole file. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + page-id (uuid/next) + shape-id (uuid/next) + shape (cts/setup-shape {:id shape-id :type :rect :name "rect" + :x 0 :y 0 :width 10 :height 10 + :parent-id uuid/zero :frame-id uuid/zero + :svg-attrs {:fillRule "nonzero"}}) + page (-> (ctp/make-empty-page {:id page-id :name "Copy"}) + (assoc-in [:objects shape-id] shape) + (update-in [:objects uuid/zero :shapes] (fnil conj []) shape-id)) + out (th/command! {::th/type :update-file + ::rpc/profile-id (:id profile) + :id (:id file) + :session-id (uuid/random) + :revn 0 + :vern 0 + :features cfeat/supported-features + :changes [{:type :add-page :id page-id :page page}]}) + stored (bfc/get-file th/*system* (:id file) :migrate? false :realize? true)] + (t/is (th/ex-of-type? (:error out) :assertion)) + (t/is (th/ex-of-code? (:error out) :data-validation)) + (t/is (not (contains? (get-in stored [:data :pages-index]) page-id))))) diff --git a/backend/test/backend_tests/rpc_file_update_test.clj b/backend/test/backend_tests/rpc_file_update_test.clj index c5a8ba9b6d..34e4fe8b00 100644 --- a/backend/test/backend_tests/rpc_file_update_test.clj +++ b/backend/test/backend_tests/rpc_file_update_test.clj @@ -282,18 +282,20 @@ (defn- update-file-logged "Updates the file and returns the response in `:out` and, in `:rejected`, - the props of the log record of the rejection, if any." + the props of the log record of the rejection, if any. Keeps the watch + until the record arrives: the logger writes it on another thread, so it + can land after the update returns." [ctx params] - (let [records (atom []) - out (try - (add-watch l/log-record ::capture - (fn [_ _ _ record] (swap! records conj record))) - (update-file ctx params) - (finally - (remove-watch l/log-record ::capture)))] - {:out out - :rejected (some-> (wait-for-record records "update-file commit rejected") - ::l/props)})) + (let [records (atom [])] + (try + (add-watch l/log-record ::capture + (fn [_ _ _ record] (swap! records conj record))) + (let [out (update-file ctx params)] + {:out out + :rejected (some-> (wait-for-record records "update-file commit rejected") + ::l/props)}) + (finally + (remove-watch l/log-record ::capture))))) (t/deftest bisects-the-commits-to-find-the-invalid-one (let [{:keys [page-id] :as ctx} (create-file) diff --git a/common/src/app/common/files/changes.cljc b/common/src/app/common/files/changes.cljc index b2b0e6de60..b81764e580 100644 --- a/common/src/app/common/files/changes.cljc +++ b/common/src/app/common/files/changes.cljc @@ -930,6 +930,13 @@ (when (and id name page) (ex/raise :type :conflict :hint "id+name or page should be provided, never both")) + + ;; NOTE: we only perform hard validation on backend. The page arrives + ;; whole (duplicate page, undo of a page deletion), as the client holds + ;; it, so each of its shapes is validated as `:add-obj` validates one. + #?(:clj (doseq [shape (vals (:objects page))] + (validate-shape shape (:id page)))) + (let [page (if (and (string? name) (uuid? id)) (ctp/make-empty-page {:id id :name name}) page)] diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 70e78627af..b930a88385 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -2345,19 +2345,22 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0035-fix-shape-svg-attrs-and-defs" +(defmethod migrate-data "0037-fix-shape-svg-attrs-and-defs" ;; Stored svg attribute keys are kebab-case (issue #11947): the v3 reader ;; already outputs kebab and birth normalizes any outside spelling, so ;; export/import converges to the same state. Old birth and the cleaner ;; era stored camelCase React prop names instead, which the reader - ;; kebab-ized on import beyond repair. Convert once, with the same - ;; birth transforms: `:svg-attrs` and `:svg-defs` node attrs go through + ;; kebab-ized on import beyond repair. Convert with the same birth + ;; transforms: `:svg-attrs` and `:svg-defs` node attrs go through ;; the whitelist+kebab transform (unknown attrs drop, same as birth), ;; while svg-raw `:content` attrs are kebab-ized spelling-only (unknown ;; keys kept: the sidebar reads keys outside the whitelist). Kebab ;; input passes through unchanged, so re-running is safe. This replaces - ;; "0021-fix-shape-svg-attrs" (id withdrawn from `available-migrations` - ;; below): files that recorded it migrate again through this one. + ;; "0021-fix-shape-svg-attrs" and "0035-fix-shape-svg-attrs-and-defs" + ;; (ids withdrawn from `available-migrations` below): files that + ;; recorded either migrate again through this one. 0035 was withdrawn + ;; because a client that loaded a file before 0035 ran could still save + ;; camelCase keys into it afterwards (#13107). ;; Node tags in defs and svg-raw content are stored as keywords; files ;; that hold them as strings get them converted here, since the closed ;; node schemas reject strings. `:svg-transform` held as a plain map @@ -2502,5 +2505,5 @@ "0033-normalize-main-instance-names" "0034-fix-path-content-offset" "0035-fix-dangling-children" - "0035-fix-shape-svg-attrs-and-defs" - "0036-clear-nil-keys-in-objects"])) + "0036-clear-nil-keys-in-objects" + "0037-fix-shape-svg-attrs-and-defs"])) diff --git a/common/test/common_tests/files_migrations_0035_test.cljc b/common/test/common_tests/files_migrations_0035_test.cljc index 078bc895e4..c64ce69527 100644 --- a/common/test/common_tests/files_migrations_0035_test.cljc +++ b/common/test/common_tests/files_migrations_0035_test.cljc @@ -7,8 +7,6 @@ (ns common-tests.files-migrations-0035-test (:require [app.common.files.migrations :as cfm] - [app.common.geom.matrix :as gmt] - [app.common.geom.rect :as grc] [app.common.types.file :as ctf] [app.common.types.objects-map :as omap] [app.common.types.shape :as cts] @@ -189,333 +187,3 @@ (t/is (some? (get objects' frame-id)) "the frame that exists is kept")))) -;; ====================================================================== -;; 0035-fix-shape-svg-attrs-and-defs -;; Stored svg attribute keys are kebab-case (issue #11947). Old birth and -;; the cleaner era stored camelCase React prop names, which the v3 reader -;; kebab-ized on import beyond repair. This migration converts once, with -;; the same birth transforms: `:svg-attrs` and `:svg-defs` node attrs go -;; through the whitelist+kebab transform, svg-raw `:content` attrs are -;; kebab-ized spelling-only. It replaces "0021-fix-shape-svg-attrs" (id -;; withdrawn): files that recorded it migrate again through this one. -;; ====================================================================== - -(def ^:private migration-id - "0035-fix-shape-svg-attrs-and-defs") - -(defn- make-shape - [shape-id shape] - (assoc shape :id shape-id)) - -(defn- camel-shape - [shape-id] - (make-shape shape-id - {:type :rect - :svg-attrs {:fillRule "evenodd" - :stroke-width "2" - :stroke-style "dotted" - :data-foo "drop-me"}})) - -(defn- camel-defs - [] - {"g1" {:tag :linearGradient - :attrs {:id "g1" - :gradientUnits "userSpaceOnUse" - :xlink:href "#base" - :data-x "drop-me"} - :content [{:tag :stop - :attrs {:offset "0" - :stopColor "#ffffff"} - :content []}]}}) - -(t/deftest migration-0035-converts-camel-svg-attrs-to-kebab-in-pages - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (camel-shape shape-id)}}}} - data' (cfm/migrate-data data migration-id) - attrs (get-in data' [:pages-index page-id :objects shape-id :svg-attrs])] - (t/is (= "evenodd" (:fill-rule attrs))) - (t/is (= "2" (:stroke-width attrs))) - (t/is (= "dotted" (:stroke-style attrs)) "penpot-extra-attrs survive") - (t/is (not (contains? attrs :data-foo)) "unknown attrs are dropped") - (t/is (not (contains? attrs :fillRule)) "no camel keys survive"))) - -(t/deftest migration-0035-converts-camel-svg-attrs-to-kebab-in-components - (let [shape-id (uuid/next) - component-id (uuid/next) - data {:components - {component-id - {:objects - {shape-id (camel-shape shape-id)}}}} - data' (cfm/migrate-data data migration-id) - attrs (get-in data' [:components component-id :objects shape-id :svg-attrs])] - (t/is (= "evenodd" (:fill-rule attrs))) - (t/is (not (contains? attrs :fillRule)) "no camel keys survive"))) - -(t/deftest migration-0035-normalizes-def-node-attrs - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :rect - :svg-defs (camel-defs)})}}}} - data' (cfm/migrate-data data migration-id) - grad (get-in data' [:pages-index page-id :objects shape-id :svg-defs "g1"])] - (t/is (= :linearGradient (:tag grad)) "tags are untouched") - (t/is (= "g1" (get-in grad [:attrs :id])) ":id survives") - (t/is (= "userSpaceOnUse" (get-in grad [:attrs :gradient-units]))) - (t/is (= "#base" (get-in grad [:attrs :xlink-href]))) - (t/is (not (contains? (:attrs grad) :data-x)) "unknown def attrs are dropped") - (t/is (= "#ffffff" (get-in grad [:content 0 :attrs :stop-color])) - "nested stops are normalized"))) - -(t/deftest migration-0035-kebabizes-content-spelling-only - (let [shape-id (uuid/next) - page-id (uuid/next) - raw-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :path - :content [{:marker "untouched"}]}) - raw-id (make-shape raw-id - {:type :svg-raw - :content {:tag :g - :attrs {:strokeWidth "2" - :data-foo "keep-me"} - :content [{:tag :path - :attrs {:strokeLinecap "round"} - :content []}]}})}}}} - data' (cfm/migrate-data data migration-id) - path (get-in data' [:pages-index page-id :objects shape-id]) - raw (get-in data' [:pages-index page-id :objects raw-id]) - content (:content raw)] - (t/is (= [{:marker "untouched"}] (:content path)) - "non-svg-raw content is untouched") - (t/is (= "2" (get-in content [:attrs :stroke-width]))) - (t/is (= "keep-me" (get-in content [:attrs :data-foo])) - "unknown content keys survive") - (t/is (= "round" (get-in content [:content 0 :attrs :stroke-linecap]))))) - -(defn- string-tag-defs - [] - {:a {:tag "mask" - :attrs {:id "a" :maskUnits "userSpaceOnUse"} - :content [{:tag "rect" - :attrs {:width "10" :height "10" :fill "#FFF"} - :content []}]}}) - -(defn- string-tag-content - [] - {:tag "g" - :attrs {} - :content [{:tag "text" - :attrs {:fontSize "12"} - :content ["Hi"]} - {:tag "style" - :attrs nil - :content [".a{fill:red}"]}]}) - -(t/deftest migration-0035-keywordizes-string-tags - (let [shape-id (uuid/next) - raw-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :rect - :svg-defs (string-tag-defs)}) - raw-id (make-shape raw-id - {:type :svg-raw - :content (string-tag-content)})}}}} - data' (cfm/migrate-data data migration-id) - mask (get-in data' [:pages-index page-id :objects shape-id :svg-defs :a]) - content (get-in data' [:pages-index page-id :objects raw-id :content])] - (t/is (= :mask (:tag mask))) - (t/is (= :rect (get-in mask [:content 0 :tag])) "nested def tags too") - (t/is (= "userSpaceOnUse" (get-in mask [:attrs :mask-units]))) - (t/is (= :g (:tag content))) - (t/is (= :text (get-in content [:content 0 :tag])) "nested content tags too") - (t/is (= ["Hi"] (get-in content [:content 0 :content]))) - (t/is (= {} (get-in content [:content 1 :attrs])) "nil attrs become an empty map") - (t/is (= data' (cfm/migrate-data data' migration-id)) "re-running is stable"))) - -(t/deftest migration-0035-string-tag-file-passes-file-validation - ;; `migrate-file` validates the whole file data after migrating: a file - ;; with string node tags or nil node attrs must come out valid, or it - ;; cannot be opened. - (let [rect-id (uuid/next) - raw-id (uuid/next) - file (ctf/make-file {:name "String svg tags"}) - page-id (first (get-in file [:data :pages])) - rect (-> (cts/setup-shape {:id rect-id :type :rect :name "rect"}) - (assoc :svg-defs (string-tag-defs))) - raw (-> (cts/setup-shape {:id raw-id :type :svg-raw :name "raw"}) - (assoc :content (string-tag-content))) - file (-> file - (assoc :migrations (disj cfm/available-migrations migration-id)) - (update-in [:data :pages-index page-id :objects] assoc rect-id rect raw-id raw) - (update-in [:data :pages-index page-id :objects uuid/zero :shapes] conj rect-id raw-id)) - file' (cfm/migrate-file file {})] - (t/is (= :mask (get-in file' [:data :pages-index page-id :objects rect-id :svg-defs :a :tag]))) - (t/is (= :g (get-in file' [:data :pages-index page-id :objects raw-id :content :tag]))))) - -(t/deftest migration-0035-drops-map-valued-svg-attrs - ;; Files exported by old Penpot versions hold maps where an svg attr - ;; value should be, such as a `:filter` that once read - ;; `url(#filter_2)`. The closed `:svg-attrs` schema admits scalars - ;; only, so `migrate-file` must drop them or the file cannot be - ;; imported or opened. The values below are copied from the - ;; "Avataaars" and "Tutorial for beginners" templates. - (let [shape-id (uuid/next) - file (ctf/make-file {:name "Map-valued svg attrs"}) - page-id (first (get-in file [:data :pages])) - shape (-> (cts/setup-shape {:id shape-id :type :rect :name "rect"}) - (assoc :svg-attrs {:fillRule "evenodd" - :filter {(keyword "{") "{ \"url(#filter_2) nil}\""} - :mask {(keyword "url(-filter0-d-16837-160)") nil} - :style {:fill "red" - :filter {:a "b"}}})) - file (-> file - (assoc :migrations (disj cfm/available-migrations migration-id)) - (update-in [:data :pages-index page-id :objects] assoc shape-id shape) - (update-in [:data :pages-index page-id :objects uuid/zero :shapes] conj shape-id)) - file' (cfm/migrate-file file {}) - attrs (get-in file' [:data :pages-index page-id :objects shape-id :svg-attrs])] - (t/is (= {:fill-rule "evenodd" :style {:fill "red"}} attrs) - "map values are dropped, scalars are kept"))) - -(t/deftest migration-0035-turns-plain-map-svg-transform-into-matrix - ;; The svg defs renderer multiplies `:svg-transform` as a matrix; a - ;; plain map there renders as `matrix(NaN, ...)`. - (let [plain-id (uuid/next) - matrix-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {plain-id (make-shape plain-id - {:type :path - :svg-transform {:a 1.0 :b 0.0 :c 0.0 :d 1.0 :e 10.0 :f 20.0}}) - matrix-id (make-shape matrix-id - {:type :path - :svg-transform (gmt/matrix 2 0 0 2 0 0)})}}}} - data' (cfm/migrate-data data migration-id) - plain' (get-in data' [:pages-index page-id :objects plain-id :svg-transform]) - matrix' (get-in data' [:pages-index page-id :objects matrix-id :svg-transform])] - (t/is (gmt/matrix? plain')) - (t/is (= (gmt/matrix 1 0 0 1 10 20) plain')) - (t/is (= (gmt/matrix 2 0 0 2 0 0) matrix') "matrices are left as they are") - (t/is (= data' (cfm/migrate-data data' migration-id)) "re-running is stable"))) - -(t/deftest migration-0035-leaves-kebab-files-equal - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :rect - :svg-attrs {:fill-rule "evenodd" - :stroke-width "2"} - :svg-defs {"g1" {:tag :linearGradient - :attrs {:id "g1" - :gradient-units "userSpaceOnUse"} - :content []}}})}}}} - data' (cfm/migrate-data data migration-id)] - (t/is (= data data') "kebab is a fixed point"))) - -(t/deftest migration-0035-is-idempotent - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (camel-shape shape-id)}}}} - data' (cfm/migrate-data data migration-id)] - (t/is (= data' (cfm/migrate-data data' migration-id))))) - -(t/deftest migration-0035-normalizes-flat-svg-viewbox-to-rect - ;; Carried over from the withdrawn 0021: legacy files hold - ;; `:svg-viewbox` as a plain map instead of a rect record. - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :rect - :svg-viewbox {:x 1 :y 2 :width 3 :height 4}})}}}} - data' (cfm/migrate-data data migration-id) - viewbox (get-in data' [:pages-index page-id :objects shape-id :svg-viewbox])] - (t/is (grc/rect? viewbox) "flat viewbox becomes a rect record") - (t/is (= 1 (:x viewbox))) - (t/is (= 2 (:y viewbox))) - (t/is (= 3 (:width viewbox))) - (t/is (= 4 (:height viewbox))) - (t/is (= data' (cfm/migrate-data data' migration-id)) - "re-running is stable"))) - -(t/deftest migration-0035-passes-non-maps-through - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :rect - :svg-attrs "junk" - :svg-defs "junk"})}}}} - data' (cfm/migrate-data data migration-id) - shape (get-in data' [:pages-index page-id :objects shape-id])] - (t/is (= "junk" (:svg-attrs shape)) "non-maps reach validation untouched") - (t/is (= "junk" (:svg-defs shape))))) - -(t/deftest migration-0035-junk-viewbox-becomes-default-rect - ;; Unlike `:svg-attrs`/`:svg-defs` (passed through to validation), - ;; `:svg-viewbox` goes through `make-rect`, which does not throw on - ;; junk: it falls back to the default rect. Inherited from 0021. - (let [shape-id (uuid/next) - page-id (uuid/next) - data {:pages-index - {page-id - {:objects - {shape-id (make-shape shape-id - {:type :rect - :svg-viewbox "junk"})}}}} - data' (cfm/migrate-data data migration-id) - viewbox (get-in data' [:pages-index page-id :objects shape-id :svg-viewbox])] - (t/is (grc/rect? viewbox) "junk viewbox becomes a rect, not an error") - (t/is (= 0 (:x viewbox))) - (t/is (= 0.01 (:width viewbox))))) - -(t/deftest migration-0035-runs-through-file-migration - ;; A file that recorded the withdrawn "0021-fix-shape-svg-attrs" (and - ;; everything else current) still needs migration: 0021 left - ;; `available-migrations`, which re-arms it for 0035. - (let [shape-id (uuid/next) - file (ctf/make-file {:name "Legacy camel svg attrs"}) - page-id (first (get-in file [:data :pages])) - shape (-> (cts/setup-shape {:id shape-id :type :rect}) - (assoc :svg-attrs {:fillRule "evenodd"})) - file (-> file - (assoc :migrations (-> cfm/available-migrations - (disj migration-id) - (conj "0021-fix-shape-svg-attrs"))) - (assoc-in [:data :pages-index page-id :objects shape-id] shape)) - file' (cfm/migrate-file file {})] - (t/is (cfm/need-migration? file) "new migration detected") - (t/is (not (cfm/need-migration? file')) "new migration recorded") - (t/is (contains? (:migrations file') migration-id) "migration id persisted") - (t/is (not (contains? cfm/available-migrations "0021-fix-shape-svg-attrs")) - "the withdrawn 0021 id stays out: its camel-restoring transform must never run again") - (t/is (= "evenodd" - (get-in file' [:data :pages-index page-id :objects shape-id :svg-attrs :fill-rule])) - "migration repaired file data before schema validation"))) diff --git a/common/test/common_tests/files_migrations_0037_test.cljc b/common/test/common_tests/files_migrations_0037_test.cljc new file mode 100644 index 0000000000..eff0afb82a --- /dev/null +++ b/common/test/common_tests/files_migrations_0037_test.cljc @@ -0,0 +1,371 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0037-test + (:require + [app.common.files.migrations :as cfm] + [app.common.geom.matrix :as gmt] + [app.common.geom.rect :as grc] + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; ====================================================================== +;; 0037-fix-shape-svg-attrs-and-defs +;; Stored svg attribute keys are kebab-case (issue #11947). Old birth and +;; the cleaner era stored camelCase React prop names, which the v3 reader +;; kebab-ized on import beyond repair. This migration converts them with +;; the same birth transforms: `:svg-attrs` and `:svg-defs` node attrs go +;; through the whitelist+kebab transform, svg-raw `:content` attrs are +;; kebab-ized spelling-only. It replaces "0021-fix-shape-svg-attrs" and +;; "0035-fix-shape-svg-attrs-and-defs" (ids withdrawn): files that +;; recorded either migrate again through this one. +;; ====================================================================== + +(def ^:private migration-id + "0037-fix-shape-svg-attrs-and-defs") + +(defn- make-shape + [shape-id shape] + (assoc shape :id shape-id)) + +(defn- camel-shape + [shape-id] + (make-shape shape-id + {:type :rect + :svg-attrs {:fillRule "evenodd" + :stroke-width "2" + :stroke-style "dotted" + :data-foo "drop-me"}})) + +(defn- camel-defs + [] + {"g1" {:tag :linearGradient + :attrs {:id "g1" + :gradientUnits "userSpaceOnUse" + :xlink:href "#base" + :data-x "drop-me"} + :content [{:tag :stop + :attrs {:offset "0" + :stopColor "#ffffff"} + :content []}]}}) + +(t/deftest migration-0037-converts-camel-svg-attrs-to-kebab-in-pages + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (camel-shape shape-id)}}}} + data' (cfm/migrate-data data migration-id) + attrs (get-in data' [:pages-index page-id :objects shape-id :svg-attrs])] + (t/is (= "evenodd" (:fill-rule attrs))) + (t/is (= "2" (:stroke-width attrs))) + (t/is (= "dotted" (:stroke-style attrs)) "penpot-extra-attrs survive") + (t/is (not (contains? attrs :data-foo)) "unknown attrs are dropped") + (t/is (not (contains? attrs :fillRule)) "no camel keys survive"))) + +(t/deftest migration-0037-converts-camel-svg-attrs-to-kebab-in-components + (let [shape-id (uuid/next) + component-id (uuid/next) + data {:components + {component-id + {:objects + {shape-id (camel-shape shape-id)}}}} + data' (cfm/migrate-data data migration-id) + attrs (get-in data' [:components component-id :objects shape-id :svg-attrs])] + (t/is (= "evenodd" (:fill-rule attrs))) + (t/is (not (contains? attrs :fillRule)) "no camel keys survive"))) + +(t/deftest migration-0037-normalizes-def-node-attrs + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :rect + :svg-defs (camel-defs)})}}}} + data' (cfm/migrate-data data migration-id) + grad (get-in data' [:pages-index page-id :objects shape-id :svg-defs "g1"])] + (t/is (= :linearGradient (:tag grad)) "tags are untouched") + (t/is (= "g1" (get-in grad [:attrs :id])) ":id survives") + (t/is (= "userSpaceOnUse" (get-in grad [:attrs :gradient-units]))) + (t/is (= "#base" (get-in grad [:attrs :xlink-href]))) + (t/is (not (contains? (:attrs grad) :data-x)) "unknown def attrs are dropped") + (t/is (= "#ffffff" (get-in grad [:content 0 :attrs :stop-color])) + "nested stops are normalized"))) + +(t/deftest migration-0037-kebabizes-content-spelling-only + (let [shape-id (uuid/next) + page-id (uuid/next) + raw-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :path + :content [{:marker "untouched"}]}) + raw-id (make-shape raw-id + {:type :svg-raw + :content {:tag :g + :attrs {:strokeWidth "2" + :data-foo "keep-me"} + :content [{:tag :path + :attrs {:strokeLinecap "round"} + :content []}]}})}}}} + data' (cfm/migrate-data data migration-id) + path (get-in data' [:pages-index page-id :objects shape-id]) + raw (get-in data' [:pages-index page-id :objects raw-id]) + content (:content raw)] + (t/is (= [{:marker "untouched"}] (:content path)) + "non-svg-raw content is untouched") + (t/is (= "2" (get-in content [:attrs :stroke-width]))) + (t/is (= "keep-me" (get-in content [:attrs :data-foo])) + "unknown content keys survive") + (t/is (= "round" (get-in content [:content 0 :attrs :stroke-linecap]))))) + +(defn- string-tag-defs + [] + {:a {:tag "mask" + :attrs {:id "a" :maskUnits "userSpaceOnUse"} + :content [{:tag "rect" + :attrs {:width "10" :height "10" :fill "#FFF"} + :content []}]}}) + +(defn- string-tag-content + [] + {:tag "g" + :attrs {} + :content [{:tag "text" + :attrs {:fontSize "12"} + :content ["Hi"]} + {:tag "style" + :attrs nil + :content [".a{fill:red}"]}]}) + +(t/deftest migration-0037-keywordizes-string-tags + (let [shape-id (uuid/next) + raw-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :rect + :svg-defs (string-tag-defs)}) + raw-id (make-shape raw-id + {:type :svg-raw + :content (string-tag-content)})}}}} + data' (cfm/migrate-data data migration-id) + mask (get-in data' [:pages-index page-id :objects shape-id :svg-defs :a]) + content (get-in data' [:pages-index page-id :objects raw-id :content])] + (t/is (= :mask (:tag mask))) + (t/is (= :rect (get-in mask [:content 0 :tag])) "nested def tags too") + (t/is (= "userSpaceOnUse" (get-in mask [:attrs :mask-units]))) + (t/is (= :g (:tag content))) + (t/is (= :text (get-in content [:content 0 :tag])) "nested content tags too") + (t/is (= ["Hi"] (get-in content [:content 0 :content]))) + (t/is (= {} (get-in content [:content 1 :attrs])) "nil attrs become an empty map") + (t/is (= data' (cfm/migrate-data data' migration-id)) "re-running is stable"))) + +(t/deftest migration-0037-string-tag-file-passes-file-validation + ;; `migrate-file` validates the whole file data after migrating: a file + ;; with string node tags or nil node attrs must come out valid, or it + ;; cannot be opened. + (let [rect-id (uuid/next) + raw-id (uuid/next) + file (ctf/make-file {:name "String svg tags"}) + page-id (first (get-in file [:data :pages])) + rect (-> (cts/setup-shape {:id rect-id :type :rect :name "rect"}) + (assoc :svg-defs (string-tag-defs))) + raw (-> (cts/setup-shape {:id raw-id :type :svg-raw :name "raw"}) + (assoc :content (string-tag-content))) + file (-> file + (assoc :migrations (disj cfm/available-migrations migration-id)) + (update-in [:data :pages-index page-id :objects] assoc rect-id rect raw-id raw) + (update-in [:data :pages-index page-id :objects uuid/zero :shapes] conj rect-id raw-id)) + file' (cfm/migrate-file file {})] + (t/is (= :mask (get-in file' [:data :pages-index page-id :objects rect-id :svg-defs :a :tag]))) + (t/is (= :g (get-in file' [:data :pages-index page-id :objects raw-id :content :tag]))))) + +(t/deftest migration-0037-drops-map-valued-svg-attrs + ;; Files exported by old Penpot versions hold maps where an svg attr + ;; value should be, such as a `:filter` that once read + ;; `url(#filter_2)`. The closed `:svg-attrs` schema admits scalars + ;; only, so `migrate-file` must drop them or the file cannot be + ;; imported or opened. The values below are copied from the + ;; "Avataaars" and "Tutorial for beginners" templates. + (let [shape-id (uuid/next) + file (ctf/make-file {:name "Map-valued svg attrs"}) + page-id (first (get-in file [:data :pages])) + shape (-> (cts/setup-shape {:id shape-id :type :rect :name "rect"}) + (assoc :svg-attrs {:fillRule "evenodd" + :filter {(keyword "{") "{ \"url(#filter_2) nil}\""} + :mask {(keyword "url(-filter0-d-16837-160)") nil} + :style {:fill "red" + :filter {:a "b"}}})) + file (-> file + (assoc :migrations (disj cfm/available-migrations migration-id)) + (update-in [:data :pages-index page-id :objects] assoc shape-id shape) + (update-in [:data :pages-index page-id :objects uuid/zero :shapes] conj shape-id)) + file' (cfm/migrate-file file {}) + attrs (get-in file' [:data :pages-index page-id :objects shape-id :svg-attrs])] + (t/is (= {:fill-rule "evenodd" :style {:fill "red"}} attrs) + "map values are dropped, scalars are kept"))) + +(t/deftest migration-0037-turns-plain-map-svg-transform-into-matrix + ;; The svg defs renderer multiplies `:svg-transform` as a matrix; a + ;; plain map there renders as `matrix(NaN, ...)`. + (let [plain-id (uuid/next) + matrix-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {plain-id (make-shape plain-id + {:type :path + :svg-transform {:a 1.0 :b 0.0 :c 0.0 :d 1.0 :e 10.0 :f 20.0}}) + matrix-id (make-shape matrix-id + {:type :path + :svg-transform (gmt/matrix 2 0 0 2 0 0)})}}}} + data' (cfm/migrate-data data migration-id) + plain' (get-in data' [:pages-index page-id :objects plain-id :svg-transform]) + matrix' (get-in data' [:pages-index page-id :objects matrix-id :svg-transform])] + (t/is (gmt/matrix? plain')) + (t/is (= (gmt/matrix 1 0 0 1 10 20) plain')) + (t/is (= (gmt/matrix 2 0 0 2 0 0) matrix') "matrices are left as they are") + (t/is (= data' (cfm/migrate-data data' migration-id)) "re-running is stable"))) + +(t/deftest migration-0037-leaves-kebab-files-equal + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :rect + :svg-attrs {:fill-rule "evenodd" + :stroke-width "2"} + :svg-defs {"g1" {:tag :linearGradient + :attrs {:id "g1" + :gradient-units "userSpaceOnUse"} + :content []}}})}}}} + data' (cfm/migrate-data data migration-id)] + (t/is (= data data') "kebab is a fixed point"))) + +(t/deftest migration-0037-is-idempotent + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (camel-shape shape-id)}}}} + data' (cfm/migrate-data data migration-id)] + (t/is (= data' (cfm/migrate-data data' migration-id))))) + +(t/deftest migration-0037-normalizes-flat-svg-viewbox-to-rect + ;; Carried over from the withdrawn 0021: legacy files hold + ;; `:svg-viewbox` as a plain map instead of a rect record. + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :rect + :svg-viewbox {:x 1 :y 2 :width 3 :height 4}})}}}} + data' (cfm/migrate-data data migration-id) + viewbox (get-in data' [:pages-index page-id :objects shape-id :svg-viewbox])] + (t/is (grc/rect? viewbox) "flat viewbox becomes a rect record") + (t/is (= 1 (:x viewbox))) + (t/is (= 2 (:y viewbox))) + (t/is (= 3 (:width viewbox))) + (t/is (= 4 (:height viewbox))) + (t/is (= data' (cfm/migrate-data data' migration-id)) + "re-running is stable"))) + +(t/deftest migration-0037-passes-non-maps-through + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :rect + :svg-attrs "junk" + :svg-defs "junk"})}}}} + data' (cfm/migrate-data data migration-id) + shape (get-in data' [:pages-index page-id :objects shape-id])] + (t/is (= "junk" (:svg-attrs shape)) "non-maps reach validation untouched") + (t/is (= "junk" (:svg-defs shape))))) + +(t/deftest migration-0037-junk-viewbox-becomes-default-rect + ;; Unlike `:svg-attrs`/`:svg-defs` (passed through to validation), + ;; `:svg-viewbox` goes through `make-rect`, which does not throw on + ;; junk: it falls back to the default rect. Inherited from 0021. + (let [shape-id (uuid/next) + page-id (uuid/next) + data {:pages-index + {page-id + {:objects + {shape-id (make-shape shape-id + {:type :rect + :svg-viewbox "junk"})}}}} + data' (cfm/migrate-data data migration-id) + viewbox (get-in data' [:pages-index page-id :objects shape-id :svg-viewbox])] + (t/is (grc/rect? viewbox) "junk viewbox becomes a rect, not an error") + (t/is (= 0 (:x viewbox))) + (t/is (= 0.01 (:width viewbox))))) + +(t/deftest migration-0037-reruns-for-files-that-recorded-withdrawn-0021 + ;; A file that recorded the withdrawn "0021-fix-shape-svg-attrs" (and + ;; everything else current) still needs migration: 0021 left + ;; `available-migrations`, which re-arms it for 0037. + (let [shape-id (uuid/next) + file (ctf/make-file {:name "Legacy camel svg attrs"}) + page-id (first (get-in file [:data :pages])) + shape (-> (cts/setup-shape {:id shape-id :type :rect}) + (assoc :svg-attrs {:fillRule "evenodd"})) + file (-> file + (assoc :migrations (-> cfm/available-migrations + (disj migration-id) + (conj "0021-fix-shape-svg-attrs"))) + (assoc-in [:data :pages-index page-id :objects shape-id] shape)) + file' (cfm/migrate-file file {})] + (t/is (cfm/need-migration? file) "new migration detected") + (t/is (not (cfm/need-migration? file')) "new migration recorded") + (t/is (contains? (:migrations file') migration-id) "migration id persisted") + (t/is (not (contains? cfm/available-migrations "0021-fix-shape-svg-attrs")) + "the withdrawn 0021 id stays out of available-migrations") + (t/is (= "evenodd" + (get-in file' [:data :pages-index page-id :objects shape-id :svg-attrs :fill-rule])) + "migration repaired file data before schema validation"))) + +(t/deftest migration-0037-reruns-for-files-that-recorded-withdrawn-0035 + ;; A file that recorded 0035 holds camelCase keys when a client that + ;; loaded it before 0035 ran saved them afterwards (#13107). 0035 also + ;; left `available-migrations`, so 0037 must run on it again. + (let [shape-id (uuid/next) + file (ctf/make-file {:name "Legacy camel svg attrs"}) + page-id (first (get-in file [:data :pages])) + shape (-> (cts/setup-shape {:id shape-id :type :rect}) + (assoc :svg-attrs {:fillRule "evenodd"})) + file (-> file + (assoc :migrations (-> cfm/available-migrations + (disj migration-id) + (conj "0035-fix-shape-svg-attrs-and-defs"))) + (assoc-in [:data :pages-index page-id :objects shape-id] shape)) + file' (cfm/migrate-file file {})] + (t/is (cfm/need-migration? file) "new migration detected") + (t/is (not (cfm/need-migration? file')) "new migration recorded") + (t/is (contains? (:migrations file') migration-id) "migration id persisted") + (t/is (not (contains? cfm/available-migrations "0035-fix-shape-svg-attrs-and-defs")) + "the withdrawn 0035 id stays out of available-migrations") + (t/is (= "evenodd" + (get-in file' [:data :pages-index page-id :objects shape-id :svg-attrs :fill-rule])) + "migration repaired file data before schema validation"))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index d3cdb08456..29c80c981b 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -26,6 +26,7 @@ [common-tests.files-migrations-0034-test] [common-tests.files-migrations-0035-test] [common-tests.files-migrations-0036-test] + [common-tests.files-migrations-0037-test] [common-tests.files-migrations-test] [common-tests.files.comp-processors-test] [common-tests.files.helpers-test] @@ -144,6 +145,7 @@ 'common-tests.files-migrations-0034-test 'common-tests.files-migrations-0035-test 'common-tests.files-migrations-0036-test + 'common-tests.files-migrations-0037-test 'common-tests.files-migrations-test 'common-tests.files.comp-processors-test 'common-tests.files.helpers-test diff --git a/frontend/test/frontend_tests/plugins/wait_for_save_test.cljs b/frontend/test/frontend_tests/plugins/wait_for_save_test.cljs index e214d301ad..c9951aa657 100644 --- a/frontend/test/frontend_tests/plugins/wait_for_save_test.cljs +++ b/frontend/test/frontend_tests/plugins/wait_for_save_test.cljs @@ -40,6 +40,14 @@ [outcome] (some-> (:error outcome) .-code)) +(defn- saves + "The `update-file` commands the transport received. Every flow of the + editor shares the mocked transport and a lingering one can still reach + it, so a stray call must neither pass for a save nor release a wait on + one." + [requests] + (filterv #(= :update-file (first %)) @requests)) + (t/deftest ^:async empty-persistence-resolves-after-intake (await (with-persistence @@ -50,7 +58,7 @@ (let [outcome (await (await-outcome (persist/wait-for-save 2000)))] (t/is (contains? outcome :ok) "empty buffer and queue resolve after intake") - (t/is (empty? @requests) + (t/is (empty? (saves requests)) "nothing to save sends no update-file"))))))))) (t/deftest ^:async context-wait-for-save-resolves @@ -133,7 +141,7 @@ (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) "commit is buffered")) (let [p (persist/wait-for-save 5000)] - (await (async/wait-for #(seq @requests) + (await (async/wait-for #(seq (saves requests)) "force-persist flushed the buffer")) (rx/error! response (ex-info "rejected" {:type :validation :code :invalid-data})) (let [outcome (await (await-outcome p))] @@ -189,7 +197,8 @@ (js/setTimeout (fn [] (ptk/emit! store (local-commit file-id))) 0) - (await (async/wait-for #(seq @requests) "force-persist sent the edit")) + (await (async/wait-for #(seq (saves requests)) + "force-persist sent the edit")) (rx/push! response {:revn 1}) (let [outcome (await (await-outcome p))] (t/is (contains? outcome :ok) @@ -206,7 +215,8 @@ (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) "commit is buffered")) (let [p (persist/wait-for-save 5000)] - (await (async/wait-for #(seq @requests) "force-persist flushed the buffer")) + (await (async/wait-for #(seq (saves requests)) + "force-persist flushed the buffer")) (rx/push! response {:revn 1}) (let [outcome (await (await-outcome p))] (t/is (contains? outcome :ok) @@ -255,7 +265,7 @@ (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) "commit is buffered")) (let [p (persist/wait-for-save 5000)] - (await (async/wait-for #(seq @requests) + (await (async/wait-for #(seq (saves requests)) "force-persist flushed the buffer")) (rx/push! response (thp/rejection (:id @commit))) (let [outcome (await (await-outcome p))] @@ -276,7 +286,7 @@ (await (async/wait-for #(seq (get-in @store [:persistence :buffer])) "commit is buffered")) (let [p (persist/wait-for-save 5000)] - (await (async/wait-for #(seq @requests) + (await (async/wait-for #(seq (saves requests)) "force-persist flushed the buffer")) ;; Rollback starts before settle can publish empty :saved; ;; never emit ::rolled-back (reload-delay case).