diff --git a/.serena/memories/common/file-change-validation-migration-subtleties.md b/.serena/memories/common/file-change-validation-migration-subtleties.md index 7333faadc3..a6b5121bf7 100644 --- a/.serena/memories/common/file-change-validation-migration-subtleties.md +++ b/.serena/memories/common/file-change-validation-migration-subtleties.md @@ -30,4 +30,5 @@ - Migrations are an ordered set mixing legacy version-derived ids and newer named ids. Keep append order stable; `migrate` applies the set difference between available migrations and file migrations. - `migrate-file` synthesizes legacy migration ids from old numeric versions when `:migrations` is absent, migrates legacy features, and records feature flags created through `cfeat/*new*`. - When a file had no previous `:migrations`, `migrate-file` marks all migrations as migrated in metadata so callers persist the complete migration set, not only transformations that changed data. -- Closed schemas (`schema:stroke-attrs` and others) reject a key that was dropped from the declaration, so removing a declared attr needs a migration in the same change, for data already written with it. \ No newline at end of file +- Closed schemas (`schema:stroke-attrs` and others) reject a key that was dropped from the declaration, so removing a declared attr needs a migration in the same change, for data already written with it. +- Withdrawing a migration id: remove both the `defmethod` and the id from `available-migrations`; the replacement migration subsumes it and files that recorded the old id re-migrate through the replacement (persisted `:migrations` keep the old id via union — do not rewrite them, including playwright fixture JSONs). Idempotent/pure-cleanup migrations only; never withdraw one whose transform must not run again unless the replacement is safe for those files. Precedent: "legacy-36" -> "0036-clear-nil-keys-in-objects". diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index c2eba53139..6e24e1dc04 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -714,17 +714,6 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "legacy-36" - [data _] - (letfn [(update-container [container] - (d/update-when container :objects (fn [objects] - (if (contains? objects nil) - (dissoc objects nil) - objects))))] - (-> data - (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container)))) - (defmethod migrate-data "legacy-37" [data _] (d/without-nils data)) @@ -2354,13 +2343,13 @@ (let [shapes' (filterv #(contains? objects %) shapes)] (when (not= shapes shapes') (l/trc :hint "fix dangling children" - :container container-id - :shape-id (:id shape) + :container (str container-id) + :shape-id (str (:id shape)) :removed (remove #(contains? objects %) shapes))) shapes')))) (fix-objects [container-id objects] - (update-vals objects (partial fix-shape container-id objects))) + (d/update-vals objects (partial fix-shape container-id objects))) (update-container [container] (d/update-when container :objects #(fix-objects (:id container) %)))] @@ -2369,6 +2358,26 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) +(defmethod migrate-data "0036-clear-nil-keys-in-objects" + [data _] + ;; Soft-delete can leave a nil key in `:objects` (gsh/move on a missing + ;; main, then index-by :id). That fails the file schema on duplicate-file + ;; (#12336). "legacy-36" did the same cleanup but its id was withdrawn + ;; from `available-migrations` below: files that recorded it migrate + ;; again through this one. + (letfn [(update-container [container] + (d/update-when container :objects + (fn [objects] + (if (contains? objects nil) + (do + (l/trc :hint "clear nil key in objects" + :container-id (:id container)) + (dissoc objects nil)) + objects))))] + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + (def available-migrations (into (d/ordered-set) ["legacy-2" @@ -2396,7 +2405,6 @@ "legacy-32" "legacy-33" "legacy-34" - "legacy-36" "legacy-37" "legacy-38" "legacy-39" @@ -2461,4 +2469,5 @@ "0032-fix-children-not-in-parent" "0033-normalize-main-instance-names" "0034-fix-path-content-offset" - "0035-fix-dangling-children"])) + "0035-fix-dangling-children" + "0036-clear-nil-keys-in-objects"])) diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index 444484b68f..8206b66088 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -616,16 +616,28 @@ ([file-data component] (load-component-objects file-data component (gpt/point 0 0))) ([file-data component delta] - (if (and component (empty? (:objects component))) ;; This operation may be called twice, e.g. in an idempotent change + ;; nil? not empty?: a missing-main soft-delete stores {}, which must + ;; still count as already loaded so render/library callers do not re-enter. + (if (and component (nil? (:objects component))) ;; This operation may be called twice, e.g. in an idempotent change (let [component-page (get-component-page file-data component) page-objects (:objects component-page) - objects (->> (cons (:main-instance-id component) - (cfh/get-children-ids page-objects (:main-instance-id component))) - (map #(get page-objects %)) + ids (into [] + (cons (:main-instance-id component) + (cfh/get-children-ids page-objects (:main-instance-id component)))) + ;; Skip ids already gone from the page. Calling gsh/move on nil + ;; builds {:selrect nil :points []} and index-by :id stores it + ;; under nil, which later fails duplicate-file schema checks. + shapes (into [] (keep #(get page-objects %)) ids) + objects (->> shapes ;; when it is an undo of a cut-paste, we need to undo the movement ;; of the shapes so we need to move them delta (map #(gsh/move % delta)) (d/index-by :id))] + (when (not= (count shapes) (count ids)) + (l/wrn :hint "skipping missing shapes when loading component objects" + :component-id (str (:id component)) + :main-instance-id (str (:main-instance-id component)) + :file-id (str (:id file-data)))) (assoc component :objects objects)) component))) diff --git a/common/test/common_tests/files_migrations_0036_test.cljc b/common/test/common_tests/files_migrations_0036_test.cljc new file mode 100644 index 0000000000..d9789ea072 --- /dev/null +++ b/common/test/common_tests/files_migrations_0036_test.cljc @@ -0,0 +1,130 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0036-test + (:require + [app.common.files.migrations :as cfm] + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; ====================================================================== +;; 0036-clear-nil-keys-in-objects +;; Soft-delete can leave a nil key in a deleted component's `:objects` +;; (`{nil {:selrect nil :points []}}`). That fails the file schema on +;; duplicate-file (#12336). It replaces `legacy-36` (id withdrawn from +;; `available-migrations`): files that recorded it migrate again +;; through this one. +;; ====================================================================== + +(def ^:private migration "0036-clear-nil-keys-in-objects") + +(def ^:private nil-placeholder + {:selrect nil :points []}) + +(defn- root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(t/deftest removes-nil-key-from-component-objects + (let [component-id (uuid/next) + data {:components + {component-id + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page (uuid/next) + :objects {nil nil-placeholder}}}} + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "nil key is removed from the deleted component :objects") + (t/is (= {} objects') + "only the nil placeholder was present, so objects is empty"))) + +(t/deftest removes-nil-key-from-page-objects + (let [page-id (uuid/next) + frame-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame" :shapes []}) + data {:pages-index + {page-id + {:id page-id + :name "Page" + :objects {uuid/zero (root [frame-id]) + frame-id frame + nil nil-placeholder}}}} + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "nil key is removed from the page :objects") + (t/is (some? (get objects' frame-id)) + "real shapes are kept"))) + +(t/deftest migrated-file-data-passes-schema + ;; Same failure mode duplicate-file hits: full schema check after migrate. + (let [file-id (uuid/next) + page-id (uuid/next) + component-id (uuid/next) + data (-> (ctf/make-file-data file-id page-id) + (assoc-in [:components component-id] + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page page-id + :objects {nil nil-placeholder}}))] + + (t/is (thrown? #?(:clj Exception :cljs js/Error) + (ctf/check-file-data data)) + "corrupt trash objects fail the file schema") + + (let [data' (cfm/migrate-data data migration)] + (t/is (not (contains? (get-in data' [:components component-id :objects]) nil))) + (t/is (= data' (ctf/check-file-data data')) + "after clearing nil keys the file data passes the schema")))) + +(t/deftest migration-0036-runs-through-file-migration + ;; A file that recorded the withdrawn "legacy-36" (and everything else + ;; current) still needs migration: legacy-36 left + ;; `available-migrations`, which re-arms it for 0036. + (let [component-id (uuid/next) + file (ctf/make-file {:name "Legacy nil objects"}) + page-id (first (get-in file [:data :pages])) + file (-> file + (assoc :migrations (-> cfm/available-migrations + (disj migration) + (conj "legacy-36"))) + (assoc-in [:data :components component-id] + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page page-id + :objects {nil nil-placeholder}})) + file' (cfm/migrate-file file {})] + + (t/is (cfm/need-migration? file) + "the withdrawn legacy-36 re-arms the cleanup for this file") + (t/is (not (cfm/need-migration? file')) + "new migration recorded") + (t/is (contains? (:migrations file') migration) + "0036 migration id persisted") + (t/is (contains? (:migrations file') "legacy-36") + "the old id stays in the recorded list") + (t/is (not (contains? cfm/available-migrations "legacy-36")) + "the withdrawn legacy-36 id stays out of available-migrations") + (t/is (not (contains? (get-in file' [:data :components component-id :objects]) nil)) + "nil key is cleared through the full migrate-file path"))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index c82551cfea..df2eadcd5f 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -25,6 +25,7 @@ [common-tests.files-migrations-0033-test] [common-tests.files-migrations-0034-test] [common-tests.files-migrations-0035-test] + [common-tests.files-migrations-0036-test] [common-tests.files-migrations-test] [common-tests.files.comp-processors-test] [common-tests.files.helpers-test] @@ -103,6 +104,7 @@ [common-tests.types.color-test] [common-tests.types.components-test] [common-tests.types.container-test] + [common-tests.types.file-load-component-objects-test] [common-tests.types.fill-test] [common-tests.types.font-test] [common-tests.types.modifiers-test] @@ -139,6 +141,7 @@ 'common-tests.files-migrations-0033-test 'common-tests.files-migrations-0034-test 'common-tests.files-migrations-0035-test + 'common-tests.files-migrations-0036-test 'common-tests.files-migrations-test 'common-tests.files.comp-processors-test 'common-tests.files.helpers-test @@ -217,6 +220,7 @@ 'common-tests.types.color-test 'common-tests.types.components-test 'common-tests.types.container-test + 'common-tests.types.file-load-component-objects-test 'common-tests.types.fill-test 'common-tests.types.font-test 'common-tests.types.modifiers-test diff --git a/common/test/common_tests/types/file_load_component_objects_test.cljc b/common/test/common_tests/types/file_load_component_objects_test.cljc new file mode 100644 index 0000000000..a1c8bfb0d7 --- /dev/null +++ b/common/test/common_tests/types/file_load_component_objects_test.cljc @@ -0,0 +1,107 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.types.file-load-component-objects-test + (:require + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; Soft-delete snapshots the main instance into the component via +;; `load-component-objects`. When the main id is already missing from +;; the page, the old path called `gsh/move` on nil and indexed by `:id`, +;; storing `{nil {:selrect nil :points []}}` — which later breaks +;; duplicate-file schema validation (#12336). + +(defn- root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- file-data + [page-id component] + {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [])}}} + :components {(:id component) component}}) + +(t/deftest load-component-objects-skips-missing-main + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + component {:id component-id + :name "Orphan" + :path "" + :main-instance-id main-id + :main-instance-page page-id} + result (ctf/load-component-objects + (file-data page-id component) + component) + objects (:objects result)] + + (t/is (not (contains? objects nil)) + "missing main must not create a nil key in :objects") + (t/is (= {} objects) + "missing main yields an empty snapshot, not a placeholder shape"))) + +(t/deftest load-component-objects-keeps-present-main + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + main (cts/setup-shape {:id main-id + :type :frame + :name "Main" + :shapes []}) + component {:id component-id + :name "Present" + :path "" + :main-instance-id main-id + :main-instance-page page-id} + file-data {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [main-id]) + main-id main}}} + :components {component-id component}} + result (ctf/load-component-objects file-data component) + objects (:objects result)] + + (t/is (contains? objects main-id) + "present main is stored under its uuid") + (t/is (not (contains? objects nil)) + "present main never introduces a nil key") + (t/is (= main-id (get-in objects [main-id :id]))))) + +(t/deftest load-component-objects-treats-empty-objects-as-already-loaded + ;; A missing-main soft-delete stores `:objects {}`. Render/library paths + ;; call load-component-objects again on that component. The guard must + ;; treat `{}` as already loaded — otherwise it reloads from the page + ;; (and re-warns). Put the main on the page now so a mistaken re-entry + ;; would populate objects and fail this assertion. + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + main (cts/setup-shape {:id main-id + :type :frame + :name "Main" + :shapes []}) + component {:id component-id + :name "AlreadySnapshotted" + :path "" + :main-instance-id main-id + :main-instance-page page-id + :objects {}} + file-data {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [main-id]) + main-id main}}} + :components {component-id component}} + result (ctf/load-component-objects file-data component)] + + (t/is (= {} (:objects result)) + "empty :objects means already loaded; do not reload from the page"))) diff --git a/tracebitmap.txt b/tracebitmap.txt deleted file mode 100644 index a14ad92425..0000000000 --- a/tracebitmap.txt +++ /dev/null @@ -1,28 +0,0 @@ -How to trace a bitmap to svg in multiple colors: - - -Dependencies: imagemagick, netpbm, potrace - -Generate the PPM format from PNG: - - convert download.png download.ppm - -Generate a mask file for each color of the file: - - ppmcolormask "#184d47" download.ppm > download-mask1.pbm - ppmcolormask "#66462C" download.ppm > download-mask2.pbm - -Then trace the bitmap (bitmap -> svg): - - potrace --flat -b svg download-mask1.pbm -o download1.svg - potrace --flat -b svg download-mask2.pbm -o download2.svg - -Extract a list of colors from an image: - - convert download.png +dither -colors 10 -unique-colors txt: - - -Articles: - -- http://confluence.concord.org/display/CCTR/Removing%20Antialiasing.html -- http://confluence.concord.org/display/CCTR/Tracing%20Color%20Raster%20Images.html