From aa35bc78c9ee52dfa4ef50a864c7b99ad7c96560 Mon Sep 17 00:00:00 2001 From: Pablo Alba Date: Tue, 17 Nov 2026 09:36:40 +0100 Subject: [PATCH] :bug: Fix duplicate-file crash on nil keys in deleted component objects (#13045) * :bug: Fix duplicate-file crash on nil keys in deleted component objects * :recycle: Withdraw the legacy-36 migration in favor of 0036 legacy-36 and 0036-clear-nil-keys-in-objects perform the same cleanup of nil keys in `:objects`. Migrations only clean up invalid data here, so they carry no order dependency between them and running both is wasteful. Remove the legacy-36 defmethod and its id from `available-migrations`, the same way 0021-fix-shape-svg-attrs was withdrawn in favor of 0035. Files that recorded legacy-36 re-migrate through 0036; their persisted `:migrations` lists keep the old id, and the playwright fixtures recording it stay untouched. Add a migrate-file test proving a file that recorded legacy-36 is re-armed for 0036 and passes schema validation after. Document the withdrawal pattern in the migration subtleties memory. AI-assisted-by: inkling-2-small-free --------- Co-authored-by: Andrey Antukh --- ...-change-validation-migration-subtleties.md | 3 +- common/src/app/common/files/migrations.cljc | 35 +++-- common/src/app/common/types/file.cljc | 20 ++- .../files_migrations_0036_test.cljc | 130 ++++++++++++++++++ common/test/common_tests/runner.cljc | 4 + .../file_load_component_objects_test.cljc | 107 ++++++++++++++ 6 files changed, 281 insertions(+), 18 deletions(-) create mode 100644 common/test/common_tests/files_migrations_0036_test.cljc create mode 100644 common/test/common_tests/types/file_load_component_objects_test.cljc diff --git a/.serena/memories/common/file-change-validation-migration-subtleties.md b/.serena/memories/common/file-change-validation-migration-subtleties.md index 75d9d80b4c..fab22a45fd 100644 --- a/.serena/memories/common/file-change-validation-migration-subtleties.md +++ b/.serena/memories/common/file-change-validation-migration-subtleties.md @@ -33,4 +33,5 @@ - `migrate-file` synthesizes legacy migration ids from old numeric versions when `:migrations` is absent, migrates legacy features, and records feature flags created through `cfeat/*new*`. - When a file had no previous `:migrations`, `migrate-file` marks all migrations as migrated in metadata so callers persist the complete migration set, not only transformations that changed data. - Closed schemas (`schema:stroke-attrs` and others) reject a key that was dropped from the declaration, so removing a declared attr needs a migration in the same change, for data already written with it. -- Rebase collision on a migration number: upstream keeps the id. Renumber the branch migration to the next free one, keep it last in `available-migrations`, and rename everything that spells the number — its test file and namespace (`files_migrations_00NN_test.cljc`), the test names, and comments in production code or memories that name it. \ No newline at end of file +- Rebase collision on a migration number: upstream keeps the id. Renumber the branch migration to the next free one, keep it last in `available-migrations`, and rename everything that spells the number — its test file and namespace (`files_migrations_00NN_test.cljc`), the test names, and comments in production code or memories that name it. +- Withdrawing a migration id: remove both the `defmethod` and the id from `available-migrations`; the replacement migration subsumes it and files that recorded the old id re-migrate through the replacement (persisted `:migrations` keep the old id via union — do not rewrite them, including playwright fixture JSONs). Idempotent/pure-cleanup migrations only; never withdraw one whose transform must not run again unless the replacement is safe for those files. Precedents: "0021-fix-shape-svg-attrs" -> "0035-fix-shape-svg-attrs-and-defs"; "legacy-36" -> "0036-clear-nil-keys-in-objects". diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 1d935272d8..4f9d142e8e 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -716,17 +716,6 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "legacy-36" - [data _] - (letfn [(update-container [container] - (d/update-when container :objects (fn [objects] - (if (contains? objects nil) - (dissoc objects nil) - objects))))] - (-> data - (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container)))) - (defmethod migrate-data "legacy-37" [data _] (d/without-nils data)) @@ -2402,6 +2391,26 @@ (update :pages-index d/update-vals migrate-container) (d/update-when :components d/update-vals migrate-container)))) +(defmethod migrate-data "0036-clear-nil-keys-in-objects" + [data _] + ;; Soft-delete can leave a nil key in `:objects` (gsh/move on a missing + ;; main, then index-by :id). That fails the file schema on duplicate-file + ;; (#12336). "legacy-36" did the same cleanup but its id was withdrawn + ;; from `available-migrations` below: files that recorded it migrate + ;; again through this one. + (letfn [(update-container [container] + (d/update-when container :objects + (fn [objects] + (if (contains? objects nil) + (do + (l/trc :hint "clear nil key in objects" + :container-id (:id container)) + (dissoc objects nil)) + objects))))] + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + (def available-migrations (into (d/ordered-set) ["legacy-2" @@ -2429,7 +2438,6 @@ "legacy-32" "legacy-33" "legacy-34" - "legacy-36" "legacy-37" "legacy-38" "legacy-39" @@ -2494,4 +2502,5 @@ "0033-normalize-main-instance-names" "0034-fix-path-content-offset" "0035-fix-dangling-children" - "0035-fix-shape-svg-attrs-and-defs"])) + "0035-fix-shape-svg-attrs-and-defs" + "0036-clear-nil-keys-in-objects"])) diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index 78fa9cf3b4..359cbd3d6d 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -615,16 +615,28 @@ ([file-data component] (load-component-objects file-data component (gpt/point 0 0))) ([file-data component delta] - (if (and component (empty? (:objects component))) ;; This operation may be called twice, e.g. in an idempotent change + ;; nil? not empty?: a missing-main soft-delete stores {}, which must + ;; still count as already loaded so render/library callers do not re-enter. + (if (and component (nil? (:objects component))) ;; This operation may be called twice, e.g. in an idempotent change (let [component-page (get-component-page file-data component) page-objects (:objects component-page) - objects (->> (cons (:main-instance-id component) - (cfh/get-children-ids page-objects (:main-instance-id component))) - (map #(get page-objects %)) + ids (into [] + (cons (:main-instance-id component) + (cfh/get-children-ids page-objects (:main-instance-id component)))) + ;; Skip ids already gone from the page. Calling gsh/move on nil + ;; builds {:selrect nil :points []} and index-by :id stores it + ;; under nil, which later fails duplicate-file schema checks. + shapes (into [] (keep #(get page-objects %)) ids) + objects (->> shapes ;; when it is an undo of a cut-paste, we need to undo the movement ;; of the shapes so we need to move them delta (map #(gsh/move % delta)) (d/index-by :id))] + (when (not= (count shapes) (count ids)) + (l/wrn :hint "skipping missing shapes when loading component objects" + :component-id (str (:id component)) + :main-instance-id (str (:main-instance-id component)) + :file-id (str (:id file-data)))) (assoc component :objects objects)) component))) diff --git a/common/test/common_tests/files_migrations_0036_test.cljc b/common/test/common_tests/files_migrations_0036_test.cljc new file mode 100644 index 0000000000..d9789ea072 --- /dev/null +++ b/common/test/common_tests/files_migrations_0036_test.cljc @@ -0,0 +1,130 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0036-test + (:require + [app.common.files.migrations :as cfm] + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; ====================================================================== +;; 0036-clear-nil-keys-in-objects +;; Soft-delete can leave a nil key in a deleted component's `:objects` +;; (`{nil {:selrect nil :points []}}`). That fails the file schema on +;; duplicate-file (#12336). It replaces `legacy-36` (id withdrawn from +;; `available-migrations`): files that recorded it migrate again +;; through this one. +;; ====================================================================== + +(def ^:private migration "0036-clear-nil-keys-in-objects") + +(def ^:private nil-placeholder + {:selrect nil :points []}) + +(defn- root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(t/deftest removes-nil-key-from-component-objects + (let [component-id (uuid/next) + data {:components + {component-id + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page (uuid/next) + :objects {nil nil-placeholder}}}} + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "nil key is removed from the deleted component :objects") + (t/is (= {} objects') + "only the nil placeholder was present, so objects is empty"))) + +(t/deftest removes-nil-key-from-page-objects + (let [page-id (uuid/next) + frame-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame" :shapes []}) + data {:pages-index + {page-id + {:id page-id + :name "Page" + :objects {uuid/zero (root [frame-id]) + frame-id frame + nil nil-placeholder}}}} + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "nil key is removed from the page :objects") + (t/is (some? (get objects' frame-id)) + "real shapes are kept"))) + +(t/deftest migrated-file-data-passes-schema + ;; Same failure mode duplicate-file hits: full schema check after migrate. + (let [file-id (uuid/next) + page-id (uuid/next) + component-id (uuid/next) + data (-> (ctf/make-file-data file-id page-id) + (assoc-in [:components component-id] + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page page-id + :objects {nil nil-placeholder}}))] + + (t/is (thrown? #?(:clj Exception :cljs js/Error) + (ctf/check-file-data data)) + "corrupt trash objects fail the file schema") + + (let [data' (cfm/migrate-data data migration)] + (t/is (not (contains? (get-in data' [:components component-id :objects]) nil))) + (t/is (= data' (ctf/check-file-data data')) + "after clearing nil keys the file data passes the schema")))) + +(t/deftest migration-0036-runs-through-file-migration + ;; A file that recorded the withdrawn "legacy-36" (and everything else + ;; current) still needs migration: legacy-36 left + ;; `available-migrations`, which re-arms it for 0036. + (let [component-id (uuid/next) + file (ctf/make-file {:name "Legacy nil objects"}) + page-id (first (get-in file [:data :pages])) + file (-> file + (assoc :migrations (-> cfm/available-migrations + (disj migration) + (conj "legacy-36"))) + (assoc-in [:data :components component-id] + {:id component-id + :name "Trashed" + :path "" + :deleted true + :main-instance-id (uuid/next) + :main-instance-page page-id + :objects {nil nil-placeholder}})) + file' (cfm/migrate-file file {})] + + (t/is (cfm/need-migration? file) + "the withdrawn legacy-36 re-arms the cleanup for this file") + (t/is (not (cfm/need-migration? file')) + "new migration recorded") + (t/is (contains? (:migrations file') migration) + "0036 migration id persisted") + (t/is (contains? (:migrations file') "legacy-36") + "the old id stays in the recorded list") + (t/is (not (contains? cfm/available-migrations "legacy-36")) + "the withdrawn legacy-36 id stays out of available-migrations") + (t/is (not (contains? (get-in file' [:data :components component-id :objects]) nil)) + "nil key is cleared through the full migrate-file path"))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index df2662df08..d3cdb08456 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -25,6 +25,7 @@ [common-tests.files-migrations-0033-test] [common-tests.files-migrations-0034-test] [common-tests.files-migrations-0035-test] + [common-tests.files-migrations-0036-test] [common-tests.files-migrations-test] [common-tests.files.comp-processors-test] [common-tests.files.helpers-test] @@ -104,6 +105,7 @@ [common-tests.types.color-test] [common-tests.types.components-test] [common-tests.types.container-test] + [common-tests.types.file-load-component-objects-test] [common-tests.types.fill-test] [common-tests.types.font-test] [common-tests.types.modifiers-test] @@ -141,6 +143,7 @@ 'common-tests.files-migrations-0033-test 'common-tests.files-migrations-0034-test 'common-tests.files-migrations-0035-test + 'common-tests.files-migrations-0036-test 'common-tests.files-migrations-test 'common-tests.files.comp-processors-test 'common-tests.files.helpers-test @@ -220,6 +223,7 @@ 'common-tests.types.color-test 'common-tests.types.components-test 'common-tests.types.container-test + 'common-tests.types.file-load-component-objects-test 'common-tests.types.fill-test 'common-tests.types.font-test 'common-tests.types.modifiers-test diff --git a/common/test/common_tests/types/file_load_component_objects_test.cljc b/common/test/common_tests/types/file_load_component_objects_test.cljc new file mode 100644 index 0000000000..a1c8bfb0d7 --- /dev/null +++ b/common/test/common_tests/types/file_load_component_objects_test.cljc @@ -0,0 +1,107 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.types.file-load-component-objects-test + (:require + [app.common.types.file :as ctf] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +;; Soft-delete snapshots the main instance into the component via +;; `load-component-objects`. When the main id is already missing from +;; the page, the old path called `gsh/move` on nil and indexed by `:id`, +;; storing `{nil {:selrect nil :points []}}` — which later breaks +;; duplicate-file schema validation (#12336). + +(defn- root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- file-data + [page-id component] + {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [])}}} + :components {(:id component) component}}) + +(t/deftest load-component-objects-skips-missing-main + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + component {:id component-id + :name "Orphan" + :path "" + :main-instance-id main-id + :main-instance-page page-id} + result (ctf/load-component-objects + (file-data page-id component) + component) + objects (:objects result)] + + (t/is (not (contains? objects nil)) + "missing main must not create a nil key in :objects") + (t/is (= {} objects) + "missing main yields an empty snapshot, not a placeholder shape"))) + +(t/deftest load-component-objects-keeps-present-main + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + main (cts/setup-shape {:id main-id + :type :frame + :name "Main" + :shapes []}) + component {:id component-id + :name "Present" + :path "" + :main-instance-id main-id + :main-instance-page page-id} + file-data {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [main-id]) + main-id main}}} + :components {component-id component}} + result (ctf/load-component-objects file-data component) + objects (:objects result)] + + (t/is (contains? objects main-id) + "present main is stored under its uuid") + (t/is (not (contains? objects nil)) + "present main never introduces a nil key") + (t/is (= main-id (get-in objects [main-id :id]))))) + +(t/deftest load-component-objects-treats-empty-objects-as-already-loaded + ;; A missing-main soft-delete stores `:objects {}`. Render/library paths + ;; call load-component-objects again on that component. The guard must + ;; treat `{}` as already loaded — otherwise it reloads from the page + ;; (and re-warns). Put the main on the page now so a mistaken re-entry + ;; would populate objects and fail this assertion. + (let [page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + main (cts/setup-shape {:id main-id + :type :frame + :name "Main" + :shapes []}) + component {:id component-id + :name "AlreadySnapshotted" + :path "" + :main-instance-id main-id + :main-instance-page page-id + :objects {}} + file-data {:pages-index {page-id {:id page-id + :name "Page" + :objects {uuid/zero (root [main-id]) + main-id main}}} + :components {component-id component}} + result (ctf/load-component-objects file-data component)] + + (t/is (= {} (:objects result)) + "empty :objects means already loaded; do not reload from the page")))