diff --git a/.serena/memories/clojure/logging.md b/.serena/memories/clojure/logging.md new file mode 100644 index 0000000000..56fba6a1b3 --- /dev/null +++ b/.serena/memories/clojure/logging.md @@ -0,0 +1,7 @@ +# Clojure Logging Conventions + +Structured logging through `app.common.logging` (`l/trc`/`l/dbg`/`l/info`/`l/wrn`/`l/error`/`l/fatal`): `:hint` carries the human-readable message, every other datum goes in its own keyword field (`:id`, `:container`, `:removed`, ...). + +- `:hint` values start lowercase: `"fix dangling children"`, not `"Fix dangling children"`. Uppercase-first hints exist as legacy (~57 occurrences, the densest in `backend/src`; also `l/trc` in migration `legacy-28` and `log/error` in `app.common.files.repair`) — do not copy them, and do not mass-rewrite them unless the surrounding code needs the change. +- A name that requires uppercase (an env var like `PENPOT_MEDIA_PROCESSING_SERVICE_URI`, an acronym like `MCP` or `LDAP`) stays uppercase inside an otherwise lowercase sentence. +- Repair work done by a file migration logs at `l/trc`, only when it changes something, and with the ids it touched: the house style of `migrations.cljc` (`legacy-28` precedes; the file has no `warn`/`info` lines). The shipped `log4j2.xml` pins the `app` logger to `info` (static file, `monitorInterval="60"`), so those trace lines appear only after the operator raises the level; the durable record of a repaired file is the migration id in its `:migrations`. diff --git a/.serena/memories/critical-info.md b/.serena/memories/critical-info.md index 43f678a643..ac15e674bd 100644 --- a/.serena/memories/critical-info.md +++ b/.serena/memories/critical-info.md @@ -6,6 +6,7 @@ You are working on the GitHub project `penpot/penpot`, a monorepo. - A section's top-level memory is `
/core`. When a section is relevant, read the core memory before focused memories. - Edits/stale refs/duplication cleanup: `mem:memory-maintenance`. - Cross-cutting testing principles, TDD workflow, and anti-patterns: `mem:testing`. +- Structured logging conventions (`:hint` casing and fields, log levels): `mem:clojure/logging` — read before adding or changing any log call. - Verified Clojure language behaviors that contradict common assumptions (e.g. `int?` covers `Long`; `integer?` is the general predicate): `mem:clojure/idioms` — read before assuming stdlib predicate semantics. - Read `mem:clojure/design-rules` (function shape, boundary checks, naming) before writing or modifying Clojure/ClojureScript code — not only when designing a solution or an API. diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 6a82e463ef..c2eba53139 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -2246,44 +2246,49 @@ ;; been added to enforce this from now on. ;; Main instances in variants are a special case because they need to match the variant name. (letfn [(update-shape [container shape-id] - (let [shape (ctn/get-shape container shape-id) + (let [shape (ctn/get-shape container shape-id)] + ;; A shape may reference a child that does not exist in the + ;; container (a dangling reference). There is nothing to update + ;; on it, and storing it would add a nil entry to the objects + ;; map, making the whole file invalid. + (if (nil? shape) + container + (let [shape (cond + ;; Variant container must have a normalized name + (ctk/is-variant-container? shape) + (let [name (:name shape) + clean-name (cpn/clean-path name) + clean-name (if (str/blank? clean-name) + "(empty)" ;; this may occur if the name contains only separators + clean-name)] + (if (= name clean-name) + shape + (assoc shape :name clean-name))) - shape (cond - ;; Variant container must have a normalized name - (ctk/is-variant-container? shape) - (let [name (:name shape) - clean-name (cpn/clean-path name) - clean-name (if (str/blank? clean-name) - "(empty)" ;; this may occur if the name contains only separators - clean-name)] - (if (= name clean-name) - shape - (assoc shape :name clean-name))) + ;; Main instance inside a variant must have the same name as the container + (ctk/main-instance? shape) + (if (ctk/is-variant? shape) + (let [variant-container (ctn/get-shape container (:parent-id shape))] + (if (or (= (:name shape) (:name variant-container)) + (nil? variant-container)) + shape + (assoc shape :name (:name variant-container)))) - ;; Main instance inside a variant must have the same name as the container - (ctk/main-instance? shape) - (if (ctk/is-variant? shape) - (let [variant-container (ctn/get-shape container (:parent-id shape))] - (if (or (= (:name shape) (:name variant-container)) - (nil? variant-container)) - shape - (assoc shape :name (:name variant-container)))) + ;; Other main instances must have the same name as the path+name of the component + (let [component (ctkl/get-component data (:component-id shape) true) + expected-name (when component + (cpn/merge-path-item (:path component) (:name component)))] + (if (or (= expected-name (:name shape)) + (nil? expected-name)) + shape + (assoc shape :name expected-name)))) - ;; Other main instances must have the same name as the path+name of the component - (let [component (ctkl/get-component data (:component-id shape) true) - expected-name (when component - (cpn/merge-path-item (:path component) (:name component)))] - (if (or (= expected-name (:name shape)) - (nil? expected-name)) - shape - (assoc shape :name expected-name)))) + ;; Any other shape does not need processing + :else shape)] - ;; Any other shape does not need processing - :else shape)] - - (reduce update-shape - (ctt/set-shape container shape) - (:shapes shape)))) + (reduce update-shape + (ctt/set-shape container shape) + (:shapes shape)))))) (update-container [container] @@ -2335,6 +2340,35 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container))))) +(defmethod migrate-data "0035-fix-dangling-children" + [data _] + ;; A shape may list a child id that does not exist in the objects map of + ;; the page or component that owns it (a dangling reference). Such a + ;; reference is invalid: any operation that walks the shape tree through + ;; the children lists trips on it, and the file cannot be migrated nor + ;; opened. Here we remove the dangling ids from the children lists, + ;; keeping every shape that really exists. + (letfn [(fix-shape [container-id objects shape] + (d/update-when shape :shapes + (fn [shapes] + (let [shapes' (filterv #(contains? objects %) shapes)] + (when (not= shapes shapes') + (l/trc :hint "fix dangling children" + :container container-id + :shape-id (:id shape) + :removed (remove #(contains? objects %) shapes))) + shapes')))) + + (fix-objects [container-id objects] + (update-vals objects (partial fix-shape container-id objects))) + + (update-container [container] + (d/update-when container :objects #(fix-objects (:id container) %)))] + + (-> data + (update :pages-index d/update-vals update-container) + (d/update-when :components d/update-vals update-container)))) + (def available-migrations (into (d/ordered-set) ["legacy-2" @@ -2426,4 +2460,5 @@ "0031-migrate-stroke-width-token-attr" "0032-fix-children-not-in-parent" "0033-normalize-main-instance-names" - "0034-fix-path-content-offset"])) + "0034-fix-path-content-offset" + "0035-fix-dangling-children"])) diff --git a/common/test/common_tests/files_migrations_0033_test.cljc b/common/test/common_tests/files_migrations_0033_test.cljc new file mode 100644 index 0000000000..6f56602b77 --- /dev/null +++ b/common/test/common_tests/files_migrations_0033_test.cljc @@ -0,0 +1,138 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0033-test + "Regression coverage for the shape tree traversal of the + `0033-normalize-main-instance-names` migration. + + The migration walks a container from its root down, following the + `:shapes` vectors. A file can hold a `:shapes` entry pointing to a + shape that is not in the `:objects` map (a dangling reference). The + traversal must skip it instead of storing a nil shape into the + `:objects` map, and must keep normalizing the names of the shapes + that do exist." + (:require + [app.common.files.migrations :as cfm] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +(def ^:private migration "0033-normalize-main-instance-names") + +(defn- make-root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- make-main-instance + "A frame declared as the main instance of a component." + [main-id file-id component-id name] + (-> (cts/setup-shape {:id main-id :type :frame :name name}) + (assoc :main-instance true + :component-id component-id + :component-file file-id))) + +(defn- page-data + "File data with a single page owning `objects`." + [page-id objects] + {:pages-index {page-id {:id page-id :objects objects}}}) + +(defn- component-data + "File data with a single component owning `objects`." + [component-id objects] + {:components {component-id {:id component-id :objects objects}}}) + +;; --- The guard: a missing child must not become a nil shape + +(t/deftest missing-child-of-a-page-does-not-create-a-nil-shape + (let [page-id (uuid/next) + child-id (uuid/next) + root (make-root [child-id]) + data (page-data page-id {uuid/zero root}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= 1 (count objects')) + "the objects map keeps only the shapes that exist") + (t/is (= root (get objects' uuid/zero)) + "the root is left untouched"))) + +(t/deftest missing-child-of-a-component-does-not-create-a-nil-shape + (let [component-id (uuid/next) + child-id (uuid/next) + root (make-root [child-id]) + data (component-data component-id {uuid/zero root}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= 1 (count objects')) + "the objects map keeps only the shapes that exist") + (t/is (= root (get objects' uuid/zero)) + "the component root is left untouched"))) + +(t/deftest missing-child-deep-in-the-tree-does-not-create-a-nil-shape + (let [page-id (uuid/next) + frame-id (uuid/next) + missing-id (uuid/next) + frame (-> (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + (assoc :shapes [missing-id])) + data (page-data page-id {uuid/zero (make-root [frame-id]) + frame-id frame}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= frame (get objects' frame-id)) + "the parent of the missing child is left untouched"))) + +;; --- The behavior: names keep being normalized next to a missing child + +(t/deftest main-instance-of-a-page-is-renamed-next-to-a-missing-child + (let [file-id (uuid/next) + page-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + missing-id (uuid/next) + main-shape (make-main-instance main-id file-id component-id "Group/Name") + data (assoc (page-data page-id {uuid/zero (make-root [missing-id main-id]) + main-id main-shape}) + :components {component-id {:id component-id + :name "Name" + :path "Group" + :main-instance-id main-id}}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= "Group / Name" (get-in objects' [main-id :name])) + "the main instance takes the path and name of its component"))) + +(t/deftest main-instance-of-a-component-is-renamed-next-to-a-missing-child + (let [file-id (uuid/next) + component-id (uuid/next) + main-id (uuid/next) + missing-id (uuid/next) + main-shape (make-main-instance main-id file-id component-id "Group/Name") + data (component-data component-id {uuid/zero (make-root [missing-id main-id]) + main-id main-shape}) + data (assoc-in data [:components component-id :name] "Name") + data (assoc-in data [:components component-id :path] "Group") + data (assoc-in data [:components component-id :main-instance-id] main-id) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (not (contains? objects' nil)) + "no nil key is added to the objects map") + (t/is (= "Group / Name" (get-in objects' [main-id :name])) + "the main instance takes the path and name of its component"))) diff --git a/common/test/common_tests/files_migrations_0035_test.cljc b/common/test/common_tests/files_migrations_0035_test.cljc new file mode 100644 index 0000000000..3680e8daf9 --- /dev/null +++ b/common/test/common_tests/files_migrations_0035_test.cljc @@ -0,0 +1,186 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.files-migrations-0035-test + "Coverage for `0035-fix-dangling-children`. + + A shape can list a child id that is not present in the `:objects` map + of the page or component owning it (a dangling reference). Any + traversal of the shape tree trips on it, and the file becomes + unopenable. The migration drops those ids and keeps every shape that + really exists." + (:require + [app.common.files.migrations :as cfm] + [app.common.types.file :as ctf] + [app.common.types.objects-map :as omap] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +(def ^:private migration "0035-fix-dangling-children") + +(defn- make-root + [children] + (cts/setup-shape {:id uuid/zero + :type :frame + :name "Root" + :shapes children})) + +(defn- make-frame + [frame-id child-ids] + (-> (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + (assoc :shapes child-ids))) + +(defn- page-data + [page-id objects] + {:pages-index {page-id {:id page-id :objects objects}}}) + +(defn- component-data + [component-id objects] + {:components {component-id {:id component-id :objects objects}}}) + +(t/deftest removes-the-dangling-children-of-a-page + (let [page-id (uuid/next) + frame-id (uuid/next) + child-id (uuid/next) + frame (make-frame frame-id [child-id]) + data (page-data page-id {uuid/zero (make-root [frame-id child-id]) + frame-id frame}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling id is removed from the root children list") + (t/is (= [] (get-in objects' [frame-id :shapes])) + "the dangling id is removed from the frame children list") + (t/is (= 2 (count objects')) + "every shape that exists is kept") + (t/is (some? (get objects' frame-id)) + "the frame itself is kept"))) + +(t/deftest removes-the-dangling-children-of-a-component + (let [component-id (uuid/next) + frame-id (uuid/next) + child-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + data (component-data component-id {uuid/zero (make-root [frame-id child-id]) + frame-id frame}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:components component-id :objects])] + + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling id is removed from the children list") + (t/is (= 2 (count objects')) + "every shape that exists is kept") + (t/is (= frame (get objects' frame-id)) + "the existing shape is left untouched"))) + +(t/deftest migration-works-with-an-objects-map + ;; The backend stores the page :objects as an ObjectsMap (the lazy + ;; representation enabled by the fdata/objects-map feature). It is an + ;; IPersistentMap that does not implement IFn on the JVM: a migration + ;; must consult it with contains?/get, never by calling it as a + ;; function. + (let [page-id (uuid/next) + frame-id (uuid/next) + child-id (uuid/next) + frame (make-frame frame-id [child-id]) + objects (omap/wrap {uuid/zero (make-root [frame-id child-id]) + frame-id frame}) + data (page-data page-id objects) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling id is removed with the ObjectsMap representation") + (t/is (= [] (get-in objects' [frame-id :shapes])) + "the frame children list is pruned the same way"))) + +(t/deftest keeps-the-order-and-the-shapes-of-a-valid-tree + (let [page-id (uuid/next) + frame-id (uuid/next) + rect-id (uuid/next) + frame (cts/setup-shape {:id frame-id :type :frame :name "Frame"}) + rect (cts/setup-shape {:id rect-id :type :rect :name "Rect"}) + data (page-data page-id {uuid/zero (make-root [frame-id rect-id]) + frame-id frame + rect-id rect}) + migrated (cfm/migrate-data data migration) + objects' (get-in migrated [:pages-index page-id :objects])] + + (t/is (= [frame-id rect-id] (get-in objects' [uuid/zero :shapes])) + "the children list is kept as it was") + (t/is (= data migrated) + "a tree without dangling references is not modified"))) + +(t/deftest leaves-shapes-with-empty-children-alone + (let [page-id (uuid/next) + frame (cts/setup-shape {:id (uuid/next) :type :frame :name "Frame"}) + data (page-data page-id {uuid/zero (make-root []) + (:id frame) frame}) + migrated (cfm/migrate-data data migration)] + + (t/is (= data migrated) + "empty children vectors are left alone"))) + +(t/deftest leaves-shapes-without-children-key-alone + (let [page-id (uuid/next) + rect-id (uuid/next) + rect (cts/setup-shape {:id rect-id :type :rect :name "Rect"}) + data (page-data page-id {uuid/zero (make-root []) + rect-id rect}) + migrated (cfm/migrate-data data migration)] + + (t/is (= data migrated) + "shapes without a children key are left alone") + (t/is (not (contains? (get-in migrated [:pages-index page-id :objects rect-id]) :shapes)) + "no children key is created on the shape"))) + +(t/deftest migration-is-idempotent + (let [page-id (uuid/next) + child-id (uuid/next) + data (page-data page-id {uuid/zero (make-root [child-id])}) + migrated (cfm/migrate-data data migration)] + + (t/is (= migrated (cfm/migrate-data migrated migration)) + "applying the migration twice does not change the result"))) + +(t/deftest file-with-dangling-children-passes-the-whole-migration + (let [file-id (uuid/next) + page-id (uuid/next) + frame-id (uuid/next) + missing-id (uuid/next) + frame (make-frame frame-id [missing-id]) + base (ctf/make-file-data file-id page-id) + data (-> base + (assoc-in [:pages-index page-id :objects uuid/zero] + (make-root [frame-id missing-id])) + (assoc-in [:pages-index page-id :objects frame-id] frame)) + ;; The file reached the state before 0033 ran, so 0033, 0034 and + ;; 0035 are all pending, exactly as in the reported failure. + file {:id file-id + :data data + :migrations (disj cfm/available-migrations + "0033-normalize-main-instance-names" + "0034-fix-path-content-offset" + migration)}] + + (t/is (cfm/need-migration? file) + "the file is scheduled for migration") + + (let [migrated (cfm/migrate-file file {}) + objects' (get-in migrated [:data :pages-index page-id :objects])] + + (t/is (contains? (:migrations migrated) migration) + "the migration is recorded as applied") + (t/is (not (contains? objects' nil)) + "the migrated data has no nil shape entry") + (t/is (= [frame-id] (get-in objects' [uuid/zero :shapes])) + "the dangling reference of the root is removed") + (t/is (= [] (get-in objects' [frame-id :shapes])) + "the dangling reference of the frame is removed") + (t/is (some? (get objects' frame-id)) + "the frame that exists is kept")))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index 5d24e4ea9f..c82551cfea 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -22,7 +22,9 @@ [common-tests.files-changes-test] [common-tests.files-migrations-0025-test] [common-tests.files-migrations-0026-test] + [common-tests.files-migrations-0033-test] [common-tests.files-migrations-0034-test] + [common-tests.files-migrations-0035-test] [common-tests.files-migrations-test] [common-tests.files.comp-processors-test] [common-tests.files.helpers-test] @@ -134,7 +136,9 @@ 'common-tests.files-changes-test 'common-tests.files-migrations-0025-test 'common-tests.files-migrations-0026-test + 'common-tests.files-migrations-0033-test 'common-tests.files-migrations-0034-test + 'common-tests.files-migrations-0035-test 'common-tests.files-migrations-test 'common-tests.files.comp-processors-test 'common-tests.files.helpers-test