From f104cc5477c44ee1b938d0c91f051af9403a0bc2 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 28 Jan 2024 00:34:30 +0100 Subject: [PATCH 01/45] :zap: Improve performance on creating component from graphic About 25% speed improvement on average on single file migration process --- common/src/app/common/data.cljc | 8 + common/src/app/common/files/helpers.cljc | 11 +- .../app/common/files/libraries_helpers.cljc | 67 ++++--- .../src/app/common/files/shapes_helpers.cljc | 165 ++++++++++-------- .../src/app/main/data/workspace/shapes.cljs | 12 +- 5 files changed, 148 insertions(+), 115 deletions(-) diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index 12e3f7762b..196307bc82 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -57,6 +57,14 @@ #?(:cljs (instance? lkm/LinkedMap o) :clj (instance? LinkedMap o))) +(defn vec2 + "Creates a optimized vector compatible type of length 2 backed + internally with MapEntry impl because it has faster access method + for its fields." + [o1 o2] + #?(:clj (clojure.lang.MapEntry. o1 o2) + :cljs (cljs.core/->MapEntry o1 o2 nil))) + #?(:clj (defmethod print-method clojure.lang.PersistentQueue [q, w] ;; Overload the printer for queues so they look like fish diff --git a/common/src/app/common/files/helpers.cljc b/common/src/app/common/files/helpers.cljc index 46b9ac66e8..ea53bc2d69 100644 --- a/common/src/app/common/files/helpers.cljc +++ b/common/src/app/common/files/helpers.cljc @@ -484,7 +484,7 @@ (letfn [(red-fn [cur-idx id] (let [[prev-idx _] (first cur-idx) prev-idx (or prev-idx 0) - cur-idx (conj cur-idx [(inc prev-idx) id])] + cur-idx (conj cur-idx (d/vec2 (inc prev-idx) id))] (rec-index cur-idx id))) (rec-index [cur-idx id] (let [object (get objects id)] @@ -509,10 +509,11 @@ (defn order-by-indexed-shapes [objects ids] - (->> (indexed-shapes objects) - (sort-by first) - (filter (comp (into #{} ids) second)) - (map second))) + (let [ids (if (set? ids) ids (set ids))] + (->> (indexed-shapes objects) + (filter (fn [o] (contains? ids (val o)))) + (sort-by key) + (map val)))) (defn get-index-replacement "Given a collection of shapes, calculate their positions diff --git a/common/src/app/common/files/libraries_helpers.cljc b/common/src/app/common/files/libraries_helpers.cljc index bd750dee52..8b7f34acae 100644 --- a/common/src/app/common/files/libraries_helpers.cljc +++ b/common/src/app/common/files/libraries_helpers.cljc @@ -6,6 +6,7 @@ (ns app.common.files.libraries-helpers (:require + [app.common.data :as d] [app.common.files.changes-builder :as pcb] [app.common.files.helpers :as cfh] [app.common.types.component :as ctk] @@ -37,41 +38,50 @@ use it as root. Otherwise, create a frame (v2) or group (v1) that contains all ids. Then, make a component with it, and link all shapes to their corresponding one in the component." [it shapes objects page-id file-id components-v2 prepare-create-group prepare-create-board] - (let [changes (pcb/empty-changes it page-id) - from-singe-frame? (and (= 1 (count shapes)) (-> shapes first cfh/frame-shape?)) + (let [changes (pcb/empty-changes it page-id) + shapes-count (count shapes) + first-shape (first shapes) + + from-singe-frame? + (and (= 1 shapes-count) + (cfh/frame-shape? first-shape)) + [root changes old-root-ids] - (if (and (= (count shapes) 1) - (or (and (= (:type (first shapes)) :group) (not components-v2)) - (= (:type (first shapes)) :frame)) - (not (ctk/instance-head? (first shapes)))) - - [(first shapes) + (if (and (= shapes-count 1) + (or (and (cfh/group-shape? first-shape) + (not components-v2)) + (cfh/frame-shape? first-shape)) + (not (ctk/instance-head? first-shape))) + [first-shape (-> (pcb/empty-changes it page-id) (pcb/with-objects objects)) - (:shapes (first shapes))] + (:shapes first-shape)] - (let [root-name (if (= 1 (count shapes)) - (:name (first shapes)) + (let [root-name (if (= 1 shapes-count) + (:name first-shape) "Component 1") - [root changes] (if-not components-v2 - (prepare-create-group it ; These functions needs to be passed as argument - objects ; to avoid a circular dependence - page-id - shapes - root-name - (not (ctk/instance-head? (first shapes)))) - (prepare-create-board changes - (uuid/next) - (:parent-id (first shapes)) - objects - (map :id shapes) - nil - root-name - true))] + shape-ids (into (d/ordered-set) (map :id) shapes) - [root changes (map :id shapes)])) + [root changes] + (if-not components-v2 + (prepare-create-group it ; These functions needs to be passed as argument + objects ; to avoid a circular dependence + page-id + shapes + root-name + (not (ctk/instance-head? first-shape))) + (prepare-create-board changes + (uuid/next) + (:parent-id first-shape) + objects + shape-ids + nil + root-name + true))] + + [root changes shape-ids])) changes (cond-> changes @@ -79,8 +89,7 @@ (pcb/update-shapes (:shapes root) (fn [shape] - (-> shape - (assoc :constraints-h :scale :constraints-v :scale))))) + (assoc shape :constraints-h :scale :constraints-v :scale)))) objects' (assoc objects (:id root) root) diff --git a/common/src/app/common/files/shapes_helpers.cljc b/common/src/app/common/files/shapes_helpers.cljc index 03e3e89c1c..f9f814186c 100644 --- a/common/src/app/common/files/shapes_helpers.cljc +++ b/common/src/app/common/files/shapes_helpers.cljc @@ -39,16 +39,17 @@ (defn prepare-move-shapes-into-frame [changes frame-id shapes objects] - (let [ordered-indexes (cfh/order-by-indexed-shapes objects shapes) - parent-id (get-in objects [frame-id :parent-id]) - ordered-indexes (->> ordered-indexes (remove #(= % parent-id))) - to-move-shapes (map (d/getf objects) ordered-indexes)] - (if (d/not-empty? to-move-shapes) + (let [parent-id (dm/get-in objects [frame-id :parent-id]) + shapes (remove #(= % parent-id) shapes) + to-move (->> shapes + (map (d/getf objects)) + (not-empty))] + (if to-move (-> changes (cond-> (not (ctl/any-layout? objects frame-id)) - (pcb/update-shapes ordered-indexes ctl/remove-layout-item-data)) - (pcb/update-shapes ordered-indexes #(cond-> % (cfh/frame-shape? %) (assoc :hide-in-viewer true))) - (pcb/change-parent frame-id to-move-shapes 0) + (pcb/update-shapes shapes ctl/remove-layout-item-data)) + (pcb/update-shapes shapes #(cond-> % (cfh/frame-shape? %) (assoc :hide-in-viewer true))) + (pcb/change-parent frame-id to-move 0) (cond-> (ctl/grid-layout? objects frame-id) (-> (pcb/update-shapes [frame-id] ctl/assign-cells {:with-objects? true}) (pcb/reorder-grid-children [frame-id])))) @@ -60,90 +61,102 @@ changes id parent-id objects selected index frame-name without-fill? nil)) ([changes id parent-id objects selected index frame-name without-fill? target-cell-id] - (let [selected-objs (map #(get objects %) selected) - new-index (or index - (cfh/get-index-replacement selected objects))] - (when (d/not-empty? selected) - (let [srect (gsh/shapes->rect selected-objs) - selected-id (first selected) + (when-let [selected-objs (->> selected + (map (d/getf objects)) + (not-empty))] - frame-id (dm/get-in objects [selected-id :frame-id]) - parent-id (or parent-id (dm/get-in objects [selected-id :parent-id])) - base-parent (get objects parent-id) + (let [;; We calculate here the ordered selection because it is used + ;; multiple times and this avoid the need of creating the index + ;; manytimes for single operation. + selected' (cfh/order-by-indexed-shapes objects selected) + new-index (or index + (->> (first selected') + (cfh/get-position-on-parent objects) + (inc))) - layout-props - (when (and (= 1 (count selected)) - (ctl/any-layout? base-parent)) - (let [shape (get objects selected-id)] - (select-keys shape ctl/layout-item-props))) + srect (gsh/shapes->rect selected-objs) + selected-id (first selected) + selected-obj (get objects selected-id) - target-cell-id - (if (and (nil? target-cell-id) - (ctl/grid-layout? objects parent-id)) - ;; Find the top-left grid cell of the selected elements - (let [ncols (count (:layout-grid-columns base-parent))] - (->> selected - (map #(ctl/get-cell-by-shape-id base-parent %)) - (apply min-key (fn [{:keys [row column]}] (+ (* ncols row) column))) - :id)) - target-cell-id) + frame-id (get selected-obj :frame-id) + parent-id (or parent-id (get selected-obj :parent-id)) + base-parent (get objects parent-id) - attrs {:type :frame - :x (:x srect) - :y (:y srect) - :width (:width srect) - :height (:height srect)} + layout-props + (when (and (= 1 (count selected)) + (ctl/any-layout? base-parent)) + (select-keys selected-obj ctl/layout-item-props)) - shape (cts/setup-shape - (cond-> attrs - (some? id) - (assoc :id id) + target-cell-id + (if (and (nil? target-cell-id) + (ctl/grid-layout? objects parent-id)) + ;; Find the top-left grid cell of the selected elements + (let [ncols (count (:layout-grid-columns base-parent))] + (->> selected + (map #(ctl/get-cell-by-shape-id base-parent %)) + (apply min-key (fn [{:keys [row column]}] (+ (* ncols row) column))) + :id)) + target-cell-id) - (some? frame-name) - (assoc :name frame-name) + attrs + {:type :frame + :x (:x srect) + :y (:y srect) + :width (:width srect) + :height (:height srect)} - :always - (assoc :frame-id frame-id - :parent-id parent-id - :shapes (into [] selected)) + shape + (cts/setup-shape + (cond-> attrs + (some? id) + (assoc :id id) - (some? layout-props) - (d/patch-object layout-props) + (some? frame-name) + (assoc :name frame-name) - (or (not= frame-id uuid/zero) without-fill?) - (assoc :fills [] :hide-in-viewer true))) + :always + (assoc :frame-id frame-id + :parent-id parent-id + :shapes (into [] selected)) - shape (with-meta shape {:index new-index}) + (some? layout-props) + (d/patch-object layout-props) - [shape changes] - (prepare-add-shape changes shape objects) + (or (not= frame-id uuid/zero) without-fill?) + (assoc :fills [] :hide-in-viewer true))) - changes - (prepare-move-shapes-into-frame changes (:id shape) selected objects) + shape + (with-meta shape {:index new-index}) - changes - (cond-> changes - (ctl/grid-layout? objects (:parent-id shape)) - (-> (pcb/update-shapes - [(:parent-id shape)] - (fn [parent objects] - ;; This restores the grid layout before adding and moving the shapes - ;; this is done because the add+move could have altered the layout and we - ;; want to do it after both operations are completed. Also here we could - ;; asign the new element to a target-cell - (-> parent - (assoc :layout-grid-cells (:layout-grid-cells base-parent)) - (assoc :layout-grid-rows (:layout-grid-rows base-parent)) - (assoc :layout-grid-columns (:layout-grid-columns base-parent)) + [shape changes] + (prepare-add-shape changes shape objects) - (cond-> (some? target-cell-id) - (assoc-in [:layout-grid-cells target-cell-id :shapes] [(:id shape)])) - (ctl/assign-cells objects))) - {:with-objects? true}) + changes + (prepare-move-shapes-into-frame changes (:id shape) selected' objects) - (pcb/reorder-grid-children [(:parent-id shape)])))] + changes + (cond-> changes + (ctl/grid-layout? objects (:parent-id shape)) + (-> (pcb/update-shapes + [(:parent-id shape)] + (fn [parent objects] + ;; This restores the grid layout before adding and moving the shapes + ;; this is done because the add+move could have altered the layout and we + ;; want to do it after both operations are completed. Also here we could + ;; asign the new element to a target-cell + (-> parent + (assoc :layout-grid-cells (:layout-grid-cells base-parent)) + (assoc :layout-grid-rows (:layout-grid-rows base-parent)) + (assoc :layout-grid-columns (:layout-grid-columns base-parent)) - [shape changes]))))) + (cond-> (some? target-cell-id) + (assoc-in [:layout-grid-cells target-cell-id :shapes] [(:id shape)])) + (ctl/assign-cells objects))) + {:with-objects? true}) + + (pcb/reorder-grid-children [(:parent-id shape)])))] + + [shape changes])))) (defn prepare-create-empty-artboard diff --git a/frontend/src/app/main/data/workspace/shapes.cljs b/frontend/src/app/main/data/workspace/shapes.cljs index 6e47b0a13f..5288239acb 100644 --- a/frontend/src/app/main/data/workspace/shapes.cljs +++ b/frontend/src/app/main/data/workspace/shapes.cljs @@ -72,13 +72,15 @@ (watch [it state _] (let [page-id (:current-page-id state) objects (wsh/lookup-page-objects state page-id) - shapes (->> shapes (remove #(dm/get-in objects [% :blocked]))) + shapes (->> shapes + (remove #(dm/get-in objects [% :blocked])) + (cfh/order-by-indexed-shapes objects)) + changes (-> (pcb/empty-changes it page-id) (pcb/with-objects objects)) - changes (cfsh/prepare-move-shapes-into-frame changes - frame-id - shapes - objects)] + + changes (cfsh/prepare-move-shapes-into-frame changes frame-id shapes objects)] + (if (some? changes) (rx/of (dch/commit-changes changes)) (rx/empty)))))) From 6d73685f3ac322bb2c1aa5a82a61233a62cf183b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 09:26:08 +0100 Subject: [PATCH 02/45] :zap: Optimize file validation process --- common/src/app/common/files/validate.cljc | 86 +++++++++++------------ 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/common/src/app/common/files/validate.cljc b/common/src/app/common/files/validate.cljc index 01373f93c8..a5ce2e1da5 100644 --- a/common/src/app/common/files/validate.cljc +++ b/common/src/app/common/files/validate.cljc @@ -98,7 +98,8 @@ (defn- check-geometry "Validate that the shape has valid coordinates, selrect and points." [shape file page] - (when (and (not (#{:path :bool} (:type shape))) + (when (and (not (or (cfh/path-shape? shape) + (cfh/bool-shape? shape))) (or (nil? (:x shape)) ; This may occur in root shape (uuid/zero) in old files (nil? (:y shape)) (nil? (:width shape)) @@ -112,61 +113,64 @@ (defn- check-parent-children "Validate parent and children exists, and the link is bidirectional." [shape file page] - (let [parent (ctst/get-shape page (:parent-id shape))] + (let [parent (ctst/get-shape page (:parent-id shape)) + shape-id (:id shape) + shapes (:shapes shape)] + (if (nil? parent) (report-error :parent-not-found (str/ffmt "Parent % not found" (:parent-id shape)) shape file page) (do (when-not (cfh/root? shape) - (when-not (some #{(:id shape)} (:shapes parent)) + (when-not (some #(= shape-id %) (:shapes parent)) (report-error :child-not-in-parent - (str/ffmt "Shape % not in parent's children list" (:id shape)) + (str/ffmt "Shape % not in parent's children list" shape-id) shape file page))) - (when-not (= (count (:shapes shape)) (count (distinct (:shapes shape)))) + (when-not (= (count shapes) (count (distinct shapes))) (report-error :duplicated-children - (str/ffmt "Shape % has duplicated children" (:id shape)) + (str/ffmt "Shape % has duplicated children" shape-id) shape file page)) - (doseq [child-id (:shapes shape)] + (doseq [child-id shapes] (let [child (ctst/get-shape page child-id)] (if (nil? child) (report-error :child-not-found - (str/ffmt "Child % not found in parent %" child-id (:id shape)) + (str/ffmt "Child % not found in parent %" child-id shape-id) shape file page - :parent-id (:id shape) + :parent-id shape-id :child-id child-id) - (when (not= (:parent-id child) (:id shape)) + (when (not= (:parent-id child) shape-id) (report-error :invalid-parent - (str/ffmt "Child % has invalid parent %" child-id (:id shape)) + (str/ffmt "Child % has invalid parent %" child-id shape-id) child file page - :parent-id (:id shape)))))))))) + :parent-id shape-id))))))))) (defn- check-frame "Validate that the frame-id shape exists and is indeed a frame. Also it must point to the parent shape (if this is a frame) or to the frame-id of the parent (if not)." - [shape file page] - (let [frame (ctst/get-shape page (:frame-id shape))] + [{:keys [frame-id] :as shape} file page] + (let [frame (ctst/get-shape page frame-id)] (if (nil? frame) (report-error :frame-not-found - (str/ffmt "Frame % not found" (:frame-id shape)) + (str/ffmt "Frame % not found" frame-id) shape file page) (if (not= (:type frame) :frame) (report-error :invalid-frame - (str/ffmt "Frame % is not actually a frame" (:frame-id shape)) + (str/ffmt "Frame % is not actually a frame" frame-id) shape file page) (let [parent (ctst/get-shape page (:parent-id shape))] (when (some? parent) (if (= (:type parent) :frame) - (when-not (= (:frame-id shape) (:id parent)) + (when-not (= frame-id (:id parent)) (report-error :invalid-frame (str/ffmt "Frame-id should point to parent %" (:id parent)) shape file page)) - (when-not (= (:frame-id shape) (:frame-id parent)) + (when-not (= frame-id (:frame-id parent)) (report-error :invalid-frame - (str/ffmt "Frame-id should point to parent frame %" (:frame-id parent)) + (str/ffmt "Frame-id should point to parent frame %" frame-id) shape file page))))))))) (defn- check-component-main-head @@ -289,8 +293,7 @@ (check-component-main-head shape file page libraries) (check-component-root shape file page) (check-component-not-ref shape file page) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :main-top))) + (run! #(check-shape % file page libraries :context :main-top) (:shapes shape))) (defn- check-shape-main-root-nested "Root shape of a nested main instance @@ -301,8 +304,7 @@ (check-component-main-head shape file page libraries) (check-component-not-root shape file page) (check-component-not-ref shape file page) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :main-nested))) + (run! #(check-shape % file page libraries :context :main-nested) (:shapes shape))) (defn- check-shape-copy-root-top "Root shape of a top copy instance @@ -314,8 +316,7 @@ (check-component-not-main-head shape file page libraries) (check-component-root shape file page) (check-component-ref shape file page libraries) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :copy-top))) + (run! #(check-shape % file page libraries :context :copy-top) (:shapes shape))) (defn- check-shape-copy-root-nested "Root shape of a nested copy instance @@ -326,8 +327,7 @@ (check-component-not-main-head shape file page libraries) (check-component-not-root shape file page) (check-component-ref shape file page libraries) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :copy-nested))) + (run! #(check-shape % file page libraries :context :copy-nested) (:shapes shape))) (defn- check-shape-main-not-root "Not-root shape of a main instance (not any attribute)" @@ -335,8 +335,7 @@ (check-component-not-main-not-head shape file page) (check-component-not-root shape file page) (check-component-not-ref shape file page) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :main-any))) + (run! #(check-shape % file page libraries :context :main-any) (:shapes shape))) (defn- check-shape-copy-not-root "Not-root shape of a copy instance :shape-ref" @@ -344,8 +343,7 @@ (check-component-not-main-not-head shape file page) (check-component-not-root shape file page) (check-component-ref shape file page libraries) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :copy-any))) + (run! #(check-shape % file page libraries :context :copy-any) (:shapes shape))) (defn- check-shape-not-component "Shape is not in a component or is a fostered children (not any @@ -354,8 +352,7 @@ (check-component-not-main-not-head shape file page) (check-component-not-root shape file page) (check-component-not-ref shape file page) - (doseq [child-id (:shapes shape)] - (check-shape child-id file page libraries :context :not-component))) + (run! #(check-shape % file page libraries :context :not-component) (:shapes shape))) (defn- check-shape "Validate referential integrity and semantic coherence of @@ -439,6 +436,11 @@ "Objects list cannot be nil" component file nil))) +(defn- get-orphan-shapes + [{:keys [objects] :as page}] + (let [xf (comp (map #(contains? objects (:parent-id %))) + (map :id))] + (into [] xf (vals objects)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; PUBLIC API: VALIDATION FUNCTIONS @@ -451,18 +453,14 @@ [{:keys [data features] :as file} libraries] (when (contains? features "components/v2") (binding [*errors* (volatile! [])] - (doseq [page (filter :id (ctpl/pages-seq data))] - (let [orphans (->> page - :objects - vals - (filter #(not (contains? (:objects page) (:parent-id %)))) - (map :id))] - (check-shape uuid/zero file page libraries) - (doseq [shape-id orphans] - (check-shape shape-id file page libraries)))) - (doseq [component (vals (:components data))] - (check-component component file)) + (doseq [page (filter :id (ctpl/pages-seq data))] + (check-shape uuid/zero file page libraries) + (->> (get-orphan-shapes page) + (run! #(check-shape % file page libraries)))) + + (->> (vals (:components data)) + (run! #(check-component % file))) (-> *errors* deref not-empty)))) From 0fd6cacd17fc499739305a52cc1dc2403000a2fd Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Fri, 26 Jan 2024 16:22:30 +0100 Subject: [PATCH 03/45] :bug: Fix parent not found, adding migration --- backend/src/app/features/components_v2.clj | 19 +------------------ common/src/app/common/files/defaults.cljc | 2 +- common/src/app/common/files/migrations.cljc | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 548255a5a4..4b388f3058 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -667,23 +667,7 @@ :component-root)))] (-> file-data (update :pages-index update-vals fix-container) - (d/update-when :components update-vals fix-container)))) - - fix-shape-nil-parent-id - (fn [file-data] - ;; Ensure that parent-id and frame-id are not nil - (letfn [(fix-container [container] - (d/update-when container :objects update-vals fix-shape)) - - (fix-shape [shape] - (let [frame-id (or (:frame-id shape) - uuid/zero) - parent-id (or (:parent-id shape) - frame-id)] - (assoc shape :frame-id frame-id - :parent-id parent-id)))] - (-> file-data - (update :pages-index update-vals fix-container))))] + (d/update-when :components update-vals fix-container))))] (-> file-data (fix-file-data) @@ -708,7 +692,6 @@ (fix-frame-ids) (fix-component-nil-objects) (fix-false-copies) - (fix-shape-nil-parent-id) (fix-copies-of-detached)))) ; <- Do not add fixes after this one ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/common/src/app/common/files/defaults.cljc b/common/src/app/common/files/defaults.cljc index e35914d73f..aae6cf1ef1 100644 --- a/common/src/app/common/files/defaults.cljc +++ b/common/src/app/common/files/defaults.cljc @@ -6,4 +6,4 @@ (ns app.common.files.defaults) -(def version 44) +(def version 45) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 025f1d8417..49e37e0b77 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -843,3 +843,18 @@ (-> data (update :pages-index update-vals update-container) (update :components update-vals update-container)))) + +(defmethod migrate 45 + [data] + (letfn [(fix-shape [shape] + (let [frame-id (or (:frame-id shape) + uuid/zero) + parent-id (or (:parent-id shape) + frame-id)] + (assoc shape :frame-id frame-id + :parent-id parent-id))) + + (update-container [container] + (d/update-when container :objects update-vals fix-shape))] + (-> data + (update :pages-index update-vals update-container)))) \ No newline at end of file From daf77ecc5f4c2704e8099cd4e86a6a2ad4bb9c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Fri, 26 Jan 2024 12:21:29 +0100 Subject: [PATCH 04/45] :bug: Enhande handling of detached shapes during migration fixes --- backend/src/app/features/components_v2.clj | 59 ++++++++++++++-------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 4b388f3058..36d9f3dba7 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -127,15 +127,35 @@ or that are the result of old bugs." [file-data libraries] (let [detached-ids (volatile! #{}) + detach-shape (fn [container shape] - ;; Detach a shape. If it's inside a component, add it to detached-ids. This list - ;; is used later to process any other copy that was referencing a detached copy. + ;; Detach a shape and make necessary adjustments. (let [is-component? (let [root-shape (ctst/get-shape container (:id container))] - (and (some? root-shape) (nil? (:parent-id root-shape))))] - (when is-component? - (vswap! detached-ids conj (:id shape))) - (ctk/detach-shape shape))) + (and (some? root-shape) (nil? (:parent-id root-shape)))) + parent (ctst/get-shape container (:parent-id shape)) + in-copy? (ctn/in-any-component? (:objects container) parent)] + + (letfn [(detach-recursive [container shape first?] + + ;; If the shape is inside a component, add it to detached-ids. This list is used + ;; later to process other copies that was referencing a detached nested copy. + (when is-component? + (vswap! detached-ids conj (:id shape))) + + ;; Detach the shape and all children until we find a subinstance. + (if (or first? in-copy? (not (ctk/instance-head? shape))) + (as-> container $ + (ctn/update-shape $ (:id shape) ctk/detach-shape) + (reduce #(detach-recursive %1 %2 false) + $ + (map (d/getf (:objects container)) (:shapes shape)))) + + ;; If this is a subinstance head and the initial shape whas not itself a + ;; nested copy, stop detaching and promote it to root. + (ctn/update-shape container (:id shape) #(assoc % :component-root true))))] + + (detach-recursive container shape true)))) fix-bad-children (fn [file-data] @@ -476,15 +496,16 @@ (fn [file-data] ;; Detach shapes that were inside a copy (have :shape-ref) but now they aren't. (letfn [(fix-container [container] - (d/update-when container :objects update-vals (partial fix-shape container))) + (reduce fix-shape container (ctn/shapes-seq container))) (fix-shape [container shape] - (let [parent (ctst/get-shape container (:parent-id shape))] + (let [shape (ctst/get-shape container (:id shape)) ; Get the possibly updated shape + parent (ctst/get-shape container (:parent-id shape))] (if (and (ctk/in-component-copy? shape) (not (ctk/instance-head? shape)) (not (ctk/in-component-copy? parent))) (detach-shape container shape) - shape)))] + container)))] (-> file-data (update :pages-index update-vals fix-container) @@ -523,11 +544,9 @@ (if (some? direct-shape-2) ;; If it exists, there is nothing else to do. container - ;; If not found, detach shape and all children (stopping if a nested instance is reached) - (let [children (ctn/get-children-in-instance (:objects container) (:id shape))] - (reduce #(ctn/update-shape %1 (:id %2) (partial detach-shape %1)) - container - children)))))))) + ;; If not found, detach shape and all children. + ;; container + (detach-shape container shape))))))) container))] (-> file-data @@ -539,14 +558,14 @@ ;; If the user has created a copy and then converted into a path or bool, ;; detach it because the synchronization will no longer work. (letfn [(fix-container [container] - (d/update-when container :objects update-vals (partial fix-shape container))) + (reduce fix-shape container (ctn/shapes-seq container))) (fix-shape [container shape] (if (and (ctk/instance-head? shape) (or (cfh/path-shape? shape) (cfh/bool-shape? shape))) (detach-shape container shape) - shape))] + container))] (-> file-data (update :pages-index update-vals fix-container) @@ -631,9 +650,8 @@ (fn [file-data] ;; Find component heads that are not main-instance but have not :shape-ref. ;; Also shapes that have :shape-ref but are not in a copy. - (letfn [(fix-container - [container] - (d/update-when container :objects update-vals (partial fix-shape container))) + (letfn [(fix-container [container] + (reduce fix-shape container (ctn/shapes-seq container))) (fix-shape [container shape] @@ -643,7 +661,8 @@ (and (ctk/in-component-copy? shape) (nil? (ctn/get-head-shape (:objects container) shape {:allow-main? true})))) (detach-shape container shape) - shape))] + container))] + (-> file-data (update :pages-index update-vals fix-container) (d/update-when :components update-vals fix-container)))) From e4e566240feffa797daa4471a863814844183dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Mon, 29 Jan 2024 09:54:13 +0100 Subject: [PATCH 05/45] :bug: Add fix for removing v2 remains in v1 files --- backend/src/app/features/components_v2.clj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 36d9f3dba7..ab0ac200dd 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -302,9 +302,15 @@ 0 gap))) + ;; Fix name if missing (nil? (:name shape)) (assoc :name (d/name (:type shape))) + ;; Remove v2 info from components that have been copied and pasted + ;; from a v2 file + (some? (:main-instance shape)) + (dissoc shape :main-instance) + ;; Fix broken fills (seq (:fills shape)) (update :fills (fn [fills] (filterv valid-fill? fills))) From 04be6b13bee577d68f84df112a4f696d111bc4ad Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 14:52:05 +0100 Subject: [PATCH 06/45] :bug: Fix invalid colors on file library on comp-v2 migration --- backend/src/app/features/components_v2.clj | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index ab0ac200dd..5285183079 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -105,7 +105,12 @@ ;; FILE PREPARATION BEFORE MIGRATION ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(def valid-color? (sm/lazy-validator ::ctc/recent-color)) +(def valid-recent-color? + (sm/lazy-validator ::ctc/recent-color)) + +(def valid-color? + (sm/lazy-validator ::ctc/color)) + (def valid-fill? (sm/lazy-validator ::cts/fill)) (def valid-stroke? (sm/lazy-validator ::cts/stroke)) (def valid-flow? (sm/lazy-validator ::ctp/flow)) @@ -223,9 +228,17 @@ ;; fix that issues. fix-file-data (fn [file-data] - (-> file-data - (d/update-when :colors dissoc nil) - (d/update-when :typographies dissoc nil))) + (letfn [(fix-colors-library [colors] + (let [colors (dissoc colors nil)] + (reduce-kv (fn [colors id color] + (if (valid-color? color) + colors + (dissoc colors id))) + colors + colors)))] + (-> file-data + (d/update-when :colors fix-colors-library) + (d/update-when :typographies dissoc nil)))) delete-big-geometry-shapes (fn [file-data] @@ -416,7 +429,7 @@ ;; Remove invalid colors in :recent-colors (d/update-when file-data :recent-colors (fn [colors] - (filterv valid-color? colors)))) + (filterv valid-recent-color? colors)))) fix-broken-parents (fn [file-data] From 295d9568c8bc203bcdc97b0bf1333d0b19195712 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 15:25:27 +0100 Subject: [PATCH 07/45] :bug: Fix incompatibilities of old file migrations with new code --- common/src/app/common/files/migrations.cljc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 49e37e0b77..774a71717f 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -109,11 +109,14 @@ (assoc :points (grc/rect->points selrect)))))) (fix-empty-points [shape] - (let [shape (cond-> shape - (empty? (:selrect shape)) (cts/setup-rect))] - (cond-> shape - (empty? (:points shape)) - (assoc :points (grc/rect->points (:selrect shape)))))) + (if (empty? (:points shape)) + (-> shape + (update :selrect (fn [selrect] + (if (map? selrect) + (grc/make-rect selrect) + selrect))) + (cts/setup-shape)) + shape)) (update-object [object] (cond-> object @@ -857,4 +860,4 @@ (update-container [container] (d/update-when container :objects update-vals fix-shape))] (-> data - (update :pages-index update-vals update-container)))) \ No newline at end of file + (update :pages-index update-vals update-container)))) From 9ca7c4280c2bf4b2a82582755065e5eaf92950ae Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 15:38:44 +0100 Subject: [PATCH 08/45] :lipstick: Fix minor cosmetic issues on components-v2 feature ns --- backend/src/app/features/components_v2.clj | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 5285183079..2716673546 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -185,12 +185,10 @@ fix-missing-image-metadata (fn [file-data] ;; Delete broken image shapes with no metadata. - (letfn [(fix-container - [container] + (letfn [(fix-container [container] (d/update-when container :objects #(reduce-kv fix-shape % %))) - (fix-shape - [objects id shape] + (fix-shape [objects id shape] (if (and (cfh/image-shape? shape) (nil? (:metadata shape))) (-> objects @@ -435,12 +433,10 @@ (fn [file-data] ;; Find children shapes whose parent-id is not set to the parent that contains them. ;; Remove them from the parent :shapes list. - (letfn [(fix-container - [container] + (letfn [(fix-container [container] (d/update-when container :objects #(reduce-kv fix-shape % %))) - (fix-shape - [objects id shape] + (fix-shape [objects id shape] (reduce (fn [objects child-id] (let [child (get objects child-id)] (cond-> objects From e43380ad6198e3e7298a76d21c818491947cf23d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 15:39:04 +0100 Subject: [PATCH 09/45] :bug: Properly remove invalid text shapes on comp-v2 migration --- backend/src/app/features/components_v2.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 2716673546..e90e977934 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -284,7 +284,7 @@ (fn [shapes] (filterv #(not= id %) shapes))))) (and (cfh/text-shape? shape) - (not (seq (:content shape)))) + (not (valid-text-content? (:content shape)))) (dissoc objects id) :else From 5baa9e8fb6645faefa55f3cc661843332d1671bf Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Mon, 29 Jan 2024 16:18:13 +0100 Subject: [PATCH 10/45] :bug: Fix shape should not be component root migration error --- backend/src/app/features/components_v2.clj | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index e90e977934..9f1f381915 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -701,7 +701,25 @@ :component-root)))] (-> file-data (update :pages-index update-vals fix-container) - (d/update-when :components update-vals fix-container))))] + (d/update-when :components update-vals fix-container)))) + + fix-component-root-without-component + (fn [file-data] + ;; Ensure that parent-id and frame-id are not nil + (letfn [(fix-container [container] + (d/update-when container :objects update-vals fix-shape)) + + (fix-shape [shape] + (cond-> shape + (and (ctk/instance-root? shape) + (or (not (ctk/instance-head? shape)) + (not (some? (:component-file shape))))) + (dissoc shape + :component-id + :component-file + :component-root)))] + (-> file-data + (update :pages-index update-vals fix-container))))] (-> file-data (fix-file-data) @@ -726,6 +744,7 @@ (fix-frame-ids) (fix-component-nil-objects) (fix-false-copies) + (fix-component-root-without-component) (fix-copies-of-detached)))) ; <- Do not add fixes after this one ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; From 0ea623487ca7103cd3557ae8131374176b328116 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 18:31:55 +0100 Subject: [PATCH 11/45] :sparkles: Add better validation for point, matrix and rect types --- common/src/app/common/geom/matrix.cljc | 13 +++++++++---- common/src/app/common/geom/point.cljc | 18 ++++++++++------- common/src/app/common/geom/rect.cljc | 27 ++++++++++++++++++++++++++ common/src/app/common/types/shape.cljc | 23 ++-------------------- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/common/src/app/common/geom/matrix.cljc b/common/src/app/common/geom/matrix.cljc index 52efef50e1..d435d861cc 100644 --- a/common/src/app/common/geom/matrix.cljc +++ b/common/src/app/common/geom/matrix.cljc @@ -67,7 +67,8 @@ ([a b c d e f] (pos->Matrix a b c d e f))) -(def number-regex #"[+-]?\d*(\.\d+)?(e[+-]?\d+)?") +(def number-regex + #"[+-]?\d*(\.\d+)?([eE][+-]?\d+)?") (defn str->matrix [matrix-str] @@ -76,8 +77,8 @@ (map (comp d/parse-double first)))] (apply matrix params))) -(sm/def! ::matrix-map - [:map {:title "MatrixMap"} +(def ^:private schema:matrix-attrs + [:map {:title "MatrixAttrs"} [:a ::sm/safe-double] [:b ::sm/safe-double] [:c ::sm/safe-double] @@ -85,6 +86,10 @@ [:e ::sm/safe-double] [:f ::sm/safe-double]]) +(def valid-matrix? + (sm/lazy-validator + [:and [:fn matrix?] schema:matrix-attrs])) + (sm/def! ::matrix (letfn [(decode [o] (if (map? o) @@ -101,7 +106,7 @@ (dm/get-prop o :f) ","))] {:type ::matrix - :pred matrix? + :pred valid-matrix? :type-properties {:title "matrix" :description "Matrix instance" diff --git a/common/src/app/common/geom/point.cljc b/common/src/app/common/geom/point.cljc index fbe7e8e416..0a04fa7476 100644 --- a/common/src/app/common/geom/point.cljc +++ b/common/src/app/common/geom/point.cljc @@ -41,12 +41,6 @@ [v] (instance? Point v)) -(sm/def! ::point-map - [:map {:title "PointMap"} - [:x ::sm/safe-number] - [:y ::sm/safe-number]]) - - ;; FIXME: deprecated (s/def ::x ::us/safe-number) (s/def ::y ::us/safe-number) @@ -57,6 +51,16 @@ (s/def ::point (s/and ::point-attrs point?)) + +(def ^:private schema:point-attrs + [:map {:title "PointAttrs"} + [:x ::sm/safe-number] + [:y ::sm/safe-number]]) + +(def valid-point? + (sm/lazy-validator + [:and [:fn point?] schema:point-attrs])) + (sm/def! ::point (letfn [(decode [p] (if (map? p) @@ -71,7 +75,7 @@ (dm/get-prop p :y)))] {:type ::point - :pred point? + :pred valid-point? :type-properties {:title "point" :description "Point" diff --git a/common/src/app/common/geom/rect.cljc b/common/src/app/common/geom/rect.cljc index 445eb06cc0..ce01fb0cb1 100644 --- a/common/src/app/common/geom/rect.cljc +++ b/common/src/app/common/geom/rect.cljc @@ -12,6 +12,8 @@ [app.common.geom.point :as gpt] [app.common.math :as mth] [app.common.record :as rc] + [app.common.schema :as sm] + [app.common.schema.generators :as sg] [app.common.transit :as t])) (rc/defrecord Rect [x y width height x1 y1 x2 y2]) @@ -66,6 +68,31 @@ h (mth/max height 0.01)] (pos->Rect x y w h x y (+ x w) (+ y h)))))) +(def ^:private schema:rect-attrs + [:map {:title "RectAttrs"} + [:x ::sm/safe-number] + [:y ::sm/safe-number] + [:width ::sm/safe-number] + [:height ::sm/safe-number] + [:x1 ::sm/safe-number] + [:y1 ::sm/safe-number] + [:x2 ::sm/safe-number] + [:y2 ::sm/safe-number]]) + +(sm/define! ::rect + [:and + {:gen/gen (->> (sg/tuple (sg/small-double) + (sg/small-double) + (sg/small-double) + (sg/small-double)) + (sg/fmap #(apply make-rect %)))} + [:fn rect?] + schema:rect-attrs]) + +(def valid-rect? + (sm/lazy-validator + [:and [:fn rect?] schema:rect-attrs])) + (def empty-rect (make-rect 0 0 0.01 0.01)) diff --git a/common/src/app/common/types/shape.cljc b/common/src/app/common/types/shape.cljc index cdc7ccb163..46c2074b6b 100644 --- a/common/src/app/common/types/shape.cljc +++ b/common/src/app/common/types/shape.cljc @@ -79,25 +79,6 @@ (def text-align-types #{"left" "right" "center" "justify"}) -(sm/define! ::selrect - [:and - {:title "Selrect" - :gen/gen (->> (sg/tuple (sg/small-double) - (sg/small-double) - (sg/small-double) - (sg/small-double)) - (sg/fmap #(apply grc/make-rect %)))} - [:fn grc/rect?] - [:map - [:x ::sm/safe-number] - [:y ::sm/safe-number] - [:x1 ::sm/safe-number] - [:x2 ::sm/safe-number] - [:y1 ::sm/safe-number] - [:y2 ::sm/safe-number] - [:width ::sm/safe-number] - [:height ::sm/safe-number]]]) - (sm/define! ::points [:vector {:gen/max 4 :gen/min 4} ::gpt/point]) @@ -133,7 +114,7 @@ [:id ::sm/uuid] [:name :string] [:type [::sm/one-of shape-types]] - [:selrect ::selrect] + [:selrect ::grc/rect] [:points ::points] [:transform ::gmt/matrix] [:transform-inverse ::gmt/matrix] @@ -156,7 +137,7 @@ [:main-instance {:optional true} :boolean] [:remote-synced {:optional true} :boolean] [:shape-ref {:optional true} ::sm/uuid] - [:selrect {:optional true} ::selrect] + [:selrect {:optional true} ::grc/rect] [:points {:optional true} ::points] [:blocked {:optional true} :boolean] [:collapsed {:optional true} :boolean] From 3ea737deb1627371889cfcc34dec8f4e836885d1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 18:33:52 +0100 Subject: [PATCH 12/45] :bug: Remove paths that can't be repaired on comp-v2 migration --- backend/src/app/features/components_v2.clj | 26 ++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 9f1f381915..b1d42def05 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -127,6 +127,9 @@ (def valid-rgb-color-string? (sm/lazy-validator ::ctc/rgb-color)) +(def valid-shape-points? + (sm/lazy-validator ::cts/points)) + (defn- prepare-file-data "Apply some specific migrations or fixes to things that are allowed in v1 but not in v2, or that are the result of old bugs." @@ -287,6 +290,13 @@ (not (valid-text-content? (:content shape)))) (dissoc objects id) + (and (cfh/path-shape? shape) + (not (valid-path-content? (:content shape)))) + (-> objects + (dissoc id) + (d/update-in-when [(:parent-id shape) :shapes] + (fn [shapes] (filterv #(not= id %) shapes)))) + :else objects)) @@ -379,13 +389,15 @@ (and (cfh/path-shape? shape) (seq (:content shape)) (not (valid-path-content? (:content shape)))) - (let [shape (update shape :content fix-path-content) - [points selrect] (gshp/content->points+selrect shape (:content shape))] - (-> shape - (dissoc :bool-content) - (dissoc :bool-type) - (assoc :points points) - (assoc :selrect selrect))) + (let [shape (update shape :content fix-path-content)] + (if (not (valid-path-content? (:content shape))) + shape + (let [[points selrect] (gshp/content->points+selrect shape (:content shape))] + (-> shape + (dissoc :bool-content) + (dissoc :bool-type) + (assoc :points points) + (assoc :selrect selrect))))) ;; When we fount a bool shape with no content, ;; we convert it to a simple rect From 03eca0d9a29cb45161b20354d174e7c07c8ab11b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 18:34:56 +0100 Subject: [PATCH 13/45] :bug: Repair shape points if it is possible on comp-v2 migration --- backend/src/app/features/components_v2.clj | 43 +++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index b1d42def05..7bb70cb716 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -332,6 +332,47 @@ (some? (:main-instance shape)) (dissoc shape :main-instance) + (and (cfh/text-shape? shape) + (valid-text-content? (:content shape)) + (not (valid-shape-points? (:points shape))) + (seq (:position-data shape))) + (as-> shape + (let [selrect (->> (:position-data shape) + (map (juxt :x :y :width :height)) + (map #(apply grc/make-rect %)) + (grc/join-rects)) + points (grc/rect->points selrect)] + + (assoc shape + :x (:x selrect) + :y (:y selrect) + :width (:width selrect) + :height (:height selrect) + :selrect selrect + :points points))) + + (and (or (cfh/rect-shape? shape) + (cfh/svg-raw-shape? shape)) + (not (valid-shape-points? (:points shape))) + (grc/valid-rect? (:selrect shape))) + (as-> shape + (let [selrect (if (grc/valid-rect? (:svg-viewbox shape)) + (:svg-viewbox shape) + (:selrect shape)) + points (grc/rect->points selrect)] + (assoc shape + :x (:x selrect) + :y (:y selrect) + :width (:width selrect) + :height (:height selrect) + :selrect selrect + :points points))) + + (and (cfh/group-shape? shape) + (grc/valid-rect? (:selrect shape)) + (not (valid-shape-points? (:points shape)))) + (assoc :points (grc/rect->points (:selrect shape))) + ;; Fix broken fills (seq (:fills shape)) (update :fills (fn [fills] (filterv valid-fill? fills))) @@ -736,7 +777,6 @@ (-> file-data (fix-file-data) (fix-page-invalid-options) - (fix-completly-broken-shapes) (fix-bad-children) (fix-misc-shape-issues) (fix-recent-colors) @@ -744,6 +784,7 @@ (fix-text-shapes-converted-to-path) (fix-broken-paths) (delete-big-geometry-shapes) + (fix-completly-broken-shapes) (fix-broken-parents) (fix-orphan-shapes) (fix-orphan-copies) From c4ee88dc6627f5401025d2dd7bac90d2b2e32756 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 29 Jan 2024 23:47:54 +0100 Subject: [PATCH 14/45] :bug: Add fix for percent number on style attrs on parsing svg --- common/src/app/common/svg.cljc | 44 +++++++++++++------ common/src/app/common/svg/shapes_builder.cljc | 3 +- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/common/src/app/common/svg.cljc b/common/src/app/common/svg.cljc index 6a486f5768..1800a8a20a 100644 --- a/common/src/app/common/svg.cljc +++ b/common/src/app/common/svg.cljc @@ -895,9 +895,10 @@ (defn map-nodes [mapfn node] (let [update-content - (fn [content] (cond->> content - (vector? content) - (mapv (partial map-nodes mapfn))))] + (fn [content] + (cond->> content + (vector? content) + (mapv (partial map-nodes mapfn))))] (cond-> node (map? node) @@ -922,7 +923,8 @@ value))) (defn fix-default-values - "Gives values to some SVG elements which defaults won't work when imported into the platform" + "Gives values to some SVG elements which defaults won't work when + imported into the platform" [svg-data] (let [add-defaults (fn [{:keys [tag attrs] :as node}] @@ -984,29 +986,43 @@ (fix-percent-attrs-viewbox [attrs] (d/mapm fix-percent-attr-viewbox attrs)) - (fix-percent-attr-numeric [_ attr-val] - (let [is-percent? (str/ends-with? attr-val "%")] - (if is-percent? - (str (let [attr-num (d/parse-double (str/rtrim attr-val "%"))] - (/ attr-num 100))) - attr-val))) + (fix-percent-attr-numeric-val [val] + (let [val (d/parse-double (str/rtrim val "%"))] + (str (/ val 100)))) - (fix-percent-attrs-numeric [attrs] - (d/mapm fix-percent-attr-numeric attrs)) + (fix-percent-attr-numeric [attrs key val] + (cond + (= key :style) + (let [val (->> (str/split val ";") + (map (fn [val] + (if (str/ends-with? val "%") + (let [[k v] (str/split val ":" 2) + v (fix-percent-attr-numeric-val v)] + (str k ":" v)) + val))) + (str/join ";"))] + (assoc attrs key val)) + + (str/ends-with? val "%") + (assoc attrs key (fix-percent-attr-numeric-val val)) + + :else + attrs)) (fix-percent-values [node] (let [units (or (get-in node [:attrs :filterUnits]) (get-in node [:attrs :gradientUnits]) (get-in node [:attrs :patternUnits]) (get-in node [:attrs :clipUnits]))] + (cond-> node (or (= "objectBoundingBox" units) (nil? units)) - (update :attrs fix-percent-attrs-numeric) + (update :attrs #(reduce-kv fix-percent-attr-numeric % %)) (not= "objectBoundingBox" units) (update :attrs fix-percent-attrs-viewbox))))] - (->> svg-data (map-nodes fix-percent-values))))) + (map-nodes fix-percent-values svg-data)))) (defn collect-images [svg-data] (let [redfn (fn [acc {:keys [tag attrs]}] diff --git a/common/src/app/common/svg/shapes_builder.cljc b/common/src/app/common/svg/shapes_builder.cljc index a831d3c0ea..8dbf12999a 100644 --- a/common/src/app/common/svg/shapes_builder.cljc +++ b/common/src/app/common/svg/shapes_builder.cljc @@ -193,7 +193,8 @@ (defn create-group [name frame-id {:keys [x y width height offset-x offset-y] :as svg-data} {:keys [attrs]}] (let [transform (csvg/parse-transform (:transform attrs)) - attrs (-> (d/without-keys attrs csvg/inheritable-props) + attrs (-> attrs + (d/without-keys csvg/inheritable-props) (csvg/attrs->props)) vbox (grc/make-rect offset-x offset-y width height)] (cts/setup-shape From 036bf84ecd99751f0d7be034b4b240bc3ae96721 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Jan 2024 09:52:48 +0100 Subject: [PATCH 15/45] :bug: Set proper order on comp-v2 migration fixes --- backend/src/app/features/components_v2.clj | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 7bb70cb716..91449025ae 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -168,12 +168,10 @@ fix-bad-children (fn [file-data] ;; Remove any child that does not exist. And also remove duplicated children. - (letfn [(fix-container - [container] + (letfn [(fix-container [container] (d/update-when container :objects update-vals (partial fix-shape container))) - (fix-shape - [container shape] + (fix-shape [container shape] (let [objects (:objects container)] (d/update-when shape :shapes (fn [shapes] @@ -241,7 +239,7 @@ (d/update-when :colors fix-colors-library) (d/update-when :typographies dissoc nil)))) - delete-big-geometry-shapes + fix-big-geometry-shapes (fn [file-data] ;; At some point in time, we had a bug that generated shapes ;; with huge geometries that did not validate the @@ -777,14 +775,14 @@ (-> file-data (fix-file-data) (fix-page-invalid-options) - (fix-bad-children) (fix-misc-shape-issues) (fix-recent-colors) (fix-missing-image-metadata) (fix-text-shapes-converted-to-path) (fix-broken-paths) - (delete-big-geometry-shapes) + (fix-big-geometry-shapes) (fix-completly-broken-shapes) + (fix-bad-children) (fix-broken-parents) (fix-orphan-shapes) (fix-orphan-copies) From 26f4082b5fd04d7f07d814c6acb7def8dc3f1b36 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Jan 2024 13:22:56 +0100 Subject: [PATCH 16/45] :paperclip: Add debug helpers for jvm/tap --- backend/dev/user.clj | 11 +++++----- common/src/app/common/debug.clj | 36 +++++++++++++++++++++++++++++++ common/src/app/common/pprint.cljc | 19 +++++++++++++++- 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 common/src/app/common/debug.clj diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 5d0cb7e371..efe2c935a3 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -7,6 +7,7 @@ (ns user (:require [app.common.data :as d] + [app.common.debug :as debug] [app.common.exceptions :as ex] [app.common.files.helpers :as cfh] [app.common.fressian :as fres] @@ -55,8 +56,12 @@ [promesa.exec :as px])) (repl/disable-reload! (find-ns 'integrant.core)) +(repl/disable-reload! (find-ns 'app.common.debug)) + (set! *warn-on-reflection* true) +(add-tap #'debug/tap-handler) + ;; --- Benchmarking Tools (defmacro run-quick-bench @@ -132,12 +137,6 @@ ;; :v6 v6 ;; }]))) -(defonce debug-tap - (do - (add-tap #(locking debug-tap - (prn "tap debug:" %))) - 1)) - (defn calculate-frames [{:keys [data]}] diff --git a/common/src/app/common/debug.clj b/common/src/app/common/debug.clj new file mode 100644 index 0000000000..f23c498ed1 --- /dev/null +++ b/common/src/app/common/debug.clj @@ -0,0 +1,36 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.common.debug + (:require + [app.common.logging :as l] + [app.common.pprint :as pp])) + +(defn pprint + [expr] + (l/raw! :debug + (binding [*print-level* pp/default-level + *print-length* pp/default-length] + (with-out-str + (println "tap dbg:") + (pp/pprint expr {:max-width pp/default-width}))))) + + +(def store (atom {})) + +(defn get-stored + [] + (deref store)) + +(defn tap-handler + [v] + (if (and (vector? v) + (keyword (first v))) + (let [[command obj] v] + (case command + (:print :prn :pprint) (pprint obj) + :store (reset! store obj))) + (pprint v))) diff --git a/common/src/app/common/pprint.cljc b/common/src/app/common/pprint.cljc index 66925e7efc..e1c9ea39e5 100644 --- a/common/src/app/common/pprint.cljc +++ b/common/src/app/common/pprint.cljc @@ -9,9 +9,26 @@ (:require [me.flowthing.pp :as pp])) +(def default-level 8) +(def default-length 25) +(def default-width 120) + +#?(:clj + (defn set-defaults + [& {:keys [level width length]}] + (when length + (alter-var-root #'default-length (constantly length))) + (when width + (alter-var-root #'default-width (constantly width))) + (when level + (alter-var-root #'default-level (constantly level))) + nil)) + (defn pprint [expr & {:keys [width level length] - :or {width 120 level 8 length 25}}] + :or {width default-width + level default-level + length default-length}}] (binding [*print-level* level *print-length* length] (pp/pprint expr {:max-width width}))) From 7c0a63c7da11721a80096230bf75d97d8dbd644d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Jan 2024 13:30:48 +0100 Subject: [PATCH 17/45] :sparkles: Add minor improvements to comp-v2 migration srepl helper --- backend/src/app/srepl/components_v2.clj | 40 +++++++++++++++++-------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index 3db7746001..ffbd01d66e 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -188,17 +188,27 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn migrate-file! - [file-id & {:keys [rollback? validate? label] :or {rollback? true validate? false}}] + [file-id & {:keys [rollback? validate? label cache skip-on-graphic-error?] + :or {rollback? true + validate? false + skip-on-graphic-error? true}}] (l/dbg :hint "migrate:start" :rollback rollback?) - (let [tpoint (dt/tpoint) + (let [tpoint (dt/tpoint) file-id (if (string? file-id) (parse-uuid file-id) - file-id)] - (binding [feat/*stats* (atom {})] + file-id) + cache (if (int? cache) + (cache/create :executor (::wrk/executor main/system) + :max-items cache) + nil)] + + (binding [feat/*stats* (atom {}) + feat/*cache* cache] (try (-> (assoc main/system ::db/rollback rollback?) (feat/migrate-file! file-id :validate? validate? + :skip-on-graphic-error? skip-on-graphic-error? :label label)) (-> (deref feat/*stats*) @@ -212,22 +222,28 @@ (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed))))))) (defn migrate-team! - [team-id & {:keys [rollback? skip-on-graphic-error? validate? label] + [team-id & {:keys [rollback? skip-on-graphic-error? validate? label cache] :or {rollback? true validate? true - skip-on-graphic-error? false}}] + skip-on-graphic-error? true}}] (l/dbg :hint "migrate:start" :rollback rollback?) - (let [team-id (if (string? team-id) - (parse-uuid team-id) - team-id) - stats (atom {}) - tpoint (dt/tpoint)] + (let [team-id (if (string? team-id) + (parse-uuid team-id) + team-id) + stats (atom {}) + tpoint (dt/tpoint) + + cache (if (int? cache) + (cache/create :executor (::wrk/executor main/system) + :max-items cache) + nil)] (add-watch stats :progress-report (report-progress-files tpoint)) - (binding [feat/*stats* stats] + (binding [feat/*stats* stats + feat/*cache* cache] (try (-> (assoc main/system ::db/rollback rollback?) (feat/migrate-team! team-id From 2924791cb03e1aab4ba59d2b173fac590bef3ce5 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Tue, 30 Jan 2024 13:38:31 +0100 Subject: [PATCH 18/45] :bug: Fix non-root copy only allowed inside a copy migration error --- backend/src/app/features/components_v2.clj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 91449025ae..9cc15aaefa 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -746,10 +746,7 @@ (fix-shape [shape] (cond-> shape (@detached-ids (:shape-ref shape)) - (dissoc shape - :component-id - :component-file - :component-root)))] + (ctk/detach-shape)))] (-> file-data (update :pages-index update-vals fix-container) (d/update-when :components update-vals fix-container)))) From 8cc3669aac8612e4c4c04abaf5c58ea26db09194 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Tue, 30 Jan 2024 13:40:34 +0100 Subject: [PATCH 19/45] :sparkles: Improve validator syntax --- backend/src/app/features/components_v2.clj | 50 +++++++++++----------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 9cc15aaefa..49ede7ee80 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -328,7 +328,7 @@ ;; Remove v2 info from components that have been copied and pasted ;; from a v2 file (some? (:main-instance shape)) - (dissoc shape :main-instance) + (dissoc :main-instance) (and (cfh/text-shape? shape) (valid-text-content? (:content shape)) @@ -733,13 +733,31 @@ (update :pages-index update-vals fix-container) (d/update-when :components update-vals fix-container)))) + + fix-component-root-without-component + (fn [file-data] + ;; Ensure that if component-root is set component-file and component-id are set too + (letfn [(fix-container [container] + (d/update-when container :objects update-vals fix-shape)) + + (fix-shape [shape] + (cond-> shape + (and (ctk/instance-root? shape) + (or (not (ctk/instance-head? shape)) + (not (some? (:component-file shape))))) + (dissoc :component-id + :component-file + :component-root)))] + (-> file-data + (update :pages-index update-vals fix-container)))) + fix-copies-of-detached (fn [file-data] - ;; Find any copy that is referencing a shape inside a component that have - ;; been detached in a previous fix. If so, undo the nested copy, converting - ;; it into a direct copy. - ;; - ;; WARNING: THIS SHOULD BE CALLED AT THE END OF THE PROCESS. + ;; Find any copy that is referencing a shape inside a component that have + ;; been detached in a previous fix. If so, undo the nested copy, converting + ;; it into a direct copy. + ;; + ;; WARNING: THIS SHOULD BE CALLED AT THE END OF THE PROCESS. (letfn [(fix-container [container] (d/update-when container :objects update-vals fix-shape)) @@ -749,25 +767,7 @@ (ctk/detach-shape)))] (-> file-data (update :pages-index update-vals fix-container) - (d/update-when :components update-vals fix-container)))) - - fix-component-root-without-component - (fn [file-data] - ;; Ensure that parent-id and frame-id are not nil - (letfn [(fix-container [container] - (d/update-when container :objects update-vals fix-shape)) - - (fix-shape [shape] - (cond-> shape - (and (ctk/instance-root? shape) - (or (not (ctk/instance-head? shape)) - (not (some? (:component-file shape))))) - (dissoc shape - :component-id - :component-file - :component-root)))] - (-> file-data - (update :pages-index update-vals fix-container))))] + (d/update-when :components update-vals fix-container))))] (-> file-data (fix-file-data) From fc0a4fa5b7a6c927dee90714fb3d8f7e329913df Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Jan 2024 16:49:18 +0100 Subject: [PATCH 20/45] :bug: Ignore style attr on fix-percents function --- common/src/app/common/svg.cljc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/common/src/app/common/svg.cljc b/common/src/app/common/svg.cljc index 1800a8a20a..62aed04d3b 100644 --- a/common/src/app/common/svg.cljc +++ b/common/src/app/common/svg.cljc @@ -993,15 +993,7 @@ (fix-percent-attr-numeric [attrs key val] (cond (= key :style) - (let [val (->> (str/split val ";") - (map (fn [val] - (if (str/ends-with? val "%") - (let [[k v] (str/split val ":" 2) - v (fix-percent-attr-numeric-val v)] - (str k ":" v)) - val))) - (str/join ";"))] - (assoc attrs key val)) + attrs (str/ends-with? val "%") (assoc attrs key (fix-percent-attr-numeric-val val)) From a8ab883c0741f358b30cbe5ed8b4c0373826b238 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Jan 2024 17:49:07 +0100 Subject: [PATCH 21/45] :bug: Use correct default for `r` on parsing svg circle --- common/src/app/common/svg/shapes_builder.cljc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/src/app/common/svg/shapes_builder.cljc b/common/src/app/common/svg/shapes_builder.cljc index 8dbf12999a..1796d08a56 100644 --- a/common/src/app/common/svg/shapes_builder.cljc +++ b/common/src/app/common/svg/shapes_builder.cljc @@ -305,6 +305,8 @@ rx (d/nilv r rx) ry (d/nilv r ry) + rx (d/nilv rx 0) + ry (d/nilv ry 0) ;; There are some svg circles in the internet that does not ;; have cx and cy attrs, so we default them to 0 From db622cece8e1e9cc5d6a8af43fff1591b44388bd Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Jan 2024 17:49:49 +0100 Subject: [PATCH 22/45] :sparkles: Use better default for svgo on frontend code --- frontend/src/app/main/data/workspace/media.cljs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/frontend/src/app/main/data/workspace/media.cljs b/frontend/src/app/main/data/workspace/media.cljs index 58b8c7736b..c68dfe00a2 100644 --- a/frontend/src/app/main/data/workspace/media.cljs +++ b/frontend/src/app/main/data/workspace/media.cljs @@ -39,18 +39,7 @@ (def ^:private svgo-config {:multipass false - :plugins - [{:name "safePreset" - :params {:overrides - {:convertColors - {:names2hex true - :shorthex false - :shortname false} - :convertTransform - {:matrixToTransform false - :convertToShorts false - :transformPrecision 4 - :leadingZero false}}}}]}) + :plugins ["safeAndFastPreset"]}) (defn svg->clj [[name text]] From b6d53c9dede85c3e88964254805b3908d2af4b98 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 09:06:27 +0100 Subject: [PATCH 23/45] :paperclip: Update devenv logging config --- backend/resources/log4j2-devenv.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/resources/log4j2-devenv.xml b/backend/resources/log4j2-devenv.xml index ca1ab6739a..46c0ae1504 100644 --- a/backend/resources/log4j2-devenv.xml +++ b/backend/resources/log4j2-devenv.xml @@ -21,7 +21,7 @@ - + From 43c13ed432b20eb28b3b4d2385b7b4727fc0e508 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 09:06:51 +0100 Subject: [PATCH 24/45] :bug: Add additional fix to fix-percents function on svg parse --- common/src/app/common/svg.cljc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/src/app/common/svg.cljc b/common/src/app/common/svg.cljc index 62aed04d3b..d400f01f1a 100644 --- a/common/src/app/common/svg.cljc +++ b/common/src/app/common/svg.cljc @@ -995,6 +995,9 @@ (= key :style) attrs + (str/starts-with? (d/name key) "data-") + attrs + (str/ends-with? val "%") (assoc attrs key (fix-percent-attr-numeric-val val)) From db5946d1ab5a3bfd33c7b56ff545b27dddbc29ff Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 11:10:44 +0100 Subject: [PATCH 25/45] :bug: Fix broken points on image shapes in comp-v2 migration --- backend/src/app/features/components_v2.clj | 31 +++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 49ede7ee80..3da089f40e 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -111,9 +111,14 @@ (def valid-color? (sm/lazy-validator ::ctc/color)) -(def valid-fill? (sm/lazy-validator ::cts/fill)) -(def valid-stroke? (sm/lazy-validator ::cts/stroke)) -(def valid-flow? (sm/lazy-validator ::ctp/flow)) +(def valid-fill? + (sm/lazy-validator ::cts/fill)) + +(def valid-stroke? + (sm/lazy-validator ::cts/stroke)) + +(def valid-flow? + (sm/lazy-validator ::ctp/flow)) (def valid-text-content? (sm/lazy-validator ::ctsx/content)) @@ -130,6 +135,9 @@ (def valid-shape-points? (sm/lazy-validator ::cts/points)) +(def valid-image-attrs? + (sm/lazy-validator ::cts/image-attrs)) + (defn- prepare-file-data "Apply some specific migrations or fixes to things that are allowed in v1 but not in v2, or that are the result of old bugs." @@ -330,6 +338,23 @@ (some? (:main-instance shape)) (dissoc :main-instance) + (and (cfh/image-shape? shape) + (valid-image-attrs? shape) + (grc/valid-rect? (:selrect shape)) + (not (valid-shape-points? (:points shape)))) + (as-> shape + (let [selrect (:selrect shape) + metadata (:metadata shape) + selrect (grc/make-rect + (:x selrect) + (:y selrect) + (:width metadata) + (:height metadata)) + points (grc/rect->points selrect)] + (assoc shape + :selrect selrect + :points points))) + (and (cfh/text-shape? shape) (valid-text-content? (:content shape)) (not (valid-shape-points? (:points shape))) From 13ca5d1f1a5da46836d2fe082eb8b8bf0e8b08d4 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 12:18:13 +0100 Subject: [PATCH 26/45] :sparkles: Add minor improvements to process-files! srepl helper --- backend/src/app/srepl/helpers.clj | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index cd5c3f56ad..c83bc6ceed 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -258,8 +258,10 @@ max-jobs start-at on-file + validate? rollback?] :or {max-jobs 1 + validate? true rollback? true}}] (l/dbg :hint "process:start" @@ -282,10 +284,10 @@ file (binding [*system* system] (on-file file'))] - (when (and (some? file) - (not (identical? file file'))) + (when (and (some? file) (not (identical? file file'))) - (cfv/validate-file-schema! file) + (when validate? + (cfv/validate-file-schema! file)) (let [file (if (contains? (:features file) "fdata/objects-map") (feat.fdata/enable-objects-map file) @@ -300,6 +302,9 @@ (db/update! conn :file {:data (blob/encode (:data file)) + :deleted-at (:deleted-at file) + :created-at (:created-at file) + :modified-at (:modified-at file) :features (db/create-array conn "text" (:features file)) :revn (:revn file)} {:id file-id})))))) @@ -314,7 +319,6 @@ :file-id (str file-id) :elapsed elapsed)))))] - (try (db/tx-run! main/system (fn [{:keys [::db/conn] :as system}] From f97931647c47f6f496fa686178a7a2e4d2e11ad9 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 12:55:16 +0100 Subject: [PATCH 27/45] :sparkles: Add helper for mark for deletion invalid files --- backend/src/app/srepl/components_v2.clj | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index ffbd01d66e..ff915b1db5 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -8,6 +8,7 @@ (:require [app.common.data :as d] [app.common.logging :as l] + [app.common.types.shape :as cts] [app.common.uuid :as uuid] [app.db :as db] [app.features.components-v2 :as feat] @@ -17,6 +18,7 @@ [app.util.events :as events] [app.util.time :as dt] [app.worker :as-alias wrk] + [clojure.set :as set] [cuerdas.core :as str] [promesa.exec :as px] [promesa.exec.semaphore :as ps] @@ -398,3 +400,31 @@ (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; FILE PROCESS HELPERS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn delete-broken-files + [{:keys [id data] :as file}] + (let [xform (comp + (map val) + (map :objects) + (mapcat vals) + (map :type)) + stypes (-> (into #{} xform (:pages-index data)) + (set/difference cts/shape-types))] + + (cond + (seq stypes) + (do + (l/wrn :hint "found shapes with unknown types" :file-id (str id) :types stypes) + (assoc file :deleted-at (dt/now))) + + (-> data :options :components-v2 true?) + (do + (l/wrn :hint "found old components-v2 format" :file-id (str id)) + (assoc file :deleted-at (dt/now))) + + :else + file))) From 893a8992c3bbd6f22890c42fdf102ea74af27711 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 12:55:40 +0100 Subject: [PATCH 28/45] :sparkles: Add progress reporting on process-files! srepl helper --- backend/src/app/srepl/helpers.clj | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index c83bc6ceed..4d5ac73712 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -275,9 +275,9 @@ sjobs (ps/create :permits max-jobs) process-file - (fn [file-id tpoint] + (fn [file-id idx tpoint] (try - (l/trc :hint "process:file:start" :file-id (str file-id)) + (l/trc :hint "process:file:start" :file-id (str file-id) :index idx) (db/tx-run! (assoc main/system ::db/rollback rollback?) (fn [{:keys [::db/conn] :as system}] (let [file' (get-file* system file-id) @@ -311,12 +311,14 @@ (catch Throwable cause (l/wrn :hint "unexpected error on processing file (skiping)" :file-id (str file-id) + :index idx :cause cause)) (finally (ps/release! sjobs) (let [elapsed (dt/format-duration (tpoint))] (l/trc :hint "process:file:end" :file-id (str file-id) + :index idx :elapsed elapsed)))))] (try @@ -325,15 +327,18 @@ (db/exec! conn ["SET statement_timeout = 0"]) (db/exec! conn ["SET idle_in_transaction_session_timeout = 0"]) - (run! (fn [file-id] - (ps/acquire! sjobs) - (px/run! executor (partial process-file file-id (dt/tpoint)))) - (->> (db/cursor conn [sql:get-file-ids (or start-at (dt/now))]) - (take max-items) - (map :id))) - - ;; Close and await tasks - (pu/close! executor))) + (try + (reduce (fn [idx file-id] + (ps/acquire! sjobs) + (px/run! executor (partial process-file file-id idx (dt/tpoint))) + (inc idx)) + 1 + (->> (db/cursor conn [sql:get-file-ids (or start-at (dt/now))]) + (take max-items) + (map :id))) + (finally + ;; Close and await tasks + (pu/close! executor))))) (catch Throwable cause (l/dbg :hint "process:error" :cause cause)) From 733825edfacf10fc5823788fc83edf801caa2aab Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 12:56:17 +0100 Subject: [PATCH 29/45] :zap: Optimize d/mapm helper using reduce-kv --- common/src/app/common/data.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index 196307bc82..958a3b9b08 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -316,9 +316,12 @@ (defn mapm "Map over the values of a map" ([mfn] - (map (fn [[key val]] [key (mfn key val)]))) + (map (fn [[key val]] (vec2 key (mfn key val))))) ([mfn coll] - (into {} (mapm mfn) coll))) + (reduce-kv (fn [coll k v] + (assoc coll k (mfn k v))) + coll + coll))) (defn removev "Returns a vector of the items in coll for which (fn item) returns logical false" From 7968bffc38852053f051e35f0f28ab726761b855 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 12:56:37 +0100 Subject: [PATCH 30/45] :lipstick: Minor cosmetic change on pages-seq helper --- common/src/app/common/types/pages_list.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/app/common/types/pages_list.cljc b/common/src/app/common/types/pages_list.cljc index 1a1dbb5664..a5bbda63e6 100644 --- a/common/src/app/common/types/pages_list.cljc +++ b/common/src/app/common/types/pages_list.cljc @@ -34,7 +34,7 @@ (defn pages-seq [fdata] - (vals (:pages-index fdata))) + (-> fdata :pages-index vals seq)) (defn update-page [file-data page-id f] From 5e4e706033df89aa8f44dbdb99901b985f259681 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 13:05:19 +0100 Subject: [PATCH 31/45] :sparkles: Use proper executor for cache on comp-v2 migration --- backend/src/app/srepl/components_v2.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index ff915b1db5..0c37856831 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -304,7 +304,7 @@ sprocs (ps/create :permits max-procs) cache (if (int? cache) - (cache/create :executor executor + (cache/create :executor (::wrk/executor main/system) :max-items cache) nil) migrate-team From e216b1071642bbbf2b12ed800785a298ed162e81 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 16:38:52 +0100 Subject: [PATCH 32/45] :bug: Fix raw data export import on debug interface --- backend/src/app/http/debug.clj | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 3f5d6a7b79..fe1fddc401 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -100,11 +100,11 @@ (let [profile (profile/get-profile pool profile-id) project-id (:default-project-id profile)] - (db/run! pool (fn [{:keys [::db/conn]}] - (create-file conn {:id file-id - :name (str "Cloned file: " filename) - :project-id project-id - :profile-id profile-id}) + (db/run! pool (fn [{:keys [::db/conn] :as cfg}] + (create-file cfg {:id file-id + :name (str "Cloned file: " filename) + :project-id project-id + :profile-id profile-id}) (db/update! conn :file {:data data} {:id file-id}) @@ -141,11 +141,11 @@ {::rres/status 200 ::rres/body "OK UPDATED"}) - (db/run! pool (fn [{:keys [::db/conn]}] - (create-file conn {:id file-id - :name fname - :project-id project-id - :profile-id profile-id}) + (db/run! pool (fn [{:keys [::db/conn] :as cfg}] + (create-file cfg {:id file-id + :name fname + :project-id project-id + :profile-id profile-id}) (db/update! conn :file {:data data} {:id file-id}) From d4c91ae44ba792921c239c1e4f7130b372994b7c Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 16:39:14 +0100 Subject: [PATCH 33/45] :lipstick: Print file name on process-file! helper progress report --- backend/src/app/srepl/components_v2.clj | 10 ++++++++-- backend/src/app/srepl/helpers.clj | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index 0c37856831..873a7904ad 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -418,12 +418,18 @@ (cond (seq stypes) (do - (l/wrn :hint "found shapes with unknown types" :file-id (str id) :types stypes) + (l/wrn :hint "found shapes with unknown types" + :file-id (str id) + :file-name (:name file) + :types stypes) (assoc file :deleted-at (dt/now))) (-> data :options :components-v2 true?) (do - (l/wrn :hint "found old components-v2 format" :file-id (str id)) + (l/wrn :hint "found old components-v2 format" + :file-id (str id) + :file-name (:name file)) + (assoc file :deleted-at (dt/now))) :else diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 4d5ac73712..9f9ccc7cc1 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -261,6 +261,7 @@ validate? rollback?] :or {max-jobs 1 + max-items Long/MAX_VALUE validate? true rollback? true}}] @@ -332,7 +333,7 @@ (ps/acquire! sjobs) (px/run! executor (partial process-file file-id idx (dt/tpoint))) (inc idx)) - 1 + 0 (->> (db/cursor conn [sql:get-file-ids (or start-at (dt/now))]) (take max-items) (map :id))) From 09ba1c262bf18254182528cca10117e723c174aa Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 16:52:03 +0100 Subject: [PATCH 34/45] :bug: Add minor adjustment to migration 33 to accept and fix invalid root frames --- common/src/app/common/files/migrations.cljc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 774a71717f..dfce09424a 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -623,6 +623,10 @@ (-> object (assoc :parent-id uuid/zero) (assoc :frame-id uuid/zero) + ;; We explicitly dissoc them and let the shape-setup + ;; to regenerate it with valid values. + (dissoc :selrect) + (dissoc :points) (cts/setup-shape)) object)) From fb7d3676d2541b015fcd74b57795070da6822abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Wed, 31 Jan 2024 15:09:10 +0100 Subject: [PATCH 35/45] :bug: Fix components without id --- backend/src/app/features/components_v2.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 3da089f40e..f985ec5f96 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -602,6 +602,17 @@ (update :pages-index update-vals fix-container) (d/update-when :components update-vals fix-container)))) + fix-components-without-id + (fn [file-data] + ;; We have detected some components that have no :id attribute. + ;; Regenerate it from the components map. + (letfn [(fix-component [id component] + (if (some? (:id component)) + component + (assoc component :id id)))] + (-> file-data + (d/update-when :components #(d/mapm fix-component %))))) + remap-refs (fn [file-data] ;; Remap shape-refs so that they point to the near main. @@ -810,6 +821,7 @@ (fix-orphan-copies) (remove-nested-roots) (add-not-nested-roots) + (fix-components-without-id) (remap-refs) (fix-converted-copies) (transform-to-frames) From 8528de642f3330ec2377b21da23207dab89f0fe6 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 18:39:01 +0100 Subject: [PATCH 36/45] :bug: Remove `:thumbnail` prop from all shapes on comp-v2 migration --- common/src/app/common/files/defaults.cljc | 2 +- common/src/app/common/files/migrations.cljc | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/common/src/app/common/files/defaults.cljc b/common/src/app/common/files/defaults.cljc index aae6cf1ef1..61cd7f1188 100644 --- a/common/src/app/common/files/defaults.cljc +++ b/common/src/app/common/files/defaults.cljc @@ -6,4 +6,4 @@ (ns app.common.files.defaults) -(def version 45) +(def version 46) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index dfce09424a..ccb2560f53 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -865,3 +865,14 @@ (d/update-when container :objects update-vals fix-shape))] (-> data (update :pages-index update-vals update-container)))) + +(defmethod migrate 46 + [data] + (letfn [(update-object [object] + (dissoc object :thumbnail)) + + (update-container [container] + (d/update-when container :objects update-vals update-object))] + (-> data + (update :pages-index update-vals update-container) + (update :components update-vals update-container)))) From 3d462e382113efa5dfe4e9bce2871fc12deb4c84 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 18:39:55 +0100 Subject: [PATCH 37/45] :sparkles: Split geometry fixes form fix-misc fn on comp-v2 migration --- backend/src/app/features/components_v2.clj | 154 +++++++++++---------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index f985ec5f96..72e3fda923 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -313,21 +313,96 @@ (update :pages-index update-vals update-container) (update :components update-vals update-container)))) - fix-misc-shape-issues + fix-shape-geometry (fn [file-data] (letfn [(fix-container [container] (d/update-when container :objects update-vals fix-shape)) + (fix-shape [shape] + (cond + (and (cfh/image-shape? shape) + (valid-image-attrs? shape) + (grc/valid-rect? (:selrect shape)) + (not (valid-shape-points? (:points shape)))) + (let [selrect (:selrect shape) + metadata (:metadata shape) + selrect (grc/make-rect + (:x selrect) + (:y selrect) + (:width metadata) + (:height metadata)) + points (grc/rect->points selrect)] + (assoc shape + :selrect selrect + :points points)) + + (and (cfh/text-shape? shape) + (valid-text-content? (:content shape)) + (not (valid-shape-points? (:points shape))) + (seq (:position-data shape))) + (let [selrect (->> (:position-data shape) + (map (juxt :x :y :width :height)) + (map #(apply grc/make-rect %)) + (grc/join-rects)) + points (grc/rect->points selrect)] + + (assoc shape + :x (:x selrect) + :y (:y selrect) + :width (:width selrect) + :height (:height selrect) + :selrect selrect + :points points)) + + (and (or (cfh/rect-shape? shape) + (cfh/svg-raw-shape? shape)) + (not (valid-shape-points? (:points shape))) + (grc/valid-rect? (:selrect shape))) + (let [selrect (if (grc/valid-rect? (:svg-viewbox shape)) + (:svg-viewbox shape) + (:selrect shape)) + points (grc/rect->points selrect)] + (assoc shape + :x (:x selrect) + :y (:y selrect) + :width (:width selrect) + :height (:height selrect) + :selrect selrect + :points points)) + + (and (cfh/group-shape? shape) + (grc/valid-rect? (:selrect shape)) + (not (valid-shape-points? (:points shape)))) + (assoc shape :points (grc/rect->points (:selrect shape))) + + :else + shape))] + + (-> file-data + (update :pages-index update-vals fix-container) + (d/update-when :components update-vals fix-container)))) + + fix-misc-shape-issues + (fn [file-data] + (letfn [(fix-container [container] + (d/update-when container :objects update-vals fix-shape)) + + (fix-gap-value [gap] + (if (or (= gap ##Inf) + (= gap ##-Inf)) + 0 + gap)) + (fix-shape [shape] (cond-> shape ;; Some shapes has invalid gap value (contains? shape :layout-gap) - (d/update-in-when [:layout-gap :column-gap] - (fn [gap] - (if (or (= gap ##Inf) - (= gap ##-Inf)) - 0 - gap))) + (update :layout-gap (fn [layout-gap] + (if (number? layout-gap) + {:row-gap layout-gap :column-gap layout-gap} + (-> layout-gap + (d/update-when :column-gap fix-gap-value) + (d/update-when :row-gap fix-gap-value))))) ;; Fix name if missing (nil? (:name shape)) @@ -338,64 +413,6 @@ (some? (:main-instance shape)) (dissoc :main-instance) - (and (cfh/image-shape? shape) - (valid-image-attrs? shape) - (grc/valid-rect? (:selrect shape)) - (not (valid-shape-points? (:points shape)))) - (as-> shape - (let [selrect (:selrect shape) - metadata (:metadata shape) - selrect (grc/make-rect - (:x selrect) - (:y selrect) - (:width metadata) - (:height metadata)) - points (grc/rect->points selrect)] - (assoc shape - :selrect selrect - :points points))) - - (and (cfh/text-shape? shape) - (valid-text-content? (:content shape)) - (not (valid-shape-points? (:points shape))) - (seq (:position-data shape))) - (as-> shape - (let [selrect (->> (:position-data shape) - (map (juxt :x :y :width :height)) - (map #(apply grc/make-rect %)) - (grc/join-rects)) - points (grc/rect->points selrect)] - - (assoc shape - :x (:x selrect) - :y (:y selrect) - :width (:width selrect) - :height (:height selrect) - :selrect selrect - :points points))) - - (and (or (cfh/rect-shape? shape) - (cfh/svg-raw-shape? shape)) - (not (valid-shape-points? (:points shape))) - (grc/valid-rect? (:selrect shape))) - (as-> shape - (let [selrect (if (grc/valid-rect? (:svg-viewbox shape)) - (:svg-viewbox shape) - (:selrect shape)) - points (grc/rect->points selrect)] - (assoc shape - :x (:x selrect) - :y (:y selrect) - :width (:width selrect) - :height (:height selrect) - :selrect selrect - :points points))) - - (and (cfh/group-shape? shape) - (grc/valid-rect? (:selrect shape)) - (not (valid-shape-points? (:points shape)))) - (assoc :points (grc/rect->points (:selrect shape))) - ;; Fix broken fills (seq (:fills shape)) (update :fills (fn [fills] (filterv valid-fill? fills))) @@ -407,11 +424,7 @@ ;; Fix some broken layout related attrs, probably ;; of copypaste on flex layout betatest period (true? (:layout shape)) - (assoc :layout :flex) - - (number? (:layout-gap shape)) - (as-> shape (let [n (:layout-gap shape)] - (assoc shape :layout-gap {:row-gap n :column-gap n})))))] + (assoc :layout :flex)))] (-> file-data (update :pages-index update-vals fix-container) @@ -814,6 +827,7 @@ (fix-text-shapes-converted-to-path) (fix-broken-paths) (fix-big-geometry-shapes) + (fix-shape-geometry) (fix-completly-broken-shapes) (fix-bad-children) (fix-broken-parents) From a8cf072bda87cc231500541948c4de11a21d5976 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 18:49:21 +0100 Subject: [PATCH 38/45] :sparkles: Add proper error report on binfile/v1 exportation --- backend/src/app/rpc/commands/binfile.clj | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 2621cce6a2..8f2216e636 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -8,6 +8,7 @@ (:refer-clojure :exclude [assert]) (:require [app.binfile.v1 :as bf.v1] + [app.common.logging :as l] [app.common.schema :as sm] [app.db :as db] [app.http.sse :as sse] @@ -50,11 +51,16 @@ ::rres/headers {"content-type" "application/octet-stream"} ::rres/body (reify rres/StreamableResponseBody (-write-body-to-stream [_ _ output-stream] - (-> cfg - (assoc ::bf.v1/ids #{file-id}) - (assoc ::bf.v1/embed-assets embed-assets) - (assoc ::bf.v1/include-libraries include-libraries) - (bf.v1/export-files! output-stream))))})) + (try + (-> cfg + (assoc ::bf.v1/ids #{file-id}) + (assoc ::bf.v1/embed-assets embed-assets) + (assoc ::bf.v1/include-libraries include-libraries) + (bf.v1/export-files! output-stream)) + (catch Throwable cause + (l/err :hint "exception on exporting file" + :file-id (str file-id) + :cause cause)))))})) ;; --- Command: import-binfile From e8c35c2de6a903cd42d34fbbe6d5ea99dea541af Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 18:56:06 +0100 Subject: [PATCH 39/45] :bug: Fix invalid shape type `:icon` on comp-v2 migration --- backend/src/app/features/components_v2.clj | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 72e3fda923..9036d105ff 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -370,6 +370,16 @@ :selrect selrect :points points)) + (and (= :icon (:type shape)) + (grc/valid-rect? (:selrect shape)) + (valid-shape-points? (:points shape))) + (-> shape + (assoc :type :rect) + (dissoc :content) + (dissoc :metadata) + (dissoc :segments) + (dissoc :x1 :y1 :x2 :y2)) + (and (cfh/group-shape? shape) (grc/valid-rect? (:selrect shape)) (not (valid-shape-points? (:points shape)))) From 3a401f69fd81beb39e26cf67de862c04e552fb4c Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 31 Jan 2024 18:59:45 +0100 Subject: [PATCH 40/45] :sparkles: Simplify srepl helper for shape deletion --- backend/src/app/srepl/components_v2.clj | 36 +++++-------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index 873a7904ad..4adf552939 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -8,7 +8,6 @@ (:require [app.common.data :as d] [app.common.logging :as l] - [app.common.types.shape :as cts] [app.common.uuid :as uuid] [app.db :as db] [app.features.components-v2 :as feat] @@ -18,7 +17,6 @@ [app.util.events :as events] [app.util.time :as dt] [app.worker :as-alias wrk] - [clojure.set :as set] [cuerdas.core :as str] [promesa.exec :as px] [promesa.exec.semaphore :as ps] @@ -407,30 +405,10 @@ (defn delete-broken-files [{:keys [id data] :as file}] - (let [xform (comp - (map val) - (map :objects) - (mapcat vals) - (map :type)) - stypes (-> (into #{} xform (:pages-index data)) - (set/difference cts/shape-types))] - - (cond - (seq stypes) - (do - (l/wrn :hint "found shapes with unknown types" - :file-id (str id) - :file-name (:name file) - :types stypes) - (assoc file :deleted-at (dt/now))) - - (-> data :options :components-v2 true?) - (do - (l/wrn :hint "found old components-v2 format" - :file-id (str id) - :file-name (:name file)) - - (assoc file :deleted-at (dt/now))) - - :else - file))) + (if (-> data :options :components-v2 true?) + (do + (l/wrn :hint "found old components-v2 format" + :file-id (str id) + :file-name (:name file)) + (assoc file :deleted-at (dt/now))) + file)) From dba10ffd9bc2a19afe66296ea433d5533cd9b103 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Feb 2024 08:49:58 +0100 Subject: [PATCH 41/45] :bug: Fix broken circle shapes geometry on comp-v2 migration --- backend/src/app/features/components_v2.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 9036d105ff..194c7821b9 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -355,7 +355,8 @@ :points points)) (and (or (cfh/rect-shape? shape) - (cfh/svg-raw-shape? shape)) + (cfh/svg-raw-shape? shape) + (cfh/circle-shape? shape)) (not (valid-shape-points? (:points shape))) (grc/valid-rect? (:selrect shape))) (let [selrect (if (grc/valid-rect? (:svg-viewbox shape)) From 9eb902c68257aab5b39bc54cb20aadb759586a28 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Feb 2024 09:04:26 +0100 Subject: [PATCH 42/45] :bug: Fix broken grids defaults on comp-v2 migration --- backend/src/app/features/components_v2.clj | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 194c7821b9..d365ca10e6 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -31,6 +31,7 @@ [app.common.types.components-list :as ctkl] [app.common.types.container :as ctn] [app.common.types.file :as ctf] + [app.common.types.grid :as ctg] [app.common.types.page :as ctp] [app.common.types.pages-list :as ctpl] [app.common.types.shape :as cts] @@ -138,6 +139,13 @@ (def valid-image-attrs? (sm/lazy-validator ::cts/image-attrs)) +(def valid-column-grid-params? + (sm/lazy-validator ::ctg/column-params)) + +(def valid-square-grid-params? + (sm/lazy-validator ::ctg/square-params)) + + (defn- prepare-file-data "Apply some specific migrations or fixes to things that are allowed in v1 but not in v2, or that are the result of old bugs." @@ -221,11 +229,28 @@ (dissoc options :background) options)) + (fix-saved-grids [options] + (d/update-when options :saved-grids + (fn [grids] + (cond-> grids + (and (contains? grids :column) + (not (valid-column-grid-params? (:column grids)))) + (dissoc :column) + + (and (contains? grids :row) + (not (valid-column-grid-params? (:row grids)))) + (dissoc :row) + + (and (contains? grids :square) + (not (valid-square-grid-params? (:square grids)))) + (dissoc :square))))) + (fix-options [options] (-> options ;; Some pages has invalid data on flows, we proceed just to ;; delete them. (d/update-when :flows #(filterv valid-flow? %)) + (fix-saved-grids) (fix-background)))] (update file-data :pages-index update-vals update-page))) From 04fd4e12d77c251f8e68ab54e485adfb2381af61 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Feb 2024 09:18:42 +0100 Subject: [PATCH 43/45] :bug: Fix invalid transforms on comp-v2 migration --- backend/src/app/features/components_v2.clj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index d365ca10e6..89d0bd8e8e 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -449,6 +449,14 @@ (some? (:main-instance shape)) (dissoc :main-instance) + (and (contains? shape :transform) + (not (gmt/valid-matrix? (:transform shape)))) + (assoc :transform (gmt/matrix)) + + (and (contains? shape :transform-inverse) + (not (gmt/valid-matrix? (:transform-inverse shape)))) + (assoc :transform-inverse (gmt/matrix)) + ;; Fix broken fills (seq (:fills shape)) (update :fills (fn [fills] (filterv valid-fill? fills))) From 750ea4fe3ff3c9c83aa187ddfe3de11ac45874e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Thu, 1 Feb 2024 14:50:38 +0100 Subject: [PATCH 44/45] :bug: Add a fix for component heads that are not groups or frames --- backend/src/app/features/components_v2.clj | 55 +++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 89d0bd8e8e..94094db5e2 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -667,6 +667,7 @@ (if (some? (:id component)) component (assoc component :id id)))] + (-> file-data (d/update-when :components #(d/mapm fix-component %))))) @@ -730,9 +731,59 @@ (update :pages-index update-vals fix-container) (d/update-when :components update-vals fix-container)))) + wrap-non-group-component-roots + (fn [file-data] + ;; Some components have a root that is not a group nor a frame + ;; (e.g. a path or a svg-raw). We need to wrap them in a frame + ;; for this one to became the root. + (letfn [(fix-component [component] + (let [root-shape (ctst/get-shape component (:id component))] + (if (or (cfh/group-shape? root-shape) + (cfh/frame-shape? root-shape)) + component + (let [new-id (uuid/next) + frame (-> (cts/setup-shape + {:type :frame + :id (:id component) + :x (:x (:selrect root-shape)) + :y (:y (:selrect root-shape)) + :width (:width (:selrect root-shape)) + :height (:height (:selrect root-shape)) + :name (:name component) + :shapes [new-id]}) + (assoc :frame-id nil + :parent-id nil)) + root-shape' (assoc root-shape + :id new-id + :parent-id (:id frame) + :frame-id (:id frame))] + (update component :objects assoc + (:id frame) frame + (:id root-shape') root-shape')))))] + + (-> file-data + (d/update-when :components update-vals fix-component)))) + + detach-non-group-instance-roots + (fn [file-data] + ;; If there is a copy instance whose root is not a frame or a group, it cannot + ;; be easily repaired, and anyway it's not working in production, so detach it. + (letfn [(fix-container [container] + (reduce fix-shape container (ctn/shapes-seq container))) + + (fix-shape [container shape] + (if (and (ctk/instance-head? shape) + (not (#{:group :frame} (:type shape)))) + (detach-shape container shape) + container))] + + (-> file-data + (update :pages-index update-vals fix-container) + (d/update-when :components update-vals fix-container)))) + transform-to-frames (fn [file-data] - ;; Transform component and copy heads to frames, and set the + ;; Transform component and copy heads fron group to frames, and set the ;; frame-id of its childrens (letfn [(fix-container [container] (d/update-when container :objects update-vals fix-shape)) @@ -882,6 +933,8 @@ (fix-components-without-id) (remap-refs) (fix-converted-copies) + (wrap-non-group-component-roots) + (detach-non-group-instance-roots) (transform-to-frames) (remap-frame-ids) (fix-frame-ids) From c6fb21187454d053183a29ecd303337ca410e5e1 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Thu, 1 Feb 2024 11:25:16 +0100 Subject: [PATCH 45/45] :bug: Fix copies of detached elements migration --- backend/src/app/features/components_v2.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 94094db5e2..71b0c538a6 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -941,8 +941,9 @@ (fix-component-nil-objects) (fix-false-copies) (fix-component-root-without-component) - (fix-copies-of-detached)))) ; <- Do not add fixes after this one - + (fix-copies-of-detached); <- Do not add fixes after this and fix-orphan-copies call + ; This extra call to fix-orphan-copies after fix-copies-of-detached because we can have detached subtrees with invalid shape-ref attributes + (fix-orphan-copies)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; COMPONENTS MIGRATION ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;