diff --git a/backend/resources/app/templates/debug.tmpl b/backend/resources/app/templates/debug.tmpl index 42894b570c..635380282b 100644 --- a/backend/resources/app/templates/debug.tmpl +++ b/backend/resources/app/templates/debug.tmpl @@ -190,6 +190,20 @@ Debug Main Page + +
+ Validate file: + Given an FILE-ID, check the referential integrity. +
+
+ +
+
+ +
+
+
+
@@ -222,6 +236,7 @@ Debug Main Page
+
Import binfile: Import penpot file in binary format. @@ -236,6 +251,34 @@ Debug Main Page
+ +
+ Repair file: + Given an FILE-ID, repair the referential integrity errors. +
+
+ WARNING: the reparation is not guaranteed and may cause loss of data! +
+
+ You may need to give several repair rounds until all errors are cleared. + +
+
+ +
+
+ + +
+ + A snapshot is made just before the validation, unless skipped. + +
+
+ +
+
+
{% endblock %} diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 7c5956b375..28687d6ac5 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -13,6 +13,9 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.features :as cfeat] + [app.common.files.changes :as cfc] + [app.common.files.repair :as cfr] + [app.common.files.validate :as cfv] [app.common.logging :as l] [app.common.pprint :as pp] [app.common.time :as ct] @@ -28,6 +31,7 @@ [app.rpc.commands.teams :as teams] [app.setup :as-alias setup] [app.setup.clock :as clock] + [app.srepl.helpers :as h] [app.srepl.main :as srepl] [app.storage :as-alias sto] [app.storage.tmp :as tmp] @@ -130,7 +134,7 @@ :hint "invalid button")) (ex/raise :type :not-found - :code :enpty-data + :code :empty-data :hint "empty response")))) (defn- is-file-exists? @@ -484,6 +488,89 @@ {::yres/status 302 ::yres/headers {"location" "/dbg"}})))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; VALIDATE / REPAIR +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- validate-file + [cfg {:keys [params] :as request}] + (let [file-id (some-> params :file-id parse-uuid)] + + (when-not file-id + (ex/raise :type :validation + :code :missing-arguments)) + + (db/tx-run! (assoc cfg ::db/rollback true) + (fn [cfg] + (let [file (bfc/get-file cfg file-id) + libs (bfc/get-resolved-file-libraries cfg file-id)] + (if file + (let [errors (cfv/validate-file file libs)] + {::yres/status 200 + ::yres/headers {"content-type" "text/plain"} + ::yres/body (if (empty? errors) + "NO VALIDATION ERRORS FOUND" + (pp/pprint-str errors))}) + (ex/raise :type :not-found + :code :empty-data + :hint "empty response"))))))) + +(defn- repair-file + [cfg {:keys [params] :as request}] + (let [file-id (some-> params :file-id parse-uuid) + skip-snapshot? (contains? params :skip-snapshot) + profile-id (:app.http.session/profile-id request)] + + (when-not file-id + (ex/raise :type :validation + :code :missing-arguments)) + + (let [output (StringBuilder.) + + repair-file + (fn [file libs _] + (let [errors (cfv/validate-file file libs)] + (.append output (if (empty? errors) + "NO VALIDATION ERRORS FOUND\n" + (str "VALIDATION ERRORS FOUND:\n" + (pp/pprint-str errors) "\n"))) + (if (empty? errors) + file + (let [changes (cfr/repair-file file libs errors)] + (-> file + (update :revn inc) + (update :data cfc/process-changes changes))))))] + + (add-watch l/log-record ::repair-watcher + (fn [_ _ _ record] + (when (= "app.common.files.repair" (::l/logger record)) + (let [props (::l/props record) + hint (get props :hint "") + args (dissoc props :hint) + message (str hint " " + (when-not (empty? args) + args) + "\n")] + (.append output message))))) + (try + (db/tx-run! cfg + h/process-file! + file-id + repair-file + {::h/with-libraries? true + ::h/validate? false + ::h/profile-id profile-id + ::h/snapshot-label (when-not skip-snapshot? "repair")}) + + (.append output "\nREPAIR FINISHED") + + {::yres/status 200 + ::yres/headers {"content-type" "text/plain"} + ::yres/body (.toString output)} + + (finally + (remove-watch l/log-record ::repair-watcher)))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; OTHER SMALL VIEWS/HANDLERS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -578,5 +665,7 @@ {:handler (partial handle-team-features cfg)}] ["/file-export" {:handler (partial export-handler cfg)}] ["/file-import" {:handler (partial import-handler cfg)}] - ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]]) + ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}] + ["/file-validate" {:handler (partial validate-file cfg)}] + ["/file-repair" {:handler (partial repair-file cfg)}]]]]) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 0836429f91..18221f9434 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -153,7 +153,7 @@ (defn process-file! [system file-id update-fn - & {:keys [::snapshot-label ::validate? ::with-libraries?] + & {:keys [::profile-id ::snapshot-label ::validate? ::with-libraries?] :or {validate? true} :as opts}] (let [file (bfc/get-file system file-id :lock-for-update? true @@ -177,8 +177,9 @@ (when (string? snapshot-label) (fsnap/create! system file {:label snapshot-label + :profile-id profile-id :deleted-at (ct/in-future {:days 30}) - :created-by "admin"})) + :created-by "system"})) (let [file' (update file' :revn inc)] (bfc/update-file! system file' opts) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index d6d81f38c6..c174f29f7a 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -398,10 +398,6 @@ (println (sm/humanize-explain explain)) (ex/print-throwable cause)))))))) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; PROCESSING -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - (defn repair-file! "Repair the list of errors detected by validation." [file-id & {:keys [rollback?] :or {rollback? true} :as options}] @@ -410,6 +406,10 @@ options (assoc options ::h/with-libraries? true)] (db/tx-run! system h/process-file! file-id procs.file-repair/repair-file options))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PROCESSING +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn update-file! "Apply a function to the file. Optionally save the changes or not. The function receives the decoded and migrated file data." diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index ad611b1e41..bcc4ddc44d 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -28,7 +28,8 @@ [app.common.types.shape :as cts] [app.common.types.shape-tree :as ctst] [app.common.types.text :as txt] - [app.common.types.tokens-lib :refer [schema:tokens-lib]] + [app.common.types.tokens-lib :as ctob] + [app.common.types.tokens-status :as ctos] [app.common.types.typographies-list :as ctyl] [app.common.types.typography :as cty] [app.common.uuid :as uuid] @@ -86,7 +87,9 @@ [:components {:optional true} schema:components] [:typographies {:optional true} schema:typographies] [:plugin-data {:optional true} schema:plugin-data] - [:tokens-lib {:optional true} schema:tokens-lib]]) + [:tokens-source {:optional true} ::sm/uuid] ;; Forward-compat: UUID of external library containing tokens-lib (full support in follow-up PR) + [:tokens-lib {:optional true} ctob/schema:tokens-lib] + [:tokens-status {:optional true} ctos/schema:tokens-status]]) (def schema:file-metadata [:map {:title "Metadata"} @@ -316,6 +319,7 @@ (update-objects-tree container f))))) ;; Asset helpers + (defn find-component-file [file libraries component-file] (if (and (some? file) (= component-file (:id file))) diff --git a/common/src/app/common/types/tokens_status.cljc b/common/src/app/common/types/tokens_status.cljc new file mode 100644 index 0000000000..c2953cc4c1 --- /dev/null +++ b/common/src/app/common/types/tokens_status.cljc @@ -0,0 +1,147 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.common.types.tokens-status + (:require + #?(:clj [app.common.fressian :as fres]) + #?(:clj [clojure.data.json :as c.json]) + [app.common.schema :as sm] + [app.common.schema.generators :as sg] + [app.common.transit :as t] + [clojure.core.protocols :as cp] + [clojure.datafy :refer [datafy]] + [clojure.pprint :as pp])) + +;; TokensStatus datatype contains the activation status of the themes and sets +;; in a tokens library. + +(defprotocol ITokensStatus + (get-active-theme-ids [_] "Return a clojure set of active theme ids") + (get-active-set-ids [_] "Return a clojure set of active set ids") + (theme-active? [_ theme-id] "Check if a theme is active") + (set-active? [_ set-id] "Check if a set is active") + (set-tokens-status [_ theme-ids set-ids] "Set the activation status of the themes and sets")) + +(deftype TokensStatus [active-theme-ids active-set-ids] + cp/Datafiable + (datafy [_] + {:active-theme-ids active-theme-ids + :active-set-ids active-set-ids}) + + #?@(:clj + [c.json/JSONWriter + (-write [this writter options] + (c.json/-write (datafy this) writter options))]) + + ITokensStatus + (get-active-theme-ids [_] + active-theme-ids) + + (get-active-set-ids [_] + active-set-ids) + + (theme-active? [_ theme-id] + (assert (uuid? theme-id)) + (contains? active-theme-ids theme-id)) + + (set-active? [_ set-id] + (assert (uuid? set-id)) + (contains? active-set-ids set-id)) + + (set-tokens-status [_ theme-ids set-ids] + (assert (set? theme-ids)) + (assert (set? set-ids)) + (TokensStatus. theme-ids set-ids))) + +;; === Helper & Predicate === + +(defn map->TokensStatus + [{:keys [active-theme-ids active-set-ids]}] + (TokensStatus. active-theme-ids active-set-ids)) + +(defn tokens-status? + [o] + (instance? TokensStatus o)) + +;; === Schemas, Check functions & Constructor === + +(declare make-tokens-status) + +(def schema:tokens-status-attrs + [:map {:title "TokensStatus"} + [:active-theme-ids {:optional true} [:set {:gen/max 5} ::sm/uuid]] + [:active-set-ids {:optional true} [:set {:gen/max 5} ::sm/uuid]]]) + +(def schema:tokens-status + [:and {:gen/gen (->> (sg/generator schema:tokens-status-attrs) + (sg/fmap #(make-tokens-status %)))} + [:fn tokens-status?]]) + +(def ^:private check-tokens-status-attrs + (sm/check-fn schema:tokens-status-attrs + :hint "expected valid params for tokens-status")) + +(def check-tokens-status + (sm/check-fn schema:tokens-status + :hint "expected valid tokens-status")) + +(defn make-tokens-status + [& {:as attrs}] + (-> attrs + (update :active-theme-ids #(or % #{})) + (update :active-set-ids #(or % #{})) + (check-tokens-status-attrs) + (map->TokensStatus))) + +;; === Pretty-print for debugging === + +(defmethod pp/simple-dispatch TokensStatus [^TokensStatus obj] + (.write *out* "#penpot/tokens-status ") + (pp/pprint-newline :miser) + (pp/pprint (datafy obj))) + +#?(:clj + (do + (defmethod print-method TokensStatus + [^TokensStatus this ^java.io.Writer w] + (.write w "#penpot/tokens-status ") + (print-method (datafy this) w)) + + (defmethod print-dup TokensStatus + [^TokensStatus this ^java.io.Writer w] + (print-method this w))) + + :cljs + (extend-type TokensStatus + cljs.core/IPrintWithWriter + (-pr-writer [this writer opts] + (-write writer "#penpot/tokens-status ") + (-pr-writer (datafy this) writer opts)) + + cljs.core/IEncodeJS + (-clj->js [this] + (clj->js (datafy this))))) + +;; === Transit serialization === + +(t/add-handlers! + {:id "penpot/tokens-status" + :class TokensStatus + :wfn datafy + :rfn #(make-tokens-status %)}) + +;; === Fressian serialization === + +#?(:clj + (fres/add-handlers! + {:name "penpot/tokens-status/v1" + :class TokensStatus + :wfn (fn [n w o] + (fres/write-tag! w n 1) + (fres/write-object! w (datafy o))) + :rfn (fn [r] + (let [obj (fres/read-object! r)] + (make-tokens-status obj)))})) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index db0dde0bbb..ab075151ee 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -86,6 +86,7 @@ [common-tests.types.shape-layout-test] [common-tests.types.token-test] [common-tests.types.tokens-lib-test] + [common-tests.types.tokens-status-test] [common-tests.undo-stack-test] [common-tests.uuid-test])) @@ -164,6 +165,7 @@ 'common-tests.types.shape-layout-test 'common-tests.types.token-test 'common-tests.types.tokens-lib-test + 'common-tests.types.tokens-status-test 'common-tests.undo-stack-test 'common-tests.uuid-test]) diff --git a/common/test/common_tests/types/tokens_status_test.cljc b/common/test/common_tests/types/tokens_status_test.cljc new file mode 100644 index 0000000000..d2ebb81dd3 --- /dev/null +++ b/common/test/common_tests/types/tokens_status_test.cljc @@ -0,0 +1,101 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns common-tests.types.tokens-status-test + (:require + #?(:clj [app.common.fressian :as fres]) + #?(:clj [clojure.data.json :as json]) + [app.common.transit :as tr] + [app.common.types.tokens-status :as ctos] + [app.common.uuid :as uuid] + [clojure.datafy :refer [datafy]] + [clojure.test :as t])) + +(t/deftest make-tokens-status + (let [theme-id (uuid/next) + set-id (uuid/next) + status (ctos/make-tokens-status :active-theme-ids #{theme-id} + :active-set-ids #{set-id})] + (t/is (ctos/tokens-status? status)) + (t/is (ctos/check-tokens-status status)) + (t/is (= 1 (count (ctos/get-active-theme-ids status)))) + (t/is (ctos/theme-active? status theme-id)) + (t/is (= 1 (count (ctos/get-active-set-ids status)))) + (t/is (ctos/set-active? status set-id)))) + +(t/deftest make-tokens-status-defaults + (let [status (ctos/make-tokens-status)] + (t/is (ctos/tokens-status? status)) + (t/is (ctos/check-tokens-status status)) + (t/is (= 0 (count (ctos/get-active-theme-ids status)))) + (t/is (= 0 (count (ctos/get-active-set-ids status)))))) + +(t/deftest make-invalid-tokens-status + (t/testing "non-set for active-themes" + (t/is (thrown-with-msg? #?(:cljs js/Error :clj Exception) + #"expected valid params for tokens-status" + (ctos/make-tokens-status :active-theme-ids [])))) + (t/testing "non-uuid in active-sets" + (t/is (thrown-with-msg? #?(:cljs js/Error :clj Exception) + #"expected valid params for tokens-status" + (ctos/make-tokens-status :active-set-ids #{"not-a-uuid"}))))) + +(t/deftest set-tokens-status + (let [theme1-id (uuid/next) + theme2-id (uuid/next) + theme3-id (uuid/next) + set1-id (uuid/next) + set2-id (uuid/next) + set3-id (uuid/next) + status (-> (ctos/make-tokens-status {:active-theme-ids #{theme3-id} + :active-set-ids #{set3-id}}) + (ctos/set-tokens-status #{theme1-id theme2-id} #{set1-id set2-id}))] + (t/is (= #{theme1-id theme2-id} (ctos/get-active-theme-ids status))) + (t/is (= #{set1-id set2-id} (ctos/get-active-set-ids status))))) + +(t/deftest datafy-tokens-status + (let [theme-id (uuid/next) + set-id (uuid/next) + status (ctos/make-tokens-status :active-theme-ids #{theme-id} + :active-set-ids #{set-id}) + result (datafy status)] + (t/is (map? result)) + (t/is (not (ctos/tokens-status? result))) + (t/is (= (:active-theme-ids result) #{theme-id})) + (t/is (= (:active-set-ids result) #{set-id})))) + +(t/deftest transit-serialization + (let [theme-id (uuid/next) + set-id (uuid/next) + status (ctos/make-tokens-status :active-theme-ids #{theme-id} + :active-set-ids #{set-id}) + encoded (tr/encode-str status) + status' (tr/decode-str encoded)] + (t/is (ctos/tokens-status? status')) + (t/is (= (datafy status') (datafy status))))) + +#?(:clj + (t/deftest fressian-serialization + (let [theme-id (uuid/next) + set-id (uuid/next) + status (ctos/make-tokens-status :active-theme-ids #{theme-id} + :active-set-ids #{set-id}) + encoded (fres/encode status) + status' (fres/decode encoded)] + (t/is (ctos/tokens-status? status')) + (t/is (= (datafy status') (datafy status)))))) + +#?(:clj + (t/deftest json-serialization + (let [theme-id (uuid/next) + set-id (uuid/next) + status (ctos/make-tokens-status :active-theme-ids #{theme-id} + :active-set-ids #{set-id}) + json-str (json/write-str status) + parsed (json/read-str json-str :key-fn keyword)] + (t/is (map? parsed)) + (t/is (= [(str theme-id)] (:active-theme-ids parsed))) + (t/is (= [(str set-id)] (:active-set-ids parsed))))))