diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 311cbc96aa..37e9821dff 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -15,6 +15,7 @@ [app.common.exceptions :as ex] [app.common.features :as cfeat] [app.common.files.migrations :as-alias fmg] + [app.common.files.shape-compact :as fsc] [app.common.json :as json] [app.common.logging :as l] [app.common.media :as cmedia] @@ -45,6 +46,7 @@ java.io.InputStream java.io.OutputStreamWriter java.lang.AutoCloseable + java.nio.charset.StandardCharsets java.util.zip.ZipEntry java.util.zip.ZipFile java.util.zip.ZipOutputStream)) @@ -55,6 +57,7 @@ [:map {:title "Manifest"} [:version ::sm/int] [:type :string] + [:format {:optional true} [:enum "compact" "legacy"]] [:referer {:optional true} :string] [:generated-by {:optional true} :string] @@ -63,7 +66,8 @@ [:map [:id ::sm/uuid] [:name :string] - [:features ::cfeat/features]]]] + [:features ::cfeat/features] + [:format {:optional true} [:enum "compact" "legacy"]]]]] [:relations {:optional true} [:vector @@ -217,6 +221,16 @@ (.flush writer)) (.closeEntry output)) +(defn- write-compact-entry! + [^ZipOutputStream output ^String path data] + (.putNextEntry output (ZipEntry. path)) + (let [sw (java.io.StringWriter.)] + (json/write sw data :indent false :key-fn json/write-camel-key) + (.flush sw) + (let [^bytes bytes (.getBytes ^String (str sw) StandardCharsets/UTF_8)] + (.write output bytes 0 (alength bytes)))) + (.closeEntry output)) + (defn- get-file [{:keys [::bfc/embed-assets ::bfc/include-libraries] :as cfg} file-id] @@ -286,7 +300,8 @@ (vswap! bfc/*state* update :files assoc file-id {:id file-id :name (:name file) - :features (:features file)}) + :features (:features file) + :format (if (contains? cf/flags :binfile-v3-compact) "compact" "legacy")}) (let [file (cond-> (select-keys file bfc/file-attrs) (:options data) @@ -302,22 +317,29 @@ (doseq [[index page-id] (d/enumerate pages)] - (let [path (str "files/" file-id "/pages/" page-id ".json") - page (get pages-index page-id) - objects (:objects page) - page (-> page - (dissoc :objects) - (assoc :index index)) - page (encode-page page)] - - (write-entry! output path page) - - (events/tap :progress {:section :page :id page-id :name (:name page) :file-id file-id}) - - (doseq [[shape-id shape] objects] - (let [path (str "files/" file-id "/pages/" page-id "/" shape-id ".json") - shape (encode-shape shape)] - (write-entry! output path shape))))) + (let [page (get pages-index page-id) + objects (:objects page)] + (if (contains? cf/flags :binfile-v3-compact) + (let [path (str "files/" file-id "/pages/" page-id ".json") + objects (d/update-vals objects + (fn [shape] + (-> shape fsc/compact-shape fsc/round-values encode-shape))) + page (-> page + (assoc :objects objects :index index) + (dissoc :options))] + (events/tap :progress {:section :page :id page-id :name (:name page) :file-id file-id}) + (write-compact-entry! output path page)) + (let [path (str "files/" file-id "/pages/" page-id ".json") + page (-> page + (dissoc :objects) + (assoc :index index)) + page (encode-page page)] + (write-entry! output path page) + (events/tap :progress {:section :page :id page-id :name (:name page) :file-id file-id}) + (doseq [[shape-id shape] objects] + (let [path (str "files/" file-id "/pages/" page-id "/" shape-id ".json") + shape (encode-shape shape)] + (write-entry! output path shape))))))) (vswap! bfc/*state* bfc/collect-storage-objects media) (vswap! bfc/*state* bfc/collect-storage-objects thumbnails) @@ -371,11 +393,12 @@ (defn- export-files [{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}] - (let [ids (into ids (when include-libraries (bfc/get-libraries cfg ids))) - rels (if include-libraries - (->> (bfc/get-files-rels cfg ids) - (mapv (juxt :file-id :library-file-id))) - [])] + (let [ids (into ids (when include-libraries (bfc/get-libraries cfg ids))) + rels (if include-libraries + (->> (bfc/get-files-rels cfg ids) + (mapv (juxt :file-id :library-file-id))) + []) + compact? (contains? cf/flags :binfile-v3-compact)] (vswap! bfc/*state* assoc :files (d/ordered-map)) @@ -389,7 +412,8 @@ ;; Write manifest file (let [files (:files @bfc/*state*) params {:type "penpot/export-files" - :version 1 + :version (if compact? 2 1) + :format (if compact? "compact" "legacy") :generated-by (str "penpot/" (:full cf/version)) :refer "penpot" :files (vec (vals files)) @@ -685,7 +709,7 @@ (not-empty))) (defn- read-file-pages - [{:keys [::bfc/input ::entries] :as cfg} file-id] + [{:keys [::bfc/input ::entries ::compact?] :as cfg} file-id] (->> (keep (match-page-entry-fn file-id) entries) (keep (fn [{:keys [id entry]}] (let [page (->> (read-entry input entry) @@ -693,8 +717,17 @@ page (dissoc page :options)] (events/tap :progress {:section :page :id id :file-id file-id}) (when (= id (:id page)) - (let [objects (read-file-shapes cfg file-id id)] - (assoc page :objects objects)))))) + (if compact? + (let [objects (d/update-vals (:objects page) + (fn [shape] + (-> shape + (bfl/clean-shape-pre-decode) + (decode-shape) + (fsc/expand-shape) + (bfl/clean-shape-post-decode))))] + (assoc page :objects objects)) + (let [objects (read-file-shapes cfg file-id id)] + (assoc page :objects objects))))))) (sort-by :index) (reduce (fn [result {:keys [id] :as page}] (assoc result id (dissoc page :index))) @@ -936,10 +969,12 @@ (let [manifest (-> (read-manifest input) (validate-manifest)) + compact? (= 2 (:version manifest)) entries (read-zip-entries input) cfg (-> cfg (assoc ::entries entries) (assoc ::manifest manifest) + (assoc ::compact? compact?) (assoc ::bfc/timestamp timestamp))] (when-not (= "penpot/export-files" (:type manifest)) diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 232a44f87c..23c1d1e795 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -14,6 +14,7 @@ [app.common.thumbnails :as thc] [app.common.types.shape :as cts] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.db.sql :as sql] [app.http :as http] @@ -105,3 +106,60 @@ (v3/import-files!))] (t/is (= (count result) 1)) (t/is (every? uuid? result))))) + +(t/deftest export-binfile-v3-compact + (let [profile (th/create-profile* 1) + file (prepare-simple-file profile) + output (tmp/tempfile :suffix ".zip")] + + (with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)] + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output))) + + (let [result (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output) + (v3/import-files!))] + (t/is (= (count result) 1)) + (t/is (every? uuid? result))))) + +(t/deftest export-binfile-v3-compact-round-trip + (let [profile (th/create-profile* 1) + file (prepare-simple-file profile) + output1 (tmp/tempfile :suffix ".zip") + output2 (tmp/tempfile :suffix ".zip")] + + (with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)] + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output1))) + + (let [result (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output1) + (v3/import-files!)) + imported-id (first result)] + + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{imported-id}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output2)) + + (let [result2 (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output2) + (v3/import-files!))] + (t/is (= (count result2) 1)) + (t/is (every? uuid? result2)))))) diff --git a/common/src/app/common/flags.cljc b/common/src/app/common/flags.cljc index 62ce48aa9a..524e13314b 100644 --- a/common/src/app/common/flags.cljc +++ b/common/src/app/common/flags.cljc @@ -170,7 +170,11 @@ :mcp :background-blur :available-viewer-wasm - :stroke-path}) + :stroke-path + + ;; Compact binfile-v3 export: one JSON per page with embedded + ;; shapes instead of one JSON per shape. + :binfile-v3-compact}) (def all-flags (set/union email login varia))