diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 37e9821dff..78d354b243 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -57,7 +57,6 @@ [:map {:title "Manifest"} [:version ::sm/int] [:type :string] - [:format {:optional true} [:enum "compact" "legacy"]] [:referer {:optional true} :string] [:generated-by {:optional true} :string] @@ -66,8 +65,7 @@ [:map [:id ::sm/uuid] [:name :string] - [:features ::cfeat/features] - [:format {:optional true} [:enum "compact" "legacy"]]]]] + [:features ::cfeat/features]]]] [:relations {:optional true} [:vector @@ -300,8 +298,7 @@ (vswap! bfc/*state* update :files assoc file-id {:id file-id :name (:name file) - :features (:features file) - :format (if (contains? cf/flags :binfile-v3-compact) "compact" "legacy")}) + :features (:features file)}) (let [file (cond-> (select-keys file bfc/file-attrs) (:options data) @@ -413,7 +410,6 @@ (let [files (:files @bfc/*state*) params {:type "penpot/export-files" :version (if compact? 2 1) - :format (if compact? "compact" "legacy") :generated-by (str "penpot/" (:full cf/version)) :refer "penpot" :files (vec (vals files)) diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 23c1d1e795..38c86b7d2b 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -9,7 +9,10 @@ (:require [app.binfile.common :as bfc] [app.binfile.v3 :as v3] + [app.common.data.macros :as dm] [app.common.features :as cfeat] + [app.common.json :as json] + [app.common.math :as mth] [app.common.pprint :as pp] [app.common.thumbnails :as thc] [app.common.types.shape :as cts] @@ -21,11 +24,14 @@ [app.rpc :as-alias rpc] [app.storage :as sto] [app.storage.tmp :as tmp] + [app.util.blob :as blob] [backend-tests.helpers :as th] [clojure.test :as t] [cuerdas.core :as str] [datoteka.fs :as fs] - [datoteka.io :as io])) + [datoteka.io :as io]) + (:import + java.util.zip.ZipFile)) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) @@ -87,6 +93,51 @@ (dissoc file :data))) +(defn- prepare-simple-file-with-ids + [profile] + (let [page-id-1 (uuid/custom 1 1) + page-id-2 (uuid/custom 1 2) + shape-id (uuid/custom 2 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false})] + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-page + :name "test 1" + :id page-id-1} + {:type :add-page + :name "test 2" + :id page-id-2}]) + + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-obj + :page-id page-id-1 + :id shape-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id shape-id + :name "image" + :frame-id uuid/zero + :parent-id uuid/zero + :type :rect})}]) + + {:file-id (:id file) + :page-id-1 page-id-1 + :page-id-2 page-id-2 + :shape-id shape-id})) + (t/deftest export-binfile-v3 (let [profile (th/create-profile* 1) file (prepare-simple-file profile) @@ -109,39 +160,63 @@ (t/deftest export-binfile-v3-compact (let [profile (th/create-profile* 1) - file (prepare-simple-file profile) + {:keys [file-id page-id-1 shape-id]} (prepare-simple-file-with-ids profile) + file {:id file-id} output (tmp/tempfile :suffix ".zip")] (with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/ids #{file-id}) (assoc ::bfc/embed-assets false) (assoc ::bfc/include-libraries false)) (io/output-stream output))) + ;; Verify manifest version + (let [manifest (v3/get-manifest (str output))] + (t/is (= 2 (:version manifest)))) + (let [result (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) (assoc ::bfc/profile-id (:id profile)) (assoc ::bfc/input output) (v3/import-files!))] (t/is (= (count result) 1)) - (t/is (every? uuid? result))))) + (t/is (every? uuid? result)) + + ;; Verify imported data has correct shapes with preserved geometry + (let [imported-id (first result) + imported-data (-> (bfc/get-file th/*system* imported-id {:realize? true}) + :data) + pages-index (get imported-data :pages-index) + page (get pages-index page-id-1) + shape (get-in page [:objects shape-id])] + (t/is page "page should exist after import") + (t/is shape "shape should exist after import") + (t/is (= :rect (:type shape))) + (t/is (mth/close? (:x shape) 0)) + (t/is (mth/close? (:y shape) 0)) + (t/is (mth/close? (:width shape) 0.01)) + (t/is (mth/close? (:height shape) 0.01)))))) (t/deftest export-binfile-v3-compact-round-trip (let [profile (th/create-profile* 1) - file (prepare-simple-file profile) + {:keys [file-id page-id-1 shape-id]} (prepare-simple-file-with-ids profile) output1 (tmp/tempfile :suffix ".zip") output2 (tmp/tempfile :suffix ".zip")] (with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/ids #{file-id}) (assoc ::bfc/embed-assets false) (assoc ::bfc/include-libraries false)) (io/output-stream output1))) + ;; Verify compact manifest + (let [manifest (v3/get-manifest (str output1))] + (t/is (= 2 (:version manifest)))) + (let [result (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) (assoc ::bfc/profile-id (:id profile)) @@ -149,6 +224,7 @@ (v3/import-files!)) imported-id (first result)] + ;; Re-export as legacy (without compact flag) and verify (v3/export-files! (-> th/*system* (assoc ::bfc/ids #{imported-id}) @@ -156,10 +232,64 @@ (assoc ::bfc/include-libraries false)) (io/output-stream output2)) - (let [result2 (-> th/*system* - (assoc ::bfc/project-id (:default-project-id profile)) - (assoc ::bfc/profile-id (:id profile)) - (assoc ::bfc/input output2) - (v3/import-files!))] + (let [manifest2 (v3/get-manifest (str output2)) + result2 (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output2) + (v3/import-files!))] + (t/is (= 1 (:version manifest2)) "re-export without flag should be legacy") (t/is (= (count result2) 1)) - (t/is (every? uuid? result2)))))) + (t/is (every? uuid? result2)) + + ;; Verify shapes survive the compact -> import -> legacy re-export round-trip + (let [reimported-id (first result2) + reimported-data (-> (bfc/get-file th/*system* reimported-id {:realize? true}) + :data) + pages-index (get reimported-data :pages-index) + page (get pages-index page-id-1) + shape (get-in page [:objects shape-id])] + (t/is page "page should exist after round-trip") + (t/is shape "shape should exist after round-trip") + (t/is (= :rect (:type shape))) + (t/is (mth/close? (:x shape) 0)) + (t/is (mth/close? (:y shape) 0)) + (t/is (mth/close? (:width shape) 0.01)) + (t/is (mth/close? (:height shape) 0.01)))))) + + (t/deftest export-binfile-v3-compact-page-structure + (let [profile (th/create-profile* 1) + {:keys [file-id page-id-1 shape-id]} (prepare-simple-file-with-ids profile) + output (tmp/tempfile :suffix ".zip")] + + (with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)] + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{file-id}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output))) + + (with-open [^ZipFile zip (ZipFile. (fs/file output))] + (let [entries (iterator-seq (.entries zip)) + + ;; Page entry should exist with embedded objects + page-path (str "files/" file-id "/pages/" page-id-1 ".json") + page-entry (.getEntry zip page-path)] + (t/is page-entry "compact page entry should exist") + + ;; Per-shape entries should NOT exist in compact format + (doseq [^java.util.zip.ZipEntry entry entries] + (let [name (.getName entry)] + (t/is (not (.startsWith name (str "files/" file-id "/pages/" page-id-1 "/"))) + (str "should not have per-shape entries: " name)))) + + ;; Verify page entry contains objects + (with-open [reader (io/reader (.getInputStream zip page-entry))] + (let [page-data (json/read reader :key-fn json/read-kebab-key)] + (t/is (contains? page-data :objects) + "compact page should contain embedded objects") + (t/is (> (count (:objects page-data)) 0) + "compact page objects should not be empty") + (t/is (contains? (:objects page-data) shape-id) + "compact page should contain the shape")))))))) diff --git a/library/src/lib/export.cljs b/library/src/lib/export.cljs index 024e4e0d92..172e54d498 100644 --- a/library/src/lib/export.cljs +++ b/library/src/lib/export.cljs @@ -9,6 +9,7 @@ (:require [app.common.data :as d] [app.common.files.builder :as fb] + [app.common.files.shape-compact :as fsc] [app.common.json :as json] [app.common.media :as media] [app.common.schema :as sm] @@ -94,7 +95,7 @@ (-> shape encode-shape json/encode))) (defn- generate-file-export-procs - [{:keys [id data] :as file}] + [format {:keys [id data] :as file}] (cons (let [file (cond-> (select-keys file file-attrs) (:options data) @@ -104,7 +105,8 @@ (concat (let [pages (get data :pages) - pages-index (get data :pages-index)] + pages-index (get data :pages-index) + compact? (= format "compact")] (->> (d/enumerate pages) (mapcat @@ -114,13 +116,31 @@ page (-> page (dissoc :objects) (assoc :index index))] - (cons - [(str "files/" id "/pages/" page-id ".json") - (delay (-> page encode-page json/encode))] - (map (fn [[shape-id shape]] - [(str "files/" id "/pages/" page-id "/" shape-id ".json") - (delay (encode-shape* shape))]) - objects))))))) + (if compact? + (let [compacted-objects + (reduce-kv + (fn [m shape-id shape] + (let [shape (-> shape + (cond-> (or (= (:type shape) :path) + (= (:type shape) :bool)) + (update :content vec)) + fsc/compact-shape + fsc/round-values + encode-shape)] + (assoc m shape-id shape))) + {} + objects)] + (list + [(str "files/" id "/pages/" page-id ".json") + (delay (-> (assoc page :objects compacted-objects) + json/encode))])) + (cons + [(str "files/" id "/pages/" page-id ".json") + (delay (-> page encode-page json/encode))] + (map (fn [[shape-id shape]] + [(str "files/" id "/pages/" page-id "/" shape-id ".json") + (delay (encode-shape* shape))]) + objects)))))))) (->> (get data :components) (map (fn [[component-id component]] @@ -155,9 +175,9 @@ json/encode))]))))) (defn- generate-files-export-procs - [state] + [state format] (->> (vals (get state ::fb/files)) - (mapcat generate-file-export-procs))) + (mapcat #(generate-file-export-procs format %)))) (defn- generate-media-export-procs [state] @@ -181,7 +201,7 @@ (json/encode)))])))))) (defn- generate-manifest-procs - [state] + [state format] (let [opts (get state :options) files (->> (get state ::fb/files) (mapv (fn [[file-id file]] @@ -189,7 +209,7 @@ :name (:name file) :features (:features file)}))) params {:type "penpot/export-files" - :version 1 + :version (if (= format "compact") 2 1) :generated-by "penpot-library/%version%" :referer (get opts :referer) :files files @@ -201,11 +221,11 @@ (delay (json/encode params))])) (defn- generate-procs - [state] + [state format] (let [state (deref state)] - (cons (generate-manifest-procs state) + (cons (generate-manifest-procs state format) (concat - (generate-files-export-procs state) + (generate-files-export-procs state format) (generate-media-export-procs state))))) (def ^:private @@ -218,8 +238,8 @@ (constantly nil)) (defn- export - [state writer progress-fn] - (let [procs (into [] xf:add-proc-index (generate-procs state)) + [state writer progress-fn format] + (let [procs (into [] xf:add-proc-index (generate-procs state format)) total (count procs)] (->> (p/reduce (fn [writer [path data index]] (let [data (if (delay? data) (deref data) data) @@ -237,7 +257,7 @@ (defn export-bytes ([state] - (export state (zip/writer (zip/bytes-writer)) noop-fn)) + (export state (zip/writer (zip/bytes-writer)) noop-fn "legacy")) ([state options] (let [options (if (object? options) @@ -245,13 +265,16 @@ options) progress-fn - (get options :on-progress noop-fn)] + (get options :on-progress noop-fn) - (export state (zip/writer (zip/bytes-writer)) progress-fn)))) + format + (get options :format "legacy")] + + (export state (zip/writer (zip/bytes-writer)) progress-fn format)))) (defn export-blob ([state] - (export state (zip/writer (zip/blob-writer)) noop-fn)) + (export state (zip/writer (zip/blob-writer)) noop-fn "legacy")) ([state options] (let [options (if (object? options) @@ -259,13 +282,16 @@ options) progress-fn - (get options :on-progress noop-fn)] + (get options :on-progress noop-fn) - (export state (zip/writer (zip/blob-writer)) progress-fn)))) + format + (get options :format "legacy")] + + (export state (zip/writer (zip/blob-writer)) progress-fn format)))) (defn export-stream ([state stream] - (export state (zip/writer stream) noop-fn)) + (export state (zip/writer stream) noop-fn "legacy")) ([state stream options] (let [options (if (object? options) @@ -273,5 +299,9 @@ options) progress-fn - (get options :on-progress noop-fn)] - (export state (zip/writer stream) progress-fn)))) + (get options :on-progress noop-fn) + + format + (get options :format "legacy")] + + (export state (zip/writer stream) progress-fn format)))) diff --git a/library/test/builder.test.js b/library/test/builder.test.js index e4c7bf8a37..36096e33a7 100644 --- a/library/test/builder.test.js +++ b/library/test/builder.test.js @@ -3,6 +3,7 @@ import test from "node:test"; import * as fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { BlobReader, ZipReader, TextWriter } from "@zip.js/zip.js"; import * as penpot from "#self"; @@ -218,3 +219,97 @@ test("create context with tokens lib as obj", () => { assert.ok(file.data); assert.ok(file.data.tokensLib) }); + +test("export compact format produces page-level objects", async () => { + const context = penpot.createBuildContext(); + const fileId = context.addFile({name: "test file"}); + const pageId = context.addPage({name: "test page"}); + + context.addRect({name: "rect1", x: 100, y: 200, width: 50, height: 30}); + + const blob = await penpot.exportAsBlob(context, {format: "compact"}); + + const zipReader = new ZipReader(new BlobReader(new Blob([blob]))); + const entries = await zipReader.getEntries(); + + const manifestEntry = entries.find(e => e.filename === "manifest.json"); + assert.ok(manifestEntry, "manifest should exist"); + + const manifest = JSON.parse(await manifestEntry.getData(new TextWriter())); + assert.equal(manifest.version, 2); + + const pageEntry = entries.find(e => + e.filename === `files/${fileId}/pages/${pageId}.json`); + assert.ok(pageEntry, "page entry should exist"); + + const pageData = JSON.parse(await pageEntry.getData(new TextWriter())); + assert.ok(pageData.objects, "page should contain objects"); + assert.ok(Object.keys(pageData.objects).length > 0, + "objects should not be empty"); + + const shapeEntries = entries.filter(e => + e.filename.startsWith(`files/${fileId}/pages/${pageId}/`) && + e.filename !== `files/${fileId}/pages/${pageId}.json`); + assert.equal(shapeEntries.length, 0, "should not have per-shape entries"); + + await zipReader.close(); +}); + +test("export legacy format produces per-shape entries", async () => { + const context = penpot.createBuildContext(); + const fileId = context.addFile({name: "test file"}); + const pageId = context.addPage({name: "test page"}); + + context.addRect({name: "rect1", x: 100, y: 200, width: 50, height: 30}); + + const blob = await penpot.exportAsBlob(context, {format: "legacy"}); + + const zipReader = new ZipReader(new BlobReader(new Blob([blob]))); + const entries = await zipReader.getEntries(); + + const manifestEntry = entries.find(e => e.filename === "manifest.json"); + assert.ok(manifestEntry, "manifest should exist"); + + const manifest = JSON.parse(await manifestEntry.getData(new TextWriter())); + assert.equal(manifest.version, 1); + + const pageEntry = entries.find(e => + e.filename === `files/${fileId}/pages/${pageId}.json`); + assert.ok(pageEntry, "page entry should exist"); + + const pageData = JSON.parse(await pageEntry.getData(new TextWriter())); + assert.ok(!pageData.objects, "legacy page should not contain objects"); + + const shapeEntries = entries.filter(e => + e.filename.startsWith(`files/${fileId}/pages/${pageId}/`) && + e.filename !== `files/${fileId}/pages/${pageId}.json`); + assert.ok(shapeEntries.length > 0, "should have per-shape entries"); + + await zipReader.close(); +}); + +test("export default format produces per-shape entries", async () => { + const context = penpot.createBuildContext(); + const fileId = context.addFile({name: "test file"}); + const pageId = context.addPage({name: "test page"}); + + context.addRect({name: "rect1", x: 100, y: 200, width: 50, height: 30}); + + const blob = await penpot.exportAsBlob(context); + + const zipReader = new ZipReader(new BlobReader(new Blob([blob]))); + const entries = await zipReader.getEntries(); + + const manifest = JSON.parse( + await entries.find(e => e.filename === "manifest.json") + .getData(new TextWriter())); + assert.equal(manifest.version, 1); + + const shapeEntries = entries.filter(e => + e.filename.startsWith(`files/${fileId}/pages/${pageId}/`) && + e.filename !== `files/${fileId}/pages/${pageId}.json`); + assert.ok(shapeEntries.length > 0, + "default (legacy) should have per-shape entries"); + + await zipReader.close(); +});