Add library compact export, tests, and manifest cleanup

Embed compact shapes in page entries for library exports, add geometry
verification and structure tests for backend, and remove the redundant
:format field from the manifest — version alone determines format.

AI-assisted-by: deepseek-v4-pro
This commit is contained in:
Andrey Antukh 2026-07-16 13:22:54 +00:00
parent 54c692d10c
commit aabf1b0bdb
4 changed files with 296 additions and 45 deletions

View File

@ -57,7 +57,6 @@
[:map {:title "Manifest"}
[:version ::sm/int]
[:type :string]
[:format {:optional true} [:enum "compact" "legacy"]]
[:referer {:optional true} :string]
[:generated-by {:optional true} :string]
@ -66,8 +65,7 @@
[:map
[:id ::sm/uuid]
[:name :string]
[:features ::cfeat/features]
[:format {:optional true} [:enum "compact" "legacy"]]]]]
[:features ::cfeat/features]]]]
[:relations {:optional true}
[:vector
@ -300,8 +298,7 @@
(vswap! bfc/*state* update :files assoc file-id
{:id file-id
:name (:name file)
:features (:features file)
:format (if (contains? cf/flags :binfile-v3-compact) "compact" "legacy")})
:features (:features file)})
(let [file (cond-> (select-keys file bfc/file-attrs)
(:options data)
@ -413,7 +410,6 @@
(let [files (:files @bfc/*state*)
params {:type "penpot/export-files"
:version (if compact? 2 1)
:format (if compact? "compact" "legacy")
:generated-by (str "penpot/" (:full cf/version))
:refer "penpot"
:files (vec (vals files))

View File

@ -9,7 +9,10 @@
(:require
[app.binfile.common :as bfc]
[app.binfile.v3 :as v3]
[app.common.data.macros :as dm]
[app.common.features :as cfeat]
[app.common.json :as json]
[app.common.math :as mth]
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.types.shape :as cts]
@ -21,11 +24,14 @@
[app.rpc :as-alias rpc]
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.blob :as blob]
[backend-tests.helpers :as th]
[clojure.test :as t]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io]))
[datoteka.io :as io])
(:import
java.util.zip.ZipFile))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
@ -87,6 +93,51 @@
(dissoc file :data)))
(defn- prepare-simple-file-with-ids
[profile]
(let [page-id-1 (uuid/custom 1 1)
page-id-2 (uuid/custom 1 2)
shape-id (uuid/custom 2 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared false})]
(update-file!
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-page
:name "test 1"
:id page-id-1}
{:type :add-page
:name "test 2"
:id page-id-2}])
(update-file!
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id-1
:id shape-id
:parent-id uuid/zero
:frame-id uuid/zero
:components-v2 true
:obj (cts/setup-shape
{:id shape-id
:name "image"
:frame-id uuid/zero
:parent-id uuid/zero
:type :rect})}])
{:file-id (:id file)
:page-id-1 page-id-1
:page-id-2 page-id-2
:shape-id shape-id}))
(t/deftest export-binfile-v3
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
@ -109,39 +160,63 @@
(t/deftest export-binfile-v3-compact
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
{:keys [file-id page-id-1 shape-id]} (prepare-simple-file-with-ids profile)
file {:id file-id}
output (tmp/tempfile :suffix ".zip")]
(with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{(:id file)})
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets false)
(assoc ::bfc/include-libraries false))
(io/output-stream output)))
;; Verify manifest version
(let [manifest (v3/get-manifest (str output))]
(t/is (= 2 (:version manifest))))
(let [result (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output)
(v3/import-files!))]
(t/is (= (count result) 1))
(t/is (every? uuid? result)))))
(t/is (every? uuid? result))
;; Verify imported data has correct shapes with preserved geometry
(let [imported-id (first result)
imported-data (-> (bfc/get-file th/*system* imported-id {:realize? true})
:data)
pages-index (get imported-data :pages-index)
page (get pages-index page-id-1)
shape (get-in page [:objects shape-id])]
(t/is page "page should exist after import")
(t/is shape "shape should exist after import")
(t/is (= :rect (:type shape)))
(t/is (mth/close? (:x shape) 0))
(t/is (mth/close? (:y shape) 0))
(t/is (mth/close? (:width shape) 0.01))
(t/is (mth/close? (:height shape) 0.01))))))
(t/deftest export-binfile-v3-compact-round-trip
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
{:keys [file-id page-id-1 shape-id]} (prepare-simple-file-with-ids profile)
output1 (tmp/tempfile :suffix ".zip")
output2 (tmp/tempfile :suffix ".zip")]
(with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{(:id file)})
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets false)
(assoc ::bfc/include-libraries false))
(io/output-stream output1)))
;; Verify compact manifest
(let [manifest (v3/get-manifest (str output1))]
(t/is (= 2 (:version manifest))))
(let [result (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
@ -149,6 +224,7 @@
(v3/import-files!))
imported-id (first result)]
;; Re-export as legacy (without compact flag) and verify
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{imported-id})
@ -156,10 +232,64 @@
(assoc ::bfc/include-libraries false))
(io/output-stream output2))
(let [result2 (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output2)
(v3/import-files!))]
(let [manifest2 (v3/get-manifest (str output2))
result2 (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output2)
(v3/import-files!))]
(t/is (= 1 (:version manifest2)) "re-export without flag should be legacy")
(t/is (= (count result2) 1))
(t/is (every? uuid? result2))))))
(t/is (every? uuid? result2))
;; Verify shapes survive the compact -> import -> legacy re-export round-trip
(let [reimported-id (first result2)
reimported-data (-> (bfc/get-file th/*system* reimported-id {:realize? true})
:data)
pages-index (get reimported-data :pages-index)
page (get pages-index page-id-1)
shape (get-in page [:objects shape-id])]
(t/is page "page should exist after round-trip")
(t/is shape "shape should exist after round-trip")
(t/is (= :rect (:type shape)))
(t/is (mth/close? (:x shape) 0))
(t/is (mth/close? (:y shape) 0))
(t/is (mth/close? (:width shape) 0.01))
(t/is (mth/close? (:height shape) 0.01))))))
(t/deftest export-binfile-v3-compact-page-structure
(let [profile (th/create-profile* 1)
{:keys [file-id page-id-1 shape-id]} (prepare-simple-file-with-ids profile)
output (tmp/tempfile :suffix ".zip")]
(with-redefs [cf/flags (conj cf/flags :binfile-v3-compact)]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets false)
(assoc ::bfc/include-libraries false))
(io/output-stream output)))
(with-open [^ZipFile zip (ZipFile. (fs/file output))]
(let [entries (iterator-seq (.entries zip))
;; Page entry should exist with embedded objects
page-path (str "files/" file-id "/pages/" page-id-1 ".json")
page-entry (.getEntry zip page-path)]
(t/is page-entry "compact page entry should exist")
;; Per-shape entries should NOT exist in compact format
(doseq [^java.util.zip.ZipEntry entry entries]
(let [name (.getName entry)]
(t/is (not (.startsWith name (str "files/" file-id "/pages/" page-id-1 "/")))
(str "should not have per-shape entries: " name))))
;; Verify page entry contains objects
(with-open [reader (io/reader (.getInputStream zip page-entry))]
(let [page-data (json/read reader :key-fn json/read-kebab-key)]
(t/is (contains? page-data :objects)
"compact page should contain embedded objects")
(t/is (> (count (:objects page-data)) 0)
"compact page objects should not be empty")
(t/is (contains? (:objects page-data) shape-id)
"compact page should contain the shape"))))))))

View File

@ -9,6 +9,7 @@
(:require
[app.common.data :as d]
[app.common.files.builder :as fb]
[app.common.files.shape-compact :as fsc]
[app.common.json :as json]
[app.common.media :as media]
[app.common.schema :as sm]
@ -94,7 +95,7 @@
(-> shape encode-shape json/encode)))
(defn- generate-file-export-procs
[{:keys [id data] :as file}]
[format {:keys [id data] :as file}]
(cons
(let [file (cond-> (select-keys file file-attrs)
(:options data)
@ -104,7 +105,8 @@
(concat
(let [pages (get data :pages)
pages-index (get data :pages-index)]
pages-index (get data :pages-index)
compact? (= format "compact")]
(->> (d/enumerate pages)
(mapcat
@ -114,13 +116,31 @@
page (-> page
(dissoc :objects)
(assoc :index index))]
(cons
[(str "files/" id "/pages/" page-id ".json")
(delay (-> page encode-page json/encode))]
(map (fn [[shape-id shape]]
[(str "files/" id "/pages/" page-id "/" shape-id ".json")
(delay (encode-shape* shape))])
objects)))))))
(if compact?
(let [compacted-objects
(reduce-kv
(fn [m shape-id shape]
(let [shape (-> shape
(cond-> (or (= (:type shape) :path)
(= (:type shape) :bool))
(update :content vec))
fsc/compact-shape
fsc/round-values
encode-shape)]
(assoc m shape-id shape)))
{}
objects)]
(list
[(str "files/" id "/pages/" page-id ".json")
(delay (-> (assoc page :objects compacted-objects)
json/encode))]))
(cons
[(str "files/" id "/pages/" page-id ".json")
(delay (-> page encode-page json/encode))]
(map (fn [[shape-id shape]]
[(str "files/" id "/pages/" page-id "/" shape-id ".json")
(delay (encode-shape* shape))])
objects))))))))
(->> (get data :components)
(map (fn [[component-id component]]
@ -155,9 +175,9 @@
json/encode))])))))
(defn- generate-files-export-procs
[state]
[state format]
(->> (vals (get state ::fb/files))
(mapcat generate-file-export-procs)))
(mapcat #(generate-file-export-procs format %))))
(defn- generate-media-export-procs
[state]
@ -181,7 +201,7 @@
(json/encode)))]))))))
(defn- generate-manifest-procs
[state]
[state format]
(let [opts (get state :options)
files (->> (get state ::fb/files)
(mapv (fn [[file-id file]]
@ -189,7 +209,7 @@
:name (:name file)
:features (:features file)})))
params {:type "penpot/export-files"
:version 1
:version (if (= format "compact") 2 1)
:generated-by "penpot-library/%version%"
:referer (get opts :referer)
:files files
@ -201,11 +221,11 @@
(delay (json/encode params))]))
(defn- generate-procs
[state]
[state format]
(let [state (deref state)]
(cons (generate-manifest-procs state)
(cons (generate-manifest-procs state format)
(concat
(generate-files-export-procs state)
(generate-files-export-procs state format)
(generate-media-export-procs state)))))
(def ^:private
@ -218,8 +238,8 @@
(constantly nil))
(defn- export
[state writer progress-fn]
(let [procs (into [] xf:add-proc-index (generate-procs state))
[state writer progress-fn format]
(let [procs (into [] xf:add-proc-index (generate-procs state format))
total (count procs)]
(->> (p/reduce (fn [writer [path data index]]
(let [data (if (delay? data) (deref data) data)
@ -237,7 +257,7 @@
(defn export-bytes
([state]
(export state (zip/writer (zip/bytes-writer)) noop-fn))
(export state (zip/writer (zip/bytes-writer)) noop-fn "legacy"))
([state options]
(let [options
(if (object? options)
@ -245,13 +265,16 @@
options)
progress-fn
(get options :on-progress noop-fn)]
(get options :on-progress noop-fn)
(export state (zip/writer (zip/bytes-writer)) progress-fn))))
format
(get options :format "legacy")]
(export state (zip/writer (zip/bytes-writer)) progress-fn format))))
(defn export-blob
([state]
(export state (zip/writer (zip/blob-writer)) noop-fn))
(export state (zip/writer (zip/blob-writer)) noop-fn "legacy"))
([state options]
(let [options
(if (object? options)
@ -259,13 +282,16 @@
options)
progress-fn
(get options :on-progress noop-fn)]
(get options :on-progress noop-fn)
(export state (zip/writer (zip/blob-writer)) progress-fn))))
format
(get options :format "legacy")]
(export state (zip/writer (zip/blob-writer)) progress-fn format))))
(defn export-stream
([state stream]
(export state (zip/writer stream) noop-fn))
(export state (zip/writer stream) noop-fn "legacy"))
([state stream options]
(let [options
(if (object? options)
@ -273,5 +299,9 @@
options)
progress-fn
(get options :on-progress noop-fn)]
(export state (zip/writer stream) progress-fn))))
(get options :on-progress noop-fn)
format
(get options :format "legacy")]
(export state (zip/writer stream) progress-fn format))))

View File

@ -3,6 +3,7 @@ import test from "node:test";
import * as fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { BlobReader, ZipReader, TextWriter } from "@zip.js/zip.js";
import * as penpot from "#self";
@ -218,3 +219,97 @@ test("create context with tokens lib as obj", () => {
assert.ok(file.data);
assert.ok(file.data.tokensLib)
});
test("export compact format produces page-level objects", async () => {
const context = penpot.createBuildContext();
const fileId = context.addFile({name: "test file"});
const pageId = context.addPage({name: "test page"});
context.addRect({name: "rect1", x: 100, y: 200, width: 50, height: 30});
const blob = await penpot.exportAsBlob(context, {format: "compact"});
const zipReader = new ZipReader(new BlobReader(new Blob([blob])));
const entries = await zipReader.getEntries();
const manifestEntry = entries.find(e => e.filename === "manifest.json");
assert.ok(manifestEntry, "manifest should exist");
const manifest = JSON.parse(await manifestEntry.getData(new TextWriter()));
assert.equal(manifest.version, 2);
const pageEntry = entries.find(e =>
e.filename === `files/${fileId}/pages/${pageId}.json`);
assert.ok(pageEntry, "page entry should exist");
const pageData = JSON.parse(await pageEntry.getData(new TextWriter()));
assert.ok(pageData.objects, "page should contain objects");
assert.ok(Object.keys(pageData.objects).length > 0,
"objects should not be empty");
const shapeEntries = entries.filter(e =>
e.filename.startsWith(`files/${fileId}/pages/${pageId}/`) &&
e.filename !== `files/${fileId}/pages/${pageId}.json`);
assert.equal(shapeEntries.length, 0, "should not have per-shape entries");
await zipReader.close();
});
test("export legacy format produces per-shape entries", async () => {
const context = penpot.createBuildContext();
const fileId = context.addFile({name: "test file"});
const pageId = context.addPage({name: "test page"});
context.addRect({name: "rect1", x: 100, y: 200, width: 50, height: 30});
const blob = await penpot.exportAsBlob(context, {format: "legacy"});
const zipReader = new ZipReader(new BlobReader(new Blob([blob])));
const entries = await zipReader.getEntries();
const manifestEntry = entries.find(e => e.filename === "manifest.json");
assert.ok(manifestEntry, "manifest should exist");
const manifest = JSON.parse(await manifestEntry.getData(new TextWriter()));
assert.equal(manifest.version, 1);
const pageEntry = entries.find(e =>
e.filename === `files/${fileId}/pages/${pageId}.json`);
assert.ok(pageEntry, "page entry should exist");
const pageData = JSON.parse(await pageEntry.getData(new TextWriter()));
assert.ok(!pageData.objects, "legacy page should not contain objects");
const shapeEntries = entries.filter(e =>
e.filename.startsWith(`files/${fileId}/pages/${pageId}/`) &&
e.filename !== `files/${fileId}/pages/${pageId}.json`);
assert.ok(shapeEntries.length > 0, "should have per-shape entries");
await zipReader.close();
});
test("export default format produces per-shape entries", async () => {
const context = penpot.createBuildContext();
const fileId = context.addFile({name: "test file"});
const pageId = context.addPage({name: "test page"});
context.addRect({name: "rect1", x: 100, y: 200, width: 50, height: 30});
const blob = await penpot.exportAsBlob(context);
const zipReader = new ZipReader(new BlobReader(new Blob([blob])));
const entries = await zipReader.getEntries();
const manifest = JSON.parse(
await entries.find(e => e.filename === "manifest.json")
.getData(new TextWriter()));
assert.equal(manifest.version, 1);
const shapeEntries = entries.filter(e =>
e.filename.startsWith(`files/${fileId}/pages/${pageId}/`) &&
e.filename !== `files/${fileId}/pages/${pageId}.json`);
assert.ok(shapeEntries.length > 0,
"default (legacy) should have per-shape entries");
await zipReader.close();
});