✨ Add the :export-assets job-def (external exporter)

The job-def freezes the export items the frontend prepares (file, page,
object, type, scale, name and the optional share-id) and routes the job
to the `:exporter` queue through the new `::jobs/queue-name` metadata,
which the creation command reads and the substrate never does. The
handler only raises: the work runs on the external worker, and a wrong
configuration that claimed the job on a JVM runner fails visibly.

Export has one command per job type: `create-export-binfile-job` keeps
the exact behavior the export command had, and `create-export-assets-job`
checks the read permission of every file named in the items before
anything is stored. The family quote still counts every export job of
either kind.

AI-assisted-by: glm-5.3-flash
This commit is contained in:
Andrey Antukh 2026-10-06 10:18:53 +00:00
parent 27ccb32ea1
commit c05a05b4c3
4 changed files with 247 additions and 7 deletions

View File

@ -591,6 +591,7 @@
:offload-file-data (ig/ref :app.tasks.offload-file-data/job-def)
:objects-gc (ig/ref :app.tasks.objects-gc/job-def)
:export-binfile (ig/ref :app.tasks.export-binfile/job-def)
:export-assets (ig/ref :app.tasks.export-assets/job-def)
:import-binfile (ig/ref :app.tasks.import-binfile/job-def)
:storage-gc-deleted (ig/ref ::sto.gc-deleted/job-def)
:storage-gc-touched (ig/ref ::sto.gc-touched/job-def)
@ -657,6 +658,11 @@
::mtx/metrics (ig/ref ::mtx/metrics)
::mbus/msgbus (ig/ref ::mbus/msgbus)}
;; the export of shapes/frames: the work runs on the external exporter
;; worker, so this def carries no handler work and no components
:app.tasks.export-assets/job-def
{}
:app.storage.gc-deleted/job-def
{::db/pool (ig/ref ::db/pool)
::sto/storage (ig/ref ::sto/storage)

View File

@ -32,6 +32,7 @@
[app.rpc.doc :as-alias doc]
[app.rpc.quotes :as quotes]
[app.storage :as sto]
[app.tasks.export-assets :as export-assets]
[app.tasks.export-binfile :as export-binfile]
[app.tasks.import-binfile :as import-binfile]
[app.util.services :as sv]
@ -44,6 +45,13 @@
[:map {:title "create-export-binfile-job" :closed true}
[:params export-binfile/schema:params]])
(def ^:private schema:create-export-assets-job
"The shapes and frames frozen in the params, rendered as image files
by the external exporter worker; the business params are what the
`:export-assets` job-def declares."
[:map {:title "create-export-assets-job" :closed true}
[:params export-assets/schema:params]])
(def ^:private schema:create-import-binfile-job
"The `.penpot` package the caller uploaded, imported into a project by
the backend runner; the business params are what the `:import-binfile`
@ -110,6 +118,18 @@
;; EXPORT
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- check-file-permissions
"The files the job will read, checked before anything is stored: a
refusal arrives synchronously. The handler checks it again before
reading, because a permission can be revoked while the job waits."
[cfg profile-id file-ids]
(when (empty? file-ids)
(ex/raise :type :validation
:code :no-files-to-export
:hint "expected at least one file to export"))
(doseq [file-id file-ids]
(files/check-read-permissions! cfg profile-id file-id)))
(sv/defmethod ::create-export-binfile-job
"Create a durable job that exports a set of files as a `.penpot`
package.
@ -134,13 +154,7 @@
params (jobs/validate-params job-def (:params envelope))
file-ids (:file-ids params)]
(when (empty? file-ids)
(ex/raise :type :validation
:code :no-files-to-export
:hint "expected at least one file to export"))
(doseq [file-id file-ids]
(files/check-read-permissions! cfg profile-id file-id))
(check-file-permissions cfg profile-id file-ids)
(quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile
::quotes/profile-id profile-id})
@ -152,6 +166,40 @@
:files (count file-ids)
:export-type (d/name (:export-type params))}}))))
(sv/defmethod ::create-export-assets-job
"Create a durable job that renders shapes and frames as image files.
Its params freeze the items to render (file, page, object, type,
scale, name and the optional share id) the way the exporter receives
them. The work runs on the external exporter worker, which consumes
the queue the job-def names; the backend ships no runner for it.
The job is created, not run: the caller follows it by its id. The
read permission of every file named in the items is checked here so a
refusal arrives synchronously, and the handler checks it again before
reading."
{::doc/added "2.20"
::webhooks/event? true
::sm/params schema:create-export-assets-job
::sm/result schema:job-summary}
[cfg {:keys [::rpc/profile-id] :as envelope}]
(let [job-def (resolve-job-def cfg :export :export-assets)
;; the RPC layer decoded the params against the job-def schema
;; before the command ran
params (jobs/validate-params job-def (:params envelope))]
(check-file-permissions cfg profile-id
(into #{} (map :file-id) (:exports params)))
(quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile
::quotes/profile-id profile-id})
(let [summary (get-job-summary cfg (submit-job cfg job-def :export-assets
params profile-id))]
(with-meta summary
{::audit/props {:job-id (:id summary)
:files (count (:exports params))}}))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMPORT
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

View File

@ -0,0 +1,79 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.tasks.export-assets
"The `:export-assets` job: the shapes and frames a caller froze at
creation time, rendered as image files.
This is an EXTERNAL job: the work runs on the exporter worker, a
process outside the JVM, so the job is created with `::jobs/queue-name
:exporter` (read by the creation command; the substrate never looks at
it) and the backend ships no runner for that queue. The handler here
is a guard, not a worker: it is never executed under a correct
configuration, because the only runner that could claim the job is
external, and `claim-job` does not look at the queue. Should a wrong
configuration ever route the job to a JVM runner, it fails visibly
here instead of \"completing\" empty.
Progress vocabulary of the external worker (the substrate validates
the shape, the worker owns the words): stages `:preparing`,
`:rendering` and `:packaging`, with counters `objects` (one per
exported shape) and `pages` (one per exported frame)."
(:require
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.jobs :as jobs]
[integrant.core :as ig]))
(def schema:export-item
"One thing to render: where it lives (file, page, object), what to
produce of it (type, scale) and what to call it. The same item shape
the legacy `/api/export` surface received, frozen in the params of the
job; the optional `share-id` marks the item as coming from a public
share."
[:map {:title "export-item" :closed true}
[:file-id ::sm/uuid]
[:page-id ::sm/uuid]
[:object-id ::sm/uuid]
[:type [::sm/one-of #{:png :jpeg :webp :svg :pdf}]]
[:name ::sm/text]
[:suffix {:optional true} ::sm/text]
[:scale ::sm/number]
[:share-id {:optional true} ::sm/uuid]])
(def schema:params
"Business params of an asset export: the items to render, frozen when
the job was created, plus the options that shape the output. There is
no `wait` here: a job is followed by its id, not waited on."
[:map {:title "export-assets-params" :closed true}
[:exports [:vector {:min 1} schema:export-item]]
[:name {:optional true} ::sm/text]
[:skip-children {:optional true} ::sm/boolean]
[:force-multiple {:optional true} ::sm/boolean]
[:is-wasm {:optional true} ::sm/boolean]])
(defn- execute-export-assets
"Plain handler, importable and testable without integrant.
It only raises: the work of this job belongs to the external worker.
See the namespace docstring."
[_context _params]
(ex/raise :type :internal
:code :no-handler-for-external
:hint "this job runs on the external exporter worker"))
(defmethod ig/init-key ::job-def
[_ _cfg]
{::jobs/name :export-assets
::jobs/family :export
::jobs/resource-role :output
;; the queue the creation command routes the job to: an external
;; worker consumes it, the backend ships no runner for it
::jobs/queue-name :exporter
::jobs/schema schema:params
::jobs/handler execute-export-assets
::jobs/decoder (sm/decoder schema:params sm/json-transformer)
::jobs/validator (sm/validator schema:params)})

View File

@ -179,6 +179,113 @@
(t/is (= "detach-libraries" (:export-type props)))
(t/is (= #{:job-id :files :export-type} (set (keys props)))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ASSETS EXPORT
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- export-item
[file-id & {:keys [page-id object-id type name scale suffix share-id]}]
(cond-> {:file-id file-id
:page-id (or page-id (uuid/next))
:object-id (or object-id (uuid/next))
:type (or type :png)
:name (or name "shape")
:scale (or scale 1)}
suffix (assoc :suffix suffix)
share-id (assoc :share-id share-id)))
(defn- create-export-assets-job
[profile-id exports & {:as extra}]
(th/command! (merge {::th/type :create-export-assets-job
::rpc/profile-id profile-id
:params {:exports exports}}
extra)))
(t/deftest create-export-assets-job-answers-with-a-pending-job
(let [profile (th/create-profile* 1)
file-id (first (:file-ids (import-fixture! profile)))
item (export-item file-id)
before (count-storage-objects)
out (create-export-assets-job (:id profile) [item])]
(t/is (th/success? out))
(let [result (:result out)
job (jobs/get-job th/*system* (:id result))]
(t/testing "the answer is enough to follow the job"
(t/is (uuid? (:id result)))
(t/is (= "pending" (:status result)))
(t/is (= "export-assets" (:name result)))
(t/is (ct/inst? (:created-at result))))
(t/testing "and the row routes to the external worker queue"
(let [job-def (jobs/get-job-def (::jobs/defs th/*system*) :export-assets)
params (jobs/decode-params job-def (:params job))]
(t/is (= "exporter" (:queue job)))
(t/is (= 0 (:max-retries job)))
(t/is (= (:id profile) (:profile-id job)))
(t/is (= 1 (count (:exports params))))
(t/is (= :png (:type (first (:exports params)))))))
(t/testing "creating the job stores no artifact"
(t/is (= before (count-storage-objects)))))))
(t/deftest create-export-assets-job-checks-the-read-permission
(let [owner (th/create-profile* 1)
other (th/create-profile* 2)
file-id (first (:file-ids (import-fixture! owner)))
out (create-export-assets-job (:id other) [(export-item file-id)])]
(t/testing "a file the caller cannot read is not exported"
(t/is (not (th/success? out)))
(t/is (= :not-found (th/ex-type (:error out)))))
(t/testing "and no job was created for it"
(t/is (zero? (count-jobs))))))
(t/deftest create-export-assets-job-rejects-an-unknown-type
(let [profile (th/create-profile* 1)
file-id (first (:file-ids (import-fixture! profile)))
out (create-export-assets-job (:id profile)
[(export-item file-id :type :psd)])]
(t/is (not (th/success? out)))
(t/is (= :data-validation (th/ex-code (:error out))))
(t/testing "and nothing was created"
(t/is (zero? (count-jobs))))))
(t/deftest create-export-assets-job-adds-the-job-id-to-the-audit-props
(let [profile (th/create-profile* 1)
file-id (first (:file-ids (import-fixture! profile)))
out (create-export-assets-job (:id profile)
[(export-item file-id)])
props (::audit/props (meta (:result out)))]
(t/is (th/success? out))
(t/testing "the audit of the call points at the job and its items"
(t/is (= (:id (:result out)) (:job-id props)))
(t/is (= 1 (:files props)))
(t/is (= #{:job-id :files} (set (keys props)))))))
(t/deftest create-export-assets-job-counts-in-the-export-quote
;; the quote of the family counts every job of it, so a mixed backlog
;; of binfile and assets jobs fills the same quote
(with-mocks [mock {:target 'app.config/get
:return (th/config-get-mock
{:quotes-export-jobs-per-profile 1})}]
(let [profile (th/create-profile* 1)
file-id (first (:file-ids (import-fixture! profile)))]
(t/is (th/success? (create-export-assets-job (:id profile)
[(export-item file-id)])))
(t/testing "the next export job of the family is refused, whatever its type"
(let [out (create-export-binfile-job (:id profile) #{file-id})]
(t/is (not (th/success? out)))
(t/is (= :max-quote-reached (th/ex-code (:error out))))))
(t/is (= 1 (count-jobs))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMPORT
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;