mirror of
https://github.com/penpot/penpot.git
synced 2026-11-16 15:11:17 +00:00
✨ Add the :export-assets job-def (external exporter)
The job-def freezes the export items the frontend prepares (file, page, object, type, scale, name and the optional share-id) and routes the job to the `:exporter` queue through the new `::jobs/queue-name` metadata, which the creation command reads and the substrate never does. The handler only raises: the work runs on the external worker, and a wrong configuration that claimed the job on a JVM runner fails visibly. Export has one command per job type: `create-export-binfile-job` keeps the exact behavior the export command had, and `create-export-assets-job` checks the read permission of every file named in the items before anything is stored. The family quote still counts every export job of either kind. AI-assisted-by: glm-5.3-flash
This commit is contained in:
parent
27ccb32ea1
commit
c05a05b4c3
@ -591,6 +591,7 @@
|
||||
:offload-file-data (ig/ref :app.tasks.offload-file-data/job-def)
|
||||
:objects-gc (ig/ref :app.tasks.objects-gc/job-def)
|
||||
:export-binfile (ig/ref :app.tasks.export-binfile/job-def)
|
||||
:export-assets (ig/ref :app.tasks.export-assets/job-def)
|
||||
:import-binfile (ig/ref :app.tasks.import-binfile/job-def)
|
||||
:storage-gc-deleted (ig/ref ::sto.gc-deleted/job-def)
|
||||
:storage-gc-touched (ig/ref ::sto.gc-touched/job-def)
|
||||
@ -657,6 +658,11 @@
|
||||
::mtx/metrics (ig/ref ::mtx/metrics)
|
||||
::mbus/msgbus (ig/ref ::mbus/msgbus)}
|
||||
|
||||
;; the export of shapes/frames: the work runs on the external exporter
|
||||
;; worker, so this def carries no handler work and no components
|
||||
:app.tasks.export-assets/job-def
|
||||
{}
|
||||
|
||||
:app.storage.gc-deleted/job-def
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::sto/storage (ig/ref ::sto/storage)
|
||||
|
||||
@ -32,6 +32,7 @@
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.quotes :as quotes]
|
||||
[app.storage :as sto]
|
||||
[app.tasks.export-assets :as export-assets]
|
||||
[app.tasks.export-binfile :as export-binfile]
|
||||
[app.tasks.import-binfile :as import-binfile]
|
||||
[app.util.services :as sv]
|
||||
@ -44,6 +45,13 @@
|
||||
[:map {:title "create-export-binfile-job" :closed true}
|
||||
[:params export-binfile/schema:params]])
|
||||
|
||||
(def ^:private schema:create-export-assets-job
|
||||
"The shapes and frames frozen in the params, rendered as image files
|
||||
by the external exporter worker; the business params are what the
|
||||
`:export-assets` job-def declares."
|
||||
[:map {:title "create-export-assets-job" :closed true}
|
||||
[:params export-assets/schema:params]])
|
||||
|
||||
(def ^:private schema:create-import-binfile-job
|
||||
"The `.penpot` package the caller uploaded, imported into a project by
|
||||
the backend runner; the business params are what the `:import-binfile`
|
||||
@ -110,6 +118,18 @@
|
||||
;; EXPORT
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- check-file-permissions
|
||||
"The files the job will read, checked before anything is stored: a
|
||||
refusal arrives synchronously. The handler checks it again before
|
||||
reading, because a permission can be revoked while the job waits."
|
||||
[cfg profile-id file-ids]
|
||||
(when (empty? file-ids)
|
||||
(ex/raise :type :validation
|
||||
:code :no-files-to-export
|
||||
:hint "expected at least one file to export"))
|
||||
(doseq [file-id file-ids]
|
||||
(files/check-read-permissions! cfg profile-id file-id)))
|
||||
|
||||
(sv/defmethod ::create-export-binfile-job
|
||||
"Create a durable job that exports a set of files as a `.penpot`
|
||||
package.
|
||||
@ -134,13 +154,7 @@
|
||||
params (jobs/validate-params job-def (:params envelope))
|
||||
file-ids (:file-ids params)]
|
||||
|
||||
(when (empty? file-ids)
|
||||
(ex/raise :type :validation
|
||||
:code :no-files-to-export
|
||||
:hint "expected at least one file to export"))
|
||||
|
||||
(doseq [file-id file-ids]
|
||||
(files/check-read-permissions! cfg profile-id file-id))
|
||||
(check-file-permissions cfg profile-id file-ids)
|
||||
|
||||
(quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile
|
||||
::quotes/profile-id profile-id})
|
||||
@ -152,6 +166,40 @@
|
||||
:files (count file-ids)
|
||||
:export-type (d/name (:export-type params))}}))))
|
||||
|
||||
(sv/defmethod ::create-export-assets-job
|
||||
"Create a durable job that renders shapes and frames as image files.
|
||||
|
||||
Its params freeze the items to render (file, page, object, type,
|
||||
scale, name and the optional share id) the way the exporter receives
|
||||
them. The work runs on the external exporter worker, which consumes
|
||||
the queue the job-def names; the backend ships no runner for it.
|
||||
|
||||
The job is created, not run: the caller follows it by its id. The
|
||||
read permission of every file named in the items is checked here so a
|
||||
refusal arrives synchronously, and the handler checks it again before
|
||||
reading."
|
||||
{::doc/added "2.20"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:create-export-assets-job
|
||||
::sm/result schema:job-summary}
|
||||
[cfg {:keys [::rpc/profile-id] :as envelope}]
|
||||
(let [job-def (resolve-job-def cfg :export :export-assets)
|
||||
;; the RPC layer decoded the params against the job-def schema
|
||||
;; before the command ran
|
||||
params (jobs/validate-params job-def (:params envelope))]
|
||||
|
||||
(check-file-permissions cfg profile-id
|
||||
(into #{} (map :file-id) (:exports params)))
|
||||
|
||||
(quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile
|
||||
::quotes/profile-id profile-id})
|
||||
|
||||
(let [summary (get-job-summary cfg (submit-job cfg job-def :export-assets
|
||||
params profile-id))]
|
||||
(with-meta summary
|
||||
{::audit/props {:job-id (:id summary)
|
||||
:files (count (:exports params))}}))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; IMPORT
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
79
backend/src/app/tasks/export_assets.clj
Normal file
79
backend/src/app/tasks/export_assets.clj
Normal file
@ -0,0 +1,79 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.tasks.export-assets
|
||||
"The `:export-assets` job: the shapes and frames a caller froze at
|
||||
creation time, rendered as image files.
|
||||
|
||||
This is an EXTERNAL job: the work runs on the exporter worker, a
|
||||
process outside the JVM, so the job is created with `::jobs/queue-name
|
||||
:exporter` (read by the creation command; the substrate never looks at
|
||||
it) and the backend ships no runner for that queue. The handler here
|
||||
is a guard, not a worker: it is never executed under a correct
|
||||
configuration, because the only runner that could claim the job is
|
||||
external, and `claim-job` does not look at the queue. Should a wrong
|
||||
configuration ever route the job to a JVM runner, it fails visibly
|
||||
here instead of \"completing\" empty.
|
||||
|
||||
Progress vocabulary of the external worker (the substrate validates
|
||||
the shape, the worker owns the words): stages `:preparing`,
|
||||
`:rendering` and `:packaging`, with counters `objects` (one per
|
||||
exported shape) and `pages` (one per exported frame)."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.jobs :as jobs]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(def schema:export-item
|
||||
"One thing to render: where it lives (file, page, object), what to
|
||||
produce of it (type, scale) and what to call it. The same item shape
|
||||
the legacy `/api/export` surface received, frozen in the params of the
|
||||
job; the optional `share-id` marks the item as coming from a public
|
||||
share."
|
||||
[:map {:title "export-item" :closed true}
|
||||
[:file-id ::sm/uuid]
|
||||
[:page-id ::sm/uuid]
|
||||
[:object-id ::sm/uuid]
|
||||
[:type [::sm/one-of #{:png :jpeg :webp :svg :pdf}]]
|
||||
[:name ::sm/text]
|
||||
[:suffix {:optional true} ::sm/text]
|
||||
[:scale ::sm/number]
|
||||
[:share-id {:optional true} ::sm/uuid]])
|
||||
|
||||
(def schema:params
|
||||
"Business params of an asset export: the items to render, frozen when
|
||||
the job was created, plus the options that shape the output. There is
|
||||
no `wait` here: a job is followed by its id, not waited on."
|
||||
[:map {:title "export-assets-params" :closed true}
|
||||
[:exports [:vector {:min 1} schema:export-item]]
|
||||
[:name {:optional true} ::sm/text]
|
||||
[:skip-children {:optional true} ::sm/boolean]
|
||||
[:force-multiple {:optional true} ::sm/boolean]
|
||||
[:is-wasm {:optional true} ::sm/boolean]])
|
||||
|
||||
(defn- execute-export-assets
|
||||
"Plain handler, importable and testable without integrant.
|
||||
|
||||
It only raises: the work of this job belongs to the external worker.
|
||||
See the namespace docstring."
|
||||
[_context _params]
|
||||
(ex/raise :type :internal
|
||||
:code :no-handler-for-external
|
||||
:hint "this job runs on the external exporter worker"))
|
||||
|
||||
(defmethod ig/init-key ::job-def
|
||||
[_ _cfg]
|
||||
{::jobs/name :export-assets
|
||||
::jobs/family :export
|
||||
::jobs/resource-role :output
|
||||
;; the queue the creation command routes the job to: an external
|
||||
;; worker consumes it, the backend ships no runner for it
|
||||
::jobs/queue-name :exporter
|
||||
::jobs/schema schema:params
|
||||
::jobs/handler execute-export-assets
|
||||
::jobs/decoder (sm/decoder schema:params sm/json-transformer)
|
||||
::jobs/validator (sm/validator schema:params)})
|
||||
@ -179,6 +179,113 @@
|
||||
(t/is (= "detach-libraries" (:export-type props)))
|
||||
(t/is (= #{:job-id :files :export-type} (set (keys props)))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; ASSETS EXPORT
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- export-item
|
||||
[file-id & {:keys [page-id object-id type name scale suffix share-id]}]
|
||||
(cond-> {:file-id file-id
|
||||
:page-id (or page-id (uuid/next))
|
||||
:object-id (or object-id (uuid/next))
|
||||
:type (or type :png)
|
||||
:name (or name "shape")
|
||||
:scale (or scale 1)}
|
||||
suffix (assoc :suffix suffix)
|
||||
share-id (assoc :share-id share-id)))
|
||||
|
||||
(defn- create-export-assets-job
|
||||
[profile-id exports & {:as extra}]
|
||||
(th/command! (merge {::th/type :create-export-assets-job
|
||||
::rpc/profile-id profile-id
|
||||
:params {:exports exports}}
|
||||
extra)))
|
||||
|
||||
(t/deftest create-export-assets-job-answers-with-a-pending-job
|
||||
(let [profile (th/create-profile* 1)
|
||||
file-id (first (:file-ids (import-fixture! profile)))
|
||||
item (export-item file-id)
|
||||
before (count-storage-objects)
|
||||
out (create-export-assets-job (:id profile) [item])]
|
||||
|
||||
(t/is (th/success? out))
|
||||
|
||||
(let [result (:result out)
|
||||
job (jobs/get-job th/*system* (:id result))]
|
||||
|
||||
(t/testing "the answer is enough to follow the job"
|
||||
(t/is (uuid? (:id result)))
|
||||
(t/is (= "pending" (:status result)))
|
||||
(t/is (= "export-assets" (:name result)))
|
||||
(t/is (ct/inst? (:created-at result))))
|
||||
|
||||
(t/testing "and the row routes to the external worker queue"
|
||||
(let [job-def (jobs/get-job-def (::jobs/defs th/*system*) :export-assets)
|
||||
params (jobs/decode-params job-def (:params job))]
|
||||
(t/is (= "exporter" (:queue job)))
|
||||
(t/is (= 0 (:max-retries job)))
|
||||
(t/is (= (:id profile) (:profile-id job)))
|
||||
(t/is (= 1 (count (:exports params))))
|
||||
(t/is (= :png (:type (first (:exports params)))))))
|
||||
|
||||
(t/testing "creating the job stores no artifact"
|
||||
(t/is (= before (count-storage-objects)))))))
|
||||
|
||||
(t/deftest create-export-assets-job-checks-the-read-permission
|
||||
(let [owner (th/create-profile* 1)
|
||||
other (th/create-profile* 2)
|
||||
file-id (first (:file-ids (import-fixture! owner)))
|
||||
out (create-export-assets-job (:id other) [(export-item file-id)])]
|
||||
|
||||
(t/testing "a file the caller cannot read is not exported"
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :not-found (th/ex-type (:error out)))))
|
||||
|
||||
(t/testing "and no job was created for it"
|
||||
(t/is (zero? (count-jobs))))))
|
||||
|
||||
(t/deftest create-export-assets-job-rejects-an-unknown-type
|
||||
(let [profile (th/create-profile* 1)
|
||||
file-id (first (:file-ids (import-fixture! profile)))
|
||||
out (create-export-assets-job (:id profile)
|
||||
[(export-item file-id :type :psd)])]
|
||||
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :data-validation (th/ex-code (:error out))))
|
||||
(t/testing "and nothing was created"
|
||||
(t/is (zero? (count-jobs))))))
|
||||
|
||||
(t/deftest create-export-assets-job-adds-the-job-id-to-the-audit-props
|
||||
(let [profile (th/create-profile* 1)
|
||||
file-id (first (:file-ids (import-fixture! profile)))
|
||||
out (create-export-assets-job (:id profile)
|
||||
[(export-item file-id)])
|
||||
props (::audit/props (meta (:result out)))]
|
||||
(t/is (th/success? out))
|
||||
(t/testing "the audit of the call points at the job and its items"
|
||||
(t/is (= (:id (:result out)) (:job-id props)))
|
||||
(t/is (= 1 (:files props)))
|
||||
(t/is (= #{:job-id :files} (set (keys props)))))))
|
||||
|
||||
(t/deftest create-export-assets-job-counts-in-the-export-quote
|
||||
;; the quote of the family counts every job of it, so a mixed backlog
|
||||
;; of binfile and assets jobs fills the same quote
|
||||
(with-mocks [mock {:target 'app.config/get
|
||||
:return (th/config-get-mock
|
||||
{:quotes-export-jobs-per-profile 1})}]
|
||||
(let [profile (th/create-profile* 1)
|
||||
file-id (first (:file-ids (import-fixture! profile)))]
|
||||
|
||||
(t/is (th/success? (create-export-assets-job (:id profile)
|
||||
[(export-item file-id)])))
|
||||
|
||||
(t/testing "the next export job of the family is refused, whatever its type"
|
||||
(let [out (create-export-binfile-job (:id profile) #{file-id})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :max-quote-reached (th/ex-code (:error out))))))
|
||||
|
||||
(t/is (= 1 (count-jobs))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; IMPORT
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user