diff --git a/.gitignore b/.gitignore index 5e38fb2361..24a52dca38 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,7 @@ opencode.json /exporter/target /exporter/.shadow-cljs /exporter/resources/wasm/ -/exporter/src/app/wasm/shared.js +/exporter/src/exporter/wasm/shared.js /frontend/.storybook/preview-body.html /frontend/.storybook/preview-head.html /frontend/playwright-report/ @@ -99,7 +99,6 @@ opencode.json # render-wasm build artifacts copied into the exporter (autogenerated) /exporter/resources/wasm/render-wasm.js /exporter/resources/wasm/render-wasm.wasm -/exporter/src/app/wasm/shared.js /media-processor/dist/ /media-processor/target/ diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index 7a70eb4669..9431fe4d00 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -70,11 +70,54 @@ Every background job is a job-def: a plain `(defn execute-X [cfg params] ...)` i - The domain cfg of a binfile request lives in `app.binfile.common` (`export-cfg`, `import-cfg`), so the legacy RPC and the jobs freeze the same ids, resolve the same team features and apply the same anti-zip-bomb limits. `resolve-export-type` is the translation the legacy RPC needs for its deprecated boolean flags; the creation command asks the caller for the type instead of guessing a default. - Two retention knobs, easy to confuse: `:jobs-retention` sweeps terminal internal rows (the ones without a profile), while `:jobs-user-ttl` is the default expiry a user-facing job gets when it is created; the jobs GC deletes the row once `expires_at` passes and touches its resource. The heavy work of a user job runs on its own `:binfile` queue (`:worker-binfile-parallelism`, default 1) so it does not block the default runner. - The quote of a family of jobs (`::export-jobs-per-profile`, `::import-jobs-per-profile`) counts the jobs of the profile that are still on their way (`new`/`scheduled`/`running`/`retry`) whose name the registry gives for that family, so a new job of the family is covered without touching the quote. -- A user-facing job is created through `app.rpc.commands.jobs`. One command per job type: `create-export-binfile-job` and `create-import-binfile-job` name the job they create and their body carries only `:params`, whose schema the command imports from the job namespace (`::sm/params`), so the RPC validates the business params up front, before the command runs; the job-def is still the place that owns them and validates them again at run time. The commands check the permission and the quote of the family before the expensive part, submit with `max-retries 0`, an expiry from `:jobs-user-ttl` and the queue the job-def routes to through `::jobs/queue-name` (`:binfile` when the def says nothing; read by the command, never by the substrate), and answer with a summary of the row. The import command has its own creation schema (`import-binfile/schema:create-params`, version optional) because the request is not the stored params: the command fills the version (from the caller or the package header) and the manifest metadata kept for audit. It assembles the upload, reads the manifest of a version 3 package for the audit (never its content) and stores the package in `job-resource` as the resource of the job, releasing it when the job never comes to own it. +- The `:exporter` queue has no runner in the backend: its consumer is the exporter process (a separate Node worker; see `mem:exporter/core`). The backend serves it through `app.rpc.management.jobs`, shared-key authenticated (`X-Shared-Key: exporter[:] `, the key the exporter derives from the same secret with a per-tenant salt; the bare `exporter` id is the legacy form): `claim-job {job-id scheduled-at}` (exact `scheduled-at` match; a stale payload answers `{:action :skip :status ...}`; the row's params come back decoded with the job-def, so the worker gets typed values — a row whose def is gone answers the stored shapes), `report-job-progress {job-id progress {stage counters}}` (the worker owns the milestone words; the substrate validates the shape), `create-job-session {job-id}` (a real `http_session_v2` of the owner profile, so the assets fetch as the user; TTL `:job-session-ttl` default 1h), `complete-job` (the params form carries `result`/`resource-id`; the artifact form carries a multipart `content` part, the single source of `filename`/`mtype`, declared with `::rpc/multipart-only-params` so a transit body cannot name a server path — path confined by `media.v/check-upload-path`, allowlist `common.media/export-artifact-types`, cap `:exporter-max-result-size`; the checks run before any bytes are stored and a lost race stores nothing) and `fail-job {job-id error session-id}`. Every method answers `{:action :skip :status ...}` on a row it cannot write and reports the row status, and a settle named with `session-id` closes that render session whatever the write says — only when the session belongs to the job's `profile-id`. Every method also checks the row belongs to the calling exporter's tenant (suffix of a per-tenant key id, own tenant for the bare id): anything else answers a bare `{:action :skip}` (no status, no write, no session close), so no oracle leaks tenant rows. `report-job-progress` carries the tenant in the heartbeat write itself (`:tenant` on `jobs/heartbeat`, filtered in the touch and progress statements), so the hot path needs no row read; a filter that matches nothing raises `:interrupt` with `:reason :foreign-tenant`. The tenant is normalized to lowercase when the configuration is read (`PENPOT_TENANT=ACME` behaves as `acme`), and `app.setup.keys/derive-exporter-key` is the single definition of the exporter key on the backend (registry and middleware both call it). +- A user-facing job is created through `app.rpc.commands.jobs`. One command per job type: `create-export-binfile-job`, `create-export-assets-job` and `create-import-binfile-job` name the job they create and their body carries only `:params`, whose schema the command imports from the job namespace (`::sm/params`), so the RPC validates the business params up front, before the command runs; the job-def is still the place that owns them and validates them again at run time. The commands check the permission and the quote of the family before the expensive part — binfile checks the files it reads by id, and assets checks every item on its own: an item carrying `:share-id` validates through the share-link permissions the same way `get-file-fragment` does (a logged viewer of the share is enough; the command stays authenticated, so the anonymous visitor is refused, the same as the exporter cookie it replaces) —, submit with `max-retries 0`, an expiry from `:jobs-user-ttl` (or ten minutes when the creation says `:transient`, the single-use exports of the clipboard and the Plugin API) and the queue the job-def routes to through `::jobs/queue-name` (`:binfile` when the def says nothing; read by the command, never by the substrate), and answer with a summary of the row. Assets params carry an optional `:kind` (`:shapes`/`:frames`, declared by the frames dialog; absent means shapes): the worker routes on it instead of inferring the export from the items. The import command has its own creation schema (`import-binfile/schema:create-params`, version optional) because the request is not the stored params: the command fills the version (from the caller or the package header) and the manifest metadata kept for audit. It assembles the upload, reads the manifest of a version 3 package for the audit (never its content) and stores the package in `job-resource` as the resource of the job, releasing it when the job never comes to own it. - `get-job` is the read surface a profile has over its own jobs: it answers with the same summary the creation commands return plus the `result` of a finished job or its public error (through `decode-job-error`, so the keywords come back), and a job of another profile is not distinguishable from one that does not exist: both answer `not-found`. `decode-row` decodes `params`, `result` and `error`, and the result is then decoded with the decoder the substrate compiled from the result schema the job-def the row names declares (`decode-result`; a def without one, or a row whose job-def is gone, keeps the plain JSON the row holds: a uuid as the text it is on disk, a uuid used as a map key as a keyword). The event log still has no read surface: a client that follows a job takes its progress from the websocket. - `heartbeat` uses named options `[cfg & {:keys [job-id progress] :as options}]` and returns the number of durable writes, so 0 means there was nothing to write: no job context, or the throttle did not allow it. A write that finds the job no longer active (terminal or gone) raises an `:interrupt` (`code :job-interrupted`, carrying the job id and the status it found) through the shared `jobs/check-active` instead of counting zero, so a task in course stops at its next beat; the interrupt is internal and never reaches HTTP (the management progress report answers `:skip` for it). The job id falls back to `::jobs/job-id` on the cfg and then to the runner-bound `*job-id*`. The `modified_at` touch is throttled to ~1s and the progress event to ~200ms, so the check fires at most once per second with no extra query (the read of the row happens only when a write already found nothing). A progress report is a milestone (`schema:progress`): a required `:stage` keyword plus an optional `:counters` map of `scope -> {:current, :total?}`, and no other key. Counters are non-negative, a `total` is positive and never lower than its `current`, and stage and counter keys are names of at most 64 characters. A milestone without counters is valid (a point of the run with no units to count), the stage vocabulary belongs to the worker and not to the substrate, and a client must degrade for a stage it does not know. Read a stored payload back with `jobs/decode-progress`: the database holds the keywords as strings. The 1s beat (down from 60s, so a cancelled heavy job is seen as gone at its next beat) costs one `modified_at` touch per beating job per second: cheap while the only heavy beater runs alone on its queue, revisit if concurrent beating jobs ever grow. - `job.error` conforms to `app.jobs/schema:job-error` (`type` and `code` keywords, `hint` text, extra details allowed). Read it back with `jobs/decode-job-error`; a plain `db/decode-json-pgobject` leaves the keywords as strings. `complete` takes named options and an optional `resource-id`, which is only ever set, never replaced: it reads the row locked inside its own transaction and applies the rules in a fixed order, skipping a job that is no longer completable before refusing one for its resource, so the answer is a function of the row and not of the order two callers arrive in. It checks the result against the schema the job-def of the row declares before it writes (a violation raises `:invalid-result` and nothing is stored), and only for a transition that is going to write: a terminal job answers `:skip` without validating. `complete` is the single door of a result, so the check covers the runner and the external worker that completes a job through the management RPC (`::complete-job`), which stores a result the substrate would otherwise take blind. +## The wake-up contract of a worker queue (public surface) + +What an out-of-JVM process needs to learn about work WITHOUT reading the +dispatcher's guts. Verified against `app.worker/queue-key`, +`encode-payload` (`app.worker.dispatcher`) and the runner's local +`decode-payload`; a change there is a change of this contract. + +- The key: `penpot.worker.queue::` (`app.worker/queue-key`). + `job.queue` stores the bare name, so the prefix is composed at use. The + tenant prefix keeps instances sharing one Redis from handing each other + jobs. +- The payload: a JSON array of exactly two items, `["", + ""]`, the instant formatted by `ct/format-inst`. Fewer + than two is not a payload; the runner's decoder reads exactly those + two and drops and logs anything it cannot parse as uuid + instant. +- Dispatcher order: the rows are marked `scheduled` FIRST, pushed LAST + (RPUSH). A crash between the two leaves `scheduled` rows that + `reschedule-lost-jobs` re-queues; the reverse order would leave a + duplicate payload nothing deduplicates. +- The `scheduled-at` in the payload must be handed back AS-IS on `claim-job`: + the claim demands an exact match and answers `{:action :skip :status ...}` + for a stale payload. That is what makes a claim idempotent and what turns + the queue into a mere notification: a pop that is not confirmed by a claim + is not work. +- The queue is not the truth: the management API is (`claim-job`, + `report-job-progress`, `complete-job`, `fail-job` in + `app/rpc/management/jobs.clj`), shared-key authenticated by the route + resolver. Everything a consumer decides about a job goes through it. +- The queue has NO owner: anyone with Redis access can BLPOP it. Operational + rule that follows: do NOT wire a JVM runner for a queue an external + process consumes — the dispatcher hands each payload to whoever pops, so + both would start taking the same jobs. +- Cancellation travels the beats: `report-job-progress` answers + `{:action :skip}` once the row stopped being active — the only signal that + distinguishes "cancelled" from "network down". Same mechanism as the + internal `heartbeat` `:interrupt` (see the heartbeat note above); do not + build a second channel. +- Not part of this contract: `app.jobs/request`/`reply` (the ephemeral lane; + in retirement, `P1/F5/T16`). A comment in `app/jobs.clj` (2026-09-17) saying + that lane "waits for a producer" predates that decision and must not guide + anything here. + For worker dispatch, cron, retry semantics (`ex/raise :type ::wrk/retry` with `:delay`/`:strategy`), deduplication, job events, and queue internals: `mem:backend/subtleties`. ## Jobs metrics diff --git a/.serena/memories/clojure/promesa.md b/.serena/memories/clojure/promesa.md index 75cb5f6642..8ff6fc4f0f 100644 --- a/.serena/memories/clojure/promesa.md +++ b/.serena/memories/clojure/promesa.md @@ -1,9 +1,78 @@ -# Promesa +# Promesa (funcool, 12.0.1) — verified API shapes + +The promise lib of the monorepo: JVM (backend, `promesa.exec` thread +pools) and CLJS (frontend, exporter). Declared in `common/deps.edn`; +heavy users: `frontend/src/app/main/**` (Rx interop), `exporter/src/app/**` +(management client, auth), `backend/src/app/storage/s3.clj` and +`worker/cron.clj` (JVM await paths). Doc: `doc/promises.md` upstream. + +## Two API families (verified against the 12.0.1 jar) + +Argument order is inconsistent BY DESIGN; each op belongs to one of +two families: + +**promise-first `([p f ...])` — pairs with `->` threading** (the + promise arrives as the threaded first argument): +- `then`, `then'`, `bind`, `handle`, `finally`, `catch` (`([p f])`, + 2-arity with predicate: `([p pred-or-type f])`), `chain` + `([p f & fs])`. + +**fn-first `([f p ...])` — pairs with `->>` threading** (the promise + arrives as the threaded LAST argument): +- `map`/`fmap`, `mapcat`/`mcat`, `hmap`/`hcat`, `fnly`, `merr`. + +Every 3-arity in a family inserts its executor FIRST for the fn-first +family (`([executor f p])`) and LAST for the promise-first family +(`([p f executor])`). + +**What `f` must return:** +- `then`/`catch`/`fmap`/`hmap`: plain value OK — auto unwrapping. +- the bind-style ops `mcat` and `merr`: **`f` MUST return a promise + instance** (`merr` may also throw). A plain value does NOT get + wrapped: the impl's `fbind` pushes a BIND task that demands a + thenable and crashes with `TypeError: expected thenable` + (first-hand crash in the exporter worker, 2026-10-07). + +`finally`/`fnly`: run `f`, mirror the original promise value. + +## promesa's own -> / ->> macros (do NOT confuse with clojure.core) + +`p/->` and `p/->>` exist in `promesa.core` (12.x). Their purpose is NOT +arg positioning: they expand to a `chain` over `then` and REALIZE the +promise at each step, so every form receives the realized value +(e.g. `(p/-> (js/fetch ...) .-body .-status)`). + +## Repo convention + +**Prefer `->>` composition** over the map family (fn-first ops); it is +how new promise code should read: + + (->> (fetch ...) + (p/fmap (fn [v] ...)) + (p/mcat (fn [v] (p/...)))) + +A `p/do` terminator inside `->>` receives the threaded promise as its +LAST expression — it runs AFTER the earlier exprs, so it can never act +as the chain's continuation; terminate by wrapping the chain in an +explicit `(p/do chain ...)` (`p/let` avoids the same trap). + +Legacy code threads promises with `->` over the promise-first +family (`(-> p (p/then ...))`, `(p/then p f)` calls). Read the args in +that order when touching it; do not rewrite it unless the task touches +that code anyway. + +For JVM blocking code, `promesa.exec/await!` exists but is deprecated +since 12.0.0 → `p/await`. + +## Async test gotcha (CLJS) + +An async cljs.test var only proves its chain when the chain is really +awaited: a broken link outside the `p/let`/`p/do` body ends silently as +"Ran N tests containing 0 assertions" with 0 failures. If a test passes +with 0 assertions, the chain broke upstream; add a print in the last +link (`CHAIN RAN`) to prove plumbing before trusting green. + +## Misc -Promise library of the monorepo (frontend CLJS and common, `funcool/promesa` 12.0.1 in `common/deps.edn`). Its API has two call styles and mixing them in one expression is the trap: `fmap`/`mcat`/`merr` are **fn first, promise last** (`->>` friendly) and `then`/`then'`/`catch`/`handle` are **promise first, fn last** (`->` friendly). Convention: `fmap`/`mcat`/`merr` inside `->>` pipelines, `then`/`catch` with `->`. A `then` written inside a `->>` reads backwards and hides whether the step carries a value or a promise. -- `fmap` (alias `map`, both `(f p)`) is functor map: `f` returns a plain value. -- `mcat` (alias `mapcat`, both `(f p)`) is monadic bind: `f` must return a promise, and the result mirrors it. -- `then` also unwraps a promise returned by `f`, which is why it covers both cases at the cost of not naming which one it is; `then'` maps without unwrapping. -- `merr` (fn first) and `catch` (promise first) are the error side. - `all` takes a collection of promises (a lazy seq works); `let` chains sequential bindings. - Seam with the rx pipelines of the frontend: `beicon.v2.core/from` builds an observable from "arrays, collections, and promise instance" (`external/beicon`), so `(->> p (p/mcat f) (rx/from) (rx/map g))` is the usual bridge (`app.main.data.imports`, `app.util.zip`). diff --git a/.serena/memories/common/core.md b/.serena/memories/common/core.md index 816d131e14..3e82207713 100644 --- a/.serena/memories/common/core.md +++ b/.serena/memories/common/core.md @@ -10,6 +10,7 @@ - `app.common.logic.*`: higher-level workflows/algorithms over files, shapes, components, variants, libraries, tokens, etc. - `app.common.geom.*`: geometry helpers and transformations. - `app.common.schema` / `app.common.schema.*`: Malli abstraction layer. +- `app.common.export`: asset-export job item/params schemas, the single source of truth the backend job-def and the exporter worker validate and decode against. - `app.common.math`, `app.common.time`, `app.common.uuid`, `app.common.json`, etc.: cross-runtime utilities. - `app.common.test_helpers.*`: test builders and production-path helpers. diff --git a/.serena/memories/critical-info.md b/.serena/memories/critical-info.md index 28e1043431..02dc149207 100644 --- a/.serena/memories/critical-info.md +++ b/.serena/memories/critical-info.md @@ -8,7 +8,7 @@ You are working on the GitHub project `penpot/penpot`, a monorepo. - Cross-cutting testing principles, TDD workflow, and anti-patterns: `mem:testing`. - Verified Clojure language behaviors that contradict common assumptions (e.g. `int?` covers `Long`; `integer?` is the general predicate): `mem:clojure/idioms` — read before assuming stdlib predicate semantics. - Read `mem:clojure/design-rules` (function shape, boundary checks, naming) before writing or modifying Clojure/ClojureScript code — not only when designing a solution or an API. -- Promises: which call style to use where (`fmap`/`mcat`/`merr` vs `then`/`catch`) and how they bridge to rx: `mem:clojure/promesa` — read before chaining promises or mixing them with observables. +- Promise code (JVM + CLJS): promesa's two API families and their arg orders (fmap/mcat fn-first, then/catch promise-first, `mcat` f must return a promise), the repo `->>` preference, and async-test silent-green traps: `mem:clojure/promesa` — read before writing or reading promise chains. # Development workflow diff --git a/.serena/memories/devenv/core.md b/.serena/memories/devenv/core.md index 3507a5d6e0..434123b30e 100644 --- a/.serena/memories/devenv/core.md +++ b/.serena/memories/devenv/core.md @@ -76,7 +76,7 @@ No `--delete` on the working-tree pass: gitignored caches in the workspace survi - `run-devenv-shell [--instance 0|wsN|N] [cmd...]`: bash in target instance. (`--instance` flag not yet renamed to `--ws`.) - `start-devenv` / `log-devenv` / `drop-devenv`: legacy paths around ws0 + shared infra. `drop-devenv` never removes volumes. -`exporter/scripts/run` and `wait-and-start.sh` source `backend/scripts/_env` then `_env.local` if present. +`exporter/scripts/run` and `wait-and-start.sh` source `backend/scripts/_env` then `_env.local` if present, and default `PENPOT_INTERNAL_URI` to the in-container nginx loopback (`http://localhost:4449`, `:-` so an explicit value wins). The loopback is invariant across instances, so it is defined at the exporter start scripts — the only in-container consumer — not in compose or `_env`. ## Image and dependency pins diff --git a/.serena/memories/exporter/core.md b/.serena/memories/exporter/core.md index f48948d8d6..0d4b2fd7ba 100644 --- a/.serena/memories/exporter/core.md +++ b/.serena/memories/exporter/core.md @@ -1,36 +1,77 @@ # Exporter Architecture and Workflow -`exporter/`: CLJS/Node headless export service. Depends on `common/`; uses Playwright plus export JS/CLJS deps for SVG/PDF/assets. +`exporter/`: CLJS/Node **worker process** that consumes the backend's +`:exporter` job queue and renders shapes, frames and files to bitmap, SVG and +PDF. Depends on `common/`; uses Playwright plus export JS/CLJS deps for +SVG/PDF/assets. There is no HTTP surface of its own and nothing is persisted +on this side: the row of the job, its cancel and its result are the backend's. ## Layout and commands -- Source: `exporter/src/`; config: `deps.edn`, `shadow-cljs.edn`, `package.json`; runtime helpers/assets: `vendor/`, `scripts/`. -- `exporter/resources/wasm/render-wasm.{js,wasm}` and `exporter/src/app/wasm/shared.js` are generated render-wasm build output, not source: they are gitignored. Never hand-edit or commit them; regenerate by rebuilding render-wasm. -- From `exporter/`: setup `./scripts/setup`; watch `pnpm run watch` or `pnpm run watch:app`; production build `pnpm run build`; test bundle `pnpm run build:test`; tests `pnpm run test` or `pnpm run test:quiet`; lint `pnpm run lint:clj`; format check/fix `pnpm run check-fmt:clj` / `pnpm run fmt:clj`. +- Source: `exporter/src/` (one tree, `exporter.*`; the legacy `app.*` tree is + gone); config: `deps.edn`, `shadow-cljs.edn`, `package.json`; runtime helpers/assets: `vendor/`, `scripts/`. +- `exporter/resources/wasm/render-wasm.{js,wasm}` and `exporter/src/exporter/wasm/shared.js` are generated render-wasm build output, not source: they are gitignored. Never hand-edit or commit them; regenerate by rebuilding render-wasm. +- From `exporter/`: setup `./scripts/setup`; watch `pnpm run watch` or `pnpm run watch:app` (both build render-wasm first); run what watch built with `pnpm run start:dev` (dev env via `scripts/run`; changes arrive through shadow hot-reload while `watch` runs) or `pnpm run start` (env as-is, same as the container `CMD`); production build `pnpm run build` (wasm + shadow release, same chain `scripts/build ` uses for the release bundle before stamping the version and generating `setup`); test bundle `pnpm run build:test`; tests `pnpm run test` or `pnpm run test:quiet`; lint `pnpm run lint:clj`; format check/fix `pnpm run check-fmt:clj` / `pnpm run fmt:clj`. `./scripts/ci exporter` in the repo root is green (test-quiet defaults `PENPOT_SECRET_KEY` like `test` does). - Because exporter consumes `common/`, shared file/shape/model changes may need exporter verification even when the immediate change is not under `exporter/`. - Cross-cutting testing principles and anti-patterns: `mem:testing`. - Exporter test conventions and CI: `mem:exporter/testing`. +- No promesa chains in `src` (only raw JS promise interop where a JS API demands + it); async is `^:async` + `await`. Promise arg orders and the async-test + silent-green trap: `mem:clojure/promesa` — read before touching async code. -## HTTP and browser pool +## The worker anatomy -- POST body limit is about 60 MB. Exporter supports `application/transit+json`; request params merge query params and body params. -- Map response bodies are Transit JSON and force HTTP 200; nil 200 bodies become 204. -- Auth token comes from cookie `auth-token`, then uploads use Bearer auth plus the management shared key. -- Each export job gets a fresh Playwright browser context. On success, the context closes and the browser returns to the pool; on error, the browser is destroyed instead of reused. -- Borrow validates browser connection. Pool acquire timeout is about 10s; font loading timeout logs a warning and continues after about 15s. +- The queue wake-up contract (key shape, the two-item payload, mark-then-push, the claim-back rule) is the backend's public surface now: `mem:backend/core`, "The wake-up contract of a worker queue". This process was its first external consumer; read it before writing another. +- Main thread boots pools + K pollers (`exporter.main`): browser pool, wasm pool, temp area (`:exporter/tmpdir`, which also cleans what a previous process left on its init), then `consumer.worker/start`. Shutdown: SIGTERM/SIGINT run `shutdown` (the worker stops intake first, in-flight runs drain within their grace period, then the system halts in reverse dependency order) before exiting; `stop` alone halts the same way without exiting. +- K pollers (`exporter.consumer.worker`): one Redis connection per poller (connection = slot + backpressure), BLPOP on `penpot.worker.queue::exporter`, payload `[job-id scheduled-at]` JSON; corrupt payload warns and drops, dead Redis delays 1s and retries. `K = PENPOT_EXPORTER_WORKER_CONCURRENCY` (default 2, floored at 1); replicas × K is the total concurrency. Pools size themselves to K (both read `consumer.config/concurrency`); there is no independent pool-max config anymore, and the wasm pool takes `:min` (`PENPOT_WASM_WORKER_POOL_MIN`, clamped to the max) plus `:idle-timeout` (`PENPOT_WASM_WORKER_IDLE_TIMEOUT`, seconds turned into milliseconds). The other legacy pool/queue keys were deleted: nothing reads them. +- Management client (`exporter.consumer.api`): claim/progress/create-job-session/complete-job-with-artifact (multipart via undici FormData)/fail-job, POST to `/api/management/methods/` with `X-Shared-Key: exporter: ` (key = `PENPOT_EXPORTER_SHARED_KEY` or HKDF-derived from the secret with a per-tenant salt; the backend only lets the id touch its own tenant's jobs); transit bodies, non-2xx errors carry `:status` and the error body. One module-level undici `Agent` (TLS posture unchanged, closed in `shutdown` via `close-client!`) and a per-call `AbortSignal.timeout`: 10s for claim/beat/session, 120s for the artifact upload, so a hung backend cannot pin a poller slot. `fetch` is a plain def seam for hermetic tests. +- Runner (`exporter.consumer`, `run-export [render tmpdir claim params]`): mints a render session (`create-job-session`), plans via `make-plan` (single → artifact of its type; multi → zip; frames → pdf via pdfunite), settles via `complete-job-with-artifact` multipart or `fail-job`; the settle never rejects. `exporter.consumer.plan` holds the render-plan pieces (name transducers, partition of 50, grouping by `[scale type]`), dispatching on `exporter.renderer/wasm?`. The job kind is declared (`:kind` in the frozen params), never sniffed. +- Beats: per-object milestones `:preparing/:rendering/:packaging` with `objects`/`pages` counters, throttled at 250ms; a watchdog repeats the last milestone every 1s. +- Beats are serialized (`consumer/enqueue-beat`): one report in flight, the next waiting, so the watchdog repeating the last report can never overwrite a newer milestone. The chain is safe because every call carries a timeout. +- Cancellation: a `skip` answer on any beat means the row is terminal (cancelled, aborted, settled). The runner raises `:job-cancelled` between units of work; mid-Skia the per-lease interval fires the SharedArrayBuffer signal and terminates the leased worker thread. The runner owns the local registry entry via `exporter.jobs` and releases it on settle. +- `exporter.jobs` is the **local cancel arm only**: one flag per claimed job (`register`/`mark-cancelled` returning true/nil/`cancelled?`/`release`) plus one zero-arg check constructor per job (`check-cancelled`, raising `:job-cancelled`). No terminate callbacks and no shared-array signal travel through here — the abort belongs to whoever holds the render (the per-lease interval of `exporter.wasm.scope` terminates its own worker). +- Temp files: `exporter.tmpdir/track`/`release` per job id; the `:exporter/tmpdir` init cleans orphans (aged `:job-ttl`). No redis-side store, cancel topic or abandoned-job sweep: an unclaimed-in-time row goes `aborted` by the backend's own lease GC. +- Run artifacts: `exporter.resources` (`create [tmpdir type name]`, `create-zip`/`add-to-zip`/`close-zip`); no per-file deletion timers — the tmpdir registry owns every path. -## Export batching and async behavior +## Running it (devenv and deploy) -- `prepare-exports` groups entries by `[scale type]` and partitions groups into chunks of 50. Each partition uses file/page/share/name from its first item, so be careful if entries might cross those boundaries. -- Single-export response is used only when multiple export is not forced and there is exactly one prepared export containing exactly one object. -- Multi-object export can run async: when `wait` is false it returns a resource immediately and publishes progress/end/error to Redis by profile topic; when `wait` is true it waits for upload and returns the uploaded resource. -- Frame export returns a resource immediately and publishes Redis updates; it does not follow the same `wait` option path. -- ZIP entry names are sanitized and duplicates receive numeric suffixes. +- Devenv: the tmux `exporter` window (`./scripts/watch` + `wait-and-start.sh`) IS the worker now — no separate window, no role env. It needs the backend and valkey of the devenv up; `watch:app` rebuilds with the same shadow build the `main` build uses. +- Deploy: the compose service is one `penpot-exporter` (the same image, `CMD ["node", "app.js"]`); scale by replicas, K is per process. Nginx has no `/api/export` route anymore: the frontend talks to the backend's RPC only (`create-export-assets-job`), and downloads ride `/assets/by-id/?share-id=`/auth of the job resource. +- Useful beats while debugging: the boot log names `:workers K`; each poller logs `running job`/`job settled`; the runner logs `export job settled` with `:outcome`, and a cancel logs `job cancelled by backend` on the watchdog. + +## The tree + +- Lifecycle is `exporter.utils.system`: async-aware Integrant-style init/halt (`init-key`/`halt-key` multimethods, `ref`s, `weavejester.dependency` ordering, `^:async` + `cljs.core/await`, no promesa chains); tests are `^:async deftest` with `await` directly in the body. +- `exporter.main` mirrors the backend's `app.main`: a `system-config` def reading env at the wiring layer, a public `system` atom with the running map, `start`/`stop`/`restart` (`^:async`, resolving `:started`/`:stopped`/`:restarted`), a thread guard in `start` (a render worker thread runs this same bundle and boots the `exporter.wasm.worker` engine instead of the system — without it a warmed pool worker would nest pools and pollers and never post `ready`), `start-custom` as the test seam, idempotent `install-process-handlers` for the `uncaughtException`/SIGTERM/SIGINT wiring (signals drain through `shutdown`, whose `exit` is a test seam; runs on `start`, never on namespace load), `^:dev/before-load-async`/`^:dev/after-load-async` hooks bridging the promise lifecycle to shadow's `done` callbacks (a hook failure logs, never sticks the reload), and `main` as the entry point: shadow boots `exporter.main/main`. The entry leaves info logging enabled for the new tree (`setup!` on the `exporter` logger); without it the default level filters everything below fatal. +- Service config maps take plain (non-namespaced) param keys (`{:exporter.browser/pool {:max 2}}`): the top-level key already scopes the component, so namespacing the params only adds noise. Component keys and error reasons stay namespaced. +- `exporter.config` is the process configuration: `PENPOT_`-prefixed env over defaults, malli-decoded and validated; `get`/`get-internal-uri`/`management-key` (HKDF)/`flags`/`version`. `prepare-config` over an explicit env object is pure and throws `:invalid-config` (tests exercise it); only the wiring entry `load-config!` reads the live environment and exits on it. +- `exporter.browser` is the browser pool service: the running instance is the raw pool, the public API takes it first (`exec` fails fast on a nil pool), the browser factory is injected through config for hermetic tests. +- `exporter.wasm.pool` follows the backend `db/run!` shape: `run [cfg f & args]` resolves the pool from `cfg` (the `:exporter.wasm/pool` service instance, or a raw pool), acquires a worker, applies `f` to the cfg carrying it under `:exporter.wasm.pool/worker` plus any extra args, and returns the worker — or destroys it when `f` fails, like a rollback. Pure pool mechanics only (`init`/`halt`, `terminate`, `await-ready`); the render protocol lives one level up. +- `exporter.wasm.scope` owns the lease: `run [cfg on-scope]` lends one worker for the whole batch (serializer, per-lease signal buffer, mid-render abort interval), and the public low-level `render-on [cfg params]` renders on the leased worker — asserting `:worker-not-leased`/`:timeout-ms-missing` (`:assertion`, never a silent default: a bare `::key` resolves to the CURRENT ns, never to an alias, so cross-ns keys are literals or `::alias/key`) — with the rest preconfigured and namespaced by setter (`:exporter.renderer/on-object` from the caller, `:exporter.wasm.scope/cancel-buffer` from the lease); the check is mandatory at the lease entry and runs here when configured. +- `exporter.renderer.browser` merges the bitmap/pdf/svg flows into one `render [cfg params on-object check-cancelled]` (branching by type, `on-object` always awaited, unknown types raise); `cfg` carries what the job params do not freeze (`:exporter.browser/pool`, `:base-uri`, `:public-uri`, `:svgo?`). +- `exporter.browser.scope` owns the browser batch: `run [cfg exports]` fires every export through its own checkout, all at once, bounded by the pool. No lease — a browser batch shares nothing but the pool, so fan-out only. The facade validates once, partitions by backend and joins both scopes. +- The wasm driver is `exporter.wasm.scope/run`, called from the facade; there is no `exporter.renderer.wasm` wrapper. +- `exporter.renderer` is the malli-validated facade with one entry point, the `render-fn` closure over `cfg`: it validates the task map (`:exports :on-object :check-cancelled`) and dispatches it through `render [cfg task]`, which assumes a valid task and only partitions and joins — the browser exports through `exporter.browser.scope`, the wasm ones through one leased worker — resolving nil once every export landed (objects travel through `on-object`). The `:exporter/renderer` service builds the closure from pools plus tmpdir (refs) and static config (`:base-uri`, `:public-uri`, `:management-key`, `:image-cache-size`, `:svgo?`). The facade merges the render env (`::wrender/env`) into every wasm export's params before the lease: the worker thread reads no config and no globals. The consumer renders through the injected fn and never requires the facade. `cfg` is task-free infrastructure with process lifetime (pools, uris, key, flags, tmpdir); everything per-task rides in the validated task map. One closed schema validates data plus injections, so a bad task — or a misspelled key — never touches a pool; task validation raises `:data-validation`. Each driver takes the same `[cfg params on-object check-cancelled]` shape (`headless?` is now `wasm?`: the browser also renders headless, so the old name lied) and implements cancel internally: cooperative checkpoints on the browser, checkpoints plus a mid-render abort of its own leased worker on wasm. The lease lives one level down, in `exporter.wasm.scope/run`, which the facade calls with the task keys set once in `cfg`; the browser batch goes through `exporter.browser.scope/run [cfg exports]`. +- `exporter.consumer.worker` is the queue poller (`read-payload`, `process`, `poll-once`, `start`/`stop` with a running flag, no promesa): K connections over the `:exporter` queue, claim → `run-export` through the injected `:renderer` and `:exporter/tmpdir` → settle before the next pop. The worker config carries the queue knobs plus both by ref; no render statics nest inside it. `stop` quits every connection and ends every loop; the temp area boots clean through the tmpdir init. +- `exporter.consumer.config` holds the consumer knobs (`concurrency` floored at 1, `queue-key`, `queue-name`): the runner, the worker and `exporter.main` read it. +- `exporter.consumer.api` is the backend management client (claim/progress/session/settle, transit bodies, multipart artifact), with the same `fetch` def seam for hermetic tests. +- `exporter.wasm` is the headless render-wasm driver under Node (artifact load + headless boot, font provisioning, shape rendering); `exporter.wasm.enums` binds the generated `shared.js` the `render-wasm/build export` step emits next to it (not committed). `exporter.wasm.text`/`serialize` own the browser-free call sequencing over the shared serializers. +- `exporter.wasm.render` is the headless pipeline (fetch scene → serialize → provision fonts/images → relayout → render each object) as `[env params on-object]`: `env` carries `:internal-uri`, `:public-uri`, `:tmpdir`, `:image-cache-size`, and `fetch` is a plain def seam. Asset fetches (images `by-file-media-id`, team fonts `by-id`) carry the item's `:share-id` through `app.common.uri/append-query-param`, so an export started from a share finds its images and fonts; the render headers carry no `X-Shared-Key` (those routes authenticate the session, never the worker). `exporter.wasm.worker` is the worker-thread entry: it splits `::render/env` off the posted params (asserting `:env-missing`), assocs the cancel fn from the shared buffer, runs the pipeline posting `object`/`done`, and posts `error` instead of ever rejecting. +- `exporter.main` wires both pools (`:exporter.browser/pool` and `:exporter.wasm.pool/pool`, each sized to the worker concurrency) plus `:exporter/tmpdir` (path + job ttl from env) plus `:exporter/renderer` (pools and tmpdir as `ref`s, static uris/key/flags) plus `:exporter.consumer/worker` (`:renderer` and `:exporter/tmpdir` by ref, `:concurrency`, `:queue-key`). Service keys follow the domain (`:exporter.wasm/pool`, `:exporter.consumer/worker`), written `::alias/name` where an alias covers them. No test boots the production wiring: the wasm pool warms one worker eagerly and the worker script in a test process would be the test bundle itself; the wiring is asserted as data and the lifecycle rides fake configs. +- Async test vars prove their chain only when really awaited: a test passing with 0 assertions means the chain broke upstream. ## Render details +- Headless engines (wasm/Skia) lease one render worker for the whole run (`scope/run`); browser renders go one DOM page per partition of 50. +- Each export gets a fresh Playwright browser context. On success the context closes and the browser returns to the pool; on error the browser is destroyed instead of reused. Borrow validates the connection; pool acquire timeout about 10s; font loading timeout logs a warning and continues after about 15s. - Bitmap export differs for WASM vs non-WASM render paths: WASM forces Playwright `deviceScaleFactor` to 1 and passes scale through the render URL; non-WASM uses `deviceScaleFactor = scale`. - WebP is produced by taking a PNG screenshot and converting it with ImageMagick. - SVG export rasterizes text foreignObjects to PNG, converts through PPM/color masks/potrace, and reassembles SVG paths. It also replaces non-breaking spaces for SVG compatibility and drops empty defs/paths. - PDF export injects `@page` sizing through raw browser `evaluate` JavaScript; that code cannot rely on CLJS runtime helpers. -- Temporary resources schedule local deletion, then uploads POST to `/api/management/methods/upload-tempfile` with `X-Shared-Key: exporter ` and Bearer auth. +- ZIP entry names are sanitized and duplicates receive numeric suffixes. + +## On the render engines (do not lose in refactors) + +- The wasm engines run on worker threads; `../render-wasm/build export` must run first: it emits the artifact into `exporter/resources/wasm` and `src/exporter/wasm/shared.js` (both uncommitted) before compiling. +- A render thread cannot read a flag: the cancel reaches it through the SharedArrayBuffer signal, and a render that never answers is killed by the silence watchdog of `exporter.wasm.scope` (the same mechanism a hard cancel rides on). +- `exporter.main/start` decides what boots by thread: the main thread consumes the queue, a render worker (`exporter.wasm.worker/main`) renders. diff --git a/.serena/memories/exporter/testing.md b/.serena/memories/exporter/testing.md index 189c1e852c..ab066cca6d 100644 --- a/.serena/memories/exporter/testing.md +++ b/.serena/memories/exporter/testing.md @@ -12,5 +12,6 @@ - Focus a test var with `node target/tests/test.js --focus exporter-tests.renderer-svg-test/creates-the-correct-gradient-element`. - Set app log level by appending `--log-level warn` (or `trace|debug|info|warn|error`). - `test:quiet` accepts forwarded options but rebuilds the bundle; prefer the direct runner after `build:test` for focused runs. +- The direct runner and `test`/`test:quiet` default `PENPOT_SECRET_KEY=test-secret-key` when the environment says nothing: the config needs a secret to boot. - From `exporter/`: `pnpm run check-fmt:clj` checks ClojureScript formatting. - From `exporter/`: `pnpm run lint:clj` runs ClojureScript linting. diff --git a/.serena/memories/frontend/core.md b/.serena/memories/frontend/core.md index 9afe4e20a7..2f7a7bb2ea 100644 --- a/.serena/memories/frontend/core.md +++ b/.serena/memories/frontend/core.md @@ -5,7 +5,7 @@ Frontend: CLJS SPA; React/Rumext; Potok; RxJS; okulary refs; SCSS modules; share ## Stable namespace map - `app.main.ui.*`: Rumext/React UI components for workspace, dashboard, viewer, settings, auth, nitrate, etc. -- `app.main.data.*`: Potok event handlers and side effects. +- `app.main.data.*`: Potok event handlers and side effects. Exports live in `app.main.data.exports.*`: assets (shapes and frames) and files (binfile) both run as jobs of the jobs substrate (`create-export-*-job` + `dj/watch-job` on the websocket; the widget `ui.exports.assets/progress-widget` shows milestones `{stage counters}`, downloads on the completed row's `result` and cancels by job id). With render-wasm active the export never leaves the browser: the shared worker WASM module renders it (one file, a ZIP of shapes or a multi-page boards PDF) and the client downloads it locally, with no job. - `app.main.refs`: reactive refs/lenses over store and derived workspace data. - `app.main.store`: Potok store and `emit!`. - `app.plugins.*` and `app.plugins`: CLJS implementation of Plugin JS API proxies. diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 99f47ea40c..7f63dbdd30 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -81,6 +81,8 @@ :jobs-retention (ct/duration {:days 7}) :jobs-user-ttl (ct/duration {:days 7}) :jobs-request-timeout (ct/duration {:minutes 2}) + :job-session-ttl (ct/duration {:hours 1}) + :exporter-max-result-size (* 1024 1024 350) ; 350MiB :media-max-file-size (* 1024 1024 30) ; 30MiB :font-max-file-size (* 1024 1024 30) ; 30MiB @@ -214,6 +216,8 @@ [:jobs-retention {:optional true} ::ct/duration] [:jobs-user-ttl {:optional true} ::ct/duration] [:jobs-request-timeout {:optional true} ::ct/duration] + [:job-session-ttl {:optional true} ::ct/duration] + [:exporter-max-result-size {:optional true} ::sm/int] [:telemetry-enabled {:optional true} ::sm/boolean] [:default-blob-version {:optional true} ::sm/int] [:allow-demo-users {:optional true} ::sm/boolean] @@ -504,6 +508,25 @@ (or (c/get config :jobs-user-ttl) (ct/duration {:days 7}))) +(defn get-job-session-ttl + "How long the token minted by `create-job-session` verifies, beyond + the session row's own lifetime: an export never runs for hours, so a + short expiry is defense in depth for the worker that dies without + settling. The session row itself is swept by the sessions idle GC." + [] + (or (c/get config :job-session-ttl) + (ct/duration {:hours 1}))) + +(defn get-exporter-max-result-size + "How big the artifact a worker sends with `complete-job` may be. The + ceiling is the same the multipart body already has at the HTTP server + (`:http/max-multipart-body-size`); lowering it rejects a result + before any object is stored. Raising it past the server cap is an + operational change of both this and nginx `client_max_body_size`." + [] + (or (c/get config :exporter-max-result-size) + (* 1024 1024 350))) + (defn get "A configuration getter. Helps code be more testable." ([key] diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index ad80bfe9c4..9968715860 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -15,6 +15,7 @@ [app.http :as-alias http] [app.http.errors :as errors] [app.metrics :as mtx] + [app.setup.keys :as keys] [app.tokens :as tokens] [app.util.inet :as inet] [app.util.pointer-map :as pmap] @@ -394,16 +395,34 @@ [^String a ^String b] (MessageDigest/isEqual (.getBytes a "UTF-8") (.getBytes b "UTF-8"))) +(def ^:private exporter-key-prefix + "Prefix of the per-tenant exporter key ids (`exporter:`). The + setup only registers the ids it knows, but any id with this prefix + the secret derives is a key this deployment minted: the tenant travels + in the id, so no enumeration is needed." + "exporter:") + +(defn- derive-tenant-key + "The expected secret of a per-tenant exporter id: the shared derivation + with the tenant the id names, so the registry and the middleware can + never build two different salts." + [secret key-id] + (keys/derive-exporter-key secret (subs (name key-id) (count exporter-key-prefix)))) + (defn- wrap-shared-key-auth - [handler keys] + [handler keys secret] (if (seq keys) (fn [request] (if-let [[key-id key] (some-> (yreq/get-header request "x-shared-key") (str/split #"\s+" 2))] (let [key-id (-> key-id str/lower keyword)] (if (and (string? key) - (contains? keys key-id) - (constant-time-eq? key (get keys key-id))) + (or (and (contains? keys key-id) + (constant-time-eq? key (get keys key-id))) + ;; Per-tenant exporter ids need no registry entry. + (and (string? secret) + (str/starts-with? (name key-id) exporter-key-prefix) + (constant-time-eq? key (derive-tenant-key secret key-id))))) (-> request (assoc ::http/auth-key-id key-id) (handler)) diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index 4e9ce3bd77..4409bee8ea 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -157,22 +157,34 @@ (declare ^:private assign-session-cookie) (declare ^:private clear-session-cookie) +(defn generate-token + "The authentication token of a session, with exactly the claims the + authz resolver accepts: the session id (`:sid`) and the profile it + belongs to (`:uid`). The optional `:exp` bounds how long the token + verifies, independent of the session row: without it, the expiry of a + login session is its absolute max-age counted from `:created-at`." + [cfg session & {:keys [exp]}] + (let [absolute-max-age (cf/get :auth-token-cookie-max-age-absolute + default-cookie-max-age-absolute) + claims (cond-> {:iss "authentication" + :aud "penpot" + :sid (:id session) + :iat (:modified-at session) + :uid (:profile-id session) + :sso-provider-id (:sso-provider-id session) + :sso-session-id (:sso-session-id session)} + exp + (assoc :exp exp) + + (and (nil? exp) (ct/inst? (:created-at session))) + (assoc :exp (ct/plus (:created-at session) + absolute-max-age))) + header {:kid 1 :ver 1}] + (tokens/generate cfg claims header))) + (defn- assign-token [cfg session] - (let [absolute-max-age (cf/get :auth-token-cookie-max-age-absolute default-cookie-max-age-absolute) - claims {:iss "authentication" - :aud "penpot" - :sid (:id session) - :iat (:modified-at session) - :uid (:profile-id session) - :sso-provider-id (:sso-provider-id session) - :sso-session-id (:sso-session-id session)} - claims (if (:created-at session) - (assoc claims :exp (ct/plus (:created-at session) absolute-max-age)) - claims) - header {:kid 1 :ver 1} - token (tokens/generate cfg claims header)] - (assoc session :token token))) + (assoc session :token (generate-token cfg session))) (defn create-fn [{:keys [::manager] :as cfg} {profile-id :id :as profile} diff --git a/backend/src/app/jobs.clj b/backend/src/app/jobs.clj index 271414ebcc..196def9a6f 100644 --- a/backend/src/app/jobs.clj +++ b/backend/src/app/jobs.clj @@ -807,15 +807,31 @@ WHERE id = ? AND status IN ('new', 'scheduled', 'running', 'retry')") +(def ^:private sql:touch-heartbeat-of-tenant + "UPDATE job + SET modified_at = ? + WHERE id = ? + AND tenant = ? + AND status IN ('new', 'scheduled', 'running', 'retry')") + (def ^:private sql:lock-active-job "SELECT id FROM job WHERE id = ? AND status IN ('new', 'scheduled', 'running', 'retry') FOR UPDATE") +(def ^:private sql:lock-active-job-of-tenant + "SELECT id FROM job + WHERE id = ? + AND tenant = ? + AND status IN ('new', 'scheduled', 'running', 'retry') + FOR UPDATE") + (defn- touch-job - [connectable job-id now] - (-> (db/exec-one! connectable [sql:touch-heartbeat now job-id]) + [connectable job-id now tenant] + (-> (if (some? tenant) + (db/exec-one! connectable [sql:touch-heartbeat-of-tenant now job-id tenant]) + (db/exec-one! connectable [sql:touch-heartbeat now job-id])) (db/get-update-count))) (defn- report-progress @@ -823,14 +839,18 @@ The row is locked first: a concurrent complete, fail or cancel blocks here, so a progress event can never land on an already terminal job. + With a `tenant`, the lock also filters by it, so a foreign report + matches nothing and writes nothing. Returns 1 when the event was stored. Like every lifecycle write it owns its transaction (see `own-transaction`), so a beat survives the rollback of the work around it." - [cfg job-id progress] + [cfg job-id progress tenant] (db/tx-run! (own-transaction cfg) (fn [{:keys [::db/conn] :as tx-cfg}] - (if (db/exec-one! conn [sql:lock-active-job job-id]) + (if (if (some? tenant) + (db/exec-one! conn [sql:lock-active-job-of-tenant job-id tenant]) + (db/exec-one! conn [sql:lock-active-job job-id])) (do (insert-event tx-cfg job-id "progress" progress) 1) @@ -844,19 +864,35 @@ not. The status is read from the row, because a write can only tell that it matched no row, not why. + With a `tenant`, a row of another tenant is inactive too, and the + interrupt carries `:reason :foreign-tenant` (and no `:status`): the + caller answers it like a missing row, without revealing state. + The interrupt is an internal signal, not an error: the caller is a task that must stop the work in course. `heartbeat` calls this when one of its writes finds the job gone, and a caller can also ask explicitly, which is what the binfile adapter does before and after its run. Answers with the cfg, so a caller can thread the check into its run." - [cfg job-id] + [cfg job-id & {:keys [tenant]}] (let [job (get-job cfg job-id)] - (when-not (and (some? job) (contains? active-statuses (:status job))) + (cond + ;; a row of another tenant is not this caller's business: same + ;; signal, no state + (and (some? job) (some? tenant) (not= tenant (:tenant job))) (ex/raise :type :interrupt :code :job-interrupted + :reason :foreign-tenant + :hint "the job belongs to another tenant" + :job-id job-id) + + (not (and (some? job) (contains? active-statuses (:status job)))) + (ex/raise :type :interrupt + :code :job-interrupted + :reason (if job :terminal :gone) :hint "the job is no longer active" :job-id job-id :status (or (:status job) "gone"))) + cfg)) (defn heartbeat @@ -867,6 +903,9 @@ - `:job-id` explicit job id; defaults to `::jobs/job-id` on the cfg and then to the runner-bound `*job-id*`. - `:progress` optional progress report (see `schema:progress`). + - `:tenant` optional tenant the beat belongs to: both writes filter + by it, so a foreign beat matches nothing. Internal + runners omit it and beat as their own tenant. - `::force?` internal: skips only the progress throttle. Returns the number of durable writes performed, so 0 means there was @@ -892,7 +931,7 @@ forgiving on the handler path, where it is logged and ignored; the forced path used by the management API propagates it so the external worker can retry." - [cfg & {:keys [job-id progress] :as options}] + [cfg & {:keys [job-id progress tenant] :as options}] (let [job-id (or job-id (get cfg ::job-id) *job-id*) ;; a malformed report is a caller bug: never swallow it progress (some-> progress validate-progress)] @@ -902,16 +941,16 @@ force? (boolean (::force? options)) writes (volatile! 0)] (when (should-write? heartbeats job-id now heartbeat-interval) - (let [n (touch-job connectable job-id now)] + (let [n (touch-job connectable job-id now tenant)] ;; the touch is conditional on the row still being active: a ;; write that matched no row is what raises the interrupt (when (zero? n) - (check-active cfg job-id)) + (check-active cfg job-id :tenant tenant)) (vswap! writes + n))) (when (and (some? progress) (or force? (should-write? progresses job-id now progress-interval))) - (let [store #(report-progress cfg job-id progress) + (let [store #(report-progress cfg job-id progress tenant) n (if force? (store) (try @@ -925,7 +964,7 @@ ;; the raise stays outside the catch above: an interrupt is ;; not a transient failure and must not be swallowed with it (when (zero? n) - (check-active cfg job-id)) + (check-active cfg job-id :tenant tenant)) (vswap! writes + n))) (int @writes))))) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 9267c889f4..f1bde59523 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -665,6 +665,7 @@ :offload-file-data (ig/ref :app.tasks.offload-file-data/job-def) :objects-gc (ig/ref :app.tasks.objects-gc/job-def) :export-binfile (ig/ref :app.tasks.export-binfile/job-def) + :export-assets (ig/ref :app.tasks.export-assets/job-def) :import-binfile (ig/ref :app.tasks.import-binfile/job-def) :storage-gc-deleted (ig/ref ::sto.gc-deleted/job-def) :storage-gc-touched (ig/ref ::sto.gc-touched/job-def) @@ -731,6 +732,11 @@ ::mtx/metrics (ig/ref ::mtx/metrics) ::mbus/msgbus (ig/ref ::mbus/msgbus)} + ;; the export of shapes/frames: the work runs on the external exporter + ;; worker, so this def carries no handler work and no components + :app.tasks.export-assets/job-def + {} + :app.storage.gc-deleted/job-def {::db/pool (ig/ref ::db/pool) ::sto/storage (ig/ref ::sto/storage) diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index f9c9d7eabf..abdf309788 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -370,14 +370,19 @@ "Authorizes a shared-key caller against the method's `::allowed-shared-keys`. A management method declares the shared-key ids allowed to call it with - `::allowed-shared-keys`. When the metadata is absent any configured shared key is - accepted, preserving the previous behavior. When it is present and empty no - key is accepted." + `::allowed-shared-keys`: a set of ids, or a predicate over the calling + key id (e.g. the exporter family with per-tenant ids). When the metadata + is absent any configured shared key is accepted, preserving the previous + behavior. When it is present and empty no key is accepted." [_ f mdata] (if (contains? mdata ::allowed-shared-keys) - (let [allowed-keys (::allowed-shared-keys mdata)] + (let [allowed (::allowed-shared-keys mdata)] (fn [cfg params] - (if (contains? allowed-keys (::auth-key-id params)) + (if (if (set? allowed) + (contains? allowed (::auth-key-id params)) + ;; A predicate (e.g. the exporter family with per-tenant + ;; ids): the set form stays for every other method. + (allowed (::auth-key-id params))) (f cfg params) (do (l/wrn :hint "shared key is not allowed for this method" @@ -541,7 +546,7 @@ ["/api" ["/management" ["/methods/:method-name" - {:middleware [[mw/shared-key-auth shared-keys] + {:middleware [[mw/shared-key-auth shared-keys (:secret-key (::setup/props cfg))] [session/authz cfg]] :handler (make-rpc-handler management-methods)}] diff --git a/backend/src/app/rpc/commands/jobs.clj b/backend/src/app/rpc/commands/jobs.clj index c3f590315b..e184f1014e 100644 --- a/backend/src/app/rpc/commands/jobs.clj +++ b/backend/src/app/rpc/commands/jobs.clj @@ -16,6 +16,7 @@ [app.binfile.v3 :as bf.v3] [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.export :as export] [app.common.schema :as sm] [app.common.time :as ct] [app.config :as cf] @@ -44,6 +45,19 @@ [:map {:title "create-export-binfile-job" :closed true} [:params export-binfile/schema:params]]) +(def ^:private schema:create-export-assets-job + "The shapes and frames frozen in the params, rendered as image files + by the external exporter worker; the business params are what the + `:export-assets` job-def declares. + + `:transient` marks a single-use export (a clipboard image, a plugin + shape export): the caller consumes the artifact in the same flow and + never comes back for it, so its row and object live minutes, not the + user ledger's days." + [:map {:title "create-export-assets-job" :closed true} + [:params export/schema:params] + [:transient {:optional true} ::sm/boolean]]) + (def ^:private schema:create-import-binfile-job "The `.penpot` package the caller uploaded, imported into a project by the backend runner; the business params are what the `:import-binfile` @@ -93,19 +107,27 @@ (defn- submit-job "Create the job row with what every user job shares: the queue its job-def routes the heavy work to (`::jobs/queue-name`, `:binfile` when - the def says nothing), no automatic retry and the expiry of the ledger." - [cfg job-def name params profile-id & {:keys [resource-id]}] + the def says nothing), no automatic retry and the expiry of the ledger + (or the one the caller names, for a single-use job)." + [cfg job-def name params profile-id & {:keys [resource-id expires-at]}] (jobs/submit cfg (cond-> {::jobs/name name ::jobs/params params ::jobs/queue (or (::jobs/queue-name job-def) :binfile) ::jobs/profile-id profile-id ::jobs/max-retries 0 - ::jobs/expires-at (ct/plus (ct/now) - (cf/get-jobs-user-ttl))} + ::jobs/expires-at (or expires-at + (ct/plus (ct/now) + (cf/get-jobs-user-ttl)))} (some? resource-id) (assoc ::jobs/resource-id resource-id)))) +(def ^:private transient-job-ttl + "How long a single-use export lives. The legacy tempfile paths these + flows replace lived ten minutes, and the artifact is downloaded in the + same flow: nothing reads it later." + (ct/duration {:minutes 10})) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; EXPORT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -129,6 +151,49 @@ (= 1 (count teams))) (first teams))))))) +(defn- check-file-permissions + "The files the job will read, checked before anything is stored: a + refusal arrives synchronously. The handler checks it again before + reading, because a permission can be revoked while the job waits. + This is the binfile flavour: whole files, no share ids." + [cfg profile-id file-ids] + (when (empty? file-ids) + (ex/raise :type :validation + :code :no-files-to-export + :hint "expected at least one file to export")) + (doseq [file-id file-ids] + (files/check-read-permissions! cfg profile-id file-id))) + +(defn- check-items-scope! + "One job renders one batch: every item names the same file, page and + share (a missing share-id is its own group). The worker groups by + scale and type and renders each group against the page of its first + item, so a mixed batch would render objects against the wrong page + instead of failing. The legacy surface only took one file; refuse + here, synchronously, the way permissions refuse." + [items] + (let [scopes (into #{} (map (juxt :file-id :page-id :share-id)) items)] + (when (> (count scopes) 1) + (ex/raise :type :validation + :code :mixed-export-scope + :hint "all the items of an export belong to the same file, page and share")))) + +(defn- check-item-permissions + "The assets flavour: the items name files, and a viewer that reached + them through a share link names the share, so the permission goes the + same way every share fetch goes (a logged viewer of a public share is + allowed; the workspace path stays for the plain items). The command + is not anonymous: an anonymous visitor has no profile to own the + artifact, the same as the exporter cookie this cutover replaces." + [cfg profile-id items] + (check-items-scope! items) + (doseq [item items] + (let [file-id (:file-id item) + share-id (:share-id item)] + (if (some? share-id) + (files/check-read-permissions! cfg profile-id file-id share-id) + (files/check-read-permissions! cfg profile-id file-id))))) + (sv/defmethod ::create-export-binfile-job "Create a durable job that exports a set of files as a `.penpot` package. @@ -153,13 +218,7 @@ params (jobs/validate-params job-def (:params envelope)) file-ids (:file-ids params)] - (when (empty? file-ids) - (ex/raise :type :validation - :code :no-files-to-export - :hint "expected at least one file to export")) - - (doseq [file-id file-ids] - (files/check-read-permissions! cfg profile-id file-id)) + (check-file-permissions cfg profile-id file-ids) (quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile ::quotes/profile-id profile-id}) @@ -173,6 +232,42 @@ :export-type (d/name (:export-type params))} team-id (assoc :team-id team-id))})))) +(sv/defmethod ::create-export-assets-job + "Create a durable job that renders shapes and frames as image files. + + Its params freeze the items to render (file, page, object, type, + scale, name and the optional share id) the way the exporter receives + them. The work runs on the external exporter worker, which consumes + the queue the job-def names; the backend ships no runner for it. + + The job is created, not run: the caller follows it by its id. The + read permission of every file named in the items is checked here so a + refusal arrives synchronously, and the handler checks it again before + reading." + {::doc/added "2.20" + ::webhooks/event? true + ::sm/params schema:create-export-assets-job + ::sm/result schema:job-summary} + [cfg {:keys [::rpc/profile-id] :as envelope}] + (let [job-def (resolve-job-def cfg :export :export-assets) + ;; the RPC layer decoded the params against the job-def schema + ;; before the command ran + params (jobs/validate-params job-def (:params envelope))] + + (check-item-permissions cfg profile-id (:exports params)) + + (quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile + ::quotes/profile-id profile-id}) + + (let [expires-at (when (:transient envelope) + (ct/plus (ct/now) transient-job-ttl)) + summary (get-job-summary cfg (submit-job cfg job-def :export-assets + params profile-id + :expires-at expires-at))] + (with-meta summary + {::audit/props {:job-id (:id summary) + :files (count (:exports params))}})))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPORT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/src/app/rpc/management/jobs.clj b/backend/src/app/rpc/management/jobs.clj index 3fd79ff3d0..da1737a681 100644 --- a/backend/src/app/rpc/management/jobs.clj +++ b/backend/src/app/rpc/management/jobs.clj @@ -15,19 +15,63 @@ (app.http.middleware) to authenticate the ` ` header, and wrap-management applies wrap-shared-key-authorization (app.rpc) to check the key id against ::rpc/allowed-shared-keys. If the route - middleware or wrap-management changes, revisit these methods." + middleware or wrap-management changes, revisit these methods. + + Every write method answers `{:action :run}` when it landed and + `{:action :skip}` when it did not, with the state of the row it found + (read fresh after the failed write) in `:status`: the worker tells a + cancelled job apart from one it was too late for, without guessing." (:require [app.common.exceptions :as ex] + [app.common.media :as cm] [app.common.schema :as sm] [app.common.time :as ct] + [app.config :as cf] + [app.db :as db] + [app.http.session :as session] [app.jobs :as jobs] + [app.jobs.storage :as js] + [app.media.validation :as media.v] [app.rpc :as-alias rpc] [app.rpc.doc :as doc] - [app.util.services :as sv])) + [app.setup :as-alias setup] + [app.storage :as sto] + [app.util.services :as sv] + [clojure.string :as str])) -(def ^:private allowed-shared-key-ids - "Only the exporter worker claims and settles durable jobs today." - #{:exporter}) +(def ^:private exporter-key-prefix + "Prefix of the per-tenant exporter key ids (`exporter:`). The + bare `exporter` id is the legacy form, bound to this backend's own + tenant below." + "exporter:") + +(defn- exporter-key? + "A key id of the exporter family: the bare legacy id, or one suffixed + with the tenant it renders for (lowercase — the middleware lowercases + every id before comparing)." + [key-id] + (and (some? key-id) + (let [id-name (if (keyword? key-id) (name key-id) (str key-id))] + (or (= id-name "exporter") + (str/starts-with? id-name exporter-key-prefix))))) + +(defn- key-tenant + "The tenant the calling exporter renders for: the suffix of a + per-tenant key id, or this backend's own tenant for the bare legacy + id (single-tenant deploys only ever see their own)." + [key-id] + (let [id-name (name key-id)] + (if (str/starts-with? id-name exporter-key-prefix) + (subs id-name (count exporter-key-prefix)) + (cf/get :tenant)))) + +(defn- tenant-ok? + "The job belongs to the calling exporter's tenant. Anything else is + answered exactly like a missing row, so the caller learns nothing + about rows it cannot see." + [key-id job] + (and (some? job) + (= (key-tenant key-id) (:tenant job)))) ;; ---- RPC METHOD: CLAIM-JOB @@ -40,27 +84,66 @@ [:map {:title "claim-job-result"} [:action [:enum :run :skip]] [:name {:optional true} ::sm/text] - ;; Opaque per-job params blob; intentionally untyped. - [:params {:optional true} :any]]) + ;; Params decoded with the row's job-def (typed values); a row + ;; whose job-def is gone answers the stored shapes untouched. + [:params {:optional true} :any] + ;; On `:skip`, the state of the row the write lost against. + [:status {:optional true} ::sm/text]]) + +(defn- decode-claim-params + "The frozen params through the decoder of the row's job-def, so the + worker gets typed values (uuids, keywords, instants) instead of the + stored JSON shapes — transit carries the types natively. A row whose + job-def is gone (retired) answers untouched, mirroring what + `jobs/decode-result-of` does for results." + [cfg row] + (let [params (:params row)] + (if-let [job-def (and (some? params) + (get (jobs/get-defs cfg) (keyword (:name row))))] + (jobs/decode-params job-def params) + params))) + +(defn- row-status + "The state of the row after a write that touched nothing: one fresh + read, never the row the method already saw, so the answer names the + state the race produced." + [cfg job-id] + (:status (jobs/get-job cfg job-id))) + +(defn- skip-with-status + "The answer of a landed-nowhere write: one fresh read of the row + after the failure, so the worker sees the state the race produced." + [cfg job-id] + (if-let [status (row-status cfg job-id)] + {:action :skip :status status} + {:action :skip})) (sv/defmethod ::claim-job {::doc/added "2.20" ::sm/params schema:claim-job-params ::sm/result schema:claim-job-result ::rpc/auth false - ::rpc/allowed-shared-keys allowed-shared-key-ids} - [cfg {:keys [job-id scheduled-at]}] + ::rpc/allowed-shared-keys exporter-key?} + [cfg {:keys [job-id scheduled-at] :as params}] (let [row (jobs/get-job cfg job-id)] ;; Race condition: if the job is claimed between get-job and claim! by ;; another worker, claim! returns 0 and we return :skip. The conditional ;; claim (UPDATE ... WHERE status IN ('new','scheduled','retry') AND ;; scheduled_at=?) handles this correctly: only a pending row with an ;; exact scheduled-at match is claimed, a stale payload is skipped. - (if (and row (pos? (jobs/claim cfg job-id scheduled-at))) + (cond + ;; A job of another tenant is indistinguishable from a missing one: + ;; a bare skip, never the row's state. + (and (some? row) (not (tenant-ok? (::rpc/auth-key-id params) row))) + {:action :skip} + + (and row (pos? (jobs/claim cfg job-id scheduled-at))) {:action :run :name (:name row) - :params (:params row)} - {:action :skip}))) + :params (decode-claim-params cfg row)} + + :else + (skip-with-status cfg job-id)))) ;; ---- RPC METHOD: REPORT-JOB-PROGRESS @@ -74,15 +157,16 @@ (def ^:private schema:report-job-progress-result [:map {:title "report-job-progress-result"} - [:action [:enum :run :skip]]]) + [:action [:enum :run :skip]] + [:status {:optional true} ::sm/text]]) (sv/defmethod ::report-job-progress {::doc/added "2.20" ::sm/params schema:report-job-progress-params ::sm/result schema:report-job-progress-result ::rpc/auth false - ::rpc/allowed-shared-keys allowed-shared-key-ids} - [cfg {:keys [job-id progress]}] + ::rpc/allowed-shared-keys exporter-key?} + [cfg {:keys [job-id progress] :as params}] ;; A lost race (row already terminal) reports :skip so the worker ;; stops retrying a report that can never land, mirroring claim-job. ;; The forced route propagates database errors: the external worker @@ -90,62 +174,237 @@ ;; the beat raises when the job is no longer active is caught here for ;; the same reason, and it never reaches the HTTP error mapping: it is ;; an internal signal, not a failure of the request. + ;; + ;; The tenant rides in the write itself (`:tenant`), so the hot path is + ;; one statement and needs no row read. A beat for another tenant's job + ;; matches nothing and the substrate says why; the answer is a bare + ;; skip, never the row's state. (let [result (ex/try! (jobs/heartbeat cfg :job-id job-id :progress progress + :tenant (key-tenant (::rpc/auth-key-id params)) ::jobs/force? true))] (cond (ex/exception? result) - (if (= :interrupt (:type (ex-data result))) - {:action :skip} - (throw result)) + (let [data (ex-data result)] + (cond + (= :foreign-tenant (:reason data)) + {:action :skip} + + (= :interrupt (:type data)) + (skip-with-status cfg job-id) + + :else + (throw result))) (pos? result) {:action :run} :else - {:action :skip}))) + (skip-with-status cfg job-id)))) ;; ---- RPC METHOD: COMPLETE-JOB (def ^:private schema:complete-job-params - [:map {:title "complete-job-params"} - [:job-id ::sm/uuid] - ;; Per-job result: any JSON-encodable value, nil when the job returns - ;; nothing. Nested maps and vectors are allowed; unserializable - ;; values are dropped to nil by jobs/encode-result with a warning. A - ;; job whose job-def declares a result schema has its result validated - ;; against it: one that does not conform is rejected and the job is - ;; not completed. - [:result [:maybe :any]] - ;; Storage object the worker produced for this job. A resource is only - ;; set, never replaced: a job that already has one is rejected. - [:resource-id {:optional true} ::sm/uuid]]) + [:and + [:map {:title "complete-job-params"} + [:job-id ::sm/uuid] + ;; Per-job result: any JSON-encodable value, nil when the job returns + ;; nothing. Nested maps and vectors are allowed; unserializable + ;; values are dropped to nil by jobs/encode-result with a warning. A + ;; job whose job-def declares a result schema has its result validated + ;; against it: one that does not conform is rejected and the job is + ;; not completed. + [:result {:optional true} [:maybe :any]] + ;; Storage object the worker produced for this job. A resource is only + ;; set, never replaced: a job that already has one is rejected. + [:resource-id {:optional true} ::sm/uuid] + ;; The multipart form: the worker sends the artifact itself, the + ;; server stores it. The part is the single source of what the user + ;; will download: its filename and type travel in the part itself + ;; (`schema:upload` requires the name; `check-artifact!` enforces + ;; the allowlist on the type), never as separate text fields. + [:content {:optional true} media.v/schema:upload] + ;; The render session the worker used, closed once the job settles: + ;; complete is terminal, the session has no reader after it. + [:session-id {:optional true} ::sm/uuid]] + [:fn {:error/message "one of :content, :result or :resource-id is required"} + (fn [{:keys [content resource-id] :as params}] + (boolean (or (some? content) + (contains? params :result) + (some? resource-id))))]]) + (def ^:private schema:complete-job-result [:map {:title "complete-job-result"} - [:action [:enum :run :skip]]]) + [:action [:enum :run :skip]] + [:status {:optional true} ::sm/text]]) + +(defn- close-job-session! + "The session a worker rendered with, deleted once the job settles, and + only when it really belongs to that job's owner: the id travels in the + request, so a shared-key holder must not be able to delete another + profile's login. + + Best-effort: the settle already decided the outcome, and a session that + cannot be deleted now is swept later by the sessions idle GC. With no + job row to match (it was already gone) nothing is deleted." + [cfg job-id session-id] + (when (and (uuid? session-id) (uuid? job-id)) + (ex/ignoring + (when-let [job (jobs/get-job cfg job-id)] + (when-let [session (session/read-session (::session/manager cfg) session-id)] + (when (= (:profile-id job) (:profile-id session)) + (session/delete-session (::session/manager cfg) session-id))))))) + +(defn- check-artifact! + "The bytes a worker offers as the result: the path must be one the + multipart parser wrote (`media.v/check-upload-path`), the type one the + export produces and the size under the configured cap. Every check + refuses to *use* the bytes, not to receive them (the parser already + wrote them to a server tempfile); refusing early costs nothing, + storing early costs a GC cycle." + [content] + (media.v/check-upload-path content) + (media.v/check-media-type content cm/export-artifact-types) + (let [max-size (cf/get-exporter-max-result-size)] + (when (> (:size content) max-size) + (ex/raise :type :validation + :code :request-body-too-large + :hint "the result of the job is over the maximum size" + :size (:size content) + :max-size max-size))) + content) + +(defn- completable-job + "The row a completion may land on: running (or retrying), whatever + the JSON form asks for." + [cfg job-id] + (let [job (jobs/get-job cfg job-id)] + (when (and (some? job) + (contains? #{"running" "retry"} (:status job))) + job))) + +(defn- artifact-owner + "The job an artifact completion may land on: completable AND owned, + because the object needs an owner and a job without one owns nothing + (it may only settle without an artifact). Read before anything is + stored: a lost race with a cancel never turns the bytes of the worker + into an object." + [cfg job-id] + (let [job (completable-job cfg job-id)] + (when (and (some? job) + (uuid? (:profile-id job))) + job))) + +(defn- complete-with-artifact + "The multipart form of a completion, on a job that is still + completable: the artifact is stored out of any transaction (storage + waits for no database), owned by the profile of the job, and the + completion itself writes the `resource_id` and the result descriptor + `{resource-uri filename mtype size}` in its own transaction, where + the first terminal state wins. A completion that lost the race + releases the artifact it stored: no object lingers for the storage GC + to find later." + [cfg job content session-id] + (let [resource (js/put-resource cfg + (:profile-id job) + {:content (sto/content (:path content)) + :filename (:filename content) + :mtype (:mtype content)})] + (if (pos? (jobs/complete cfg + :job-id (:id job) + :resource-id (:resource-id resource) + :result {:resource-uri (:resource-uri resource) + :filename (:filename content) + :mtype (:mtype content) + :size (:size resource)})) + (do + (close-job-session! cfg (:id job) session-id) + {:action :run}) + (do + (js/release-resource cfg (:resource-id resource)) + (close-job-session! cfg (:id job) session-id) + (skip-with-status cfg (:id job)))))) (sv/defmethod ::complete-job - "Complete a job on behalf of the worker that ran it, with the result it - produced and, when it wrote one, the resource it stores. + "Close a running job as completed, in one of two forms. + + The params form carries the plain `:result` the job produced and an + optional storage object (`:resource-id`) the worker wrote itself: the + internal binfile path. + + The artifact form carries the artifact itself (`:content`, a multipart + part with `:filename` and `:mtype` naming the download): the server + stores it, owned by the profile of the job, and coalesces the resource + into the same completion. The checks (completable, path, type, size) + run before any object is written: a lost race against a cancel never + stores bytes. The result of a job whose job-def declares a result schema is validated against it before it is stored: one that does not conform is rejected and the job is not completed. A job that is no longer completable is skipped, - and nothing is validated for it." + and nothing is validated for it. + + Both forms belong to the calling exporter's tenant: a job of another + tenant answers like a missing row, and nothing is stored, written or + revealed for it. + + With `:session-id`, the render session the worker minted through + `create-job-session` is deleted once the completion is answered, + whatever it says: the settle of the run is what ends the life of the + session it minted, even a settle that lost against a cancel. The + session is deleted only when it belongs to the job's owner. + + A completion of a job that already ended answers `{:action :skip, + :status ...}` and changes nothing." {::doc/added "2.20" ::sm/params schema:complete-job-params ::sm/result schema:complete-job-result ::rpc/auth false - ::rpc/allowed-shared-keys allowed-shared-key-ids} - [cfg {:keys [job-id result resource-id]}] - (if (pos? (jobs/complete cfg - :job-id job-id - :result result - :resource-id resource-id)) - {:action :run} - {:action :skip})) + ;; The artifact arrives as a multipart part: the parser writes it to a + ;; server tempfile and only a request flagged multipart server-side can + ;; carry it. Without this, a transit body could name any internal path. + ::rpc/multipart-only-params [:content] + ::rpc/allowed-shared-keys exporter-key?} + [cfg {:keys [job-id result resource-id content session-id] :as params}] + (cond + ;; the multipart form stores the artifact itself: only for a job + ;; that can still complete and has an owner, because the object + ;; needs one + (some? content) + (if-let [job (artifact-owner cfg job-id)] + (if (tenant-ok? (::rpc/auth-key-id params) job) + (do + (check-artifact! content) + (complete-with-artifact cfg job content session-id)) + ;; a job of another tenant answers like a missing row, and its + ;; session is not ours to close + {:action :skip}) + (do + ;; a completion that lost the race is not a session to keep either + (close-job-session! cfg job-id session-id) + (skip-with-status cfg job-id))) + + ;; the params form: nothing to store here, the resource (if any) + ;; already lives in storage + :else + (if-let [job (completable-job cfg job-id)] + (if (tenant-ok? (::rpc/auth-key-id params) job) + (if (pos? (jobs/complete cfg + :job-id job-id + :result result + :resource-id resource-id)) + (do + (close-job-session! cfg job-id session-id) + {:action :run}) + (do + (close-job-session! cfg job-id session-id) + (skip-with-status cfg job-id))) + {:action :skip}) + (do + (close-job-session! cfg job-id session-id) + (skip-with-status cfg job-id))))) ;; ---- RPC METHOD: FAIL-JOB @@ -154,19 +413,125 @@ [:job-id ::sm/uuid] ;; Rich error report: type/code/hint are required, the map stays ;; open to worker-defined details (see jobs/fail). - [:error jobs/schema:job-error]]) + [:error jobs/schema:job-error] + ;; The render session the worker used, closed once the job settles: + ;; fail is terminal too. + [:session-id {:optional true} ::sm/uuid]]) (def ^:private schema:fail-job-result [:map {:title "fail-job-result"} - [:action [:enum :run :skip]]]) + [:action [:enum :run :skip]] + [:status {:optional true} ::sm/text]]) (sv/defmethod ::fail-job {::doc/added "2.20" ::sm/params schema:fail-job-params ::sm/result schema:fail-job-result ::rpc/auth false - ::rpc/allowed-shared-keys allowed-shared-key-ids} - [cfg {:keys [job-id error]}] - (if (pos? (jobs/fail cfg job-id error)) - {:action :run} - {:action :skip})) + ::rpc/allowed-shared-keys exporter-key?} + [cfg {:keys [job-id error session-id] :as params}] + ;; A job of another tenant answers like a missing row, before any + ;; write lands and without touching its session. + (let [job (jobs/get-job cfg job-id)] + (if (and (some? job) + (not (tenant-ok? (::rpc/auth-key-id params) job))) + {:action :skip} + ;; The session is closed whatever the write says: the worker that + ;; settles (even a settle that loses against a cancel it cannot + ;; see) is what ends the life of the render session it minted. It + ;; is deleted only when it belongs to the job's owner. + (let [landed? (pos? (jobs/fail cfg job-id error))] + (close-job-session! cfg job-id session-id) + (if landed? + {:action :run} + (skip-with-status cfg job-id)))))) + +;; ---- RPC METHOD: CREATE-JOB-SESSION + +(def ^:private schema:create-job-session-params + [:map {:title "create-job-session-params"} + [:job-id ::sm/uuid]]) + +(def ^:private schema:create-job-session-result + [:map {:title "create-job-session-result"} + [:session-id ::sm/uuid] + [:session-token ::sm/text]]) + +(defn- validate-job! + "The job whose owner will render: it must exist, be `running` (the + claim already passed: a session for a waiting job would outlive a + job that may never run) and have an owner. A job of another tenant + is indistinguishable from a missing one on purpose: the tenant is + the boundary of what the key can touch, and the profile the boundary + of what the session can touch." + [cfg key-id job-id] + (let [job (jobs/get-job cfg job-id)] + (when (nil? job) + (ex/raise :type :not-found + :code :job-not-found + :hint "the job does not exist" + :job-id job-id)) + (when-not (tenant-ok? key-id job) + (ex/raise :type :not-found + :code :job-not-found + :hint "the job does not exist" + :job-id job-id)) + (when-not (= "running" (:status job)) + (ex/raise :type :validation + :code :invalid-job-state + :hint "the session is minted for a running job" + :status (:status job) + :job-id job-id)) + (when-not (uuid? (:profile-id job)) + (ex/raise :type :validation + :code :job-without-profile + :hint "the job has no owner profile to mint the session for" + :job-id job-id)) + (:profile-id job))) + +(defn- validate-owner! + "The profile the session will render as: the same one the login + accepts (exists, active, not blocked, not deleted). A session is a + key to everything the owner can touch, so a worker that settles a job + of a dead profile never gets one." + [cfg profile-id] + (let [profile (db/get* cfg :profile {:id profile-id})] + (when (or (nil? profile) (some? (:deleted-at profile))) + (ex/raise :type :not-found + :code :profile-not-found + :hint "the owner profile of the job does not exist" + :profile-id profile-id)) + (when-not (:is-active profile) + (ex/raise :type :validation + :code :profile-not-active + :hint "the owner profile of the job is not active" + :profile-id profile-id)) + (when (:is-blocked profile) + (ex/raise :type :restriction + :code :profile-blocked + :hint "the owner profile of the job is marked as blocked" + :profile-id profile-id)))) + +(sv/defmethod ::create-job-session + "Mint an authentication session of the owner of a running job, for + the worker to render as the user: render.html and assets authenticate + by cookie and the RPC by Bearer, through the surfaces that already + exist. Every call mints a fresh session (worker retries never wait on + one a settle may have closed), and the token carries the short `:exp` + of `:job-session-ttl`: a worker that dies without settling leaves + nothing usable for long, and the sessions idle GC sweeps the row." + {::doc/added "2.20" + ::sm/params schema:create-job-session-params + ::sm/result schema:create-job-session-result + ::rpc/auth false + ::rpc/allowed-shared-keys exporter-key?} + [cfg {:keys [job-id] :as params}] + (let [profile-id (validate-job! cfg (::rpc/auth-key-id params) job-id)] + (validate-owner! cfg profile-id) + (let [session (session/create-session (::session/manager cfg) + {:profile-id profile-id}) + token (session/generate-token cfg session + :exp (ct/plus (ct/now) + (cf/get-job-session-ttl)))] + {:session-id (:id session) + :session-token token}))) diff --git a/backend/src/app/setup.clj b/backend/src/app/setup.clj index 2afb5955b4..e5bb697520 100644 --- a/backend/src/app/setup.clj +++ b/backend/src/app/setup.clj @@ -103,8 +103,10 @@ (let [secret (get props :secret-key)] (reduce (fn [keys id] (let [key (or (get cfg id) - (-> (keys/derive secret :salt (name id)) - (bc/bytes->b64-str true)))] + (if (= id :exporter) + (keys/derive-exporter-key secret (cf/get :tenant)) + (-> (keys/derive secret :salt (name id)) + (bc/bytes->b64-str true))))] (if (or (str/empty? key) (str/blank? key)) (do diff --git a/backend/src/app/setup/keys.clj b/backend/src/app/setup/keys.clj index aac9d60bd7..74c7cdf82b 100644 --- a/backend/src/app/setup/keys.clj +++ b/backend/src/app/setup/keys.clj @@ -8,6 +8,7 @@ "Keys derivation service." (:refer-clojure :exclude [derive]) (:require + [buddy.core.codecs :as bc] [buddy.core.kdf :as bk])) (defn derive @@ -20,3 +21,17 @@ :alg :hkdf :digest :blake2b-512})] (bk/get-bytes engine size))) + +(defn derive-exporter-key + "The management key of the exporter that renders for `tenant`: HKDF + blake2b512 over the salt `exporter:`, base64url. + + The single definition of the exporter key on the backend: the shared-key + registry (bare `exporter` id, own tenant) and the per-tenant ids of the + middleware both come through here. The Node worker derives the same + value; both sides pin the same golden vector, so a change on one side + fails a test instead of a deployment." + [secret-key tenant] + (assert (string? tenant) "expect string tenant") + (-> (derive secret-key :salt (str "exporter:" tenant)) + (bc/bytes->b64-str true))) diff --git a/backend/src/app/tasks/export_assets.clj b/backend/src/app/tasks/export_assets.clj new file mode 100644 index 0000000000..8ea0759209 --- /dev/null +++ b/backend/src/app/tasks/export_assets.clj @@ -0,0 +1,55 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.tasks.export-assets + "The `:export-assets` job: the shapes and frames a caller froze at + creation time, rendered as image files. + + This is an EXTERNAL job: the work runs on the exporter worker, a + process outside the JVM, so the job is created with `::jobs/queue-name + :exporter` (read by the creation command; the substrate never looks at + it) and the backend ships no runner for that queue. The handler here + is a guard, not a worker: it is never executed under a correct + configuration, because the only runner that could claim the job is + external, and `claim-job` does not look at the queue. Should a wrong + configuration ever route the job to a JVM runner, it fails visibly + here instead of \"completing\" empty. + + Progress vocabulary of the external worker (the substrate validates + the shape, the worker owns the words): stages `:preparing`, + `:rendering` and `:packaging`, with counters `objects` (one per + exported shape) and `pages` (one per exported frame)." + (:require + [app.common.exceptions :as ex] + [app.common.export :as export] + [app.common.schema :as sm] + [app.jobs :as jobs] + [integrant.core :as ig])) + +(defn- execute-export-assets + "Plain handler, importable and testable without integrant. + + It only raises: the work of this job belongs to the external worker. + See the namespace docstring." + [_context _params] + (ex/raise :type :internal + :code :no-handler-for-external + :hint "this job runs on the external exporter worker")) + +(defmethod ig/init-key ::job-def + [_ _cfg] + {::jobs/name :export-assets + ::jobs/family :export + ::jobs/resource-role :output + ;; the queue the creation command routes the job to: an external + ;; worker consumes it, the backend ships no runner for it + ::jobs/queue-name :exporter + ;; the item shapes live in app.common.export, the single source of + ;; truth both runtimes validate and decode against + ::jobs/schema export/schema:params + ::jobs/handler execute-export-assets + ::jobs/decoder (sm/decoder export/schema:params sm/json-transformer) + ::jobs/validator (sm/validator export/schema:params)}) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 24c433b012..403f163f7a 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -573,7 +573,11 @@ (:app.http/auth-key-id request) :else - (first (::rpc/allowed-shared-keys mdata))) + ;; A predicate declares a family, not an enumerable + ;; set: fall back to the legacy id, which every such + ;; method accepts. + (let [allowed (::rpc/allowed-shared-keys mdata)] + (if (set? allowed) (first allowed) :exporter))) data (if (or (contains? data ::rpc/auth-key-id) (nil? key-id)) data diff --git a/backend/test/backend_tests/rpc/commands/jobs_test.clj b/backend/test/backend_tests/rpc/commands/jobs_test.clj index 5ced5072ee..3d41d7ec73 100644 --- a/backend/test/backend_tests/rpc/commands/jobs_test.clj +++ b/backend/test/backend_tests/rpc/commands/jobs_test.clj @@ -212,6 +212,244 @@ (t/is (= #{:job-id :files :export-type} (set (keys props)))) (t/is (not (contains? (:result out) :team-id))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; ASSETS EXPORT +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- export-item + [file-id & {:keys [page-id object-id type name scale suffix share-id]}] + (cond-> {:file-id file-id + :page-id (or page-id (uuid/next)) + :object-id (or object-id (uuid/next)) + :type (or type :png) + :name (or name "shape") + :scale (or scale 1)} + suffix (assoc :suffix suffix) + share-id (assoc :share-id share-id))) + +(defn- create-export-assets-job + [profile-id exports & {:as extra}] + (th/command! (merge {::th/type :create-export-assets-job + ::rpc/profile-id profile-id + :params {:exports exports}} + extra))) + +(t/deftest create-export-assets-job-answers-with-a-pending-job + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + item (export-item file-id) + before (count-storage-objects) + out (create-export-assets-job (:id profile) [item])] + + (t/is (th/success? out)) + + (let [result (:result out) + job (jobs/get-job th/*system* (:id result))] + + (t/testing "the answer is enough to follow the job" + (t/is (uuid? (:id result))) + (t/is (= "pending" (:status result))) + (t/is (= "export-assets" (:name result))) + (t/is (ct/inst? (:created-at result)))) + + (t/testing "and the row routes to the external worker queue" + (let [job-def (jobs/get-job-def (::jobs/defs th/*system*) :export-assets) + params (jobs/decode-params job-def (:params job))] + (t/is (= "exporter" (:queue job))) + (t/is (= 0 (:max-retries job))) + (t/is (= (:id profile) (:profile-id job))) + (t/is (= 1 (count (:exports params)))) + (t/is (= :png (:type (first (:exports params))))))) + + (t/testing "creating the job stores no artifact" + (t/is (= before (count-storage-objects))))))) + +(t/deftest create-export-assets-job-checks-the-read-permission + (let [owner (th/create-profile* 1) + other (th/create-profile* 2) + file-id (first (:file-ids (import-fixture! owner))) + out (create-export-assets-job (:id other) [(export-item file-id)])] + + (t/testing "a file the caller cannot read is not exported" + (t/is (not (th/success? out))) + (t/is (= :not-found (th/ex-type (:error out))))) + + (t/testing "and no job was created for it" + (t/is (zero? (count-jobs)))))) + +(t/deftest create-export-assets-job-rejects-an-unknown-type + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + out (create-export-assets-job (:id profile) + [(export-item file-id :type :psd)])] + + (t/is (not (th/success? out))) + ;; refused by the RPC wrapper before the command runs (the same + ;; `:params-validation` a real caller sees over HTTP) + (t/is (= :params-validation (th/ex-code (:error out)))) + (t/testing "and nothing was created" + (t/is (zero? (count-jobs)))))) + +(t/deftest create-export-assets-job-transient-expires-in-minutes + ;; A single-use export (clipboard image, plugin shape export) is + ;; consumed in the same flow: minutes, not the user ledger's days. + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + item (export-item file-id) + life (fn [out] + (let [row (th/db-get :job {:id (:id (:result out))} + :expires-at :created-at)] + (- (inst-ms (:expires-at row)) + (inst-ms (:created-at row))))) + plain (create-export-assets-job (:id profile) [item]) + quick (create-export-assets-job (:id profile) [item] :transient true)] + (t/is (th/success? plain)) + (t/is (th/success? quick)) + (t/testing "a plain export keeps the user ledger" + ;; the expiry is computed before the row is written, so the life is + ;; at most the ttl and at least it minus the insert + (t/is (<= (- (inst-ms (cf/get-jobs-user-ttl)) 1000) + (life plain) + (inst-ms (cf/get-jobs-user-ttl))))) + (t/testing "a transient export lives minutes" + (t/is (<= (- (inst-ms (ct/duration {:minutes 10})) 1000) + (life quick) + (inst-ms (ct/duration {:minutes 10}))))))) + +(t/deftest create-export-assets-job-rejects-a-mixed-scope + ;; One job renders one batch: the worker groups by scale and type and + ;; renders each group against the page of its first item, so items + ;; from two files, pages or shares would render against the wrong + ;; page. Refused here, synchronously, before any permission check. + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + page-id (uuid/next) + one (export-item file-id :page-id page-id) + other (export-item (uuid/next) :page-id page-id)] + (t/testing "items from two files are refused" + (let [out (create-export-assets-job (:id profile) [one other])] + (t/is (not (th/success? out))) + (t/is (= :mixed-export-scope (th/ex-code (:error out)))))) + (t/testing "items from two pages are refused" + (let [out (create-export-assets-job (:id profile) [one (export-item file-id)])] + (t/is (not (th/success? out))) + (t/is (= :mixed-export-scope (th/ex-code (:error out)))))) + (t/testing "and nothing was created" + (t/is (zero? (count-jobs)))))) + +(t/deftest create-export-assets-job-accepts-an-empty-suffix + ;; the export form sends `""` when the user types no suffix (it is the + ;; normal case, not an edge): the legacy `/api/export` surface took a + ;; plain string, so the job must take it too + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + item (assoc (export-item file-id) :suffix "") + out (create-export-assets-job (:id profile) [item])] + + (t/is (th/success? out)) + + (t/testing "and the empty suffix is frozen as-is for the worker" + (let [job (jobs/get-job th/*system* (:id (:result out))) + job-def (jobs/get-job-def (::jobs/defs th/*system*) :export-assets) + params (jobs/decode-params job-def (:params job))] + (t/is (= "" (:suffix (first (:exports params))))))))) + +(t/deftest create-export-assets-job-accepts-a-frames-kind + ;; the frames dialog freezes fully typed pages plus an explicit kind, + ;; so the worker joins them into one pdf instead of inferring it + ;; from missing keys + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + item (assoc (export-item file-id) :type :pdf :scale 1 :suffix "") + out (th/command! {::th/type :create-export-assets-job + ::rpc/profile-id (:id profile) + :params {:exports [item] + :kind :frames}})] + + (t/is (th/success? out)) + + (t/testing "and the kind is frozen as-is for the worker" + (let [job (jobs/get-job th/*system* (:id (:result out))) + job-def (jobs/get-job-def (::jobs/defs th/*system*) :export-assets) + params (jobs/decode-params job-def (:params job))] + (t/is (= :frames (:kind params))))))) + +(t/deftest create-export-assets-job-accepts-a-file-nameable-through-a-share + ;; the viewer that renders a file through a public share exports the + ;; same way every share fetch goes: the share id names the file, the + ;; share-link permissions answer (a logged viewer is enough), and the + ;; workspace membership is never touched + (let [owner (th/create-profile* 1 {:is-active true}) + viewer (th/create-profile* 2 {:is-active true}) + file-id (first (:file-ids (import-fixture! owner))) + share (th/command! {::th/type :create-share-link + ::rpc/profile-id (:id owner) + :file-id file-id + :pages #{(uuid/next)} + :who-comment "team" + :who-inspect "all"}) + share-id (:id (:result share)) + + out (create-export-assets-job (:id viewer) + [(export-item file-id + :share-id share-id)])] + + (t/testing "a logged viewer of the share can have the job" + (t/is (th/success? out)) + (t/is (= 1 (count-jobs))))) + + (t/testing "a share id that names another file is not a pass" + (let [owner (th/create-profile* 3 {:is-active true}) + viewer (th/create-profile* 4 {:is-active true}) + file-a (first (:file-ids (import-fixture! owner))) + file-b (first (:file-ids (import-fixture! viewer))) + share (th/command! {::th/type :create-share-link + ::rpc/profile-id (:id viewer) + :file-id file-b + :pages #{(uuid/next)} + :who-comment "team" + :who-inspect "all"}) + share-id (:id (:result share)) + out (create-export-assets-job (:id viewer) + [(export-item file-a + :share-id share-id)])] + (t/is (not (th/success? out))) + (t/is (= :not-found (th/ex-type (:error out)))) + (t/testing "and nothing was created for it" + ;; only the accepted share job of the first case exists + (t/is (= 1 (count-jobs))))))) + +(t/deftest create-export-assets-job-adds-the-job-id-to-the-audit-props + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + out (create-export-assets-job (:id profile) + [(export-item file-id)]) + props (::audit/props (meta (:result out)))] + (t/is (th/success? out)) + (t/testing "the audit of the call points at the job and its items" + (t/is (= (:id (:result out)) (:job-id props))) + (t/is (= 1 (:files props))) + (t/is (= #{:job-id :files} (set (keys props))))))) + +(t/deftest create-export-assets-job-counts-in-the-export-quote + ;; the quote of the family counts every job of it, so a mixed backlog + ;; of binfile and assets jobs fills the same quote + (with-mocks [mock {:target 'app.config/get + :return (th/config-get-mock + {:quotes-export-jobs-per-profile 1})}] + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile)))] + + (t/is (th/success? (create-export-assets-job (:id profile) + [(export-item file-id)]))) + + (t/testing "the next export job of the family is refused, whatever its type" + (let [out (create-export-binfile-job (:id profile) #{file-id})] + (t/is (not (th/success? out))) + (t/is (= :max-quote-reached (th/ex-code (:error out)))))) + + (t/is (= 1 (count-jobs)))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPORT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/test/backend_tests/rpc_management_jobs_test.clj b/backend/test/backend_tests/rpc_management_jobs_test.clj index c228289528..d5697597f1 100644 --- a/backend/test/backend_tests/rpc_management_jobs_test.clj +++ b/backend/test/backend_tests/rpc_management_jobs_test.clj @@ -8,14 +8,24 @@ (:require [app.common.logging :as l] [app.common.time :as ct] + [app.common.transit :as transit] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] + [app.http.middleware :as mw] + [app.http.session :as session] [app.jobs :as jobs] [app.metrics :as-alias mtx] - [app.rpc :as-alias rpc] + [app.rpc :as rpc] + [app.setup :as-alias setup] + [app.setup.keys :as keys] + [app.storage.tmp :as tmp] [backend-tests.helpers :as th] - [clojure.test :as t])) + [clojure.test :as t] + [cuerdas.core :as str] + [datoteka.fs :as fs] + [datoteka.io :as io] + [mockery.core :refer [with-mocks]])) (t/use-fixtures :once th/state-init) @@ -33,7 +43,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn- mk-job - [{:keys [name status params scheduled-at] + [{:keys [name status params scheduled-at profile-id tenant] :or {name "media-process" status "new" params {:x 1} @@ -41,23 +51,24 @@ (let [id (uuid/next)] (th/db-insert! :job {:id id :name name - :tenant (cf/get :tenant) + :tenant (or tenant (cf/get :tenant)) :queue "media" :params (db/json params) :priority 100 :max-retries 3 :retry-num 0 :status status + :profile-id profile-id :scheduled-at scheduled-at :created-at (ct/now) - :modified-at (ct/now)}) + :modified-at (ct/now)}) id)) (defn- get-row [id] (let [row (th/db-get :job {:id id} - :status :result :error - :started-at :completed-at :modified-at)] + :status :result :error :resource-id + :started-at :completed-at :modified-at :name)] (reduce (fn [row key] (update row key #(cond-> % (db/pgobject? %) db/decode-json-pgobject))) row @@ -85,7 +96,27 @@ (defn- mgmt [type params] - (th/management-command! (assoc params ::th/type type))) + ;; The legacy key id, pinned explicitly: the jobs methods declare a + ;; predicate instead of a set now, and the shared fallback cannot pick + ;; a first id from it. + (th/management-command! (merge {::th/type type + ::rpc/auth-key-id :exporter} + params))) + +(defn- mk-profile + ([] (mk-profile 1)) + ([i] + ;; the created profile is inactive until the email is verified, and + ;; a session can only be minted for an owner the login would accept + (th/create-profile* i {:is-active true}))) + +(defn- get-session-row + [id] + (th/db-get :http-session-v2 {:id id})) + +(defn- decode-token + [token] + (session/decode-token {::setup/props (get th/*system* :app.setup/props)} token)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; TESTS @@ -112,20 +143,22 @@ out (mgmt :claim-job {:job-id job-id :scheduled-at (ct/in-future {:minutes 5})})] (t/is (nil? (:error out))) - (t/is (= {:action :skip} (:result out))) + (t/is (= {:action :skip :status "new"} (:result out))) (t/is (= "new" (:status (get-row job-id)))))) (t/deftest claim-job-skips-terminal-and-cancelled-rows + ;; the skip says the state of the row that won: a worker that carries + ;; a stale payload sees cancelled/aborted, not a plain skip to retry on (let [completed-id (mk-job {:status "completed"}) cancelled-id (mk-job {:status "cancelled"}) aborted-id (mk-job {:status "aborted"})] - (t/is (= {:action :skip} + (t/is (= {:action :skip :status "completed"} (:result (mgmt :claim-job {:job-id completed-id :scheduled-at (ct/now)})))) - (t/is (= {:action :skip} + (t/is (= {:action :skip :status "cancelled"} (:result (mgmt :claim-job {:job-id cancelled-id :scheduled-at (ct/now)})))) - (t/is (= {:action :skip} + (t/is (= {:action :skip :status "aborted"} (:result (mgmt :claim-job {:job-id aborted-id :scheduled-at (ct/now)})))) (t/is (= "completed" (:status (get-row completed-id)))) @@ -180,7 +213,7 @@ :progress {:stage :pages :counters {:pages {:current 100 :total 100}}}})] (t/is (nil? (:error out))) - (t/is (= {:action :skip} (:result out))) + (t/is (= {:action :skip :status "completed"} (:result out))) (t/is (= [] (get-progresss job-id))))) (t/deftest report-job-progress-persists-rapid-reports @@ -322,19 +355,19 @@ (t/testing "complete on a terminal row reports skip" (let [out (mgmt :complete-job {:job-id done-id :result {:x 1}})] (t/is (nil? (:error out))) - (t/is (= {:action :skip} (:result out))))) + (t/is (= {:action :skip :status "completed"} (:result out))))) (t/testing "fail on a terminal row reports skip" (let [out (mgmt :fail-job {:job-id done-id :error {:type :internal :code "processing-error" :hint "bad image"}})] (t/is (nil? (:error out))) - (t/is (= {:action :skip} (:result out))))) + (t/is (= {:action :skip :status "completed"} (:result out))))) (t/testing "progress on a terminal row reports skip" (let [out (mgmt :report-job-progress {:job-id done-id :progress {:stage :pages :counters {:pages {:current 4 :total 10}}}})] (t/is (nil? (:error out))) - (t/is (= {:action :skip} (:result out)))))) + (t/is (= {:action :skip :status "completed"} (:result out)))))) (let [cfg (make-cfg) job-id (mk-job {}) _ (jobs/claim cfg job-id (:scheduled-at (th/db-get :job {:id job-id} :id :scheduled-at)))] @@ -393,7 +426,7 @@ :result {:v 2} :resource-id second})] (t/is (nil? (:error out))) - (t/is (= {:action :skip} (:result out))))) + (t/is (= {:action :skip :status "completed"} (:result out))))) (t/testing "the row keeps the resource it was given and no event is added" (let [row (get-row job-id)] @@ -448,3 +481,563 @@ :attempt 3})] (t/is (nil? (:error out))) (t/is (= 3 (:attempt (jobs/decode-job-error (:error (get-row job-id)))))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; CREATE-JOB-SESSION +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest create-job-session-mints-a-session-of-the-owner + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + out (mgmt :create-job-session {:job-id job-id}) + result (:result out)] + + (t/is (nil? (:error out))) + + (t/testing "the answer is a real session of the job owner" + (t/is (uuid? (:session-id result))) + (t/is (string? (:session-token result))) + (let [row (get-session-row (:session-id result))] + (t/is (some? row)) + (t/is (= (:id profile) (:profile-id row))))) + + (t/testing "the token verifies and names the session and its owner" + (let [claims (decode-token (:session-token result))] + (t/is (some? claims)) + (t/is (= (:session-id result) (:sid claims))) + (t/is (= (:id profile) (:uid claims))) + ;; the short :exp of a job session: at most an hour by default, + ;; never the absolute max-age of a login session + (t/is (pos? (- (inst-ms (:exp claims)) (inst-ms (ct/now))))) + (t/is (pos? (compare (ct/plus (ct/now) (ct/duration {:hours 1})) + (:exp claims)))))))) + +(t/deftest create-job-session-honours-the-configured-ttl + (with-mocks [mock {:target 'app.config/get-job-session-ttl + :return (ct/duration {:minutes 5})}] + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + out (mgmt :create-job-session {:job-id job-id}) + claims (decode-token (get-in out [:result :session-token]))] + (t/is (nil? (:error out))) + (t/is (pos? (- (inst-ms (:exp claims)) (inst-ms (ct/now))))) + (t/is (pos? (compare (ct/plus (ct/now) (ct/duration {:minutes 5})) + (:exp claims))))))) + +(t/deftest create-job-session-mints-a-fresh-session-every-call + ;; every call is self-contained: a worker retry never waits on a + ;; session that a settle may already have closed + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + first (mgmt :create-job-session {:job-id job-id}) + second (mgmt :create-job-session {:job-id job-id})] + (t/is (nil? (:error first))) + (t/is (nil? (:error second))) + (t/is (not= (get-in first [:result :session-id]) + (get-in second [:result :session-id]))))) + +(t/deftest create-job-session-refuses-a-job-that-is-not-running + (let [profile (mk-profile)] + (doseq [status ["new" "scheduled" "completed" "failed" "cancelled" "aborted"]] + (let [job-id (mk-job {:status status :profile-id (:id profile)}) + out (mgmt :create-job-session {:job-id job-id})] + (t/is (some? (:error out))) + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :invalid-job-state (th/ex-code (:error out)))) + (t/testing "and no session was minted" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM http_session_v2"]))))))))) + +(t/deftest create-job-session-refuses-an-unknown-job + (let [out (mgmt :create-job-session {:job-id (uuid/next)})] + (t/is (some? (:error out))) + (t/is (= :not-found (th/ex-type (:error out)))) + (t/is (= :job-not-found (th/ex-code (:error out)))))) + +(t/deftest create-job-session-refuses-a-job-without-owner + (let [job-id (mk-job {:status "running"}) + out (mgmt :create-job-session {:job-id job-id})] + (t/is (some? (:error out))) + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :job-without-profile (th/ex-code (:error out)))))) + +(t/deftest create-job-session-refuses-an-owner-the-login-would-refuse + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)})] + + (t/testing "a blocked owner is a restriction, not a fault" + (th/db-update! :profile {:is-blocked true} {:id (:id profile)}) + (let [out (mgmt :create-job-session {:job-id job-id})] + (t/is (= :restriction (th/ex-type (:error out)))) + (t/is (= :profile-blocked (th/ex-code (:error out)))))) + + (t/testing "an inactive owner is refused like the login does" + (th/db-update! :profile {:is-blocked false :is-active false} {:id (:id profile)}) + (let [out (mgmt :create-job-session {:job-id job-id})] + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :profile-not-active (th/ex-code (:error out))))))) + + (t/testing "a deleted owner does not exist" + (let [profile (mk-profile 2) + job-id (mk-job {:status "running" + :profile-id (:id profile)})] + (th/db-update! :profile {:deleted-at (ct/now)} {:id (:id profile)}) + (let [out (mgmt :create-job-session {:job-id job-id})] + (t/is (= :not-found (th/ex-type (:error out)))) + (t/is (= :profile-not-found (th/ex-code (:error out)))))))) + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; COMPLETE-JOB MULTIPART +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- artifact-tempfile + "An on-disk artifact, as the multipart parser would leave one for the + server: a path with the bytes of the produced export in it." + [content] + (let [path (fs/create-tempfile :dir tmp/default-tmp-dir + :prefix "test-artifact-")] + (with-open [out (io/output-stream path)] + (io/write* out (.getBytes ^String content "UTF-8"))) + path)) + +(defn- multipart-complete + [job-id path & {:as extra}] + (-> (merge {:job-id job-id + :content {:path path + :filename "result.zip" + :size (fs/size path) + :mtype "application/zip"}} + extra) + (assoc ::th/type :complete-job + ::rpc/auth-key-id :exporter) + (th/with-multipart) + (th/management-command!))) + +(t/deftest claim-job-skips-a-job-of-another-tenant-like-a-missing-one + (let [job-id (mk-job {:status "new" :tenant "other-tenant"})] + ;; indistinguishable from a missing row: a bare skip, no status + (t/is (= {:action :skip} + (:result (mgmt :claim-job {:job-id job-id + :scheduled-at (ct/now)})))))) + +(t/deftest claim-job-accepts-the-suffixed-key-of-its-tenant + (let [scheduled-at (ct/now) + job-id (mk-job {:scheduled-at scheduled-at}) + tenant-key (keyword (str "exporter:" (cf/get :tenant))) + out (mgmt :claim-job {:job-id job-id + :scheduled-at scheduled-at + ::rpc/auth-key-id tenant-key})] + (t/is (nil? (:error out))) + (t/is (= :run (:action (:result out)))))) + +(t/deftest report-job-progress-skips-a-job-of-another-tenant-without-writing + (let [job-id (mk-job {:status "running" :tenant "other-tenant"}) + out (mgmt :report-job-progress {:job-id job-id + :progress {:stage :pages + :counters {:pages {:current 1 + :total 2}}}})] + (t/is (nil? (:error out))) + (t/is (= {:action :skip} (:result out))) + (t/testing "no progress event was stored" + (t/is (zero? (count-job-events job-id)))))) + +(t/deftest create-job-session-rejects-a-job-of-another-tenant-as-missing + (let [job-id (mk-job {:status "running" :tenant "other-tenant"}) + out (mgmt :create-job-session {:job-id job-id})] + (t/is (= :not-found (th/ex-type (:error out)))) + (t/is (= :job-not-found (th/ex-code (:error out)))))) + +(t/deftest complete-job-skips-a-job-of-another-tenant-without-storing + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile) + :tenant "other-tenant"}) + path (artifact-tempfile "penpot") + session-id (uuid/next) + out (multipart-complete job-id path :session-id session-id)] + (t/is (nil? (:error out))) + ;; bare skip, like a missing row: no status to tell tenants apart + (t/is (= {:action :skip} (:result out))) + (t/testing "no artifact was stored and the row is untouched" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))) + (t/is (= "running" (:status (get-row job-id))))))) + +(t/deftest fail-job-skips-a-job-of-another-tenant + (let [job-id (mk-job {:status "running" :tenant "other-tenant"}) + out (mgmt :fail-job {:job-id job-id + :error {:type :internal + :code :boom + :hint "boom"}})] + (t/is (nil? (:error out))) + (t/is (= {:action :skip} (:result out))) + (t/is (= "running" (:status (get-row job-id)))))) + +(t/deftest complete-job-params-form-skips-a-job-of-another-tenant + ;; The form without `:content` carries `:result`: a foreign tenant gets + ;; a bare skip, so neither the write nor the row's state leaks. + (let [job-id (mk-job {:status "running" :tenant "other-tenant"}) + out (mgmt :complete-job {:job-id job-id + :result {:x 1}})] + (t/is (nil? (:error out))) + (t/is (= {:action :skip} (:result out))) + (t/is (= "running" (:status (get-row job-id)))) + (t/is (nil? (:completed-at (get-row job-id)))))) + +(t/deftest complete-job-refuses-an-upload-outside-multipart + ;; The upload map names a server path: only a request the multipart + ;; parser flagged may carry one. A transit body is refused before any + ;; store. + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot") + out (mgmt :complete-job {:job-id job-id + :content {:path path + :filename "result.zip" + :size (fs/size path) + :mtype "application/zip"}})] + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :params-validation (th/ex-code (:error out)))) + (t/testing "nothing was stored and the row is untouched" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))) + (t/is (= "running" (:status (get-row job-id))))))) + +(t/deftest complete-job-refuses-an-artifact-path-outside-the-upload-areas + ;; Even over multipart, the parser only writes under its own temp + ;; areas: anything else (e.g. an internal config file) is refused by + ;; the same media check the other upload methods use. + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (fs/create-tempfile :prefix "outside-upload-")] + (with-open [out (io/output-stream path)] + (io/write* out (.getBytes "secret" "UTF-8"))) + (let [out (multipart-complete job-id path)] + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :params-validation (th/ex-code (:error out)))) + (t/testing "nothing was stored and the row is untouched" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))) + (t/is (= "running" (:status (get-row job-id)))))))) + +(t/deftest close-job-session-leaves-a-session-of-another-profile-alone + ;; The session id travels in the request: a shared-key holder must not + ;; be able to delete another profile's login through a settle. + (let [owner (mk-profile 1) + other (mk-profile 2) + job-id (mk-job {:status "running" + :profile-id (:id owner)}) + stranger (session/create-session (get th/*system* :app.http.session/manager) + {:profile-id (:id other)}) + out (multipart-complete job-id (artifact-tempfile "penpot") + :session-id (:id stranger))] + (t/is (nil? (:error out))) + (t/is (= {:action :run} (:result out))) + (t/testing "the stranger session is still there" + (t/is (some? (get-session-row (:id stranger))))))) + +(t/deftest derive-exporter-key-matches-the-node-vector + ;; The same golden value the exporter test pins: a change on either + ;; side of the derivation fails here instead of in a deployment. + (t/is (= "LXzydEEZEVAxbiNAzXV7mPSm6wYbrS7Omp4yMZ4gmOY" + (keys/derive-exporter-key "test-secret-key" "default")))) + +(t/deftest complete-job-multipart-stores-the-artifact-and-completes + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot") + out (multipart-complete job-id path) + row (get-row job-id) + rid (:resource-id row)] + (t/is (nil? (:error out))) + (t/is (= {:action :run} (:result out))) + (t/is (= "completed" (:status row))) + + (t/testing "the artifact lives in the job-resource bucket, owned by the profile" + (let [{:keys [metadata]} (first (th/db-exec! ["SELECT metadata + FROM storage_object WHERE id = ?" rid])) + metadata (cond-> metadata (db/pgobject? metadata) db/decode-transit-pgobject)] + (t/is (some? rid)) + (t/is (= "job-resource" (:bucket metadata))) + (t/is (= (:id profile) (:profile-id metadata))))) + + (t/testing "the result names the artifact and how to download it" + (let [result (:result row)] + (t/is (= "result.zip" (:filename result))) + (t/is (= "application/zip" (:mtype result))) + (t/is (= (fs/size path) (:size result))) + (t/is (str/ends-with? (:resource-uri result) (str "/assets/by-id/" rid))))))) + +(t/deftest complete-job-multipart-ignores-legacy-text-fields + ;; The part is the single source: an exporter that still sends the old + ;; `filename`/`mtype` text fields completes fine, and the part wins. + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot") + out (multipart-complete job-id path + :filename "ignored.zip" + :mtype "image/gif")] + (t/is (nil? (:error out))) + (t/is (= {:action :run} (:result out))) + (let [result (:result (get-row job-id))] + (t/is (= "result.zip" (:filename result))) + (t/is (= "application/zip" (:mtype result)))))) + +(t/deftest complete-job-multipart-closes-the-render-session + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + session-id (get-in (mgmt :create-job-session {:job-id job-id}) + [:result :session-id]) + path (artifact-tempfile "penpot") + out (multipart-complete job-id path :session-id session-id)] + (t/is (nil? (:error out))) + (t/is (= {:action :run} (:result out))) + (t/testing "the session is gone, deleted after the commit" + (t/is (nil? (get-session-row session-id)))))) + +(t/deftest complete-job-multipart-refuses-a-mtype-outside-the-allowlist + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot") + out (multipart-complete job-id path + :content {:path path + :filename "result.gif" + :size (fs/size path) + :mtype "image/gif"})] + (t/is (some? (:error out))) + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :media-type-not-allowed (th/ex-code (:error out)))) + (t/testing "no artifact was stored and the row is untouched" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))) + (t/is (= "running" (:status (get-row job-id))))))) + +(t/deftest complete-job-multipart-refuses-a-part-without-filename + ;; The part is the single source of the download name: a part without + ;; one fails the upload schema before anything is stored. + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot") + out (mgmt :complete-job {:job-id job-id + :content {:path path + :size (fs/size path) + :mtype "application/zip"}})] + (t/is (some? (:error out))) + (t/is (= :validation (th/ex-type (:error out)))) + (t/testing "no artifact was stored and the row is untouched" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))) + (t/is (= "running" (:status (get-row job-id))))))) + +(t/deftest complete-job-multipart-refuses-a-result-over-the-cap + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot")] + (with-mocks [mock {:target 'app.config/get-exporter-max-result-size + :return 0}] + (let [out (multipart-complete job-id path)] + (t/is (some? (:error out))) + (t/is (= :validation (th/ex-type (:error out)))) + (t/is (= :request-body-too-large (th/ex-code (:error out)))) + (t/testing "no artifact was stored and the row is untouched" + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))) + (t/is (= "running" (:status (get-row job-id))))))))) + +(t/deftest complete-job-multipart-closes-the-session-on-a-lost-race + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + session-id (get-in (mgmt :create-job-session {:job-id job-id}) + [:result :session-id]) + path (artifact-tempfile "penpot")] + (with-mocks [mock {:target 'app.jobs/complete + :return 0}] + (let [out (multipart-complete job-id path :session-id session-id)] + (t/is (nil? (:error out))) + (t/is (= :skip (-> out :result :action))) + (t/testing "the session is gone even though the write lost" + (t/is (nil? (get-session-row session-id)))) + (t/testing "the stored artifact was released, the row untouched" + (t/is (nil? (:resource-id (get-row job-id)))) + (t/is (= "running" (:status (get-row job-id))))))))) + +(t/deftest complete-job-multipart-of-a-terminal-job-skips-and-stores-nothing + (let [profile (mk-profile) + job-id (mk-job {:status "cancelled" + :profile-id (:id profile)}) + path (artifact-tempfile "penpot") + out (multipart-complete job-id path)] + (t/is (nil? (:error out))) + ;; the bytes are not used: the completion lost the race with the + ;; cancel, and no object is left for the storage GC to find + (t/is (= {:action :skip :status "cancelled"} (:result out))) + (t/is (zero? (count (th/db-exec! ["SELECT id FROM storage_object"])))))) + +(t/deftest fail-job-closes-the-render-session + (let [profile (mk-profile) + job-id (mk-job {:status "running" + :profile-id (:id profile)}) + session-id (get-in (mgmt :create-job-session {:job-id job-id}) + [:result :session-id]) + out (mgmt :fail-job {:job-id job-id + :error {:type :internal + :code "processing-error" + :hint "bad image"} + :session-id session-id})] + (t/is (nil? (:error out))) + (t/is (= {:action :run} (:result out))) + (t/is (= "failed" (:status (get-row job-id)))) + (t/is (nil? (get-session-row session-id))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; WIRE CONTRACT: what the exporter actually receives and sends +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; The claim carries the row's stored shapes (plain JSON: every id a +;; string, every enum its name). The exporter normalizes them to typed +;; values. These tests pin the backend half of that contract: the claim +;; returns stored shapes untouched, the dispatcher bytes are exact, and +;; the HTTP edges (auth, dispatch, body parsing) run production code. + +(t/deftest claim-job-returns-decoded-typed-params + ;; The row stores JSON (strings); the claim decodes with the row's + ;; job-def, so the worker gets typed values (uuids, keywords) that + ;; transit carries natively. A row whose job-def is gone answers + ;; untouched (see the media-process claims above). + (let [file-id (uuid/next) + object-id (uuid/next) + scheduled-at (ct/now) + job-id (mk-job {:name "export-assets" + :scheduled-at scheduled-at + :params {:exports [{:file-id (str file-id) + :page-id (str (uuid/next)) + :object-id (str object-id) + :type "png" + :name "Board" + :suffix "" + :scale 2}] + :name "the export"}}) + out (mgmt :claim-job {:job-id job-id + :scheduled-at scheduled-at}) + params (get-in out [:result :params])] + (t/is (nil? (:error out))) + (t/is (= "export-assets" (get-in out [:result :name]))) + (t/testing "ids travel as typed uuids" + (t/is (= file-id (get-in params [:exports 0 :file-id]))) + (t/is (= object-id (get-in params [:exports 0 :object-id])))) + (t/testing "enums travel as keywords" + (t/is (= :png (get-in params [:exports 0 :type])))))) + +(t/deftest claim-params-decode-to-typed-values-with-the-job-def + ;; The row stores JSON (strings); the decoder the runner would use + ;; turns them into the typed values the exporter validates against. + ;; The claim path returns the stored shapes (see above); this pins + ;; the decode half both sides rely on. + (let [file-id (uuid/next) + object-id (uuid/next) + stored {:exports [{:file-id (str file-id) + :page-id (str (uuid/next)) + :object-id (str object-id) + :type "png" + :name "Board" + :suffix "" + :scale 2}] + :name "the export"} + decoded (jobs/decode-params + (jobs/get-job-def (jobs/get-defs (make-cfg)) :export-assets) + stored)] + (t/is (= file-id (get-in decoded [:exports 0 :file-id]))) + (t/is (= object-id (get-in decoded [:exports 0 :object-id]))) + (t/is (= :png (get-in decoded [:exports 0 :type]))) + (t/is (= "the export" (:name decoded))))) + +(t/deftest shared-key-middleware-allows-the-exporter-key + (let [secret "test-secret" + derived (keys/derive-exporter-key secret "acme") + handler (#'mw/wrap-shared-key-auth (fn [request] request) + {:exporter "test-key"} + secret) + allowed (th/make-dummy-request :method :post + :headers {"x-shared-key" "exporter test-key"}) + denied (th/make-dummy-request :method :post + :headers {"x-shared-key" "exporter wrong-key"}) + missing (th/make-dummy-request :method :post) + tenant (th/make-dummy-request :method :post + :headers {"x-shared-key" (str "exporter:acme " derived)}) + foreign (th/make-dummy-request :method :post + :headers {"x-shared-key" "exporter:acme wrong-key"})] + (t/testing "the declared key reaches the handler with its id" + (let [request (handler allowed)] + (t/is (= :exporter (:app.http/auth-key-id request))))) + (t/testing "a wrong key is a 403" + (t/is (= 403 (:yetti.response/status (handler denied))))) + (t/testing "a missing key is a 403" + (t/is (= 403 (:yetti.response/status (handler missing))))) + (t/testing "a per-tenant id the secret derives reaches the handler too" + (let [request (handler tenant)] + (t/is (= :exporter:acme (:app.http/auth-key-id request))))) + (t/testing "a per-tenant id with a wrong key is a 403" + (t/is (= 403 (:yetti.response/status (handler foreign))))))) + +(t/deftest management-dispatch-routes-by-method-name + (t/testing "the stub method runs with the multipart flag of the request" + (let [handler (rpc/make-rpc-handler + {:ping [{} (fn [data] + (select-keys data [:app.rpc/auth-key-id + :app.rpc/is-multipart]))]}) + request (-> (th/make-dummy-request + :method :post + :headers {"content-type" "multipart/form-data; boundary=test"}) + (assoc :path-params {:method-name "ping"} + :params {} + :app.http/auth-key-id :exporter)) + response (handler request)] + (t/is (= 200 (:yetti.response/status response))) + (t/is (true? (:app.rpc/is-multipart (:yetti.response/body response)))) + (t/is (= :exporter (:app.rpc/auth-key-id (:yetti.response/body response)))))) + (t/testing "the real claim method resolves through the same dispatch" + (let [scheduled-at (ct/now) + job-id (mk-job {:scheduled-at scheduled-at}) + handler (rpc/make-rpc-handler + (get th/*system* :app.rpc/management-methods)) + request (-> (th/make-dummy-request + :method :post + :headers {"content-type" "application/transit+json"}) + (assoc :path-params {:method-name "claim-job"} + :params {:job-id job-id :scheduled-at scheduled-at} + :app.http/auth-key-id :exporter)) + response (handler request) + body (:yetti.response/body response)] + (t/is (= 200 (:yetti.response/status response))) + (t/is (= :run (:action body))) + (t/is (= "media-process" (:name body)))))) + +(t/deftest management-body-parses-transit-by-content-type + (let [payload {:job-id (uuid/next) + :scheduled-at (ct/inst "2026-10-07T09:00:00Z")} + request (th/make-dummy-request + :method :post + :headers {"content-type" "application/transit+json"} + :body-bytes (transit/encode-str payload))] + (t/testing "a transit body decodes to the params map" + (t/is (= payload (#'mw/read-body-params + request nil "application/transit+json"))))) + (t/testing "a broken body raises malformed, not a 500" + (let [request (th/make-dummy-request + :method :post + :headers {"content-type" "application/transit+json"} + :body-bytes "{nope") + metrics (get th/*system* :app.metrics/metrics)] + (try + (#'mw/read-body-params request metrics "application/transit+json") + (t/is false "should have raised") + (catch clojure.lang.ExceptionInfo cause + (t/is (= :validation (th/ex-type cause))) + (t/is (= :malformed-json (th/ex-code cause)))))))) diff --git a/backend/test/backend_tests/rpc_management_shared_keys_test.clj b/backend/test/backend_tests/rpc_management_shared_keys_test.clj index ea78696293..adc2a5b869 100644 --- a/backend/test/backend_tests/rpc_management_shared_keys_test.clj +++ b/backend/test/backend_tests/rpc_management_shared_keys_test.clj @@ -29,11 +29,21 @@ "every management namespace must be scanned") (doseq [[method-name [mdata _]] methods] (let [ns-name (:ns mdata) - expected (get expected-shared-keys ns-name)] + expected (get expected-shared-keys ns-name) + allowed (::rpc/allowed-shared-keys mdata)] (t/is (some? expected) (str "unexpected management namespace for " method-name ": " ns-name)) - (t/is (= expected (::rpc/allowed-shared-keys mdata)) - (str "wrong ::rpc/allowed-shared-keys for " method-name " in " ns-name)))))) + (if (= "app.rpc.management.jobs" ns-name) + ;; The jobs methods declare a predicate: the bare legacy id + ;; plus any `exporter:` suffix. + (do + (t/is (fn? allowed)) + (t/is (allowed :exporter)) + (t/is (allowed :exporter:acme)) + (t/is (not (allowed :nexus))) + (t/is (not (allowed nil)))) + (t/is (= expected allowed) + (str "wrong ::rpc/allowed-shared-keys for " method-name " in " ns-name))))))) (t/deftest management-command-uses-the-key-from-the-request-metadata (let [out (th/management-command! diff --git a/backend/test/backend_tests/worker_dispatcher_test.clj b/backend/test/backend_tests/worker_dispatcher_test.clj index c498214beb..fb0f9ceff0 100644 --- a/backend/test/backend_tests/worker_dispatcher_test.clj +++ b/backend/test/backend_tests/worker_dispatcher_test.clj @@ -325,3 +325,15 @@ (t/is (= "running" (:status row))) (t/is (nil? (:error row))) (t/is (= [] (:events row))))))) + +(t/deftest dispatcher-payload-bytes-are-the-exporter-contract + ;; The only bytes the exporter parses: a JSON array of exactly two + ;; items, [job-id scheduled-at], the instant in ct/format-inst shape. + ;; The worker echoes scheduled-at back byte-identical on claim, so any + ;; drift here breaks the claim-back rule (and the exporter test pins + ;; the same literal from its side). + (let [payload (#'wdisp/encode-payload + {:id (uuid/parse "03b7300f-11e6-8131-8008-e4b09b780a0e") + :scheduled-at (ct/inst "2026-10-07T09:00:00Z")})] + (t/is (= "[\"03b7300f-11e6-8131-8008-e4b09b780a0e\",\"2026-10-07T09:00:00Z\"]" + payload)))) diff --git a/common/src/app/common/export.cljc b/common/src/app/common/export.cljc new file mode 100644 index 0000000000..dad8e37644 --- /dev/null +++ b/common/src/app/common/export.cljc @@ -0,0 +1,56 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.common.export + "The shapes of an asset export job, shared by its producer (the + backend, which freezes them in the job row) and its consumer (the + exporter worker, which renders them). + + One source of truth on purpose: the row stores plain JSON (every id + and enum a string) and each side decodes with the repo transformer + (`sm/json-transformer`), so a drift between what the backend writes + and what the exporter reads fails in tests instead of in production." + (:require + [app.common.schema :as sm])) + +(def schema:export-item + "One thing to render: where it lives (file, page, object), what to + produce of it (type, scale) and what to call it. The same item shape + the legacy `/api/export` surface received, frozen in the params of + the job; the optional `share-id` marks the item as coming from a + public share." + [:map {:title "export-item" :closed true} + [:file-id ::sm/uuid] + [:page-id ::sm/uuid] + [:object-id ::sm/uuid] + [:type [::sm/one-of #{:png :jpeg :webp :svg :pdf}]] + [:name ::sm/text] + ;; a plain string on purpose, empty included: the export form sends + ;; `""` when the user types no suffix, and the legacy `/api/export` + ;; surface took it that way; the worker concatenates it as-is + [:suffix {:optional true} :string] + [:scale ::sm/number] + [:share-id {:optional true} ::sm/uuid]]) + +(def schema:export-items + "The items of one assets run, as the exporter validates them after + normalizing the claim." + [:vector {:min 1} schema:export-item]) + +(def schema:params + "Business params of an asset export: the items to render, frozen when + the job was created, plus the options that shape the output. There is + no `wait` here: a job is followed by its id, not waited on. The + optional `kind` declares the export the caller asked for (`:frames` + joins pages into one pdf); absent it is a shapes export. The worker + reads it as declared instead of inferring it from the items." + [:map {:title "export-assets-params" :closed true} + [:exports [:vector {:min 1} schema:export-item]] + [:kind {:optional true} [::sm/one-of #{:shapes :frames}]] + [:name {:optional true} ::sm/text] + [:skip-children {:optional true} ::sm/boolean] + [:force-multiple {:optional true} ::sm/boolean] + [:is-wasm {:optional true} ::sm/boolean]]) diff --git a/common/src/app/common/media.cljc b/common/src/app/common/media.cljc index a82759a7bd..99dfad3c3c 100644 --- a/common/src/app/common/media.cljc +++ b/common/src/app/common/media.cljc @@ -25,6 +25,18 @@ (def tempfile-types (conj image-types "application/pdf" "application/zip")) +(def export-artifact-types + "The content types an export job may leave as its result artifact: + what the exporter produces, nothing more. The `complete-job` call of + the management API refuses any other type for the bytes a worker + sends." + #{"application/zip" + "application/pdf" + "image/png" + "image/jpeg" + "image/webp" + "image/svg+xml"}) + (def storage-object-types "Every content-type the system stores in storage objects: uploaded images and fonts, generated thumbnails, temporary files and the diff --git a/common/test/common_tests/export_test.cljc b/common/test/common_tests/export_test.cljc new file mode 100644 index 0000000000..c35ca0edc2 --- /dev/null +++ b/common/test/common_tests/export_test.cljc @@ -0,0 +1,62 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.export-test + "The asset-export shapes both runtimes share: stored JSON (every id + and enum a string) decodes to typed values with the repo transformer, + and typed values decode to themselves. Runs on JVM and JS because the + backend writes the row and the exporter reads the claim." + (:require + [app.common.export :as export] + [app.common.schema :as sm] + [clojure.test :as t])) + +(def ^:private stored-item + "One item as the job row holds it: plain JSON, strings everywhere." + {:file-id "c9c0b00d-4f50-8024-8008-d07a282a13ee" + :page-id "c9c0b00d-4f50-8024-8008-d07a282a13ef" + :object-id "1ac0612f-1ee8-8023-8008-d07a2aa96040" + :type "png" + :name "Board" + :suffix "" + :scale 2}) + +(def ^:private decode + (sm/decoder export/schema:export-item sm/json-transformer)) + +(t/deftest stored-strings-decode-to-typed-values + (let [item (decode stored-item)] + (t/testing "ids decode to uuids" + (t/is (uuid? (:file-id item))) + (t/is (uuid? (:page-id item))) + (t/is (uuid? (:object-id item))) + (t/is (= #uuid "1ac0612f-1ee8-8023-8008-d07a2aa96040" + (:object-id item)))) + (t/testing "the type decodes to a keyword" + (t/is (= :png (:type item)))) + (t/testing "plain values ride through untouched" + (t/is (= "Board" (:name item))) + (t/is (= "" (:suffix item))) + (t/is (= 2 (:scale item)))))) + +(t/deftest typed-values-decode-to-themselves + (let [typed {:file-id #uuid "c9c0b00d-4f50-8024-8008-d07a282a13ee" + :page-id #uuid "c9c0b00d-4f50-8024-8008-d07a282a13ef" + :object-id #uuid "1ac0612f-1ee8-8023-8008-d07a2aa96040" + :type :png + :name "Board" + :suffix "" + :scale 2}] + (t/is (= typed (decode typed))))) + +(t/deftest params-schema-owns-kind-and-options + (let [decode-params (sm/decoder export/schema:params sm/json-transformer) + params (decode-params {:exports [stored-item] + :kind "frames" + :name "the export"})] + (t/is (= :frames (:kind params))) + (t/is (= "the export" (:name params))) + (t/is (= 1 (count (:exports params)))))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index 9a774d271d..af6d785561 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -18,6 +18,7 @@ [common-tests.colors-test] [common-tests.data-test] [common-tests.exceptions-test] + [common-tests.export-test] [common-tests.files-builder-test] [common-tests.files-changes-test] [common-tests.files-migrations-0025-test] @@ -126,6 +127,7 @@ 'common-tests.colors-test 'common-tests.data-test 'common-tests.exceptions-test + 'common-tests.export-test 'common-tests.files-builder-test #?(:cljs 'common-tests.fonts-test) 'common-tests.files-changes-test diff --git a/docker/devenv/defaults.env b/docker/devenv/defaults.env index 7f9c107494..efae883c50 100644 --- a/docker/devenv/defaults.env +++ b/docker/devenv/defaults.env @@ -23,6 +23,13 @@ PENPOT_USER_DATA_VOLUME=penpotdev_user_data # database number (0 for ws0, 1 for ws1, ...) set by instance-env-overrides. PENPOT_HOST=devenv PENPOT_PUBLIC_URI=https://localhost:3449 +# In-container address of the platform edge (nginx), used to reach the +# backend without leaving through the public URI (which is host-side and +# unreachable from inside the container). Caddy only emulates the external +# HTTPS layer; no service goes outside to come back in. The exporter worker +# uses it for the management API; per-container loopback, so ws1+ overrides +# never touch it. +PENPOT_INTERNAL_URI=http://localhost:4449 PENPOT_DATABASE_URI=postgresql://postgres/penpot PENPOT_DATABASE_USERNAME=penpot PENPOT_DATABASE_PASSWORD=penpot diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index 9a3cfd3222..edcb87f14d 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -137,10 +137,6 @@ http { } # On production, this is controlled by ELB - location /api/export { - proxy_pass http://127.0.0.1:6061; - } - location = /.well-known/oauth-authorization-server/api/mcp/oauth { proxy_pass http://127.0.0.1:6060; proxy_http_version 1.1; diff --git a/docker/images/docker-compose.yaml b/docker/images/docker-compose.yaml index 5bca9bd6a0..05b2519293 100644 --- a/docker/images/docker-compose.yaml +++ b/docker/images/docker-compose.yaml @@ -225,6 +225,9 @@ services: # PENPOT_MCP_BACKEND_URI: http://penpot-backend:6060/ penpot-exporter: + # The exporter is a consumer's process: it polls the backend's + # `:exporter` job queue through valkey and calls the backend's + # management API to claim, report and settle every export job. image: "penpotapp/exporter:${PENPOT_VERSION:-2.18}" restart: always @@ -239,18 +242,17 @@ services: << : [*penpot-flags, *penpot-secret-key, *penpot-public-uri] # Don't touch it; this uses an internal docker network to - # communicate with the frontend. + # reach the backend's management API and the frontend of the + # assets the exports render. PENPOT_INTERNAL_URI: http://penpot-frontend:8080 - ## Valkey (or previously Redis) is used for the websockets notifications - ## and for storing the state export jobs + ## Valkey (or previously Redis) carries the export jobs queue the + ## backend dispatcher pushes to, plus the websockets + ## notifications PENPOT_REDIS_URI: redis://penpot-valkey/0 - # PENPOT_EXPORTER_MAX_CONCURRENT_JOBS: 4 - # PENPOT_EXPORTER_MAX_JOBS_PER_PROFILE: 2 - # PENPOT_EXPORTER_QUEUE_MAX: 64 # PENPOT_EXPORTER_JOB_TTL: 3600 - # PENPOT_WASM_WORKER_POOL_MAX: 2 + # PENPOT_EXPORTER_WORKER_CONCURRENCY: 2 # PENPOT_WASM_WORKER_POOL_MIN: 1 # PENPOT_WASM_WORKER_IDLE_TIMEOUT: 300 # PENPOT_WASM_WORKER_IMAGE_CACHE_SIZE: 134217728 diff --git a/docker/images/files/nginx-entrypoint.sh b/docker/images/files/nginx-entrypoint.sh index d60a5d691f..e43473ed72 100644 --- a/docker/images/files/nginx-entrypoint.sh +++ b/docker/images/files/nginx-entrypoint.sh @@ -69,13 +69,12 @@ else fi export PENPOT_BACKEND_URI=${PENPOT_BACKEND_URI:-http://penpot-backend:6060} -export PENPOT_EXPORTER_URI=${PENPOT_EXPORTER_URI:-http://penpot-exporter:6061} export PENPOT_HTTP_SERVER_MAX_BODY_SIZE=${PENPOT_HTTP_SERVER_MAX_BODY_SIZE:-367001600} # Default to 350MiB export PENPOT_IPV6_LISTEN_DIRECTIVE=${PENPOT_IPV6_LISTEN_DIRECTIVE:-"listen [::]:8080 default_server reuseport backlog=16384;"} if is_truthy "${PENPOT_DISABLE_IPV6_LISTEN:-}"; then export PENPOT_IPV6_LISTEN_DIRECTIVE="" fi -envsubst "\$PENPOT_BACKEND_URI,\$PENPOT_EXPORTER_URI,\$PENPOT_HTTP_SERVER_MAX_BODY_SIZE,\$PENPOT_IPV6_LISTEN_DIRECTIVE,\$PENPOT_PUBLIC_PATH,\$PENPOT_PUBLIC_PATH_DIRECTIVES" \ +envsubst "\$PENPOT_BACKEND_URI,\$PENPOT_HTTP_SERVER_MAX_BODY_SIZE,\$PENPOT_IPV6_LISTEN_DIRECTIVE,\$PENPOT_PUBLIC_PATH,\$PENPOT_PUBLIC_PATH_DIRECTIVES" \ < /tmp/nginx.conf.template > /etc/nginx/nginx.conf if [[ $PENPOT_FLAGS == *"enable-admin-console"* ]]; then diff --git a/docker/images/files/nginx.conf.template b/docker/images/files/nginx.conf.template index 54d62e18eb..c1cce625cb 100644 --- a/docker/images/files/nginx.conf.template +++ b/docker/images/files/nginx.conf.template @@ -150,11 +150,6 @@ http { proxy_pass $PENPOT_BACKEND_URI/link-preview$is_args$args; } - location /api/export { - proxy_set_header Host $proxy_host; - proxy_pass $PENPOT_EXPORTER_URI; - } - location = /.well-known/oauth-authorization-server/api/mcp/oauth { proxy_set_header Host $proxy_host; proxy_pass $PENPOT_BACKEND_URI$request_uri; diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 4759855e14..164c811c5a 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -898,15 +898,14 @@ them. In case you have a custom setup, you probably need to configure the following environment variables on the frontend container: -To connect the frontend to the exporter and backend, you need to fill out these environment variables. +To connect the frontend to the backend, you need to fill out this environment variable. ```bash # Frontend PENPOT_BACKEND_URI: http://your-penpot-backend:6060 -PENPOT_EXPORTER_URI: http://your-penpot-exporter:6061 ``` -These variables are used for generate correct nginx.conf file on container startup. +This variable is used to generate a correct nginx.conf file on container startup. ### Exporter @@ -917,9 +916,10 @@ The exporter uses this variable: PENPOT_INTERNAL_URI: http://penpot-frontend:8080 ``` -- `PENPOT_INTERNAL_URI`: The URI used by the exporter's headless browser to - communicate with the frontend (internal Docker network). Defaults to - `PENPOT_PUBLIC_URI` if not set. The default value +- `PENPOT_INTERNAL_URI`: The URI the exporter uses to reach the backend + management API, where it claims, reports and settles its export jobs, and + the frontend that serves the assets it renders (internal Docker network). + Defaults to `PENPOT_PUBLIC_URI` if not set. The default value `http://penpot-frontend:8080` used in the docker-compose is a good default and it is recommended to keep it unchanged. diff --git a/exporter/README.md b/exporter/README.md index 5e3986b90c..ed6271b289 100644 --- a/exporter/README.md +++ b/exporter/README.md @@ -1,100 +1,74 @@ # Exporter -Node service that renders shapes and files to bitmap, SVG and PDF. Wasm exports -are **jobs**: created over HTTP, admitted by a scheduler with bounded -concurrency, and persisted in Redis so their state can be queried and cancelled. -The legacy entry point, which is what the browser backend still goes through, -runs the export as soon as it is asked for, with no admission control. +Node process that renders shapes, frames and files to bitmap, SVG and PDF. It +is a **consumer**: it polls the backend's `:exporter` job queue on Redis and +drives each export through the backend's management API — claim, report the +milestones, and settle (`complete-job` multipart with the artifact, or +`fail-job` on error). There is no HTTP surface of its own and nothing is +persisted on this side: the row of the job, its cancel button and its result +all live in the backend. -## HTTP API +## How a job runs -Mounted under `/api/export` (the router matches on the path *after* that prefix, -so it also works when the process is hit directly on `/`). +1. The backend dispatcher pushes `[job-id, scheduled-at]` (JSON) into + `penpot.worker.queue::exporter`. +2. One of the `K` pollers pops it and owns that connection for the whole run: + the blocking pop is both the semaphore of the slot and the backpressure, so + the process never runs more exports than pollers. `K` is + `PENPOT_EXPORTER_WORKER_CONCURRENCY` (default 2, and the browser and wasm + pools size themselves to the same figure). +3. The poller claims the job over the management API and, on `:run`, mints a + render session (`create-job-session`) through which every asset fetch + happens as the owner of the job. +4. The milestones are reported as beats (`:preparing`, `:rendering`, + `:packaging`, with the `objects` and `pages` counters). A watchdog re-sends + the last one every second, whatever the run is doing. +5. It settles with `complete-job` (the artifact rides multipart, the session + is closed in the same step) or `fail-job`. The settle never rejects: a + worker that cannot talk leaves the row to the backend's lease GC. -| Method | Path | Description | -|----------|-----------------|----------------------------------------------------| -| `POST` | `/` | Legacy command multiplex; runs unscheduled | -| `POST` | `/jobs` | Create an export job | -| `GET` | `/jobs/{id}` | Job record | -| `DELETE` | `/jobs/{id}` | Request cancellation | +## Cancellation -Job states: `queued` -> `running` -> `ended` | `error` | `cancelled`. The last -three are terminal. +A cancellation reaches the worker through the beats: a `skip` answer means the +backend no longer listens (the row is terminal), so the runner stops between +units of work; mid-Skia the watchdog terminates the leased render worker. The +row was already `cancelled` by the backend's `cancel-job`. -## Redis layout +## Running it -Every key is namespaced with `penpot.exporter.` plus the tenant -(`PENPOT_TENANT`, `default` in code but set to the workspace name in devenv, -e.g. `devenv-ws0`). +Local (devenv): `./scripts/setup` once (installs deps + Playwright +chromium), then `pnpm run watch` in one pane (builds render-wasm and +recompiles on change) and `pnpm run start:dev` in another (loads the +backend dev env and runs `target/app.js`). Code changes arrive through +shadow hot-reload: `on-before-load` halts the running system and +`on-after-load` reboots it in the same process. The tmux `exporter` +window does the same split (`./scripts/watch` + +`./scripts/wait-and-start.sh`). -``` -penpot.exporter.{tenant}.job.{job-id} hash field: data (transit blob of the - whole record) -penpot.exporter.{tenant}.job-cancel pubsub payload: the job id, one line -``` - -There is no index: the keyspace is one self-expiring hash per job and nothing -else. Each hash carries the same TTL as the exported file -(`PENPOT_EXPORTER_JOB_TTL`, default 3600s), refreshed on every write and never -after the job settles. - -## Inspecting Redis - -Redis is not published on the host, so `redis-cli` from your machine gets -connection refused. Run it **inside the devenv container**, against the `valkey` -host on database 0: - -```bash -redis-cli -h valkey -n 0 -``` - -`redis-cli -u "$PENPOT_REDIS_URI"` does the same and follows whatever the env is -set to (`redis://valkey/0` in devenv). - -Keys carry the tenant, which in devenv is the **workspace name** -(`$PENPOT_TENANT`, e.g. `devenv-ws0`), not `default`. From the prompt: - -``` -# every job record -KEYS penpot.exporter.devenv-ws0.job.* - -# the whole record, transit-json in the `data` field -HGET penpot.exporter.devenv-ws0.job. data - -# seconds left before the record expires -TTL penpot.exporter.devenv-ws0.job. - -# watch cancellations as they are published (blocks the connection) -SUBSCRIBE penpot.exporter.devenv-ws0.job-cancel - -# drop one record -DEL penpot.exporter.devenv-ws0.job. -``` - -`KEYS` is fine here -- the keyspace is a handful of job hashes. On a real -deployment use `SCAN 0 MATCH penpot.exporter..job.* COUNT 100` instead. -Do not `FLUSHDB`: the backend shares this database. - -The backend debug UI also renders these records: `/dbg` has an *Export jobs* -section, with a `?job-id=` filter. +`pnpm run start` runs the compiled `target/app.js` with the env as-is +(the same `node target/app.js` the container image uses as `CMD`). The +release bundle is `./scripts/build ` (wasm + shadow release + +version stamp + generated `setup`); `./manage.sh build-exporter-bundle` +packs it for the image. ## Configuration -| Variable | Default | Description | -|---------------------------------------|---------|--------------------------------------| -| `PENPOT_REDIS_URI` | `redis://redis/0` | Job store and cancel topic | -| `PENPOT_TENANT` | `default` | Key and topic prefix | -| `PENPOT_EXPORTER_JOB_TTL` | `3600` | Lifetime of a job record, in seconds | -| `PENPOT_EXPORTER_MAX_CONCURRENT_JOBS` | `4` | Admission limit | -| `PENPOT_EXPORTER_MAX_JOBS_PER_PROFILE`| `2` | Per-profile admission limit | -| `PENPOT_EXPORTER_QUEUE_MAX` | `64` | Queue cap; over it, `429 :queue-full` | -| `PENPOT_EXPORTER_REPLICA_ID` | hostname | Names this replica; boot cleanup only cancels its own jobs | -| `PENPOT_WASM_WORKER_POOL_MAX` | `2` | Headless render worker threads; min 1 | -| `PENPOT_WASM_WORKER_POOL_MIN` | `1` | Workers kept warm; clamped to the max | -| `PENPOT_WASM_WORKER_IDLE_TIMEOUT` | `300` | Silence before a worker is terminated, in seconds | -| `PENPOT_WASM_WORKER_IMAGE_CACHE_SIZE` | `134217728` | Per-worker image cache budget, in bytes | +| Variable | Default | Description | +|------------------------------------------|---------|----------------------------------------------------| +| `PENPOT_PUBLIC_URI` | — | The frontend the assets render through | +| `PENPOT_INTERNAL_URI` | `public-uri` | Same network's frontend; the management API too | +| `PENPOT_REDIS_URI` | `redis://redis/0` | The queue and the rest of redis | +| `PENPOT_TENANT` | `default` | Queue prefix (workspace name in devenv) | +| `PENPOT_TEMPDIR` | `/tmp/penpot` | Temp area owned by the tmpdir service | +| `PENPOT_SECRET_KEY` | — | Required; the management key derives from it when no shared key is set | +| `PENPOT_EXPORTER_SHARED_KEY` | — | The shared key; derived from the secret otherwise | +| `PENPOT_EXPORTER_JOB_TTL` | `3600` | Lifetime of the temp files a job owns, in seconds | +| `PENPOT_EXPORTER_WORKER_CONCURRENCY` | `2` | `K`: pollers, running exports, pool sizes | +| `PENPOT_WASM_WORKER_POOL_MIN` | `1` | Render workers kept warm; clamped to the max | +| `PENPOT_WASM_WORKER_IDLE_TIMEOUT` | `300` | Silence before a worker is terminated, in seconds | +| `PENPOT_WASM_WORKER_IMAGE_CACHE_SIZE` | `134217728` | Per-worker image cache budget, in bytes | -A headless job leases one render worker for its whole run, so it is admitted -only when a worker is free: `PENPOT_WASM_WORKER_POOL_MAX` is the real limit for -them, and `PENPOT_EXPORTER_MAX_CONCURRENT_JOBS` bounds the browser ones -alongside. +## Inspecting the backend + +Nothing about a job is stored here: `get-job` in the backend answers the full +row, and its `app.debug`/admin surfaces render the same records. diff --git a/exporter/package.json b/exporter/package.json index d50b5b7a40..909d59d3ca 100644 --- a/exporter/package.json +++ b/exporter/package.json @@ -32,9 +32,11 @@ "clear:shadow-cache": "rm -rf .shadow-cljs && rm -rf target", "watch:app": "pnpm run build:wasm && pnpm run clear:shadow-cache && clojure -M:dev:shadow-cljs watch main", "watch": "pnpm run watch:app", + "start": "node target/app.js", + "start:dev": "./scripts/run", "build:app": "clojure -M:dev:shadow-cljs release main", "build:wasm": "../render-wasm/build export", - "build": "pnpm run clear:shadow-cache && pnpm run build:app", + "build": "pnpm run build:wasm && pnpm run clear:shadow-cache && pnpm run build:app", "fmt:clj": "cljfmt fix --parallel=true src/ test/", "check-fmt:clj": "cljfmt check --parallel=true src/ test/", "lint:clj": "clj-kondo --fail-level error --parallel --lint src/ test/", diff --git a/exporter/scripts/build b/exporter/scripts/build index 5c2b0c40d4..8647758af4 100755 --- a/exporter/scripts/build +++ b/exporter/scripts/build @@ -6,10 +6,12 @@ export CURRENT_VERSION=$1; export NODE_ENV=production; pnpm install || exit 1; -pnpm run build:wasm; + +# Build the application (wasm artifact first, then the shadow release) +pnpm run build; WASM_SRC="resources/wasm"; -WASM_SHARED="src/app/wasm/shared.js"; +WASM_SHARED="src/exporter/wasm/shared.js"; if [ ! -f "$WASM_SRC/render-wasm.wasm" ] || [ ! -f "$WASM_SHARED" ]; then echo "ERROR: the render-wasm build did not produce:" >&2; echo " $WASM_SRC/render-wasm.wasm" >&2; @@ -17,11 +19,6 @@ if [ ! -f "$WASM_SRC/render-wasm.wasm" ] || [ ! -f "$WASM_SHARED" ]; then exit 1; fi -rm -rf target - -# Build the application -pnpm run build; - cp pnpm-lock.yaml target/; cp pnpm-workspace.yaml target/; cp package.json target/; diff --git a/exporter/scripts/run b/exporter/scripts/run index a7f8836f74..909db28060 100755 --- a/exporter/scripts/run +++ b/exporter/scripts/run @@ -7,4 +7,9 @@ if [ -f $SCRIPT_DIR/../../backend/scripts/_env.local ]; then source $SCRIPT_DIR/../../backend/scripts/_env.local; fi +# Same default as wait-and-start.sh: the platform edge is always the +# in-container nginx loopback, so it is defined at the start scripts, +# not in compose or _env; an explicitly exported value always wins. +export PENPOT_INTERNAL_URI="${PENPOT_INTERNAL_URI:-http://localhost:4449}" + exec node target/app.js diff --git a/exporter/scripts/test-quiet.js b/exporter/scripts/test-quiet.js index b1be0dd682..c7f7c610aa 100644 --- a/exporter/scripts/test-quiet.js +++ b/exporter/scripts/test-quiet.js @@ -24,6 +24,13 @@ progress("Running tests..."); const result = spawnSync( "node", ["target/tests/test.js", ...process.argv.slice(2)], - { stdio: "inherit" }, -); + { + stdio: "inherit", + // the config of the process needs a secret to even boot: same + // default the `test` script uses when the environment says nothing + env: { + ...process.env, + PENPOT_SECRET_KEY: process.env.PENPOT_SECRET_KEY || "test-secret-key", + }, + }); process.exit(result.status ?? 1); diff --git a/exporter/scripts/wait-and-start.sh b/exporter/scripts/wait-and-start.sh index 53d1bc73f0..a280b2782c 100755 --- a/exporter/scripts/wait-and-start.sh +++ b/exporter/scripts/wait-and-start.sh @@ -7,6 +7,11 @@ if [ -f $SCRIPT_DIR/../../backend/scripts/_env.local ]; then source $SCRIPT_DIR/../../backend/scripts/_env.local; fi +# In-container address of the platform edge (nginx). Invariant across +# instances (per-container loopback), so it lives here, not in compose +# or _env; an explicitly exported value always wins. +export PENPOT_INTERNAL_URI="${PENPOT_INTERNAL_URI:-http://localhost:4449}" + bb -i '(babashka.wait/wait-for-port "localhost" 9630)'; bb -i '(babashka.wait/wait-for-path "target/app.js")'; sleep 2; diff --git a/exporter/shadow-cljs.edn b/exporter/shadow-cljs.edn index 076e6a10f6..ea9d38db52 100644 --- a/exporter/shadow-cljs.edn +++ b/exporter/shadow-cljs.edn @@ -6,13 +6,11 @@ {:target :esm :runtime :node :output-dir "target/" - :devtools {:before-load-async app.core/stop - :after-load app.core/start} :modules {:app {:entries [] - :init-fn app.core/main}} + :init-fn exporter.main/main}} :js-options {:entry-keys ["module" "browser" "main"] diff --git a/exporter/src/app/auth.cljs b/exporter/src/app/auth.cljs deleted file mode 100644 index 0bcfb27199..0000000000 --- a/exporter/src/app/auth.cljs +++ /dev/null @@ -1,98 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.auth - "Resolves the caller's session cookie to a real profile id. - - The export commands take `:profile-id` from the request body, which was - harmless while it only picked a pub/sub topic. The job API can read and - cancel other people's work, so its ownership comes from the session: the - token goes to the backend's `get-profile` command, which answers with the - anonymous profile (`uuid/zero`) when it is not a valid session. - - Results are memoized briefly, so a burst of export calls from one client is - one round trip rather than one per request." - (:require - ["undici" :as http] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.common.transit :as t] - [app.common.uri :as u] - [app.common.uuid :as uuid] - [app.config :as cf] - [promesa.core :as p])) - -(def ^:private cache-ttl-ms 60000) - -(defonce ^:private cache (atom {})) - -(defn- put-in-cache - "Stores the resolution by session token and removes expired entries. - Without cleanup, long-lived exporters accumulate stale entries" - [cache token profile-id now] - (-> (into {} (remove (fn [[_ {:keys [expires-at]}]] (<= expires-at now))) cache) - (assoc token {:profile-id profile-id - :expires-at (+ now cache-ttl-ms)}))) - -(defn- rpc-uri - [] - (-> (cf/get-internal-uri) - (u/ensure-path-slash) - (u/join "api/rpc/command/get-profile") - (str))) - -(defn- fetch-profile-id - [token] - (let [uri (rpc-uri) - headers #js {"Content-Type" "application/transit+json" - "X-Shared-Key" (str "exporter " cf/management-key) - "Cookie" (str "auth-token=" token)}] - (->> (p/do (http/fetch uri #js {:method "POST" :headers headers :body (t/encode-str {})})) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.text resp) - (p/resolved nil)))) - (p/fmap (fn [body] - (some-> body t/decode-str :id))) - (p/merr (fn [cause] - (l/warn :hint "unable to resolve session profile" :uri uri :cause cause) - (p/resolved nil)))))) - -(defn resolve-profile-id - "Promise of the authenticated profile id, or nil for an anonymous or absent - session." - [token] - (if (nil? token) - (p/resolved nil) - (let [{:keys [profile-id expires-at]} (get @cache token)] - (if (and expires-at (> expires-at (js/Date.now))) - (p/resolved profile-id) - (->> (fetch-profile-id token) - (p/fmap (fn [profile-id] - (let [profile-id (when (and profile-id (not= uuid/zero profile-id)) profile-id)] - (swap! cache put-in-cache token profile-id (js/Date.now)) - profile-id)))))))) - -(defn require-profile-id - "Like `resolve-profile-id`, but rejects anonymous callers." - [token] - (->> (resolve-profile-id token) - (p/mcat (fn [profile-id] - (if profile-id - (p/resolved profile-id) - (ex/raise :type :authentication - :code :authentication-required - :hint "no valid session for this request")))))) - -(defn check-owner! - "Raises unless `profile-id` owns `job`." - [job profile-id] - (when (or (nil? job) - (not= (str (:profile-id job)) (str profile-id))) - (ex/raise :type :not-found - :code :object-not-found - :hint "job does not exist")) - job) diff --git a/exporter/src/app/browser.cljs b/exporter/src/app/browser.cljs deleted file mode 100644 index 9a02e0a5ae..0000000000 --- a/exporter/src/app/browser.cljs +++ /dev/null @@ -1,217 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.browser - (:require - ["generic-pool" :as gp] - ["generic-pool/lib/errors.js" :as gpe] - ["playwright" :as pw] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.config :as cf] - [app.util.object :as obj] - [promesa.core :as p])) - -(l/set-level! :trace) - -(def TimeoutError gpe/TimeoutError) - -;; --- BROWSER API - -(def default-timeout 30000) -(def default-viewport-width 1920) -(def default-viewport-height 1080) -(def default-user-agent - (str "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/99.0.3729.169 Safari/537.36")) - -(defn create-cookies - [uri {:keys [name token] :or {name "auth-token"}}] - (let [domain (str (:host uri) - (when (:port uri) - (str ":" (:port uri))))] - #js [#js {:domain domain - :path "/" - :name name - :value token}])) - -(defn nav! - ([page url] (nav! page url nil)) - ([page url {:keys [wait-until timeout] :or {wait-until "networkidle" timeout 20000}}] - (.goto ^js page (str url) #js {:waitUntil wait-until :timeout timeout}))) - -(defn sleep - [page ms] - (.waitForTimeout ^js page ms)) - -(defn wait-for-fonts - "Wait until the browser has finished loading all fonts. - - Checking `document.fonts.status === 'loaded'` on its own is not enough: - the render page injects its `@font-face` rules asynchronously and the - browser only loads a face lazily, when some painted text actually uses - it. A face that is declared but not yet requested stays in the `unloaded` - state, which does NOT keep `document.fonts.status` at `loading`, so the - check can pass before any real font glyphs are available. The export is - then captured with a (usually wider) fallback font, and auto-width text - laid out with the real font metrics overflows its bounds and gets clipped. - - To avoid that we explicitly request every declared face and await - `document.fonts.ready` before checking the status." - ([page] (wait-for-fonts page nil)) - ([page {:keys [timeout] :or {timeout 15000}}] - (-> (.waitForFunction ^js page - "async () => { - if (!document.fonts) return true; - try { - await Promise.all(Array.from(document.fonts, (face) => face.load())); - } catch (e) {} - await document.fonts.ready; - return document.fonts.status === 'loaded'; - }" - nil - #js {:timeout timeout}) - (p/catch (fn [cause] - (l/warn :hint "wait-for-fonts timed out; continuing anyway" - :cause (ex-message cause)) - (p/resolved nil)))))) - -(defn wait-for - ([locator] (wait-for locator nil)) - ([locator {:keys [state timeout] :or {state "visible" timeout 10000}}] - (.waitFor ^js locator #js {:state state :timeout timeout}))) - -(defn screenshot - ([frame] (screenshot frame {})) - ([frame {:keys [full-page? omit-background? type quality path] - :or {type "png" full-page? false omit-background? false quality 95}}] - (let [options (-> (obj/new) - (obj/set! "type" (name type)) - (obj/set! "omitBackground" omit-background?) - (cond-> path (obj/set! "path" path)) - (cond-> (= "jpeg" type) (obj/set! "quality" quality)) - (cond-> full-page? (-> (obj/set! "fullPage" true) - (obj/set! "clip" nil))))] - (.screenshot ^js frame options)))) - -(defn emulate-media! - [page {:keys [media]}] - (.emulateMedia ^js page #js {:media media}) - page) - -(defn pdf - ([page] (pdf page {})) - ([page {:keys [scale path page-ranges] - :or {page-ranges "1" - scale 1}}] - (.pdf ^js page #js {:path path - :scale scale - :pageRanges page-ranges - :printBackground true - :preferCSSPageSize true}))) -(defn eval! - [frame f] - (.evaluate ^js frame f)) - -(defn select - [frame selector] - (.locator ^js frame selector)) - -(defn select-all - [frame selector] - (.$$ ^js frame selector)) - - -;; --- BROWSER STATE - -(defonce pool (atom nil)) -(defonce pool-browser-id (atom 1)) - -(def browser-pool-factory - (letfn [(create [] - (p/let [opts #js {:args #js ["--allow-insecure-localhost" "--font-render-hinting=none"]} - browser (.launch pw/chromium opts) - id (swap! pool-browser-id inc)] - (l/info :origin "factory" :action "create" :browser-id id) - (unchecked-set browser "__id" id) - browser)) - - (destroy [obj] - (let [id (unchecked-get obj "__id")] - (l/info :origin "factory" :action "destroy" :browser-id id) - (.close ^js obj))) - - (validate [obj] - (let [id (unchecked-get obj "__id")] - (l/info :origin "factory" :action "validate" :browser-id id :obj obj) - (p/resolved (.isConnected ^js obj))))] - - #js {:create create - :destroy destroy - :validate validate})) - -(defn init - [] - (let [opts #js {:max (cf/get :browser-pool-max 5) - :min (cf/get :browser-pool-min 0) - :testOnBorrow true - :evictionRunIntervalMillis 5000 - :numTestsPerEvictionRun 5 - ;; :acquireTimeoutMillis 120000 ; 2min - :acquireTimeoutMillis 10000 ; 10 s - :idleTimeoutMillis 10000}] - - (l/info :hint "initializing browser pool" :opts opts) - (reset! pool (gp/createPool browser-pool-factory opts)) - (p/resolved nil))) - -(defn stop - [] - (when-let [pool (deref pool)] - (l/info :hint "finalizing browser pool") - (p/do! - (.drain ^js pool) - (.clear ^js pool)))) - -(defn- ex-ignore - [p] - (p/handle p (constantly nil))) - -(defn- translate-browser-errors - [cause] - (if (instance? TimeoutError cause) - (ex/raise :type :internal - :code :timeout - :hint (ex-message cause) - :cause cause) - (p/rejected cause))) - -(defn exec! - [config handle] - (letfn [(handle-browser [browser] - (p/let [id (unchecked-get browser "__id") - context (.newContext ^js browser config)] - (l/trace :hint "exec:handle:start" :browser-id id) - (p/let [page (.newPage ^js context) - result (handle page)] - (.close ^js context) - (l/trace :hint "exec:handle:end" :browser-id id) - result))) - - (on-acquire [pool browser] - (-> (handle-browser browser) - (p/then (fn [result] - (.release ^js pool browser) - result)) - (p/catch (fn [cause] - (p/do! - (ex-ignore (.destroy ^js pool browser)) - (p/rejected cause))))))] - - (when-let [pool (deref pool)] - (-> (p/do! (.acquire ^js pool)) - (p/then (partial on-acquire pool)) - (p/catch translate-browser-errors))))) diff --git a/exporter/src/app/core.cljs b/exporter/src/app/core.cljs deleted file mode 100644 index 7dcf3a5b71..0000000000 --- a/exporter/src/app/core.cljs +++ /dev/null @@ -1,105 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.core - (:require - ["node:process" :as proc] - ["node:worker_threads" :as wt] - [app.browser :as bwr] - [app.common.logging :as l] - [app.config :as cf] - [app.http :as http] - [app.jobs :as jobs] - [app.jobs.utils :as job.utils] - [app.redis :as redis] - [app.wasm :as wasm] - [app.wasm.pool :as wasm.pool] - [app.wasm.worker :as wasm.worker] - [promesa.core :as p])) - -(enable-console-print!) -(l/setup! {:app :info}) - -(defn start - "Render workers run this same bundle, so the thread decides what gets booted: - the http server and its pools, or one render worker." - [& _] - (if-not ^boolean wt/isMainThread - (wasm.worker/main) - (do - (l/info :msg "initializing" - :public-uri (str (cf/get :public-uri)) - :internal-uri (str (cf/get-internal-uri)) - :replica cf/replica-id - :version (:full cf/version)) - (l/info :msg "headless wasm export available" - :wasm-dir wasm/artifact-dir - :workers (cf/get :wasm-worker-pool-max) - :image-cache-size (cf/get :wasm-worker-image-cache-size)) - (p/do - (bwr/init) - (redis/init) - (jobs/init) - (job.utils/init) - (wasm.pool/init) - (http/init))))) - -(def main start) - -;; Draining a pool waits for every checked-out resource to come back, which an -;; export in flight can hold for as long as its own timeout. On a hot reload -;; that would block `start` from ever running again, leaving a drained pool that -;; fails every later job. -(def ^:private shutdown-step-timeout 3000) - -(defn- shutdown-step - [label f] - (-> (p/race [(p/do (f)) - (p/fmap (constantly ::timeout) (p/delay shutdown-step-timeout))]) - (p/handle (fn [result cause] - (when (or (some? cause) (= ::timeout result)) - (l/warn :hint "shutdown step did not finish cleanly" - :step label - :cause cause)) - nil)))) - -(defn stop - [done] - ;; an empty line for visual feedback of restart - (js/console.log "") - - (if-not ^boolean wt/isMainThread - ;; A render worker owns no server, pools or connections; nothing to unwind. - (done) - (do - (l/info :msg "stopping") - (p/do - (shutdown-step "browser-pool" bwr/stop) - (shutdown-step "wasm-worker-pool" wasm.pool/stop) - (shutdown-step "redis" redis/stop) - (shutdown-step "http" http/stop) - (done))))) - -(.on proc/default "uncaughtException" - (fn [cause] - (js/console.error cause))) - -(defonce ^:private exiting? (atom false)) - -(defn- exit! - "Cancels this process's jobs before exiting, so their records and clients - do not keep waiting on work that dies with it. A second signal exits at once." - [] - (if (compare-and-set! exiting? false true) - (->> (shutdown-step "cancel-jobs" jobs/cancel-all!) - (p/fnly (fn [_ _] (proc/exit 0)))) - (proc/exit 0))) - -;; Signals are only delivered to the main thread, and `exit` in a worker would -;; take down that worker rather than the process. -(when ^boolean wt/isMainThread - (.on proc/default "SIGTERM" exit!) - (.on proc/default "SIGINT" exit!)) diff --git a/exporter/src/app/handlers.cljs b/exporter/src/app/handlers.cljs deleted file mode 100644 index bd2b453951..0000000000 --- a/exporter/src/app/handlers.cljs +++ /dev/null @@ -1,110 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.handlers - (:require - [app.auth :as auth] - [app.common.data :as d] - [app.common.logging :as l] - [app.common.spec :as us] - [app.handlers.export :as export] - [app.util.transit :as t] - [clojure.spec.alpha :as s] - [promesa.core :as p])) - -(l/set-level! :debug) - -(def ^:private error-codes - #{:queue-full}) - -(defn on-error - [error exchange] - (let [{:keys [type code] :as data} (ex-data error)] - (cond - (and (= :validation type) - (contains? error-codes code)) - (let [data {:type :validation - :code code - :hint (ex-message error)}] - (l/warn :hint "rejecting export request" :code code) - (-> exchange - (assoc :response/status 429) - (assoc :response/body (t/encode data)) - (assoc :response/headers {"content-type" "application/transit+json"}))) - - (= :authentication type) - (let [data {:type :authentication - :code code - :hint (ex-message error)}] - (-> exchange - (assoc :response/status 401) - (assoc :response/body (t/encode data)) - (assoc :response/headers {"content-type" "application/transit+json"}))) - - (or (= :validation type) - (= :assertion type)) - (let [explain (us/pretty-explain data) - data (-> data - (assoc :explain explain) - (assoc :type :validation) - (dissoc ::s/problems ::s/value ::s/spec))] - (-> exchange - (assoc :response/status 400) - (assoc :response/body (t/encode data)) - (assoc :response/headers {"content-type" "application/transit+json"}))) - - (= :not-found type) - (-> exchange - (assoc :response/status 404) - (assoc :response/body (t/encode data)) - (assoc :response/headers {"content-type" "application/transit+json"})) - - (and (= :internal type) - (= :browser-not-ready code)) - (let [data {:type :server-error - :code :internal - :hint (ex-message error) - :data data}] - (-> exchange - (assoc :response/status 503) - (assoc :response/body (t/encode data)) - (assoc :response/headers {"content-type" "application/transit+json"}))) - - :else - (let [data {:type :server-error - :code code - :hint (ex-message error) - :data data}] - (l/error :hint "unexpected internal error" :cause error) - (-> exchange - (assoc :response/status 500) - (assoc :response/body (t/encode (d/without-nils data))) - (assoc :response/headers {"content-type" "application/transit+json"})))))) - -(defn handler - "The original `POST /api/export` entry point, and the one the browser backend - still goes through. The export runs as soon as it is asked for, and the - contract is unchanged: `:wait` answers with the finished resource, otherwise - with the resource handle while the work runs." - [{:keys [:request/params :request/auth-token] :as exchange}] - (let [{:keys [cmd wait] :as params} (export/conform-params params)] - (l/debug :hint "process-request" :cmd cmd) - (->> (auth/resolve-profile-id auth-token) - (p/mcat (fn [profile-id] - ;; The session wins when there is one; the body value stays - ;; the fallback so nothing that used to work stops working. - (export/export! auth-token (cond-> params - (some? profile-id) - (assoc :profile-id profile-id))))) - (p/mcat (fn [{:keys [resource pending]}] - (if wait - (p/fmap (fn [resource] - (assoc exchange :response/body resource)) - pending) - (do - (p/merr (constantly nil) pending) - (p/resolved - (assoc exchange :response/body (dissoc resource :path)))))))))) diff --git a/exporter/src/app/handlers/export.cljs b/exporter/src/app/handlers/export.cljs deleted file mode 100644 index 8ba9a9c322..0000000000 --- a/exporter/src/app/handlers/export.cljs +++ /dev/null @@ -1,100 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.handlers.export - "Handle export jobs" - (:require - [app.common.spec :as us] - [app.handlers.export-frames :as export-frames] - [app.handlers.export-shapes :as export-shapes] - [app.jobs :as jobs] - [app.jobs.scheduler :as scheduler] - [app.jobs.utils :as job.utils] - [clojure.spec.alpha :as s] - [promesa.core :as p])) - -;; --- PARAMS - -(defmulti command-spec :cmd) - -(s/def ::cmd ::us/keyword) -(s/def ::wait ::us/boolean) - -(defmethod command-spec :export-shapes [_] ::export-shapes/params) -(defmethod command-spec :export-frames [_] ::export-frames/params) - -(s/def ::params - (s/and (s/keys :req-un [::cmd] - :opt-un [::wait]) - (s/multi-spec command-spec :cmd))) - -(defn conform-params - [params] - (us/conform ::params params)) - -(defn- prepare - [cmd auth-token params] - (case cmd - :export-shapes (export-shapes/prepare auth-token params) - :export-frames (export-frames/prepare auth-token params))) - -(defn- current - [job] - (or (jobs/lookup (:id job)) job)) - -(defn- run-and-track - [job run] - (->> (p/do (run job)) - (p/mcat (fn [resource] - (->> (jobs/complete! (current job) resource) - (p/fmap (constantly resource))))) - (p/merr (fn [cause] - (if (jobs/cancelled? (:id job)) - (p/rejected cause) - (->> (jobs/fail! (current job) cause) - (p/mcat (fn [_] (p/rejected cause))))))))) - -(defn- run-now! - "Runs the job as soon as it is created, outside the scheduler." - [job] - (->> (p/do (jobs/start! job)) - (p/mcat (fn [job] (p/do ((jobs/run-fn (:id job)) job)))) - (p/fnly (fn [_ _] - (jobs/release! (:id job)) - (job.utils/release! (:id job)))))) - -(defn- create! - [auth-token {:keys [cmd profile-id] :as params} start] - (let [{:keys [resource total headless run]} (prepare cmd auth-token params)] - (->> (jobs/create! {:profile-id profile-id - :cmd cmd - ;; Backend used for admission and cancel UI: headless - ;; WASM when `:is-wasm` is set, otherwise browser. - :backend (if headless "wasm" "browser") - :total total - :name (:name resource) - :resource-id (:id resource)} - (fn [job] (run-and-track job run))) - (p/fmap (fn [job] - (try - {:job job - :resource resource - :pending (start job)} - (catch :default cause - (jobs/fail! job cause) - (jobs/release! (:id job)) - (throw cause)))))))) - -(defn create-job! - "Returns a promise of `{:job :resource :pending}`." - [auth-token params] - (create! auth-token params scheduler/submit!)) - -(defn export! - "Like `create-job!`, but the work starts right away: This is what - keeps browser exports behaving exactly as they did before there were jobs." - [auth-token params] - (create! auth-token params run-now!)) diff --git a/exporter/src/app/handlers/export_frames.cljs b/exporter/src/app/handlers/export_frames.cljs deleted file mode 100644 index ed290d4b50..0000000000 --- a/exporter/src/app/handlers/export_frames.cljs +++ /dev/null @@ -1,93 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.handlers.export-frames - (:require - [app.common.spec :as us] - [app.handlers.export-shapes :refer [count-objects headless-exports? prepare-exports]] - [app.handlers.resources :as rsc] - [app.jobs :as jobs] - [app.jobs.utils :as job.utils] - [app.renderer :as rd] - [app.util.shell :as sh] - [cljs.spec.alpha :as s] - [cuerdas.core :as str] - [promesa.core :as p])) - -(declare ^:private join-pdf) -(declare ^:private move-file) - -(s/def ::name ::us/string) -(s/def ::file-id ::us/uuid) -(s/def ::page-id ::us/uuid) -(s/def ::object-id ::us/uuid) -(s/def ::is-wasm ::us/boolean) - -(s/def ::export - (s/keys :req-un [::file-id ::page-id ::object-id ::name])) - -(s/def ::exports - (s/every ::export :kind vector? :min-count 1)) - -(s/def ::params - (s/keys :req-un [::exports] - :opt-un [::name ::is-wasm])) - -(defn- run-export - [job auth-token resource {:keys [exports is-wasm file-id]}] - (let [rendered (atom []) - - on-object - (fn [{:keys [path] :as _object}] - (job.utils/track! (:id job) path) - (jobs/progress! job (count (swap! rendered conj path)))) - - exports - (map #(assoc % :is-wasm is-wasm :job-id (:id job)) exports)] - - (job.utils/track! (:id job) (:path resource)) - (->> (rd/with-scope exports - (fn [render] - (jobs/check-cancelled! job) - (->> exports - (map (fn [export] (render export on-object))) - (p/all)))) - (p/fmap (fn [_] @rendered)) - (p/mcat (partial join-pdf job file-id)) - (p/mcat (partial move-file resource)) - (p/fmap (constantly resource)) - (p/mcat (partial rsc/upload-resource auth-token)) - (p/mcat (fn [resource] - (->> (sh/stat (:path resource)) - (p/fmap #(merge resource %))))) - (p/fmap (fn [resource] (dissoc resource :path)))))) - -(defn prepare - [auth-token {:keys [exports name is-wasm] :as _params}] - (let [exports (-> (map #(assoc % :type :pdf :scale 1 :suffix "") exports) - (prepare-exports auth-token is-wasm)) - resource (rsc/create :pdf (or name (-> exports first :name))) - file-id (-> exports first :file-id)] - {:resource resource - :total (count-objects exports) - :headless (headless-exports? exports is-wasm) - :run (fn [job] (run-export job auth-token resource - {:exports exports - :is-wasm is-wasm - :file-id file-id}))})) - -(defn- join-pdf - [job file-id paths] - (p/let [prefix (str/concat "penpot.pdfunite." file-id ".") - path (job.utils/track! (:id job) (sh/tempfile :prefix prefix :suffix ".pdf"))] - (apply sh/run-cmd! "pdfunite" (conj (vec paths) path)) - path)) - -(defn- move-file - [{:keys [path] :as resource} output-path] - (p/do - (sh/move! output-path path) - resource)) diff --git a/exporter/src/app/handlers/export_shapes.cljs b/exporter/src/app/handlers/export_shapes.cljs deleted file mode 100644 index 27f5844e98..0000000000 --- a/exporter/src/app/handlers/export_shapes.cljs +++ /dev/null @@ -1,197 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.handlers.export-shapes - (:require - [app.common.data :as d] - [app.common.spec :as us] - [app.handlers.resources :as rsc] - [app.jobs :as jobs] - [app.jobs.utils :as job.utils] - [app.renderer :as rd] - [app.util.mime :as mime] - [app.util.shell :as sh] - [cljs.spec.alpha :as s] - [cuerdas.core :as str] - [promesa.core :as p])) - -(declare prepare-exports) - -;; Regex to clean namefiles -(def sanitize-file-regex #"[\\/:*?\"<>|]") - -(s/def ::file-id ::us/uuid) -(s/def ::filename ::us/string) -(s/def ::name ::us/string) -(s/def ::object-id ::us/uuid) -(s/def ::page-id ::us/uuid) -(s/def ::share-id ::us/uuid) -(s/def ::profile-id ::us/uuid) -(s/def ::scale ::us/number) -(s/def ::suffix ::us/string) -(s/def ::type ::us/keyword) -(s/def ::wait ::us/boolean) -(s/def ::is-wasm ::us/boolean) - -(s/def ::export - (s/keys :req-un [::page-id ::file-id ::object-id ::type ::suffix ::scale ::name] - :opt-un [::share-id])) - -(s/def ::exports - (s/coll-of ::export :kind vector? :min-count 1)) - -(s/def ::params - (s/keys :req-un [::exports ::profile-id] - :opt-un [::wait ::name ::skip-children ::force-multiple ::is-wasm])) - -(defn count-objects - [exports] - (reduce + 0 (map (comp count :objects) exports))) - -(defn- render! - [job export on-object] - (jobs/check-cancelled! job) - (rd/render (assoc export :job-id (:id job)) on-object)) - -(defn- scoped-renders - "Renders every export, the headless ones sharing a single worker." - [job exports on-object] - (rd/with-scope exports - (fn [render] - (jobs/check-cancelled! job) - (->> exports - (map (fn [export] (render export on-object))) - (p/all))))) - -(defn- run-single - [job auth-token resource {:keys [export is-wasm skip-children]}] - (job.utils/track! (:id job) (:path resource)) - (->> (render! job - (assoc export :skip-children skip-children :is-wasm (boolean is-wasm)) - (fn [{:keys [path] :as _object}] - (job.utils/track! (:id job) path) - (sh/move! path (:path resource)))) - (p/fmap (constantly resource)) - (p/mcat (partial rsc/upload-resource auth-token)) - (p/fmap (fn [resource] (dissoc resource :path))))) - -(defn- run-multiple - [job auth-token resource {:keys [exports is-wasm]}] - (let [failure (volatile! nil) - - zip (rsc/create-zip :resource resource - :on-error (fn [cause] (vreset! failure cause)) - :on-progress (fn [{:keys [done]}] - (jobs/progress! job done))) - - append (fn [{:keys [filename path] :as _object}] - (job.utils/track! (:id job) path) - (rsc/add-to-zip zip path (str/replace filename sanitize-file-regex "_")))] - - (job.utils/track! (:id job) (:path resource)) - (->> (scoped-renders job - (map #(assoc % :is-wasm (boolean is-wasm) :job-id (:id job)) exports) - append) - (p/mcat (fn [_] - (if-let [cause @failure] - (p/rejected cause) - (rsc/close-zip zip)))) - (p/fmap (constantly resource)) - (p/mcat (partial rsc/upload-resource auth-token)) - (p/fmap (fn [resource] (dissoc resource :path)))))) - -(defn headless-exports? - "Whether any of `exports` renders headless, and so whether the job leases a - render worker. Mirrors what `rd/with-scope` decides at run time." - [exports is-wasm] - (boolean (some #(rd/headless? {:is-wasm is-wasm :type (:type %)}) exports))) - -(defn prepare - [auth-token {:keys [exports force-multiple name skip-children is-wasm] :as _params}] - (let [exports (prepare-exports exports auth-token is-wasm) - headless? (headless-exports? exports is-wasm) - single? (and (not force-multiple) - (= 1 (count exports)) - (= 1 (count (-> exports first :objects))))] - (if single? - (let [export (first exports) - resource (rsc/create (:type export) (or name (:name export)))] - {:resource resource - :total 1 - :headless headless? - :run (fn [job] (run-single job auth-token resource - {:export export - :is-wasm is-wasm - :skip-children skip-children}))}) - - (let [resource (rsc/create :zip (or name (-> exports first :name)))] - {:resource resource - :total (count-objects exports) - :headless headless? - :run (fn [job] (run-multiple job auth-token resource - {:exports exports :is-wasm is-wasm}))})))) - -(defn- assoc-file-name - "A transducer that assocs a candidate filename and avoid duplicates" - [] - (letfn [(find-candidate [params used] - (loop [index 0] - (let [candidate (str (:name params) - (:suffix params "") - (when (pos? index) - (str/concat "-" (inc index))) - (mime/get-extension (:type params)))] - (if (contains? used candidate) - (recur (inc index)) - candidate))))] - (fn [rf] - (let [used (volatile! #{})] - (fn - ([] (rf)) - ([result] (rf result)) - ([result params] - (let [candidate (find-candidate params @used) - params (assoc params :filename candidate)] - (vswap! used conj candidate) - (rf result params)))))))) - -(def ^:const ^:private - default-partition-size 50) - -(defn prepare-exports - [exports token is-wasm] - (letfn [(process-group [[part1 :as group]] - ;; Browser path renders one DOM page per partition; headless WASM - ;; takes the whole group in one render. - (if (rd/headless? {:is-wasm is-wasm :type (:type part1)}) - [(build-render group)] - (sequence (comp (partition-all default-partition-size) - (map build-render)) - group))) - - (build-render [[part1 :as part]] - {:file-id (:file-id part1) - :page-id (:page-id part1) - :share-id (:share-id part1) - :name (:name part1) - :token token - :type (:type part1) - :scale (:scale part1) - :objects (mapv part-entry->object part)}) - - (part-entry->object [entry] - {:id (:object-id entry) - :filename (:filename entry) - :name (:name entry) - :suffix (:suffix entry)})] - - (let [xform (comp - (map #(assoc % :token token)) - (assoc-file-name))] - (->> (sequence xform exports) - (d/group-by (juxt :scale :type)) - (map second) - (into [] (mapcat process-group)))))) diff --git a/exporter/src/app/handlers/jobs.cljs b/exporter/src/app/handlers/jobs.cljs deleted file mode 100644 index 08c687a3d7..0000000000 --- a/exporter/src/app/handlers/jobs.cljs +++ /dev/null @@ -1,60 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.handlers.jobs - "REST surface for export jobs, under `/api/export/jobs`. - - Ownership always comes from the session (see `app.auth`), never from the - request body." - (:require - [app.auth :as auth] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.handlers.export :as export] - [app.jobs :as jobs] - [promesa.core :as p])) - -(defn create - [{:keys [:request/auth-token :request/params] :as exchange}] - (->> (auth/require-profile-id auth-token) - (p/mcat (fn [profile-id] - (let [params (-> params - (assoc :profile-id profile-id) - (export/conform-params))] - (l/dbg :hint "create export job" :cmd (:cmd params) :profile-id (str profile-id)) - (export/create-job! auth-token params)))) - (p/fmap (fn [{:keys [job resource pending]}] - ;; A failure is reported through the job record, so the - ;; promise must not surface as an unhandled rejection. - (p/merr (constantly nil) pending) - (-> exchange - (assoc :response/body (-> (or (jobs/lookup (:id job)) job) - (assoc :filename (:filename resource)) - (assoc :mtype (:mtype resource))))))))) - -(defn fetch - [{:keys [:request/auth-token] :as exchange} job-id] - (->> (auth/require-profile-id auth-token) - (p/mcat (fn [profile-id] - (->> (jobs/fetch job-id) - (p/fmap #(auth/check-owner! % profile-id))))) - (p/fmap (fn [job] - (assoc exchange :response/body job))))) - -(defn cancel - [{:keys [:request/auth-token] :as exchange} job-id] - (->> (auth/require-profile-id auth-token) - (p/mcat (fn [profile-id] - (->> (jobs/fetch job-id) - (p/fmap #(auth/check-owner! % profile-id))))) - (p/mcat (fn [job] (jobs/cancel! (:id job)))) - (p/mcat (fn [_] (jobs/fetch job-id))) - (p/fmap (fn [job] - (if job - (assoc exchange :response/body job) - (ex/raise :type :not-found - :code :object-not-found - :hint "job does not exist")))))) diff --git a/exporter/src/app/handlers/resources.cljs b/exporter/src/app/handlers/resources.cljs deleted file mode 100644 index d703821f00..0000000000 --- a/exporter/src/app/handlers/resources.cljs +++ /dev/null @@ -1,98 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.handlers.resources - "Temporal resources management." - (:require - ["archiver" :as arc] - ["node:fs" :as fs] - ["node:fs/promises" :as fsp] - ["node:path" :as path] - ["undici" :as http] - [app.common.exceptions :as ex] - [app.common.transit :as t] - [app.common.uri :as u] - [app.common.uuid :as uuid] - [app.config :as cf] - [app.util.mime :as mime] - [app.util.shell :as sh] - [cljs.core :as c] - [cuerdas.core :as str] - [promesa.core :as p])) - -(defn- get-path - [type id] - (path/join sh/tmpdir (str/concat "penpot.resource." (c/name type) "." id))) - -(defn create - "Generates ephimeral resource object." - [type name] - (let [task-id (uuid/next) - path (-> (get-path type task-id) - (sh/schedule-deletion))] - {:path path - :mtype (mime/get type) - :name name - :filename (str/concat (str/replace name #"[\\/:*?\"<>|]" "_") (mime/get-extension type)) - :id task-id})) - -(defn create-zip - [& {:keys [resource on-complete on-progress on-error]}] - (let [^js zip (new arc/ZipArchive) - ^js out (fs/createWriteStream (:path resource)) - on-complete (or on-complete (constantly nil)) - progress (atom 0)] - (.on zip "error" on-error) - (.on zip "end" on-complete) - (.on zip "entry" (fn [data] - (let [name (unchecked-get data "name") - num (swap! progress inc)] - (on-progress {:done num :filename name})))) - (.pipe zip out) - zip)) - -(defn add-to-zip - [zip path name] - (.file ^js zip path #js {:name name})) - -(defn close-zip - [zip] - (p/create (fn [resolve] - (.on ^js zip "close" resolve) - (.finalize ^js zip)))) - -(defn upload-resource - [auth-token resource] - (->> (fsp/readFile (:path resource)) - (p/fmap (fn [buffer] - (new js/Blob #js [buffer] #js {:type (:mtype resource)}))) - (p/mcat (fn [blob] - (let [fdata (new http/FormData) - agent (new http/Agent #js {:connect #js {:rejectUnauthorized false}}) - headers #js {"X-Shared-Key" (str "exporter " cf/management-key) - "Authorization" (str "Bearer " auth-token)} - - request #js {:headers headers - :method "POST" - :body fdata - :dispatcher agent} - uri (-> (cf/get-internal-uri) - (u/ensure-path-slash) - (u/join "api/management/methods/upload-tempfile") - (str))] - - (.append fdata "content" blob (:filename resource)) - (http/fetch uri request)))) - - (p/mcat (fn [response] - (if (not= (.-status response) 200) - (ex/raise :type :internal - :code :unable-to-upload-resource - :response-status (.-status response)) - (.text response)))) - (p/fmap t/decode-str) - (p/fmap (fn [result] - (merge resource (dissoc result :id)))))) diff --git a/exporter/src/app/http.cljs b/exporter/src/app/http.cljs deleted file mode 100644 index 93100885f0..0000000000 --- a/exporter/src/app/http.cljs +++ /dev/null @@ -1,189 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.http - (:require - ["cookies$default" :as Cookies] - ["inflation$default" :as inflate] - ["node:http" :as http] - ["node:stream$default" :as stream] - ["raw-body$default" :as raw-body] - [app.common.logging :as l] - [app.common.transit :as t] - [app.config :as cf] - [app.handlers :as handlers] - [app.router :as router] - [cuerdas.core :as str] - [lambdaisland.uri :as u] - [promesa.core :as p])) - -(l/set-level! :info) - -(defprotocol IStreamableResponseBody - (write-body! [_ response])) - -(extend-protocol IStreamableResponseBody - string - (write-body! [data response] - (.write ^js response data) - (.end ^js response)) - - js/Buffer - (write-body! [data response] - (.write ^js response data) - (.end ^js response)) - - stream/Stream - (write-body! [data response] - (.pipe ^js data response) - (.on ^js data "error" (fn [cause] - (js/console.error cause) - (.end response))))) - -(defn- handle-response - [{:keys [:response/body - :response/headers - :response/status - response] - :as exchange}] - (let [status (or status 200) - headers (clj->js headers) - body (or body "")] - (.writeHead ^js response status headers) - (write-body! body response))) - -(defn- parse-headers - [req] - (let [orig (unchecked-get req "headers")] - (persistent! - (reduce #(assoc! %1 (str/lower %2) (unchecked-get orig %2)) - (transient {}) - (js/Object.keys orig))))) - -(defn- wrap-body-params - [handler] - (let [opts #js {:limit "60mb" :encoding "utf8"}] - (fn [{:keys [:request/method :request/headers request] :as exchange}] - (let [ctype (get headers "content-type")] - (if (= method "post") - (-> (raw-body (inflate request) opts) - (p/then (fn [data] - (cond-> data - (= ctype "application/transit+json") - (t/decode-str)))) - (p/then (fn [data] - (handler (assoc exchange :request/body-params data))))) - (handler exchange)))))) - -(defn- wrap-params - [handler] - (fn [{:keys [:request/body-params :request/query-params] :as exchange}] - (handler (assoc exchange :request/params (merge query-params body-params))))) - -(defn- wrap-response-format - [handler] - (fn [exchange] - (p/then - (handler exchange) - (fn [{:keys [:response/body :response/status] :as exchange}] - (cond - (map? body) - (let [data (t/encode-str body {:type :json-verbose}) - size (js/Buffer.byteLength data "utf-8")] - (-> exchange - (assoc :response/body data) - (assoc :response/status (or status 200)) - (update :response/headers assoc "content-type" "application/transit+json") - (update :response/headers assoc "content-length" size))) - - (and (nil? body) - (= 200 status)) - (-> exchange - (assoc :response/body "") - (assoc :response/status 204) - (assoc :response/headers {"content-length" 0})) - - :else - exchange))))) - -(defn- wrap-query-params - [handler] - (fn [{:keys [:request/uri] :as exchange}] - (handler (assoc exchange :request/query-params (u/query-string->map (:query uri)))))) - -(defn- wrap-error - [handler on-error] - (fn [exchange] - (-> (p/do (handler exchange)) - (p/catch (fn [cause] (on-error cause exchange)))))) - -(defn- wrap-auth - [handler cookie-name] - (fn [{:keys [:request/cookies] :as exchange}] - (let [token (.get ^js cookies cookie-name)] - (handler (cond-> exchange token (assoc :request/auth-token token)))))) - -(defn- wrap-health - "Add /healthz entry point intercept." - [handler] - (fn [{:keys [:request/path] :as exchange}] - (if (= path "/readyz") - (assoc exchange - :response/status 200 - :response/body "OK") - (handler exchange)))) - -(defn- create-adapter - [handler] - (fn [req res] - (let [cookies (Cookies. req res) - headers (parse-headers req) - uri (u/uri (unchecked-get req "url")) - exchange {:request/method (str/lower (unchecked-get req "method")) - :request/path (:path uri) - :request/uri uri - :request/headers headers - :request/cookies cookies - :request req - :response res}] - (-> (p/do (handler exchange)) - (p/then handle-response))))) - -(defn- create-server - [handler] - (.createServer ^js http (create-adapter handler))) - -(def instance (atom nil)) - -(defn init - [] - (let [handler (-> (router/create handlers/handler) - (wrap-health) - (wrap-auth "auth-token") - (wrap-response-format) - (wrap-params) - (wrap-query-params) - (wrap-body-params) - (wrap-error handlers/on-error)) - server (create-server handler) - port (cf/get :http-server-port 6061)] - - (.listen server port) - (l/info :hint "welcome to penpot" - :module "exporter" - :flags cf/flags - :version (:full cf/version)) - (l/info :hint "starting http server" :port port) - (reset! instance server))) - -(defn stop - [] - (if-let [server @instance] - (p/create (fn [resolve] - (.close server (fn [] - (l/info :hint "shutdown http server") - (resolve))))) - (p/resolved nil))) diff --git a/exporter/src/app/jobs.cljs b/exporter/src/app/jobs.cljs deleted file mode 100644 index d555269a52..0000000000 --- a/exporter/src/app/jobs.cljs +++ /dev/null @@ -1,307 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.jobs - "Export job model and lifecycle. - - The record is persisted in redis (`app.jobs.store`); the runtime bits that - cannot be serialized -- cancel callbacks, the cancel signal shared with a - render worker, the throttling bookkeeping -- stay in this process, keyed by - job id. - - Every state change also publishes the same `:export-update` message the - exporter has always published, so websocket clients keep working unchanged." - (:require - [app.common.data :as d] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.common.time :as ct] - [app.common.uuid :as uuid] - [app.config :as cf] - [app.jobs.store :as store] - [app.redis :as redis] - [promesa.core :as p])) - -(l/set-level! :debug) - -;; A large export reports progress per object; persisting each one would be -;; hundreds of writes for information nobody reads at that resolution. -(def ^:private progress-throttle-ms 250) - -(def ^:private terminal-states #{"ended" "error" "cancelled"}) - -(defonce ^:private registry (atom {})) - -(defn- now-ms - [] - (inst-ms (ct/now))) - -(defn- runtime - [job-id] - (get @registry (str job-id))) - -(defn lookup - "The live record of a job running in this process, or nil." - [job-id] - (:job (runtime job-id))) - -(defn fetch - "The job record from the shared store." - [job-id] - (store/fetch job-id)) - -(defn- publish! - [{:keys [id profile-id resource-id state done total name filename mtype - resource-uri error]}] - (redis/pub! (redis/->tenant-key (str profile-id)) - (d/without-nils - {:type :export-update - :job-id id - :resource-id resource-id - :status state - :done done - :total total - :name name - :filename filename - :mtype mtype - :resource-uri resource-uri - :cause error}))) - -(defn- store-job! - [job] - (swap! registry update (str (:id job)) assoc :job job) - job) - -(defn create! - "Builds a queued job and persists it. `run-fn` is a 1-arg fn of the job that - performs the export and returns a promise; the caller decides when to run - it." - [{:keys [profile-id cmd backend total name resource-id]} run-fn] - (let [job {:id (uuid/next) - :profile-id profile-id - :replica cf/replica-id - :cmd cmd - :backend backend - :state "queued" - :done 0 - :total total - :name name - :resource-id resource-id - :created-at (now-ms)}] - (swap! registry assoc (str (:id job)) {:job job :run-fn run-fn :cancelled? false}) - (->> (store/persist! job) - (p/fmap (constantly job))))) - -(defn run-fn - [job-id] - (:run-fn (runtime job-id))) - -(defn cancelled? - [job-id] - (boolean (:cancelled? (runtime job-id)))) - -(defn terminal? - [job] - (contains? terminal-states (:state job))) - -(defn check-cancelled! - "Raises when the job has been cancelled. Called as each render's turn comes - up, so a cancellation stops the ones that have not started yet." - [job] - (when (cancelled? (:id job)) - (ex/raise :type :internal - :code :job-cancelled - :hint "export job was cancelled"))) - -(defn cancel-signal - "Int32Array over a SharedArrayBuffer, readable from a worker thread: 0 while - the job is live, 1 once it has been cancelled. Nil once the job has been - released -- writing the signal back would leave an entry for a settled job in - the registry that nothing would ever remove." - [job-id] - (let [k (str job-id)] - (when-let [rt (get @registry k)] - (or (:cancel-signal rt) - (let [signal (js/Int32Array. (js/SharedArrayBuffer. 4))] - (swap! registry update k (fn [rt] (some-> rt (assoc :cancel-signal signal)))) - (when (cancelled? job-id) - (js/Atomics.store signal 0 1)) - signal))))) - -(defn on-cancel - "Registers a callback used to abort the job's in-flight work (terminating a - render worker). A job fans out over several renders, so callbacks - accumulate. One registered for a job that already settled is dropped: keeping - it would revive that job's registry entry for good." - [job-id f] - (swap! registry update (str job-id) - (fn [rt] (some-> rt (update :cancel-fns (fnil conj []) f))))) - -(defn release! - "Drops the runtime entry once the job settled. The persisted record stays - until its TTL." - [job-id] - (swap! registry dissoc (str job-id))) - -(defn- live - "The job as the lifecycle last left it, or nil once it settled. Callers hold - the snapshot handed to them when their work started; writing that back would - resurrect a failed export as running and drop the error with it." - [job] - (when-let [current (:job (runtime (:id job)))] - (when-not (terminal? current) - current))) - -(defn- persist-and-publish! - [job] - (store-job! job) - (publish! job) - (store/persist! job)) - -(defn transition! - "Moves the job on. The first terminal state wins: anything arriving after it - is dropped, so a late failure cannot overwrite a cancellation, nor a straggler - overwrite either." - [job data] - (if-let [job (live job)] - (let [job (merge job data)] - (l/info :hint "export job" :job-id (str (:id job)) :state (:state job) - :replica (:replica job)) - (persist-and-publish! job)) - (p/resolved job))) - -(defn start! - [job] - (transition! job {:state "running" :started-at (now-ms)})) - -(defn progress! - "Reports `done` objects completed. Writes are throttled, so the caller need - not care how often it calls this." - [{:keys [id] :as job} done] - (if-let [job (live job)] - (let [k (str id) - now (now-ms) - last (:last-progress-ms (runtime id) 0) - job (assoc job :done done) - write? (>= (- now last) progress-throttle-ms)] - (store-job! job) - (if write? - (do - (swap! registry update k assoc :last-progress-ms now) - (persist-and-publish! job)) - (p/resolved job))) - (p/resolved job))) - -(defn complete! - [job {:keys [uri filename mtype size] :as _resource}] - (transition! job {:state "ended" - :ended-at (now-ms) - :done (:total job) - :resource-uri uri - :filename filename - :mtype mtype - :size size})) - -(defn fail! - [job cause] - (l/error :hint "export job failed" :job-id (str (:id job)) :cause cause) - (transition! job {:state "error" - :ended-at (now-ms) - :error (ex-message cause)})) - -(defn- cancel-local! - [job-id & {:as data}] - (let [k (str job-id) - rt (get @registry k) - job (:job rt)] - (if (or (nil? job) (terminal? job)) - (p/resolved job) - (do - (swap! registry update k (fn [rt] (some-> rt (assoc :cancelled? true)))) - (when-let [signal (:cancel-signal rt)] - (js/Atomics.store signal 0 1)) - ;; Recorded before the callbacks run, not after: one of them - ;; (`scheduler/drop-queued!`) releases the job, and `transition!` on a - ;; released job is a no-op, so a queued job would keep claiming to be - ;; queued in the store and never publish its `cancelled` update. - (let [result (transition! job (merge {:state "cancelled" :ended-at (now-ms)} data))] - (doseq [f (:cancel-fns rt)] - (try - (f) - (catch :default cause - (l/warn :hint "error on job cancel callback" :job-id k :cause cause)))) - result))))) - -(defn cancel! - "Cancels a job. One this process does not own is broadcast over the cancel - topic, so whoever runs it acts on it. Idempotent." - [job-id] - (if (some? (runtime job-id)) - (cancel-local! job-id) - (->> (fetch job-id) - (p/mcat (fn [job] - (cond - (nil? job) (p/resolved nil) - (terminal? job) (p/resolved job) - :else (p/do - (store/request-cancel! (:id job)) - job))))))) - -(defn cancel-all! - "Cancels every job this process holds, on shutdown. Nothing in flight survives - the process, and clients learn it now rather than when this replica boots - again, if it ever does under the same id." - [] - (let [job-ids (keys @registry)] - (when (seq job-ids) - (l/info :hint "cancelling jobs on shutdown" :count (count job-ids))) - (->> job-ids - (map #(cancel-local! % :interrupted true)) - (p/all)))) - -(defn- abandoned? - "A job this replica's previous run left mid-flight. A sibling's jobs are its - own business: they may well be running." - [job] - (and (= cf/replica-id (:replica job)) - (not (terminal? job)))) - -(defn- clean-abandoned! - "Marks every job left mid-flight by a previous run of this replica as - cancelled. - - A queue and its running jobs live in the memory of the process that owns - them, so nothing in flight when it died can be resumed; without this the - record would keep claiming to be running until its TTL expires. A replica - that never comes back under the same id leaves its records to the TTL." - [] - (->> (store/fetch-all) - (p/mcat (fn [jobs] - (let [abandoned (filter abandoned? jobs)] - (when (seq abandoned) - (l/warn :hint "cancelling jobs abandoned by a previous process" - :replica cf/replica-id - :count (count abandoned))) - (->> abandoned - (map (fn [job] - (store/persist! (assoc job - :state "cancelled" - :interrupted true - :ended-at (now-ms))))) - (p/all))))) - (p/fmap (fn [result] (count result))) - (p/merr (fn [cause] - (l/warn :hint "unable to clean abandoned jobs" :cause cause) - (p/resolved 0))))) - -(defn init - [] - (store/on-cancel-request - (fn [job-id] - (when (some? (runtime job-id)) - (l/info :hint "remote cancel request" :job-id job-id) - (cancel-local! job-id)))) - (clean-abandoned!)) diff --git a/exporter/src/app/jobs/scheduler.cljs b/exporter/src/app/jobs/scheduler.cljs deleted file mode 100644 index a986019a57..0000000000 --- a/exporter/src/app/jobs/scheduler.cljs +++ /dev/null @@ -1,150 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.jobs.scheduler - "Admission control for export jobs. - - Limits concurrent jobs and rejects work rather than allowing an unbounded backlog. - Every limit is per replica: the queue lives in this process only. - Queue order is FIFO, except jobs whose profile is already at its cap are skipped, - as are headless jobs once every render worker is busy." - (:require - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.config :as cf] - [app.jobs :as jobs] - [app.jobs.utils :as job.utils] - [app.wasm.pool :as pool] - [promesa.core :as p])) - -(l/set-level! :debug) - -(defonce ^:private state - (atom {:running {} ;; job-id -> {:profile-id :headless?} - :queue []})) ;; vector of {:job :resolve :reject} - -(defn- max-concurrent [] (cf/get :exporter-max-concurrent-jobs 4)) -(defn- max-per-profile [] (cf/get :exporter-max-jobs-per-profile 2)) -(defn- max-queued [] (cf/get :exporter-queue-max 64)) - -(defn- headless? - [job] - (= "wasm" (:backend job))) - -(defn- running-for - [{:keys [running]} profile-id] - (count (filter #(= profile-id (:profile-id %)) (vals running)))) - -(defn- running-headless - [{:keys [running]}] - (count (filter :headless? (vals running)))) - -(defn- eligible? - [state job] - (and (< (count (:running state)) (max-concurrent)) - (< (running-for state (:profile-id job)) (max-per-profile)) - ;; A headless job holds one render worker for its whole run, so admitting - ;; more of them than there are workers would only move the wait inside - ;; the pool, with the job already reporting itself as running. - (or (not (headless? job)) - (< (running-headless state) (pool/capacity))))) - -(declare ^:private pump!) - -(defn- finish! - [job-id] - (swap! state update :running dissoc (str job-id)) - (jobs/release! job-id) - (job.utils/release! job-id) - (pump!)) - -(defn- execute! - [{:keys [id profile-id] :as job}] - (swap! state update :running assoc (str id) {:profile-id profile-id - :headless? (headless? job)}) - (if (jobs/cancelled? id) - (do (finish! id) - (p/resolved job)) - (let [run-fn (jobs/run-fn id)] - (->> (p/do (jobs/start! job)) - (p/mcat (fn [job] (p/do (run-fn job)))) - (p/fnly (fn [_ _] (finish! id))))))) - -(defn- drop-queued! - "Removes a queued job and settles its promise, freeing its queue slot on cancellation." - [job-id] - (let [entry (volatile! nil)] - (swap! state (fn [state] - (let [queue (:queue state) - idx (->> (map-indexed vector queue) - (some (fn [[idx entry]] - (when (= (str job-id) (str (-> entry :job :id))) - idx))))] - (if idx - (do (vreset! entry (nth queue idx)) - (assoc state :queue (into (subvec queue 0 idx) (subvec queue (inc idx))))) - state)))) - (when-let [{:keys [resolve]} @entry] - (jobs/release! job-id) - (job.utils/release! job-id) - (resolve nil)))) - -(defn- take-eligible - "Pops the first queued entry that can run now, or nil." - [state] - (let [queue (:queue state) - idx (->> (map-indexed vector queue) - (some (fn [[idx entry]] - (when (eligible? state (:job entry)) - idx))))] - (when idx - [(assoc state :queue (into (subvec queue 0 idx) (subvec queue (inc idx)))) - (nth queue idx)]))) - -(defn- pump! - [] - (loop [] - (let [entry (volatile! nil)] - (swap! state (fn [state] - (if-let [[next-state next-entry] (take-eligible state)] - (do (vreset! entry next-entry) next-state) - (do (vreset! entry nil) state)))) - (when-let [{:keys [job resolve reject]} @entry] - (-> (execute! job) - (p/then resolve) - (p/catch reject)) - (recur))))) - -(defn submit! - "Registers `job` for execution. Returns a promise of the job's result, which - resolves when the export actually finishes; callers that only need the handle - can ignore it. Raises when the exporter is saturated." - [job] - (let [resolve* (volatile! nil) - reject* (volatile! nil) - pending (p/create (fn [resolve reject] - (vreset! resolve* resolve) - (vreset! reject* reject)))] - (if (eligible? @state job) - (-> (execute! job) - (p/then @resolve*) - (p/catch @reject*)) - - (let [queued (volatile! false)] - (swap! state (fn [state] - (if (< (count (:queue state)) (max-queued)) - (do (vreset! queued true) - (update state :queue conj {:job job - :resolve @resolve* - :reject @reject*})) - (do (vreset! queued false) state)))) - (when-not @queued - (ex/raise :type :validation - :code :queue-full - :hint "too many queued export jobs")) - (jobs/on-cancel (:id job) (fn [] (drop-queued! (:id job)))) - (l/dbg :hint "export job queued" :job-id (str (:id job))))) - pending)) diff --git a/exporter/src/app/jobs/store.cljs b/exporter/src/app/jobs/store.cljs deleted file mode 100644 index 3cfb27b951..0000000000 --- a/exporter/src/app/jobs/store.cljs +++ /dev/null @@ -1,87 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.jobs.store - "Redis persistence for export jobs. - - Stores each job as a single blob with a TTL matching the exported file, - so records expire with their files. - - Reads use Redis. Cancellation requires the process running the job." - (:require - [app.common.logging :as l] - [app.common.transit :as t] - [app.config :as cf] - [app.redis :as redis] - [promesa.core :as p])) - -(def ^:private cancel-topic (redis/->key "job-cancel")) - -(defn- job-key - [job-id] - (redis/->key "job." job-id)) - -(defn- ttl - [] - (cf/get :exporter-job-ttl 3600)) - -(defn persist! - "Writes the job record and refreshes its TTL. - - If the write fails, log it and continue. The export still runs - and publishes websocket updates, but the job can't be fetched - afterward (fetch returns nil, REST returns 404)." - [{:keys [id state] :as job}] - (let [jkey (job-key id)] - (->> (p/do - (redis/hset! jkey {:data (t/encode-str job)}) - (redis/expire! jkey (ttl)) - job) - (p/merr (fn [cause] - (if (= :redis-not-available (:code (ex-data cause))) - (l/warn :hint "job record not persisted, no redis connection" - :job-id (str id) :state state) - (l/error :hint "unable to persist job record" - :job-id (str id) :state state :cause cause)) - (p/resolved job)))))) - -(defn fetch - "The job record, or nil when unknown or expired." - [job-id] - (->> (redis/hgetall (job-key job-id)) - (p/fmap (fn [data] - (when-let [blob (get data "data")] - (try - (t/decode-str blob) - (catch :default cause - (l/warn :hint "unable to decode job record" :job-id (str job-id) :cause cause) - nil))))))) - -(defn fetch-all - [] - (->> (redis/scan (redis/->key "job.*")) - (p/mcat (fn [keys] - (->> (map (fn [k] - (->> (redis/hgetall k) - (p/fmap (fn [data] - (when-let [blob (get data "data")] - (try - (t/decode-str blob) - (catch :default _ nil))))))) - keys) - (p/all)))) - (p/fmap (fn [jobs] (vec (remove nil? jobs)))))) - -(defn request-cancel! - "Asks every exporter to cancel `job-id`. Only the one running it will act." - [job-id] - (redis/pub! cancel-topic (str job-id))) - -(defn on-cancel-request - "Registers `handler` (fn of the job-id string) for cancel requests. Returns an - unsubscribe fn." - [handler] - (redis/sub! cancel-topic handler)) diff --git a/exporter/src/app/jobs/utils.cljs b/exporter/src/app/jobs/utils.cljs deleted file mode 100644 index 1aca381616..0000000000 --- a/exporter/src/app/jobs/utils.cljs +++ /dev/null @@ -1,86 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.jobs.utils - "Temp file ownership for export jobs. - - Temp files used to be cleaned only by the per-file timer in `app.util.shell`, - an hour after creation and lost entirely on restart. Here each job owns the - paths it creates, so they are dropped as soon as it settles and whatever a - crash left behind is cleaned at boot." - (:require - ["node:fs/promises" :as fsp] - ["node:path" :as path] - [app.common.logging :as l] - [app.config :as cf] - [app.util.shell :as sh] - [cuerdas.core :as str] - [promesa.core :as p])) - -(def ^:private managed-prefix "penpot.") - -(defonce ^:private tracked (atom {})) - -(defn track! - "Registers `path` as owned by `job-id`, so it is removed when the job settles." - [job-id path] - (when (and job-id path) - (swap! tracked update (str job-id) (fnil conj #{}) path)) - path) - -(defn- remove-path! - [path] - (->> (p/do (fsp/rm path #js {:recursive true :force true})) - (p/merr (fn [cause] - (l/warn :hint "unable to remove job temp file" :path path :cause cause) - (p/resolved nil))))) - -(defn release! - "Removes every file the job owns. Called once the job reached a terminal - state and its result has already been uploaded, so nothing else reads them." - [job-id] - (let [k (str job-id) - paths (get @tracked k)] - (swap! tracked dissoc k) - (if (seq paths) - (->> (map remove-path! paths) - (p/all) - (p/fmap (fn [_] - (l/dbg :hint "released job temp files" :job-id k :count (count paths)) - nil))) - (p/resolved nil)))) - -(defn- clean! - "Removes managed temp files older than the job TTL. They can only be leftovers - of a previous process: every live one belongs to a job of this process." - [] - (let [max-age (* 1000 (cf/get :exporter-job-ttl 3600)) - now (js/Date.now)] - (->> (p/do (fsp/readdir sh/tmpdir)) - (p/mcat (fn [entries] - (->> (filter #(str/starts-with? % managed-prefix) entries) - (map (fn [entry] - (let [fpath (path/join sh/tmpdir entry)] - (->> (p/do (fsp/stat fpath)) - (p/mcat (fn [^js stat] - (if (> (- now (inst-ms (.-mtime stat))) max-age) - (->> (remove-path! fpath) - (p/fmap (constantly 1))) - (p/resolved 0)))) - (p/merr (fn [_] (p/resolved 0))))))) - (p/all)))) - (p/fmap (fn [results] - (let [removed (reduce + 0 results)] - (when (pos? removed) - (l/info :hint "removed orphaned export temp files" :count removed)) - removed))) - (p/merr (fn [cause] - (l/warn :hint "temp file cleanup failed" :cause cause) - (p/resolved 0)))))) - -(defn init - [] - (clean!)) diff --git a/exporter/src/app/redis.cljs b/exporter/src/app/redis.cljs deleted file mode 100644 index 8fca0c335c..0000000000 --- a/exporter/src/app/redis.cljs +++ /dev/null @@ -1,177 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.redis - (:require - ["ioredis" :as redis] - [app.common.data.macros :as dm] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.common.transit :as t] - [app.config :as cf] - [promesa.core :as p])) - -(l/set-level! :trace) - -(def client (atom nil)) - -;; A connection in subscriber mode rejects every other command, so the -;; subscriptions need a connection of their own. -(def ^:private subscriber (atom nil)) - -(def ^:private subscriptions (atom {})) - -(defn- create-client - [uri role] - (let [^js client (new redis/default uri)] - (.on client "connect" - (fn [] (l/info :hint "redis connection established" :uri uri :role role))) - (.on client "error" - (fn [cause] (l/error :hint "error on redis connection" :role role :cause cause))) - (.on client "close" - (fn [] (l/warn :hint "connection closed" :role role))) - (.on client "reconnect" - (fn [ms] (l/warn :hint "reconnecting to redis" :role role :ms ms))) - (.on client "end" - (fn [] (l/warn :hint "client ended, no more connections will be attempted" :role role))) - client)) - -(defn- dispatch-message - [topic payload] - (doseq [handler (get @subscriptions topic)] - (try - (handler payload) - (catch :default cause - (l/error :hint "error on redis subscription handler" :topic topic :cause cause))))) - -(defn init - [] - (let [uri (cf/get :redis-uri)] - (swap! client (fn [prev] - (when prev (.disconnect ^js prev)) - (create-client uri "commands"))) - (swap! subscriber (fn [prev] - (when prev (.disconnect ^js prev)) - (let [^js conn (create-client uri "subscriber")] - (.on conn "message" (fn [topic payload] (dispatch-message topic payload))) - ;; Reinstate subscriptions after a reconnection. - (.on conn "connect" - (fn [] - (doseq [topic (keys @subscriptions)] - (.subscribe conn topic)))) - conn))))) - -(defn stop - [] - (reset! subscriptions {}) - (swap! subscriber (fn [conn] - (when conn (.quit ^js conn)) - nil)) - (swap! client (fn [client] - (when client (.quit ^js client)) - nil))) - -(def ^:private tenant (cf/get :tenant)) - -(defn ->tenant-key - "Namespaces `parts` under the tenant, the prefix the backend msgbus uses." - [& parts] - (dm/str tenant "." (apply str parts))) - -(defn ->key - "Namespaces `parts` under the exporter, inside the tenant." - [& parts] - (dm/str "penpot.exporter." tenant "." (apply str parts))) - -(defn pub! - "Publishes on `topic`, which must already be namespaced." - [topic payload] - (let [payload (if (map? payload) (t/encode-str payload) payload)] - (when-let [client @client] - (.publish ^js client topic payload)))) - -(defn sub! - "Subscribes `handler` (fn of the raw payload string) to `topic`, which must - already be namespaced. Returns a 0-arg fn that removes this handler." - [topic handler] - (swap! subscriptions update topic (fnil conj []) handler) - (when-let [conn @subscriber] - (.subscribe ^js conn topic)) - (fn [] - (swap! subscriptions update topic (fn [handlers] (vec (remove #(= % handler) handlers)))))) - -(defn- with-client - "Runs `f` against the command connection. Rejects when there is no connection - or the command fails: whether a failure is survivable depends on what the - caller was doing, and only the caller knows." - [f] - (if-let [client @client] - (p/do (f client)) - (p/rejected (ex/error :type :internal - :code :redis-not-available - :hint "no redis connection")))) - -(defn- with-client-lenient - "For reads, where an unreachable redis is reported as \"nothing there\"." - [f] - (->> (with-client f) - (p/merr (fn [cause] - (l/warn :hint "redis command failed" :cause cause) - (p/resolved nil))))) - -(defn hset! - "Writes `data` (a map of string/keyword -> value) as a hash. Nil values are - dropped, since redis has no null." - [k data] - (let [obj (reduce-kv (fn [obj field value] - (if (some? value) - (doto obj (unchecked-set (name field) (str value))) - obj)) - #js {} - data)] - (if (zero? (alength (js/Object.keys obj))) - (p/resolved nil) - (with-client (fn [^js client] (.hset client k obj)))))) - -(defn hgetall - "Returns the hash as a map of string keys, or nil when it does not exist." - [k] - (->> (with-client-lenient (fn [^js client] (.hgetall client k))) - (p/fmap (fn [result] - (when (and result (pos? (alength (js/Object.keys result)))) - (persistent! - (reduce (fn [res field] - (assoc! res field (unchecked-get result field))) - (transient {}) - (js/Object.keys result)))))))) - -(defn expire! - [k seconds] - (with-client (fn [^js client] (.expire client k seconds)))) - -(defn del! - [k] - (with-client (fn [^js client] (.del client k)))) - -(defn scan - "Every key matching `pattern`, walked in cursor batches so a large keyspace is - never blocked the way `KEYS` would block it. - - Batches are accumulated in memory rather than consumed as a stream, which a - promise-returning fn cannot express. Fine for the job keyspace, but reading - redis wants a streaming or reactive interface before it is used for more." - [pattern] - (letfn [(step [cursor found] - (->> (with-client-lenient (fn [^js client] (.scan client cursor "MATCH" pattern "COUNT" 200))) - (p/mcat (fn [result] - (if (nil? result) - (p/resolved found) - (let [next-cursor (aget result 0) - found (into found (aget result 1))] - (if (= "0" next-cursor) - (p/resolved found) - (step next-cursor found))))))))] - (step "0" []))) diff --git a/exporter/src/app/renderer.cljs b/exporter/src/app/renderer.cljs deleted file mode 100644 index 98f97f3443..0000000000 --- a/exporter/src/app/renderer.cljs +++ /dev/null @@ -1,79 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.renderer - "Common renderer interface." - (:require - [app.common.logging :as l] - [app.common.spec :as us] - [app.renderer.bitmap :as rb] - [app.renderer.pdf :as rp] - [app.renderer.svg :as rs] - [app.renderer.wasm :as rw] - [cljs.spec.alpha :as s])) - -(s/def ::name ::us/string) -(s/def ::suffix ::us/string) -(s/def ::type #{:png :jpeg :webp :pdf :svg}) -(s/def ::page-id ::us/uuid) -(s/def ::file-id ::us/uuid) -(s/def ::share-id ::us/uuid) -(s/def ::scale ::us/number) -(s/def ::token ::us/string) -(s/def ::filename ::us/string) -(s/def ::is-wasm ::us/boolean) -(s/def ::job-id ::us/uuid) - -(s/def ::object - (s/keys :req-un [::id ::name ::suffix ::filename] - :opt-un [::share-id])) - -(s/def ::objects - (s/coll-of ::object :min-count 1)) - -(s/def ::render-params - (s/keys :req-un [::file-id ::page-id ::scale ::token ::type ::objects] - :opt-un [::is-wasm ::job-id])) - -(defn headless? - "Whether `params` renders with render-wasm rather than a browser." - [{:keys [is-wasm]}] - (boolean is-wasm)) - -(defn render - [{:keys [type is-wasm] :as params} on-object] - (us/verify ::render-params params) - (us/verify fn? on-object) - (let [headless? (headless? params)] - (when is-wasm - (l/info :hint "render" - :type type - :backend (if headless? "wasm" "browser"))) - (if headless? - (rw/render params on-object) - (case type - :png (rb/render params on-object) - :jpeg (rb/render params on-object) - :webp (rb/render params on-object) - :pdf (rp/render params on-object) - :svg (rs/render params on-object))))) - -(defn with-scope - "Runs `f`, a fn of a render fn with the same signature as `render`. Exports - that render headless share one worker for the whole call instead of acquiring - one per render; the browser backend keeps rendering them in parallel." - [exports f] - (if (some headless? exports) - (rw/with-scope (:job-id (first exports)) - (fn [render-leased] - (f (fn [params on-object] - (us/verify ::render-params params) - (us/verify fn? on-object) - (if (headless? params) - (render-leased params on-object) - (render params on-object)))))) - (f render))) - diff --git a/exporter/src/app/renderer/bitmap.cljs b/exporter/src/app/renderer/bitmap.cljs deleted file mode 100644 index ceefc75df5..0000000000 --- a/exporter/src/app/renderer/bitmap.cljs +++ /dev/null @@ -1,67 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.renderer.bitmap - "A bitmap renderer." - (:require - [app.browser :as bw] - [app.common.logging :as l] - [app.common.uri :as u] - [app.config :as cf] - [app.util.mime :as mime] - [app.util.shell :as sh] - [cuerdas.core :as str] - [promesa.core :as p])) - -(defn render - [{:keys [file-id page-id share-id token scale type objects skip-children] :as params} on-object] - (letfn [(prepare-options [uri] - #js {:screen #js {:width bw/default-viewport-width - :height bw/default-viewport-height} - :viewport #js {:width bw/default-viewport-width - :height bw/default-viewport-height} - :locale "en-US" - :storageState #js {:cookies (bw/create-cookies uri {:token token})} - :deviceScaleFactor scale - :userAgent bw/default-user-agent}) - - (render-object [page {:keys [id] :as object}] - (p/let [path (sh/tempfile :prefix "penpot.tmp.bitmap." :suffix (mime/get-extension type)) - node (bw/select page (str/concat "#screenshot-" id))] - (bw/wait-for node) - (case type - :png (bw/screenshot node {:omit-background? true :type type :path path}) - :jpeg (bw/screenshot node {:omit-background? false :type type :path path}) - :webp (p/let [png-path (sh/tempfile :prefix "penpot.tmp.bitmap." :suffix ".png")] - ;; playwright only supports jpg and png, we need to convert it afterwards - (bw/screenshot node {:omit-background? true :type :png :path png-path}) - (sh/run-cmd! "convert" png-path "-quality" "100" (str "WEBP:" path)))) - (on-object (assoc object :path path)))) - - (render [uri page] - (l/info :uri uri) - (p/do - ;; navigate to the page and perform basic setup - (bw/nav! page (str uri)) - (bw/sleep page 1000) ; the good old fix with sleep - (bw/wait-for-fonts page) - (bw/eval! page (js* "() => document.body.style.background = 'transparent'")) - - ;; take the screnshot of requested objects, one by one - (p/run (partial render-object page) objects) - nil))] - - (p/let [params {:file-id file-id - :page-id page-id - :share-id share-id - :object-id (mapv :id objects) - :route "objects" - :skip-children skip-children} - uri (-> (cf/get-internal-uri) - (u/ensure-path-slash) - (u/join "render.html") - (assoc :query (u/map->query-string params)))] - (bw/exec! (prepare-options uri) (partial render uri))))) diff --git a/exporter/src/app/renderer/pdf.cljs b/exporter/src/app/renderer/pdf.cljs deleted file mode 100644 index 5d983d80c0..0000000000 --- a/exporter/src/app/renderer/pdf.cljs +++ /dev/null @@ -1,83 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.renderer.pdf - "A pdf renderer." - (:require - [app.browser :as bw] - [app.common.data.macros :as dm] - [app.common.logging :as l] - [app.common.uri :as u] - [app.config :as cf] - [app.util.mime :as mime] - [app.util.shell :as sh] - [promesa.core :as p])) - -(defn render - [{:keys [file-id page-id share-id token scale type objects] :as params} on-object] - (letfn [(prepare-options [uri] - #js {:screen #js {:width bw/default-viewport-width - :height bw/default-viewport-height} - :viewport #js {:width bw/default-viewport-width - :height bw/default-viewport-height} - :locale "en-US" - :storageState #js {:cookies (bw/create-cookies uri {:token token})} - :deviceScaleFactor scale - :userAgent bw/default-user-agent}) - - (prepare-uri [base-uri object-id] - (let [params {:file-id file-id - :page-id page-id - :share-id share-id - :object-id object-id - :route "objects"}] - (-> base-uri - (u/join "render.html") - (assoc :query (u/map->query-string params))))) - - (sync-page-size! [dom] - (bw/eval! dom - (fn [elem] - ;; IMPORTANT: No CLJS runtime allowed. Use only JS - ;; primitives. This runs in a context without access to - ;; cljs.core. Avoid any functions that transpile to - ;; cljs.core/* calls, as they will break in the browser - ;; runtime. - - (let [width (.getAttribute ^js elem "width") - height (.getAttribute ^js elem "height") - style-node (let [node (.createElement js/document "style")] - (.appendChild (.-head js/document) node) - node)] - (set! (.-textContent style-node) - (dm/str "@page { size: " width "px " height "px; margin: 0; }\n" - "html, body, #app { margin: 0; padding: 0; width: " width "px; height: " height "px; overflow: visible; }")))))) - - (render-object [page base-uri {:keys [id] :as object}] - (p/let [uri (prepare-uri base-uri id) - path (sh/tempfile :prefix "penpot.tmp.pdf." :suffix (mime/get-extension type))] - (l/info :uri uri) - (bw/nav! page uri) - (p/let [dom (bw/select page (dm/str "#screenshot-" id))] - (bw/wait-for dom) - (sync-page-size! dom) - (bw/screenshot dom {:full-page? true}) - (bw/sleep page 2000) ; the good old fix with sleep - (bw/wait-for-fonts page) - (bw/pdf page {:path path}) - path))) - - (render [base-uri page] - (p/loop [objects (seq objects)] - (when-let [object (first objects)] - (p/let [path (render-object page base-uri object)] - (on-object (assoc object :path path)) - (p/recur (rest objects))))))] - - (let [base-uri (-> (cf/get-internal-uri) - (u/ensure-path-slash))] - (bw/exec! (prepare-options base-uri) - (partial render base-uri))))) diff --git a/exporter/src/app/renderer/svg.cljs b/exporter/src/app/renderer/svg.cljs deleted file mode 100644 index fdd9b7c0de..0000000000 --- a/exporter/src/app/renderer/svg.cljs +++ /dev/null @@ -1,356 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.renderer.svg - (:require - ["@penpot/svgo" :as svgo] - ["xml-js" :as xml] - [app.browser :as bw] - [app.common.data :as d] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.common.types.color :as ctc] - [app.common.uri :as u] - [app.config :as cf] - [app.renderer.svg-gradient :as svg-gradient] - [app.util.mime :as mime] - [app.util.shell :as sh] - [clojure.walk :as walk] - [cuerdas.core :as str] - [promesa.core :as p])) - -(l/set-level! :trace) - -(defn- xml->clj - [data] - (js->clj (xml/xml2js data))) - -(defn- clj->xml - [data] - (xml/js2xml (clj->js data))) - -(defn ^boolean element? - [item] - (and (map? item) - (= "element" (get item "type")))) - -(defn ^boolean group-element? - [item] - (and (element? item) - (= "g" (get item "name")))) - -(defn ^boolean shape-element? - [item] - (and (element? item) - (str/starts-with? (get-in item ["attributes" "id"]) "shape-"))) - -(defn ^boolean foreign-object-element? - [item] - (and (element? item) - (= "foreignObject" (get item "name")))) - -(defn ^boolean empty-defs-element? - [item] - (and (= (get item "name") "defs") - (nil? (get item "attributes")) - (nil? (get item "elements")))) - -(defn ^boolean empty-path-element? - [item] - (and (= (get item "name") "path") - (let [d (get-in item ["attributes" "d"])] - (or (str/blank? d) - (nil? d) - (str/empty? d))))) - -(defn flatten-toplevel-svg-elements - "Flattens XML data structure if two nested top-side SVG elements found." - [item] - (if (and (= "svg" (get-in item ["elements" 0 "name"])) - (= "svg" (get-in item ["elements" 0 "elements" 0 "name"]))) - (update-in item ["elements" 0] assoc "elements" (get-in item ["elements" 0 "elements" 0 "elements"])) - item)) - -(defn replace-text-nodes - "Function responsible of replace the foreignObject elements on the - provided XML with the previously rasterized PATH's." - [xmldata nodes] - (letfn [(replace-fobject [item] - (if (foreign-object-element? item) - (let [id (get-in item ["attributes" "id"]) - node (get nodes id)] - (if node - (:svgdata node) - item)) - item)) - - (process-element [item xform] - (d/update-when item "elements" #(into [] xform %)))] - - (let [xform (comp (remove empty-defs-element?) - (remove empty-path-element?) - (map replace-fobject))] - (->> xmldata - (xml->clj) - (flatten-toplevel-svg-elements) - (walk/prewalk (fn [item] - (cond-> item - (element? item) - (process-element xform)))) - (clj->xml))))) - -(defn parse-viewbox - "Parses viewBox string into width & height map." - [data] - (let [[width height] (->> (str/split data #"\s+") - (drop 2) - (map d/parse-double))] - {:width width - :height height})) - -(defn- replace-internal-uris - "Replaces internal-uri references with public-uri in SVG output. - This ensures that font URLs and other resource references in the - exported SVG use the public-facing URI accessible to end users." - [svg-content] - (let [internal-uri (str (cf/get-internal-uri)) - public-uri (str (cf/get :public-uri))] - (if (and (not= internal-uri public-uri) - (str/includes? svg-content internal-uri)) - (str/replace svg-content internal-uri public-uri) - svg-content))) - -(defn render - [{:keys [page-id file-id share-id objects token scale type]} on-object] - (letfn [(convert-to-ppm [pngpath] - (let [ppmpath (str/concat pngpath "origin.ppm")] - (l/trace :fn :convert-to-ppm :path ppmpath) - (-> (sh/run-cmd! "convert" pngpath ppmpath) - (p/then (constantly ppmpath))))) - - (trace-color-mask [pbmpath] - (l/trace :fn :trace-color-mask :pbmpath pbmpath) - (let [svgpath (str/concat pbmpath ".svg")] - (-> (sh/run-cmd! "potrace" "--flat" "-b" "svg" pbmpath "-o" svgpath) - (p/then (constantly svgpath))))) - - (generate-color-layer [ppmpath color] - (when-not (ctc/hex-color-string? color) - (ex/raise :type :validation - :code :invalid-color - :hint (str "invalid hex color: " color))) - (l/trace :fn :generate-color-layer :ppmpath ppmpath :color color) - (let [pbmpath (str/concat ppmpath ".mask-" (subs color 1) ".pbm")] - (-> (sh/run-cmd! "ppmcolormask" color ppmpath) - (p/then (fn [stdout] - (-> (sh/write-file! pbmpath stdout) - (p/then (constantly pbmpath))))) - (p/then trace-color-mask) - (p/then sh/read-file) - (p/then (fn [data] - (p/let [data (xml->clj data) - data (get-in data ["elements" 1])] - {:color color - :svgdata data})))))) - - (set-path-color [id color mapping node] - (let [color-mapping (get mapping color)] - (cond - (and (some? color-mapping) - (= "transparent" (get color-mapping "type"))) - (update node "attributes" assoc - "fill" (get color-mapping "hex") - "fill-opacity" (get color-mapping "opacity")) - - (and (some? color-mapping) - (= "gradient" (get color-mapping "type"))) - (update node "attributes" assoc - "fill" (str "url(#gradient-" id "-" (subs color 1) ")")) - - :else - (update node "attributes" assoc "fill" color)))) - - (get-gradients [id mapping] - (->> mapping - (filter (fn [[_color data]] - (= (get data "type") "gradient"))) - (mapv (partial svg-gradient/data->gradient-def id)))) - - (join-color-layers [{:keys [id x y width height mapping] :as node} layers] - (l/trace :fn :join-color-layers :mapping mapping) - (loop [result (-> (:svgdata (first layers)) - (assoc "elements" [])) - layers (seq layers)] - (if-let [{:keys [color svgdata]} (first layers)] - (recur (->> (get svgdata "elements") - (filter #(= (get % "name") "g")) - (map (partial set-path-color id color mapping)) - (update result "elements" into)) - (rest layers)) - - ;; Now we have the result containing the svgdata of a - ;; SVG with all text layers. Now we need to transform - ;; this SVG to G (Group) and remove unnecessary metadata - ;; objects. - (let [vbox (-> (get-in result ["attributes" "viewBox"]) - (parse-viewbox)) - transform (str/fmt "translate(%s, %s) scale(%s, %s)" x y - (/ width (:width vbox)) - (/ height (:height vbox))) - - gradient-defs (get-gradients id mapping) - - elements - (->> (get result "elements") - (mapv (fn [group] - (let [paths (get group "elements")] - (if (= 1 (count paths)) - (let [path (first paths)] - (update path "attributes" - (fn [attrs] - (-> attrs - (d/merge (get group "attributes")) - (update "transform" #(str transform " " %)))))) - (update-in group ["attributes" "transform"] #(str transform " " %))))))) - - - elements (cond->> elements - (seq gradient-defs) - (into [{"type" "element" "name" "defs" "attributes" {} - "elements" gradient-defs}]))] - - (-> result - (assoc "name" "g") - (assoc "attributes" {}) - (assoc "elements" elements)))))) - - (convert-to-svg [ppmpath {:keys [colors] :as node}] - (l/trace :fn :convert-to-svg :ppmpath ppmpath :colors colors) - (-> (p/all (map (partial generate-color-layer ppmpath) colors)) - (p/then (partial join-color-layers node)))) - - (trace-node [{:keys [data] :as node}] - (l/trace :fn :trace-node) - (p/let [pngpath (sh/tempfile :prefix "penpot.tmp.render.svg.parse." - :suffix ".origin.png") - _ (sh/write-file! pngpath data) - ppmpath (convert-to-ppm pngpath) - svgdata (convert-to-svg ppmpath node)] - (-> node - (dissoc :data) - (assoc :svgdata svgdata)))) - - (extract-element-attrs [^js element] - (let [^js attrs (.. element -attributes) - ^js colors (.. element -dataset -colors) - ^js mapping (.. element -dataset -mapping)] - #js {:id (.. attrs -id -value) - :x (.. attrs -x -value) - :y (.. attrs -y -value) - :width (.. attrs -width -value) - :height (.. attrs -height -value) - :colors (.split colors ",") - :mapping (js/JSON.parse mapping)})) - - (extract-single-node [[shot node]] - (l/trace :fn :extract-single-node) - - (p/let [attrs (bw/eval! node extract-element-attrs)] - {:id (unchecked-get attrs "id") - :x (unchecked-get attrs "x") - :y (unchecked-get attrs "y") - :width (unchecked-get attrs "width") - :height (unchecked-get attrs "height") - :colors (vec (unchecked-get attrs "colors")) - :mapping (js->clj (unchecked-get attrs "mapping")) - :data shot})) - - (resolve-text-node [page node] - (p/let [attrs (bw/eval! node extract-element-attrs) - id (unchecked-get attrs "id") - text-node (bw/select page (str "#screenshot-text-" id " foreignObject")) - shot (bw/screenshot text-node {:omit-background? true :type "png"})] - [shot node])) - - (extract-txt-node [page item] - (-> (p/resolved item) - (p/then (partial resolve-text-node page)) - (p/then extract-single-node) - (p/then trace-node))) - - (extract-txt-nodes [page {:keys [id] :as objects}] - (l/trace :fn :process-text-nodes) - (-> (bw/select-all page (str/concat "#screenshot-" id " foreignObject")) - (p/then (fn [nodes] (p/all (map (partial extract-txt-node page) nodes)))) - (p/then (fn [nodes] (d/index-by :id nodes))))) - - (extract-svg [page {:keys [id] :as object}] - (let [node (bw/select page (str/concat "#screenshot-" id))] - (bw/wait-for node) - (bw/eval! node (fn [elem] (.-outerHTML ^js elem))))) - - (prepare-options [uri] - #js {:screen #js {:width bw/default-viewport-width - :height bw/default-viewport-height} - :viewport #js {:width bw/default-viewport-width - :height bw/default-viewport-height} - :locale "en-US" - :storageState #js {:cookies (bw/create-cookies uri {:token token})} - :deviceScaleFactor scale - :userAgent bw/default-user-agent}) - - (render-object [page {:keys [id] :as object}] - (p/let [path (sh/tempfile :prefix "penpot.tmp.render.svg." :suffix (mime/get-extension type)) - node (bw/select page (str/concat "#screenshot-" id))] - (bw/wait-for node) - (p/let [xmldata (extract-svg page object) - txtdata (extract-txt-nodes page object) - result (replace-text-nodes xmldata txtdata) - - ;; SVG standard don't allow the entity - ;; nbsp.   is equivalent but compatible - ;; with SVG. - result (str/replace result " " " ") - - result (if (contains? cf/flags :exporter-svgo) - (svgo/optimize result svgo/defaultOptions) - result) - - result (replace-internal-uris result)] - - ;; (println "------- ORIGIN:") - ;; (cljs.pprint/pprint (xml->clj xmldata)) - ;; (println "------- RESULT:") - ;; (cljs.pprint/pprint (xml->clj result)) - ;; (println "-------") - - (sh/write-file! path result) - (on-object (assoc object :path path)) - path))) - - (render [uri page] - (l/info :uri uri) - (p/do - ;; navigate to the page and perform basic setup - (bw/nav! page (str uri)) - (bw/sleep page 1000) ; the good old fix with sleep - (bw/wait-for-fonts page) - - ;; take the screnshot of requested objects, one by one - (p/run (partial render-object page) objects) - nil))] - (p/let [params {:file-id file-id - :page-id page-id - :share-id share-id - :render-embed true - :object-id (mapv :id objects) - :route "objects"} - uri (-> (cf/get-internal-uri) - (u/ensure-path-slash) - (u/join "render.html") - (assoc :query (u/map->query-string params)))] - (bw/exec! (prepare-options uri) - (partial render uri))))) diff --git a/exporter/src/app/renderer/wasm.cljs b/exporter/src/app/renderer/wasm.cljs deleted file mode 100644 index a284e35269..0000000000 --- a/exporter/src/app/renderer/wasm.cljs +++ /dev/null @@ -1,50 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.renderer.wasm - "Main-thread side of the headless renderer. - - Renders run on pooled workers because Skia calls are synchronous and would block - the HTTP server and other exports. Each job keeps one worker for all its renders, - sharing its caches and pool slot." - (:require - [app.jobs :as jobs] - [app.wasm.pool :as pool] - [promesa.core :as p])) - -(defn- serializer - "Chains thunks so a job's renders run one at a time on its worker. A failure - is isolated: it doesn't break the chain for the next one." - [] - (let [queue (atom (p/resolved nil))] - (fn [thunk] - (let [result (p/handle @queue (fn [_ _] (thunk)))] - (reset! queue (p/handle result (fn [_ _] nil))) - result)))) - -(defn with-scope - "Runs `f`, a fn of a 2-arg render fn. Every render goes to the same worker, - one at a time, so the cancel check runs as each render's turn comes up." - [job-id f] - (pool/with-worker - (fn [worker] - (let [chain (serializer) - live (volatile! worker) - signal (when job-id (jobs/cancel-signal job-id)) - opts {:cancel-buffer (some-> signal (.-buffer)) - :cancelled? (when job-id #(jobs/cancelled? job-id))}] - (when job-id - ;; Between objects the worker sees the flag; inside a render only - ;; terminating the thread stops it. Cleared on the way out so a later - ;; cancel cannot terminate a worker that is by then somebody else's. - (jobs/on-cancel job-id (fn [] (pool/terminate! @live)))) - (->> (p/do (f (fn [params on-object] - (chain #(pool/render-on worker params on-object opts))))) - (p/fnly (fn [_ _] (vreset! live nil)))))))) - -(defn render - [params on-object] - (with-scope (:job-id params) (fn [render*] (render* params on-object)))) diff --git a/exporter/src/app/router.cljs b/exporter/src/app/router.cljs deleted file mode 100644 index 7b763edc8a..0000000000 --- a/exporter/src/app/router.cljs +++ /dev/null @@ -1,61 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.router - "Method + path dispatch. - - Requests arrive with whatever prefix the proxy in front uses (`/api/export` - in devenv, `/` when talking to the process directly), so routes are matched - on the remainder after that prefix." - (:require - [app.common.exceptions :as ex] - [app.handlers.jobs :as jobs.handlers] - [cuerdas.core :as str])) - -(def ^:private mount-point "/api/export") - -(defn- route-path - [path] - (let [path (or path "/") - path (if (str/starts-with? path mount-point) - (subs path (count mount-point)) - path) - path (str/rtrim path "/")] - (if (str/empty? path) "/" path))) - -(defn- job-id - [path prefix] - (let [id (subs path (count prefix))] - (when-not (or (str/empty? id) (str/includes? id "/")) - id))) - -(defn create - "Builds the request handler. `legacy-handler` serves the original - `POST /api/export` command multiplex." - [legacy-handler] - (fn [{:keys [:request/method :request/path] :as exchange}] - (let [path (route-path path)] - (cond - (and (= "post" method) (= "/" path)) - (legacy-handler exchange) - - (and (= "post" method) (= "/jobs" path)) - (jobs.handlers/create exchange) - - (and (= "get" method) (str/starts-with? path "/jobs/")) - (if-let [id (job-id path "/jobs/")] - (jobs.handlers/fetch exchange id) - (ex/raise :type :not-found :code :object-not-found :hint "unknown route")) - - (and (= "delete" method) (str/starts-with? path "/jobs/")) - (if-let [id (job-id path "/jobs/")] - (jobs.handlers/cancel exchange id) - (ex/raise :type :not-found :code :object-not-found :hint "unknown route")) - - :else - (ex/raise :type :not-found - :code :route-not-found - :hint (str "no route for " method " " path)))))) diff --git a/exporter/src/app/util/shell.cljs b/exporter/src/app/util/shell.cljs deleted file mode 100644 index 929924899b..0000000000 --- a/exporter/src/app/util/shell.cljs +++ /dev/null @@ -1,107 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.util.shell - "Shell & FS utilities." - (:require - ["node:child_process" :as proc] - ["node:fs" :as fs] - ["node:path" :as path] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.common.time :as ct] - [app.common.uuid :as uuid] - [app.config :as cf] - [cuerdas.core :as str] - [promesa.core :as p])) - -(l/set-level! :trace) - -(def ^:const default-deletion-delay - (* 60 60 1)) ;; 1h - -(def tmpdir - (let [path (cf/get :tempdir)] - (l/inf :hint "tmptdir setup" :path path) - (when-not (fs/existsSync path) - (fs/mkdirSync path #js {:recursive true})) - path)) - -(defn schedule-deletion - ([path] (schedule-deletion path default-deletion-delay)) - ([path delay] - (let [remove-path - (fn [] - (try - (when (fs/existsSync path) - (fs/rmSync path #js {:recursive true}) - (l/trc :hint "tempfile permanently deleted" :path path)) - (catch :default cause - (l/err :hint "error on deleting temporal file" - :path path - :cause cause)))) - scheduled-at - (-> (ct/now) (ct/plus #js {:seconds delay}))] - - (l/trc :hint "schedule tempfile deletion" - :path path - :scheduled-at (ct/format-inst scheduled-at)) - - (js/setTimeout remove-path (* delay 1000)) - path))) - -(defn tempfile - [& {:keys [prefix suffix] - :or {prefix "penpot." - suffix ".tmp"}}] - (loop [i 0] - (if (< i 1000) - (let [path (path/join tmpdir (str/concat prefix (uuid/next) "-" i suffix))] - (if (fs/existsSync path) - (recur (inc i)) - (schedule-deletion path))) - (ex/raise :type :internal - :code :unable-to-locate-temporal-file - :hint "unable to find a tempfile candidate")))) - -(defn move! - [origin-path dest-path] - (.rename fs/promises origin-path dest-path)) - -(defn stat - [path] - (->> (.stat fs/promises path) - (p/fmap (fn [data] - {:path path - :created-at (inst-ms (.-ctime ^js data)) - :size (.-size data)})) - (p/merr (fn [_cause] - (p/resolved nil))))) - -(defn rmdir! - [path] - (.rm fs/promises path #js {:recursive true})) - -(defn write-file! - [fpath content] - (.writeFile fs/promises fpath content)) - -(defn read-file - [fpath] - (.readFile fs/promises fpath)) - -(defn run-cmd! - [cmd & args] - (p/create - (fn [resolve reject] - (l/trace :fn :run-cmd :cmd cmd :args args) - (proc/execFile cmd (clj->js args) #js {:encoding "buffer"} - (fn [error stdout _stderr] - ;; (l/trace :fn :run-cmd :stdout stdout) - (if error - (reject error) - (resolve stdout))))))) - diff --git a/exporter/src/app/util/transit.cljs b/exporter/src/app/util/transit.cljs deleted file mode 100644 index 859606e3ca..0000000000 --- a/exporter/src/app/util/transit.cljs +++ /dev/null @@ -1,29 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.util.transit - (:require - [cognitect.transit :as t])) - -;; --- Transit Handlers - -(def ^:private +read-handlers+ - {"u" uuid}) - -(def ^:private +write-handlers+ - {}) - -;; --- Public Api - -(defn decode - [data] - (let [r (t/reader :json {:handlers +read-handlers+})] - (t/read r data))) - -(defn encode - [data] - (let [w (t/writer :json-verbose {:handlers +write-handlers+})] - (t/write w data))) diff --git a/exporter/src/app/wasm/pool.cljs b/exporter/src/app/wasm/pool.cljs deleted file mode 100644 index 31a20527e7..0000000000 --- a/exporter/src/app/wasm/pool.cljs +++ /dev/null @@ -1,240 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.wasm.pool - "Pool of headless render workers. - - Mirrors `app.browser`: a `generic-pool` whose objects are `worker_threads` - instead of browsers, so acquisition and eviction behave the same way for both - render backends. A worker is expensive to build (it boots its own render-wasm - module), hence the pooling. - - Acquisition is not capped: the admission scheduler is the backpressure, and - the idle watchdog guarantees a wedged worker gives its slot back. - - Workers run the same bundle as the main thread; `app.core/start` branches on - `isMainThread`. There is always at least one worker, since a headless render - has nowhere else to go." - (:require - ["generic-pool" :as gp] - ["node:path" :as path] - ["node:process" :as proc] - ["node:worker_threads" :as wt] - [app.common.exceptions :as ex] - [app.common.logging :as l] - [app.common.transit :as t] - [app.config :as cf] - [promesa.core :as p])) - -(l/set-level! :info) - -(defonce pool (atom nil)) -(defonce ^:private worker-id (atom 0)) - -(def ^:private ready-timeout-ms 60000) - -(defn- idle-timeout-ms - "How long a render may go silent before the worker is presumed wedged. Reset - on every message, so a long export keeps its worker as long as it keeps - reporting objects; only a thread stuck inside Skia, which reports nothing and - emits no `exit`, runs it out." - [] - (* 1000 (cf/get :wasm-worker-idle-timeout 300))) - -(defn- worker-script - [] - (path/resolve (aget (.-argv proc/default) 1))) - -(defn- create-worker - [] - (p/create - (fn [resolve reject] - (let [script (worker-script) - id (swap! worker-id inc) - worker (new wt/Worker script) - timer (js/setTimeout - (fn [] - (l/error :hint "render worker did not become ready" :worker-id id) - (.terminate ^js worker) - (reject (ex/error :type :internal - :code :worker-not-ready - :hint "render worker did not become ready"))) - ready-timeout-ms)] - - (unchecked-set worker "__id" id) - (unchecked-set worker "__alive" true) - - (.on ^js worker "error" - (fn [cause] - (l/error :hint "render worker error" :worker-id id :cause cause) - (unchecked-set worker "__alive" false) - ;; A worker that dies while booting has to fail its own creation; - ;; rejecting after `resolve` is a no-op, so this is safe for the - ;; errors that arrive once it is already in the pool. - (js/clearTimeout timer) - (reject cause))) - - (.on ^js worker "exit" - (fn [code] - (l/info :hint "render worker exited" :worker-id id :code code) - (unchecked-set worker "__alive" false))) - - ;; Not `.once`: a stray message before the handshake would consume the - ;; listener and leave the worker hanging until `ready-timeout-ms`. - (letfn [(on-ready [data] - (when (= "ready" (unchecked-get data "type")) - (js/clearTimeout timer) - (.off ^js worker "message" on-ready) - (l/info :origin "factory" :action "create" :worker-id id) - (resolve worker)))] - (.on ^js worker "message" on-ready)))))) - -(def ^:private worker-pool-factory - #js {:create create-worker - :destroy (fn [worker] - (l/info :origin "factory" :action "destroy" - :worker-id (unchecked-get worker "__id")) - (.terminate ^js worker)) - :validate (fn [worker] - (p/resolved (true? (unchecked-get worker "__alive"))))}) - -(defn capacity - "How many renders can run at once, and so how many headless jobs the - scheduler may admit." - [] - ;; Clamped rather than rejected: a bad value should not stop the exporter - ;; from booting, and a headless render has no other backend to fall back to. - (max 1 (cf/get :wasm-worker-pool-max 2))) - -(defn init - [] - (let [configured (cf/get :wasm-worker-pool-max 2) - max-workers (capacity) - opts #js {:max max-workers - :min (min max-workers (cf/get :wasm-worker-pool-min 1)) - :testOnBorrow true - :evictionRunIntervalMillis 30000 - :numTestsPerEvictionRun 2 - :idleTimeoutMillis 300000}] - (when (not= configured max-workers) - (l/warn :hint "wasm-worker-pool-max raised to the minimum of one" - :configured configured)) - (l/info :hint "initializing render worker pool" :opts opts) - (reset! pool (gp/createPool worker-pool-factory opts)) - (p/resolved nil))) - -(defn stop - [] - (when-let [instance @pool] - (l/info :hint "finalizing render worker pool") - (reset! pool nil) - (p/do - (.drain ^js instance) - (.clear ^js instance)))) - -(defn- run-on-worker - "Settles when the worker reports the render finished, failed, or the thread - went away. That last case matters: a terminated worker (how a cancel stops a - render mid-Skia) emits `exit` and never `error`, and a promise left pending - there would keep its pool slot borrowed for the life of the process." - [^js worker params cancel-buffer on-object] - (p/create - (fn [resolve reject] - (let [timer (volatile! nil)] - (letfn [(disarm [] - (when-let [t @timer] - (js/clearTimeout t) - (vreset! timer nil))) - - (rearm [] - (disarm) - (vreset! timer (js/setTimeout - (fn [] - (l/error :hint "render worker went silent, terminating" - :worker-id (unchecked-get worker "__id")) - (cleanup) - ;; Terminating is what frees the pool slot: - ;; the `exit` it raises has no listener left. - (unchecked-set worker "__alive" false) - (.terminate ^js worker) - (reject (ex/error :type :internal - :code :render-timeout - :hint "render worker stopped responding"))) - (idle-timeout-ms)))) - - (cleanup [] - (disarm) - (.off worker "message" on-message) - (.off worker "error" on-error) - (.off worker "exit" on-exit)) - - (on-error [cause] - (cleanup) - (reject cause)) - - (on-exit [code] - (cleanup) - (reject (ex/error :type :internal - :code :worker-exited - :hint (str "render worker exited with code " code)))) - - (on-message [data] - (rearm) - (case (unchecked-get data "type") - ;; A failure while the main thread handles the object (moving - ;; the file, appending to the zip) has to end the render too, - ;; or nothing ever settles this promise. - "object" (try - (on-object (t/decode-str (unchecked-get data "payload"))) - (catch :default cause - (cleanup) - (reject cause))) - "done" (do (cleanup) (resolve nil)) - "error" (do (cleanup) - (reject (ex/error :type :internal - :code (or (some-> (unchecked-get data "code") keyword) - :wasm-render-error) - :hint (unchecked-get data "message")))) - nil))] - - (.on worker "message" on-message) - (.once worker "error" on-error) - (.once worker "exit" on-exit) - (rearm) - (.postMessage worker #js {:type "render" - :params (t/encode-str params) - :cancel cancel-buffer})))))) - -(defn with-worker - "Acquires one worker for the whole of `f`, a fn of that worker." - [f] - (let [instance @pool] - (->> (p/do (.acquire ^js instance)) - (p/mcat (fn [worker] - (->> (p/do (f worker)) - (p/fmap (fn [result] - (.release ^js instance worker) - result)) - (p/merr (fn [cause] - ;; The module may be aborted or mid-write, and - ;; a terminated worker cannot be reused. - (-> (p/do (.destroy ^js instance worker)) - (p/handle (fn [_ _] (p/rejected cause)))))))))))) - -(defn render-on - "Renders `params` on an already acquired worker." - [worker params on-object {:keys [cancel-buffer cancelled?]}] - (if (and cancelled? (cancelled?)) - (p/rejected (ex/error :type :internal - :code :job-cancelled - :hint "export job was cancelled")) - (run-on-worker worker params cancel-buffer on-object))) - -(defn terminate! - [^js worker] - (when worker - (unchecked-set worker "__alive" false) - (.terminate worker))) diff --git a/exporter/src/app/wasm/render.cljs b/exporter/src/app/wasm/render.cljs deleted file mode 100644 index 7889a6e3c5..0000000000 --- a/exporter/src/app/wasm/render.cljs +++ /dev/null @@ -1,502 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.wasm.render - "Headless render pipeline: renders exports with the render-wasm Skia pipeline, - with no browser and no WebGL. - - Per request: fetch scene (get-page RPC) -> serialize -> provision fonts and - images -> relayout text with the real fonts -> render each object. - - This runs inside a render worker (`app.wasm.worker`), one WASM design state - per worker, so the synchronous Skia calls never block the process that serves - HTTP. `app.renderer.wasm` is the main-thread side that drives it. - - Moved here verbatim from `app.renderer.wasm`; git reads it as a new file only - because that namespace still exists as the proxy. Reviewable as a rename: - `git show :exporter/src/app/renderer/wasm.cljs | diff -u - `. - - Handles png/jpeg/webp (Skia encodes all three), pdf and svg." - (:require - ["node:fs" :as fs] - ["undici" :as http] - [app.common.data :as d] - [app.common.exceptions :as ex] - [app.common.fonts :as cfnt] - ;; Required for side effects: these register the transit read handlers and - ;; deftype impls the `get-page` response is decoded into. - [app.common.geom.matrix] - [app.common.geom.point] - [app.common.geom.rect] - [app.common.logging :as l] - [app.common.transit :as t] - [app.common.types.fills.impl] - [app.common.types.objects-map] - [app.common.types.path.impl] - [app.common.types.shape] - [app.common.types.shape.images :as images] - [app.common.uri :as u] - [app.common.uuid :as uuid] - [app.config :as cf] - [app.util.mime :as mime] - [app.util.shell :as sh] - [app.wasm :as wasm] - [app.wasm.serialize :as serialize] - [cuerdas.core :as str] - [promesa.core :as p])) - -;; --- module lifecycle (one shared, lazily-initialized instance) - -(defonce ^:private module* (atom nil)) - -(defn- ensure-module! - [] - (or @module* - (reset! module* (wasm/init!)))) - -;; --- backend endpoints -;; -;; Every fetch targets the internal endpoint (falling back to public-uri), -;; in a deployment the exporter reaches the backend over the container network - -(defn- internal-uri - "Absolute URI for `path` on the internal (backend) endpoint." - [path] - (-> (cf/get-internal-uri) - (u/ensure-path-slash) - (u/join path) - (str))) - -(defn- public-uri - "Absolute URI for `path` on the public endpoint. Whoever opens an exported SVG - resolves its `@font-face` sources, so those cannot use the internal endpoint." - [path] - (-> (cf/get :public-uri) - (u/ensure-path-slash) - (u/join path) - (str))) - -(defn- error-detail - "Node's fetch reports every transport failure as a bare `TypeError: fetch - failed`; the actual reason (TLS rejection, DNS, ECONNREFUSED) is buried in a - nested `cause` chain that the logger does not print. Flattens the chain into - one readable string." - [cause] - (->> (iterate (fn [^js e] (unchecked-get e "cause")) cause) - (take-while some?) - (take 5) - (map (fn [^js e] - (let [code (unchecked-get e "code") - msg (or (unchecked-get e "message") (str e))] - (if code (str code ": " msg) msg)))) - (str/join " <- "))) - -(defn- fetch! - "`undici/fetch` that fails with an ex-info carrying the target uri and the - unwrapped cause chain, so a failed request says what actually went wrong and - against which endpoint." - [uri opts] - (->> (p/do (http/fetch uri opts)) - (p/merr (fn [cause] - (p/rejected (ex-info "http fetch failed" - {:uri uri :detail (error-detail cause)} - cause)))))) - -(defn- explain - "Log-friendly reason for `cause`: the detail `fetch!` already attached, or a - freshly unwrapped chain for anything else (WASM aborts, decode errors)." - [cause] - (or (:detail (ex-data cause)) - (error-detail cause))) - -(defn- rpc-headers - "Auth headers for backend RPC calls (management key + bearer)." - [token] - #js {"Content-Type" "application/transit+json" - "X-Shared-Key" (str "exporter " cf/management-key) - "Authorization" (str "Bearer " token)}) - -(defn- asset-headers - "Auth headers for `/assets/*`. Cookie, not Bearer: those endpoints redirect to - a presigned S3/minio URL, and a Bearer header makes S3 400 (\"multiple - authentication types\")." - [token] - #js {"X-Shared-Key" (str "exporter " cf/management-key) - "Cookie" (str "auth-token=" token)}) - -;; --- shape bundle fetch (backend RPC) - -(defn- fetch-objects - "Fetches the exported roots and their children from the backend via the - `get-page` RPC (`:object-id`, as the browser render path does), using the - same auth the exporter uses elsewhere (management key + bearer)." - [{:keys [file-id page-id share-id token objects]}] - (let [headers (rpc-headers token) - root-ids (into #{} (map :id) objects) - body (t/encode-str (cond-> {:file-id file-id - :page-id page-id} - (seq root-ids) (assoc :object-id root-ids) - share-id (assoc :share-id share-id))) - uri (internal-uri "api/rpc/command/get-page")] - (l/dbg :hint "wasm render: get-page" - :uri uri - :file-id (str file-id) - :page-id (str page-id) - :roots (count root-ids)) - (->> (fetch! uri #js {:method "POST" :headers headers :body body}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.text resp) - (->> (.text resp) - (p/mcat (fn [resp-body] - (l/error :hint "wasm render: get-page failed" - :uri uri - :status (.-status resp) - :body resp-body) - (p/rejected (ex-info "get-page failed" - {:status (.-status resp) - :body resp-body})))))))) - (p/fmap t/decode-str) - (p/fmap :objects)))) - -;; --- font resolution -;; -;; The text serializer keeps each font's real uuid, so `wasm/fonts-for-shape` -;; reports it. Custom (team) fonts resolve through the file's font variants, -;; google fonts through the shared `app.common.fonts` catalog; builtin -;; fonts through its bundled family + the frontend's static `/fonts/`. - -(defn- fetch-font-variants - "Team (custom) font variants for the file, or nil — a failure here degrades - to fallback fonts, it does not fail the export." - [{:keys [file-id share-id token]}] - (let [headers (rpc-headers token) - body (t/encode-str (cond-> {:file-id file-id} - share-id (assoc :share-id share-id))) - uri (internal-uri "api/rpc/command/get-font-variants")] - (->> (fetch! uri #js {:method "POST" :headers headers :body body}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.text resp) - (p/resolved nil)))) - (p/fmap (fn [s] (when s (t/decode-str s)))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: get-font-variants failed" - :uri uri :detail (explain cause) :cause cause) - (p/resolved nil)))))) - -(defn- fetch-ttf-bytes - "Downloads a TTF, returning a promise of an ArrayBuffer (or nil). A failure - here degrades to fallback fonts, it does not fail the export." - ([uri] (fetch-ttf-bytes uri #js {:method "GET"})) - ([uri opts] - (->> (fetch! uri opts) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.arrayBuffer resp) - (p/resolved nil)))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: font fetch failed" - :uri uri :detail (explain cause) :cause cause) - (p/resolved nil)))))) - -;; TTF bytes cached for the process lifetime, keyed by whatever identifies the -;; variant (a gfont id+weight+style, a builtin file name). -(defonce ^:private font-bytes* (atom {})) - -(defn- cached-ttf-bytes - [cache-key fetch-fn] - (if-let [bytes (get @font-bytes* cache-key)] - (p/resolved bytes) - (->> (fetch-fn) - (p/fmap (fn [buf] - (when buf (swap! font-bytes* assoc cache-key buf)) - buf))))) - -(defn- fetch-asset-bytes - [asset-id {:keys [token]}] - (fetch-ttf-bytes (internal-uri (str "assets/by-id/" asset-id)) - #js {:method "GET" :headers (asset-headers token)})) - -(defn- fetch-gfont-bytes - [ttf-url] - (fetch-ttf-bytes (cfnt/gstatic->proxy-url ttf-url (internal-uri "internal/gfonts/font")))) - -(defn- fetch-builtin-font-bytes - [ttf-file] - (cached-ttf-bytes ttf-file #(fetch-ttf-bytes (internal-uri (str "fonts/" ttf-file))))) - -(defn- family-uuid - [id] - (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3))) - -(defn- find-variant - "Custom variant for a family: uuid+weight+style first, degrading to - uuid+weight then uuid." - [variants font-uuid weight style] - (let [style-str (if (zero? style) "normal" "italic")] - (or (d/seek (fn [v] (and (= (:font-id v) font-uuid) - (= (:font-weight v) weight) - (= (name (:font-style v)) style-str))) - variants) - (d/seek (fn [v] (and (= (:font-id v) font-uuid) - (= (:font-weight v) weight))) - variants) - (d/seek (fn [v] (= (:font-id v) font-uuid)) variants)))) - -(defn- make-resolve-font - "Builds a `resolve-font` fn (family map -> promise of TTF bytes). Custom - variants first; the bundled fonts for `uuid/zero`, which is what - `font-id->uuid` maps every builtin family to; google catalog otherwise." - [variants params] - (fn [{:keys [id weight style]}] - (let [font-uuid (family-uuid id) - variant (find-variant variants font-uuid weight style)] - (cond - (:ttf-file-id variant) - (fetch-asset-bytes (:ttf-file-id variant) params) - - (= uuid/zero font-uuid) - (fetch-builtin-font-bytes (cfnt/resolve-ttf-file weight style)) - - :else - (if-let [gurl (cfnt/resolve-ttf-url font-uuid weight style)] - (fetch-gfont-bytes gurl) - (p/resolved nil)))))) - -(defn- make-font-url - "Builds a `font-url` fn (family map -> public URL of its TTF), the same - sources `make-resolve-font` downloads from but addressed publicly. The SVG - export emits one `@font-face` per family from these." - [variants] - (fn [{:keys [id weight style]}] - (let [font-uuid (family-uuid id) - variant (find-variant variants font-uuid weight style)] - (cond - (:ttf-file-id variant) - (public-uri (str "assets/by-id/" (:ttf-file-id variant))) - - (= uuid/zero font-uuid) - (public-uri (str "fonts/" (cfnt/resolve-ttf-file weight style))) - - :else - (some-> (cfnt/resolve-ttf-url font-uuid weight style) - (cfnt/gstatic->proxy-url (public-uri "internal/gfonts/font"))))))) - -;; --- fallback fonts (emoji + per-script noto fonts) -;; -;; Emoji and non-latin scripts render through fallback families, not through -;; any span's font family, so `wasm/fonts-for-shape` never reports them and the -;; provisioning above never uploads them. Must run per request, since -;; `clear-fonts!` empties the store; the TTF bytes stay cached per process. - -(defn- scene-fallback-fonts - "Fallback font descriptors needed by the scene's text. Deduped because - several languages map to one noto family and provisioning is concurrent — - otherwise they all miss the byte cache at once and refetch the same TTF." - [scene] - (let [texts (for [shape (vals scene) - :when (= :text (:type shape)) - node (or (some->> (:content shape) (tree-seq :children :children)) []) - :let [text (:text node)] - :when (string? text)] - text) - emoji? (boolean (some cfnt/contains-emoji? texts)) - langs (reduce cfnt/collect-used-languages #{} texts)] - (distinct - (cond-> (cfnt/add-noto-fonts [] langs) - emoji? (cfnt/add-emoji-font))))) - -(defn- fetch-fallback-font-bytes - "Downloads one fallback font's TTF. Cached by the whole variant, not just - `font-id`: `resolve-ttf-url` picks a different TTF per weight/style, so a - font-id-only key would serve the first downloaded variant for every other one." - [{:keys [font-id weight style]}] - (if-let [ttf-url (some-> (cfnt/gfont-id->uuid font-id) (cfnt/resolve-ttf-url weight style))] - (cached-ttf-bytes [font-id weight style] #(fetch-gfont-bytes ttf-url)) - (p/resolved nil))) - -(defn- provision-fallback-fonts! - [scene] - (->> (scene-fallback-fonts scene) - (map (fn [{:keys [font-id weight style is-emoji is-fallback] :as font}] - (if-let [font-uuid (cfnt/gfont-id->uuid font-id)] - (->> (fetch-fallback-font-bytes font) - (p/fmap (fn [buf] - (if buf - (wasm/store-font! {:id (uuid/get-u32 font-uuid) - :weight weight - :style style - :emoji? (boolean is-emoji) - :fallback? (boolean is-fallback)} - buf) - (l/warn :hint "wasm render: fallback font unavailable" - :font-id font-id))))) - (p/resolved nil)))) - (p/all))) - -;; --- image resolution -;; -;; Image fills reference file-media ids; the encoded bytes go straight to -;; `_store_image` (Skia decodes, no WebGL), keyed by media uuid so this happens -;; once per request rather than per rendered object. - -(defn- fetch-file-media-bytes - "Downloads an image fill's encoded bytes by file-media id." - [media-id {:keys [token]}] - (let [headers (asset-headers token) - uri (internal-uri (str "assets/by-file-media-id/" media-id))] - (->> (fetch! uri #js {:method "GET" :headers headers}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.arrayBuffer resp) - (do - (l/warn :hint "wasm render: image fetch non-200" - :media-id (str media-id) - :uri uri - :status (.-status resp)) - (p/resolved nil))))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: image fetch failed" - :media-id (str media-id) :uri uri - :detail (explain cause) :cause cause) - (p/resolved nil)))))) - -(defn- provision-images! - "Fetches and stores every image the scene references (shape, stroke and - text-span fills, enumerated by `app.common.types.shape.images`). Unlike fonts, - the image store is not reset per request, so already-held images are skipped - and repeated exports of a file reuse them. - - Always registers a public media URL for each id so SVG export can emit linked - `` even when the encoded bytes were already cached." - [scene params] - (let [all-ids (images/scene-image-ids scene) - new-ids (remove wasm/image-cached? all-ids)] - (l/dbg :hint "wasm render: provisioning images" - :total (count all-ids) - :cached (- (count all-ids) (count new-ids))) - (doseq [image-id all-ids] - (wasm/store-image-url! image-id (public-uri (str "assets/by-file-media-id/" image-id)))) - (->> new-ids - (map (fn [image-id] - (->> (fetch-file-media-bytes image-id params) - (p/fmap (fn [buf] - (if buf - (do - (l/dbg :hint "wasm render: image stored" - :media-id (str image-id) - :bytes (.-byteLength ^js buf)) - (wasm/store-image! image-id buf)) - (l/warn :hint "wasm render: image unavailable" - :media-id (str image-id)))))))) - (p/all)))) - -(defn- relayout-text! - "Recomputes layout for every text shape, once the real fonts are provisioned - (serialize-time layout used the fallback)." - [scene] - (doseq [shape (vals scene) - :when (= :text (:type shape))] - (wasm/update-text-layout! (:id shape)))) - -;; --- render - -(defn- check-cancelled! - "Cancellation is cooperative: a render already inside Skia cannot be - interrupted, so the flag is only observed between objects. Killing a job - mid-object is the caller's job (terminating the worker)." - [{:keys [cancelled?] :as _params}] - (when (and cancelled? (cancelled?)) - (ex/raise :type :internal - :code :job-cancelled - :hint "export job was cancelled"))) - -(defn- render-object-bytes - [type id scale] - (case type - :pdf (let [bytes (wasm/render-shape-pdf id)] - (l/dbg :hint "PDF generated via Skia (render-wasm headless)" - :object-id (str id) - :backend "skia-wasm" - :bytes (.-length bytes)) - bytes) - :svg (let [bytes (wasm/render-shape-svg id scale)] - (l/dbg :hint "SVG generated via Skia (render-wasm headless)" - :object-id (str id) - :backend "skia-wasm" - :bytes (.-length bytes)) - bytes) - (wasm/render-shape-raster id scale type))) - -(defn- render* - [{:keys [scale type objects] :as params} on-object] - (l/dbg :hint "wasm render: start" - :type type - :scale scale - :objects (count objects) - :file-id (str (:file-id params)) - :page-id (str (:page-id params))) - (->> (ensure-module!) - (p/mcat (fn [_] (fetch-objects params))) - (p/mcat (fn [scene] - (l/dbg :hint "wasm render: scene fetched" :shapes (count scene)) - (serialize/serialize-scene! scene) - (l/dbg :hint "wasm render: scene serialized") - ;; So fonts from a previous request don't leak into this one. - (wasm/clear-fonts!) - (->> (p/all [(fetch-font-variants params) - (provision-images! scene params) - (provision-fallback-fonts! scene)]) - (p/mcat - (fn [[variants _]] - (let [variants (or variants []) - resolve-font (make-resolve-font variants params) - font-url (make-font-url variants)] - ;; Before rendering, so the relayout below sees real - ;; font metrics. Deduped across objects: shapes - ;; sharing one family download its TTF once. - (wasm/provision-fonts! (map :id objects) resolve-font - :font-url font-url)))) - (p/mcat - (fn [_] - (relayout-text! scene) - (p/run - (fn [{:keys [id] :as object}] - (check-cancelled! params) - (let [bytes (render-object-bytes type id scale) - path (sh/tempfile :prefix "penpot.tmp.wasm." - :suffix (mime/get-extension type))] - (l/dbg :hint "wasm render: object rendered" - :object-id (str id) :bytes (.-length bytes)) - (fs/writeFileSync path bytes) - ;; `on-object` returns a plain value (zip append) or - ;; a promise (single export's file move); `p/do` - ;; normalizes both to a thenable. - (p/do (on-object (assoc object :path path))))) - objects)))))) - (p/fmap (fn [result] - ;; After the request, never mid-render, so an image can't - ;; disappear under a running export. - (let [evicted (wasm/evict-images! (cf/get :wasm-worker-image-cache-size wasm/image-cache-size))] - (when (pos? evicted) - (l/info :hint "wasm render: evicted cached images" :count evicted))) - result)) - (p/merr (fn [cause] - (l/error :hint "wasm render: failed" - :detail (explain cause) - :internal-uri (str (cf/get-internal-uri)) - :cause cause) - ;; A panic can leave the mem buffer allocated or the instance - ;; aborted; drop it so the next request rebuilds a fresh one. - (reset! module* nil) - (p/rejected cause))))) - -(defn render - "Public entry. Renders every object of `params`, calling `on-object` with - `{:id :filename :path ...}` as each one is written out." - [params on-object] - (render* params on-object)) diff --git a/exporter/src/app/wasm/worker.cljs b/exporter/src/app/wasm/worker.cljs deleted file mode 100644 index 5e5aeec5fc..0000000000 --- a/exporter/src/app/wasm/worker.cljs +++ /dev/null @@ -1,71 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.wasm.worker - "Render worker entry point. - - Owns one render-wasm module and renders one export request at a time. The - Skia calls are synchronous, so running them here is what lets several exports - progress at once: the main thread keeps serving HTTP, zipping and uploading - while this thread is blocked inside a render. - - Messages in: {type: \"render\", params: , cancel: SharedArrayBuffer} - Messages out: {type: \"ready\"} - {type: \"object\", payload: } one per rendered object - {type: \"done\"} | {type: \"error\", message, code}" - (:require - ["node:worker_threads" :as wt] - [app.common.logging :as l] - [app.common.transit :as t] - [app.wasm.render :as render] - [promesa.core :as p])) - -(defn- post! - [message] - (.postMessage ^js wt/parentPort message)) - -(defn- cancelled-fn - [buffer] - (if (some? buffer) - (let [signal (js/Int32Array. buffer)] - (fn [] (pos? (js/Atomics.load signal 0)))) - (constantly false))) - -(defn- handle-render - [data] - (let [params (-> (unchecked-get data "params") - (t/decode-str) - (assoc :cancelled? (cancelled-fn (unchecked-get data "cancel"))))] - (->> (render/render params - (fn [object] - (post! #js {:type "object" :payload (t/encode-str object)}))) - (p/fmap (fn [_] (post! #js {:type "done"}))) - (p/merr (fn [cause] - (l/warn :hint "render worker: request failed" :cause cause) - (post! #js {:type "error" - :message (or (ex-message cause) (str cause)) - :code (some-> cause ex-data :code name)}) - (p/resolved nil)))))) - -(defn- on-message - [data] - (case (unchecked-get data "type") - "render" (handle-render data) - (l/warn :hint "render worker: unknown message" :type (unchecked-get data "type")))) - -(defonce ^:private listening - ;; `defonce` survives a hot reload, so a reload does not stack a second - ;; listener on the port. The indirection through the var keeps the reloaded - ;; `on-message` in play instead of pinning the one captured at boot. - (delay - (.on ^js wt/parentPort "message" (fn [data] (on-message data))) - true)) - -(defn main - [& _] - @listening - (post! #js {:type "ready"}) - (l/info :hint "render worker ready")) diff --git a/exporter/src/exporter/browser.cljs b/exporter/src/exporter/browser.cljs new file mode 100644 index 0000000000..49427c2552 --- /dev/null +++ b/exporter/src/exporter/browser.cljs @@ -0,0 +1,246 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.browser + "The browser pool service and the page API its pages speak. + + The pool is a system component (`:exporter.browser/pool`): `init-key` + builds a `generic-pool` of Chromium browsers, `halt-key` drains it. + Everything else here is the plain page API the renderers use; none of + it touches the pool, so none of it needs the system. + + There are no promesa chains in this namespace: Playwright and + `generic-pool` already speak native promises, and `await` covers the + rest." + (:refer-clojure :exclude [eval]) + (:require + ["generic-pool" :as gp] + ["generic-pool/lib/errors.js" :as gpe] + ["playwright" :as pw] + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [exporter.util.object :as obj] + [exporter.utils.system :as system])) + +(l/set-level! :trace) + +(def ^:private TimeoutError gpe/TimeoutError) + +;; --- PAGE API + +(def default-timeout 30000) +(def default-viewport-width 1920) +(def default-viewport-height 1080) +(def default-user-agent + (str "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/99.0.3729.169 Safari/537.36")) + +(defn create-cookies + [uri {:keys [name token] :or {name "auth-token"}}] + (let [domain (str (:host uri) + (when (:port uri) + (str ":" (:port uri))))] + #js [#js {:domain domain + :path "/" + :name name + :value token}])) + +(defn nav + ([page url] (nav page url nil)) + ([page url {:keys [wait-until timeout] :or {wait-until "networkidle" timeout 20000}}] + (.goto ^js page (str url) #js {:waitUntil wait-until :timeout timeout}))) + +(defn sleep + [page ms] + (.waitForTimeout ^js page ms)) + +(defn- ^:async wait-for-fonts-async + [page timeout] + (try + (await (.waitForFunction ^js page + "async () => { + if (!document.fonts) return true; + try { + await Promise.all(Array.from(document.fonts, (face) => face.load())); + } catch (e) {} + await document.fonts.ready; + return document.fonts.status === 'loaded'; + }" + nil + #js {:timeout timeout})) + (catch :default cause + (l/warn :hint "wait-for-fonts timed out; continuing anyway" + :cause (ex-message cause)) + nil))) + +(defn wait-for-fonts + "Wait until the browser has finished loading all fonts. + + Checking `document.fonts.status === 'loaded'` on its own is not enough: + the render page injects its `@font-face` rules asynchronously and the + browser only loads a face lazily, when some painted text actually uses + it. A face that is declared but not yet requested stays in the `unloaded` + state, which does NOT keep `document.fonts.status` at `loading`, so the + check can pass before any real font glyphs are available. The export is + then captured with a (usually wider) fallback font, and auto-width text + laid out with the real font metrics overflows its bounds and gets clipped. + + To avoid that we explicitly request every declared face and await + `document.fonts.ready` before checking the status." + ([page] (wait-for-fonts page nil)) + ([page {:keys [timeout] :or {timeout 15000}}] + (wait-for-fonts-async page timeout))) + +(defn wait-for + ([locator] (wait-for locator nil)) + ([locator {:keys [state timeout] :or {state "visible" timeout 10000}}] + (.waitFor ^js locator #js {:state state :timeout timeout}))) + +(defn screenshot + ([frame] (screenshot frame {})) + ([frame {:keys [full-page? omit-background? type quality path] + :or {type "png" full-page? false omit-background? false quality 95}}] + (let [options (-> (obj/new) + (obj/set! "type" (name type)) + (obj/set! "omitBackground" omit-background?) + (cond-> path (obj/set! "path" path)) + (cond-> (= "jpeg" type) (obj/set! "quality" quality)) + (cond-> full-page? (-> (obj/set! "fullPage" true) + (obj/set! "clip" nil))))] + (.screenshot ^js frame options)))) + +(defn emulate-media + [page {:keys [media]}] + (.emulateMedia ^js page #js {:media media}) + page) + +(defn pdf + ([page] (pdf page {})) + ([page {:keys [scale path page-ranges] + :or {page-ranges "1" + scale 1}}] + (.pdf ^js page #js {:path path + :scale scale + :pageRanges page-ranges + :printBackground true + :preferCSSPageSize true}))) + +(defn eval + [frame f] + (.evaluate ^js frame f)) + +(defn select + [frame selector] + (.locator ^js frame selector)) + +(defn select-all + [frame selector] + (.$$ ^js frame selector)) + +;; --- POOL SERVICE + +(def defaults + "Static pool defaults. The sizing (`:max`) mirrors the worker + concurrency default; the wiring layer passes the env-derived value." + {:max 2 + :acquire-timeout 10000 + :idle-timeout 10000 + :launch-args ["--allow-insecure-localhost" "--font-render-hinting=none"]}) + +(defn- default-create-browser + [launch-args] + (fn [] + (.launch pw/chromium #js {:args (clj->js launch-args)}))) + +(defn- ^:async create-instance + [create-browser id-counter] + (let [browser (await (create-browser)) + id (swap! id-counter inc)] + (l/info :origin "factory" :action "create" :browser-id id) + (unchecked-set browser "__id" id) + browser)) + +(defn- destroy-instance + [browser] + (l/info :origin "factory" :action "destroy" + :browser-id (unchecked-get browser "__id")) + (.close ^js browser)) + +(defn- validate-instance + [browser] + (l/info :origin "factory" :action "validate" + :browser-id (unchecked-get browser "__id")) + (.isConnected ^js browser)) + +(defn- make-factory + [create-browser] + (let [id-counter (atom 0)] + #js {:create (fn [] (create-instance create-browser id-counter)) + :destroy destroy-instance + :validate validate-instance})) + +(defmethod system/init-key ::pool + [_ cfg] + (let [opts (merge defaults (d/without-nils cfg)) + create-browser (or (:create-browser opts) + (default-create-browser (:launch-args opts)))] + (l/info :hint "initializing browser pool" :max (:max opts)) + (gp/createPool (make-factory create-browser) + #js {:max (:max opts) + :min 0 + :testOnBorrow true + :evictionRunIntervalMillis 5000 + :numTestsPerEvictionRun 5 + :acquireTimeoutMillis (:acquire-timeout opts) + :idleTimeoutMillis (:idle-timeout opts)}))) + +(defn- ^:async drain-pool + [pool] + (await (.drain ^js pool)) + (await (.clear ^js pool)) + nil) + +(defmethod system/halt-key ::pool + [_ pool] + (when pool + (l/info :hint "finalizing browser pool") + (drain-pool pool))) + +(defn- throw-browser-error + [cause] + (if (instance? TimeoutError cause) + (ex/raise :type :internal + :code :timeout + :hint (ex-message cause) + :cause cause) + (throw cause))) + +(defn ^:async exec + "Check out a browser, run `handle` with a fresh page, and put the + browser back. `handle` takes the page and may return a plain value or + a promise. On failure the browser is destroyed instead of reused, and + a Playwright timeout surfaces as an `:timeout` error." + [pool context-opts handle] + (when-not pool + (throw (ex-info "browser pool is not started" {:reason ::not-started}))) + (let [browser (await (.acquire ^js pool))] + (try + (let [context (await (.newContext ^js browser context-opts)) + _ (l/trace :hint "exec:handle:start" :browser-id (unchecked-get browser "__id")) + page (await (.newPage ^js context)) + result (await (handle page))] + (await (.close ^js context)) + (l/trace :hint "exec:handle:end" :browser-id (unchecked-get browser "__id")) + (await (.release ^js pool browser)) + result) + (catch :default cause + (try + (await (.destroy ^js pool browser)) + (catch :default destroy-cause + (l/warn :hint "browser destroy failed" + :cause (ex-message destroy-cause)))) + (throw-browser-error cause))))) diff --git a/exporter/src/exporter/browser/scope.cljs b/exporter/src/exporter/browser/scope.cljs new file mode 100644 index 0000000000..de28d1ec3f --- /dev/null +++ b/exporter/src/exporter/browser/scope.cljs @@ -0,0 +1,26 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.browser.scope + "One batch through browsers: every export gets its own checkout, + all at once, bounded by the pool. No lease — a browser batch shares + nothing but the pool, so this level owns fan-out only, the mirror of + `exporter.wasm.scope` holding a lease instead. Task keys ride in + `cfg` under `:exporter.renderer/*`, set by whoever built the batch." + (:require + [exporter.renderer.browser :as driver])) + +(defn ^:async run + "Renders every export through its own browser checkout. Rejects on + the first failure; progress and cancel flow per export through the + task's own callbacks." + [cfg exports] + (let [on-object (:exporter.renderer/on-object cfg) + check-cancelled (:exporter.renderer/check-cancelled cfg)] + (await (js/Promise.all + (mapv (fn [params] + (driver/render cfg params on-object check-cancelled)) + exports))))) diff --git a/exporter/src/app/config.cljs b/exporter/src/exporter/config.cljs similarity index 52% rename from exporter/src/app/config.cljs rename to exporter/src/exporter/config.cljs index 4f94887ce0..c26ebef534 100644 --- a/exporter/src/app/config.cljs +++ b/exporter/src/exporter/config.cljs @@ -4,14 +4,23 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.config +(ns exporter.config + "The process configuration, read from the environment. + + Same shape as the legacy `app.config` it replaces: `PENPOT_`-prefixed + variables become kebab-case keys over the defaults, decoded and + validated against the schema. The one deliberate split: `prepare` + over an explicit env object is pure and throws on invalid input (so + tests exercise it), while only the wiring entry `load-config!` reads + the live environment and exits on it — the process never boots with + a config it cannot derive its management key from." (:refer-clojure :exclude [get]) (:require ["node:buffer" :as buffer] ["node:crypto" :as crypto] - ["node:os" :as os] ["node:process" :as process] [app.common.data :as d] + [app.common.exceptions :as ex] [app.common.flags :as flags] [app.common.logging :as l] [app.common.schema :as sm] @@ -25,16 +34,12 @@ {:public-uri "http://localhost:3449" ;; :internal-uri nil ;; internal-uri cannot be nil :tenant "default" - :host "localhost" - :http-server-port 6061 - :http-server-host "0.0.0.0" :tempdir "/tmp/penpot" :redis-uri "redis://redis/0" - :exporter-max-concurrent-jobs 4 - :exporter-max-jobs-per-profile 2 - :exporter-queue-max 64 :exporter-job-ttl 3600 - :wasm-worker-pool-max 2 + ;; how many export jobs run at once: every poller owns one Redis + ;; connection, and the pools size themselves to the same number + :exporter-worker-concurrency 2 :wasm-worker-pool-min 1 :wasm-worker-idle-timeout 300 :wasm-worker-image-cache-size (* 128 1024 1024)}) @@ -45,19 +50,12 @@ [:public-uri {:optional true} ::sm/uri] [:internal-uri {:optional true} ::sm/uri] [:exporter-shared-key {:optional true} :string] - [:host {:optional true} :string] [:tenant {:optional true} :string] [:flags {:optional true} [::sm/set :keyword]] [:redis-uri {:optional true} :string] [:tempdir {:optional true} :string] - [:browser-pool-max {:optional true} ::sm/int] - [:browser-pool-min {:optional true} ::sm/int] - [:exporter-max-concurrent-jobs {:optional true} ::sm/int] - [:exporter-max-jobs-per-profile {:optional true} ::sm/int] - [:exporter-queue-max {:optional true} ::sm/int] [:exporter-job-ttl {:optional true} ::sm/int] - [:exporter-replica-id {:optional true} :string] - [:wasm-worker-pool-max {:optional true} ::sm/int] + [:exporter-worker-concurrency {:optional true} ::sm/int] [:wasm-worker-pool-min {:optional true} ::sm/int] [:wasm-worker-idle-timeout {:optional true} ::sm/int] [:wasm-worker-image-cache-size {:optional true} ::sm/int]]) @@ -75,12 +73,13 @@ [config] (flags/parse (:flags config))) -(defn- read-env - [prefix] - (let [env (unchecked-get process "env") - kwd (fn [s] (-> (str/kebab s) (str/keyword))) - prefix (str prefix "_") - len (count prefix)] +(defn read-env + "The `PENPOT_`-prefixed variables of `env` as kebab-case keys, the + rest ignored. Takes the env object explicitly so tests pass a fake." + [env] + (let [prefix "penpot_" + len (count prefix) + kwd (fn [s] (-> (str/kebab s) (str/keyword)))] (reduce (fn [res key] (let [val (unchecked-get env key) key (str/lower key)] @@ -90,22 +89,32 @@ {} (js/Object.keys env)))) -(defn- prepare-config - [] - (let [env (read-env "penpot") - env (d/without-nils env) - data (merge defaults env) - data (decode-config data)] - +(defn prepare-config + "Defaults plus `env`, decoded and validated. Throws on invalid input + instead of exiting: only the wiring entry exits, on the live + environment." + [env] + (let [data (-> (merge defaults (d/without-nils (read-env env))) + (decode-config))] (when-not (valid-config? data) - (let [explain (explain-config data)] - (println (sm/humanize-explain explain)) - (process/exit -1))) - + (throw (ex/error :type :internal + :code :invalid-config + :hint (sm/humanize-explain (explain-config data))))) data)) +(defn load-config! + "The live environment as validated config. Exits the process on + invalid input: without a secret there is no management key to derive, + so there is nothing to boot." + [] + (try + (prepare-config (unchecked-get process "env")) + (catch :default cause + (println (ex-message cause)) + (process/exit -1)))) + (def config - (prepare-config)) + (load-config!)) (def version (v/parse "%version%")) @@ -126,17 +135,19 @@ (or (c/get config :internal-uri) (c/get config :public-uri))) -(def replica-id - "Names this exporter among its siblings. It must survive a restart, since - the boot sweep uses it to find the jobs the previous run left behind." - (or (c/get config :exporter-replica-id) - (os/hostname))) +(defn derive-management-key + "The management key for `secret` and `tenant`: HKDF blake2b512 over + \"exporter:\", 32 bytes, base64url. Pure so tests pin the + formula the backend derives on its side. The tenant rides in the + salt (and in the key id the client sends), so a key only ever opens + its own tenant's jobs." + [secret tenant] + (-> (.from buffer/Buffer (crypto/hkdfSync "blake2b512" secret (str "exporter:" tenant) "" 32)) + (.toString "base64url"))) (def management-key (let [key (or (c/get config :exporter-shared-key) - (let [secret-key (c/get config :secret-key) - derived-key (crypto/hkdfSync "blake2b512" secret-key, "exporter" "" 32)] - (-> (.from buffer/Buffer derived-key) - (.toString "base64url"))))] + (derive-management-key (c/get config :secret-key) + (c/get config :tenant)))] (l/inf :hint "exporter key initialized" :key (d/obfuscate-string key)) key)) diff --git a/exporter/src/exporter/consumer.cljs b/exporter/src/exporter/consumer.cljs new file mode 100644 index 0000000000..f036112d46 --- /dev/null +++ b/exporter/src/exporter/consumer.cljs @@ -0,0 +1,406 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.consumer + "The runner of the `:export-assets` job. + + Runs one export end to end: the render plan of + `exporter.consumer.plan` (the name transducers and the partitioning + the export surfaces always kept), the pdf assembly of the frames + run, the beats shaped by the progress contract (`:preparing`, + `:rendering`, `:packaging`, with the `objects` and `pages` counters + of every partition) and one settle: a `complete-job` multipart call + that stores the artifact and closes the session the backend minted + in the same step. The render runs as the job's owner through the + session `create-job-session` opened. The batch renders through the + injected render fn (`:exporter/renderer` once wired): this namespace + never calls the facade directly. + + Cancellation arrives through the beats: a `skip` answer from + `report-job-progress` says the job can no longer hear the worker + (cancelled, aborted, settled). Between units of work the runner + raises, with the same check the drivers run; inside a render the + lease holder aborts its own worker. The local registry entry is + released on every settle — unlike the legacy runner, which only + released the temp files and leaked one entry per job. + + The run never rejects on an export failure: whatever goes wrong + reaches the backend as `fail-job` (with the session id, so the render + session does not outlive the settle). Only bugs of this process + reject. There are no promesa chains in this namespace." + (:require + [app.common.exceptions :as ex] + [app.common.export :as export] + [app.common.logging :as l] + [app.common.schema :as sm] + [app.common.uuid :as uuid] + [cuerdas.core :as str] + [exporter.consumer.api :as api] + [exporter.consumer.plan :as plan] + [exporter.jobs :as jobs] + [exporter.resources :as rsc] + [exporter.shell :as shell] + [exporter.tmpdir :as tdir])) + +(def ^:private report-throttle-ms 250) +(def ^:private watchdog-interval-ms 1000) + +;; ---- PROGRESS AND CANCEL (the milestones of the contract) + +(defn enqueue-beat + "Chain `f` after the promise in `chain`, so only one beat is in flight + at a time: the watchdog and a unit beat would otherwise race and a + stale milestone could overwrite a newer one. A failed link does not + break the chain. Returns the promise of this link." + [chain f] + (let [prev @chain + next (-> (if (some? prev) + (.catch prev (fn [_] nil)) + (js/Promise.resolve)) + (.then (fn [_] (f))))] + (reset! chain next) + next)) + +(defn- start-beats + "The reporter of one run's beats. + + The beat reports the stage and the counters and asks the backend to + go on; a `skip` answer means nobody is listening anymore — the job + was cancelled or settled elsewhere — and the export must stop: the + reporter fires `on-cancel` once (what marks the local flag the + checks read; the in-flight render aborts itself) and returns the + answer, so the runner sees it at the next unit of work and raises. + + Between forced beats the reporter throttles the way the legacy export + did, since hundreds of objects would otherwise be hundreds of HTTP + calls for information nobody reads at that resolution. An errored + beat answers `run`: the export itself must not fail because the beat + could not fly, and the backend that is unreachable has the lease to + decide what happens to the row. The beat never rejects. + + Beats are serialized: the watchdog repeats the last report while a + unit beat may be in flight, and two POSTs arriving out of order would + let the older milestone win." + [job-id on-cancel] + (let [last-beat (atom 0) + last (atom nil) + fired (atom false) + chain (atom nil) + check (fn [answer] + (when (= :skip (:action answer)) + (when (compare-and-set! fired false true) + (l/info :hint "job cancelled by backend" + :job-id (str job-id)) + (on-cancel))) + answer) + report (^:async fn [stage counters] + (try + (check (await (api/report-job-progress job-id {:stage stage + :counters counters}))) + (catch :default cause + (l/warn :hint "beat failed to land" + :job-id (str job-id) + :cause cause) + {:action :run}))) + beat (^:async fn [stage counters & {:keys [force?]}] + (let [now (js/Date.now) + now? (or force? + (>= (- now @last-beat) report-throttle-ms))] + (if now? + (do (reset! last-beat now) + (reset! last {:stage stage :counters counters}) + (await (enqueue-beat chain #(report stage counters)))) + {:action :run :throttled? true})))] + {:beat beat :last last})) + +;; ---- THE WATCHDOG + +(defn- with-watchdog + "The forced beats of one run: one every `watchdog-interval-ms`, and + they repeat the last report the run made, so a cancellation that + lands mid-render (no object landing, no unit of work ending) is seen + within that interval. `stop` clears it once the run settles." + [beat last] + (let [timer (js/setInterval + #(when (some? @last) + (beat (:stage @last) (:counters @last) :force? true)) + watchdog-interval-ms)] + (fn stop [] (js/clearInterval timer)))) + +(defn- counter + "One counter of the vocabulary: `kind` is `objects` or `pages`." + [kind current total] + {kind {:current current :total total}}) + +;; ---- THE PARAMS + +(defn- ->uuid + "One id of the claim into what the renderer wants: the claim carries + the job row JSON, every id a string, while the render spec only + takes uuid objects. Values that already are uuids (tests, callers in + process) pass through." + [v] + (if (string? v) (uuid/uuid v) v)) + +(defn- normalize-items + "The items of one claim, decoded with the row's job-def: every id a + uuid object and the type a keyword. Plain JSON shapes (tests, edn + callers) still pass through, so string ids and type names are + tolerated here." + [items] + (->> items + (mapv (fn [item] + (cond-> item + (string? (:type item)) (update :type keyword) + (string? (:file-id item)) (update :file-id ->uuid) + (string? (:page-id item)) (update :page-id ->uuid) + (string? (:object-id item)) (update :object-id ->uuid) + (string? (:share-id item)) (update :share-id ->uuid)))))) + +(def ^:private check-items + "The normalized claim items against the shared contract: what the + backend froze, decoded. A mistyped backend fails here, at the + boundary, instead of three stages deep in the render." + (sm/check-fn export/schema:export-items :hint "invalid export items")) + +(defn- make-plan + "The render plan the legacy handlers prepare: the same transducers of + names and partition size they served the old surface with, the same + `single?` rule (one prepared export with one object, not forced), + and the artifact the whole run fills." + [job-id token tmpdir {:keys [exports kind force-multiple name skip-children is-wasm]}] + (let [items (check-items (normalize-items exports)) + ;; declared by the caller, never sniffed: typeless items do not + ;; exist on the validated contract, so there is nothing to infer + frames? (= :frames (keyword kind)) + prepared (plan/prepare-exports items token is-wasm) + single? (and (not frames?) + (not (true? force-multiple)) + (= 1 (count prepared)) + (= 1 (count (-> prepared first :objects)))) + artifact-kind (cond + single? (-> prepared first :type) + frames? :pdf + :else :zip) + ;; The artifact parks in the injected area, so it is tracked + ;; where it is born: single, multi and frames all settle + ;; through it, and the release wipes the job area once the + ;; settle uploaded it. + resource (rsc/create tmpdir + artifact-kind + (or name (-> prepared first :name)))] + (tdir/track job-id (:path resource)) + {:frames? frames? + :single? single? + :job-id (->uuid job-id) + :prepared prepared + :total (plan/count-objects prepared) + :skip-children skip-children + :is-wasm (boolean is-wasm) + :counter-kind (if frames? :pages :objects) + :resource resource})) + +;; ---- THE RUNNERS + +(defn- check-beat + "A forced beat after a unit of work: a `skip` there is a cancelled + job, the same exception the legacy surfaces raised, so everything + below unwinds." + [answer] + (when (= :skip (:action answer)) + (throw (ex/error :type :internal + :code :job-cancelled + :hint "export job was cancelled"))) + answer) + +(defn- ^:async run-single + "One render, one object: the artifact IS the object, moved into the + resource path the multipart settle will name." + [render beat plan check-cancelled] + (let [{:keys [job-id resource counter-kind]} plan + export (-> plan :prepared first) + object (atom nil)] + (check-beat (await (beat :rendering (counter counter-kind 0 1) :force? true))) + (await (render {:exports [(cond-> (assoc export + :job-id job-id) + (some? (:skip-children plan)) + (assoc :skip-children (:skip-children plan)))] + :on-object (fn [obj] + (reset! object obj) + (tdir/track job-id (:path obj)) + nil) + :check-cancelled check-cancelled})) + (await (shell/move (:path @object) (:path resource))) + (check-beat (await (beat :packaging (counter counter-kind 1 1) :force? true))) + resource)) + +(defn- ^:async run-multiple + "The multi-object render: every prepared export renders through the + one batch of the facade, appending each object to the zip as it + lands, and the progress counts what has landed so far. Any zipping + error surfaces after the renders, so the failures of the writer do + not compete with the ones of the render." + [render beat plan check-cancelled] + (let [{:keys [job-id resource prepared total counter-kind]} plan + failure (volatile! nil) + rendered (volatile! 0) + zip (rsc/create-zip :resource resource + :on-error (fn [cause] (vreset! failure cause)) + :on-progress (fn [_] nil)) + append (^:async fn [{:keys [filename path]}] + (check-cancelled) + (tdir/track job-id path) + (vswap! rendered inc) + (await (beat :rendering + (counter counter-kind @rendered total))) + (rsc/add-to-zip zip path + (str/replace filename + plan/sanitize-file-regex + "_")) + nil)] + (check-beat (await (beat :rendering (counter counter-kind 0 total) :force? true))) + (await (render {:exports (mapv #(cond-> (assoc % :job-id job-id) + (some? (:skip-children plan)) + (assoc :skip-children (:skip-children plan))) + prepared) + :on-object append + :check-cancelled check-cancelled})) + (when-let [cause @failure] + (throw cause)) + (await (rsc/close-zip zip)) + (check-beat (await (beat :packaging (counter counter-kind total total) :force? true))) + resource)) + +(defn- ^:async join-pdf + "The pages render one file per page; `pdfunite` stitches one pdf out + of them all, the way the legacy export did." + [job-id tmpdir file-id paths] + (let [path (tdir/track job-id + (shell/tempfile tmpdir + :prefix (str/concat "penpot.pdfunite." file-id ".") + :suffix ".pdf"))] + (await (shell/run-cmd "pdfunite" (into [] (concat (vec paths) [path])))) + path)) + +(defn- ^:async run-frames + "The frames render: a file per page, joined into the pdf of the file + once every page has landed." + [render tmpdir beat plan check-cancelled] + (let [{:keys [job-id resource prepared total]} plan + file-id (-> prepared first :file-id) + paths (volatile! []) + rendered (volatile! 0) + on-object (^:async fn [{:keys [path]}] + (check-cancelled) + (tdir/track job-id path) + (vswap! paths conj path) + (vswap! rendered inc) + (await (beat :rendering + (counter :pages @rendered total))) + nil)] + (check-beat (await (beat :rendering (counter :pages 0 total) :force? true))) + (await (render {:exports (mapv #(cond-> (assoc % :job-id job-id + :is-wasm (:is-wasm plan)) + (some? (:skip-children plan)) + (assoc :skip-children (:skip-children plan))) + prepared) + :on-object on-object + :check-cancelled check-cancelled})) + (let [joined (await (join-pdf job-id tmpdir file-id @paths))] + (await (shell/move joined (:path resource))) + (check-beat (await (beat :packaging (counter :pages total total) :force? true))) + resource))) + +(defn- run-prepared + [render tmpdir beat check-cancelled plan] + (cond + (:single? plan) (run-single render beat plan check-cancelled) + (:frames? plan) (run-frames render tmpdir beat plan check-cancelled) + :else (run-multiple render beat plan check-cancelled))) + +;; ---- THE SETTLE + +(defn- failure-report + "The error shape the backend stores for `job.error`: type and code + keywords, hint the human message. A cause that is not a penpot error + becomes an internal one, with only its message as the hint." + [cause] + (let [data (ex-data cause)] + {:type (or (:type data) :internal) + :code (or (:code data) :export-failed) + :hint (or (:hint data) (ex-message cause) "export failed")})) + +(defn cancelled? + "Whether `cause` ends a run the user asked to end: a cancelled + export is a normal ending, reported and logged as one, never as a + failure with a trace." + [cause] + (= :job-cancelled (:code (ex-data cause)))) + +(defn- ^:async settle-failure + "Settles the job as failed, after releasing the temp files the run + owns. The settle never rejects: if the failure cannot reach the + backend, the row keeps running until the lease GC turns it `aborted`, + which is the honest state for a worker that cannot talk." + [job-id session cause] + (await (tdir/release job-id)) + (jobs/release job-id) + (try + (await (api/fail-job job-id (failure-report cause) + :session-id (some-> @session :session-id))) + (catch :default settle-cause + (l/warn :hint "unable to settle the failed job" + :job-id job-id :cause settle-cause)))) + +(defn ^:async run-export + "Runs one claimed export to its settle: first breath, render + session, plan, render batch, multipart complete. `render` is the + injected render fn (one task map per batch: `:exports`, `:on-object`, + `:check-cancelled`) plus `tmpdir`, the directory every temp path is + created under; the claim names the job and `params` carries the + frozen job params." + [render tmpdir {:keys [job-id]} params] + (let [session (atom nil) + stop-watchdog (atom nil) + _ (jobs/register job-id) + stop-cmd (fn [] (jobs/mark-cancelled job-id)) + {:keys [beat last]} (start-beats job-id stop-cmd) + check-cancelled (jobs/check-cancelled job-id)] + (try + (reset! stop-watchdog (with-watchdog beat last)) + ;; the first breath right after the claim, before the + ;; session roundtrip: the widget moves from queued to + ;; exporting on it, and a cancel that landed meanwhile + ;; raises here instead of after wasted work + (check-beat (await (beat :preparing {} :force? true))) + (let [session' (await (api/create-job-session job-id))] + (reset! session session') + (l/info :hint "render session minted" + :job-id (str job-id)) + (let [plan (make-plan job-id (:session-token session') tmpdir params) + resource (await (run-prepared render tmpdir beat check-cancelled plan))] + ;; the watchdog outlives the render, so the run stops it on + ;; the way out: a run that ends badly stops beating too, or + ;; its interval would knock on a dead job forever, one HTTP + ;; call a second that only ever answers skip + (@stop-watchdog) + (let [artifact {:path (str (:path resource)) + :filename (:filename resource) + :mtype (:mtype resource)} + answer (await (api/complete-job-with-artifact + {:job-id job-id :session-id (:session-id session')} + artifact))] + (await (tdir/release job-id)) + (jobs/release job-id) + (l/info :hint "export job settled" :job-id (str job-id) + :outcome (some-> answer :action name)) + nil))) + (catch :default cause + (when-let [stop @stop-watchdog] (stop)) + (if (cancelled? cause) + (l/info :hint "export job cancelled" :job-id (str job-id)) + (l/error :hint "export job failed" :job-id (str job-id) :cause cause)) + (await (settle-failure job-id session cause)))))) diff --git a/exporter/src/exporter/consumer/api.cljs b/exporter/src/exporter/consumer/api.cljs new file mode 100644 index 0000000000..0d467b9673 --- /dev/null +++ b/exporter/src/exporter/consumer/api.cljs @@ -0,0 +1,173 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.consumer.api + "The backend management API client. + + The only place of the worker that speaks HTTP: every method of the + backend ledger (claim, progress, session, settle) travels through + here, with the shared key the exporter was configured with and, when + a call renders as a user, the Bearer token of its session. Request + bodies travel as transit+json (except the artifact form of + `complete-job`, which is multipart, like every other upload) and the + answers come back as transit too. There are no promesa chains in + this namespace." + (:require + ["node:fs" :as fs] + ["undici" :as http] + [app.common.exceptions :as ex] + [app.common.transit :as t] + [app.common.uri :as u] + [cljs.core :as c] + [exporter.config :as cf])) + +(def fetch + "The network entry point of the client: a plain def, redefined by the + tests of the client, which never need a network." + http/fetch) + +;; ---- THE REQUEST + +;; ---- THE CONNECTION + +(def ^:private call-timeout-ms + "A management call that has not answered by then is treated as failed: + the beat path already knows how to carry on without it." + 10000) + +(def ^:private upload-timeout-ms + "The artifact upload carries the whole render, so it gets room." + 120000) + +(def ^:private agent* + "The single undici Agent of the process: keep-alive and TLS in one + place. Created on first use so requiring this namespace never opens a + socket." + (delay + (new http/Agent #js {:connect #js {:rejectUnauthorized false}}))) + +(defn close-client! + "Release the connections of the process. Called from the shutdown path." + [] + (when (realized? agent*) + (.close ^js @agent*))) + +(defn- dispatcher + "The agent of every request. Certificate verification is off on the + management calls (shared key plus Bearer tokens ride each beat), the + same way the existing uploads already accept self-signed certificates. + Conscious posture for first-party deployers with self-signed TLS; + revisit if the threat model changes." + [] + @agent*) + +(defn- method-uri + [method] + (-> (cf/get-internal-uri) + (u/ensure-path-slash) + (u/join (str "api/management/methods/" (c/name method))) + (str))) + +(defn- shared-key + "What every management call carries: the key id of this exporter plus + the key, and the wish to read transit back. The id names the tenant + this worker renders for (`exporter:`), so the backend only + lets it touch its own jobs." + [] + #js {"X-Shared-Key" (str "exporter:" (cf/get :tenant) " " cf/management-key) + "Accept" "application/transit+json"}) + +(defn- ^:async parse-response + "The answer of a management call: transit when it lands. A non-200 is + the failure of the call, with the error body the backend answered; + a body that does not decode stays as the plain text it came as." + [response] + (let [status (.-status response) + text (await (.text response)) + body (let [decoded (ex/try! (t/decode-str text))] + (if-not (ex/exception? decoded) decoded text))] + (if (= status 200) + body + (throw (ex/error :type :internal + :code :failed-management-request + :status status + :response body))))) + +(defn- ^:async request + "POST `params` to a management method as transit+json. The content + type travels on every call: without it the backend never parses the + body as transit and validates an empty map. A call that does not + answer in time is aborted and rejects like any other failure." + [method params] + (let [response (await (fetch (method-uri method) + #js {:method "POST" + :headers (doto (shared-key) + (unchecked-set "Content-Type" "application/transit+json")) + :body (t/encode-str params) + :signal (js/AbortSignal.timeout call-timeout-ms) + :dispatcher (dispatcher)}))] + (await (parse-response response)))) + +(defn- ^:async request-multipart + "POST a FormData body to a management method. The content type is + settled by the FormData itself (boundary included), so only the + shared key goes in the headers. The artifact carries the whole render, + so this call gets the longer timeout." + [method form-data] + (let [response (await (fetch (method-uri method) + #js {:method "POST" + :headers (shared-key) + :body form-data + :signal (js/AbortSignal.timeout upload-timeout-ms) + :dispatcher (dispatcher)}))] + (await (parse-response response)))) + +;; ---- THE METHODS + +(defn ^:async claim-job + "Request a queued job from the backend. The answer is either + `{:action :run :name ... :params ...}` (this poller owns the job) or + `{:action :skip ...}` (something else won it, or it is gone)." + [job-id scheduled-at] + (await (request :claim-job {:job-id job-id :scheduled-at scheduled-at}))) + +(defn ^:async report-job-progress + "The heartbeat of a running job: the stage hitos of the progress + contract plus the counters of the current work. A `:skip` says the + job can no longer hear the worker (cancelled, aborted, settled)." + [job-id progress] + (await (request :report-job-progress {:job-id job-id :progress progress}))) + +(defn ^:async create-job-session + "Mint an authentication session of the owner of a running job, so the + render views authenticate as the user, not as a machine." + [job-id] + (await (request :create-job-session {:job-id job-id}))) + +(defn ^:async complete-job-with-artifact + "Close a job whose result is the artifact itself: the backend stores + the bytes and derives the result descriptor. `artifact` says what it + is (`:path`, `:filename`, `:mtype`); `ctx` names the job and, when + the worker rendered with a session, the one to close." + [{:keys [job-id session-id]} {:keys [path filename mtype]}] + ;; The part is the single source of name and type: they travel in the + ;; part itself (name in the disposition, type in the Blob), never as + ;; separate text fields. + (let [blob (await (fs/openAsBlob (str path) #js {:type (str mtype)})) + fdata (new http/FormData)] + (.append fdata "job-id" (str job-id)) + (.append fdata "content" blob filename) + (when session-id + (.append fdata "session-id" (str session-id))) + (await (request-multipart :complete-job fdata)))) + +(defn ^:async fail-job + "Close a job as failed with the rich error the backend stores for + `job.error`. `:session-id` closes the render session the same way." + [job-id error & {:keys [session-id]}] + (await (request :fail-job + (cond-> {:job-id job-id :error error} + (some? session-id) (assoc :session-id session-id))))) diff --git a/exporter/src/exporter/consumer/config.cljs b/exporter/src/exporter/consumer/config.cljs new file mode 100644 index 0000000000..53237fea89 --- /dev/null +++ b/exporter/src/exporter/consumer/config.cljs @@ -0,0 +1,38 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.consumer.config + "The knobs of the consumer: how many jobs run at once and which + queue they come from. Read live, so a long-lived process obeys a + configuration change on the next poll." + (:require + [cljs.core :as c] + [exporter.config :as cf])) + +(def queue-name + "The queue this worker consumes: the one the `:export-assets` job-def + routes its jobs to through `::jobs/queue-name`." + "exporter") + +(defn concurrency + "K pollers: each one owns its Redis connection, and the blocking + connection is the semaphore of the slot, so K is the whole concurrency + of the process. At least one: a worker role with zero pollers is a + misconfiguration, not a rest. + + Takes the configured value as an argument, so the tests do not + journey through the global config; the zero-arity reads it live." + ([] + (concurrency (c/get cf/config :exporter-worker-concurrency))) + ([value] + (max 1 (if (number? value) value 2)))) + +(defn queue-key + "The Redis list the dispatcher pushes this queue's payloads through: + `penpot.worker.queue::`." + ([] (queue-key (cf/get :tenant))) + ([tenant] + (str "penpot.worker.queue:" tenant ":" queue-name))) diff --git a/exporter/src/exporter/consumer/plan.cljs b/exporter/src/exporter/consumer/plan.cljs new file mode 100644 index 0000000000..5f1d7ce94d --- /dev/null +++ b/exporter/src/exporter/consumer/plan.cljs @@ -0,0 +1,95 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.consumer.plan + "The render plan of an assets run. + + The items the backend froze in the job become the renders this + process runs: file names built once and kept free of duplicates, then + grouped by scale and type, one render per group on the wasm engine + and one DOM page per partition of fifty on the browser engine. + Every item of the job names the same file, page and share: the + creation command refuses mixed scopes, so grouping by scale and + type never mixes batches that render against different pages. + `exporter.consumer` runs whatever this namespace plans, and this + namespace only plans." + (:require + [app.common.data :as d] + [cuerdas.core :as str] + [exporter.renderer :as renderer] + [exporter.util.mime :as mime])) + +;; Regex to clean namefiles +(def sanitize-file-regex #"[\\/:*?\"<>|]") + +(defn count-objects + [exports] + (reduce + 0 (map (comp count :objects) exports))) + +(defn- assoc-file-name + "A transducer that assocs a candidate filename and avoid duplicates" + [] + (letfn [(find-candidate [params used] + (loop [index 0] + (let [candidate (str (:name params) + (:suffix params "") + (when (pos? index) + (str/concat "-" (inc index))) + (mime/get-extension (:type params)))] + (if (contains? used candidate) + (recur (inc index)) + candidate))))] + (fn [rf] + (let [used (volatile! #{})] + (fn + ([] (rf)) + ([result] (rf result)) + ([result params] + (let [candidate (find-candidate params @used) + params (assoc params :filename candidate)] + (vswap! used conj candidate) + (rf result params)))))))) + +(def ^:const ^:private + default-partition-size 50) + +(defn prepare-exports + [exports token is-wasm] + (letfn [(process-group [[part1 :as group]] + ;; The browser path renders one DOM page per partition; wasm + ;; takes the whole group in one render. + (if (renderer/wasm? {:is-wasm is-wasm :type (:type part1)}) + [(build-render group)] + (sequence (comp (partition-all default-partition-size) + (map build-render)) + group))) + + (build-render [[part1 :as part]] + (cond-> {:file-id (:file-id part1) + :page-id (:page-id part1) + :name (:name part1) + :token token + :type (:type part1) + :scale (:scale part1) + :is-wasm (boolean is-wasm) + :objects (mapv part-entry->object part)} + ;; a nil share-id is not an absent one for the render + ;; spec: only name it when the item came through one + (some? (:share-id part1)) (assoc :share-id (:share-id part1)))) + + (part-entry->object [entry] + {:id (:object-id entry) + :filename (:filename entry) + :name (:name entry) + :suffix (:suffix entry)})] + + (let [xform (comp + (map #(assoc % :token token)) + (assoc-file-name))] + (->> (sequence xform exports) + (d/group-by (juxt :scale :type)) + (map second) + (into [] (mapcat process-group)))))) diff --git a/exporter/src/exporter/consumer/worker.cljs b/exporter/src/exporter/consumer/worker.cljs new file mode 100644 index 0000000000..8272aa019b --- /dev/null +++ b/exporter/src/exporter/consumer/worker.cljs @@ -0,0 +1,171 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.consumer.worker + "The consumer of the backend `:exporter` queue. + + K pollers run in parallel, each one owning its Redis connection: the + blocking BLPOP of a poller holds that connection for the whole wait, + and the poller does not come back to the queue until the job it + claimed has settled, so the connection is both the semaphore of the + slot and the backpressure — nothing the queue receives can appear + twice taken, and no process ever runs more exports than it has + connections. + + The dispatcher pushes `[job-id, scheduled-at]` as JSON into + `penpot.worker.queue::exporter`. Corrupt payloads are logged + and dropped; a Redis hiccup is a lost turn, not a lost job: the + poller waits a bit and pops again (what was already delivered is + claimed, and the loop does not stop the process). Stopping quits + every connection and ends every loop: unlike the legacy worker, + whose loops kept failing turns after the stop, no poller survives + its handle. + + The worker is a system component (`:exporter.consumer/worker`): its + config carries the queue knobs (`:concurrency`, `:queue-key`, the + `:connect` factory) and the `:renderer` it renders through (the + `:exporter/renderer` service, injected by ref). There are no promesa + chains in this namespace." + (:require + ["ioredis" :as redis] + [app.common.logging :as l] + [exporter.config :as cf] + [exporter.consumer :as consumer] + [exporter.consumer.api :as api] + [exporter.consumer.config :as ccfg] + [exporter.utils.system :as system])) + +(def ^:private poll-timeout-s 5) +(def ^:private reconnect-delay-ms 1000) +(def ^:private drain-poll-ms 100) +(def ^:private default-shutdown-grace-ms 30000) + +(defn- wait + [ms] + (js/Promise. (fn [resolve] (js/setTimeout resolve ms)))) + +(defn read-payload + "The JSON payload of the dispatcher: `job-id` and the `scheduled-at` + of the row when it was pushed. Anything else is a corrupt payload: + none, and the caller drops it." + [payload] + (try + (let [[job-id scheduled-at] (js->clj (js/JSON.parse payload))] + {:job-id job-id :scheduled-at scheduled-at}) + (catch :default cause + (l/warn :hint "corrupt queue payload dropped" + :payload (str payload) :cause cause) + nil))) + +(defn ^:async process + "One payload of the queue: claim the job, and when the backend gives + it to this poller, run the export to its settle. `skip` means + something else won it (or it is gone): the payload is dropped, with + the state the backend says." + [cfg {:keys [job-id scheduled-at]}] + (let [answer (await (api/claim-job job-id scheduled-at))] + (if (= :skip (:action answer)) + (l/info :hint "skipping job" :job-id (str job-id) + :name (:name answer) :status (:status answer)) + (do + (l/info :hint "running job" :job-id (str job-id) :name (:name answer)) + (await (consumer/run-export (:renderer cfg) (:exporter/tmpdir cfg) {:job-id job-id} (:params answer))) + (l/info :hint "job settled" :job-id (str job-id)) + nil)))) + +;; ---- THE POLLER + +(defn ^:async poll-once + "One turn: pop the queue, claim what came, and wait for the settle. + An empty pop and a dropped payload both resolve; a claimed job is + awaited the whole run." + [conn key on-payload] + (let [answer (await (.blpop ^js conn key poll-timeout-s)) + payload (second answer) + claim (when (some? payload) (read-payload payload))] + (when (some? claim) + (await (on-payload claim)) + nil))) + +(defn- ^:async poll-loop + "The wait of one poller, turn after turn, until the handle stops. A + failed turn (Redis down, a bug) is logged, never allowed to end the + loop; a delay keeps the retry from spinning while the connection + rebuilds. An explicit loop, not recursion: a self-call under `await` + would retain one async frame per turn on a process that lives for + weeks." + [handle conn key on-payload] + (loop [] + (when @(:running handle) + (try + (await (poll-once conn key on-payload)) + (catch :default cause + (l/error :hint "poller turn failed" :key key :cause cause) + (await (wait reconnect-delay-ms)))) + (recur)))) + +(defn- open-connection + "One Redis connection of one poller. ioredis reconnects on its own + strategy, so a fallen Redis comes back with this process doing + nothing but wait (and the failed pop comes right after)." + [] + (new redis/default (cf/get :redis-uri))) + +;; ---- THE START + +(defn ^:async start + "Launches the pollers over the queue and resolves the handle their + `stop` needs. The temp area boots clean before the first poller: the + tmpdir service drops what a previous process left on its init." + [{:keys [concurrency queue-key connect] :as cfg}] + (let [running (atom true) + inflight (atom 0) + handle {:running running :conns (atom []) :inflight inflight} + on-payload (^:async fn [claim] + (swap! inflight inc) + (try + (await (process cfg claim)) + (finally + (swap! inflight dec)))) + open (or connect open-connection) + k (or concurrency (ccfg/concurrency))] + (l/info :hint "worker started" :pollers k :queue queue-key) + (doseq [_ (range k)] + (let [conn (open)] + (swap! (:conns handle) conj conn) + (poll-loop handle conn queue-key on-payload))) + handle)) + +(defn- ^:async drain-inflight + [inflight deadline] + (when (and (pos? @inflight) + (< (js/Date.now) deadline)) + (await (wait drain-poll-ms)) + (await (drain-inflight inflight deadline)))) + +(defn ^:async stop + "Ends the pollers, closing their connections and their loops: the + running flag falls first, so a turn that ends after the quit does + not pop again, then the in-flight runs drain within the grace + period before the pools halt behind them. Idempotent: a second stop + quits nothing." + ([handle] (stop handle nil)) + ([handle {:keys [grace-ms] :or {grace-ms default-shutdown-grace-ms}}] + (when (and (some? handle) + (compare-and-set! (:running handle) true false)) + (await (js/Promise.allSettled + (mapv (fn [conn] (.quit ^js conn)) @(:conns handle)))) + (let [inflight (or (:inflight handle) (atom 0))] + (await (drain-inflight inflight (+ (js/Date.now) grace-ms)))) + nil))) + +(defmethod system/init-key ::consumer/worker + [_ cfg] + (start cfg)) + +(defmethod system/halt-key ::consumer/worker + [_ handle] + (stop handle)) diff --git a/exporter/src/exporter/jobs.cljs b/exporter/src/exporter/jobs.cljs new file mode 100644 index 0000000000..82cece45d8 --- /dev/null +++ b/exporter/src/exporter/jobs.cljs @@ -0,0 +1,69 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.jobs + "The local arm of the job life this process runs. + + The durable row of a job is the backend's business: the worker claims + it, beats against it and settles it through the management API. This + namespace keeps only the runtime bit that cannot live in a row — the + cancel flag the parts of a run read between renders — keyed by the + job id of the claimed job. + + This is deliberately smaller than the legacy `app.jobs`: the abort + itself belongs to whoever holds the render (the per-lease interval of + `exporter.wasm.scope` terminates its own worker when the check + fails), so no terminate callbacks and no shared-array signal travel + through here. The only surface toward the renderers is one zero-arg + check constructor per job." + (:require + [app.common.exceptions :as ex])) + +(defonce ^:private registry (atom {})) + +(defn register + "The runtime record of a job this process runs: one cancel flag, + down until something marks it." + [job-id] + (swap! registry assoc (str job-id) {:cancelled? false}) + nil) + +(defn cancelled? + "True once the job has been marked cancelled in this process. The + check between objects reads it; inside a render the leased worker + reads its own abort instead." + [job-id] + (boolean (:cancelled? (get @registry (str job-id))))) + +(defn mark-cancelled + "Signals one job of this process as cancelled: the flag the checks + read goes up. A job this process does not own, or a settled one, is + a no-op. Returns true when this call marked it, nil otherwise." + [job-id] + (let [k (str job-id) + rt (get @registry k)] + (when (some? rt) + (when-not (:cancelled? rt) + (swap! registry assoc-in [k :cancelled?] true) + true)))) + +(defn release + "Drops the local runtime entry once the job settled. The row stays in + the backend until it is read once more or expires." + [job-id] + (swap! registry dissoc (str job-id)) + nil) + +(defn check-cancelled + "The zero-arg check of one job: silent while it lives, raising + `:job-cancelled` once marked. The same fn travels into the render + task and runs between units of work." + [job-id] + (fn [] + (when (cancelled? job-id) + (throw (ex/error :type :internal + :code :job-cancelled + :hint "export job was cancelled"))))) diff --git a/exporter/src/exporter/main.cljs b/exporter/src/exporter/main.cljs new file mode 100644 index 0000000000..132620915f --- /dev/null +++ b/exporter/src/exporter/main.cljs @@ -0,0 +1,185 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.main + "Entry point of the new exporter tree, modeled on the backend's + `app.main`: a `system-config` map wired with refs, and `start`, + `stop` and `restart` managing the running system. + + Render workers run this same bundle, so the thread decides what gets + booted: one render worker (`exporter.wasm.worker`, owning the + headless module), or the system of the main thread. Without the guard a + warmed pool worker would boot pools and pollers of its own and never + post its `ready` handshake." + (:require + ["node:process" :as proc] + ["node:worker_threads" :as wt] + [app.common.logging :as l] + [exporter.browser :as browser] + [exporter.config :as cf] + [exporter.consumer :as-alias consumer] + [exporter.consumer.api :as api] + [exporter.consumer.config :as ccfg] + [exporter.consumer.worker] + [exporter.tmpdir] + [exporter.utils.system :as system] + [exporter.wasm :as-alias wasm] + [exporter.wasm.pool] + [exporter.wasm.worker :as wasm.worker])) + +(l/setup! {:exporter :info}) + +(defn system-config + "The production wiring, built at every call so a reboot picks up + configuration changes without reloading namespaces. Env-derived + values are read here, at the wiring layer; the components themselves + take plain data. The render service owns the render config (pools by + ref plus the static config); the worker only names the renderer it + renders through." + [] + {::browser/pool + {:max (ccfg/concurrency)} + + ::wasm/pool + {:max (ccfg/concurrency) + :min (cf/get :wasm-worker-pool-min) + ;; the knob is in seconds, the pool wants milliseconds + :idle-timeout (* 1000 (cf/get :wasm-worker-idle-timeout))} + + :exporter/tmpdir + {:path (cf/get :tempdir) + :job-ttl (cf/get :exporter-job-ttl 3600)} + + :exporter/renderer + {::browser/pool (system/ref :exporter.browser/pool) + ::wasm/pool (system/ref :exporter.wasm/pool) + :exporter/tmpdir (system/ref :exporter/tmpdir) + :base-uri (cf/get-internal-uri) + :public-uri (cf/get :public-uri) + :image-cache-size (cf/get :wasm-worker-image-cache-size) + :svgo? (contains? cf/flags :exporter-svgo)} + + ::consumer/worker + {:renderer (system/ref :exporter/renderer) + :exporter/tmpdir (system/ref :exporter/tmpdir) + :concurrency (ccfg/concurrency) + :queue-key (ccfg/queue-key)}}) + +;; The running system map, or nil when nothing is started. +;; Counterpart of the backend's `app.system/system` var. +(defonce system + (atom nil)) + +;; The config the running system booted with. `restart` reboots it, +;; so tests can cycle a fake config without touching production. +(defonce ^:private config-used + (atom nil)) + +(defn ^:async start-custom + "Boot the given config, halting the running system first, like the + backend's `start-custom`. Resolves to `:started`." + [config] + (when-let [sys @system] + (await (system/halt sys))) + (swap! config-used (constantly config)) + (reset! system nil) + (let [sys (await (system/init config))] + (reset! system sys) + :started)) + +(declare shutdown) + +;; Install-once guard: `start` runs on every hot reload, and piling +;; process listeners would end in MaxListenersExceeded warnings. +(defonce ^:private handlers-installed? (atom false)) + +(defn install-process-handlers + "Register the process-level handlers: log uncaught exceptions instead + of dying silently, and drain on SIGTERM/SIGINT. Signals only reach the + main thread; exiting in a render worker would take down that worker + rather than the process. Runs on `start`, never on namespace load, + and only the first call takes effect." + [] + (when (compare-and-set! handlers-installed? false true) + (.on proc/default "uncaughtException" + (fn [cause] + (js/console.error cause))) + (when ^boolean wt/isMainThread + (.on proc/default "SIGTERM" (fn [] (shutdown))) + (.on proc/default "SIGINT" (fn [] (shutdown)))))) + +(defn ^:async start + "Boot the production wiring. Resolves to `:started`. On a render + worker thread boots the render worker instead: it runs this same + bundle and owns no pools or pollers." + [] + (install-process-handlers) + (if ^boolean wt/isMainThread + (do + (l/info :msg "starting exporter" + :workers (ccfg/concurrency) + :version (:full cf/version)) + (await (start-custom (system-config)))) + (do + (wasm.worker/main) + :started))) + +(defn ^:async stop + "Halt the running system, if any. Resolves to `:stopped`." + [] + (when-let [sys @system] + (await (system/halt sys))) + (reset! system nil) + :stopped) + +(defn ^:async shutdown + "Graceful shutdown: halt the running system (the worker stops + intake first, in-flight runs drain within their grace period, then + the pools halt in reverse dependency order) and exit. A halt + failure still exits, so a deploy never leaves the process behind. + `exit` is a seam for the tests, which cannot stub the process." + ([] (shutdown proc/exit)) + ([exit] + (try + (await (stop)) + (catch :default cause + (l/warn :hint "graceful shutdown halt failed" :cause cause))) + (api/close-client!) + (exit 0))) + +(defn ^:async restart + "Halt and reboot whatever config is running (or the production + wiring when nothing ever started). Resolves to `:restarted`." + [] + (await (stop)) + (await (start-custom (or @config-used (system-config)))) + :restarted) + +(defn ^:dev/before-load-async on-before-load + "Hot reload hook: halt the running system before shadow swaps the + code. `done` always runs, so a halt failure is logged, never a + stuck reload." + [done] + (-> (stop) + (.then (fn [_] (done))) + (.catch (fn [cause] + (l/warn :hint "hot reload stop failed" + :cause (ex-message cause)) + (done))))) + +(defn ^:dev/after-load-async on-after-load + "Hot reload hook: boot the production wiring after shadow swaps the + code. `done` always runs, so a boot failure is logged, never a + stuck reload." + [done] + (-> (start) + (.then (fn [_] (done))) + (.catch (fn [cause] + (l/warn :hint "hot reload start failed" + :cause (ex-message cause)) + (done))))) + +(def main start) diff --git a/exporter/src/exporter/renderer.cljs b/exporter/src/exporter/renderer.cljs new file mode 100644 index 0000000000..61a2a95646 --- /dev/null +++ b/exporter/src/exporter/renderer.cljs @@ -0,0 +1,140 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.renderer + "Common renderer interface, validated with malli from day one. + + One entry point: `render` takes `cfg` plus a task map + (`:exports`, `:on-object`, `:check-cancelled`), validates it, and + fires the whole batch at once — the browser exports through + `exporter.browser.scope`, the wasm ones through one leased + worker in `exporter.wasm.scope`. A single export is a batch of one. + Rendered objects travel through `on-object`; the batch itself + resolves nil once every export landed. + + `cfg` is task-free infrastructure with process lifetime: the pool + services under their system keys (`:exporter.browser/pool`, + `:exporter.wasm/pool`) plus the static render config + (`:base-uri`, `:public-uri`, `:svgo?`) — a view over the running + system map that each domain reads with its own keys. The task + map holds everything per-task: the `exports` data plus the two + injected indirections, `on-object` (the per-object collector) and + `check-cancelled` (the zero-arg check-cancelled raising + `:job-cancelled`). One closed schema validates the whole task, so a + bad batch — or a non-fn injection — never touches a pool. Each + driver takes the same `[cfg params on-object check-cancelled]` shape and + implements cancel internally: cooperative checkpoints on the + browser, checkpoints plus a mid-render abort of its own leased + worker on wasm. + + The renderer is also a system service (`:exporter/renderer`): its + config value carries the pools as refs plus the static config, and + its running instance is the render fn with that config closed over, + which is what the consumer renders through — the consumer never + calls this namespace directly." + (:require + [app.common.schema :as sm] + [exporter.browser.scope :as bscope] + [exporter.utils.system :as system] + [exporter.wasm.render :as wrender] + [exporter.wasm.scope :as scope])) + +(def schema:type + [:enum :png :jpeg :webp :pdf :svg]) + +(def schema:object + [:map + [:id ::sm/uuid] + [:name :string] + [:suffix :string] + [:filename :string] + [:share-id {:optional true} ::sm/uuid]]) + +(def schema:objects + [:vector {:min 1} schema:object]) + +(def schema:render-params + [:map {:closed true} + [:file-id ::sm/uuid] + [:page-id ::sm/uuid] + [:scale ::sm/number] + [:token :string] + [:type schema:type] + [:objects schema:objects] + [:name :string] + [:share-id {:optional true} ::sm/uuid] + [:skip-children {:optional true} ::sm/boolean] + [:is-wasm {:optional true} ::sm/boolean] + [:job-id {:optional true} ::sm/uuid]]) + +(def schema:exports + [:vector {:min 1} schema:render-params]) + +(def schema:task + [:map {:closed true} + [:exports schema:exports] + [:on-object ::sm/fn] + [:check-cancelled ::sm/fn]]) + +(def ^:private check-task + (sm/check-fn schema:task :hint "invalid render task")) + +(defn wasm? + "Whether `params` renders with render-wasm rather than a browser." + [{:keys [is-wasm]}] + (boolean is-wasm)) + +(defn- render-env + "The render environment of one batch: internal and public endpoints, + the temp area and the image cache budget. Read from the service config + (wired from the environment); the worker thread never resolves any of + it itself." + [cfg] + {:internal-uri (:base-uri cfg) + :public-uri (:public-uri cfg) + :tmpdir (:exporter/tmpdir cfg) + :image-cache-size (:image-cache-size cfg)}) + +(defn ^:async render + "Renders every export, firing them all at once: the browser ones + through their own scope, the wasm ones through one leased + worker. Assumes a valid task — validation lives one level up, at + the service entry, so through here a bad task never touches a pool + because it never arrives. Rejects on the first failure." + [cfg {:keys [exports on-object check-cancelled]}] + (let [cfg (assoc cfg + ::on-object on-object + ::check-cancelled check-cancelled) + ;; Everything the worker thread needs beyond the export + ;; itself rides in the posted params: the worker reads no + ;; config and no globals. + wexports (map #(assoc % ::wrender/env (render-env cfg)) + (filter wasm? exports)) + bexports (remove wasm? exports)] + (await (js/Promise.all + (cond-> [] + (seq bexports) + (conj (bscope/run cfg bexports)) + (seq wexports) + (conj (scope/run cfg + (fn [render-fn] + (js/Promise.all (map render-fn wexports)))))))) + nil)) + +(defn render-fn + "The render entry of one running system: validates the task map and + dispatches it through the facade with `cfg` closed over, resolving + nil once every export landed. The single boundary where a bad batch + is rejected before anything is leased — and the dependency the + consumer renders through, so the consumer never calls the facade + directly." + [cfg] + (fn [task] + (render cfg (check-task task)))) + +(defmethod system/init-key :exporter/renderer + [_ cfg] + (render-fn cfg)) diff --git a/exporter/src/exporter/renderer/browser.cljs b/exporter/src/exporter/renderer/browser.cljs new file mode 100644 index 0000000000..3eab6c282c --- /dev/null +++ b/exporter/src/exporter/renderer/browser.cljs @@ -0,0 +1,509 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.renderer.browser + "Renders through a browser page: bitmap, pdf and svg in one namespace. + + Every flow renders one partition of objects per browser checkout: + navigate once, then capture object by object, calling `on-object` + with each object carrying its `:path`. `on-object` may return a + plain value or a promise; it is always awaited. + + `cfg` is task-free infrastructure: the `:exporter.browser/pool`, the + render `:base-uri` and `:public-uri`, the `:exporter/tmpdir` area and + the `:svgo?` flag. The + per-task injections (`on-object`, `check-cancelled`) ride positional. There + are no promesa chains in this namespace." + (:require + ["@penpot/svgo" :as svgo] + ["xml-js" :as xml] + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.types.color :as ctc] + [app.common.uri :as u] + [clojure.walk :as walk] + [cuerdas.core :as str] + [exporter.browser :as browser] + [exporter.renderer.svg-gradient :as svg-gradient] + [exporter.shell :as shell] + [exporter.util.mime :as mime])) + +(l/set-level! :trace) + +(defn- raise-if-cancelled + "Cooperative checkpoint over the injected zero-arg check-cancelled. This + backend never blocks indefinitely (every step yields), so + yielding-point checks are the whole cancel story here. The check-cancelled is + mandatory: without one it raises `:check-cancelled-missing` instead + of running unprotected." + [check-cancelled] + (if (fn? check-cancelled) + (check-cancelled) + (throw (ex/error :type :assertion + :code :check-cancelled-missing + :hint "browser render needs a zero-arg check-cancelled")))) + +(defn- prepare-options + [uri token scale] + #js {:screen #js {:width browser/default-viewport-width + :height browser/default-viewport-height} + :viewport #js {:width browser/default-viewport-width + :height browser/default-viewport-height} + :locale "en-US" + :storageState #js {:cookies (browser/create-cookies uri {:token token})} + :deviceScaleFactor scale + :userAgent browser/default-user-agent}) + +(defn- render-uri + [base-uri query-params] + (-> base-uri + (u/ensure-path-slash) + (u/join "render.html") + (assoc :query (u/map->query-string query-params)))) + +;; --- BITMAP (png, jpeg, webp) + +(defn- ^:async render-bitmap-object + [tmpdir page type object on-object] + (let [path (await (shell/tempfile tmpdir :prefix "penpot.tmp.bitmap." :suffix (mime/get-extension type))) + node (browser/select page (str/concat "#screenshot-" (:id object)))] + (await (browser/wait-for node)) + (case type + :png (await (browser/screenshot node {:omit-background? true :type type :path path})) + :jpeg (await (browser/screenshot node {:omit-background? false :type type :path path})) + :webp (let [png-path (await (shell/tempfile tmpdir :prefix "penpot.tmp.bitmap." :suffix ".png"))] + ;; playwright only supports jpg and png, we need to convert it afterwards + (await (browser/screenshot node {:omit-background? true :type :png :path png-path})) + (await (shell/run-cmd "convert" png-path "-quality" "100" (str "WEBP:" path))))) + (await (on-object (assoc object :path path))))) + +(defn- ^:async render-bitmap-page + [tmpdir uri page type objects on-object] + (l/info :uri (str uri)) + ;; navigate to the page and perform basic setup + (await (browser/nav page (str uri))) + (await (browser/sleep page 1000)) ; the good old fix with sleep + (await (browser/wait-for-fonts page)) + (await (browser/eval page (js* "() => document.body.style.background = 'transparent'"))) + ;; take the screnshot of requested objects, one by one + (await (js/Promise.all (mapv (fn [object] (render-bitmap-object tmpdir page type object on-object)) + objects))) + nil) + +(defn- ^:async render-bitmap + [cfg {:keys [file-id page-id share-id token scale type objects skip-children]} on-object] + (let [query {:file-id file-id + :page-id page-id + :share-id share-id + :object-id (mapv :id objects) + :route "objects" + :skip-children skip-children} + uri (render-uri (:base-uri cfg) query)] + (await (browser/exec (:exporter.browser/pool cfg) + (prepare-options uri token scale) + (fn [page] (render-bitmap-page (:exporter/tmpdir cfg) uri page type objects on-object)))))) + +;; --- PDF + +(defn- prepare-pdf-uri + [base-uri {:keys [file-id page-id share-id]} object-id] + (let [query {:file-id file-id + :page-id page-id + :share-id share-id + :object-id object-id + :route "objects"}] + (-> base-uri + (u/join "render.html") + (assoc :query (u/map->query-string query))))) + +(defn- sync-page-size + [dom] + (browser/eval dom + (fn [elem] + ;; IMPORTANT: No CLJS runtime allowed. Use only JS + ;; primitives. This runs in a context without access to + ;; cljs.core. Avoid any functions that transpile to + ;; cljs.core/* calls, as they will break in the browser + ;; runtime. + + (let [width (.getAttribute ^js elem "width") + height (.getAttribute ^js elem "height") + style-node (let [node (.createElement js/document "style")] + (.appendChild (.-head js/document) node) + node)] + (set! (.-textContent style-node) + (dm/str "@page { size: " width "px " height "px; margin: 0; }\n" + "html, body, #app { margin: 0; padding: 0; width: " width "px; height: " height "px; overflow: visible; }")))))) + +(defn- ^:async render-pdf-object + [tmpdir page base-uri params object on-object] + (let [uri (prepare-pdf-uri base-uri params (:id object)) + path (await (shell/tempfile tmpdir :prefix "penpot.tmp.pdf." :suffix (mime/get-extension (:type params))))] + (l/info :uri (str uri)) + (await (browser/nav page uri)) + (let [dom (browser/select page (dm/str "#screenshot-" (:id object)))] + (await (browser/wait-for dom)) + (await (sync-page-size dom)) + (await (browser/screenshot dom {:full-page? true})) + (await (browser/sleep page 2000)) ; the good old fix with sleep + (await (browser/wait-for-fonts page)) + (await (browser/pdf page {:path path})) + (await (on-object (assoc object :path path)))))) + +(defn- ^:async render-pdf-page + [tmpdir base-uri page params objects on-object check-cancelled] + (doseq [object objects] + (raise-if-cancelled check-cancelled) + (await (render-pdf-object tmpdir page base-uri params object on-object))) + nil) + +(defn- ^:async render-pdf + [cfg {:keys [token scale] :as params} on-object check-cancelled] + (let [base-uri (-> (:base-uri cfg) + (u/ensure-path-slash))] + (await (browser/exec (:exporter.browser/pool cfg) + (prepare-options base-uri token scale) + (fn [page] + (render-pdf-page (:exporter/tmpdir cfg) base-uri page params (:objects params) on-object check-cancelled)))))) + +;; --- SVG + +(defn- xml->clj + [data] + (js->clj (xml/xml2js data))) + +(defn- clj->xml + [data] + (xml/js2xml (clj->js data))) + +(defn- ^boolean element? + [item] + (and (map? item) + (= "element" (get item "type")))) + +(defn- ^boolean foreign-object-element? + [item] + (and (element? item) + (= "foreignObject" (get item "name")))) + +(defn- ^boolean empty-defs-element? + [item] + (and (= (get item "name") "defs") + (nil? (get item "attributes")) + (nil? (get item "elements")))) + +(defn- ^boolean empty-path-element? + [item] + (and (= (get item "name") "path") + (let [d (get-in item ["attributes" "d"])] + (or (str/blank? d) + (nil? d) + (str/empty? d))))) + +(defn- flatten-toplevel-svg-elements + "Flattens XML data structure if two nested top-side SVG elements found." + [item] + (if (and (= "svg" (get-in item ["elements" 0 "name"])) + (= "svg" (get-in item ["elements" 0 "elements" 0 "name"]))) + (update-in item ["elements" 0] assoc "elements" (get-in item ["elements" 0 "elements" 0 "elements"])) + item)) + +(defn- replace-text-nodes + "Function responsible of replace the foreignObject elements on the + provided XML with the previously rasterized PATH's." + [xmldata nodes] + (letfn [(replace-fobject [item] + (if (foreign-object-element? item) + (let [id (get-in item ["attributes" "id"]) + node (get nodes id)] + (if node + (:svgdata node) + item)) + item)) + + (process-element [item xform] + (d/update-when item "elements" #(into [] xform %)))] + + (let [xform (comp (remove empty-defs-element?) + (remove empty-path-element?) + (map replace-fobject))] + (->> xmldata + (xml->clj) + (flatten-toplevel-svg-elements) + (walk/prewalk (fn [item] + (cond-> item + (element? item) + (process-element xform)))) + (clj->xml))))) + +(defn- parse-viewbox + "Parses viewBox string into width & height map." + [data] + (let [[width height] (->> (str/split data #"\s+") + (drop 2) + (map d/parse-double))] + {:width width + :height height})) + +(defn- replace-internal-uris + "Replaces internal-uri references with public-uri in SVG output. + This ensures that font URLs and other resource references in the + exported SVG use the public-facing URI accessible to end users." + [svg-content internal-uri public-uri] + (let [internal-uri (str internal-uri) + public-uri (str public-uri)] + (if (and (not= internal-uri public-uri) + (str/includes? svg-content internal-uri)) + (str/replace svg-content internal-uri public-uri) + svg-content))) + +(defn- ^:async convert-to-ppm + [pngpath] + (let [ppmpath (str/concat pngpath "origin.ppm")] + (l/trace :fn :convert-to-ppm :path ppmpath) + (await (shell/run-cmd "convert" pngpath ppmpath)) + ppmpath)) + +(defn- ^:async trace-color-mask + [pbmpath] + (l/trace :fn :trace-color-mask :pbmpath pbmpath) + (let [svgpath (str/concat pbmpath ".svg")] + (await (shell/run-cmd "potrace" "--flat" "-b" "svg" pbmpath "-o" svgpath)) + svgpath)) + +(defn- ^:async generate-color-layer + [ppmpath color] + (when-not (ctc/hex-color-string? color) + (ex/raise :type :validation + :code :invalid-color + :hint (str "invalid hex color: " color))) + (l/trace :fn :generate-color-layer :ppmpath ppmpath :color color) + (let [pbmpath (str/concat ppmpath ".mask-" (subs color 1) ".pbm") + stdout (await (shell/run-cmd "ppmcolormask" color ppmpath))] + (await (shell/write-file pbmpath stdout)) + (let [svgpath (await (trace-color-mask pbmpath)) + data (await (shell/read-file svgpath)) + data (xml->clj data) + data (get-in data ["elements" 1])] + {:color color + :svgdata data}))) + +(defn- set-path-color + [id color mapping node] + (let [color-mapping (get mapping color)] + (cond + (and (some? color-mapping) + (= "transparent" (get color-mapping "type"))) + (update node "attributes" assoc + "fill" (get color-mapping "hex") + "fill-opacity" (get color-mapping "opacity")) + + (and (some? color-mapping) + (= "gradient" (get color-mapping "type"))) + (update node "attributes" assoc + "fill" (str "url(#gradient-" id "-" (subs color 1) ")")) + + :else + (update node "attributes" assoc "fill" color)))) + +(defn- get-gradients + [id mapping] + (->> mapping + (filter (fn [[_color data]] + (= (get data "type") "gradient"))) + (mapv (partial svg-gradient/data->gradient-def id)))) + +(defn- join-color-layers + [{:keys [id x y width height mapping] :as node} layers] + (l/trace :fn :join-color-layers :mapping mapping) + (loop [result (-> (:svgdata (first layers)) + (assoc "elements" [])) + layers (seq layers)] + (if-let [{:keys [color svgdata]} (first layers)] + (recur (->> (get svgdata "elements") + (filter #(= (get % "name") "g")) + (map (partial set-path-color id color mapping)) + (update result "elements" into)) + (rest layers)) + + ;; Now we have the result containing the svgdata of a + ;; SVG with all text layers. Now we need to transform + ;; this SVG to G (Group) and remove unnecessary metadata + ;; objects. + (let [vbox (-> (get-in result ["attributes" "viewBox"]) + (parse-viewbox)) + transform (str/fmt "translate(%s, %s) scale(%s, %s)" x y + (/ width (:width vbox)) + (/ height (:height vbox))) + + gradient-defs (get-gradients id mapping) + + elements + (->> (get result "elements") + (mapv (fn [group] + (let [paths (get group "elements")] + (if (= 1 (count paths)) + (let [path (first paths)] + (update path "attributes" + (fn [attrs] + (-> attrs + (d/merge (get group "attributes")) + (update "transform" #(str transform " " %)))))) + (update-in group ["attributes" "transform"] #(str transform " " %))))))) + + elements (cond->> elements + (seq gradient-defs) + (into [{"type" "element" "name" "defs" "attributes" {} + "elements" gradient-defs}]))] + + (-> result + (assoc "name" "g") + (assoc "attributes" {}) + (assoc "elements" elements)))))) + +(defn- ^:async convert-to-svg + [ppmpath node] + (l/trace :fn :convert-to-svg :ppmpath ppmpath :colors (:colors node)) + (let [layers (await (js/Promise.all (mapv (partial generate-color-layer ppmpath) + (:colors node))))] + (join-color-layers node layers))) + +(defn- ^:async trace-node + [tmpdir node] + (l/trace :fn :trace-node) + (let [pngpath (await (shell/tempfile tmpdir :prefix "penpot.tmp.render.svg.parse." + :suffix ".origin.png"))] + (await (shell/write-file pngpath (:data node))) + (let [ppmpath (await (convert-to-ppm pngpath)) + svgdata (await (convert-to-svg ppmpath node))] + (-> node + (dissoc :data) + (assoc :svgdata svgdata))))) + +(defn- extract-element-attrs + [^js element] + (let [^js attrs (.. element -attributes) + ^js colors (.. element -dataset -colors) + ^js mapping (.. element -dataset -mapping)] + #js {:id (.. attrs -id -value) + :x (.. attrs -x -value) + :y (.. attrs -y -value) + :width (.. attrs -width -value) + :height (.. attrs -height -value) + :colors (.split colors ",") + :mapping (js/JSON.parse mapping)})) + +(defn- ^:async extract-single-node + [[shot node]] + (l/trace :fn :extract-single-node) + (let [attrs (await (browser/eval node extract-element-attrs))] + {:id (unchecked-get attrs "id") + :x (unchecked-get attrs "x") + :y (unchecked-get attrs "y") + :width (unchecked-get attrs "width") + :height (unchecked-get attrs "height") + :colors (vec (unchecked-get attrs "colors")) + :mapping (js->clj (unchecked-get attrs "mapping")) + :data shot})) + +(defn- ^:async resolve-text-node + [page node] + (let [attrs (await (browser/eval node extract-element-attrs)) + id (unchecked-get attrs "id") + text-node (browser/select page (str "#screenshot-text-" id " foreignObject")) + shot (await (browser/screenshot text-node {:omit-background? true :type "png"}))] + [shot node])) + +(defn- ^:async extract-txt-node + [tmpdir page item] + (let [[shot node] (await (resolve-text-node page item)) + single (await (extract-single-node [shot node]))] + (await (trace-node tmpdir single)))) + +(defn- ^:async extract-txt-nodes + [tmpdir page {:keys [id]}] + (l/trace :fn :process-text-nodes) + (let [nodes (await (browser/select-all page (str/concat "#screenshot-" id " foreignObject"))) + nodes (await (js/Promise.all (mapv (partial extract-txt-node tmpdir page) nodes)))] + (d/index-by :id nodes))) + +(defn- ^:async extract-svg + [page {:keys [id]}] + (let [node (browser/select page (str/concat "#screenshot-" id))] + (await (browser/wait-for node)) + (await (browser/eval node (fn [elem] (.-outerHTML ^js elem)))))) + +(defn sanitize-svg-content + "SVG standard don't allow the entity nbsp.   is equivalent but + compatible with SVG." + [content] + (str/replace content " " " ")) + +(defn- ^:async render-svg-object + [cfg page type object on-object] + (let [path (await (shell/tempfile (:exporter/tmpdir cfg) :prefix "penpot.tmp.render.svg." :suffix (mime/get-extension type))) + node (browser/select page (str/concat "#screenshot-" (:id object)))] + (await (browser/wait-for node)) + (let [xmldata (await (extract-svg page object)) + txtdata (await (extract-txt-nodes (:exporter/tmpdir cfg) page object)) + result (replace-text-nodes xmldata txtdata) + result (sanitize-svg-content result) + + result (if (:svgo? cfg) + (svgo/optimize result svgo/defaultOptions) + result) + + result (replace-internal-uris result (:base-uri cfg) (:public-uri cfg))] + (await (shell/write-file path result)) + (await (on-object (assoc object :path path)))))) + +(defn- ^:async render-svg-page + [cfg uri page type objects on-object] + (l/info :uri (str uri)) + ;; navigate to the page and perform basic setup + (await (browser/nav page (str uri))) + (await (browser/sleep page 1000)) ; the good old fix with sleep + (await (browser/wait-for-fonts page)) + ;; take the screnshot of requested objects, one by one + (await (js/Promise.all (mapv (fn [object] (render-svg-object cfg page type object on-object)) + objects))) + nil) + +(defn- ^:async render-svg + [cfg {:keys [file-id page-id share-id token scale type objects]} on-object] + (let [query {:file-id file-id + :page-id page-id + :share-id share-id + :render-embed true + :object-id (mapv :id objects) + :route "objects"} + uri (render-uri (:base-uri cfg) query)] + (await (browser/exec (:exporter.browser/pool cfg) + (prepare-options uri token scale) + (fn [page] (render-svg-page cfg uri page type objects on-object)))))) + +(defn ^:async render + "Renders one partition of objects through a browser page: bitmap + (png, jpeg, webp), pdf or svg depending on the type in `params`. + `cfg` is task-free infrastructure (pools, uris, flags), while + `on-object` and `check-cancelled` ride positional as the per-task + injections — the per-object collector and the zero-arg check-cancelled + raising `:job-cancelled`. + The check-cancelled runs here at fail-fast entry plus the sequential pdf + loop; the concurrent flows rely on the entry check-cancelled and the caller's + `on-object`. Calls `on-object` with each object carrying its + `:path`, awaiting it every time, and resolves nil." + [cfg {:keys [type] :as params} on-object check-cancelled] + (raise-if-cancelled check-cancelled) + (case type + (:png :jpeg :webp) (await (render-bitmap cfg params on-object)) + :pdf (await (render-pdf cfg params on-object check-cancelled)) + :svg (await (render-svg cfg params on-object)) + (ex/raise :type :validation + :code :unknown-render-type + :hint (str "unknown render type: " type)))) diff --git a/exporter/src/app/renderer/svg_gradient.cljs b/exporter/src/exporter/renderer/svg_gradient.cljs similarity index 96% rename from exporter/src/app/renderer/svg_gradient.cljs rename to exporter/src/exporter/renderer/svg_gradient.cljs index 897ed80304..cfacaf38f5 100644 --- a/exporter/src/app/renderer/svg_gradient.cljs +++ b/exporter/src/exporter/renderer/svg_gradient.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.renderer.svg-gradient) +(ns exporter.renderer.svg-gradient) (defn- get-stops [data] diff --git a/exporter/src/exporter/resources.cljs b/exporter/src/exporter/resources.cljs new file mode 100644 index 0000000000..4b26d503b6 --- /dev/null +++ b/exporter/src/exporter/resources.cljs @@ -0,0 +1,76 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.resources + "The run's local resources: the artifact files the runner parks in + the injected temp area until the settle moves them into the job's + completion, zipped once a run packs more than one. + + Same operations as the legacy `app.handlers.resources`, adapted: + the temp area arrives injected (no global), and the zip close is + awaited instead of chained. The per-file deletion timers of the old + shell are gone on purpose: the tmpdir registry owns every path and + releases it on the settle." + (:require + ["archiver" :as arc] + ["node:fs" :as fs] + ["node:path" :as path] + [app.common.uuid :as uuid] + [cljs.core :as c] + [cuerdas.core :as str] + [exporter.util.mime :as mime])) + +(defn- resource-path + [tmpdir type id] + (path/join tmpdir (str/concat "penpot.resource." (c/name type) "." id))) + +(defn create + "An ephemeral artifact descriptor of `type` named `name`, parked + under the injected `tmpdir`." + [tmpdir type name] + (let [task-id (uuid/next)] + {:path (resource-path tmpdir type task-id) + :mtype (mime/get type) + :name name + :filename (str/concat (str/replace name #"[\\/:*?\"<>|]" "_") (mime/get-extension type)) + :id task-id})) + +(defn create-zip + [& {:keys [resource on-complete on-progress on-error]}] + (let [^js zip (new arc/ZipArchive) + ^js out (fs/createWriteStream (:path resource)) + on-complete (or on-complete (constantly nil)) + progress (atom 0)] + (.on zip "error" on-error) + (.on zip "end" on-complete) + (.on zip "entry" (fn [data] + (let [name (unchecked-get data "name") + num (swap! progress inc)] + (on-progress {:done num :filename name})))) + (.pipe zip out) + zip)) + +(defn add-to-zip + [zip path name] + (.file ^js zip path #js {:name name})) + +(defn ^:async close-zip + ([zip] (close-zip zip nil)) + ([zip {:keys [timeout-ms] :or {timeout-ms 30000}}] + (await (js/Promise. + (fn [resolve reject] + (let [timer (js/setTimeout + (fn [] (reject (js/Error. "zip close timed out"))) + timeout-ms) + done (fn [] + (js/clearTimeout timer) + (resolve nil)) + fail (fn [cause] + (js/clearTimeout timer) + (reject cause))] + (.on ^js zip "close" (fn [] (done))) + (.on ^js zip "error" (fn [cause] (fail cause))) + (.finalize ^js zip))))))) diff --git a/exporter/src/exporter/shell.cljs b/exporter/src/exporter/shell.cljs new file mode 100644 index 0000000000..841455b2ef --- /dev/null +++ b/exporter/src/exporter/shell.cljs @@ -0,0 +1,109 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.shell + "Shell and filesystem utilities of the new tree. + + Same operations as the legacy `app.util.shell`, adapted: nothing + runs at namespace load (no log level, no global tmpdir created the + first time anything requires this) and no promesa chains (`^:async` + + `await`). Paths are explicit everywhere: `tempfile` takes the + directory to create under, so the temp area is injected by whoever + owns it — the `:exporter/tmpdir` service at boot — instead of read + from a global. Legacy operations nobody calls did not survive the + move." + (:require + ["node:child_process" :as proc] + ["node:fs" :as fs] + ["node:path" :as path] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.time :as ct] + [app.common.uuid :as uuid] + [cuerdas.core :as str])) + +(def ^:const default-deletion-delay + (* 60 60 1)) ;; 1h + +(defn ensure-dir + "Creates `dir` (and parents) when missing, returning it. The one + place a directory comes into existence in the new tree; a blank path + is a wiring error, not an empty directory." + [dir] + (when (str/blank? dir) + (throw (ex/error :type :assertion + :code :tmpdir-path-missing + :hint "shell needs a directory to ensure"))) + (when-not (fs/existsSync dir) + (fs/mkdirSync dir #js {:recursive true})) + dir) + +(defn schedule-deletion + ([path] (schedule-deletion path default-deletion-delay)) + ([path delay] + (let [remove-path + (fn [] + (try + (when (fs/existsSync path) + (fs/rmSync path #js {:recursive true}) + (l/trc :hint "tempfile permanently deleted" :path path)) + (catch :default cause + (l/err :hint "error on deleting temporal file" + :path path + :cause cause)))) + scheduled-at + (-> (ct/now) (ct/plus #js {:seconds delay}))] + + (l/trc :hint "schedule tempfile deletion" + :path path + :scheduled-at (ct/format-inst scheduled-at)) + + (js/setTimeout remove-path (* delay 1000)) + path))) + +(defn tempfile + "A fresh path under `dir` that nothing else owns yet. + + Deletion is not scheduled here: the tmpdir registry tracks every path a + job writes and releases them on the settle, and the boot sweep removes + what a crashed process left. `schedule-deletion` stays for one-off + callers that own a path the registry does not know." + [dir & {:keys [prefix suffix] + :or {prefix "penpot." + suffix ".tmp"}}] + (loop [i 0] + (if (< i 1000) + (let [path (path/join dir (str/concat prefix (uuid/next) "-" i suffix))] + (if (fs/existsSync path) + (recur (inc i)) + path)) + (throw (ex/error :type :internal + :code :unable-to-locate-temporal-file + :hint "unable to find a tempfile candidate"))))) + +(defn ^:async move + [origin-path dest-path] + (await (.rename fs/promises origin-path dest-path))) + +(defn ^:async write-file + [fpath content] + (await (.writeFile fs/promises fpath content))) + +(defn ^:async read-file + [fpath] + (await (.readFile fs/promises fpath))) + +(defn ^:async run-cmd + [cmd & args] + (await + (js/Promise. + (fn [resolve reject] + (l/trace :fn :run-cmd :cmd cmd :args args) + (proc/execFile cmd (clj->js args) #js {:encoding "buffer"} + (fn [error stdout _stderr] + (if error + (reject error) + (resolve stdout)))))))) diff --git a/exporter/src/exporter/tmpdir.cljs b/exporter/src/exporter/tmpdir.cljs new file mode 100644 index 0000000000..eedaf81703 --- /dev/null +++ b/exporter/src/exporter/tmpdir.cljs @@ -0,0 +1,100 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.tmpdir + "The temp-file area as a system component (`:exporter/tmpdir`). + + Its config value names the directory (`{:path ...}`, read from the + environment at the wiring layer) plus the age that makes a leftover + an orphan (`:job-ttl`, seconds, same unit as the environment knob); + booting ensures the directory exists and cleans whatever a previous + process left behind, before anything that writes to it boots. The + running instance is the path itself, which renderers and runners + read straight from their config. Fail fast on a blank path instead + of failing the first render that writes. Named for what it owns, + not for the legacy namespace it shadows. + + The area also owns what jobs park in it: every path a job tracks is + removed on its release, so temp files die with the settle instead of + an hour later (or never, on a crash)." + (:require + ["node:fs/promises" :as fsp] + ["node:path" :as path] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [cuerdas.core :as str] + [exporter.shell :as shell] + [exporter.utils.system :as system])) + +(def ^:private managed-prefix "penpot.") + +(defonce ^:private tracked (atom {})) + +(defn track + "Registers `path` as owned by `job-id`, so it is removed when the job + settles. Returns the path, so it wraps the call that creates it." + [job-id path] + (when (and job-id path) + (swap! tracked update (str job-id) (fnil conj #{}) path)) + path) + +(defn- ^:async remove-path + [path] + (try + (await (fsp/rm path #js {:recursive true :force true})) + (catch :default cause + (l/warn :hint "unable to remove job temp file" :path path :cause cause)))) + +(defn ^:async release + "Removes every file the job owns. Called once the job reached a + terminal state and its result has already been uploaded, so nothing + else reads them. Unknown jobs resolve at once." + [job-id] + (let [paths (get @tracked (str job-id))] + (swap! tracked dissoc (str job-id)) + (when (seq paths) + (await (js/Promise.all (mapv remove-path paths))) + (l/dbg :hint "released job temp files" :job-id (str job-id) :count (count paths))) + nil)) + +(defn ^:async clean-orphans + "Removes managed temp files older than `job-ttl-s`. They can only be + leftovers of a previous process: every live one belongs to a job of + this process. Resolves the count it removed." + [tmpdir job-ttl-s] + (let [max-age (* 1000 job-ttl-s) + now (js/Date.now)] + (try + (let [entries (await (fsp/readdir tmpdir)) + removed (volatile! 0)] + (doseq [entry (filter #(str/starts-with? % managed-prefix) entries)] + (let [fpath (path/join tmpdir entry)] + (try + (let [^js stat (await (fsp/stat fpath))] + (when (> (- now (inst-ms (.-mtime stat))) max-age) + (await (remove-path fpath)) + (vswap! removed inc))) + (catch :default _ nil)))) + (when (pos? @removed) + (l/info :hint "removed orphaned export temp files" :count @removed)) + @removed) + (catch :default cause + (l/warn :hint "temp file cleanup failed" :cause cause) + 0)))) + +(defn- ^:async boot + [path job-ttl] + (let [dir (shell/ensure-dir path)] + (await (clean-orphans dir job-ttl)) + dir)) + +(defmethod system/init-key :exporter/tmpdir + [_ {:keys [path job-ttl]}] + (when (nil? job-ttl) + (throw (ex/error :type :assertion + :code :job-ttl-missing + :hint "tmpdir needs :job-ttl (seconds) in its config"))) + (boot path job-ttl)) diff --git a/exporter/src/app/util/mime.cljs b/exporter/src/exporter/util/mime.cljs similarity index 96% rename from exporter/src/app/util/mime.cljs rename to exporter/src/exporter/util/mime.cljs index 3f6ce62fbe..1ed719bfa3 100644 --- a/exporter/src/app/util/mime.cljs +++ b/exporter/src/exporter/util/mime.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.util.mime +(ns exporter.util.mime "Mimetype and file extension helpers." (:refer-clojure :exclude [get]) (:require @@ -29,5 +29,3 @@ :jpeg "image/jpeg" :png "image/png" :webp "image/webp")) - - diff --git a/exporter/src/app/util/object.cljs b/exporter/src/exporter/util/object.cljs similarity index 88% rename from exporter/src/app/util/object.cljs rename to exporter/src/exporter/util/object.cljs index 8c22bcfe92..27128df1a2 100644 --- a/exporter/src/app/util/object.cljs +++ b/exporter/src/exporter/util/object.cljs @@ -4,9 +4,9 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.util.object +(ns exporter.util.object "A collection of helpers for work with javascript objects." - (:refer-clojure :exclude [set! get get-in merge clone contains?]) + (:refer-clojure :exclude [set! get get-in merge update clone contains?]) (:require [cuerdas.core :as str])) @@ -46,24 +46,24 @@ [a] (js/Object.assign #js {} a)) -(defn merge! +(defn merge-into ([a b] (js/Object.assign a b)) ([a b & more] - (reduce merge! (merge! a b) more))) + (reduce merge-into (merge-into a b) more))) (defn merge ([a b] (js/Object.assign #js {} a b)) ([a b & more] - (reduce merge! (merge a b) more))) + (reduce merge-into (merge a b) more))) (defn set! [obj key value] (unchecked-set obj key value) obj) -(defn update! +(defn update [obj key f & args] (let [found (get obj key ::not-found)] (if-not (identical? ::not-found found) diff --git a/exporter/src/exporter/utils/system.cljs b/exporter/src/exporter/utils/system.cljs new file mode 100644 index 0000000000..2f90b9159d --- /dev/null +++ b/exporter/src/exporter/utils/system.cljs @@ -0,0 +1,243 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.utils.system + "Async-aware Integrant-style system lifecycle for the new tree. + + The shape mirrors `integrant.core`, trimmed down: a config map of + qualified keys to values, `ref`s between them, and two multimethods, + `init-key` and `halt-key`, that turn values into running components + and back. The dependency graph rides `weavejester.dependency`, the + same library Integrant uses. + + The difference is async: `init` and `halt` are `^:async`, every step + is awaited, and an `init-key` method may return a plain value or a + promise. `init` never throws synchronously; every failure, including + invalid config, surfaces as a rejected promise. + + Deliberately left out from Integrant: refsets, profiles, vars, + expand/converge, suspend/resume, composite keys, EDN reading and + namespace loading." + (:require + [clojure.set :as set] + [clojure.string :as str] + [clojure.walk :as walk] + [weavejester.dependency :as dep])) + +(defrecord Ref [key]) + +(defn ref + "Create a reference to a top-level key in a config map. The ref + resolves to the running instance, once its target has started." + [key] + (when-not (qualified-keyword? key) + (throw (ex-info (str "Invalid reference: " key ". Must be a qualified keyword.") + {:reason ::invalid-ref + :ref key}))) + (->Ref key)) + +(defn ref? + "Return true if its argument is a ref." + [x] + (instance? Ref x)) + +(defn- derived-from? + [key candidate] + (or (= key candidate) (isa? key candidate))) + +(defn find-derived + "Return a seq of all entries in a map, m, where the key is the + candidate key or derived from it. If there are no matching keys, nil + is returned." + [m k] + (seq (filter #(or (= (key %) k) (derived-from? (key %) k)) m))) + +(defn- find-derived-1 + [m k] + (let [kvs (find-derived m k)] + (when (next kvs) + (throw (ex-info (str "Ambiguous key: " k ". Found multiple candidates: " + (str/join ", " (map key kvs))) + {:reason ::ambiguous-key + :key k + :matching (mapv key kvs)}))) + (first kvs))) + +(defn- depth-search + [pred? coll] + (filter pred? (tree-seq coll? seq coll))) + +(defn- find-derived-refs + [config v] + (->> (depth-search ref? v) + (map :key) + (mapcat #(map key (find-derived config %))))) + +(defn- dependency-graph + [config] + (reduce-kv (fn [g k v] + (reduce #(dep/depend %1 k %2) g (find-derived-refs config v))) + (dep/graph) + config)) + +(defn- key-comparator + [graph] + (dep/topo-comparator #(compare (str %1) (str %2)) graph)) + +(defn- config-keyset + [config ks] + (set (mapcat #(map key (find-derived config %)) ks))) + +(defn- validate-init + "Check the config and the requested keys, and return the relevant + keys in init order: requested, plus their transitive dependencies. + A circular dependency surfaces the `weavejester.dependency` ex-info." + [config ks] + (doseq [k (keys config)] + (when-not (qualified-keyword? k) + (throw (ex-info (str "Invalid config key: " k ". Keys must be qualified keywords.") + {:reason ::invalid-config + :key k})))) + (doseq [k ks] + (when-not (seq (find-derived config k)) + (throw (ex-info (str "Unknown key: " k ". It matches nothing in the config.") + {:reason ::unknown-key + :key k})))) + (let [graph (dependency-graph config) + keyset (config-keyset config ks) + relevant (select-keys config (set/union keyset + (dep/transitive-dependencies-set graph keyset))) + ref-keys (distinct (map :key (depth-search ref? (vals relevant)))) + missing (filterv #(nil? (find-derived relevant %)) ref-keys)] + (when (seq missing) + (throw (ex-info (str "Missing definitions for refs: " (str/join ", " missing)) + {:reason ::missing-refs + :config config + :missing-refs missing}))) + (doseq [k ref-keys] + (find-derived-1 relevant k)) + (let [comparator (key-comparator graph)] + (vec (sort comparator (keys relevant)))))) + +(defn- resolve-refs + [sys v] + (walk/postwalk #(if (ref? %) (val (find-derived-1 sys (:key %))) %) v)) + +(defmulti init-key + "Turn a config value associated with a key into a running component. + May return a plain value or a promise; the system awaits it either way." + {:arglists '([key config])} + (fn [key _config] key)) + +(defmethod init-key :default + [k _config] + (throw (ex-info (str "No init-key method for " k) + {:reason ::missing-init-key + :key k}))) + +(defmulti halt-key + "Stop the running component associated with a key. Receives the + running instance returned by `init-key`, so anything halt needs must + travel inside it. May return a plain value or a promise; the system + awaits it either way. Must be idempotent." + {:arglists '([key instance])} + (fn [key _instance] key)) + +(defmethod halt-key :default + [_instance-key _instance] + nil) + +(defn ^:private ^:async build-key + [sys k v] + (try + (assoc sys k (await (init-key k (resolve-refs sys v)))) + (catch :default cause + (throw (ex-info (str "Error on key " k " when building system") + {:reason ::init-failed + :key k + :value v + :system sys} + cause))))) + +(defn- halt-order + [sys ks] + (let [config (or (::origin (meta sys)) {}) + graph (dependency-graph config) + keyset (config-keyset config ks) + comparator (key-comparator graph)] + (->> (set/union keyset (dep/transitive-dependents-set graph keyset)) + (filter #(contains? sys %)) + (sort comparator) + (reverse) + (vec)))) + +(defn ^:private ^:async halt-keys + "Halt every key in ks, collecting failures instead of stopping at the + first one. Returns a vector of `{:key :cause}` maps." + [sys ks] + (loop [failures [] + ks ks] + (if (seq ks) + (let [k (first ks) + failures (try + (await (halt-key k (sys k))) + failures + (catch :default cause + (conj failures {:key k :cause cause})))] + (recur failures (rest ks))) + failures))) + +(defn ^:private ^:async halt-silently + [sys] + (when (map? sys) + (await (halt-keys sys (halt-order sys (keys sys))))) + nil) + +(defn ^:async init + "Turn a config map into a running system map. Keys start in + dependency order, each one awaited, with refs resolved to the running + instances. With two args, only those keys start, plus whatever they + depend on. + + Returns a promise of the system map. If a component fails to start, + what already started is halted in reverse order, and the promise + rejects with the original failure carrying `:reason ::init-failed`, + the offending `:key`, and the partial `:system`." + ([config] + (init config (keys config))) + ([config ks] + (assert (map? config) "init needs a config map") + (let [order (validate-init config ks)] + (try + (loop [sys (with-meta {} {::origin config}) + ks order] + (if (seq ks) + (recur (await (build-key sys (first ks) (config (first ks)))) + (rest ks)) + sys)) + (catch :default cause + (await (halt-silently (:system (ex-data cause)))) + (throw cause)))))) + +(defn ^:async halt + "Halt a running system map in reverse dependency order, awaiting each + component. With two args, halt those keys plus whatever depends on them. + + Returns a promise of nil. Every component is given its chance to stop: + failures are collected, and once everything ran, the promise rejects + with `:reason ::halt-failed` and the `:failures` vector." + ([sys] + (halt sys (keys sys))) + ([sys ks] + (when-not (and (map? sys) (::origin (meta sys))) + (throw (ex-info "halt needs a system map built by init." + {:reason ::not-a-system}))) + (let [failures (await (halt-keys sys (halt-order sys ks)))] + (when (seq failures) + (throw (ex-info (str "Failed to halt " (count failures) " component(s): " + (str/join ", " (map :key failures))) + {:reason ::halt-failed + :failures failures})))))) diff --git a/exporter/src/app/wasm.cljs b/exporter/src/exporter/wasm.cljs similarity index 83% rename from exporter/src/app/wasm.cljs rename to exporter/src/exporter/wasm.cljs index 88ae874009..fb1dc60186 100644 --- a/exporter/src/app/wasm.cljs +++ b/exporter/src/exporter/wasm.cljs @@ -4,13 +4,14 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.wasm +(ns exporter.wasm "Headless driver for the render-wasm module under Node: the GPU-free counterpart of `app.render-wasm.api`. Loads the emscripten artifact, boots it via `init_headless`, and exposes font provisioning + shape rendering. Serialization is reused from the portable render-wasm leaves, so this namespace owns only the Node runtime and the headless render calls. + There are no promesa chains in this namespace. Requires render-wasm built with `-sENVIRONMENT=web,node`." (:require @@ -23,10 +24,9 @@ [app.common.render-wasm.serializers :as sr] [app.common.render-wasm.wasm :as wasm] [app.common.uuid :as uuid] - ;; Required for side effects: binds the generated enums. - [app.wasm.enums] [cuerdas.core :as str] - [promesa.core :as p] + ;; Required for side effects: binds the generated enums. + [exporter.wasm.enums] [shadow.esm :refer [dynamic-import]])) (def ^:private default-viewport-width 1920) @@ -44,10 +44,6 @@ path in devenv and inside the bundle, so it is a constant." "resources/wasm") -(def image-cache-size - "Byte budget the image store is trimmed to between requests." - (* 256 1024 1024)) - (defn- read-result-bytes "Reads `len` bytes from the WASM heap starting at `offset`, copying them out (via `.slice`) before the buffer is freed." @@ -56,39 +52,35 @@ ;; --- MODULE LIFECYCLE -(defn init! +(defn ^:async init "Loads the render-wasm artifact under Node and boots it headless. Sets the shared `wasm/internal-module` so the portable serialization leaves work. Idempotent-ish: callers should hold the returned module." - ([] (init! default-viewport-width default-viewport-height)) + ([] (init default-viewport-width default-viewport-height)) ([width height] - (let [dir artifact-dir - js-path (path/resolve dir "render-wasm.js") - wasm-path (path/resolve dir "render-wasm.wasm") + (let [dir artifact-dir + js-path (path/resolve dir "render-wasm.js") + wasm-path (path/resolve dir "render-wasm.wasm") wasm-bytes (fs/readFileSync wasm-path)] (l/info :hint "loading render-wasm (headless)" :js js-path) ;; shadow-cljs :esm — use its dynamic-import helper (raw `js/import` ;; compiles to an undefined `import$`). - (->> (dynamic-import (str "file://" js-path)) - (p/mcat - (fn [mod] - (let [factory (unchecked-get mod "default")] - (factory - #js {;; Bypass the web fetch loader: instantiate from local bytes. - :instantiateWasm - (fn [imports success] - (-> (js/WebAssembly.instantiate wasm-bytes imports) - (.then (fn [result] (success (.-instance result))))) - #js {}) - :locateFile (fn [p] (path/resolve dir p)) - :printErr (fn [s] (l/warn :wasm s))})))) - (p/fmap - (fn [module] - (set! wasm/internal-module module) - (h/call module "_init_headless" width height) - (set! wasm/context-initialized? true) - (l/info :hint "render-wasm headless module ready" :width width :height height) - module)))))) + (let [mod (await (dynamic-import (str "file://" js-path))) + factory (unchecked-get mod "default") + module (await (factory + #js {;; Bypass the web fetch loader: instantiate from local bytes. + :instantiateWasm + (fn [imports success] + (-> (js/WebAssembly.instantiate wasm-bytes imports) + (.then (fn [result] (success (.-instance result))))) + #js {}) + :locateFile (fn [p] (path/resolve dir p)) + :printErr (fn [s] (l/warn :wasm s))}))] + (set! wasm/internal-module module) + (h/call module "_init_headless" width height) + (set! wasm/context-initialized? true) + (l/info :hint "render-wasm headless module ready" :width width :height height) + module)))) ;; --- FONT PROVISIONING (on demand, mirrors the browser) @@ -130,7 +122,7 @@ (d/distinct-xf font-key)) shape-ids)) -(defn store-font! +(defn store-font "Uploads one font's TTF bytes into the WASM font store, keyed by the family (uuid quartet + weight + style). `font-bytes` is a Uint8Array/Buffer. @@ -147,12 +139,12 @@ (aget id 0) (aget id 1) (aget id 2) (aget id 3) weight style (boolean emoji?) (boolean fallback?)))) -(defn store-font-url! +(defn store-font-url "Registers the public URL a font family was loaded from. The SVG export emits one `@font-face` per family from these, and skips families without one, so - this must run for every family `store-font!` uploads. + this must run for every family `store-font` uploads. - Does NOT call `mem/free`, for the same reason as `store-font!`." + Does NOT call `mem/free`, for the same reason as `store-font`." [{:keys [id weight style]} url] (let [bytes (js/Buffer.from url "utf-8") ptr (mem/alloc (.-byteLength bytes))] @@ -161,16 +153,16 @@ (aget id 0) (aget id 1) (aget id 2) (aget id 3) weight style))) -(defn clear-fonts! +(defn clear-fonts "Resets the WASM font store. Must be called once per render request because the shared module would otherwise accumulate fonts across requests." [] (h/call wasm/internal-module "_clear_fonts")) -(defn update-text-layout! +(defn update-text-layout "Recomputes a text shape's layout with the currently provisioned fonts. Text is laid out at serialize time using the fallback font (real fonts aren't uploaded - yet), so this must run again after `provision-fonts!` or glyph metrics/line + yet), so this must run again after `provision-fonts` or glyph metrics/line breaks are wrong. Forced, because provisioning a font changes nothing `update_layout` keys on: @@ -191,13 +183,13 @@ (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3) false))))) -(defn store-image-url! +(defn store-image-url "Registers the public URL an image was loaded from. The SVG export emits linked `` from these, and falls back to Skia base64 when missing, so this should run for every media id the scene references (including already-cached images). - Does NOT call `mem/free`, for the same reason as `store-font-url!`." + Does NOT call `mem/free`, for the same reason as `store-font-url`." [image-id url] (when (and (some? url) (not (str/blank? url))) (let [bytes (js/Buffer.from url "utf-8") @@ -207,7 +199,7 @@ (h/call wasm/internal-module "_store_image_url" (aget quart 0) (aget quart 1) (aget quart 2) (aget quart 3))))) -(defn store-image! +(defn store-image "Uploads one image's *encoded* bytes (PNG/JPEG — Skia decodes, no WebGL) into the WASM image store via `_store_image`. Buffer layout matches the Rust reader: [shape uuid 16][image uuid 16][is_thumbnail u32][encoded bytes]. Images are @@ -239,28 +231,30 @@ (.set heap img-u8 (+ ptr 36)) (h/call module "_store_image"))) -(defn evict-images! +(defn evict-images "Evicts least-recently-used images until the store retains at most `max-bytes` bytes. Returns the number evicted." [max-bytes] (h/call wasm/internal-module "_evict_images_to_budget" max-bytes)) -(defn provision-fonts! +(defn ^:async provision-fonts "Resolves and uploads every font needed by `shape-ids`, each family fetched once. `resolve-font` is an injected fn of the family map -> promise of TTF bytes (or nil to skip); optional `font-url` is a fn of the family map -> the public URL those bytes came from. This keeps the font *source* (gfonts proxy / custom assets / backend) out of the driver." [shape-ids resolve-font & {:keys [font-url]}] - (->> (fonts-for-shapes shape-ids) - (map (fn [family] - (->> (resolve-font family) - (p/fmap (fn [bytes] - (when bytes - (store-font! family bytes) - (when-let [url (when font-url (font-url family))] - (store-font-url! family url)))))))) - (p/all))) + (let [families (fonts-for-shapes shape-ids)] + (await (js/Promise.all + (mapv (^:async fn [family] + (let [bytes (await (resolve-font family))] + (when bytes + (store-font family bytes) + (when-let [url (when font-url (font-url family))] + (store-font-url family url))) + nil)) + families))) + nil)) ;; --- RENDER diff --git a/exporter/src/app/wasm/enums.cljs b/exporter/src/exporter/wasm/enums.cljs similarity index 96% rename from exporter/src/app/wasm/enums.cljs rename to exporter/src/exporter/wasm/enums.cljs index 5ecd5ebb43..68d9e12884 100644 --- a/exporter/src/app/wasm/enums.cljs +++ b/exporter/src/exporter/wasm/enums.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.wasm.enums +(ns exporter.wasm.enums "Binds this build's generated enums into the shared bridge. `shared.js` is emitted next to this file by `render-wasm/build export` and is diff --git a/exporter/src/exporter/wasm/pool.cljs b/exporter/src/exporter/wasm/pool.cljs new file mode 100644 index 0000000000..71319da673 --- /dev/null +++ b/exporter/src/exporter/wasm/pool.cljs @@ -0,0 +1,190 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.wasm.pool + "Pool of headless render workers. + + Mirrors `exporter.browser`: a `generic-pool` whose objects are + `worker_threads` instead of browsers, so acquisition and eviction + behave the same way for both render backends. A worker is expensive + to build (it boots its own render-wasm module), hence the pooling. + + Acquisition is not capped: the admission scheduler is the + backpressure, and the idle watchdog guarantees a wedged worker gives + its slot back. + + Workers run the same bundle as the main thread. There is always at + least one worker, since a headless render has nowhere else to go. + + Unlike the browser service, the running instance is a map: the pool + plus the silence timeout the renders await on it. There are no + promesa chains in this namespace: the pool speaks native promises, + and `await` covers the rest." + (:require + ["generic-pool" :as gp] + ["node:path" :as path] + ["node:process" :as proc] + ["node:worker_threads" :as wt] + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [exporter.utils.system :as system] + [exporter.wasm :as-alias wasm])) + +(l/set-level! :info) + +(def defaults + "Static pool defaults. The sizing (`:max`) mirrors the worker + concurrency default; the wiring layer passes the env-derived values." + {:max 2 + :min 1 + :timeout-ms 300000 + :ready-timeout-ms 60000 + :eviction-interval 30000 + :eviction-batch 2 + :idle-timeout 300000}) + +(defn await-ready + "Resolves the worker once it posts its ready handshake. A worker + that stays silent too long is terminated and rejected with + `:worker-not-ready`; a boot error rejects with its own cause." + [worker timeout-ms] + (js/Promise. + (fn [resolve reject] + (let [timer (js/setTimeout + (fn [] + (l/error :hint "render worker did not become ready") + (.terminate ^js worker) + (reject (ex/error :type :internal + :code :worker-not-ready + :hint "render worker did not become ready"))) + timeout-ms)] + (.on ^js worker "error" + (fn [cause] + (l/error :hint "render worker error" :cause cause) + (unchecked-set worker "__alive" false) + ;; Rejecting after `resolve` is a no-op, so this is safe + ;; for the errors that arrive once the worker is pooled. + (js/clearTimeout timer) + (reject cause))) + (.on ^js worker "exit" + (fn [code] + (l/info :hint "render worker exited" :code code) + (unchecked-set worker "__alive" false))) + ;; Not `.once`: a stray message before the handshake would + ;; consume the listener and hang the worker until the timeout. + (letfn [(on-ready [data] + (when (= "ready" (unchecked-get data "type")) + (js/clearTimeout timer) + (.off ^js worker "message" on-ready) + (resolve worker)))] + (.on ^js worker "message" on-ready)))))) + +(defn- default-script + [] + (path/resolve (aget (.-argv proc/default) 1))) + +(defn- default-create-worker + [script ready-timeout-ms] + (fn [] + (await-ready (new wt/Worker script) ready-timeout-ms))) + +(defn- ^:async create-instance + [create-worker id-counter] + (let [worker (await (create-worker)) + id (swap! id-counter inc)] + (l/info :origin "factory" :action "create" :worker-id id) + (unchecked-set worker "__id" id) + (unchecked-set worker "__alive" true) + worker)) + +(defn- destroy-instance + [worker] + (l/info :origin "factory" :action "destroy" + :worker-id (unchecked-get worker "__id")) + (unchecked-set worker "__alive" false) + (.terminate ^js worker)) + +(defn- validate-instance + [worker] + (true? (unchecked-get worker "__alive"))) + +(defn- make-factory + [create-worker] + (let [id-counter (atom 0)] + #js {:create (fn [] (create-instance create-worker id-counter)) + :destroy destroy-instance + :validate validate-instance})) + +(defmethod system/init-key ::wasm/pool + [_ cfg] + (let [opts (merge defaults (d/without-nils cfg)) + max-workers (:max opts) + script (or (:script opts) (default-script)) + create-worker (or (:create-worker opts) + (default-create-worker script (:ready-timeout-ms opts)))] + (l/info :hint "initializing render worker pool" :max max-workers) + {:pool (gp/createPool (make-factory create-worker) + #js {:max max-workers + :min (min max-workers (:min opts)) + :testOnBorrow true + :evictionRunIntervalMillis (:eviction-interval opts) + :numTestsPerEvictionRun (:eviction-batch opts) + :idleTimeoutMillis (:idle-timeout opts)}) + :timeout-ms (:timeout-ms opts)})) + +(defn- ^:async drain-pool + [pool] + (await (.drain ^js pool)) + (await (.clear ^js pool)) + nil) + +(defmethod system/halt-key ::wasm/pool + [_ instance] + (when-let [pool (:pool instance)] + (l/info :hint "finalizing render worker pool") + (drain-pool pool))) + +(defn- pool? + "A raw generic-pool: anything lendable through acquire and release." + [x] + (and (some? x) + (fn? (unchecked-get x "acquire")) + (fn? (unchecked-get x "release")))) + +(defn ^:async run + "Lends one worker: resolves the pool from `cfg` (the `::wasm/pool` + service instance, or a raw pool), acquires a worker, applies `f` to + the cfg carrying it under `::worker` plus any extra `args`, and + returns the worker — or destroys it when `f` fails, the way a failed + transaction rolls back instead of returning its connection: a failed + worker may be aborted or mid-write, and a terminated worker cannot + come back." + [cfg f & args] + (let [cfg (if (pool? cfg) {::wasm/pool {:pool cfg}} cfg) + pool (:pool (::wasm/pool cfg))] + (when-not (pool? pool) + (throw (ex/error :type :assertion + :code :invalid-pool-cfg + :hint "no pool in cfg"))) + (let [worker (await (.acquire ^js pool))] + (try + (let [result (await (apply f (assoc cfg ::worker worker) args))] + (await (.release ^js pool worker)) + result) + (catch :default cause + (try + (await (.destroy ^js pool worker)) + (catch :default cause' + (l/warn :hint "render worker destroy failed" + :cause cause'))) + (throw cause)))))) + +(defn terminate + [worker] + (when worker + (unchecked-set worker "__alive" false) + (.terminate worker))) diff --git a/exporter/src/exporter/wasm/render.cljs b/exporter/src/exporter/wasm/render.cljs new file mode 100644 index 0000000000..c3629cdc49 --- /dev/null +++ b/exporter/src/exporter/wasm/render.cljs @@ -0,0 +1,523 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.wasm.render + "Headless render pipeline: renders exports with the render-wasm Skia pipeline, + with no browser and no WebGL. + + Per request: fetch scene (get-page RPC) -> serialize -> provision fonts and + images -> relayout text with the real fonts -> render each object. + + This runs inside a render worker (one WASM design state per worker), so the + synchronous Skia calls never block the main thread. Everything the pipeline + needs beyond the export itself travels in `env` + (`:internal-uri`, `:public-uri`, `:tmpdir`, `:image-cache-size`): the + worker thread reads no config and no globals. + There are no promesa chains in this namespace. + + Handles png/jpeg/webp (Skia encodes all three), pdf and svg." + (:require + ["node:fs" :as fs] + ["undici" :as http] + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.common.fonts :as cfnt] + ;; Required for side effects: these register the transit read handlers and + ;; deftype impls the `get-page` response is decoded into. + [app.common.geom.matrix] + [app.common.geom.point] + [app.common.geom.rect] + [app.common.logging :as l] + [app.common.transit :as t] + [app.common.types.fills.impl] + [app.common.types.objects-map] + [app.common.types.path.impl] + [app.common.types.shape] + [app.common.types.shape.images :as images] + [app.common.uri :as u] + [app.common.uuid :as uuid] + [cuerdas.core :as str] + [exporter.shell :as shell] + [exporter.util.mime :as mime] + [exporter.wasm :as wasm] + [exporter.wasm.serialize :as serialize])) + +(def fetch + "The network entry point of the pipeline: a plain def, redefined by + the tests, which never need a network." + http/fetch) + +;; --- module lifecycle (one shared, lazily-initialized instance) + +(defonce ^:private module* (atom nil)) + +(defn- ^:async ensure-module + [] + (or @module* + (reset! module* (await (wasm/init))))) + +;; --- backend endpoints +;; +;; Every fetch targets the internal endpoint (falling back to public-uri), +;; in a deployment the exporter reaches the backend over the container network + +(defn internal-uri + "Absolute URI for `path` on the internal (backend) endpoint." + [env path] + (-> (:internal-uri env) + (u/ensure-path-slash) + (u/join path) + (str))) + +(defn public-uri + "Absolute URI for `path` on the public endpoint. Whoever opens an exported SVG + resolves its `@font-face` sources, so those cannot use the internal endpoint." + [env path] + (-> (:public-uri env) + (u/ensure-path-slash) + (u/join path) + (str))) + +(defn- share-uri + "An asset URI plus the share the export came through, when it has one: + the asset endpoints decide permissions by that parameter, so a share + viewer without it gets a 404 and the image or font silently vanishes. + The URI helpers do the merging and encoding, so the value is escaped + and an existing query survives." + [uri share-id] + (cond-> uri + (some? share-id) (u/append-query-param "share-id" (str share-id)))) + +(defn error-detail + "Node's fetch reports every transport failure as a bare `TypeError: fetch + failed`; the actual reason (TLS rejection, DNS, ECONNREFUSED) is buried in a + nested `cause` chain that the logger does not print. Flattens the chain into + one readable string." + [cause] + (->> (iterate (fn [^js e] (unchecked-get e "cause")) cause) + (take-while some?) + (take 5) + (map (fn [^js e] + (let [code (unchecked-get e "code") + msg (or (unchecked-get e "message") (str e))] + (if code (str code ": " msg) msg)))) + (str/join " <- "))) + +(defn- ^:async request + "`undici/fetch` that fails with an ex-info carrying the target uri and the + unwrapped cause chain, so a failed request says what actually went wrong and + against which endpoint." + [uri opts] + (try + (await (fetch uri opts)) + (catch :default cause + (throw (ex-info "http fetch failed" + {:uri uri :detail (error-detail cause)} + cause))))) + +(defn- explain + "Log-friendly reason for `cause`: the detail `request` already attached, or a + freshly unwrapped chain for anything else (WASM aborts, decode errors)." + [cause] + (or (:detail (ex-data cause)) + (error-detail cause))) + +(defn- rpc-headers + "Headers for backend RPC calls: transit in, Bearer out. No shared key + here: those endpoints authenticate the session, not the worker." + [_env token] + #js {"Content-Type" "application/transit+json" + "Authorization" (str "Bearer " token)}) + +(defn- asset-headers + "Auth headers for `/assets/*`. Cookie, not Bearer: those endpoints redirect to + a presigned S3/minio URL, and a Bearer header makes S3 400 (\"multiple + authentication types\")." + [_env token] + #js {"Cookie" (str "auth-token=" token)}) + +;; --- shape bundle fetch (backend RPC) + +(defn ^:async fetch-objects + "Fetches the exported roots and their children from the backend via the + `get-page` RPC (`:object-id`, as the browser render path does), using the + same auth the exporter uses elsewhere (management key + bearer)." + [env {:keys [file-id page-id share-id token objects]}] + (let [headers (rpc-headers env token) + root-ids (into #{} (map :id) objects) + body (t/encode-str (cond-> {:file-id file-id + :page-id page-id} + (seq root-ids) (assoc :object-id root-ids) + share-id (assoc :share-id share-id))) + uri (internal-uri env "api/rpc/command/get-page")] + (l/dbg :hint "wasm render: get-page" + :uri uri + :file-id (str file-id) + :page-id (str page-id) + :roots (count root-ids)) + (let [resp (await (request uri #js {:method "POST" :headers headers :body body}))] + (if (= 200 (.-status resp)) + (:objects (t/decode-str (await (.text resp)))) + (let [resp-body (await (.text resp))] + (l/error :hint "wasm render: get-page failed" + :uri uri + :status (.-status resp) + :body resp-body) + (throw (ex-info "get-page failed" + {:status (.-status resp) + :body resp-body}))))))) + +;; --- font resolution +;; +;; The text serializer keeps each font's real uuid, so `wasm/fonts-for-shape` +;; reports it. Custom (team) fonts resolve through the file's font variants, +;; google fonts through the shared `app.common.fonts` catalog; builtin +;; fonts through its bundled family + the frontend's static `/fonts/`. + +(defn ^:async fetch-font-variants + "Team (custom) font variants for the file, or nil — a failure here degrades + to fallback fonts, it does not fail the export." + [env {:keys [file-id share-id token]}] + (let [headers (rpc-headers env token) + body (t/encode-str (cond-> {:file-id file-id} + share-id (assoc :share-id share-id))) + uri (internal-uri env "api/rpc/command/get-font-variants")] + (try + (let [resp (await (request uri #js {:method "POST" :headers headers :body body}))] + (when (= 200 (.-status resp)) + (t/decode-str (await (.text resp))))) + (catch :default cause + (l/warn :hint "wasm render: get-font-variants failed" + :uri uri :detail (explain cause) :cause cause) + nil)))) + +(defn- ^:async fetch-ttf-bytes + "Downloads a TTF (an ArrayBuffer, or nil). A failure here degrades to + fallback fonts, it does not fail the export." + [uri opts] + (try + (let [resp (await (request uri opts))] + (when (= 200 (.-status resp)) + (await (.arrayBuffer resp)))) + (catch :default cause + (l/warn :hint "wasm render: font fetch failed" + :uri uri :detail (explain cause) :cause cause) + nil))) + +;; TTF bytes cached for the process lifetime, keyed by whatever identifies the +;; variant (a gfont id+weight+style, a builtin file name). +(defonce ^:private font-bytes* (atom {})) + +(defn- ^:async cached-ttf-bytes + [cache-key fetch-fn] + (if-let [bytes (get @font-bytes* cache-key)] + bytes + (let [buf (await (fetch-fn))] + (when buf (swap! font-bytes* assoc cache-key buf)) + buf))) + +(defn- ^:async fetch-asset-bytes + [env asset-id token share-id] + (fetch-ttf-bytes (share-uri (internal-uri env (str "assets/by-id/" asset-id)) share-id) + #js {:method "GET" :headers (asset-headers env token)})) + +(defn- ^:async fetch-gfont-bytes + [env ttf-url] + (fetch-ttf-bytes (cfnt/gstatic->proxy-url ttf-url (internal-uri env "internal/gfonts/font")) + #js {:method "GET"})) + +(defn- ^:async fetch-builtin-font-bytes + [env ttf-file] + (cached-ttf-bytes ttf-file #(fetch-ttf-bytes (internal-uri env (str "fonts/" ttf-file)) + #js {:method "GET"}))) + +(defn family-uuid + [id] + (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3))) + +(defn find-variant + "Custom variant for a family: uuid+weight+style first, degrading to + uuid+weight then uuid." + [variants font-uuid weight style] + (let [style-str (if (zero? style) "normal" "italic")] + (or (d/seek (fn [v] (and (= (:font-id v) font-uuid) + (= (:font-weight v) weight) + (= (name (:font-style v)) style-str))) + variants) + (d/seek (fn [v] (and (= (:font-id v) font-uuid) + (= (:font-weight v) weight))) + variants) + (d/seek (fn [v] (= (:font-id v) font-uuid)) variants)))) + +(defn make-resolve-font + "Builds a `resolve-font` fn (family map -> TTF bytes). Custom + variants first; the bundled fonts for `uuid/zero`, which is what + `font-id->uuid` maps every builtin family to; google catalog otherwise." + [env variants params] + (^:async fn [{:keys [id weight style]}] + (let [font-uuid (family-uuid id) + variant (find-variant variants font-uuid weight style)] + (cond + (:ttf-file-id variant) + (await (fetch-asset-bytes env (:ttf-file-id variant) + (:token params) (:share-id params))) + + (= uuid/zero font-uuid) + (await (fetch-builtin-font-bytes env (cfnt/resolve-ttf-file weight style))) + + :else + (if-let [gurl (cfnt/resolve-ttf-url font-uuid weight style)] + (await (fetch-gfont-bytes env gurl)) + nil))))) + +(defn- make-font-url + "Builds a `font-url` fn (family map -> public URL of its TTF), the same + sources `make-resolve-font` downloads from but addressed publicly. The SVG + export emits one `@font-face` per family from these." + [env variants share-id] + (fn [{:keys [id weight style]}] + (let [font-uuid (family-uuid id) + variant (find-variant variants font-uuid weight style)] + (cond + (:ttf-file-id variant) + (share-uri (public-uri env (str "assets/by-id/" (:ttf-file-id variant))) + share-id) + + (= uuid/zero font-uuid) + (public-uri env (str "fonts/" (cfnt/resolve-ttf-file weight style))) + + :else + (some-> (cfnt/resolve-ttf-url font-uuid weight style) + (cfnt/gstatic->proxy-url (public-uri env "internal/gfonts/font"))))))) + +;; --- fallback fonts (emoji + per-script noto fonts) +;; +;; Emoji and non-latin scripts render through fallback families, not through +;; any span's font family, so `wasm/fonts-for-shape` never reports them and the +;; provisioning above never uploads them. Must run per request, since +;; `clear-fonts` empties the store; the TTF bytes stay cached per process. + +(defn scene-fallback-fonts + "Fallback font descriptors needed by the scene's text. Deduped because + several languages map to one noto family and provisioning is concurrent — + otherwise they all miss the byte cache at once and refetch the same TTF." + [scene] + (let [texts (for [shape (vals scene) + :when (= :text (:type shape)) + node (or (some->> (:content shape) (tree-seq :children :children)) []) + :let [text (:text node)] + :when (string? text)] + text) + emoji? (boolean (some cfnt/contains-emoji? texts)) + langs (reduce cfnt/collect-used-languages #{} texts)] + (distinct + (cond-> (cfnt/add-noto-fonts [] langs) + emoji? (cfnt/add-emoji-font))))) + +(defn- ^:async fetch-fallback-font-bytes + "Downloads one fallback font's TTF. Cached by the whole variant, not just + `font-id`: `resolve-ttf-url` picks a different TTF per weight/style, so a + font-id-only key would serve the first downloaded variant for every other one." + [env {:keys [font-id weight style]}] + (if-let [ttf-url (some-> (cfnt/gfont-id->uuid font-id) (cfnt/resolve-ttf-url weight style))] + (cached-ttf-bytes [font-id weight style] #(fetch-gfont-bytes env ttf-url)) + nil)) + +(defn- ^:async provision-fallback-fonts + [env scene] + (await (js/Promise.all + (mapv (^:async fn [{:keys [font-id weight style is-emoji is-fallback] :as font}] + (if-let [font-uuid (cfnt/gfont-id->uuid font-id)] + (let [buf (await (fetch-fallback-font-bytes env font))] + (if buf + (wasm/store-font {:id (uuid/get-u32 font-uuid) + :weight weight + :style style + :emoji? (boolean is-emoji) + :fallback? (boolean is-fallback)} + buf) + (l/warn :hint "wasm render: fallback font unavailable" + :font-id font-id))) + nil)) + (scene-fallback-fonts scene)))) + nil) + +;; --- image resolution +;; +;; Image fills reference file-media ids; the encoded bytes go straight to +;; `_store_image` (Skia decodes, no WebGL), keyed by media uuid so this happens +;; once per request rather than per rendered object. + +(defn ^:async fetch-file-media-bytes + "Downloads an image fill's encoded bytes by file-media id, or nil when + they cannot be had: the object renders without its image either way. + The share the export came through rides in the URI: a share viewer is + allowed through it, and without it the fetch is a 404." + [env media-id token share-id] + (let [headers (asset-headers env token) + uri (share-uri (internal-uri env (str "assets/by-file-media-id/" media-id)) + share-id)] + (try + (let [resp (await (request uri #js {:method "GET" :headers headers}))] + (if (= 200 (.-status resp)) + (await (.arrayBuffer resp)) + (do + (l/warn :hint "wasm render: image fetch non-200" + :media-id (str media-id) + :uri uri + :status (.-status resp)) + nil))) + (catch :default cause + (l/warn :hint "wasm render: image fetch failed" + :media-id (str media-id) :uri uri + :detail (explain cause) :cause cause) + nil)))) + +(defn- ^:async provision-images + "Fetches and stores every image the scene references (shape, stroke and + text-span fills, enumerated by `app.common.types.shape.images`). Unlike fonts, + the image store is not reset per request, so already-held images are skipped + and repeated exports of a file reuse them. + + Always registers a public media URL for each id so SVG export can emit linked + `` even when the encoded bytes were already cached." + [env scene params] + (let [all-ids (images/scene-image-ids scene) + new-ids (remove wasm/image-cached? all-ids)] + (l/dbg :hint "wasm render: provisioning images" + :total (count all-ids) + :cached (- (count all-ids) (count new-ids))) + (doseq [image-id all-ids] + (wasm/store-image-url image-id + (share-uri (public-uri env (str "assets/by-file-media-id/" image-id)) + (:share-id params)))) + (await (js/Promise.all + (mapv (^:async fn [image-id] + (let [buf (await (fetch-file-media-bytes env image-id + (:token params) + (:share-id params)))] + (if buf + (do + (l/dbg :hint "wasm render: image stored" + :media-id (str image-id) + :bytes (.-byteLength ^js buf)) + (wasm/store-image image-id buf)) + (l/warn :hint "wasm render: image unavailable" + :media-id (str image-id)))) + nil) + new-ids))) + nil)) + +(defn- relayout-text + "Recomputes layout for every text shape, once the real fonts are provisioned + (serialize-time layout used the fallback)." + [scene] + (doseq [shape (vals scene) + :when (= :text (:type shape))] + (wasm/update-text-layout (:id shape)))) + +;; --- render + +(defn- check-cancelled + "Cancellation is cooperative: a render already inside Skia cannot be + interrupted, so the flag is only observed between objects. Killing a job + mid-object is the caller's job (terminating the worker)." + [{:keys [cancelled?] :as _params}] + (when (and cancelled? (cancelled?)) + (ex/raise :type :internal + :code :job-cancelled + :hint "export job was cancelled"))) + +(defn- render-object-bytes + [type id scale] + (case type + :pdf (let [bytes (wasm/render-shape-pdf id)] + (l/dbg :hint "PDF generated via Skia (render-wasm headless)" + :object-id (str id) + :backend "skia-wasm" + :bytes (.-length bytes)) + bytes) + :svg (let [bytes (wasm/render-shape-svg id scale)] + (l/dbg :hint "SVG generated via Skia (render-wasm headless)" + :object-id (str id) + :backend "skia-wasm" + :bytes (.-length bytes)) + bytes) + (wasm/render-shape-raster id scale type))) + +(defn invalidate-module-on-error? + "Whether a failed render drops the shared module: every error does, + except the cooperative `:job-cancelled` raised between objects, which + leaves the instance intact for the next render to reuse." + [cause] + (not= :job-cancelled (:code (ex-data cause)))) + +(defn- ^:async render* + [env {:keys [scale type objects] :as params} on-object] + (l/dbg :hint "wasm render: start" + :type type + :scale scale + :objects (count objects) + :file-id (str (:file-id params)) + :page-id (str (:page-id params))) + (try + (await (ensure-module)) + (let [scene (await (fetch-objects env params))] + (l/dbg :hint "wasm render: scene fetched" :shapes (count scene)) + (serialize/serialize-scene scene) + (l/dbg :hint "wasm render: scene serialized") + ;; So fonts from a previous request don't leak into this one. + (wasm/clear-fonts) + (let [[variants] (await (js/Promise.all [(fetch-font-variants env params) + (provision-images env scene params) + (provision-fallback-fonts env scene)])) + variants (or variants []) + resolve-font (make-resolve-font env variants params) + font-url (make-font-url env variants (:share-id params))] + ;; Before rendering, so the relayout below sees real + ;; font metrics. Deduped across objects: shapes + ;; sharing one family download its TTF once. + (await (wasm/provision-fonts (map :id objects) resolve-font + :font-url font-url))) + (relayout-text scene) + ;; Sequential, like the synchronous Skia calls underneath: + ;; the cancel check lands between objects. + (doseq [{:keys [id] :as object} objects] + (check-cancelled params) + (let [bytes (render-object-bytes type id scale) + path (shell/tempfile (:tmpdir env) + :prefix "penpot.tmp.wasm." + :suffix (mime/get-extension type))] + (l/dbg :hint "wasm render: object rendered" + :object-id (str id) :bytes (.-length bytes)) + (fs/writeFileSync path bytes) + (await (on-object (assoc object :path path))))) + ;; After the request, never mid-render, so an image can't + ;; disappear under a running export. + (let [evicted (wasm/evict-images (:image-cache-size env))] + (when (pos? evicted) + (l/info :hint "wasm render: evicted cached images" :count evicted))) + nil) + (catch :default cause + (l/error :hint "wasm render: failed" + :detail (explain cause) + :internal-uri (str (:internal-uri env)) + :cause cause) + ;; A panic can leave the mem buffer allocated or the instance + ;; aborted; drop it so the next request rebuilds a fresh one. A + ;; cooperative cancel leaves the instance intact by design, so it + ;; never invalidates the module the next render reuses. + (when (invalidate-module-on-error? cause) + (reset! module* nil)) + (throw cause)))) + +(defn ^:async render + "Public entry. Renders every object of `params` with the render + environment `env` (`:internal-uri`, `:public-uri`, `:tmpdir`, + `:image-cache-size`), calling `on-object` with + `{:id :filename :path ...}` as each one is written out." + [env params on-object] + (await (render* env params on-object))) diff --git a/exporter/src/exporter/wasm/scope.cljs b/exporter/src/exporter/wasm/scope.cljs new file mode 100644 index 0000000000..235a67584c --- /dev/null +++ b/exporter/src/exporter/wasm/scope.cljs @@ -0,0 +1,171 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.wasm.scope + "One leased render worker for a whole batch. + + The pool owns lending; this level owns what a lease means: every + headless export runs on the same worker, one at a time, sharing its + caches and pool slot, with the cancel abort armed for the life of + the lease. Task keys arrive in `cfg` under `:exporter.renderer/*` + (set by whoever built the batch); the pool instance under the + pool's own key." + (:require + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.transit :as t] + [exporter.wasm.pool :as pool])) + +(defn- serializer + "Chains thunks so a job's renders run one at a time on its worker. A + failure is isolated: it doesn't break the chain for the next one." + [] + (let [queue (atom (js/Promise.resolve nil))] + (fn [thunk] + (let [result (.then @queue (fn [_] (thunk)) (fn [_] (thunk)))] + (reset! queue (.then result (fn [_] nil) (fn [_] nil))) + result)))) + +(defn- render-on-worker- + "Settles when the worker reports the render finished, failed, or the + thread went away. That last case matters: a terminated worker (how a + cancel stops a render mid-Skia) emits `exit` and never `error`, and a + promise left pending there would keep its pool slot borrowed for the + life of the process." + [^js worker params cancel-buffer on-object timeout-ms] + (js/Promise. + (fn [resolve reject] + (let [timer (volatile! nil)] + (letfn [(disarm [] + (when-let [t @timer] + (js/clearTimeout t) + (vreset! timer nil))) + + (rearm [] + (disarm) + (vreset! timer (js/setTimeout + (fn [] + (l/error :hint "render worker went silent, terminating" + :worker-id (unchecked-get worker "__id")) + (cleanup) + ;; Terminating is what frees the pool slot: + ;; the `exit` it raises has no listener left. + (unchecked-set worker "__alive" false) + (.terminate ^js worker) + (reject (ex/error :type :internal + :code :render-timeout + :hint "render worker stopped responding"))) + timeout-ms))) + + (cleanup [] + (disarm) + (.off worker "message" on-message) + (.off worker "error" on-error) + (.off worker "exit" on-exit)) + + (on-error [cause] + (cleanup) + (reject cause)) + + (on-exit [code] + (cleanup) + (reject (ex/error :type :internal + :code :worker-exited + :hint (str "render worker exited with code " code)))) + + (on-message [data] + (rearm) + (case (unchecked-get data "type") + ;; A failure while the main thread handles the object + ;; (moving the file, appending to the zip) has to end + ;; the render too, or nothing ever settles this promise. + "object" (try + (on-object (t/decode-str (unchecked-get data "payload"))) + (catch :default cause + (cleanup) + (reject cause))) + "done" (do (cleanup) (resolve nil)) + "error" (do (cleanup) + (reject (ex/error :type :internal + :code (or (some-> (unchecked-get data "code") keyword) + :wasm-render-error) + :hint (unchecked-get data "message")))) + nil))] + + (.on worker "message" on-message) + (.once worker "error" on-error) + (.once worker "exit" on-exit) + (rearm) + (.postMessage worker #js {:type "render" + :params (t/encode-str params) + :cancel cancel-buffer})))))) + +(defn ^:async render-on + "Low-level render on the worker leased in `cfg`. A missing worker + or timeout raises instead of failing obscurely (`:worker-not-leased`, + `:timeout-ms-missing`): both are programmer errors, and a missing + timeout must never degrade silently into the default silence budget." + [cfg params] + (let [worker (::pool/worker cfg) + timeout-ms (or (:timeout-ms (:exporter.wasm/pool cfg)) + (throw (ex/error :type :assertion + :code :timeout-ms-missing + :hint "render-on needs :timeout-ms in cfg")))] + (when-not worker + (throw (ex/error :type :assertion + :code :worker-not-leased + :hint "render-on needs a leased worker in cfg"))) + (when-let [check-cancelled (:exporter.renderer/check-cancelled cfg)] + (check-cancelled)) + (await (render-on-worker- worker + params + (::cancel-buffer cfg) + (:exporter.renderer/on-object cfg) + timeout-ms)))) + +(defn ^:async run + "Leases one worker for `on-scope`, a fn of a single-arg render fn. + Every headless export fires through it, one at a time on the same + worker, so the check-cancelled runs as each render's turn comes up. The check-cancelled + is mandatory: without one the lease raises `:check-cancelled- + missing` before acquiring anything. The per lease signal buffer is + born here; the interval dies with the lease. A truly stuck thread + is backstopped by the pool's own silence watchdog." + [cfg on-scope] + (let [check-cancelled (:exporter.renderer/check-cancelled cfg)] + (when-not (fn? check-cancelled) + (throw (ex/error :type :assertion + :code :check-cancelled-missing + :hint "scope/run needs a zero-arg check-cancelled in cfg"))) + (await + (pool/run cfg + (^:async fn [cfg] + (let [worker (::pool/worker cfg) + chain (serializer) + live (volatile! worker) + signal (js/Int32Array. (js/SharedArrayBuffer. 4)) + timer (volatile! nil) + render-cfg (assoc cfg + ::cancel-buffer (.-buffer signal))] + (vreset! timer + (js/setInterval + (fn [] + (try + (check-cancelled) + (catch :default _cause + ;; One-shot: the lease is doomed anyway. + ;; Between objects the worker sees the + ;; flag; inside a render only terminating + ;; the thread stops it. + (some-> @timer js/clearInterval) + (js/Atomics.store signal 0 1) + (some-> @live pool/terminate)))) + (::watch-interval cfg 1000))) + (try + (await (on-scope (fn [params] (chain (fn [] (render-on render-cfg params)))))) + (finally + (some-> @timer js/clearInterval) + (vreset! live nil))))))))) diff --git a/exporter/src/app/wasm/serialize.cljs b/exporter/src/exporter/wasm/serialize.cljs similarity index 89% rename from exporter/src/app/wasm/serialize.cljs rename to exporter/src/exporter/wasm/serialize.cljs index 93071eaa31..c3ef56682e 100644 --- a/exporter/src/app/wasm/serialize.cljs +++ b/exporter/src/exporter/wasm/serialize.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.wasm.serialize +(ns exporter.wasm.serialize "Browser-free shape serialization for the headless exporter: the counterpart of `app.render-wasm.api/set-object`, which cannot be reused directly because its namespace pulls React/DOM/store. Only the call sequencing lives here — @@ -12,15 +12,15 @@ WASM are the editor's. Covers everything except svg-raw. Image bytes and fonts are provisioned - separately by `app.renderer.wasm`." + separately by the render pipeline." (:require [app.common.render-wasm.api.props :as props] [app.common.render-wasm.helpers :as h] [app.common.render-wasm.serialize-shape :as serialize-shape] [app.common.render-wasm.wasm :as wasm] - [app.wasm.text :as text])) + [exporter.wasm.text :as text])) -(defn set-shape! +(defn set-shape "Serializes a single shape into the WASM design state. The host-independent properties (base props, children, blur, shadows, svg-attrs, mask, bool-type, path geometry, grow-type) go through the shared `serialize-shape` — the same @@ -34,13 +34,13 @@ (when-not (= type :group) (props/write-shape-strokes! (:strokes prepared))) (when (= type :text) - (text/set-shape-text! (:content prepared))))) + (text/set-shape-text (:content prepared))))) -(defn serialize-scene! +(defn serialize-scene "Loads every shape of an `objects` map into the WASM design state. Resets the shapes pool first so repeated exports don't accumulate into the shared state. Order is irrelevant: shapes reference each other by id and the tree is resolved at render time." [objects] (h/call wasm/internal-module "_init_shapes_pool" (count objects)) - (run! set-shape! (vals objects))) + (run! set-shape (vals objects))) diff --git a/exporter/src/app/wasm/text.cljs b/exporter/src/exporter/wasm/text.cljs similarity index 97% rename from exporter/src/app/wasm/text.cljs rename to exporter/src/exporter/wasm/text.cljs index 13b037652b..25b4b60faf 100644 --- a/exporter/src/app/wasm/text.cljs +++ b/exporter/src/exporter/wasm/text.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns app.wasm.text +(ns exporter.wasm.text "Browser-free text-content serialization for the headless exporter. Only the paragraph walk is local: the binary layout and the font-id -> uuid mapping both come from `app.common.render-wasm.text-content`." @@ -14,7 +14,7 @@ [app.common.render-wasm.text-content :as tc] [app.common.render-wasm.wasm :as wasm])) -(defn set-shape-text! +(defn set-shape-text "Serializes a text shape's content into the current WASM shape. Mirrors the editor's sequence: clear -> vertical-align -> append each paragraph -> layout. Byte writing and font resolution are the shared diff --git a/exporter/src/exporter/wasm/worker.cljs b/exporter/src/exporter/wasm/worker.cljs new file mode 100644 index 0000000000..51638ccf83 --- /dev/null +++ b/exporter/src/exporter/wasm/worker.cljs @@ -0,0 +1,85 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter.wasm.worker + "Render worker entry point. + + Owns one render-wasm module and renders one export request at a time. The + Skia calls are synchronous, so running them here is what lets several exports + progress at once: the main thread keeps polling the queue and settling jobs + while this thread is blocked inside a render. There are no promesa chains + in this namespace. + + Messages in: {type: \"render\", params: , cancel: SharedArrayBuffer} + Messages out: {type: \"ready\"} + {type: \"object\", payload: } one per rendered object + {type: \"done\"} | {type: \"error\", message, code}" + (:require + ["node:worker_threads" :as wt] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.transit :as t] + [exporter.wasm.render :as render])) + +(def parent-port + "The channel renders report through: the parent port of a render + thread. A plain def so tests — which run on the main thread, where + the port is null — rebind it, the way they rebind `fetch`." + wt/parentPort) + +(defn- post + [message] + (.postMessage ^js parent-port message)) + +(defn- cancelled-fn + [buffer] + (if (some? buffer) + (let [signal (js/Int32Array. buffer)] + (fn [] (pos? (js/Atomics.load signal 0)))) + (constantly false))) + +(defn ^:async handle-render + "One render message: split the render env off the params, run the + pipeline, and post every object plus the settle. Never rejects: a + failure posts `error`, so the main thread always settles the lease." + [data] + (try + (let [{env ::render/env :as params} (t/decode-str (unchecked-get data "params"))] + (when (nil? env) + (throw (ex/error :type :assertion + :code :env-missing + :hint "worker render needs ::render/env in params"))) + (let [params (assoc (dissoc params ::render/env) + :cancelled? (cancelled-fn (unchecked-get data "cancel")))] + (await (render/render env params + (fn [object] + (post #js {:type "object" :payload (t/encode-str object)})))) + (post #js {:type "done"}))) + (catch :default cause + (l/warn :hint "render worker: request failed" :cause cause) + (post #js {:type "error" + :message (or (ex-message cause) (str cause)) + :code (some-> cause ex-data :code name)})))) + +(defn- on-message + [data] + (case (unchecked-get data "type") + "render" (handle-render data) + (l/warn :hint "render worker: unknown message" :type (unchecked-get data "type")))) + +(defonce ^:private listening + ;; `defonce` survives a hot reload, so a reload does not stack a second + ;; listener on the port. The indirection through the var keeps the reloaded + ;; `on-message` in play instead of pinning the one captured at boot. + (delay + (.on ^js wt/parentPort "message" (fn [data] (on-message data))) + true)) + +(defn main + [& _] + @listening + (post #js {:type "ready"}) + (l/info :hint "render worker ready")) diff --git a/exporter/test/exporter_tests/browser_scope_test.cljs b/exporter/test/exporter_tests/browser_scope_test.cljs new file mode 100644 index 0000000000..eb2d7d1cb8 --- /dev/null +++ b/exporter/test/exporter_tests/browser_scope_test.cljs @@ -0,0 +1,97 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.browser-scope-test + "The browser batch: every export through its own checkout, all at + once, against a stub pool with stub pages." + (:require + ["generic-pool" :as gp] + ["node:os" :as os] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [exporter.browser.scope :as scope] + [exporter.config :as cf])) + +(defn- stub-locator + [] + #js {:waitFor (fn [_opts] (js/Promise.resolve nil)) + :screenshot (fn [_opts] (js/Promise.resolve nil))}) + +(defn- stub-page + [calls] + #js {:goto (fn [url _opts] + (swap! calls conj [:goto (str url)]) + (js/Promise.resolve nil)) + :waitForTimeout (fn [ms] (swap! calls conj [:sleep ms]) (js/Promise.resolve nil)) + :waitForFunction (fn [_src _arg _opts] (js/Promise.resolve nil)) + :locator (fn [sel] (swap! calls conj [:select sel]) (stub-locator)) + :$$ (fn [sel] + (swap! calls conj [:select-all sel]) + (js/Promise.resolve #js [])) + :evaluate (fn [_f & _args] (js/Promise.resolve nil))}) + +(defn- stub-context + [page] + #js {:newPage (fn [] (js/Promise.resolve page)) + :close (fn [] (js/Promise.resolve nil))}) + +(defn- stub-browser + [page] + #js {:newContext (fn [_opts] (js/Promise.resolve (stub-context page))) + :isConnected (fn [] true) + :close (fn [] (js/Promise.resolve nil))}) + +(defn- stub-pool + [calls] + (let [page (stub-page calls)] + (gp/createPool #js {:create (fn [] (js/Promise.resolve (stub-browser page))) + :destroy (fn [browser] (.close ^js browser)) + :validate (fn [_] true)} + #js {:max 2 :min 0}))) + +(defn- test-cfg + [calls] + {:exporter.browser/pool (stub-pool calls) + :exporter/tmpdir (os/tmpdir) + :base-uri (cf/get-internal-uri) + :public-uri (cf/get :public-uri) + :svgo? false}) + +(defn- test-params + [id] + {:file-id (uuid/next) + :page-id (uuid/next) + :token "token" + :scale 1 + :type :png + :objects [{:id id :name id :suffix ".png" :filename (str id ".png")}]}) + +(t/deftest ^:async run-fires-every-export-through-its-own-checkout + (try + (let [calls (atom []) + seen (atom [])] + (await (scope/run (assoc (test-cfg calls) + :exporter.renderer/on-object (fn [object] (swap! seen conj object) nil) + :exporter.renderer/check-cancelled (fn [] nil)) + [(test-params "a") (test-params "b")])) + (t/is (= ["a" "b"] (mapv :id @seen))) + (t/is (= 2 (count (filter #(and (vector? %) (= :goto (first %))) @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-rejects-on-the-first-failure + (try + (let [calls (atom [])] + (try + (await (scope/run (assoc (test-cfg calls) + :exporter.renderer/on-object (fn [_] (throw (ex-info "on-object broke" {}))) + :exporter.renderer/check-cancelled (fn [] nil)) + [(test-params "a")])) + (t/is false "run should have rejected a failing on-object") + (catch :default cause + (t/is (= "on-object broke" (ex-message cause)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/browser_test.cljs b/exporter/test/exporter_tests/browser_test.cljs new file mode 100644 index 0000000000..5da5998f6b --- /dev/null +++ b/exporter/test/exporter_tests/browser_test.cljs @@ -0,0 +1,108 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.browser-test + "The browser pool service: lifecycle through the system, and the + acquire/use/release contract of exec, against a fake browser factory." + (:require + ["generic-pool/lib/errors.js" :as gpe] + [cljs.test :as t :include-macros true] + [exporter.browser :as browser] + [exporter.utils.system :as system])) + +(defn- stub-page + [] + #js {:marker :stub-page}) + +(defn- stub-context + [calls page] + #js {:newPage (fn [] (swap! calls conj :new-page) (js/Promise.resolve page)) + :close (fn [] (swap! calls conj :close-context) (js/Promise.resolve nil))}) + +(defn- stub-browser + [calls] + (let [page (stub-page)] + #js {:newContext (fn [_opts] + (swap! calls conj :new-context) + (js/Promise.resolve (stub-context calls page))) + :isConnected (fn [] true) + :close (fn [] + (swap! calls conj :close-browser) + (js/Promise.resolve nil))})) + +(defn- fake-create-browser + [calls] + (fn [] + (swap! calls conj :create-browser) + (js/Promise.resolve (stub-browser calls)))) + +(defn- test-config + [calls] + {:exporter.browser/pool {:max 1 + :create-browser (fake-create-browser calls)}}) + +(t/deftest ^:async pool-starts-and-stops-through-the-system + (try + (let [calls (atom []) + sys (await (system/init (test-config calls)))] + (t/is (some? (:exporter.browser/pool sys))) + (t/is (nil? (await (system/halt sys))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async exec-acquires-runs-handle-and-releases + (try + (let [calls (atom []) + sys (await (system/init (test-config calls))) + pool (:exporter.browser/pool sys) + seen (atom nil) + result (await (browser/exec pool {} (fn [page] (reset! seen page) :handle-result)))] + (t/is (= :handle-result result)) + (t/is (= :stub-page (.-marker @seen))) + (t/is (= [:create-browser :new-context :new-page :close-context] @calls)) + (t/is (not (some #{:close-browser} @calls))) + (t/testing "a second exec reuses the released browser: nothing is created again" + (await (browser/exec pool {} (fn [_] :again))) + (t/is (= 1 (count (filter #{:create-browser} @calls))))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async exec-destroys-the-browser-on-failure + (try + (let [calls (atom []) + sys (await (system/init (test-config calls))) + pool (:exporter.browser/pool sys)] + (try + (await (browser/exec pool {} (fn [_] (js/Promise.reject (ex-info "render broke" {}))))) + (t/is false "exec should have rejected when the handle fails") + (catch :default cause + (t/is (= "render broke" (ex-message cause))) + (t/is (some #{:close-browser} @calls)))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async exec-without-pool-fails-fast + (try + (await (browser/exec nil {} (fn [_] :never))) + (t/is false "exec should have rejected without a pool") + (catch :default cause + (t/is (= ::browser/not-started (-> cause ex-data :reason)))))) + +(t/deftest ^:async exec-translates-playwright-timeouts + (try + (let [calls (atom []) + sys (await (system/init (test-config calls))) + pool (:exporter.browser/pool sys)] + (try + (await (browser/exec pool {} (fn [_] (js/Promise.reject (gpe/TimeoutError. "timeout exceeded"))))) + (t/is false "exec should have rejected on a playwright timeout") + (catch :default cause + (t/is (= :timeout (-> cause ex-data :code))))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/consumer_api_test.cljs b/exporter/test/exporter_tests/consumer_api_test.cljs new file mode 100644 index 0000000000..25cd436116 --- /dev/null +++ b/exporter/test/exporter_tests/consumer_api_test.cljs @@ -0,0 +1,194 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.consumer-api-test + "The management client of the new tree against a fake http: what it + sends (method uri, shared key, shape of the body) is the whole + contract this client owns." + (:require + ["node:fs/promises" :as fsp] + ["node:os" :as os] + ["node:path" :as path] + ["undici" :as http] + [app.common.transit :as transit] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [cuerdas.core :as str] + [exporter.config :as cf] + [exporter.consumer.api :as api])) + +(def ^:private original-fetch api/fetch) + +(def ^:private calls + "Every fetch the client made, in order." + (atom [])) + +(defn- fake-fetch + [uri request] + (swap! calls conj [uri request]) + (js/Promise.resolve #js {:status 200 + :text (constantly + (js/Promise.resolve + (transit/encode-str {:action :run})))})) + +(defn- failure-fetch + "A fake that answers like a management endpoint that refuses the key." + [_uri _request] + (js/Promise.resolve #js {:status 403 + :text (constantly (js/Promise.resolve "{}"))})) + +(defn- install-fetch + ([] (install-fetch fake-fetch)) + ([the-fake] + (set! api/fetch the-fake) + (reset! calls []) + (fn [] (set! api/fetch original-fetch)))) + +(defn- last-request + [] + (second (last @calls))) + +(defn- fetch-header + "One header of the last request, exactly as it would travel." + [name] + (unchecked-get (.-headers (second (last @calls))) name)) + +(defn- body-value + "One field of the last FormData request." + [name] + (.get (.-body (last-request)) name)) + +(defn- decode-body + "The body of the last transit request, decoded." + [] + (transit/decode-str (.-body (last-request)))) + +(defn- last-uri + [] + (first (last @calls))) + +(t/deftest ^:async claim-job-posts-the-claim-of-one-poller + (let [restore (install-fetch)] + (try + (let [job-id (uuid/next)] + (await (api/claim-job job-id "2026-10-07T00:00:00Z")) + (t/is (str/ends-with? (last-uri) "/api/management/methods/claim-job")) + (t/is (= (str "exporter:" (cf/get :tenant) " " cf/management-key) (fetch-header "X-Shared-Key"))) + (t/is (= {:job-id job-id :scheduled-at "2026-10-07T00:00:00Z"} + (decode-body)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) + +(t/deftest ^:async json-calls-declare-the-transit-content-type + ;; without it the backend never parses the body as transit and + ;; validates an empty map (every key a `missing-key`) + (let [restore (install-fetch)] + (try + (await (api/claim-job (uuid/next) "2026-10-07T00:00:00Z")) + (t/is (= "application/transit+json" + (fetch-header "Content-Type"))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) + +(t/deftest ^:async report-job-progress-posts-the-beat + (let [restore (install-fetch)] + (try + (let [job-id (uuid/next) + progress {:stage :rendering + :counters {:objects {:current 1 :total 2}}}] + (await (api/report-job-progress job-id progress)) + (t/is (str/ends-with? (last-uri) "/api/management/methods/report-job-progress")) + (t/is (= {:job-id job-id :progress progress} (decode-body)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) + +(t/deftest ^:async fail-job-carries-the-error-and-the-session + (let [restore (install-fetch)] + (try + (let [job-id (uuid/next) + sid (uuid/next) + error {:type :internal :code :processing-error :hint "boom"}] + (await (api/fail-job job-id error)) + (await (api/fail-job job-id error :session-id sid)) + (t/is (= {:job-id job-id :error error :session-id sid} + (decode-body)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) + +(t/deftest ^:async complete-job-with-artifact-rides-multipart + (let [restore (install-fetch)] + (try + (let [job-id (uuid/next) + sid (uuid/next) + tmpfile (path/join (os/tmpdir) (str "test-artifact." (uuid/next)))] + (await (fsp/writeFile tmpfile "the export bytes")) + (await (api/complete-job-with-artifact + {:job-id job-id :session-id sid} + {:path tmpfile + :filename "export.zip" + :mtype "application/zip"})) + (t/is (instance? http/FormData (.-body (last-request)))) + (t/is (= (str job-id) (body-value "job-id"))) + (t/is (= (str sid) (body-value "session-id"))) + (t/is (= (str "exporter:" (cf/get :tenant) " " cf/management-key) + (fetch-header "X-Shared-Key"))) + (t/testing "name and type travel in the part itself, no text fields" + (let [body (.-body (last-request)) + content (.get body "content")] + (t/is (nil? (.get body "filename"))) + (t/is (nil? (.get body "mtype"))) + (t/is (= "export.zip" (.-name content))) + (t/is (= "application/zip" (.-type content))) + (t/is (= 16 (.-size content)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) + +(t/deftest ^:async complete-job-with-artifact-streams-a-large-file + (let [restore (install-fetch)] + (try + (let [job-id (uuid/next) + tmpfile (path/join (os/tmpdir) (str "test-artifact-large." (uuid/next))) + payload (.repeat "x" 262144)] + (await (fsp/writeFile tmpfile payload)) + (await (api/complete-job-with-artifact + {:job-id job-id} + {:path tmpfile + :filename "large.zip" + :mtype "application/zip"})) + (let [content (.get (.-body (last-request)) "content")] + (t/is (= 262144 (.-size content))) + (t/is (= "application/zip" (.-type content))) + (t/is (= "large.zip" (.-name content))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) + +(t/deftest ^:async a-failed-management-answer-is-the-failure-of-the-call + (let [restore (install-fetch failure-fetch)] + (try + (try + (await (api/claim-job (uuid/next) "2026-10-07T00:00:00Z")) + (t/is false "the call should have rejected on a 403") + (catch :default cause + (let [data (ex-data cause)] + (t/is (= :internal (:type data))) + (t/is (= :failed-management-request (:code data))) + (t/is (= 403 (:status data)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore))))) diff --git a/exporter/test/exporter_tests/consumer_test.cljs b/exporter/test/exporter_tests/consumer_test.cljs new file mode 100644 index 0000000000..330c84cf0c --- /dev/null +++ b/exporter/test/exporter_tests/consumer_test.cljs @@ -0,0 +1,489 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.consumer-test + "The export runner of the new tree: plan → render batch → settle, + over a fake management http and a stub renderer. The beats paint the + stage vocabulary in order and the settle carries the artifact; a + failure or a cancel settles as fail-job and never leaks the beat." + (:require + ["node:fs/promises" :as fsp] + ["node:os" :as os] + ["node:path" :as path] + ["undici" :as http] + [app.common.schema :as sm] + [app.common.transit :as transit] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [cuerdas.core :as cstr] + [exporter.consumer :as consumer] + [exporter.consumer.api :as api] + [exporter.jobs :as jobs] + [exporter.renderer :as renderer] + [exporter.shell :as shell])) + +;; ---- THE MANAGEMENT FAKE + +(def ^:private calls + "One entry per management call: [method, body]." + (atom [])) + +(defn- method-of + [uri] + (last (cstr/split (str uri) "/"))) + +(defn- steps-of + [] + (mapv first @calls)) + +(defn- call-of + [method] + (some (fn [[m body]] + (when (= m method) body)) + @calls)) + +(defn- fake-fetch + [respond] + (reset! calls []) + (let [original api/fetch] + (set! api/fetch + (fn [uri request] + (let [method (method-of uri) + body (if (instance? http/FormData (.-body request)) + (.-body request) + (transit/decode-str (.-body request)))] + (swap! calls conj [method body]) + (js/Promise.resolve #js {:status 200 + :text (constantly + (js/Promise.resolve + (transit/encode-str + (respond method body))))})))) + (fn [] (set! api/fetch original)))) + +;; ---- THE RENDER STUB + +(defn- stub-render + "A render fn in place of the service: writes one file of `content` + per export and calls back with it. When `captured` is an atom, it + also keeps the last batch it received." + ([content] (stub-render content nil)) + ([content captured] + (fn [{:keys [exports on-object] :as task}] + (when (some? captured) + (reset! captured task)) + (js/Promise. + (fn [resolve reject] + (let [writes (mapv (fn [_] + (let [path (shell/tempfile (os/tmpdir) :prefix "penpot.render." + :suffix ".png")] + (.then (fsp/writeFile path content) + (fn [_] + (on-object {:path path + :filename "rendered.png"}))))) + exports)] + (-> (js/Promise.all (clj->js writes)) + (.then (fn [_] (resolve nil)) reject)))))))) + +(defn- stub-render-fail + [] + (fn [& _] + (js/Promise.reject (ex-info "render boom" {})))) + +;; ---- THE FIXTURE + +(defn- export-params + [] + {:exports [{:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name "test shape" + :suffix "" + :scale 1}] + :name "the export"}) + +(defn- answer-with + [sid stages] + (fn [method body] + (when (= method "report-job-progress") + (vswap! stages conj (get-in body [:progress :stage]))) + (case method + "create-job-session" {:session-id sid :session-token "session-token"} + {:action :run}))) + +(t/deftest ^:async run-exports-one-shape-to-a-multipart-settle + (let [job-id (uuid/next) + sid (uuid/next) + stages (volatile! []) + restore-fetch (fake-fetch + (fn [method body] + (when (= method "report-job-progress") + (vswap! stages conj (get-in body [:progress :stage]))) + (case method + "create-job-session" {:session-id sid :session-token "session-token"} + {:action :run}))) + render (stub-render "rendered!")] + (try + (await (consumer/run-export render (os/tmpdir) {:job-id job-id} (export-params))) + (t/testing "the run went by first breath, session, beats, settle" + (t/is (= ["report-job-progress" "create-job-session" + "report-job-progress" "report-job-progress" + "complete-job"] + (steps-of)))) + (t/testing "the settle carried the real artifact, named and typed" + (let [fd (call-of "complete-job")] + (t/is (instance? http/FormData fd)) + (t/is (= (str job-id) (.get fd "job-id"))) + (t/is (= (str sid) (.get fd "session-id"))) + (t/is (= "image/png" (.-type (.get fd "content")))))) + (t/testing "the beats painted the vocabulary in order" + (t/is (= [:preparing :rendering :packaging] @stages))) + (t/testing "the local registry entry is gone with the settle" + (t/is (false? (jobs/cancelled? job-id)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async run-hands-the-renderer-uuids-not-claim-strings + (let [restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + captured (atom nil) + render (stub-render "rendered!" captured) + job-id (uuid/next) + share-id (uuid/next)] + (try + (await (consumer/run-export + render (os/tmpdir) + {:job-id job-id} + {:exports [{:file-id (str (uuid/next)) + :page-id (str (uuid/next)) + :object-id (str (uuid/next)) + :share-id (str share-id) + :type "png" + :name "test shape" + :suffix "" + :scale 1}] + :name "the export"})) + (let [export (-> @captured :exports first)] + (t/testing "every id arrived as a uuid object" + (t/is (uuid? (:file-id export))) + (t/is (uuid? (:page-id export))) + (t/is (uuid? (:share-id export))) + (t/is (= share-id (:share-id export))) + (t/is (uuid? (-> export :objects first :id)))) + (t/testing "the job-id traveled with the render" + (t/is (= job-id (:job-id export)))) + (t/testing "the task handed to the renderer satisfies the render contract" + (t/is ((sm/validator renderer/schema:task) @captured))) + (t/testing "the task carried both injections" + (t/is (fn? (:on-object @captured))) + (t/is (fn? (:check-cancelled @captured))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async run-carries-is-wasm-to-single-and-multiple + (let [mk-params (fn [n] + {:exports (mapv (fn [i] + {:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name (str "shape-" i) + :suffix "" + :scale 1}) + (range n)) + :name "the export" + :is-wasm true})] + (doseq [n [1 2]] + (let [restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + captured (atom nil) + render (stub-render "rendered!" captured) + job-id (uuid/next)] + (try + (await (consumer/run-export render (os/tmpdir) {:job-id job-id} (mk-params n))) + (let [exports (:exports @captured)] + (t/testing (str "every export of a batch of " n " carries :is-wasm true") + (t/is (seq exports)) + (t/is (every? #(true? (:is-wasm %)) exports)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))))) + +(t/deftest ^:async run-carries-skip-children-to-multiple + (let [mk-params (fn [] + {:exports [{:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name "one" + :suffix "" + :scale 1} + {:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name "two" + :suffix "" + :scale 1}] + :name "the export" + :skip-children true}) + restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + captured (atom nil) + render (stub-render "rendered!" captured) + job-id (uuid/next)] + (try + (await (consumer/run-export render (os/tmpdir) {:job-id job-id} (mk-params))) + (let [exports (:exports @captured)] + (t/is (seq exports)) + (t/is (= 2 (reduce + 0 (map (comp count :objects) exports)))) + (t/is (every? #(true? (:skip-children %)) exports))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(defn- one-shape + [n] + {:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name (str "shape-" n) + :suffix "" + :scale 1}) + +(t/deftest ^:async run-leaves-no-tracked-files-behind + ;; The artifact is tracked where make-plan creates it, so the settle + ;; release wipes the job area on every path: without the track the + ;; multi zip (and the frames pdf below) stays on disk. + (let [restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + render (stub-render "rendered!") + check-empty (^:async fn [label params] + (let [area (path/join (os/tmpdir) (str "penpot-consumer-release-test." (uuid/next))) + _ (await (fsp/mkdir area #js {:recursive true})) + job-id (uuid/next)] + (try + (await (consumer/run-export render area {:job-id job-id} params)) + (let [entries (await (fsp/readdir area))] + (t/testing (str "the job area is empty after a " label " run") + (t/is (empty? entries)))) + (finally + (await (fsp/rm area #js {:recursive true :force true}))))))] + (try + (await (check-empty "single" {:exports [(one-shape "one")] :name "the export"})) + (await (check-empty "multi" {:exports [(one-shape "one") (one-shape "two")] :name "the export"})) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async run-frames-leaves-no-tracked-files-behind + (let [restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + render (stub-render "page!") + original-cmd shell/run-cmd + area (path/join (os/tmpdir) (str "penpot-consumer-release-frames-test." (uuid/next))) + job-id (uuid/next)] + (set! shell/run-cmd + (^:async fn [_cmd & args] + (let [dest (last (filter string? (flatten args)))] + (await (shell/write-file dest "joined!")) + nil))) + (try + (await (fsp/mkdir area #js {:recursive true})) + (await (consumer/run-export render area {:job-id job-id} + {:exports [(assoc (one-shape "page") :type :pdf)] + :kind :frames + :name "the export"})) + (let [entries (await (fsp/readdir area))] + (t/testing "the job area is empty after a frames run" + (t/is (empty? entries)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch) + (set! shell/run-cmd original-cmd) + (await (fsp/rm area #js {:recursive true :force true})))))) + +(t/deftest ^:async run-packs-two-shapes-as-zip + (let [restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + render (stub-render "rendered!") + job-id (uuid/next)] + (try + (await (consumer/run-export + render (os/tmpdir) + {:job-id job-id} + {:exports [{:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name "one" + :suffix "" + :scale 1} + {:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :png + :name "two" + :suffix "" + :scale 1}] + :name "the export"})) + (t/testing "the artifact is the zip of both objects" + (let [fd (call-of "complete-job")] + (t/is (instance? http/FormData fd)) + (t/is (= "application/zip" (.-type (.get fd "content")))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async a-failed-run-settles-fail-job-and-stops-beating + (let [restore-fetch (fake-fetch (answer-with (uuid/next) (volatile! []))) + render (stub-render-fail) + job-id (uuid/next)] + (try + (await (consumer/run-export render (os/tmpdir) {:job-id job-id} (export-params))) + (await (js/Promise. (fn [resolve] (js/setTimeout resolve 1400)))) + (t/testing "the settle was a fail, not a complete" + (t/is (= "fail-job" (last (steps-of))))) + (t/testing "the failure is the shape the backend stores" + (let [fail (call-of "fail-job")] + (t/is (= :internal (get-in fail [:error :type]))) + (t/is (= :export-failed (get-in fail [:error :code]))) + (t/is (= "render boom" (get-in fail [:error :hint]))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async cancel-before-the-render-settles-cancelled + (let [restore-fetch (fake-fetch + (fn [method _body] + (case method + "create-job-session" {:session-id (uuid/next) + :session-token "session-token"} + "report-job-progress" {:action :skip} + {:action :run}))) + render (stub-render "should never render") + job-id (uuid/next)] + (try + (await (consumer/run-export render (os/tmpdir) {:job-id job-id} (export-params))) + (t/testing "the settle of a cancelled job was a fail the backend answers skip" + (t/is (= "fail-job" (last (steps-of)))) + (let [fail (call-of "fail-job")] + (t/is (= :job-cancelled (get-in fail [:error :code]))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async run-reports-progress-in-order + ;; The intermediate beats are awaited, so the counters land in the + ;; order the objects do: the widget never paints a step backwards. + (let [progresses (volatile! []) + restore-fetch (fake-fetch + (fn [method body] + (when (= method "report-job-progress") + (vswap! progresses conj (get-in body [:progress :counters]))) + (case method + "create-job-session" {:session-id (uuid/next) + :session-token "session-token"} + {:action :run}))) + render (stub-render "rendered!") + job-id (uuid/next)] + (try + (await (consumer/run-export + render (os/tmpdir) + {:job-id job-id} + {:exports [(one-shape "one") (one-shape "two")] + :name "the export"})) + (let [currents (into [] (comp (map :objects) (map :current) (filter some?)) @progresses)] + (t/is (= currents (sort currents)) + "counters never step backwards")) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async run-joins-frame-pages-as-pdf + ;; the frames path end to end except pdfunite itself: the page + ;; renders, the join runs in the injected temp area, the artifact + ;; settles as pdf. Frames travel declared (`:kind`), not sniffed: + ;; the items are fully typed pages, as the frozen job holds them. + (let [tmpdir (shell/ensure-dir (path/join (os/tmpdir) "penpot-consumer-frames-test")) + restore-fetch (fake-fetch + (fn [method _body] + (case method + "create-job-session" {:session-id (uuid/next) + :session-token "session-token"} + {:action :run}))) + render (stub-render "page!") + joined (atom nil) + original-cmd shell/run-cmd + job-id (uuid/next)] + (set! shell/run-cmd + (^:async fn [cmd & args] + (let [dest (last (filter string? (flatten args)))] + (reset! joined [cmd dest]) + (await (shell/write-file dest "joined!")) + nil))) + (try + (await (consumer/run-export render tmpdir {:job-id job-id} + {:exports [{:file-id (uuid/next) + :page-id (uuid/next) + :object-id (uuid/next) + :type :pdf + :name "page" + :suffix "" + :scale 1}] + :kind :frames + :name "the export"})) + (t/testing "pdfunite joined inside the injected temp area" + (t/is (= "pdfunite" (first @joined))) + (t/is (cstr/starts-with? (second @joined) tmpdir))) + (t/testing "the artifact settled as pdf" + (let [fd (call-of "complete-job")] + (t/is (instance? http/FormData fd)) + (t/is (= "application/pdf" (.-type (.get fd "content")))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch) + (set! shell/run-cmd original-cmd))))) + +;; ---- SERIALIZED BEATS + +(t/deftest ^:async beats-run-one-at-a-time + ;; The watchdog repeats the last report while a unit beat may be in + ;; flight: serialized, the newest milestone can never be overwritten by + ;; an older one arriving late. + (let [chain (atom nil) + live (atom 0) + peak (atom 0) + order (atom []) + task (fn [label] + (^:async fn [] + (swap! live inc) + (swap! peak max @live) + (swap! order conj [:start label]) + (await (js/Promise. + (fn [resolve] (js/setTimeout resolve 2)))) + (swap! order conj [:end label]) + (swap! live dec) + label))] + (await (js/Promise.all + [(consumer/enqueue-beat chain (task :a)) + (consumer/enqueue-beat chain (task :b)) + (consumer/enqueue-beat chain (task :c))])) + (t/testing "never two in flight" + (t/is (= 1 @peak))) + (t/testing "and in the order they were queued" + (t/is (= [[:start :a] [:end :a] + [:start :b] [:end :b] + [:start :c] [:end :c]] + @order))))) diff --git a/exporter/test/exporter_tests/consumer_worker_test.cljs b/exporter/test/exporter_tests/consumer_worker_test.cljs new file mode 100644 index 0000000000..831f58ebbd --- /dev/null +++ b/exporter/test/exporter_tests/consumer_worker_test.cljs @@ -0,0 +1,221 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.consumer-worker-test + "The queue loop of the new tree: the payload it reads, the claim it + makes, and the handoff to the runner, over a fake redis connection + and a fake management http." + (:require + ["node:fs/promises" :as fsp] + ["node:os" :as os] + ["undici" :as http] + [app.common.transit :as transit] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [cuerdas.core :as cstr] + [exporter.consumer.api :as api] + [exporter.consumer.worker :as worker])) + +;; ---- THE MANAGEMENT FAKE + +(def ^:private calls + "One entry per management call: [method, body]." + (atom [])) + +(defn- fake-fetch + [respond] + (reset! calls []) + (let [original api/fetch] + (set! api/fetch + (fn [uri request] + (let [method (last (cstr/split (str uri) "/")) + body (if (instance? http/FormData (.-body request)) + (.-body request) + (transit/decode-str (.-body request)))] + (swap! calls conj [method body]) + (js/Promise.resolve #js {:status 200 + :text (constantly + (js/Promise.resolve + (transit/encode-str + (respond method body))))})))) + (fn [] (set! api/fetch original)))) + +;; ---- THE QUEUE FAKE + +(defn- fake-conn + [queue] + #js {:blpop (fn [_key _timeout-s] + (if-let [payload (first @queue)] + (do + (swap! queue rest) + (js/Promise.resolve #js ["the.queue.key" payload])) + (js/Promise.resolve nil))) + :quit (fn [] (js/Promise.resolve nil))}) + +(defn- encoded-claim + "The dispatcher payload as the backend emits it + (`app.worker.dispatcher/encode-payload`): a JSON array of exactly two + items, `[job-id scheduled-at]` with the instant in `ct/format-inst` + shape. Byte-identical literals are asserted on the backend side; + `scheduled-at` travels back untouched on claim." + [job-id scheduled-at] + (js/JSON.stringify #js [(str job-id) scheduled-at])) + +(defn- test-cfg + [extra] + (merge {:concurrency 1 + :queue-key "the.queue.key" + :exporter/tmpdir (os/tmpdir)} + extra)) + +(t/deftest read-decodes-the-dispatcher-payload + (t/testing "a payload in shape names the job and when it was pushed" + (let [job-id (uuid/next)] + (t/is (= {:job-id (str job-id) + :scheduled-at "2026-10-07T09:00:00Z"} + (worker/read-payload (encoded-claim job-id + "2026-10-07T09:00:00Z")))))) + (t/testing "a corrupt payload is dropped, not thrown at the loop" + (t/is (nil? (worker/read-payload "{not a payload"))))) + +(t/deftest ^:async poll-once-awaits-what-it-claims + (try + (let [queue (atom [(encoded-claim (uuid/next) "2026-10-07T09:00:00Z")]) + conn (fake-conn queue) + handled (atom [])] + (await (worker/poll-once conn + "the.queue.key" + (fn [claim] + (swap! handled conj claim) + (js/Promise.resolve nil)))) + (t/is (= 1 (count @handled)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async poll-once-drops-empty-pops-and-corrupt-payloads + (try + (let [queue (atom ["{corrupt"]) + conn (fake-conn queue)] + (await (worker/poll-once conn + "the.queue.key" + (fn [_claim] + (t/is false "corrupt must not be handled")))) + (await (worker/poll-once conn + "the.queue.key" + (fn [_claim] + (t/is false "a corrupt pop has nothing to handle")))) + (t/is (empty? @queue))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async process-drops-a-claim-the-worker-never-won + (let [restore-fetch (fake-fetch + (fn [method _body] + (case method + "claim-job" {:action :skip :status "cancelled"} + {:action :run})))] + (try + (await (worker/process (test-cfg {}) + {:job-id (uuid/next) + :scheduled-at "2026-10-07T09:00:00Z"})) + (t/testing "only the claim traveled: no run, no render" + (t/is (= ["claim-job"] (mapv first @calls)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async process-renders-the-claim-through-the-injected-renderer + ;; the whole vertical slice over fakes: the claim, the runner, the + ;; injected render fn and the multipart settle + (let [restore-fetch (fake-fetch + (fn [method _body] + (case method + "claim-job" {:action :run + :name "export-assets" + :params {:exports [{:file-id (str (uuid/next)) + :page-id (str (uuid/next)) + :object-id (str (uuid/next)) + :type "png" + :name "test shape" + :suffix "" + :scale 1}] + :name "the export"}} + "create-job-session" {:session-id (uuid/next) + :session-token "session-token"} + {:action :run}))) + ran (atom nil) + render (fn [{:keys [exports on-object]}] + (reset! ran exports) + (js/Promise. + (fn [resolve reject] + (-> (.then (fsp/writeFile "/tmp/penpot-worker-render.png" "rendered!") + (fn [_] + (on-object {:path "/tmp/penpot-worker-render.png" + :filename "rendered.png"}))) + (.then (fn [_] (resolve nil)) reject)))))] + (try + (let [job-id (uuid/next)] + (await (worker/process (test-cfg {:renderer render}) + {:job-id (str job-id) + :scheduled-at "2026-10-07T09:00:00Z"})) + (t/testing "the one shape rendered with uuids and the job stamped" + (t/is (= 1 (count @ran))) + (let [export (first @ran)] + (t/is (uuid? (:file-id export))) + (t/is (= job-id (:job-id export))))) + (t/testing "the run settled as a complete" + (t/is (= "complete-job" (last (mapv first @calls)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (restore-fetch))))) + +(t/deftest ^:async start-opens-one-connection-per-poller-and-stop-quits-them + (try + (let [opened (atom []) + quit (atom []) + conn (fn [] + (let [c (fake-conn (atom []))] + (swap! opened conj c) + (set! (.-quit c) (fn [] (swap! quit conj c) (js/Promise.resolve nil))) + ;; the first pop never answers: the pollers sit in + ;; it, so the test never races their loop + (set! (.-blpop c) (fn [_k _t] (js/Promise. (fn [_res _rej])))) + c)) + handle (await (worker/start (test-cfg {:concurrency 2 :connect conn})))] + (t/is (= 2 (count @opened))) + (await (worker/stop handle)) + (t/is (= 2 (count @quit))) + (t/testing "a second stop is a no-op" + (await (worker/stop handle)) + (t/is (= 2 (count @quit))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async stop-waits-for-in-flight-runs-within-the-grace + (try + (let [handle {:running (atom true) + :conns (atom []) + :inflight (atom 1)}] + (js/setTimeout (fn [] (reset! (:inflight handle) 0)) 20) + (await (worker/stop handle {:grace-ms 2000})) + (t/is (zero? @(:inflight handle))) + (t/is (false? @(:running handle)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async stop-gives-up-after-the-grace-period + (try + (let [handle {:running (atom true) + :conns (atom []) + :inflight (atom 1)}] + (await (worker/stop handle {:grace-ms 30})) + (t/is (= 1 @(:inflight handle)) + "the grace expired with work still in flight") + (t/is (false? @(:running handle)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/export_shapes_test.cljs b/exporter/test/exporter_tests/export_shapes_test.cljs deleted file mode 100644 index 23a682a295..0000000000 --- a/exporter/test/exporter_tests/export_shapes_test.cljs +++ /dev/null @@ -1,31 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns exporter-tests.export-shapes-test - "Chunking of the browser backend." - (:require - [app.common.uuid :as uuid] - [app.handlers.export-shapes :as export-shapes] - [cljs.test :as t :include-macros true])) - -(defn- exports - [n type scale] - (let [file-id (uuid/next) - page-id (uuid/next)] - (mapv (fn [i] - {:file-id file-id - :page-id page-id - :object-id (uuid/next) - :name (str "shape-" i) - :suffix "" - :scale scale - :type type}) - (range n)))) - -(t/deftest browser-exports-are-chunked - (let [parts (export-shapes/prepare-exports (exports 120 :png 1) "token" false)] - (t/is (= 3 (count parts))) - (t/is (= [50 50 20] (mapv (comp count :objects) parts))))) diff --git a/exporter/test/exporter_tests/exporter_config_test.cljs b/exporter/test/exporter_tests/exporter_config_test.cljs new file mode 100644 index 0000000000..0d1181b38b --- /dev/null +++ b/exporter/test/exporter_tests/exporter_config_test.cljs @@ -0,0 +1,50 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-config-test + "The process configuration, read from the environment: kebab-case + keys under the `PENPOT_` prefix, decoded and validated against the + schema, with the management key derived the way the backend expects." + (:require + [cljs.test :as t :include-macros true] + [exporter.config :as cf])) + +(t/deftest read-env-keeps-only-the-prefixed-keys + (let [env (cf/read-env #js {"PENPOT_SECRET_KEY" "s" + "PENPOT_EXPORTER_WORKER_CONCURRENCY" "4" + "OTHER_THING" "y"})] + (t/is (= {:secret-key "s" + :exporter-worker-concurrency "4"} + env)))) + +(t/deftest prepare-config-applies-defaults-and-coerces + (let [config (cf/prepare-config #js {"PENPOT_SECRET_KEY" "s" + "PENPOT_EXPORTER_WORKER_CONCURRENCY" "4"})] + (t/is (= "s" (:secret-key config))) + (t/is (= "default" (:tenant config))) + (t/is (= "/tmp/penpot" (:tempdir config))) + (t/is (= 4 (:exporter-worker-concurrency config))))) + +(t/deftest prepare-config-refuses-a-missing-secret + ;; the secret is the one key without a default: without it the + ;; process cannot derive its management key, so the config is + ;; invalid. It throws (testable) instead of exiting: only the + ;; wiring entry exits, on the live environment. + (let [cause (try + (cf/prepare-config #js {}) + nil + (catch :default cause + cause))] + (t/is (some? cause)) + (t/is (= :invalid-config (:code (ex-data cause)))))) + +(t/deftest derive-management-key-matches-the-tenant-formula + ;; the backend derives the same key from the same secret and tenant + ;; (HKDF blake2b512 over "exporter:", 32 bytes, base64url): + ;; the hardcoded value pins the formula, so a refactor cannot + ;; silently mint a key the backend rejects. + (t/is (= "LXzydEEZEVAxbiNAzXV7mPSm6wYbrS7Omp4yMZ4gmOY" + (cf/derive-management-key "test-secret-key" "default")))) diff --git a/exporter/test/exporter_tests/exporter_consumer_config_test.cljs b/exporter/test/exporter_tests/exporter_consumer_config_test.cljs new file mode 100644 index 0000000000..9b7abed7bb --- /dev/null +++ b/exporter/test/exporter_tests/exporter_consumer_config_test.cljs @@ -0,0 +1,26 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-consumer-config-test + "The consumer knobs of the new tree: how many jobs run at once and + which queue they come from." + (:require + [cljs.test :as t :include-macros true] + [exporter.consumer.config :as ccfg])) + +(t/deftest concurrency-is-at-least-one + (t/testing "the configured value is taken as given" + (t/is (= 2 (ccfg/concurrency 2))) + (t/is (= 4 (ccfg/concurrency 4)))) + (t/testing "zero or less is a misconfiguration, floored to one" + (t/is (= 1 (ccfg/concurrency 0))) + (t/is (= 1 (ccfg/concurrency -3)))) + (t/testing "an unset value falls back to the default K" + (t/is (= 2 (ccfg/concurrency nil))))) + +(t/deftest queue-key-names-tenant-and-queue + (t/is (= "penpot.worker.queue:main:exporter" + (ccfg/queue-key "main")))) diff --git a/exporter/test/exporter_tests/exporter_consumer_plan_test.cljs b/exporter/test/exporter_tests/exporter_consumer_plan_test.cljs new file mode 100644 index 0000000000..22568fbf45 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_consumer_plan_test.cljs @@ -0,0 +1,44 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-consumer-plan-test + "The render plan of the new tree: same chunks and names as the + legacy planner, dispatching on the wasm flag." + (:require + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [exporter.consumer.plan :as plan])) + +(defn- exports + [n type scale] + (let [file-id (uuid/next) + page-id (uuid/next)] + (mapv (fn [i] + {:file-id file-id + :page-id page-id + :object-id (uuid/next) + :name (str "shape-" i) + :suffix "" + :scale scale + :type type}) + (range n)))) + +(t/deftest browser-exports-are-chunked + (let [parts (plan/prepare-exports (exports 120 :png 1) "token" false)] + (t/is (= 3 (count parts))) + (t/is (= [50 50 20] (mapv (comp count :objects) parts))))) + +(t/deftest wasm-exports-render-as-one-group + (let [parts (plan/prepare-exports (exports 120 :png 1) "token" true)] + (t/is (= 1 (count parts))) + (t/is (= 120 (count (:objects (first parts))))))) + +(t/deftest duplicate-names-get-suffixed + (let [items [(assoc ((exports 1 :png 1) 0) :name "same") + (assoc ((exports 1 :png 1) 0) :name "same")] + parts (plan/prepare-exports items "token" false)] + (t/is (= ["same.png" "same-2.png"] + (mapv :filename (:objects (first parts))))))) diff --git a/exporter/test/exporter_tests/exporter_jobs_test.cljs b/exporter/test/exporter_tests/exporter_jobs_test.cljs new file mode 100644 index 0000000000..9d45c8f876 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_jobs_test.cljs @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-jobs-test + "The local cancel arm of the new tree: one flag per claimed job, + and the zero-arg check the renderers run." + (:require + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [exporter.jobs :as jobs])) + +(t/deftest register-mark-and-release + (let [job-id (uuid/next)] + (t/testing "a fresh job is not cancelled" + (jobs/register job-id) + (t/is (false? (jobs/cancelled? job-id)))) + (t/testing "the first mark flips the flag and reports it" + (t/is (true? (jobs/mark-cancelled job-id))) + (t/is (true? (jobs/cancelled? job-id)))) + (t/testing "a released job is gone, not cancelled" + (jobs/release job-id) + (t/is (false? (jobs/cancelled? job-id))) + (t/is (nil? (jobs/mark-cancelled job-id)))))) + +(t/deftest check-cancelled-raises-the-cooperative-code + (let [job-id (uuid/next) + check (jobs/check-cancelled job-id)] + (jobs/register job-id) + (t/testing "a live job passes the check silently" + (t/is (nil? (check)))) + (t/testing "a marked job raises what the runner settles as cancel" + (jobs/mark-cancelled job-id) + (try + (check) + (t/is false "the check should have raised") + (catch :default cause + (t/is (= :job-cancelled (-> cause ex-data :code)))))) + (jobs/release job-id))) diff --git a/exporter/test/exporter_tests/exporter_resources_test.cljs b/exporter/test/exporter_tests/exporter_resources_test.cljs new file mode 100644 index 0000000000..92ebb0dad7 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_resources_test.cljs @@ -0,0 +1,85 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-resources-test + "The run's local resources: the artifact files parked in the injected + temp area until the settle uploads them, zipped once a run packs more + than one." + (:require + ["node:fs/promises" :as fsp] + ["node:os" :as os] + ["node:path" :as path] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [clojure.string :as cstr] + [exporter.resources :as rsc])) + +(defn- test-area + [] + (let [dir (path/join (os/tmpdir) (str "penpot-resources-test." (uuid/next)))] + dir)) + +(t/deftest create-parks-the-artifact-under-the-injected-tmpdir + (let [dir (test-area) + resource (rsc/create dir :pdf "a/b:c")] + (t/testing "the path lives in the injected area, named by type" + (t/is (cstr/starts-with? (:path resource) dir)) + (t/is (cstr/includes? (:path resource) "penpot.resource.pdf."))) + (t/testing "the descriptor names the upload" + (t/is (= "application/pdf" (:mtype resource))) + (t/is (= "a_b_c.pdf" (:filename resource))) + (t/is (= "a/b:c" (:name resource)))) + (t/is (uuid? (:id resource))))) + +(t/deftest ^:async zip-packs-added-files + (let [dir (test-area) + _ (await (fsp/mkdir dir #js {:recursive true})) + resource (rsc/create dir :zip "the export") + member (path/join dir "member.txt")] + (try + (await (fsp/writeFile member "member!")) + (let [entries (atom []) + zip (rsc/create-zip :resource resource + :on-progress (fn [e] (swap! entries conj e)))] + (rsc/add-to-zip zip member "member.txt") + (await (rsc/close-zip zip)) + (t/testing "the artifact landed" + (let [stat (await (fsp/stat (:path resource)))] + (t/is (pos? (.-size stat))))) + (t/testing "one entry traveled with its name" + (t/is (= [{:done 1 :filename "member.txt"}] @entries)))) + (finally + (await (fsp/rm dir #js {:recursive true :force true})))))) + +(defn- fake-zip + "An event-emitter stand-in for `close-zip`: records its listeners and + runs `on-finalize` with them once finalized." + [on-finalize] + (let [handlers (atom {}) + fake #js {}] + (unchecked-set fake "on" + (fn [event cb] + (swap! handlers assoc event cb) + fake)) + (unchecked-set fake "finalize" (fn [] (on-finalize @handlers))) + fake)) + +(t/deftest ^:async close-zip-rejects-on-archiver-error + (let [zip (fake-zip (fn [handlers] + ((get handlers "error") (js/Error. "boom"))))] + (try + (await (rsc/close-zip zip {:timeout-ms 1000})) + (t/is false "an archiver error must reject, not hang the poller slot") + (catch :default cause + (t/is (= "boom" (ex-message cause))))))) + +(t/deftest ^:async close-zip-rejects-when-the-close-never-lands + (let [zip (fake-zip (fn [_handlers] nil))] + (try + (await (rsc/close-zip zip {:timeout-ms 10})) + (t/is false "a missing close must reject through the timeout") + (catch :default cause + (t/is (= "zip close timed out" (ex-message cause))))))) diff --git a/exporter/test/exporter_tests/exporter_shell_test.cljs b/exporter/test/exporter_tests/exporter_shell_test.cljs new file mode 100644 index 0000000000..2c7387dbb3 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_shell_test.cljs @@ -0,0 +1,81 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-shell-test + "The shell utilities of the new tree: every path explicit, no + global tmpdir, over a scratch area of the test itself." + (:require + ["node:fs" :as fs] + ["node:os" :as os] + ["node:path" :as path] + [cljs.test :as t :include-macros true] + [cuerdas.core :as str] + [exporter.shell :as shell])) + +(defn- scratch-dir + [] + (path/join (os/tmpdir) "penpot-exporter-shell-test")) + +(t/deftest ensure-dir-creates-and-returns + (let [nested (path/join (scratch-dir) "nested" "deeper")] + (t/is (= nested (shell/ensure-dir nested))) + (t/is (true? (fs/existsSync nested))) + (t/testing "a blank path is a wiring error, not an empty directory" + (try + (shell/ensure-dir " ") + (t/is false "ensure-dir should have rejected a blank path") + (catch :default cause + (t/is (= :tmpdir-path-missing (-> cause ex-data :code)))))))) + +(t/deftest tempfile-names-live-under-the-given-dir + (let [dir (shell/ensure-dir (scratch-dir)) + path (shell/tempfile dir :prefix "penpot.test." :suffix ".png")] + (t/is (str/starts-with? path dir)) + (t/is (str/ends-with? path ".png")) + (t/is (str/includes? path "penpot.test.")))) + +(t/deftest ^:async schedule-deletion-removes + (try + (let [dir (shell/ensure-dir (scratch-dir)) + fpath (path/join dir "doomed.tmp")] + (await (shell/write-file fpath "doomed")) + (shell/schedule-deletion fpath 0.05) + (await (js/Promise. (fn [resolve] (js/setTimeout resolve 300)))) + (t/is (false? (fs/existsSync fpath)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async move-write-and-read-roundtrip + (try + (let [dir (shell/ensure-dir (scratch-dir)) + src (path/join dir "src.tmp") + dest (path/join dir "dest.tmp")] + (await (shell/write-file src "bytes")) + (await (shell/move src dest)) + (t/is (false? (fs/existsSync src))) + (t/is (= "bytes" (str (await (shell/read-file dest)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-cmd-resolves-stdout + (try + (let [out (await (shell/run-cmd "echo" "hello"))] + (t/is (str/includes? (str out) "hello"))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-cmd-passes-arguments-literally + ;; GHSA-4f36-m4hj-cv86 stays fixed: the wrapper runs execFile, so + ;; shell metacharacters in args travel literally and never execute. + (try + (let [marker (path/join (os/tmpdir) "penpot-rce-probe") + out (await (shell/run-cmd "echo" (str "#000000$(touch " marker ")")))] + (t/is (str/includes? (str out) "$(touch ") + "the metacharacters traveled, they did not run") + (t/is (not (fs/existsSync marker)) + "no RCE: the marker file was NOT created")) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/exporter_tmpdir_test.cljs b/exporter/test/exporter_tests/exporter_tmpdir_test.cljs new file mode 100644 index 0000000000..76b9fa9f1d --- /dev/null +++ b/exporter/test/exporter_tests/exporter_tmpdir_test.cljs @@ -0,0 +1,78 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-tmpdir-test + "The temp area owns what jobs park in it: every path a job tracks is + removed on its release, and the boot clean drops whatever a previous + process left behind." + (:require + ["node:fs" :as fs] + ["node:fs/promises" :as fsp] + ["node:os" :as os] + ["node:path" :as path] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [exporter.tmpdir :as tmpdir] + [exporter.utils.system :as system])) + +(defn- test-area + [] + (path/join (os/tmpdir) (str "penpot-tmpdir-test." (uuid/next)))) + +(defn- ^:async write + [dir name content] + (let [p (path/join dir name)] + (await (fsp/writeFile p content)) + p)) + +(t/deftest ^:async track-registers-and-release-removes + (let [dir (test-area) + _ (fs/mkdirSync dir #js {:recursive true}) + job-id (uuid/next) + owned (tmpdir/track job-id (await (write dir "penpot.a.txt" "a"))) + _ (tmpdir/track job-id (await (write dir "penpot.b.txt" "b")))] + (try + (t/is (= owned (path/join dir "penpot.a.txt"))) + (await (tmpdir/release job-id)) + (t/is (nil? (try (await (fsp/stat owned)) (catch :default _ nil))) + "tracked paths are gone after the release") + (t/testing "releasing twice, or an unknown job, is a no-op" + (await (tmpdir/release job-id)) + (await (tmpdir/release (uuid/next)))) + (finally + (await (fsp/rm dir #js {:recursive true :force true})))))) + +(t/deftest ^:async clean-orphans-removes-only-aged-managed-files + (let [dir (test-area) + _ (fs/mkdirSync dir #js {:recursive true}) + old (await (write dir "penpot.old.txt" "old")) + fresh (await (write dir "penpot.fresh.txt" "fresh")) + alien (await (write dir "other.txt" "other"))] + (try + (.utimesSync fs old (js/Date.) (js/Date. (- (js/Date.now) 7200000))) + (t/is (= 1 (await (tmpdir/clean-orphans dir 3600)))) + (t/testing "the aged managed file is gone" + (t/is (nil? (try (await (fsp/stat old)) (catch :default _ nil))))) + (t/testing "fresh managed and unmanaged files stay" + (t/is (some? (await (fsp/stat fresh)))) + (t/is (some? (await (fsp/stat alien))))) + (finally + (await (fsp/rm dir #js {:recursive true :force true})))))) + +(t/deftest ^:async init-ensures-the-dir-and-cleans-orphans + (let [dir (test-area)] + (try + (t/is (= dir (await (system/init-key :exporter/tmpdir {:path dir :job-ttl 3600})))) + (t/is (some? (await (fsp/stat dir)))) + (t/testing "a missing ttl fails fast instead of cleaning with a default" + (let [cause (try + (await (system/init-key :exporter/tmpdir {:path dir})) + nil + (catch :default cause + cause))] + (t/is (= :job-ttl-missing (:code (ex-data cause)))))) + (finally + (await (fsp/rm dir #js {:recursive true :force true})))))) diff --git a/exporter/test/exporter_tests/exporter_util_mime_test.cljs b/exporter/test/exporter_tests/exporter_util_mime_test.cljs new file mode 100644 index 0000000000..f0eef97725 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_util_mime_test.cljs @@ -0,0 +1,28 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-util-mime-test + "The mimetype and file extension of every artifact type the worker + settles: the suffix of the temp file and the mtype of the upload." + (:require + [cljs.test :as t :include-macros true] + [exporter.util.mime :as mime])) + +(t/deftest get-extension-names-every-artifact-type + (t/is (= ".png" (mime/get-extension :png))) + (t/is (= ".jpg" (mime/get-extension :jpeg))) + (t/is (= ".webp" (mime/get-extension :webp))) + (t/is (= ".svg" (mime/get-extension :svg))) + (t/is (= ".pdf" (mime/get-extension :pdf))) + (t/is (= ".zip" (mime/get-extension :zip)))) + +(t/deftest get-names-every-artifact-mimetype + (t/is (= "image/png" (mime/get :png))) + (t/is (= "image/jpeg" (mime/get :jpeg))) + (t/is (= "image/webp" (mime/get :webp))) + (t/is (= "image/svg+xml" (mime/get :svg))) + (t/is (= "application/pdf" (mime/get :pdf))) + (t/is (= "application/zip" (mime/get :zip)))) diff --git a/exporter/test/exporter_tests/exporter_util_object_test.cljs b/exporter/test/exporter_tests/exporter_util_object_test.cljs new file mode 100644 index 0000000000..70178b0f0a --- /dev/null +++ b/exporter/test/exporter_tests/exporter_util_object_test.cljs @@ -0,0 +1,62 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-util-object-test + "The javascript-object helpers the browser calls are built with: + reads that survive nil, writes that return the object, and the + clj→props translation of React prop names." + (:require + [cljs.test :as t :include-macros true] + [exporter.util.object :as obj])) + +(t/deftest get-reads-and-survives-nil + (let [o #js {:a 1}] + (t/is (= 1 (obj/get o "a"))) + (t/is (nil? (obj/get o "missing"))) + (t/is (nil? (obj/get nil "a"))) + (t/is (= :dflt (obj/get o "missing" :dflt))) + ;; the default fires on undefined only: a nil object reads nil. + ;; Same as the legacy (probe-verified), pinned so it stays that way. + (t/is (nil? (obj/get nil "a" :dflt))))) + +(t/deftest get-in-threads-nil-safely + (let [o #js {:a #js {:b 2}}] + (t/is (= 2 (obj/get-in o ["a" "b"]))) + (t/is (nil? (obj/get-in o ["a" "missing"]))) + (t/is (nil? (obj/get-in nil ["a"]))) + (t/is (= :dflt (obj/get-in o ["a" "missing"] :dflt))))) + +(t/deftest set-and-update-return-the-object + (let [o (obj/new)] + (t/is (identical? o (obj/set! o "a" 1))) + (t/is (= 1 (obj/get o "a"))) + (t/is (identical? o (obj/update o "a" inc))) + (t/is (= 2 (obj/get o "a"))) + ;; `update` on a missing key applies `f` to the sentinel instead of + ;; skipping it — the legacy does exactly the same (probe-verified + ;; `:app.util.object/not-found1`), so this pins parity, not intent: + ;; never "fix" one side without the other. + (t/is (identical? o (obj/update o "missing" (constantly :set)))) + (t/is (= :set (obj/get o "missing"))))) + +(t/deftest merge-copies-without-touching-the-inputs + (let [a #js {:x 1} + b #js {:y 2} + m (obj/merge a b)] + (t/is (= 1 (obj/get m "x"))) + (t/is (= 2 (obj/get m "y"))) + (t/is (nil? (obj/get a "y"))))) + +(t/deftest clj->props-camelizes-and-renames-class + (let [props (obj/clj->props {:background-color "red" :class "a"})] + (t/is (= "red" (obj/get props "backgroundColor"))) + (t/is (= "a" (obj/get props "className"))))) + +(t/deftest in-checks-ownership + (let [o #js {:a 1}] + (t/is (true? (obj/in? o "a"))) + (t/is (false? (obj/in? o "missing"))) + (t/is (true? (obj/contains? o "a"))))) diff --git a/exporter/test/exporter_tests/exporter_wasm_render_test.cljs b/exporter/test/exporter_tests/exporter_wasm_render_test.cljs new file mode 100644 index 0000000000..94510de247 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_wasm_render_test.cljs @@ -0,0 +1,199 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-wasm-render-test + "The headless render pipeline, without a render-wasm module: endpoint + building, error explaining, font-variant routing and the backend + fetches over a stubbed transport. The Skia calls themselves are + covered by the svg integration test." + (:require + [app.common.transit :as transit] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [exporter.wasm.render :as render])) + +(def ^:private env + {:internal-uri "http://internal/" + :public-uri "http://public/" + :management-key "mgmt-key" + :tmpdir "/tmp/penpot-wasm-render-test" + :image-cache-size 100}) + +(t/deftest endpoint-builders-join-against-the-env-uris + (t/is (= "http://internal/api/rpc/command/get-page" + (render/internal-uri env "api/rpc/command/get-page"))) + (t/is (= "http://public/assets/by-id/123" + (render/public-uri env "assets/by-id/123")))) + +(t/deftest error-detail-flattens-the-fetch-cause-chain + (let [root (doto (js/Error. "connect ECONNREFUSED") + (unchecked-set "code" "ECONNREFUSED")) + cause (doto (js/Error. "fetch failed") + (unchecked-set "cause" root))] + (t/is (= "fetch failed <- ECONNREFUSED: connect ECONNREFUSED" + (render/error-detail cause))) + (t/is (= "boom" (render/error-detail (js/Error. "boom")))))) + +(t/deftest find-variant-degrades-uuid-weight-then-uuid + (let [variants [{:font-id "f1" :font-weight 400 :font-style :normal} + {:font-id "f1" :font-weight 700 :font-style :italic}]] + (t/is (= {:font-id "f1" :font-weight 400 :font-style :normal} + (render/find-variant variants "f1" 400 0))) + (t/is (= {:font-id "f1" :font-weight 700 :font-style :italic} + (render/find-variant variants "f1" 700 1))) + (t/is (= {:font-id "f1" :font-weight 400 :font-style :normal} + (render/find-variant variants "f1" 400 1)) + "style degrades before weight") + (t/is (nil? (render/find-variant variants "f9" 400 0))))) + +(t/deftest family-uuid-roundtrips-through-u32 + ;; `:id` is a JS array (never a clj vector: `aget` on a vector + ;; reads `undefined`, which the u32 setter coerces to zero). + (let [id (uuid/next) + quart (uuid/get-u32 id)] + (t/is (= id (render/family-uuid #js [(aget quart 0) (aget quart 1) + (aget quart 2) (aget quart 3)]))))) + +(t/deftest scene-fallback-fonts-covers-emoji + (let [text-shape (fn [text] + {:type :text + :content {:children [{:children [{:children [{:text text}]}]}]}}) + latin {(uuid/next) (text-shape "hello")} + emoji {(uuid/next) (text-shape "hi 😀")}] + (t/is (= [] (render/scene-fallback-fonts latin))) + (t/is (true? (boolean (some :is-emoji (render/scene-fallback-fonts emoji))))))) + +(defn- stub-fetch + [calls respond] + (set! render/fetch + (fn [uri opts] + (swap! calls conj {:uri (str uri) :opts opts}) + (js/Promise.resolve (respond (str uri)))))) + +(t/deftest ^:async fetch-objects-decodes-the-get-page-answer + (let [calls (atom []) + original render/fetch + file-id (uuid/next) + page-id (uuid/next) + shape {:id "s1" :type :rect}] + (stub-fetch calls + (fn [_uri] + #js {:status 200 + :text (constantly + (js/Promise.resolve + (transit/encode-str {:objects {"s1" shape}})))})) + (try + (let [objects (await (render/fetch-objects + env {:file-id file-id + :page-id page-id + :token "tok" + :objects [{:id "s1"}]}))] + (t/testing "the scene travels back" + (t/is (= {"s1" shape} objects))) + (t/testing "the call rode the internal rpc with auth" + (let [{:keys [uri opts]} (first @calls)] + (t/is (= "http://internal/api/rpc/command/get-page" uri)) + (t/is (= "Bearer tok" (unchecked-get (.-headers opts) "Authorization"))) + (t/testing "and no shared key, which the rpc routes do not read" + (t/is (nil? (unchecked-get (.-headers opts) "X-Shared-Key"))))))) + (finally + (set! render/fetch original))))) + +(t/deftest ^:async fetch-font-variants-degrades-to-nil + (let [original render/fetch] + (try + (stub-fetch (atom []) + (fn [_uri] #js {:status 500 + :text (constantly (js/Promise.resolve ""))})) + (t/is (nil? (await (render/fetch-font-variants env {:file-id (uuid/next) + :token "tok"}))) + "a non-200 is fallback fonts, not a failed export") + (set! render/fetch (fn [_uri _opts] (js/Promise.reject (js/Error. "down")))) + (t/is (nil? (await (render/fetch-font-variants env {:file-id (uuid/next) + :token "tok"}))) + "a transport failure is fallback fonts too") + (finally + (set! render/fetch original))))) + +(t/deftest ^:async fetch-file-media-bytes-passes-bytes-through + (let [original render/fetch] + (try + (stub-fetch (atom []) + (fn [_uri] #js {:status 200 + :arrayBuffer (constantly + (js/Promise.resolve + (.from js/Buffer "img")))})) + (let [buf (await (render/fetch-file-media-bytes env (uuid/next) "tok" nil))] + (t/is (= 3 (.-byteLength buf)))) + (stub-fetch (atom []) + (fn [_uri] #js {:status 404 + :arrayBuffer (constantly + (js/Promise.resolve nil))})) + (t/is (nil? (await (render/fetch-file-media-bytes env (uuid/next) "tok" nil)))) + (finally + (set! render/fetch original))))) + +(t/deftest ^:async asset-fetches-carry-the-share + ;; The asset endpoints decide permissions by `share-id`: without it a + ;; share viewer's image or font fetch is a 404 and the export silently + ;; loses it. + (let [original render/fetch + calls (atom []) + media-id (uuid/next) + share-id (uuid/next)] + (stub-fetch calls + (fn [_uri] #js {:status 200 + :arrayBuffer (constantly + (js/Promise.resolve + (.from js/Buffer "img")))})) + (try + (await (render/fetch-file-media-bytes env media-id "tok" share-id)) + (t/is (= (str "http://internal/assets/by-file-media-id/" media-id + "?share-id=" share-id) + (:uri (first @calls)))) + (t/testing "and a plain export keeps the plain URL" + (reset! calls []) + (await (render/fetch-file-media-bytes env media-id "tok" nil)) + (t/is (= (str "http://internal/assets/by-file-media-id/" media-id) + (:uri (first @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))) + (finally + (set! render/fetch original))))) + +(t/deftest invalidate-module-on-error-spares-cooperative-cancels + (t/is (false? (render/invalidate-module-on-error? + (ex-info "export job was cancelled" + {:type :internal :code :job-cancelled}))) + "a cancel never invalidates the module the next render reuses") + (t/is (true? (render/invalidate-module-on-error? (js/Error. "boom"))) + "any other failure still drops a possibly poisoned module")) + +(t/deftest ^:async custom-font-fetch-carries-the-session-token-as-cookie + (let [original render/fetch + calls (atom []) + font-id (uuid/next) + asset-id (uuid/next) + variant {:font-id font-id :font-weight 400 :font-style :normal + :ttf-file-id asset-id}] + (stub-fetch calls + (fn [_uri] #js {:status 200 + :arrayBuffer (constantly + (js/Promise.resolve + (.from js/Buffer "ttf")))})) + (try + (let [resolve-font (render/make-resolve-font env [variant] {:token "tok"}) + _ (await (resolve-font {:id (uuid/get-u32 font-id) + :weight 400 + :style 0}))] + (t/testing "the asset fetch rode the internal endpoint with the session cookie" + (let [{:keys [uri opts]} (first @calls)] + (t/is (= (str "http://internal/assets/by-id/" asset-id) uri)) + (t/is (= "auth-token=tok" (unchecked-get (.-headers opts) "Cookie"))) + (t/testing "and no shared key, which these routes do not read" + (t/is (nil? (unchecked-get (.-headers opts) "X-Shared-Key"))))))) + (finally + (set! render/fetch original))))) diff --git a/exporter/test/exporter_tests/renderer_svg_test.cljs b/exporter/test/exporter_tests/exporter_wasm_svg_test.cljs similarity index 64% rename from exporter/test/exporter_tests/renderer_svg_test.cljs rename to exporter/test/exporter_tests/exporter_wasm_svg_test.cljs index 7d6bd68899..cf489f255e 100644 --- a/exporter/test/exporter_tests/renderer_svg_test.cljs +++ b/exporter/test/exporter_tests/exporter_wasm_svg_test.cljs @@ -4,30 +4,30 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns exporter-tests.renderer-svg-test +(ns exporter-tests.exporter-wasm-svg-test + "The SVG output of the headless pipeline over the real render-wasm + module: gradient defs and svg-attr fills survive the serialization." (:require - [app.renderer.svg-gradient :as svg-gradient] - [app.wasm :as wasm-io] - [app.wasm.serialize :as serialize] - [cljs.test :refer [deftest is testing async]] + [cljs.test :as t :include-macros true] [clojure.string :as str] - [promesa.core :as p])) + [exporter.renderer.svg-gradient :as svg-gradient] + [exporter.wasm :as wasm-io] + [exporter.wasm.serialize :as serialize])) (def gradient-stops [{"color" "#000000" "offset" 0 "opacity" 1} {"color" "#ffffff" "offset" 1 "opacity" 1}]) -(deftest creates-the-correct-gradient-element +(t/deftest creates-the-correct-gradient-element (doseq [[gradient-type element-name] [["linear" "linearGradient"] ["radial" "radialGradient"]]] - (testing gradient-type + (t/testing gradient-type (let [gradient-data {"type" "gradient" "gradient" {"type" gradient-type "stops" gradient-stops}} result (svg-gradient/data->gradient-def "text-id" ["#000001" gradient-data])] - (is (= element-name (get result "name"))))))) - + (t/is (= element-name (get result "name"))))))) (def ^:private svg-red-id (random-uuid)) (def ^:private blue-id (random-uuid)) @@ -56,17 +56,12 @@ [shape-id] (.toString (js/Buffer.from (wasm-io/render-shape-svg shape-id 1)))) -(deftest exporter-honours-svg-attr-fills - (async done - (->> (wasm-io/init!) - (p/mcat - (fn [_] - (serialize/serialize-scene! (scene)) - (let [red (render-svg-string svg-red-id) - blue (render-svg-string blue-id)] - (is (str/includes? blue "blue") - "control: user fills render, so the harness works") - (is (str/includes? red "fill=\"red\"") - "svg-attr fill survives headless serialization") - (p/resolved nil)))) - (p/mcat (fn [_] (done)))))) +(t/deftest ^:async exporter-honours-svg-attr-fills + (await (wasm-io/init)) + (serialize/serialize-scene (scene)) + (let [red (render-svg-string svg-red-id) + blue (render-svg-string blue-id)] + (t/is (str/includes? blue "blue") + "control: user fills render, so the harness works") + (t/is (str/includes? red "fill=\"red\"") + "svg-attr fill survives headless serialization"))) diff --git a/exporter/test/exporter_tests/exporter_wasm_worker_test.cljs b/exporter/test/exporter_tests/exporter_wasm_worker_test.cljs new file mode 100644 index 0000000000..8415e8c845 --- /dev/null +++ b/exporter/test/exporter_tests/exporter_wasm_worker_test.cljs @@ -0,0 +1,126 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.exporter-wasm-worker-test + "The render worker entry over the real render-wasm module: one render + message runs the pipeline and posts every object plus the settle, a + failed pipeline posts the error. The backend travels stubbed." + (:require + ["node:fs/promises" :as fsp] + ["node:os" :as os] + ["node:path" :as path] + [app.common.transit :as transit] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [exporter.wasm.render :as render] + [exporter.wasm.worker :as worker])) + +(def ^:private shape-id (random-uuid)) + +(defn- scene-shape + [] + {:id shape-id + :type :rect + :name "svg-red" + :x 10 :y 10 :width 100 :height 100 + :rotation 0 + :selrect {:x 10 :y 10 :x1 10 :y1 10 :x2 110 :y2 110 + :width 100 :height 100} + :strokes [] + :svg-attrs {:fill "#ff0000"}}) + +(defn- test-env + [tmpdir] + {:internal-uri "http://internal/" + :public-uri "http://public/" + :management-key "mgmt-key" + :tmpdir tmpdir + :image-cache-size 100}) + +(defn- test-params + [tmpdir] + {:file-id (uuid/next) + :page-id (uuid/next) + :token "tok" + :type :png + :scale 1 + :objects [{:id shape-id :name "r" :suffix "" :filename "r.png"}] + ::render/env (test-env tmpdir)}) + +(defn- install-port + [posts] + (let [original worker/parent-port] + (set! worker/parent-port + #js {:postMessage (fn [msg] (swap! posts conj (js->clj msg)))}) + (fn [] (set! worker/parent-port original)))) + +(defn- install-fetch + [respond] + (let [original render/fetch] + (set! render/fetch + (fn [uri opts] + (js/Promise.resolve (respond (str uri) opts)))) + (fn [] (set! render/fetch original)))) + +(defn- scene-fetch + [shape] + (install-fetch + (fn [uri _opts] + (cond + (str/ends-with? uri "get-page") + #js {:status 200 + :text (constantly + (js/Promise.resolve + (transit/encode-str {:objects {(str (:id shape)) shape}})))} + + :else + #js {:status 404 + :text (constantly (js/Promise.resolve "")) + :arrayBuffer (constantly (js/Promise.resolve nil))})))) + +(t/deftest ^:async handle-render-posts-objects-and-done + (let [tmpdir (path/join (os/tmpdir) (str "penpot-wasm-worker-test." (uuid/next)))] + (await (fsp/mkdir tmpdir #js {:recursive true})) + (let [posts (atom []) + restore-port (install-port posts) + restore-fetch (scene-fetch (scene-shape))] + (try + (await (worker/handle-render #js {:type "render" + :params (transit/encode-str (test-params tmpdir)) + :cancel nil})) + (let [by-type (group-by #(get % "type") @posts)] + (t/testing "one object posted with its file" + (t/is (= 1 (count (get by-type "object")))) + (let [object (transit/decode-str (get (first (get by-type "object")) "payload"))] + (t/is (= shape-id (:id object))) + (t/is (str/starts-with? (:path object) tmpdir)))) + (t/testing "the settle closed the lease" + (t/is (= 1 (count (get by-type "done")))) + (t/is (nil? (get by-type "error"))))) + (finally + (restore-fetch) + (restore-port) + (await (fsp/rm tmpdir #js {:recursive true :force true}))))))) + +(t/deftest ^:async handle-render-posts-the-error-never-rejects + (let [posts (atom []) + restore-port (install-port posts) + restore-fetch (install-fetch (fn [_uri _opts] + (js/Promise.reject (js/Error. "backend down"))))] + (try + (await (worker/handle-render #js {:type "render" + :params (transit/encode-str + (test-params (os/tmpdir))) + :cancel nil})) + (let [by-type (group-by #(get % "type") @posts)] + (t/is (nil? (get by-type "done"))) + (t/is (= 1 (count (get by-type "error")))) + (t/is (str/includes? (get (first (get by-type "error")) "message") + "http fetch failed"))) + (finally + (restore-fetch) + (restore-port))))) diff --git a/exporter/test/exporter_tests/jobs_test.cljs b/exporter/test/exporter_tests/jobs_test.cljs deleted file mode 100644 index a92c1c1790..0000000000 --- a/exporter/test/exporter_tests/jobs_test.cljs +++ /dev/null @@ -1,107 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns exporter-tests.jobs-test - "Job state machine. Runs without redis: a store write with no connection is - reported and swallowed, so only the in-process record is exercised." - (:require - [app.common.uuid :as uuid] - [app.config :as cf] - [app.jobs :as jobs] - [cljs.test :as t :include-macros true] - [promesa.core :as p])) - -(defn- create! - [] - (jobs/create! {:profile-id (uuid/next) - :cmd :export-shapes - :backend "wasm" - :total 10 - :name "test" - :resource-id (uuid/next)} - (constantly (p/resolved nil)))) - -(t/deftest progress-does-not-resurrect-a-finished-job - (t/testing "a render reporting after the export failed cannot undo the failure" - (t/async done - (p/let [job (create!) - _ (jobs/start! job) - _ (jobs/fail! (jobs/lookup (:id job)) (ex-info "boom" {})) - ;; `job` is the snapshot handed to the work when it started, which - ;; is what a straggling render still holds. - _ (jobs/progress! job 7)] - (let [current (jobs/lookup (:id job))] - (t/is (= "error" (:state current))) - (t/is (= "boom" (:error current))) - (t/is (not= 7 (:done current)))) - (jobs/release! (:id job)) - (done))))) - -(t/deftest first-terminal-state-wins - (t/testing "a failure arriving after a cancellation leaves the job cancelled" - (t/async done - (p/let [job (create!) - _ (jobs/start! job) - _ (jobs/cancel! (:id job)) - _ (jobs/fail! job (ex-info "too late" {}))] - (let [current (jobs/lookup (:id job))] - (t/is (= "cancelled" (:state current))) - (t/is (nil? (:error current)))) - (jobs/release! (:id job)) - (done))))) - -(t/deftest cancel-is-recorded-before-the-callbacks-run - (t/testing "a queued job dropped by its own cancel callback still ends cancelled" - (t/async done - (let [seen (atom ::not-called)] - (p/let [job (create!) - ;; What `scheduler/drop-queued!` does: it takes the job off the - ;; queue and releases it. Anything the lifecycle wrote after - ;; the callbacks ran would be dropped on the floor, so by the - ;; time one is called the record has to be terminal already. - _ (jobs/on-cancel (:id job) - (fn [] - (reset! seen (:state (jobs/lookup (:id job)))) - (jobs/release! (:id job)))) - _ (jobs/cancel! (:id job))] - (t/is (= "cancelled" @seen)) - (t/is (nil? (jobs/lookup (:id job)))) - (done)))))) - -(t/deftest writes-stop-once-the-job-is-released - (t/testing "a late write for a job the scheduler already settled is dropped" - (t/async done - (p/let [job (create!) - _ (jobs/start! job) - _ (jobs/complete! (jobs/lookup (:id job)) {:uri "http://example/x" - :filename "x.zip" - :mtype "application/zip"}) - ended (jobs/lookup (:id job)) - _ (jobs/release! (:id job)) - _ (jobs/progress! job 3)] - (t/is (= "ended" (:state ended))) - (t/is (nil? (jobs/lookup (:id job)))) - (done))))) - -(t/deftest boot-sweep-leaves-sibling-jobs-alone - (let [abandoned? #'jobs/abandoned?] - (t/is (abandoned? {:replica cf/replica-id :state "running"})) - (t/is (abandoned? {:replica cf/replica-id :state "queued"})) - (t/is (not (abandoned? {:replica cf/replica-id :state "ended"}))) - (t/is (not (abandoned? {:replica (str cf/replica-id "-sibling") :state "running"}))))) - -(t/deftest shutdown-cancels-all-its-jobs - (t/async done - (p/let [running (create!) - queued (create!) - _ (jobs/start! running) - _ (jobs/cancel-all!)] - (doseq [job [running queued]] - (let [current (jobs/lookup (:id job))] - (t/is (= "cancelled" (:state current))) - (t/is (true? (:interrupted current)))) - (jobs/release! (:id job))) - (done)))) diff --git a/exporter/test/exporter_tests/main_test.cljs b/exporter/test/exporter_tests/main_test.cljs new file mode 100644 index 0000000000..ebbb1219ed --- /dev/null +++ b/exporter/test/exporter_tests/main_test.cljs @@ -0,0 +1,202 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.main-test + "The entry point of the new tree: the namespace loads, and + start/stop/restart manage the running system." + (:require + ["node:os" :as os] + ["node:path" :as path] + ["node:process" :as proc] + [app.common.logging :as l] + [cljs.test :as t :include-macros true] + [exporter.browser :as browser] + [exporter.config :as cf] + [exporter.consumer.config :as ccfg] + [exporter.main :as main] + [exporter.utils.system :as system])) + +(t/deftest new-tree-loads + (t/testing "exporter.main resolves once required" + (t/is (some? (find-ns 'exporter.main))))) + +(t/deftest entry-logger-allows-info + (t/testing "the entry point leaves info logging enabled for the new tree" + (t/is (true? (l/enabled? "exporter.main" :info))))) + +(defn- stub-context + [calls] + #js {:newPage (fn [] (swap! calls conj :new-page) (js/Promise.resolve #js {})) + :close (fn [] (swap! calls conj :close-context) (js/Promise.resolve nil))}) + +(defn- stub-browser + [calls] + #js {:newContext (fn [_opts] + (swap! calls conj :new-context) + (js/Promise.resolve (stub-context calls))) + :isConnected (fn [] true) + :close (fn [] + (swap! calls conj :close-browser) + (js/Promise.resolve nil))}) + +(defn- fake-config + [calls] + {:exporter.browser/pool {:max 1 + :create-browser (fn [] + (swap! calls conj :create-browser) + (js/Promise.resolve (stub-browser calls)))} + :exporter.wasm/pool {:max 1 + :min 0 + :create-worker (fn [] + (swap! calls conj :create-worker) + (js/Promise.resolve #js {}))} + :exporter/renderer {:exporter.browser/pool (system/ref :exporter.browser/pool) + :exporter.wasm/pool (system/ref :exporter.wasm/pool) + :exporter/tmpdir (system/ref :exporter/tmpdir) + :base-uri "http://internal/" + :public-uri "http://public/" + :svgo? false} + :exporter/tmpdir {:path (path/join (os/tmpdir) "penpot-exporter-test") + :job-ttl 3600}}) + +(defn- pool-of + [] + (:exporter.browser/pool @main/system)) + +(defn- wasm-pool-of + [] + (:exporter.wasm/pool @main/system)) + +(t/deftest ^:async install-process-handlers-registers-handlers + (try + (let [listeners (fn [event] (.listenerCount proc/default event)) + current (fn [] (mapv listeners ["uncaughtException" "SIGTERM" "SIGINT"])) + before (current)] + (main/install-process-handlers) + (t/is (= (mapv inc before) (current))) + (t/testing "a second install adds nothing: reloads must not pile listeners" + (main/install-process-handlers) + (t/is (= (mapv inc before) (current))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async hot-reload-stop-halts-through-done + (try + (let [calls (atom [])] + (await (main/start-custom (fake-config calls))) + (await (browser/exec (pool-of) {} (fn [_] :ok))) + (let [stopped (js/Promise. (fn [resolve _] (main/on-before-load (fn [] (resolve :done)))))] + (t/is (= :done (await stopped)))) + (t/is (nil? @main/system)) + (t/is (some #{:close-browser} @calls)) + (t/is (= :stopped (await (main/stop))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest system-config-is-built-at-start-not-at-load + ;; The wiring reads the knobs when it is built, so a reboot picks up + ;; configuration changes without reloading namespaces: with a def + ;; this test would see the load-time values instead. + (with-redefs [ccfg/concurrency (fn ([] 7) ([_] 7))] + (t/is (= {:max 7} + (:exporter.browser/pool (main/system-config)))) + (t/is (= 7 (:concurrency (:exporter.consumer/worker (main/system-config))))))) + +(t/deftest production-wiring-declares-both-pools + (t/testing "the wiring names the browser and the wasm pools, sized to the worker concurrency" + (t/is (= {:max (ccfg/concurrency)} + (:exporter.browser/pool (main/system-config)))) + (t/testing "and the wasm pool carries the knobs operators keep warm" + (t/is (= {:max (ccfg/concurrency) + :min (cf/get :wasm-worker-pool-min) + :idle-timeout (* 1000 (cf/get :wasm-worker-idle-timeout))} + (:exporter.wasm/pool (main/system-config)))))) + (t/testing "the renderer owns the render config, pools by ref" + (let [renderer (:exporter/renderer (main/system-config))] + (t/is (= {:exporter.browser/pool (system/ref :exporter.browser/pool) + :exporter.wasm/pool (system/ref :exporter.wasm/pool)} + (select-keys renderer + [:exporter.browser/pool :exporter.wasm/pool]))) + (t/is (= (cf/get-internal-uri) (:base-uri renderer))) + (t/is (= (cf/get :public-uri) (:public-uri renderer))) + (t/is (= (cf/get :wasm-worker-image-cache-size) (:image-cache-size renderer))))) + (t/testing "the queue consumer names the renderer it renders through" + (let [worker (:exporter.consumer/worker (main/system-config))] + (t/is (= (ccfg/concurrency) (:concurrency worker))) + (t/is (= (ccfg/queue-key) (:queue-key worker))) + (t/is (= (system/ref :exporter/renderer) (:renderer worker))) + (t/is (= (system/ref :exporter/tmpdir) (:exporter/tmpdir worker))))) + (t/testing "the temp area is wired from the environment" + (t/is (= {:path (cf/get :tempdir) + :job-ttl (cf/get :exporter-job-ttl 3600)} + (:exporter/tmpdir (main/system-config)))))) + +;; NOTE: no test boots the production wiring: the wasm pool warms one +;; worker eagerly (min 1), and in this process the worker script would +;; be the test bundle itself. The hook path stays covered through the +;; fake configs below; the production map above is data, asserted +;; without booting it. + +(t/deftest ^:async start-boots-and-stop-halts + (try + (let [calls (atom [])] + (t/is (= :started (await (main/start-custom (fake-config calls))))) + (t/is (some? (pool-of))) + (t/testing "the renderer boots as a render fn over the resolved pools" + (t/is (fn? (:exporter/renderer @main/system)))) + (t/testing "the temp area boots before anything that writes to it" + (t/is (= (path/join (os/tmpdir) "penpot-exporter-test") + (:exporter/tmpdir @main/system)))) + (t/testing "the wasm pool rides the same lifecycle" + (t/is (some? (:pool (wasm-pool-of)))) + (t/is (= 300000 (:timeout-ms (wasm-pool-of))))) + (await (browser/exec (pool-of) {} (fn [_] :ok))) + (t/is (= :stopped (await (main/stop)))) + (t/is (nil? @main/system)) + (t/is (some #{:close-browser} @calls))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async restart-reboots-the-same-config + (try + (let [calls (atom [])] + (await (main/start-custom (fake-config calls))) + (await (browser/exec (pool-of) {} (fn [_] :ok))) + (t/is (= :restarted (await (main/restart)))) + (t/is (some #{:close-browser} @calls)) + (t/is (= :ok (await (browser/exec (pool-of) {} (fn [_] :ok))))) + (t/is (= 2 (count (filter #{:create-browser} @calls)))) + (await (main/stop))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async second-start-replaces-the-running-system + (try + (let [calls (atom [])] + (await (main/start-custom (fake-config calls))) + (await (browser/exec (pool-of) {} (fn [_] :ok))) + (t/is (= :started (await (main/start-custom (fake-config calls))))) + (t/is (some #{:close-browser} @calls)) + (t/is (= :ok (await (browser/exec (pool-of) {} (fn [_] :ok))))) + (await (main/stop))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async shutdown-halts-the-system-then-exits + (try + (let [calls (atom []) + exited (atom nil)] + (try + (await (main/start-custom (fake-config calls))) + (await (browser/exec (pool-of) {} (fn [_] :ok))) + (await (main/shutdown (fn [code] (reset! exited code)))) + (t/is (= 0 @exited)) + (t/is (nil? @main/system)) + (t/is (some #{:close-browser} @calls)) + (catch :default cause + (throw cause)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/renderer_browser_test.cljs b/exporter/test/exporter_tests/renderer_browser_test.cljs new file mode 100644 index 0000000000..f7337fd4fd --- /dev/null +++ b/exporter/test/exporter_tests/renderer_browser_test.cljs @@ -0,0 +1,208 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.renderer-browser-test + "The browser renderer: bitmap, pdf and svg flows through a real + pool with fake pages. No text nodes in the svg fixture means no + shell-outs; the exported file itself proves the reassembly." + (:require + ["generic-pool" :as gp] + ["node:os" :as os] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [exporter.config :as cf] + [exporter.renderer.browser :as render] + [exporter.renderer.svg-gradient :as grad] + [exporter.shell :as shell])) + +(defn- stub-locator + [calls evaluate-fn] + #js {:waitFor (fn [_opts] (swap! calls conj :locator-wait) (js/Promise.resolve nil)) + :screenshot (fn [opts] + (swap! calls conj [:screenshot (unchecked-get opts "path")]) + (js/Promise.resolve nil)) + :evaluate (fn [f & _args] (js/Promise.resolve (evaluate-fn f)))}) + +(defn- stub-page + [calls locator] + #js {:goto (fn [url _opts] + (swap! calls conj [:goto (str url)]) + (js/Promise.resolve nil)) + :waitForTimeout (fn [ms] (swap! calls conj [:sleep ms]) (js/Promise.resolve nil)) + :waitForFunction (fn [_src _arg _opts] + (swap! calls conj :wait-for-fonts) + (js/Promise.resolve nil)) + :locator (fn [sel] (swap! calls conj [:select sel]) locator) + :$$ (fn [sel] + (swap! calls conj [:select-all sel]) + (js/Promise.resolve #js [])) + :evaluate (fn [_f & _args] (swap! calls conj :page-eval) (js/Promise.resolve nil)) + :pdf (fn [opts] + (swap! calls conj [:pdf (unchecked-get opts "path")]) + (js/Promise.resolve nil))}) + +(defn- stub-context + [calls page] + #js {:newPage (fn [] (js/Promise.resolve page)) + :close (fn [] (swap! calls conj :close-context) (js/Promise.resolve nil))}) + +(defn- stub-browser + [calls page] + #js {:newContext (fn [_opts] (js/Promise.resolve (stub-context calls page))) + :isConnected (fn [] true) + :close (fn [] (js/Promise.resolve nil))}) + +(defn- test-pool + [calls page] + (gp/createPool #js {:create (fn [] (js/Promise.resolve (stub-browser calls page))) + :destroy (fn [browser] (.close ^js browser)) + :validate (fn [_] true)} + #js {:max 1 :min 0})) + +(defn- test-cfg + [pool] + {:exporter.browser/pool pool + :exporter/tmpdir (os/tmpdir) + :base-uri (cf/get-internal-uri) + :public-uri (cf/get :public-uri) + :svgo? false}) + +(defn- never-cancelled + [] + (fn [] nil)) + +(defn- raising-check + [calls raise-on-call] + (let [seen (atom 0)] + (fn [] + (swap! calls conj :check-cancelled) + (when (= raise-on-call (swap! seen inc)) + (throw (ex-info "export job was cancelled" + {:type :internal + :code :job-cancelled + :hint "export job was cancelled"})))))) + +(defn- test-object + [id] + {:id id :name id}) + +(defn- test-params + [type ids] + {:file-id (uuid/next) + :page-id (uuid/next) + :token "token" + :scale 1 + :type type + :objects (mapv test-object ids)}) + +(defn- collect-on-object + [] + (let [seen (atom [])] + {:seen seen + :fn (fn [object] (swap! seen conj object) nil)})) + +(t/deftest ^:async renders-bitmaps-through-one-page + (try + (let [calls (atom []) + page (stub-page calls (stub-locator calls (fn [_] nil))) + pool (test-pool calls page) + {:keys [seen fn]} (collect-on-object)] + (await (render/render (test-cfg pool) (test-params :png ["a" "b"]) fn (never-cancelled))) + (t/is (= 2 (count @seen))) + (t/is (every? #(str/ends-with? (:path %) ".png") @seen)) + (t/is (= ["a" "b"] (mapv :id @seen))) + (t/testing "one navigation for the whole partition" + (t/is (= 1 (count (filter #(and (vector? %) (= :goto (first %))) @calls)))) + (t/is (str/includes? (second (first (filter #(and (vector? %) (= :goto (first %))) @calls))) + "render.html"))) + (t/is (some #{[:select "#screenshot-a"]} @calls)) + (t/is (some #{[:select "#screenshot-b"]} @calls))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async renders-pdf-navigating-per-object + (try + (let [calls (atom []) + page (stub-page calls (stub-locator calls (fn [_] nil))) + pool (test-pool calls page) + {:keys [seen fn]} (collect-on-object)] + (await (render/render (test-cfg pool) (test-params :pdf ["a" "b"]) fn (never-cancelled))) + (t/is (= 2 (count @seen))) + (t/is (every? #(str/ends-with? (:path %) ".pdf") @seen)) + (t/testing "one navigation per object" + (t/is (= 2 (count (filter #(and (vector? %) (= :goto (first %))) @calls))))) + (t/is (= 2 (count (filter #(and (vector? %) (= :pdf (first %))) @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async cancelled-render-rejects-before-navigating + (try + (let [calls (atom []) + page (stub-page calls (stub-locator calls (fn [_] nil))) + pool (test-pool calls page) + cfg (test-cfg pool)] + (try + (await (render/render cfg (test-params :png ["a"]) + (fn [_] (swap! calls conj :object)) + (raising-check calls 1))) + (t/is false "render should have rejected a cancelled render") + (catch :default cause + (t/is (= :job-cancelled (-> cause ex-data :code))) + (t/is (not-any? #(and (vector? %) (= :goto (first %))) @calls)) + (t/is (not-any? #{:object} @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async cancel-mid-batch-stops-the-pdf-loop + (try + (let [calls (atom []) + page (stub-page calls (stub-locator calls (fn [_] nil))) + pool (test-pool calls page) + ;; entry check is the first call, then one per object: + ;; the cancel lands as the second object comes up + cfg (test-cfg pool) + {:keys [seen fn]} (collect-on-object)] + (try + (await (render/render cfg (test-params :pdf ["a" "b"]) fn (raising-check calls 3))) + (t/is false "render should have rejected a cancelled batch") + (catch :default cause + (t/is (= :job-cancelled (-> cause ex-data :code))) + (t/is (= ["a"] (mapv :id @seen))) + (t/is (= 1 (count (filter #(and (vector? %) (= :goto (first %))) @calls))))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async renders-svg-reassembling-the-document + (try + (let [calls (atom []) + outer " " + locator (stub-locator calls (fn [f] (f #js {:outerHTML outer}))) + page (stub-page calls locator) + pool (test-pool calls page) + {:keys [seen fn]} (collect-on-object) + _ (await (render/render (test-cfg pool) (test-params :svg ["a"]) fn (never-cancelled))) + path (:path (first @seen)) + content (await (shell/read-file path))] + (t/is (= 1 (count @seen))) + (t/is (str/ends-with? path ".svg")) + (t/is (str/includes? content "")) + (t/is (not (str/includes? content " ")))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest nbsp-entity-becomes-its-svg-compatible-form + (t/is (= "a b" (render/sanitize-svg-content "a b"))) + (t/is (= "plain" (render/sanitize-svg-content "plain")))) + +(t/deftest gradient-def-names-shapes + (let [stops [{"color" "#000000" "offset" 0 "opacity" 1}] + linear (grad/data->gradient-def "tid" ["#ff0000" {"gradient" {"type" "linear" "stops" stops}}]) + radial (grad/data->gradient-def "tid" ["#00ff00" {"gradient" {"type" "radial" "stops" stops}}])] + (t/is (= "linearGradient" (get linear "name"))) + (t/is (= "radialGradient" (get radial "name"))) + (t/is (= "gradient-tid-ff0000" (get-in linear ["attributes" "id"]))) + (t/is (= 1 (count (get linear "elements")))))) diff --git a/exporter/test/exporter_tests/renderer_test.cljs b/exporter/test/exporter_tests/renderer_test.cljs new file mode 100644 index 0000000000..d798e5af55 --- /dev/null +++ b/exporter/test/exporter_tests/renderer_test.cljs @@ -0,0 +1,300 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.renderer-test + "The renderer facade: malli validation and dispatch to the browser + or wasm drivers, against a stub browser pool and a stub wasm + pool." + (:require + ["generic-pool" :as gp] + ["node:events" :as events] + ["node:os" :as os] + [app.common.transit :as transit] + [app.common.uuid :as uuid] + [cljs.test :as t :include-macros true] + [exporter.config :as cf] + [exporter.renderer :as renderer])) + +;; --- stub browser side (bitmap only: navigate once, shoot objects) --- + +(defn- stub-locator + [calls] + #js {:waitFor (fn [_opts] (js/Promise.resolve nil)) + :screenshot (fn [opts] + (swap! calls conj [:screenshot (unchecked-get opts "path")]) + (js/Promise.resolve nil))}) + +(defn- stub-page + [calls] + #js {:goto (fn [url _opts] + (swap! calls conj [:goto (str url)]) + (js/Promise.resolve nil)) + :waitForTimeout (fn [ms] (swap! calls conj [:sleep ms]) (js/Promise.resolve nil)) + :waitForFunction (fn [_src _arg _opts] (js/Promise.resolve nil)) + :locator (fn [sel] (swap! calls conj [:select sel]) (stub-locator calls)) + :$$ (fn [sel] + (swap! calls conj [:select-all sel]) + (js/Promise.resolve #js [])) + :evaluate (fn [_f & _args] (js/Promise.resolve nil))}) + +(defn- stub-context + [page] + #js {:newPage (fn [] (js/Promise.resolve page)) + :close (fn [] (js/Promise.resolve nil))}) + +(defn- stub-browser + [page] + #js {:newContext (fn [_opts] (js/Promise.resolve (stub-context page))) + :isConnected (fn [] true) + :close (fn [] (js/Promise.resolve nil))}) + +(defn- stub-browser-pool + [calls] + (let [page (stub-page calls)] + (gp/createPool #js {:create (fn [] (js/Promise.resolve (stub-browser page))) + :destroy (fn [browser] (.close ^js browser)) + :validate (fn [_] true)} + #js {:max 1 :min 0}))) + +;; --- stub wasm side (one worker answering object+done) --- + +(defn- stub-worker + [calls] + (let [^js emitter (events/EventEmitter.)] + (unchecked-set emitter "postMessage" + (fn [msg] + (swap! calls conj [:post (unchecked-get msg "type")]) + (js/setTimeout + (fn [] + (.emit emitter "message" #js {:type "object" + :payload (transit/encode-str {:id "w1" :path "/tmp/w1.png"})}) + (.emit emitter "message" #js {:type "done"})) + 0))) + (unchecked-set emitter "terminate" + (fn [] + (swap! calls conj :terminate) + (js/Promise.resolve nil))) + emitter)) + +(defn- stub-wasm-pool + [calls worker] + #js {:acquire (fn [] (swap! calls conj :acquire) (js/Promise.resolve worker)) + :release (fn [_] (swap! calls conj :release) (js/Promise.resolve nil)) + :destroy (fn [_] (swap! calls conj :destroy) (js/Promise.resolve nil))}) + +(defn- stub-check + [calls] + (fn [] + (swap! calls conj :check-cancelled))) + +(defn- test-cfg + [calls] + (let [worker (stub-worker calls)] + {:exporter.browser/pool (stub-browser-pool calls) + :exporter/tmpdir (os/tmpdir) + :base-uri (cf/get-internal-uri) + :public-uri (cf/get :public-uri) + :management-key "mgmt-key" + :image-cache-size 100 + :svgo? false + :exporter.wasm/pool {:pool (stub-wasm-pool calls worker) :timeout-ms 5000}})) + +(defn- never-cancelled + [calls] + (stub-check calls)) + +(defn- test-params + [type opts] + (merge {:file-id (uuid/next) + :page-id (uuid/next) + :token "token" + :scale 1 + :type type + :name "shape-a" + :objects [{:id (uuid/uuid "1ac0612f-1ee8-8023-8008-d07a2aa96040") + :name "a" :suffix ".png" :filename "a.png"}]} + opts)) + +(t/deftest wasm-reads-the-wasm-flag + (t/is (true? (renderer/wasm? {:is-wasm true :type :png}))) + (t/is (false? (renderer/wasm? {:type :png}))) + (t/is (false? (renderer/wasm? {:is-wasm false :type :png})))) + +(t/deftest ^:async render-routes-bitmaps-to-the-browser + (try + (let [calls (atom []) + seen (atom [])] + (await ((renderer/render-fn (test-cfg calls)) + {:exports [(test-params :png nil)] + :on-object (fn [object] (swap! seen conj object) nil) + :check-cancelled (never-cancelled calls)})) + (t/is (= [(uuid/uuid "1ac0612f-1ee8-8023-8008-d07a2aa96040")] (mapv :id @seen))) + (t/is (some #(and (vector? %) (= :goto (first %))) @calls)) + (t/is (not-any? #(and (vector? %) (= :post (first %))) @calls))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-routes-wasm-to-the-worker + (try + (let [calls (atom []) + job-id (uuid/next) + seen (atom [])] + (await ((renderer/render-fn (test-cfg calls)) + {:exports [(test-params :png {:is-wasm true :job-id job-id})] + :on-object (fn [object] (swap! seen conj object) nil) + :check-cancelled (never-cancelled calls)})) + (t/is (= [{:id "w1" :path "/tmp/w1.png"}] @seen)) + (t/is (some #{[:post "render"]} @calls)) + (t/is (some #{:check-cancelled} @calls)) + (t/is (not-any? #(and (vector? %) (= :goto (first %))) @calls))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-posts-the-render-env-to-the-worker + ;; Everything the worker thread needs beyond the export itself + ;; travels in the posted params: the worker reads no config and no + ;; globals, so the lease message carries the render env. + (try + (let [calls (atom []) + posted (atom nil) + worker (let [^js emitter (events/EventEmitter.)] + (unchecked-set emitter "postMessage" + (fn [msg] + (reset! posted msg) + (js/setTimeout + (fn [] (.emit emitter "message" #js {:type "done"})) + 0))) + (unchecked-set emitter "terminate" + (fn [] (js/Promise.resolve nil))) + emitter) + cfg (assoc (test-cfg calls) + :exporter.wasm/pool {:pool (stub-wasm-pool calls worker) + :timeout-ms 5000})] + (await ((renderer/render-fn cfg) + {:exports [(test-params :png {:is-wasm true})] + :on-object (fn [_] nil) + :check-cancelled (never-cancelled calls)})) + (let [params (transit/decode-str (unchecked-get @posted "params")) + env (:exporter.wasm.render/env params)] + (t/is (= (cf/get-internal-uri) (:internal-uri env))) + (t/is (= (cf/get :public-uri) (:public-uri env))) + (t/is (= (os/tmpdir) (:tmpdir env))) + (t/is (= 100 (:image-cache-size env))) + (t/testing "and no management key: these routes never read one" + (t/is (not (contains? env :management-key)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-rejects-invalid-params + (try + (let [calls (atom [])] + (try + (await ((renderer/render-fn (test-cfg calls)) + {:exports [(dissoc (test-params :png nil) :objects)] + :on-object (fn [_] nil) + :check-cancelled (never-cancelled calls)})) + (t/is false "render should have rejected params without objects") + (catch :default cause + (t/is (= :data-validation (-> cause ex-data :code)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-rejects-a-non-fn-callback + (try + (let [calls (atom [])] + (try + (await ((renderer/render-fn (test-cfg calls)) + {:exports [(test-params :png nil)] + :on-object :not-a-fn + :check-cancelled (never-cancelled calls)})) + (t/is false "render should have rejected a non-fn on-object") + (catch :default cause + (t/is (= :data-validation (-> cause ex-data :code)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-shares-one-worker-across-wasm-renders + (try + (let [calls (atom []) + job-id (uuid/next) + cfg (test-cfg calls) + seen (atom [])] + (t/is (nil? (await ((renderer/render-fn cfg) + {:exports [(test-params :png {:is-wasm true :job-id job-id}) + (test-params :png {:is-wasm true :job-id job-id}) + (test-params :png {:job-id job-id})] + :on-object (fn [object] (swap! seen conj object) nil) + :check-cancelled (never-cancelled calls)})))) + (t/is (= 1 (count (filter #{:acquire} @calls)))) + (t/is (= 1 (count (filter #{:release} @calls)))) + (t/is (= 3 (count @seen)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-without-wasm-renders-directly + (try + (let [calls (atom []) + cfg (test-cfg calls) + seen (atom [])] + (await ((renderer/render-fn cfg) + {:exports [(test-params :png nil)] + :on-object (fn [object] (swap! seen conj object) nil) + :check-cancelled (never-cancelled calls)})) + (t/is (= [(uuid/uuid "1ac0612f-1ee8-8023-8008-d07a2aa96040")] (mapv :id @seen))) + (t/is (not-any? #{:acquire} @calls))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-validates-before-leasing + (try + (let [calls (atom []) + job-id (uuid/next) + cfg (test-cfg calls)] + (try + (await ((renderer/render-fn cfg) + {:exports [(test-params :png {:is-wasm true :job-id job-id}) + (dissoc (test-params :png nil) :objects)] + :on-object (fn [_] nil) + :check-cancelled (never-cancelled calls)})) + (t/is false "render should have rejected the bad batch") + (catch :default cause + (t/is (= :data-validation (-> cause ex-data :code))) + (t/testing "no worker was leased for a batch that never renders" + (t/is (not-any? #{:acquire} @calls)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-rejects-unknown-task-keys + (try + (let [calls (atom [])] + (try + (await ((renderer/render-fn (test-cfg calls)) + {:exports [(test-params :png nil)] + :on-objec (fn [_] nil) + :check-cancelled (never-cancelled calls)})) + (t/is false "render should have rejected the misspelled key") + (catch :default cause + (t/is (= :data-validation (-> cause ex-data :code))) + (t/is (not-any? #(and (vector? %) (= :goto (first %))) @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-rejects-unknown-export-keys-before-any-pool + (try + (let [calls (atom [])] + (try + (await ((renderer/render-fn (test-cfg calls)) + {:exports [(assoc (test-params :png nil) :bogus-key 1)] + :on-object (fn [_] nil) + :check-cancelled (never-cancelled calls)})) + (t/is false "render should have rejected the unknown export key") + (catch :default cause + (t/is (= :data-validation (-> cause ex-data :code))) + (t/testing "no pool was touched for a batch that never renders" + (t/is (not-any? #(and (vector? %) (= :goto (first %))) @calls)) + (t/is (not-any? #{:acquire} @calls)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/runner.cljs b/exporter/test/exporter_tests/runner.cljs index 82c3e4c46f..03c893648e 100644 --- a/exporter/test/exporter_tests/runner.cljs +++ b/exporter/test/exporter_tests/runner.cljs @@ -10,23 +10,64 @@ [cljs.test :as t] [clojure.string :as str] [clojure.tools.cli :refer [parse-opts]] - [exporter-tests.export-shapes-test] - [exporter-tests.jobs-test] - [exporter-tests.renderer-svg-test] - [exporter-tests.scheduler-test] - [exporter-tests.shell-test] - [exporter-tests.wasm-pool-test] + [exporter-tests.browser-scope-test] + [exporter-tests.browser-test] + [exporter-tests.consumer-api-test] + [exporter-tests.consumer-test] + [exporter-tests.consumer-worker-test] + [exporter-tests.exporter-config-test] + [exporter-tests.exporter-consumer-config-test] + [exporter-tests.exporter-consumer-plan-test] + [exporter-tests.exporter-jobs-test] + [exporter-tests.exporter-resources-test] + [exporter-tests.exporter-shell-test] + [exporter-tests.exporter-tmpdir-test] + [exporter-tests.exporter-util-mime-test] + [exporter-tests.exporter-util-object-test] + [exporter-tests.exporter-wasm-render-test] + [exporter-tests.exporter-wasm-svg-test] + [exporter-tests.exporter-wasm-worker-test] + [exporter-tests.main-test] + [exporter-tests.renderer-browser-test] + [exporter-tests.renderer-test] + [exporter-tests.system-test] + [exporter-tests.wasm-pool-service-test] + [exporter-tests.wasm-scope-test] [goog.object :as gobj])) (enable-console-print!) +;; The tests that ride the mock http must never leave a rejected promise +;; dangling: a dangling one kills the whole run with ERR_UNHANDLED_REJECTION +;; and without any way to point at the chain that broke. Log and keep going. +(js/process.on "unhandledRejection" + (fn [reason _promise] + (println (str "UNHANDLED-REJECTION: " reason)))) + (def test-namespaces - ['exporter-tests.export-shapes-test - 'exporter-tests.jobs-test - 'exporter-tests.renderer-svg-test - 'exporter-tests.scheduler-test - 'exporter-tests.shell-test - 'exporter-tests.wasm-pool-test]) + ['exporter-tests.browser-scope-test + 'exporter-tests.browser-test + 'exporter-tests.consumer-api-test + 'exporter-tests.consumer-test + 'exporter-tests.consumer-worker-test + 'exporter-tests.exporter-config-test + 'exporter-tests.exporter-consumer-config-test + 'exporter-tests.exporter-consumer-plan-test + 'exporter-tests.exporter-jobs-test + 'exporter-tests.exporter-resources-test + 'exporter-tests.exporter-shell-test + 'exporter-tests.exporter-tmpdir-test + 'exporter-tests.exporter-util-mime-test + 'exporter-tests.exporter-util-object-test + 'exporter-tests.exporter-wasm-render-test + 'exporter-tests.exporter-wasm-svg-test + 'exporter-tests.exporter-wasm-worker-test + 'exporter-tests.main-test + 'exporter-tests.renderer-browser-test + 'exporter-tests.renderer-test + 'exporter-tests.system-test + 'exporter-tests.wasm-pool-service-test + 'exporter-tests.wasm-scope-test]) (assert (every? find-ns-obj test-namespaces) "test-namespaces contains a namespace that isn't required in runner.cljs") @@ -72,7 +113,7 @@ "Log level example:\n" " node target/tests/test.js --focus exporter-tests.renderer-svg-test --log-level warn")) -(defn- fail! +(defn- fail [message] (js/console.error message) (.exit js/process 1)) @@ -82,7 +123,7 @@ (let [[ns-name test-name & extra] (str/split focus #"/")] (cond (or (str/blank? ns-name) (seq extra)) - (fail! (str "Invalid --focus value: " focus)) + (fail (str "Invalid --focus value: " focus)) (some? test-name) {:ns (symbol ns-name) :test test-name} @@ -118,17 +159,17 @@ (defn- selected-tests [{:keys [ns test]}] (when-not (some #{ns} test-namespaces) - (fail! (str "Unknown test namespace: " ns))) + (fail (str "Unknown test namespace: " ns))) (let [vars (vec (ns-test-vars ns))] (when (empty? vars) - (fail! (str "No tests found in namespace: " ns))) + (fail (str "No tests found in namespace: " ns))) (if test (let [test-sym (symbol test) test-var (some #(when (= test-sym (:name (meta %))) %) vars)] (if test-var {:vars [test-var] :fixtures (ns-fixtures ns [test-var])} - (fail! (str "Unknown test var: " ns "/" test)))) + (fail (str "Unknown test var: " ns "/" test)))) {:vars vars :fixtures (ns-fixtures ns vars)}))) @@ -137,7 +178,7 @@ {:once (apply merge (keep :once fixtures)) :each (apply merge (keep :each fixtures))}) -(defn- run-test-vars! +(defn- run-test-vars [tests] (let [vars (vec (mapcat :vars tests)) fixtures (merge-fixtures (map :fixtures tests)) @@ -156,16 +197,16 @@ (t/report @summary) (t/report (assoc @summary :type :end-run-tests)))])))) -(defn- run-focused-test! +(defn- run-focused-test [focus] - (run-test-vars! [(selected-tests (parse-focus focus))])) + (run-test-vars [(selected-tests (parse-focus focus))])) (defn -main [] (let [{:keys [options errors summary]} (parse-opts (argv) cli-options)] (cond (seq errors) - (fail! (str/join "\n" errors)) + (fail (str/join "\n" errors)) (:help options) (do @@ -176,5 +217,5 @@ (do (l/setup! {:app (or (:log-level options) :warn)}) (if (:focus options) - (run-focused-test! (:focus options)) - (run-test-vars! (map #(selected-tests {:ns %}) test-namespaces))))))) + (run-focused-test (:focus options)) + (run-test-vars (map #(selected-tests {:ns %}) test-namespaces))))))) diff --git a/exporter/test/exporter_tests/scheduler_test.cljs b/exporter/test/exporter_tests/scheduler_test.cljs deleted file mode 100644 index 04349480bf..0000000000 --- a/exporter/test/exporter_tests/scheduler_test.cljs +++ /dev/null @@ -1,81 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns exporter-tests.scheduler-test - "Admission control. A headless job leases one render worker for its whole run, - so no more of them may start than there are workers." - (:require - [app.common.uuid :as uuid] - [app.jobs :as jobs] - [app.jobs.scheduler :as scheduler] - [app.wasm.pool :as pool] - [cljs.test :as t :include-macros true] - [promesa.core :as p])) - -(defn- pin-capacity! - "Fixes the worker count for the test. Returns the thunk that puts it back; - `with-redefs` cannot be used here, since the scheduler keeps admitting jobs - after the body of the test has returned." - [n] - (let [original pool/capacity] - (set! pool/capacity (constantly n)) - (fn [] (set! pool/capacity original)))) - -(defn- create! - [backend run-fn] - (jobs/create! {:profile-id (uuid/next) - :cmd :export-shapes - :backend backend - :total 1 - :name "test" - :resource-id (uuid/next)} - run-fn)) - -(defn- gate - "A promise and the fn that settles it, standing in for a render in flight." - [] - (let [resolve* (volatile! nil) - pending (p/create (fn [resolve _] (vreset! resolve* resolve)))] - [pending (fn [] (@resolve* nil))])) - -(t/deftest headless-jobs-wait-for-a-render-worker - (t/testing "with one worker, the second headless job stays queued until the first ends" - (t/async done - (let [restore! (pin-capacity! 1) - [render open] (gate) - started (atom [])] - (p/let [job1 (create! "wasm" (fn [_] (swap! started conj :one) render)) - job2 (create! "wasm" (fn [_] (swap! started conj :two) (p/resolved nil)))] - (let [p1 (scheduler/submit! job1) - p2 (scheduler/submit! job2)] - (p/do - (p/delay 10) - (t/is (= [:one] @started)) - (t/is (= "running" (:state (jobs/lookup (:id job1))))) - (t/is (= "queued" (:state (jobs/lookup (:id job2))))) - (open) - (p/all [p1 p2]) - (t/is (= [:one :two] @started)) - (restore!) - (done)))))))) - -(t/deftest a-browser-job-is-not-held-back-by-the-worker-pool - (t/testing "the headless cap applies to headless jobs only" - (t/async done - (let [restore! (pin-capacity! 1) - [render open] (gate) - started (atom [])] - (p/let [job1 (create! "wasm" (fn [_] (swap! started conj :wasm) render)) - job2 (create! "browser" (fn [_] (swap! started conj :browser) (p/resolved nil)))] - (let [p1 (scheduler/submit! job1) - p2 (scheduler/submit! job2)] - (p/do - (p/delay 10) - (t/is (= [:wasm :browser] @started)) - (open) - (p/all [p1 p2]) - (restore!) - (done)))))))) diff --git a/exporter/test/exporter_tests/shell_test.cljs b/exporter/test/exporter_tests/shell_test.cljs deleted file mode 100644 index 5858328eb3..0000000000 --- a/exporter/test/exporter_tests/shell_test.cljs +++ /dev/null @@ -1,70 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns exporter-tests.shell-test - "Tests to verify GHSA-4f36-m4hj-cv86 is fixed: OS Command Injection in SVG exporter. - These tests prove that: - 1. execFile does NOT interpret shell metacharacters (safe execution) - 2. Malicious colors fail validation regex - 3. The injection does NOT execute commands (no RCE)" - (:require - ["node:child_process" :as proc] - ["node:fs" :as fs] - [cljs.test :as t :include-macros true])) - -(def ^:private hex-color-rx - #"^#(?:[0-9a-fA-F]{3}){1,2}$") - -(defn- valid-hex-color? - [color] - (and (string? color) - (some? (re-matches hex-color-rx color)))) - -(t/deftest execfile-does-not-interpret-shell-metacharacters - (t/testing "Proves execFile passes arguments literally (no shell interpretation)" - (t/async done - (let [cmd "echo" - args #js ["$(echo PWNED)"]] - (proc/execFile cmd args #js {:encoding "buffer"} - (fn [error stdout _stderr] - (if error - (do - (t/is false (str "unexpected error: " (.-message error))) - (done)) - (let [output (.toString stdout "utf8")] - (t/is (= "$(echo PWNED)\n" output) - "execFile passes $(...) literally, no shell interpretation") - (done))))))))) - -(t/deftest malicious-color-fails-validation - (t/testing "Proves malicious colors are rejected by validation" - (let [malicious "#000000$(echo PWNED)" - valid-color "#000000" - short-valid "#abc"] - (t/is (not (valid-hex-color? malicious)) - "malicious color with $(...) fails validation") - (t/is (valid-hex-color? valid-color) - "valid 6-digit hex color passes validation") - (t/is (valid-hex-color? short-valid) - "valid 3-digit hex color passes validation")))) - -(t/deftest execfile-does-not-execute-injected-commands - (t/testing "Proves execFile does NOT execute injected commands (no RCE)" - (t/async done - (let [marker "/tmp/penpot-exporter-rce-test" - malicious (str "#000000$(touch " marker ")") - cmd "echo" - args #js [malicious]] - (when (fs/existsSync marker) - (fs/unlinkSync marker)) - (proc/execFile cmd args #js {:encoding "buffer"} - (fn [_error _stdout _stderr] - ;; Command completes (or fails), but no injection occurs - (t/is (not (fs/existsSync marker)) - "no RCE: marker file was NOT created") - (when (fs/existsSync marker) - (fs/unlinkSync marker)) - (done))))))) diff --git a/exporter/test/exporter_tests/system_test.cljs b/exporter/test/exporter_tests/system_test.cljs new file mode 100644 index 0000000000..03cee34937 --- /dev/null +++ b/exporter/test/exporter_tests/system_test.cljs @@ -0,0 +1,215 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.system-test + "The async system lifecycle: dependency-ordered init and halt, with + refs resolved to the running instances." + (:require + [cljs.test :as t :include-macros true] + [exporter.utils.system :as system])) + +;; Each init method records its key on the tracker carried by its own +;; config, and returns a running instance that carries the tracker back, +;; so the matching halt method can record on the same atom. +(defn- track-init + [tracker k] + (swap! tracker conj k) + {:tracker tracker :component k}) + +(defn- track-halt + [tracker k] + (swap! tracker conj [:halted k]) + nil) + +(defmethod system/init-key :order-test/a [_ {:keys [tracker]}] (track-init tracker :order-test/a)) +(defmethod system/init-key :order-test/b [_ {:keys [tracker]}] (track-init tracker :order-test/b)) +(defmethod system/init-key :order-test/c [_ {:keys [tracker]}] (track-init tracker :order-test/c)) +(defmethod system/halt-key :order-test/a [_ {:keys [tracker]}] (track-halt tracker :order-test/a)) +(defmethod system/halt-key :order-test/b [_ {:keys [tracker]}] (track-halt tracker :order-test/b)) +(defmethod system/halt-key :order-test/c [_ {:keys [tracker]}] (track-halt tracker :order-test/c)) + +(defn- order-config + [tracker] + {:order-test/a {:tracker tracker} + :order-test/b {:tracker tracker :dep (system/ref :order-test/a)} + :order-test/c {:tracker tracker :dep (system/ref :order-test/b)}}) + +(defmethod system/init-key :refs-test/db [_ {:keys [tracker]}] + (swap! tracker conj :refs-test/db) + {:tracker tracker :component :refs-test/db :conn :fake-conn}) + +(defmethod system/init-key :refs-test/api [_ {:keys [tracker dep]}] + (swap! tracker conj [:refs-test/api-sees dep]) + {:tracker tracker :component :refs-test/api}) + +(defmethod system/init-key :async-test/db [_ _] + (js/Promise. (fn [resolve _] (js/setTimeout (fn [] (resolve :async-conn)) 5)))) + +(defmethod system/init-key :async-test/api [_ {:keys [dep]}] + {:component :async-test/api :dep dep}) + +(defmethod system/init-key :fail-test/a [_ {:keys [tracker]}] (track-init tracker :fail-test/a)) +(defmethod system/init-key :fail-test/b [_ {:keys [tracker fail?]}] + (swap! tracker conj :fail-test/b) + (if fail? + (js/Promise.reject (ex-info "cannot connect" {})) + {:tracker tracker :component :fail-test/b})) +(defmethod system/halt-key :fail-test/a [_ {:keys [tracker]}] (track-halt tracker :fail-test/a)) + +(defmethod system/init-key :haltfail-test/a [_ {:keys [tracker]}] (track-init tracker :haltfail-test/a)) +(defmethod system/init-key :haltfail-test/b [_ {:keys [tracker boom?]}] + (assoc (track-init tracker :haltfail-test/b) :boom? boom?)) +(defmethod system/halt-key :haltfail-test/a [_ {:keys [tracker]}] (track-halt tracker :haltfail-test/a)) +(defmethod system/halt-key :haltfail-test/b [_ {:keys [tracker boom?]}] + (swap! tracker conj [:haltfail-test/b-ran boom?]) + (when boom? + (throw (ex-info "cannot stop" {}))) + nil) + +(derive :derive-test/child :derive-test/parent) +(derive :derive-test/other :derive-test/parent) + +(defmethod system/init-key :derive-test/child [_ _] :child-running) +(defmethod system/init-key :derive-test/other [_ _] :other-running) +(defmethod system/init-key :derive-test/consumer [_ {:keys [dep]}] dep) +(defmethod system/init-key :derive-test/consumer2 [_ {:keys [dep]}] dep) + +(defmethod system/init-key :circ-test/a [_ _] :circ-a-running) +(defmethod system/init-key :circ-test/b [_ _] :circ-b-running) + +(defmethod system/init-key :nohalt-test/a [_ _] :nohalt-a-running) + +(t/deftest ^:async inits-keys-in-dependency-order + (try + (let [tracker (atom []) + sys (await (system/init (order-config tracker)))] + (t/is (= [:order-test/a :order-test/b :order-test/c] @tracker)) + (t/is (= #{:order-test/a :order-test/b :order-test/c} (set (keys sys))))) + (catch :default cause + (t/is false (str "unexpected init failure: " (ex-message cause)))))) + +(t/deftest ^:async resolves-refs-to-running-instances + (try + (let [tracker (atom []) + sys (await (system/init {:refs-test/db {:tracker tracker} + :refs-test/api {:tracker tracker + :dep (system/ref :refs-test/db)}}))] + (t/is (= [:refs-test/db + [:refs-test/api-sees {:tracker tracker + :component :refs-test/db + :conn :fake-conn}]] + @tracker)) + (t/is (= :refs-test/api (:component (:refs-test/api sys))))) + (catch :default cause + (t/is false (str "unexpected init failure: " (ex-message cause)))))) + +(t/deftest ^:async awaits-promised-instances + (try + (let [sys (await (system/init {:async-test/db {} + :async-test/api {:dep (system/ref :async-test/db)}}))] + (t/is (= :async-conn (:async-test/db sys))) + (t/is (= :async-conn (:dep (:async-test/api sys))))) + (catch :default cause + (t/is false (str "unexpected init failure: " (ex-message cause)))))) + +(t/deftest ^:async missing-ref-fails-init + (try + (await (system/init {:missing-test/a {:dep (system/ref :missing-test/ghost)}})) + (t/is false "init should have rejected on a missing ref") + (catch :default cause + (t/is (= ::system/missing-refs (-> cause ex-data :reason))) + (t/is (= [:missing-test/ghost] (-> cause ex-data :missing-refs)))))) + +(t/deftest ^:async failed-init-halts-what-already-started + (let [tracker (atom [])] + (try + (await (system/init {:fail-test/a {:tracker tracker} + :fail-test/b {:tracker tracker + :fail? true + :dep (system/ref :fail-test/a)}})) + (t/is false "init should have rejected when a component fails") + (catch :default cause + (t/is (= ::system/init-failed (-> cause ex-data :reason))) + (t/is (= :fail-test/b (-> cause ex-data :key))) + (t/is (= [:fail-test/a :fail-test/b [:halted :fail-test/a]] @tracker)))))) + +(t/deftest ^:async halts-in-reverse-dependency-order + (try + (let [tracker (atom []) + sys (await (system/init (order-config tracker))) + result (await (system/halt sys))] + (t/is (nil? result)) + (t/is (= [[:halted :order-test/c] + [:halted :order-test/b] + [:halted :order-test/a]] + (drop 3 @tracker)))) + (catch :default cause + (t/is false (str "unexpected halt failure: " (ex-message cause)))))) + +(t/deftest ^:async halt-continues-past-a-failing-component + (try + (let [tracker (atom []) + sys (await (system/init {:haltfail-test/a {:tracker tracker} + :haltfail-test/b {:tracker tracker + :boom? true + :dep (system/ref :haltfail-test/a)}}))] + (try + (await (system/halt sys)) + (t/is false "halt should have rejected when a component fails to stop") + (catch :default cause + (t/is (= ::system/halt-failed (-> cause ex-data :reason))) + (t/is (= [:haltfail-test/b] (mapv :key (-> cause ex-data :failures)))))) + (t/is (= [:haltfail-test/a + :haltfail-test/b + [:haltfail-test/b-ran true] + [:halted :haltfail-test/a]] + @tracker))) + (catch :default cause + (t/is false (str "unexpected init failure: " (ex-message cause)))))) + +(t/deftest ^:async init-subset-pulls-its-dependencies + (try + (let [tracker (atom []) + sys (await (system/init (order-config tracker) [:order-test/c]))] + (t/is (= [:order-test/a :order-test/b :order-test/c] @tracker)) + (t/is (= #{:order-test/a :order-test/b :order-test/c} (set (keys sys))))) + (catch :default cause + (t/is false (str "unexpected init failure: " (ex-message cause)))))) + +(t/deftest ^:async ref-to-a-parent-resolves-the-derived-child + (try + (let [sys (await (system/init {:derive-test/child {} + :derive-test/consumer {:dep (system/ref :derive-test/parent)}}))] + (t/is (= :child-running (:derive-test/consumer sys)))) + (catch :default cause + (t/is false (str "unexpected init failure: " (ex-message cause)))))) + +(t/deftest ^:async ref-to-a-parent-with-two-children-is-ambiguous + (try + (await (system/init {:derive-test/child {} + :derive-test/other {} + :derive-test/consumer2 {:dep (system/ref :derive-test/parent)}})) + (t/is false "init should have rejected on an ambiguous ref") + (catch :default cause + (t/is (= ::system/ambiguous-key (-> cause ex-data :reason)))))) + +(t/deftest ^:async circular-dependencies-fail-init + (try + (await (system/init {:circ-test/a {:dep (system/ref :circ-test/b)} + :circ-test/b {:dep (system/ref :circ-test/a)}})) + (t/is false "init should have rejected on a circular dependency") + (catch :default cause + (t/is (= :weavejester.dependency/circular-dependency + (-> cause ex-data :reason)))))) + +(t/deftest ^:async key-without-halt-method-halts-cleanly + (try + (let [sys (await (system/init {:nohalt-test/a {}})) + result (await (system/halt sys))] + (t/is (= :nohalt-a-running (:nohalt-test/a sys))) + (t/is (nil? result))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/wasm_pool_service_test.cljs b/exporter/test/exporter_tests/wasm_pool_service_test.cljs new file mode 100644 index 0000000000..34913f8f16 --- /dev/null +++ b/exporter/test/exporter_tests/wasm_pool_service_test.cljs @@ -0,0 +1,170 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.wasm-pool-service-test + "The wasm worker pool service: lifecycle through the system, the + lend protocol analogy to a transaction (return on success, destroy + on failure), and the creation handshake, against fake workers." + (:require + ["node:events" :as events] + [cljs.test :as t :include-macros true] + [exporter.utils.system :as system] + [exporter.wasm.pool :as pool])) + +(defn- fake-worker + [calls behavior] + (let [emitter (events/EventEmitter.)] + (unchecked-set emitter "postMessage" + (fn [msg] + (swap! calls conj [:post (unchecked-get msg "type")]) + (behavior emitter msg))) + (unchecked-set emitter "terminate" + (fn [] + (swap! calls conj :terminate) + (js/Promise.resolve nil))) + emitter)) + +(defn- fake-create-worker + [calls behavior] + (fn [] + (swap! calls conj :create) + (js/Promise.resolve (fake-worker calls behavior)))) + +(defn- silent-behavior + [_emitter _msg]) + +(defn- test-config + [calls behavior timeout-ms] + {:exporter.wasm/pool {:max 1 + :min 0 + :timeout-ms timeout-ms + :create-worker (fake-create-worker calls behavior)}}) + +(defn- pool-of + [sys] + (:pool (:exporter.wasm/pool sys))) + +(t/deftest ^:async pool-starts-and-stops-through-the-system + (try + (let [calls (atom []) + sys (await (system/init (test-config calls silent-behavior 5000)))] + (t/is (some? (:exporter.wasm/pool sys))) + (t/is (= 5000 (:timeout-ms (:exporter.wasm/pool sys)))) + (t/is (nil? (await (system/halt sys))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-lends-and-reclaims + (try + (let [calls (atom []) + sys (await (system/init (test-config calls silent-behavior 5000))) + pool (pool-of sys) + seen (atom nil) + result (await (pool/run pool (fn [cfg] (reset! seen cfg) :used)))] + (t/is (= :used result)) + (t/is (some? (unchecked-get (:exporter.wasm.pool/worker @seen) "__id"))) + (t/testing "a second use reuses the returned worker: nothing is created again" + (await (pool/run pool (fn [_] :again))) + (t/is (= 1 (count (filter #{:create} @calls))))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-reads-the-pool-from-cfg + (try + (let [calls (atom []) + sys (await (system/init (test-config calls silent-behavior 5000))) + seen (atom nil) + result (await (pool/run sys + (fn [cfg] (reset! seen cfg) :used)))] + (t/is (= :used result)) + (t/is (some? (unchecked-get (:exporter.wasm.pool/worker @seen) "__id"))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-threads-extra-args-to-the-body + (try + (let [calls (atom []) + sys (await (system/init (test-config calls silent-behavior 5000)))] + (t/is (= [:a :b] (await (pool/run (pool-of sys) + (fn [_ x y] [x y]) + :a :b)))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-rejects-cfg-without-pool + (try + (try + (await (pool/run {:timeout-ms 5000} (fn [_] :used))) + (t/is false "run should have rejected a cfg without pool") + (catch :default cause + (t/is (= :invalid-pool-cfg (-> cause ex-data :code))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async run-destroys-the-worker-on-body-failure + (try + (let [calls (atom []) + sys (await (system/init (test-config calls silent-behavior 5000)))] + (try + (await (pool/run (pool-of sys) + (fn [_] (throw (ex-info "body broke" {}))))) + (t/is false "run should have rejected a failed body") + (catch :default cause + (t/is (= "body broke" (ex-message cause))) + (t/is (some #{:terminate} @calls)) + (t/is (not-any? #{:release} @calls)))) + (await (system/halt sys))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async creation-handshake-resolves-on-ready + (try + (let [worker ^js (events/EventEmitter.) + ready (pool/await-ready worker 500)] + (js/setTimeout #(.emit worker "message" #js {:type "ready"}) 5) + (t/is (identical? worker (await ready)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async silent-handshake-rejects-and-terminates + (try + (let [calls (atom []) + worker (fake-worker calls silent-behavior)] + (try + (await (pool/await-ready worker 20)) + (t/is false "await-ready should have rejected a silent worker") + (catch :default cause + (t/is (= :worker-not-ready (-> cause ex-data :code))) + (t/is (some #{:terminate} @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async errored-handshake-rejects-with-the-cause + (try + (let [worker ^js (events/EventEmitter.) + ready (pool/await-ready worker 500)] + (js/setTimeout #(.emit worker "error" (ex-info "boot broke" {})) 5) + (try + (await ready) + (t/is false "await-ready should have rejected a broken worker") + (catch :default cause + (t/is (= "boot broke" (ex-message cause)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async terminate-marks-and-kills + (try + (let [calls (atom []) + worker (fake-worker calls silent-behavior)] + (t/is (nil? (pool/terminate nil))) + (pool/terminate worker) + (t/is (false? (unchecked-get worker "__alive"))) + (t/is (some #{:terminate} @calls))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/exporter/test/exporter_tests/wasm_pool_test.cljs b/exporter/test/exporter_tests/wasm_pool_test.cljs deleted file mode 100644 index 80837626e2..0000000000 --- a/exporter/test/exporter_tests/wasm_pool_test.cljs +++ /dev/null @@ -1,46 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns exporter-tests.wasm-pool-test - "Worker leasing, against a stub pool: `with-worker` must give the worker back - however its body ends." - (:require - [app.wasm.pool :as pool] - [cljs.test :as t :include-macros true] - [promesa.core :as p])) - -(defn- stub-pool! - "Installs a pool whose acquire/release/destroy only count calls." - [] - (let [calls (atom {:acquired 0 :released 0 :destroyed 0})] - (reset! pool/pool - #js {:acquire (fn [] (swap! calls update :acquired inc) (p/resolved ::worker)) - :release (fn [_] (swap! calls update :released inc) (p/resolved nil)) - :destroy (fn [_] (swap! calls update :destroyed inc) (p/resolved nil))}) - calls)) - -(t/deftest releases-the-worker-when-the-body-succeeds - (t/async done - (let [calls (stub-pool!)] - (p/let [result (pool/with-worker (fn [_] (p/resolved :ok)))] - (t/is (= :ok result)) - (t/is (= 1 (:acquired @calls))) - (t/is (= 1 (:released @calls))) - (t/is (= 0 (:destroyed @calls))) - (reset! pool/pool nil) - (done))))) - -(t/deftest gives-the-worker-back-when-the-body-throws-synchronously - (t/testing "a raise out of the scope body must not leave the worker borrowed" - (t/async done - (let [calls (stub-pool!)] - (->> (pool/with-worker (fn [_] (throw (ex-info "cancelled" {})))) - (p/hmap (fn [_ cause] - (t/is (some? cause)) - (t/is (= 1 (:acquired @calls))) - (t/is (= 1 (+ (:released @calls) (:destroyed @calls)))) - (reset! pool/pool nil) - (done)))))))) diff --git a/exporter/test/exporter_tests/wasm_scope_test.cljs b/exporter/test/exporter_tests/wasm_scope_test.cljs new file mode 100644 index 0000000000..b8e56ffc23 --- /dev/null +++ b/exporter/test/exporter_tests/wasm_scope_test.cljs @@ -0,0 +1,199 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns exporter-tests.wasm-scope-test + "The headless lease: one shared worker, sequential renders, failure + isolation and driver-owned cancel, against a stub pool and stub + workers. Task keys ride in cfg under `:exporter.renderer/*`." + (:require + ["node:events" :as events] + [app.common.transit :as transit] + [cljs.test :as t :include-macros true] + [exporter.wasm.scope :as scope])) + +(defn- stub-worker + [calls behavior] + (let [^js emitter (events/EventEmitter.)] + (unchecked-set emitter "postMessage" + (fn [msg] + (swap! calls conj [:post (unchecked-get msg "type")]) + (behavior emitter msg))) + (unchecked-set emitter "terminate" + (fn [] + (swap! calls conj :terminate) + (.emit emitter "exit" 1) + (js/Promise.resolve nil))) + emitter)) + +(defn- render-behavior + [payload] + (fn [^js emitter _msg] + (js/setTimeout + (fn [] + (.emit emitter "message" #js {:type "object" :payload (transit/encode-str payload)}) + (.emit emitter "message" #js {:type "done"})) + 0))) + +(defn- silent-behavior + [_emitter _msg]) + +(defn- stub-pool + [calls worker] + #js {:acquire (fn [] (js/Promise.resolve worker)) + :release (fn [_] (swap! calls conj :release) (js/Promise.resolve nil)) + :destroy (fn [_] (swap! calls conj :destroy) (js/Promise.resolve nil))}) + +(defn- stub-check + [calls cancelled?] + (fn [] + (swap! calls conj :check-cancelled) + (when @cancelled? + (throw (ex-info "export job was cancelled" + {:type :internal + :code :job-cancelled + :hint "export job was cancelled"}))))) + +(defn- test-cfg + ([calls worker on-object cancelled?] + (test-cfg calls worker on-object cancelled? nil)) + ([calls worker on-object cancelled? watch-interval] + (cond-> {:exporter.wasm/pool {:pool (stub-pool calls worker) :timeout-ms 5000} + :exporter.renderer/on-object on-object + :exporter.renderer/check-cancelled (stub-check calls cancelled?)} + watch-interval (assoc :exporter.wasm.scope/watch-interval watch-interval)))) + +(defn- test-params + [] + {:type :png + :objects [{:id "o1"}]}) + +(defn- posted? + [calls] + (some #(and (vector? %) (= :post (first %))) calls)) + +(t/deftest ^:async run-shares-one-worker-across-renders + (try + (let [calls (atom []) + worker (stub-worker calls (render-behavior {:id "o1" :path "/tmp/o1.png"})) + seen (atom []) + cfg (test-cfg calls worker + (fn [object] (swap! seen conj object) nil) + (atom false))] + (await (scope/run cfg + (^:async fn [render-fn] + (await (render-fn (test-params))) + (await (render-fn (test-params)))))) + (t/is (= [{:id "o1" :path "/tmp/o1.png"} + {:id "o1" :path "/tmp/o1.png"}] + @seen)) + (t/is (= 1 (count (filter #{:release} @calls)))) + (t/testing "the check runs as each render's turn comes up" + (t/is (= 2 (count (filter #{:check-cancelled} @calls))))) + (t/testing "a quiet lease leaves its worker alone" + (t/is (not-any? #{:terminate} @calls)))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async cancelled-render-rejects-before-posting + (try + (let [calls (atom []) + worker (stub-worker calls (render-behavior {:id "o1"})) + cfg (test-cfg calls worker (fn [_] nil) (atom true))] + (try + (await (scope/run cfg (fn [render-fn] (render-fn (test-params))))) + (t/is false "run should have rejected a cancelled render") + (catch :default cause + (t/is (= :job-cancelled (-> cause ex-data :code))) + (t/is (not (posted? @calls)))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async cancel-aborts-the-leased-worker-mid-render + (try + (let [calls (atom []) + worker (stub-worker calls silent-behavior) + cancelled (atom false) + cfg (test-cfg calls worker (fn [_] nil) cancelled 20)] + (try + (await (scope/run cfg + (fn [render-fn] + ;; the cancel lands mid-render, once + ;; the worker is listening + (js/setTimeout #(reset! cancelled true) 10) + (render-fn (test-params))))) + (t/is false "run should have rejected an aborted render") + (catch :default cause + (t/is (= :worker-exited (-> cause ex-data :code))) + (t/is (some #{:terminate} @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async failed-render-does-not-break-the-chain + (try + (let [calls (atom []) + worker (stub-worker calls (render-behavior {:id "o1"})) + seen (atom []) + thrown (atom false) + cfg (test-cfg calls worker + (fn [object] + (if (compare-and-set! thrown false true) + (throw (ex-info "on-object broke" {})) + (do (swap! seen conj object) nil))) + (atom false))] + (await (scope/run cfg + (^:async fn [render-fn] + (try + (await (render-fn (test-params))) + (catch :default _cause + (swap! calls conj :first-failed))) + (await (render-fn (test-params)))))) + (t/is (some #{:first-failed} @calls)) + (t/is (= [{:id "o1"}] @seen))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-on-without-lease-raises + (try + (try + (await (scope/render-on {:exporter.renderer/on-object (fn [_] nil) + :exporter.renderer/check-cancelled (fn [] nil) + :exporter.wasm/pool {:pool nil :timeout-ms 50}} + (test-params))) + (t/is false "render-on should have rejected a missing lease") + (catch :default cause + (t/is (= :worker-not-leased (-> cause ex-data :code))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async render-on-without-timeout-raises + (try + (try + (await (scope/render-on {:exporter.wasm.pool/worker #js {} + :exporter.renderer/on-object (fn [_] nil) + :exporter.renderer/check-cancelled (fn [] nil) + :exporter.wasm/pool {:pool nil}} + (test-params))) + (t/is false "render-on should have rejected a missing timeout") + (catch :default cause + (t/is (= :assertion (-> cause ex-data :type))) + (t/is (= :timeout-ms-missing (-> cause ex-data :code))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) + +(t/deftest ^:async silent-worker-times-out-and-is-terminated + (try + (let [calls (atom []) + worker (stub-worker calls silent-behavior) + cfg (-> (test-cfg calls worker (fn [_] nil) (atom false)) + (assoc-in [:exporter.wasm/pool :timeout-ms] 50))] + (try + (await (scope/run cfg (fn [render-fn] (render-fn (test-params))))) + (t/is false "run should have rejected a silent worker") + (catch :default cause + (t/is (= :render-timeout (-> cause ex-data :code))) + (t/is (some #{:terminate} @calls))))) + (catch :default cause + (t/is false (str "unexpected failure: " (ex-message cause)))))) diff --git a/frontend/src/app/main/data/exports/assets.cljs b/frontend/src/app/main/data/exports/assets.cljs index e85c1c3cd3..b10fb215d6 100644 --- a/frontend/src/app/main/data/exports/assets.cljs +++ b/frontend/src/app/main/data/exports/assets.cljs @@ -5,12 +5,28 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.main.data.exports.assets + "The shapes and frames exportation. + + A server export is one `:export-assets` job: the backend creates it + (`create-export-assets-job`) and freezes the items, the external + exporter worker renders them, and the client follows its life on the + websocket with `dj/watch-job`, the same channel the file exports of + `data.exports.files` ride since they became jobs. The milestones of + the job (`:preparing`, `:rendering`, `:packaging`, with the `objects` + and `pages` counters) become the figures the progress widget shows; + the completed row carries the artifact under its `result`, for the + download; and the X of the widget cancels the job by its id. + + With render-wasm active the export never leaves the browser: the + shared worker WASM module renders it (one file, a ZIP of shapes or a + multi-page PDF) and the client downloads it locally, with no job." (:require [app.common.files.helpers :as cfh] [app.common.time :as ct] [app.common.uuid :as uuid] [app.main.data.event :as ev] [app.main.data.helpers :as dsh] + [app.main.data.jobs :as dj] [app.main.data.modal :as modal] [app.main.data.notifications :as ntf] [app.main.data.persistence :as dwp] @@ -23,7 +39,6 @@ [app.util.dom :as dom] [app.util.i18n :refer [tr]] [app.util.webapi :as wapi] - [app.util.websocket :as ws] [beicon.v2.core :as rx] [cuerdas.core :as str] [potok.v2.core :as ptk])) @@ -49,20 +64,6 @@ :level :warning :timeout 8000})))))) -(defn- export-failed - "Shows an export failure as a toast, keeping the workspace open: the - failure is never raised to the global error handler, which would - replace the workspace with an error page." - [cause] - (ptk/reify ::export-failed - ptk/EffectEvent - (effect [_ _ _] - (errors/flash :cause cause - :type (if (#{:server-error :internal} (:type (ex-data cause))) - :unhandled - :handled) - :hint (tr "errors.export-failed"))))) - (defn- export-widget-failed "Reports a failure that happened before the export ever started: the widget settles instead of waiting for progress that will never arrive, @@ -102,12 +103,6 @@ [exports] (mapv normalize-export exports)) -(defn- normalize-export-shapes-params - [{:keys [exports] :as params}] - (cond-> params - (seq exports) - (assoc :exports (normalize-exports exports)))) - (defn toggle-detail-visibililty [] (ptk/reify ::toggle-detail-visibililty @@ -123,18 +118,18 @@ (update-in state [:export :widget-visible] not)))) (defn clear-export-state - [id] + [job-id] (ptk/reify ::clear-export-state ptk/UpdateEvent (update [_ state] - ;; only clear if the existing export is the same - (let [existing-id (-> state :export :id)] + ;; only clear if the export in the slot is the one that asked: + ;; a delayed clear of a finished run must not wipe the next one. + (let [existing-id (-> state :export :job-id)] (if (and (some? existing-id) - (not= id existing-id)) + (not= job-id existing-id)) state (dissoc state :export)))))) - (defn show-workspace-export-dialog [{:keys [selected origin]}] (ptk/reify ::show-workspace-export-dialog @@ -183,7 +178,7 @@ (cond-> share-id (assoc :share-id share-id))))] (rx/of (modal/show :export-shapes {:exports (vec exports) :origin "viewer" - :name name})))))) #_TODO + :name name})))))) (defn show-workspace-export-frames-dialog [frames] @@ -200,7 +195,12 @@ :file-id file-id :object-id (:id frame) :shape frame - :name (:name frame)}) + :name (:name frame) + ;; pages render as one pdf: the kind + ;; travels declared, the items fully typed + :type :pdf + :scale 1 + :suffix ""}) frames)] (rx/of (modal/show :export-frames @@ -208,28 +208,37 @@ :origin "workspace:menu" :name page-name})))))) +;; ---- THE PROGRESS WIDGET STATE ------------------------------------ +;; The events this namespace emits for the export progress widget. The +;; job's channel (`dj/watch-job`) turns into them on its way up. + (defn- initialize-export-status - "`job` is only present on the job API path; without it the widget counts the - exports the client submitted, exactly as it always has." - [exports cmd resource {:keys [job-id total status backend] :as job}] + "The state an export begins at, keyed by `id`: the job id of a backend + export, or the client id of a local WASM one. A backend job starts + queued until its first milestone lands — the creation answers the row + status `pending`, and nobody works on it yet; a local run already + carries its `:backend`, its resource and the figures it knows up front." + [exports cmd {:keys [id status resource-id total backend]}] (ptk/reify ::initialize-export-status ptk/UpdateEvent (update [_ state] - (assoc state :export (cond-> {:in-progress true - :resource-id (:id resource) - :healthy? true - :error false - :progress 0 - :widget-visible true - :detail-visible true - :exports exports - :last-update (ct/now) - :cmd cmd} - (some? job) - (assoc :job-id job-id - :total total - :status status - :backend backend)))))) + (assoc state :export + (cond-> {:in-progress true + :healthy? true + :error false + :progress 0 + :widget-visible true + :detail-visible true + :exports exports + :last-update (ct/now) + :cmd cmd + :job-id id} + (some? status) (assoc :status (if (= "pending" status) + "queued" + status)) + (some? resource-id) (assoc :resource-id resource-id) + (some? total) (assoc :total total) + (some? backend) (assoc :backend backend)))))) (defn- update-export-status [{:keys [done total status resource-uri filename mtype] :as data}] @@ -272,28 +281,128 @@ (watch [_ _ _] (when (= status "ended") (dom/trigger-download-uri filename mtype resource-uri))))) +(defn- counter-of + "The figures of one milestone: `objects` counts the shapes and + `pages` counts the frames, both as `current` exported and `total` + planned. A job publishes one kind or the other; this takes whatever + it carries." + [counters] + (or (:objects counters) (:pages counters))) + +(defn- add-milestone + "One beat of the job into the widget: what has landed and what is + coming, named by the stage the run is in. The status moves to running + on the first beat, figures or not. A milestone without figures (the + first breath of the worker) names the stage but keeps the figures the + widget already holds. A milestone that arrives once the widget + settled from a cancel or an error is ignored." + [payload] + (let [figures (counter-of (:counters payload))] + (ptk/reify ::add-milestone + ptk/UpdateEvent + (update [_ state] + ;; A milestone that arrives while the widget settles a cancel + ;; never flips it back to running (cancelling -> running flicker). + (if (or (not (get-in state [:export :in-progress])) + (= "cancelling" (get-in state [:export :status]))) + state + (-> (update state :export assoc + :status "running" + :stage (:stage payload) + :last-update (ct/now)) + (cond-> (some? figures) + (update :export assoc + :progress (:current figures) + :total (or (:total figures) + (get-in state [:export :total])))))))))) + +(defn- add-outcome + "The last row of the job as the end of the widget. A completed job + downloads its artifact right away and the widget goes away after a + moment; a failed one stays, naming the error; a cancelled one just + settles; and a cancelling step keeps the widget alive while the + cancel flies." + [{:keys [status result error]}] + (let [downloadable? (or (= "completed" status) (= "ended" status))] + (ptk/reify ::add-outcome + ptk/UpdateEvent + (update [_ state] + (update state :export (fn [export] + (if-not (and (map? export) + (or (:in-progress export) + (= "cancelling" (:status export)))) + export + (cond + (= "cancelling" status) + (assoc export :status "cancelling" + :last-update (ct/now)) + + downloadable? + (assoc export :status "ended" + :in-progress false + :last-update (ct/now)) + + (= "failed" status) + ;; The backend reports a cancelled job as + ;; "failed" with no error (shared + ;; user-status, also read by binfile + ;; flows): a failure with no error while + ;; the widget settles a cancel is that + ;; cancel landing, not an error. + (if (and (nil? error) + (= "cancelling" (:status export))) + (assoc export :status "cancelled" + :in-progress false + :last-update (ct/now)) + (assoc export :status "error" + :in-progress false + :error (:hint error) + :error-code (:code error) + :last-update (ct/now))) + + (= "cancelled" status) + (assoc export :status "cancelled" + :in-progress false + :last-update (ct/now)) + + :else export))))) + + ptk/WatchEvent + (watch [_ state _] + (when downloadable? + (dom/trigger-download-uri (:filename result) + (:mtype result) + (:resource-uri result))) + (when (or downloadable? (= "cancelled" status)) + ;; the widget has its moment naming what happened, and goes + ;; away; the id it carries is the run that ended, so a clear + ;; that lands late cannot wipe a newer one + (let [job-id (get-in state [:export :job-id])] + (->> (rx/of (clear-export-state job-id)) + (rx/delay default-timeout)))))))) (def ^:private local-export-job-id* "Job id of the in-flight worker multi-export, if any." (atom nil)) (defn cancel-export - "Stops the running export. On the job API path the exporter aborts; on the - local WASM path the worker is told to stop and the widget settles here. - - Local cancel settles on the client: if the worker is stuck before the first - cancel check, waiting for a `:cancelled` reply would leave the UI hung." + "Stops the running export: the X of the widget asks the backend to + cancel the job — fire and forget, the job may have just ended on its + own — and settles the widget itself, because the outcome is known once + the cancel is dispatched; waiting on the row and the websocket would + leave it stuck whenever that message is missed. The local WASM path + tells the worker to stop and settles on the client, because a worker + stuck before its first cancel check would leave the UI hung." [] (ptk/reify ::cancel-export ptk/WatchEvent (watch [_ state _] - (let [backend (get-in state [:export :backend]) - job-id (get-in state [:export :job-id]) - resource-id (get-in state [:export :resource-id]) - settle (rx/concat - (rx/of (update-export-status {:status "cancelled"})) - (->> (rx/of (clear-export-state resource-id)) - (rx/delay default-timeout)))] + (let [backend (get-in state [:export :backend]) + job-id (get-in state [:export :job-id]) + settle (rx/concat + (rx/of (update-export-status {:status "cancelled"})) + (->> (rx/of (clear-export-state job-id)) + (rx/delay default-timeout)))] (cond (= "local" backend) (do @@ -304,15 +413,66 @@ settle) (some? job-id) - (rx/concat - ;; Stopping is not instantaneous: the request has to reach the - ;; exporter and the work has to unwind. - (rx/of (update-export-status {:status "cancelling"})) - (->> (rp/cmd! :cancel-export-job {:job-id job-id}) - (rx/mapcat (fn [_] settle)) - ;; Already finished, or the exporter is gone; either way - ;; there is nothing left to stop. - (rx/catch (fn [_] settle))))))))) + (do + ;; The row is marked and the worker stops at its next beat, + ;; but the widget settles now: waiting on the row and the + ;; websocket would leave it stuck whenever that message is + ;; missed, and the cancel button is already gone by then. + (dj/cancel-job job-id) + settle)))))) + +;; ---- THE JOB FLOW ------------------------------------------------- + +(defn- ->widget-event + "One emission of the job taken from `dj/watch-job` is the event the + widget consumes: the payload of a progress event carries the + milestone, and the row at the end answers with the outcome." + [emission] + (cond + (and (= :progress (:kind emission)) + (map? (:payload emission))) + (add-milestone (:payload emission)) + + (contains? emission :status) + (add-outcome emission))) + +(defn- export-stream! + "The whole life of one export job as a stream of widget events: the + creation of the job, then its channel (`dj/watch-job`) mapped into + the vocabulary of the widget, and the stream over once the outcome + lands. The start and retry events of the channel are dropped by the + mapper. + + A creation that could not even happen (the export queue saturated, + mostly) settles the widget and shows the failure as a toast; nothing + is raised to the global error handler, which would replace the + workspace with an error page." + [ws-conn exports cmd {:keys [force-multiple name is-wasm kind]}] + (let [params (cond-> {:exports exports + :is-wasm is-wasm} + (some? name) + (assoc :name name) + + (some? kind) + (assoc :kind kind) + + (some? force-multiple) + (assoc :force-multiple force-multiple))] + + (->> (rp/cmd! :create-export-assets-job {:params params}) + (rx/mapcat + (fn [job] + (rx/concat + (rx/of (initialize-export-status exports cmd + {:id (:id job) + :status (:status job)})) + (->> (dj/watch-job ws-conn (:id job)) + (rx/map ->widget-event) + (rx/filter some?))))) + (rx/catch (fn [cause] + (rx/of (export-widget-failed exports cmd cause)))) + (rx/finalize (fn [] + (swap! st/ongoing-tasks disj :export)))))) (def ^:private wasm-export-types #{:jpeg :webp :png :pdf :svg}) @@ -371,23 +531,14 @@ :too-large (rx/of (ntf/error (tr "errors.export-too-large"))) - (let [profile-id (:profile-id state) - params (normalize-export-shapes-params {:exports [export] - :profile-id profile-id - :cmd :export-shapes - :wait true - :is-wasm (wasm-export-enabled? state)})] + (let [ws-conn (:ws-conn state) + file-ids (export-file-ids [export])] + (swap! st/ongoing-tasks conj :export) (rx/concat - (dwp/force-persist-files-and-wait (export-file-ids [export]) save-wait-timeout-ms) - (rx/of (warn-unsaved-changes (export-file-ids [export]))) - - (->> (rp/cmd! :export params) - (rx/map (fn [{:keys [filename mtype uri]}] - (dom/trigger-download-uri filename mtype uri) - (clear-export-state uuid/zero))) - (rx/catch (fn [cause] - (rx/of (clear-export-state uuid/zero) - (export-failed cause)))))))))))) + (dwp/force-persist-files-and-wait file-ids save-wait-timeout-ms) + (rx/of (warn-unsaved-changes file-ids)) + (export-stream! ws-conn [export] :export-shapes + {:is-wasm (wasm-export-enabled? state)})))))))) (defn- export-file-page-ids [state exports] @@ -443,15 +594,15 @@ objects (collect-export-objects state exports) background (export-page-background state exports) entries (worker-export-entries exports) - clear-later (->> (rx/of (clear-export-state resource-id)) + clear-later (->> (rx/of (clear-export-state job-id)) (rx/delay default-timeout))] (reset! local-export-job-id* job-id) (swap! st/ongoing-tasks conj :export) (rx/concat (rx/of (initialize-export-status exports cmd - {:id resource-id} - {:job-id job-id + {:id job-id + :resource-id resource-id :total total :status "running" :backend "local"})) @@ -519,75 +670,20 @@ (ptk/reify ::request-multiple-export-server ptk/WatchEvent (watch [_ state _] - (let [resource-id (volatile! nil) - ;; Emits when the export request fails. - failed-s (rx/subject) - profile-id (:profile-id state) - ws-conn (:ws-conn state) - params (cond-> - {:exports exports - :cmd cmd - :profile-id profile-id - :force-multiple true - :is-wasm (wasm-export-enabled? state)} - (some? name) - (assoc :name name)) - - progress-stream - (->> (ws/get-rcv-stream ws-conn) - (rx/filter ws/message-event?) - (rx/map :payload) - (rx/filter #(= :export-update (:type %))) - (rx/filter #(= @resource-id (:resource-id %))) - (rx/take-until failed-s) - (rx/share)) - - stopper - (rx/filter #(or (= "ended" (:status %)) - (= "error" (:status %)) - (= "cancelled" (:status %))) - progress-stream) - - file-ids - (export-file-ids exports)] - + (let [ws-conn (:ws-conn state) + file-ids (export-file-ids exports) + ;; the frames dialog declares its run; every other multiple + ;; export is a shapes one + kind (if (= :export-frames cmd) :frames :shapes)] (swap! st/ongoing-tasks conj :export) - (rx/concat (dwp/force-persist-files-and-wait file-ids save-wait-timeout-ms) (rx/of (warn-unsaved-changes file-ids)) - (rx/merge - ;; Updates the export state with each progress message, until - ;; shortly after the export ends or fails. - (->> progress-stream - (rx/map update-export-status) - (rx/take-until (rx/delay 500 stopper)) - (rx/finalize (fn [] - (swap! st/ongoing-tasks disj :export)))) - - ;; Clears the export state a while after the export ends or is - ;; cancelled. - (->> progress-stream - (rx/filter #(or (= "ended" (:status %)) - (= "cancelled" (:status %)))) - (rx/take 1) - (rx/delay default-timeout) - (rx/map #(clear-export-state @resource-id)) - (rx/take-until (rx/delay 6000 stopper))) - - ;; Launches the exportation process and stores the resource id - ;; locally. Goes last so a failure reaches the progress listeners - ;; above: the push to `failed-s` releases the leave-page warning - ;; even though no progress message will ever arrive, and nothing - ;; is raised to the global error handler, which would replace the - ;; workspace with an error page. - (->> (rp/cmd! :export params) - (rx/map (fn [{:keys [id] :as resource}] - (vreset! resource-id id) - (initialize-export-status exports cmd resource nil))) - (rx/catch (fn [cause] - (rx/push! failed-s cause) - (rx/of (export-widget-failed exports cmd cause))))))))))) + (export-stream! ws-conn exports cmd + {:force-multiple true + :name name + :kind kind + :is-wasm (wasm-export-enabled? state)})))))) (defn request-multiple-export [{:keys [exports cmd] diff --git a/frontend/src/app/main/data/jobs.cljs b/frontend/src/app/main/data/jobs.cljs index 9ffc801512..98513a2577 100644 --- a/frontend/src/app/main/data/jobs.cljs +++ b/frontend/src/app/main/data/jobs.cljs @@ -18,6 +18,10 @@ [beicon.v2.core :as rx])) (def ^:private terminal-statuses + ;; "cancelled" stays in the vocabulary: binfile flows map the outcome + ;; explicitly and their tests pin it, even though get-job reports + ;; cancelled rows as failed (shared user-status). The assets widget + ;; tells a mapped cancel apart through its own `cancelling` state. #{"completed" "failed" "cancelled"}) (defn- events-of @@ -84,6 +88,24 @@ (rx/merge (rx/take-until outcomes progress) outcomes))) +(defn outcome-result + "One terminal outcome of `watch-job` into its result: a completed + job answers its artifact, any other outcome throws its error. Lets + one-shot readers (clipboard, plugins) settle on failure instead of + filtering for `\"completed\"` and hanging when the job ends any + other way. The backend maps a cancelled export to `\"failed\"` + without an error (shared `get-user-status`, also read by binfile + flows, so it stays as it is): such an outcome throws a generic + export failure, and the widget tells it apart through its own + `cancelling` state." + [{:keys [status result error]}] + (if (= "completed" status) + result + (throw (ex-info "export job failed" + (or error {:type :internal + :code :export-failed + :hint "export job failed"}))))) + (defn cancel-job "Ask the server to cancel a job the caller created. Fire and forget: failures are ignored because the job may have just finished on its diff --git a/frontend/src/app/main/data/workspace/clipboard.cljs b/frontend/src/app/main/data/workspace/clipboard.cljs index 877e535f75..ebe97e3db7 100644 --- a/frontend/src/app/main/data/workspace/clipboard.cljs +++ b/frontend/src/app/main/data/workspace/clipboard.cljs @@ -37,6 +37,7 @@ [app.main.data.exports.assets :as de] [app.main.data.exports.wasm :as wasm.exports] [app.main.data.helpers :as dsh] + [app.main.data.jobs :as dj] [app.main.data.notifications :as ntf] [app.main.data.persistence :as dps] [app.main.data.workspace.media :as dwm] @@ -1208,6 +1209,23 @@ (rx/map (fn [_] (ntf/info "The id has been copied to the clipboard"))))))) +(defn image-export + "The single item of a copy-as-image export, in the shape the shared + export contract accepts: `:enabled` and other UI flags stay in the + dialog, never in the payload." + [file-id page-id object-id] + (de/normalize-export {:file-id file-id + :page-id page-id + :object-id object-id + ;; webp would be preferrable, but PNG is the most + ;; supported image MIME type by clipboard APIs. + :type :png + ;; Always use 2 to ensure good enough quality for + ;; wireframes. + :scale 2 + :suffix "" + :name ""})) + (defn copy-as-image [] (ptk/reify ::copy-as-image @@ -1217,16 +1235,7 @@ page-id (:current-page-id state) selected (first (dsh/lookup-selected state)) - export (de/normalize-export {:file-id file-id - :page-id page-id - :object-id selected - ;; webp would be preferrable, but PNG is the most supported image MIME type by clipboard APIs. - :type :png - ;; Always use 2 to ensure good enough quality for wireframes. - :scale 2 - :suffix "" - :enabled true - :name ""}) + export (image-export file-id page-id selected) ;; Create a deferred promise immediately, before any async operations. ;; Registering the clipboard write NOW preserves the user-gesture security @@ -1244,11 +1253,21 @@ ;; Call exporter to get image URI, then fetch blob and resolve the deferred. (->> (if (features/active-feature? state "render-wasm/v1") (rx/of {:uri (wasm.exports/export-image-uri export)}) - (rp/cmd! :export - {:exports [export] - :profile-id (:profile-id state) - :cmd :export-shapes - :wait true})) + + ;; One export job, its artifact under the completed row's + ;; result: the URI of the resource is what the blob fetch + ;; reads now, as the download of the exports does. + (->> (rp/cmd! :create-export-assets-job + {:params {:exports [export]} + ;; the blob is consumed right here: nothing + ;; reads the row or the object later + :transient true}) + (rx/mapcat (fn [{job-id :id}] + (->> (dj/watch-job (:ws-conn state) job-id) + (rx/filter :status) + (rx/timeout 120000) + (rx/map dj/outcome-result)))) + (rx/map (fn [result] {:uri (:resource-uri result)})))) (rx/mapcat (fn [{:keys [uri]}] (http/send! {:method :get diff --git a/frontend/src/app/main/repo.cljs b/frontend/src/app/main/repo.cljs index c3dca3b882..5150d361db 100644 --- a/frontend/src/app/main/repo.cljs +++ b/frontend/src/app/main/repo.cljs @@ -319,45 +319,6 @@ (rx/map http/conditional-decode-transit) (rx/mapcat handle-response)))) -(defn- send-export - [{:keys [blob?] :as params}] - (->> (http/send! {:method :post - :uri (u/join cf/public-uri "api/export") - :body (http/transit-data (dissoc params :blob?)) - :headers {"x-external-session-id" (cf/external-session-id) - "x-event-origin" (::ev/origin (meta params))} - :credentials "include" - :response-type (if blob? :blob :text)}) - (rx/map http/conditional-decode-transit) - (rx/mapcat handle-response))) - -(defmethod cmd! :export - [_ params] - (let [default {:wait false :blob? false}] - (send-export (merge default params)))) - -(defmethod cmd! :create-export-job - [_ params] - (->> (http/send! {:method :post - :uri (u/join cf/public-uri "api/export/jobs") - :body (http/transit-data params) - :headers {"x-external-session-id" (cf/external-session-id) - "x-event-origin" (::ev/origin (meta params))} - :credentials "include" - :response-type :text}) - (rx/map http/conditional-decode-transit) - (rx/mapcat handle-response))) - -(defmethod cmd! :cancel-export-job - [_ {:keys [job-id]}] - (->> (http/send! {:method :delete - :uri (u/join cf/public-uri "api/export/jobs/" (str job-id)) - :headers {"x-external-session-id" (cf/external-session-id)} - :credentials "include" - :response-type :text}) - (rx/map http/conditional-decode-transit) - (rx/mapcat handle-response))) - (defn- multipart-upload [id params] (->> (http/send! {:method :post diff --git a/frontend/src/app/main/ui/exports/assets.cljs b/frontend/src/app/main/ui/exports/assets.cljs index 6f7f2311aa..a407c0d98f 100644 --- a/frontend/src/app/main/ui/exports/assets.cljs +++ b/frontend/src/app/main/ui/exports/assets.cljs @@ -202,6 +202,19 @@ ;; FIXME: deprecated, should be refactored in two components and use ;; the generic progress reporter +(defn cancellable? + "Whether the progress widget offers cancel: local runs and server + jobs with an id that did not settle yet — running or still queued, + since the backend accepts a cancel for a pending job too." + [{:keys [in-progress backend job-id status]}] + (let [cancelling? (and (some? job-id) (= "cancelling" status))] + (and in-progress + (not cancelling?) + (or (= "local" backend) + (and (some? job-id) + (or (= "running" status) + (= "queued" status))))))) + (mf/defc progress-widget {::mf/wrap [mf/memo]} [] @@ -212,7 +225,8 @@ error? (:error state) ;; The exporter is at capacity: worth its own wording, so the user ;; knows retrying later is the thing to do. - busy? (= :queue-full (:error-code state)) + busy? (contains? #{:queue-full :max-quote-reached} + (:error-code state)) healthy? (:healthy? state) detail-visible? (:detail-visible state) widget-visible? (:widget-visible state) @@ -220,23 +234,33 @@ items (:exports state) job-id (:job-id state) status (:status state) + ;; The stages of the job, named: what the worker reports beats + ;; a single "exporting" wording. + stage-title (case (:stage state) + :preparing (tr "workspace.options.exporting-preparing") + :packaging (tr "workspace.options.exporting-packaging") + nil) queued? (and (some? job-id) (= "queued" status)) cancelling? (and (some? job-id) (= "cancelling" status)) cancelled? (and (some? job-id) (= "cancelled" status)) - ;; Server wasm jobs and local client-side exports can stop; a non-wasm - ;; browser render holds its pool slot until playwright gives up. - cancellable? (and (:in-progress state) - (not cancelling?) - (or (= "local" (:backend state)) - (and (some? job-id) - (= "wasm" (:backend state))))) + ;; Server export jobs and local client-side exports can stop; a + ;; non-wasm browser render holds its pool slot until playwright + ;; gives up. + cancellable? (cancellable? state) total (or (:total state) (count items)) complete? (= progress total) + ;; The figure the detail names: a percentage when the worker + ;; said how much there is, nothing when it has not said yet + ;; (queued, or a milestone without counters). + percent (when (and (number? progress) + (number? total) + (pos? total)) + (js/Math.round (* 100 (/ progress total)))) circ (* 2 Math/PI 12) pct (if (zero? total) circ (- circ (* circ (/ progress total)))) pwidth - (if error? + (if (or error? (zero? total)) 280 (/ (* progress 280) total)) @@ -263,6 +287,7 @@ cancelled? (tr "workspace.options.exporting-cancelled") queued? (tr "workspace.options.exporting-queued") complete? (tr "workspace.options.exporting-complete") + (and healthy? (some? stage-title)) stage-title healthy? (tr "workspace.options.exporting-object") (not healthy?) (tr "workspace.options.exporting-object-slow")) @@ -325,16 +350,22 @@ [:button {:class (stl/css :retry-btn) :on-click cancel-export} (tr "workspace.options.cancel-export")] - [:span {:class (stl/css :progress)} - (dm/str progress " / " total)]] + (when (some? percent) + [:span {:class (stl/css :progress)} + (dm/str percent "%")])] - ;; A counter for work that is being abandoned says nothing useful. - (or cancelling? cancelled?) + ;; A counter for work that is waiting, going away, or gone + ;; says nothing useful. + (or queued? cancelling? cancelled?) nil - :else + ;; The running figure, once the worker said how much there is. + (some? percent) [:span {:class (stl/css :progress)} - (dm/str progress " / " total)])] + (dm/str percent "%")] + + :else + nil)] [:> icon-button* {:variant "action" :tooltip-class (stl/css :progress-close-button) diff --git a/frontend/src/app/plugins/shape.cljs b/frontend/src/app/plugins/shape.cljs index 60a06e2c0f..10f1934d08 100644 --- a/frontend/src/app/plugins/shape.cljs +++ b/frontend/src/app/plugins/shape.cljs @@ -37,6 +37,7 @@ [app.common.uuid :as uuid] [app.main.data.exports.assets :as de] [app.main.data.exports.wasm :as wasm.exports] + [app.main.data.jobs :as dj] [app.main.data.persistence :as dwp] [app.main.data.plugins :as dp] [app.main.data.workspace :as dw] @@ -1676,20 +1677,18 @@ (rx/tap #(st/emit! (se/event plugin-id "export-shapes" :method "wasm"))) (rx/subs! resolve reject))))) - ;; Render via the exporter service. - (let [shape (u/locate-shape file-id page-id id) - payload - {:cmd :export-shapes - :profile-id (:profile-id @st/state) - :wait true - :is-wasm wasm-enabled? - :exports [(de/normalize-export {:file-id file-id - :page-id page-id - :object-id id - :name (:name shape) - :type export-type - :suffix (:suffix value "") - :scale (:scale value 1)})]}] + ;; One export job of the named item: the worker + ;; renders it, the completed row carries the artifact + ;; under its result, and the blob fetch reads its + ;; resource uri. + (let [shape (u/locate-shape file-id page-id id) + item (de/normalize-export {:file-id file-id + :page-id page-id + :object-id id + :name (:name shape) + :type export-type + :suffix (:suffix value "") + :scale (:scale value 1)})] (js/Promise. (fn [resolve reject] ;; Exporter reads persisted file state; flush first @@ -1699,10 +1698,20 @@ (->> (rx/concat (->> (dwp/wait-persisted 5000) (rx/ignore)) - (rp/cmd! :export payload)) - (rx/mapcat (fn [{:keys [uri]}] + (->> (rp/cmd! :create-export-assets-job + {:params {:exports [item] + :is-wasm wasm-enabled?} + ;; the plugin consumes the blob + ;; in this same call + :transient true}) + (rx/mapcat (fn [{job-id :id}] + (->> (dj/watch-job (:ws-conn @st/state) job-id) + (rx/filter :status) + (rx/timeout 120000) + (rx/map dj/outcome-result)))))) + (rx/mapcat (fn [result] (->> (http/send! {:method :get - :uri uri + :uri (:resource-uri result) :response-type :blob :omit-default-headers true}) (rx/map :body)))) diff --git a/frontend/test/frontend_tests/data/exports_assets_test.cljs b/frontend/test/frontend_tests/data/exports_assets_test.cljs index 4673b13411..a30e5a796f 100644 --- a/frontend/test/frontend_tests/data/exports_assets_test.cljs +++ b/frontend/test/frontend_tests/data/exports_assets_test.cljs @@ -5,10 +5,15 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns frontend-tests.data.exports-assets-test + "The assets export: one job per server run (its milestones into the + widget, the artifact of the completed row as the download, the cancel + from the widget) and the client-side WASM render that never leaves the + browser." (:require [app.common.uuid :as uuid] [app.main.data.exports.assets :as de] [app.main.data.exports.wasm :as wasm.exports] + [app.main.data.jobs :as dj] [app.main.data.persistence :as dwp] [app.main.errors :as errors] [app.main.repo :as repo] @@ -18,13 +23,13 @@ [app.util.dom :as dom] [app.util.i18n :as i18n] [app.util.webapi :as wapi] - [app.util.websocket :as ws] [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] [frontend-tests.helpers.async :as async] [frontend-tests.helpers.events :as the] [frontend-tests.helpers.mock :as mock] [potok.v2.core :as ptk])) + (def ^:private export {:id (uuid/next) :object-id (uuid/next) :type :png @@ -36,85 +41,17 @@ (merge export {:name name})) (defn- test-state - [] - {:profile-id (:id export) - :ws-conn nil}) + ([] (test-state nil)) + ([export-state] + (cond-> {:profile-id (:id export) + :ws-conn nil} + (some? export-state) + (assoc :export export-state)))) (defn- wasm-state [] (assoc (test-state) :features #{"render-wasm/v1"})) -(t/deftest sanitize-filename-strips-reserved-chars-and-dedupes - (t/is (= "a_b.png" - (wasm.exports/sanitize-filename {:name "a/b" :suffix "" :type :png} #{}))) - (t/is (= "board-2.pdf" - (wasm.exports/sanitize-filename {:name "board" :suffix "" :type :pdf} - #{"board.pdf"})))) - -(t/deftest normalize-export-preserves-existing-name - (t/is (= (export-with-name "Layer 1") - (de/normalize-export (export-with-name "Layer 1"))))) - -(t/deftest normalize-export-replaces-nil-name-with-object-id - (t/is (= (export-with-name (str (:object-id export))) - (de/normalize-export (assoc export :name nil))))) - -(t/deftest normalize-export-replaces-empty-name-with-object-id - (t/is (= (export-with-name (str (:object-id export))) - (de/normalize-export (assoc export :name ""))))) - -(t/deftest request-simple-export-sends-normalized-export - (t/async done - (let [export (export-with-name "") - observed (atom nil)] - (mock/with-mocks {repo/cmd! (mock/stub (fn [_ params] - (reset! observed params) - (rx/of {:filename "export.png" - :mtype "image/png" - :uri "blob:export"}))) - dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/of ::force-persisted))) - dom/trigger-download-uri (mock/stub (fn [& _] nil))} - (fn [done'] - (let [completed (fn [_state] - (t/is (= (export-with-name (str (:object-id export))) - (-> @observed :exports first))))] - (ptk/emit! (the/prepare-store (test-state) done' completed) - (de/request-simple-export {:export export}) - :the/end))) - done)))) - -(t/deftest request-multiple-export-sends-normalized-enabled-exports - (t/async done - (let [exports [{:id "enabled-1" - :object-id "enabled-1" - :shape {:id "enabled-1"} - :type :png - :suffix "" - :scale 1 - :enabled true - :name ""}] - observed (atom nil)] - (mock/with-mocks {repo/cmd! (mock/stub (fn [_ params] - (reset! observed params) - (rx/of {:id (:id export)}))) - ws/get-rcv-stream (mock/stub (fn [_] (rx/empty))) - st/ongoing-tasks (atom #{})} - (fn [done'] - (let [completed (fn [_state] - (t/is (= [{:id "enabled-1" - :object-id "enabled-1" - :shape {:id "enabled-1"} - :type :png - :suffix "" - :scale 1 - :enabled true - :name "enabled-1"}] - (:exports @observed))))] - (ptk/emit! (the/prepare-store (test-state) done' completed) - (de/request-multiple-export {:exports exports}) - :the/end))) - done)))) - (defn- export-store "A store for the export tests that records store errors in `store-errors`." [state store-errors] @@ -128,9 +65,317 @@ :persistence {:queue (conj #queue [] commit-id) :index {commit-id {:id commit-id :file-id file-id}}}))) -;; Scenario: the exporter fails a single export. Proves: the user gets a -;; toast and stays in the workspace; the failure is not raised to the global -;; error handler, which would replace the workspace with an error page. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; THE JOB FLOW +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:private fake-job + "What the creation of the job answers: enough to follow it." + {:id (uuid/next) :status "queued" :name "export-assets"}) + +(defn- milestone + [stage current total counter-kind] + {:kind :progress + :payload {:stage stage + :counters {counter-kind {:current current :total total}}}}) + +(defn- fake-watch + "The channel of the job, as a stream of emissions the flow reads." + [emissions] + (apply rx/concat (map rx/of emissions))) + +(defn- completed-emission + [result] + {:status "completed" :result result}) + +(t/deftest normalize-export-preserves-existing-name + (t/is (= (export-with-name "Layer 1") + (de/normalize-export (export-with-name "Layer 1"))))) + +(t/deftest normalize-export-replaces-nil-name-with-object-id + (t/is (= (export-with-name (str (:object-id export))) + (de/normalize-export (assoc export :name nil))))) + +(t/deftest normalize-export-replaces-empty-name-with-object-id + (t/is (= (export-with-name (str (:object-id export))) + (de/normalize-export (assoc export :name ""))))) + +(t/deftest initialize-export-status-shows-a-fresh-job-as-queued + ;; the creation answers the row status ("pending"): nobody works on + ;; the job yet, so the widget names it queued until the first + ;; milestone moves it to running + (let [state (ptk/update (#'de/initialize-export-status + [export] nil {:id (uuid/next) :status "pending"}) + {})] + (t/is (= "queued" (get-in state [:export :status]))) + (t/is (true? (get-in state [:export :in-progress]))) + (t/is (= 0 (get-in state [:export :progress]))))) + +(t/deftest add-milestone-without-figures-keeps-the-figures + ;; the first breath of the worker (:preparing) carries no counters: + ;; it moves the widget to running without blanking the figures + (let [before {:export {:in-progress true :status "queued" + :progress 2 :total 5}} + after (ptk/update (#'de/add-milestone {:stage :preparing :counters {}}) + before)] + (t/is (= "running" (get-in after [:export :status]))) + (t/is (= 2 (get-in after [:export :progress]))) + (t/is (= 5 (get-in after [:export :total]))))) + +(t/deftest add-milestone-while-cancelling-keeps-cancelling + ;; A late beat that lands while the widget settles a cancel never + ;; flips it back to running (cancelling -> running flicker). + (let [before {:export {:in-progress true :status "cancelling" + :progress 1 :total 2}} + after (ptk/update (#'de/add-milestone {:stage :rendering + :counters {:objects {:current 2 + :total 2}}}) + before)] + (t/is (= "cancelling" (get-in after [:export :status]))) + (t/is (= 1 (get-in after [:export :progress]))))) + +(t/deftest add-outcome-failed-without-error-after-cancel-settles-cancelled + ;; The backend reports a cancelled job as "failed" with no error + ;; (shared user-status): while the widget settles a cancel, that + ;; outcome is the cancel landing, not an error. + (let [before {:export {:in-progress true :status "cancelling"}} + after (ptk/update (#'de/add-outcome {:status "failed"}) before)] + (t/is (= "cancelled" (get-in after [:export :status]))) + (t/is (false? (get-in after [:export :in-progress]))))) + +(t/deftest add-outcome-failed-with-error-stays-error + (let [before {:export {:in-progress true :status "running"}} + after (ptk/update (#'de/add-outcome {:status "failed" + :error {:hint "boom" :code :boom}}) + before)] + (t/is (= "error" (get-in after [:export :status]))) + (t/is (= "boom" (get-in after [:export :error]))))) + +(t/deftest request-simple-export-runs-one-assets-job + (t/async done + (let [export (export-with-name "") + observed (atom nil) + downloaded (atom nil)] + (mock/with-mocks + {repo/cmd! (mock/stub (fn [cmd params] + (reset! observed {:cmd cmd :params (:params params)}) + (rx/of fake-job))) + dj/watch-job (mock/stub (fn [_ _] + (fake-watch + [(milestone :rendering 1 1 :objects) + (milestone :packaging 1 1 :objects) + (completed-emission {:resource-uri "https://resources/uri" + :filename "export.png" + :mtype "image/png"})]))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/of ::force-persisted))) + dom/trigger-download-uri (mock/stub (fn [filename mtype uri] + (when filename + (reset! downloaded + {:filename filename :mtype mtype :uri uri})))) + st/ongoing-tasks (atom #{})} + (fn [done'] + (let [completed (fn [state] + (t/testing "the creation froze the normalized export" + (t/is (= :create-export-assets-job (:cmd @observed))) + (t/is (= [(export-with-name (str (:object-id export)))] + (get-in @observed [:params :exports])))) + + (t/is (= 1 (get-in state [:export :progress]))) + + (t/testing "the widget settled the completed job" + (t/is (false? (get-in state [:export :in-progress]))) + (t/is (= "ended" (get-in state [:export :status]))) + (t/is (= 1 (get-in state [:export :total])))) + + (t/testing "the artifact of the row is the download" + (t/is (= {:filename "export.png" + :mtype "image/png" + :uri "https://resources/uri"} + @downloaded))))] + (ptk/emit! (the/prepare-store (test-state) done' completed) + (de/request-simple-export {:export export}) + :the/end))) + done)))) + +(t/deftest request-simple-export-reads-the-pages-of-a-frame-export + ;; A frame export is fully typed (`:type :pdf`, as the frames dialog + ;; builds it), and its milestones count pages, not objects. + (t/async done + (let [frame {:id (uuid/next) + :object-id (uuid/next) + :type :pdf + :suffix "" + :scale 1 + :name "Frame 1"} + downloaded (atom nil)] + (mock/with-mocks + {repo/cmd! (mock/stub (fn [_ _] (rx/of fake-job))) + dj/watch-job (mock/stub (fn [_ _] + (fake-watch + [(milestone :rendering 1 2 :pages) + (milestone :packaging 2 2 :pages) + (completed-emission {:resource-uri "https://resources/uri" + :filename "file.pdf" + :mtype "application/pdf"})]))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/of ::force-persisted))) + dom/trigger-download-uri (mock/stub (fn [filename _mtype _uri] + (reset! downloaded filename))) + st/ongoing-tasks (atom #{})} + (fn [done'] + (let [completed (fn [state] + (t/is (= 2 (get-in state [:export :progress]))) + (t/is (= 2 (get-in state [:export :total]))) + (t/is (= "file.pdf" @downloaded)))] + (ptk/emit! (the/prepare-store (test-state) done' completed) + (de/request-simple-export {:export frame}) + :the/end))) + done)))) + +(t/deftest request-multiple-export-freezes-the-run-of-many + (t/async done + (let [items [(assoc export :name "" :enabled true)] + observed (atom nil)] + (mock/with-mocks + {repo/cmd! (mock/stub (fn [cmd params] + (reset! observed {:cmd cmd :params (:params params)}) + (rx/of fake-job))) + dj/watch-job (mock/stub (fn [_ _] + (fake-watch + [(milestone :rendering 1 1 :objects) + (completed-emission {:resource-uri "https://resources/uri" + :filename "export.zip" + :mtype "application/zip"})]))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/of ::force-persisted))) + dom/trigger-download-uri (mock/stub (fn [& _] nil)) + st/ongoing-tasks (atom #{})} + (fn [done'] + (let [completed (fn [state] + (t/testing "the creation froze the multiple run" + (t/is (= true (get-in @observed [:params :force-multiple])))) + (t/testing "the widget followed the job" + (t/is (false? (get-in state [:export :in-progress]))) + (t/is (= "ended" (get-in state [:export :status])))))] + (ptk/emit! (the/prepare-store (test-state) done' completed) + (de/request-multiple-export {:exports items}) + :the/end))) + done)))) + +(t/deftest request-multiple-export-declares-the-frames-kind + ;; A frames run names its kind up front: the dialog freezes fully + ;; typed pages plus `:kind :frames`, so the worker joins them into + ;; one pdf instead of inferring it from missing keys. + (t/async done + (let [items [{:page-id (uuid/next) + :file-id (uuid/next) + :object-id (uuid/next) + :name "page" + :type :pdf + :scale 1 + :suffix ""}] + observed (atom nil)] + (mock/with-mocks + {repo/cmd! (mock/stub (fn [cmd params] + (reset! observed {:cmd cmd :params (:params params)}) + (rx/of fake-job))) + dj/watch-job (mock/stub (fn [_ _] + (fake-watch + [(milestone :rendering 1 1 :pages) + (completed-emission {:resource-uri "https://resources/uri" + :filename "file.pdf" + :mtype "application/pdf"})]))) + dom/trigger-download-uri (mock/stub (fn [& _] nil)) + st/ongoing-tasks (atom #{})} + (fn [done'] + (let [completed (fn [state] + (t/testing "the creation declared the frames run" + (t/is (= :create-export-assets-job (:cmd @observed))) + (t/is (= :frames (get-in @observed [:params :kind]))) + (t/is (= [{:page-id (:page-id (first items)) + :file-id (:file-id (first items)) + :object-id (:object-id (first items)) + :name "page" + :type :pdf + :scale 1 + :suffix ""}] + (get-in @observed [:params :exports])))) + (t/testing "the widget followed the job" + (t/is (false? (get-in state [:export :in-progress]))) + (t/is (= "ended" (get-in state [:export :status])))))] + (ptk/emit! (the/prepare-store (test-state) done' completed) + (de/request-multiple-export {:exports items + :cmd :export-frames + :name "the export"}) + :the/end))) + done)))) + +(t/deftest request-multiple-export-reports-the-saturation + ;; The creation refused because the queue is full: an answer, not a + ;; fault — the widget names it and offers the retry. + (t/async done + (let [items [(assoc export :name "" :enabled true)]] + (mock/with-mocks + {repo/cmd! (mock/stub (fn [_ _] + (rx/throw (ex-info "quote exceeded" {:code :max-quote-reached})))) + dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/of ::force-persisted))) + errors/flash (mock/stub (fn [& _] nil)) + st/ongoing-tasks (atom #{})} + (fn [done'] + (let [completed (fn [state] + (t/testing "the widget names the exporter busy" + (t/is (false? (get-in state [:export :in-progress]))) + (t/is (= :max-quote-reached (get-in state [:export :error-code])))))] + (ptk/emit! (the/prepare-store (test-state) done' completed) + (de/request-multiple-export {:exports items}) + :the/end))) + done)))) + +(t/deftest cancel-export-asks-the-backend-and-settles + (t/async done + (let [job-id (uuid/next) + observed (atom nil)] + (mock/with-mocks + {repo/cmd! (mock/stub (fn [_ params] + (reset! observed params) + (rx/of nil)))} + (fn [done'] + (let [completed (fn [state] + (t/testing "the cancel of the job the widget tracks" + (t/is (= {:id job-id} @observed))) + (t/testing "and the widget settles without waiting for the row" + (t/is (= "cancelled" (get-in state [:export :status]))) + (t/is (false? (get-in state [:export :in-progress])))))] + (ptk/emit! (the/prepare-store (test-state {:job-id job-id + :in-progress true + :status "running"}) + done' completed) + (de/cancel-export) + :the/end))) + done)))) + +(t/deftest clear-export-state-only-clears-the-run-it-names + ;; a delayed clear of a finished run must not wipe the next one: the + ;; guard compares the job id the slot holds + (let [old-id (uuid/next) + new-id (uuid/next) + state {:export {:job-id old-id :in-progress false}}] + (t/testing "a clear of the run in the slot removes it" + (t/is (nil? (:export (ptk/update (de/clear-export-state old-id) state))))) + (t/testing "a clear of another run leaves the slot alone" + (let [state' (assoc-in state [:export :job-id] new-id)] + (t/is (= new-id (get-in (ptk/update (de/clear-export-state old-id) state') + [:export :job-id]))))) + (t/testing "a clear with no run in the slot still clears" + (t/is (nil? (:export (ptk/update (de/clear-export-state old-id) + {:export {:job-id nil}}))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; FAILURES AND PERSISTENCE +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Scenario: the creation of the job fails. Proves: the user gets a +;; toast and stays in the workspace; the failure is not raised to the +;; global error handler, which would replace the workspace with an error +;; page. (t/deftest ^:async failed-simple-export-shows-a-toast-instead-of-raising (let [store-errors (atom []) flashed (atom []) @@ -151,11 +396,13 @@ (t/is (= [["translated:errors.export-failed" :unhandled]] @flashed) "an exporter defect reports as unhandled") (t/is (empty? @store-errors) "nothing reaches the global error handler") - (t/is (nil? (:export @store)) "the export state is cleared"))))) + (t/is (false? (get-in @store [:export :in-progress])) + "the widget settled instead of waiting forever") + (t/is (= :unhandled (get-in @store [:export :error-code]))))))) -;; Scenario: the export request of a multiple export fails. Proves: the user -;; gets a toast, and the leave-page warning the export turned on is released -;; even though no progress message will ever arrive. +;; Scenario: the creation of a multiple export fails. Proves: the user +;; gets a toast, and the leave-page warning the export turned on is +;; released even though no progress message will ever arrive. (t/deftest ^:async failed-multiple-export-shows-a-toast-and-releases-the-leave-warning (let [store-errors (atom []) flashed (atom []) @@ -167,7 +414,6 @@ (rx/throw (ex-info "exporter failed" {:type :server-error})))) dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) - ws/get-rcv-stream (mock/stub (fn [_] (rx/subject))) st/ongoing-tasks tasks errors/flash (fn [& {:keys [hint]}] (swap! flashed conj hint)) i18n/tr (mock/stub #(str "translated:" %))} @@ -187,10 +433,14 @@ store (export-store (unsaved-state file-id) (atom []))] (await (mock/with-mocks* - {repo/cmd! (mock/stub (fn [_ _] - (rx/of {:filename "export.png" - :mtype "image/png" - :uri "blob:export"}))) + {repo/cmd! (mock/stub (fn [_ _] (rx/of fake-job))) + dj/watch-job (mock/stub + (fn [_ _] + (fake-watch + [(completed-emission + {:resource-uri "blob:export" + :filename "export.png" + :mtype "image/png"})]))) dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) dom/trigger-download-uri (mock/stub (fn [& _] (reset! downloaded true))) i18n/tr (mock/stub #(str "translated:" %))} @@ -207,10 +457,14 @@ store (export-store (test-state) (atom []))] (await (mock/with-mocks* - {repo/cmd! (mock/stub (fn [_ _] - (rx/of {:filename "export.png" - :mtype "image/png" - :uri "blob:export"}))) + {repo/cmd! (mock/stub (fn [_ _] (rx/of fake-job))) + dj/watch-job (mock/stub + (fn [_ _] + (fake-watch + [(completed-emission + {:resource-uri "blob:export" + :filename "export.png" + :mtype "image/png"})]))) dwp/force-persist-files-and-wait (mock/stub (fn [_ _] (rx/empty))) dom/trigger-download-uri (mock/stub (fn [& _] (reset! downloaded true)))} (ptk/emit! store (de/request-simple-export {:export (assoc export :file-id (uuid/next))})) @@ -218,6 +472,17 @@ (await (async/settle)) (t/is (nil? (:notification @store))))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; THE CLIENT-SIDE WASM RENDER +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest sanitize-filename-strips-reserved-chars-and-dedupes + (t/is (= "a_b.png" + (wasm.exports/sanitize-filename {:name "a/b" :suffix "" :type :png} #{}))) + (t/is (= "board-2.pdf" + (wasm.exports/sanitize-filename {:name "board" :suffix "" :type :pdf} + #{"board.pdf"})))) + ;; Scenario: WASM is on and the user exports one shape. Proves: the worker ;; builds a single blob (no ZIP); the exporter is not called. ;; Stub `export-target`: without a module it is nil; routing still stays diff --git a/frontend/test/frontend_tests/data/exports_contract_test.cljs b/frontend/test/frontend_tests/data/exports_contract_test.cljs new file mode 100644 index 0000000000..6821cdead2 --- /dev/null +++ b/frontend/test/frontend_tests/data/exports_contract_test.cljs @@ -0,0 +1,55 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY LTD + +(ns frontend-tests.data.exports-contract-test + "The payload of `create-export-assets-job` must match the shared + contract (`app.common.export/schema:params`, closed): the backend + rejects anything else with a params-validation error the user only + sees as a failed export. These tests validate the items the callers + build, so a UI flag cannot slip into the wire again." + (:require + [app.common.export :as export] + [app.common.schema :as sm] + [app.common.uuid :as uuid] + [app.main.data.exports.assets :as de] + [app.main.data.workspace.clipboard :as clipboard] + [cljs.test :as t :include-macros true])) + +(defn- valid? + [params] + (sm/validate export/schema:params params)) + +(defn- ids + [] + [(uuid/next) (uuid/next) (uuid/next)]) + +(t/deftest image-export-matches-the-contract + (let [[file-id page-id object-id] (ids) + item (clipboard/image-export file-id page-id object-id)] + (t/is (valid? {:exports [item]})) + (t/testing "the name is filled from the object id, never left blank" + (t/is (= (str object-id) (:name item)))) + (t/testing "and the clipboard flow sends exactly this item" + (t/is (= :png (:type item))) + (t/is (= 2 (:scale item)))))) + +(t/deftest a-ui-flag-is-not-part-of-the-contract + (let [[file-id page-id object-id] (ids) + item (assoc (clipboard/image-export file-id page-id object-id) + :enabled true)] + (t/testing "the closed schema refuses the flag the dialog keeps" + (t/is (not (valid? {:exports [item]})))))) + +(t/deftest frames-items-match-the-contract + (let [[file-id page-id object-id] (ids) + item (de/normalize-export {:file-id file-id + :page-id page-id + :object-id object-id + :type :pdf + :suffix "" + :scale 1 + :name "Frame 1"})] + (t/is (valid? {:exports [item] :kind :frames})))) diff --git a/frontend/test/frontend_tests/data/jobs_test.cljs b/frontend/test/frontend_tests/data/jobs_test.cljs index 996d9d1f0b..d0e0dbced6 100644 --- a/frontend/test/frontend_tests/data/jobs_test.cljs +++ b/frontend/test/frontend_tests/data/jobs_test.cljs @@ -126,6 +126,39 @@ :error {:type :internal :code :boom :hint "boom"}}] @seen)))))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; TERMINAL OUTCOMES AS VALUES +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest outcome-result-returns-the-artifact-on-completed + (t/is (= {:resource-uri "uri"} + (dj/outcome-result {:status "completed" + :result {:resource-uri "uri"}})))) + +(t/deftest outcome-result-throws-on-failed-instead-of-hanging + (try + (dj/outcome-result {:status "failed" + :error {:type :internal :code :boom :hint "boom"}}) + (t/is false "a failed outcome must throw, never resolve empty") + (catch :default cause + (t/is (= :boom (:code (ex-data cause))))))) + +(t/deftest outcome-result-throws-a-generic-error-on-failed-without-error + ;; A cancelled export arrives as "failed" with no error (shared + ;; user-status); one-shot readers still settle, with a generic error. + (try + (dj/outcome-result {:status "failed"}) + (t/is false "a failed outcome without error must still throw") + (catch :default cause + (t/is (= :export-failed (:code (ex-data cause))))))) + +(t/deftest outcome-result-throws-on-cancelled + (try + (dj/outcome-result {:status "cancelled"}) + (t/is false "a cancelled outcome must throw, never resolve empty") + (catch :default cause + (t/is (= :export-failed (:code (ex-data cause))))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; THE COMMANDS THAT FEED IT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -152,9 +185,10 @@ {:file-ids #{job-id} :export-type :detach-libraries}))) (await (hva/->promise (rp/cmd! :create-import-binfile-job {:project-id (str job-id)}))) (await (hva/->promise (rp/cmd! :get-job {:id (str job-id)}))) - (await (hva/->promise (rp/cmd! :create-export-job {}))) + (await (hva/->promise (rp/cmd! :create-export-assets-job + {:params {:exports []}}))) - (let [[export import read exporter] @requests] + (let [[export import read assets] @requests] (t/testing "the binfile export asks the backend, not the exporter service" (t/is (= :post (:method export))) (t/is (str/ends-with? (:path (:uri export)) @@ -171,6 +205,7 @@ "/api/main/methods/get-job")) (t/is (= (str job-id) (:id (:query read))))) - (t/testing "and the exporter keeps its own name" - (t/is (= :post (:method exporter))) - (t/is (str/ends-with? (:path (:uri exporter)) "/api/export/jobs")))))))) + (t/testing "and the assets export joins the same door" + (t/is (= :post (:method assets))) + (t/is (str/ends-with? (:path (:uri assets)) + "/api/main/methods/create-export-assets-job")))))))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index 5b21228c1d..d624f1300a 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -13,6 +13,7 @@ [frontend-tests.data.dashboard-test] [frontend-tests.data.event-test] [frontend-tests.data.exports-assets-test] + [frontend-tests.data.exports-contract-test] [frontend-tests.data.exports-files-test] [frontend-tests.data.fix-deleted-fonts-test] [frontend-tests.data.imports-test] @@ -127,6 +128,7 @@ [frontend-tests.ui.drawarea-test] [frontend-tests.ui.ds-controls-numeric-input-test] [frontend-tests.ui.error-boundary-test] + [frontend-tests.ui.exports-widget-test] [frontend-tests.ui.gradient-handlers-test] [frontend-tests.ui.history-test] [frontend-tests.ui.inspect-geometry-test] @@ -195,6 +197,7 @@ 'frontend-tests.data.repo-test 'frontend-tests.data.store-test 'frontend-tests.data.exports-assets-test + 'frontend-tests.data.exports-contract-test 'frontend-tests.data.exports-files-test 'frontend-tests.data.imports-test 'frontend-tests.data.jobs-test @@ -305,6 +308,7 @@ 'frontend-tests.ui.history-test 'frontend-tests.ui.inspect-geometry-test 'frontend-tests.ui.inspect-stroke-tokens-test + 'frontend-tests.ui.exports-widget-test 'frontend-tests.ui.jobs-progress-test 'frontend-tests.ui.layout-container-multiple-test 'frontend-tests.ui.measurements-test diff --git a/frontend/test/frontend_tests/ui/exports_widget_test.cljs b/frontend/test/frontend_tests/ui/exports_widget_test.cljs new file mode 100644 index 0000000000..844d98700d --- /dev/null +++ b/frontend/test/frontend_tests/ui/exports_widget_test.cljs @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.ui.exports-widget-test + "The cancel affordance of the assets export widget: local runs and + server jobs with an id offer it while they did not settle — running + or still queued, since the backend accepts a cancel for a pending + job too." + (:require + [app.common.uuid :as uuid] + [app.main.ui.exports.assets :as widget] + [cljs.test :as t :include-macros true])) + +(t/deftest cancellable-allows-running-and-queued-server-jobs + (let [job-id (uuid/next)] + (t/is (true? (widget/cancellable? {:in-progress true + :backend "exporter" + :job-id job-id + :status "running"}))) + (t/is (true? (widget/cancellable? {:in-progress true + :backend "exporter" + :job-id job-id + :status "queued"}))))) + +(t/deftest cancellable-refuses-settled-and-cancelling-widgets + (let [job-id (uuid/next)] + (t/is (false? (widget/cancellable? {:in-progress true + :backend "exporter" + :job-id job-id + :status "cancelling"}))) + (t/is (false? (widget/cancellable? {:in-progress false + :backend "exporter" + :job-id job-id + :status "queued"}))) + (t/is (false? (widget/cancellable? {:in-progress true + :backend "exporter" + :job-id nil + :status "running"}))))) + +(t/deftest cancellable-allows-local-runs + (t/is (true? (widget/cancellable? {:in-progress true + :backend "local" + :job-id nil + :status "running"})))) diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 2e790e4f14..70495f599f 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -77,7 +77,7 @@ msgstr "OpenID" msgid "auth.new-password" msgstr "Type a new password" -#: src/app/main/ui/auth/verify_token.cljs:81 +#: src/app/main/ui/auth/verify_token.cljs:54 msgid "auth.notifications.organization-invitation-accepted" msgstr "You're now part of %s" @@ -93,7 +93,7 @@ msgstr "Profile is not verified, please verify profile before continue." msgid "auth.notifications.recovery-token-sent" msgstr "Password recovery link sent to your inbox." -#: src/app/main/ui/auth/verify_token.cljs:85 +#: src/app/main/ui/auth/verify_token.cljs:58 msgid "auth.notifications.team-invitation-accepted" msgstr "Joined the team successfully" @@ -280,25 +280,25 @@ msgstr "Add as Shared Library" msgid "dashboard.change-email" msgstr "Change email" -#: src/app/main/data/nitrate.cljs:602 +#: src/app/main/data/nitrate.cljs:614 msgid "dashboard.change-organization-modal.accept" msgstr "MOVE TEAM" -#: src/app/main/data/nitrate.cljs:600 +#: src/app/main/data/nitrate.cljs:612 msgid "dashboard.change-organization-modal.choose" msgstr "Move to:" -#: src/app/main/data/nitrate.cljs:599 +#: src/app/main/data/nitrate.cljs:611 msgid "dashboard.change-organization-modal.description" msgstr "" "Projects and files will remain available to team members. The team will get " "the configuration from the new organization." -#: src/app/main/data/nitrate.cljs:601 +#: src/app/main/data/nitrate.cljs:613 msgid "dashboard.change-organization-modal.select" msgstr "Select an organization" -#: src/app/main/data/nitrate.cljs:591, src/app/main/data/nitrate.cljs:598, src/app/main/ui/dashboard/team_form.cljs:159 +#: src/app/main/data/nitrate.cljs:603, src/app/main/data/nitrate.cljs:610, src/app/main/ui/dashboard/team_form.cljs:159 msgid "dashboard.change-organization-modal.title" msgstr "Change team's organization" @@ -364,15 +364,15 @@ msgstr "Clear search" msgid "dashboard.clear-trash-button" msgstr "Clear trash" -#: src/app/main/data/dashboard.cljs:300, src/app/main/data/dashboard.cljs:535, src/app/main/data/workspace/pages.cljs:201 +#: src/app/main/data/dashboard.cljs:310, src/app/main/data/dashboard.cljs:545, src/app/main/data/workspace/pages.cljs:201 msgid "dashboard.copy-suffix" msgstr "(copy)" -#: src/app/main/ui/dashboard/organization_team_switch.cljs:223, src/app/main/ui/dashboard/organization_team_switch.cljs:304 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:299, src/app/main/ui/dashboard/organization_team_switch.cljs:390 msgid "dashboard.create-new-organization" msgstr "Create new organization" -#: src/app/main/ui/dashboard/organization_team_switch.cljs:295 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:381 msgid "dashboard.create-new-team" msgstr "Create new team" @@ -394,7 +394,7 @@ msgstr "" msgid "dashboard.delete-file-forever-confirmation.description" msgstr "Are you sure you want to delete forever %s? This is a non reversible action." -#: src/app/main/data/dashboard.cljs:792 +#: src/app/main/data/dashboard.cljs:802 msgid "dashboard.delete-files-success-notification" msgstr "%s files have been successfully deleted." @@ -413,7 +413,7 @@ msgstr "" "it forever an all of the files contained in it. This is a non reversible " "action." -#: src/app/main/data/dashboard.cljs:791, src/app/main/data/dashboard.cljs:825 +#: src/app/main/data/dashboard.cljs:801, src/app/main/data/dashboard.cljs:835 msgid "dashboard.delete-success-notification" msgstr "%s has been successfully deleted." @@ -501,23 +501,23 @@ msgstr "Import a .penpot file" msgid "dashboard.empty-project.start" msgstr "Start building amazing stuff" -#: src/app/main/data/dashboard.cljs:795 +#: src/app/main/data/dashboard.cljs:805 msgid "dashboard.errors.error-on-delete-files" msgstr "There was an error while deleting the files." -#: src/app/main/data/dashboard.cljs:828 +#: src/app/main/data/dashboard.cljs:838 msgid "dashboard.errors.error-on-delete-project" msgstr "There was an error while deleting the project %s." -#: src/app/main/data/dashboard.cljs:923, src/app/main/ui/dashboard/file_menu.cljs:227 +#: src/app/main/data/dashboard.cljs:933, src/app/main/ui/dashboard/file_menu.cljs:227 msgid "dashboard.errors.error-on-restore-file" msgstr "There was an error while restoring the file %s." -#: src/app/main/data/dashboard.cljs:924 +#: src/app/main/data/dashboard.cljs:934 msgid "dashboard.errors.error-on-restore-files" msgstr "There was an error while restoring the files." -#: src/app/main/data/dashboard.cljs:956 +#: src/app/main/data/dashboard.cljs:966 msgid "dashboard.errors.error-on-restoring-project" msgstr "There was an error while restoring the project %s and its files." @@ -643,7 +643,7 @@ msgstr "" "using [Transfonter](https://transfonter.org/) to generate webfonts and fix " "errors. " -#: src/app/main/ui/dashboard/organization_team_switch.cljs:240 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:293 msgid "dashboard.go-to-admin-console" msgstr "Go to Admin Console" @@ -651,15 +651,15 @@ msgstr "Go to Admin Console" msgid "dashboard.grid.last-modified-at" msgstr "Last modified %s" -#: src/app/main/ui/dashboard/import.cljs:995, src/app/main/ui/dashboard/project_menu.cljs:110 +#: src/app/main/ui/dashboard/import.cljs:997, src/app/main/ui/dashboard/project_menu.cljs:110 msgid "dashboard.import" msgstr "Import Penpot files" -#: src/app/main/ui/dashboard/import.cljs:343, src/app/worker/import.cljs:144, src/app/worker/import.cljs:147 +#: src/app/main/data/imports.cljs:154, src/app/main/data/imports.cljs:157, src/app/main/ui/dashboard/import.cljs:314 msgid "dashboard.import.analyze-error" msgstr "Oops! We couldn't import this file" -#: src/app/main/ui/dashboard/import.cljs:597 +#: src/app/main/ui/dashboard/import.cljs:571 msgid "dashboard.import.auto-linked-libraries" msgid_plural "dashboard.import.auto-linked-libraries" msgstr[0] "1 library was automatically linked by name." @@ -669,15 +669,15 @@ msgstr[1] "%s libraries were automatically linked by name." msgid "dashboard.import.bad-url" msgstr "Import failed. The template URL is incorrect" -#: src/app/main/ui/dashboard/import.cljs:733 +#: src/app/main/ui/dashboard/import.cljs:706 msgid "dashboard.import.confirm-library-links" msgstr "Confirm library links" -#: src/app/main/ui/dashboard/import.cljs:709 +#: src/app/main/ui/dashboard/import.cljs:682 msgid "dashboard.import.connect-selected-libraries" msgstr "Connect selected libraries" -#: src/app/main/ui/dashboard/import.cljs:296 +#: src/app/main/ui/dashboard/import.cljs:267 msgid "dashboard.import.file-name-label" msgstr "File name" @@ -749,37 +749,37 @@ msgstr "Rebuilding pages, components and assets in your project." msgid "dashboard.import.import-error.check-error" msgstr "We couldn't verify this file." -#: src/app/main/ui/dashboard/import.cljs:623 +#: src/app/main/ui/dashboard/import.cljs:597 msgid "dashboard.import.import-error.corrupt-file" msgstr "This file appears to be damaged." -#: src/app/main/ui/dashboard/import.cljs:603 +#: src/app/main/ui/dashboard/import.cljs:577 msgid "dashboard.import.import-error.disclaimer" msgstr "Not all files have been imported" -#: src/app/main/ui/dashboard/import.cljs:607 +#: src/app/main/ui/dashboard/import.cljs:581 msgid "dashboard.import.import-error.message1" msgstr "The following files have errors:" -#: src/app/main/ui/dashboard/import.cljs:627 +#: src/app/main/ui/dashboard/import.cljs:601 msgid "dashboard.import.import-error.message2" msgstr "Files with errors will not be uploaded." -#: src/app/main/ui/dashboard/import.cljs:626 +#: src/app/main/ui/dashboard/import.cljs:600 msgid "dashboard.import.import-error.unknown-error" msgstr "Something went wrong while processing this file." -#: src/app/main/ui/dashboard/import.cljs:593 +#: src/app/main/ui/dashboard/import.cljs:567 msgid "dashboard.import.import-message" msgid_plural "dashboard.import.import-message" msgstr[0] "1 file has been imported successfully." msgstr[1] "%s files have been imported successfully." -#: src/app/main/ui/dashboard/import.cljs:587 +#: src/app/main/ui/dashboard/import.cljs:561 msgid "dashboard.import.import-warning" msgstr "Some files containted invalid objects that have been removed." -#: src/app/main/ui/dashboard/import.cljs:710 +#: src/app/main/ui/dashboard/import.cljs:683 msgid "dashboard.import.next-file" msgstr "Next file" @@ -801,35 +801,35 @@ msgstr "" "Some libraries couldn't be linked automatically. Select the correct library " "for each:" -#: src/app/main/ui/dashboard/import.cljs:565 +#: src/app/main/ui/dashboard/import.cljs:539 msgid "dashboard.import.resolve-libraries-summary" msgstr "Review the library links before confirming:" -#: src/app/main/ui/dashboard/import.cljs:459 +#: src/app/main/ui/dashboard/import.cljs:433 msgid "dashboard.import.resolve-libraries.connect-to" msgstr "connect to" -#: src/app/main/ui/dashboard/import.cljs:453 +#: src/app/main/ui/dashboard/import.cljs:427 msgid "dashboard.import.resolve-libraries.original-library" msgstr "original library" -#: src/app/main/ui/dashboard/import.cljs:521 +#: src/app/main/ui/dashboard/import.cljs:495 msgid "dashboard.import.summary.linked" msgstr "auto-linked" -#: src/app/main/ui/dashboard/import.cljs:529 +#: src/app/main/ui/dashboard/import.cljs:503 msgid "dashboard.import.summary.manually-linked" msgstr "linked manually" -#: src/app/main/ui/dashboard/import.cljs:537 +#: src/app/main/ui/dashboard/import.cljs:511 msgid "dashboard.import.summary.new" msgstr "New" -#: src/app/main/ui/dashboard/import.cljs:558 +#: src/app/main/ui/dashboard/import.cljs:532 msgid "dashboard.import.summary.no-selection" msgstr "No library selected" -#: src/app/main/ui/dashboard/import.cljs:534 +#: src/app/main/ui/dashboard/import.cljs:508 msgid "dashboard.import.summary.original" msgstr "Original" @@ -887,7 +887,7 @@ msgstr "Here you have some Libraries and templates you can add to your project" msgid "dashboard.libraries-and-templates.explore" msgstr "Explore more of them and know how to contribute" -#: src/app/main/ui/dashboard/import.cljs:816, src/app/main/ui/workspace/libraries.cljs:229 +#: src/app/main/ui/dashboard/import.cljs:798, src/app/main/ui/workspace/libraries.cljs:229 msgid "dashboard.libraries-and-templates.import-error" msgstr "There was a problem importing the template. The template wasn't imported." @@ -923,7 +923,7 @@ msgstr "Move to other team" msgid "dashboard.new-file" msgstr "+ New File" -#: src/app/main/data/dashboard.cljs:506, src/app/main/data/dashboard.cljs:618 +#: src/app/main/data/dashboard.cljs:516, src/app/main/data/dashboard.cljs:628 msgid "dashboard.new-file-prefix" msgstr "New File" @@ -931,7 +931,7 @@ msgstr "New File" msgid "dashboard.new-project" msgstr "+ New project" -#: src/app/main/data/dashboard.cljs:259, src/app/main/data/dashboard.cljs:621 +#: src/app/main/data/dashboard.cljs:269, src/app/main/data/dashboard.cljs:631 msgid "dashboard.new-project-prefix" msgstr "New Project" @@ -943,7 +943,7 @@ msgstr "No matches found for “%s“" msgid "dashboard.no-organization-allows-create-team.message" msgstr "You don't have permission to add teams to any of your organizations." -#: src/app/main/ui/dashboard/organization_team_switch.cljs:185 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:256 msgid "dashboard.no-organizations-yet" msgstr "No organizations yet." @@ -973,11 +973,11 @@ msgstr "Pinned projects will appear here" msgid "dashboard.notifications" msgstr "Notifications" -#: src/app/main/ui/auth/verify_token.cljs:38 +#: src/app/main/ui/auth/verify_token.cljs:36 msgid "dashboard.notifications.email-changed-successfully" msgstr "Your email address has been updated successfully" -#: src/app/main/ui/auth/verify_token.cljs:32 +#: src/app/main/ui/auth/verify_token.cljs:30 msgid "dashboard.notifications.email-verified-successfully" msgstr "Your email address has been verified successfully" @@ -1013,11 +1013,11 @@ msgstr "Order by role" msgid "dashboard.order-invitations-by-status" msgstr "Order by status" -#: src/app/main/data/dashboard.cljs:703 +#: src/app/main/data/dashboard.cljs:713 msgid "dashboard.organization-deleted" msgstr "The %s organization has been deleted." -#: src/app/main/ui/dashboard/organization_team_switch.cljs:208, src/app/main/ui/dashboard/organization_team_switch.cljs:209 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:224, src/app/main/ui/dashboard/organization_team_switch.cljs:225 msgid "dashboard.other-teams" msgstr "Other teams" @@ -1061,19 +1061,19 @@ msgstr "Cannot parse the plugin manifest" msgid "dashboard.plugins.try-plugin" msgstr "Try plugin: " -#: src/app/main/data/dashboard.cljs:736 +#: src/app/main/data/dashboard.cljs:746 msgid "dashboard.progress-notification.deleting-files" msgstr "Deleting files…" -#: src/app/main/data/dashboard.cljs:857 +#: src/app/main/data/dashboard.cljs:867 msgid "dashboard.progress-notification.restoring-files" msgstr "Restoring files…" -#: src/app/main/data/dashboard.cljs:737 +#: src/app/main/data/dashboard.cljs:747 msgid "dashboard.progress-notification.slow-delete" msgstr "Deletion unexpectedly slow" -#: src/app/main/data/dashboard.cljs:858 +#: src/app/main/data/dashboard.cljs:868 msgid "dashboard.progress-notification.slow-restore" msgstr "Restore unexpectedly slow" @@ -1101,7 +1101,7 @@ msgstr "Restore all projects and files" msgid "dashboard.restore-all-deleted-button" msgstr "Restore All" -#: src/app/main/data/dashboard.cljs:917 +#: src/app/main/data/dashboard.cljs:927 msgid "dashboard.restore-files-success-notification" msgstr "%s files have been successfully restored." @@ -1117,7 +1117,7 @@ msgstr "You're going to restore %s project and all the files contained in it." msgid "dashboard.restore-project-confirmation.title" msgstr "Restore Project" -#: src/app/main/data/dashboard.cljs:889, src/app/main/data/dashboard.cljs:916, src/app/main/data/dashboard.cljs:953, src/app/main/ui/dashboard/file_menu.cljs:224 +#: src/app/main/data/dashboard.cljs:899, src/app/main/data/dashboard.cljs:926, src/app/main/data/dashboard.cljs:963, src/app/main/ui/dashboard/file_menu.cljs:224 msgid "dashboard.restore-success-notification" msgstr "%s has been successfully restored." @@ -1133,19 +1133,19 @@ msgstr "Search…" msgid "dashboard.searching-for" msgstr "Searching for “%s“…" -#: src/app/main/ui/dashboard/organization_team_switch.cljs:180 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:251 msgid "dashboard.section.organizations" msgstr "Organizations" -#: src/app/main/ui/dashboard/organization_team_switch.cljs:249 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:351 msgid "dashboard.section.teams" msgstr "Teams" -#: src/app/main/data/nitrate.cljs:517 +#: src/app/main/data/nitrate.cljs:529 msgid "dashboard.select-organization-modal.accept" msgstr "ADD TO ORGANIZATION" -#: src/app/main/data/nitrate.cljs:515 +#: src/app/main/data/nitrate.cljs:527 msgid "dashboard.select-organization-modal.choose" msgstr "Choose an organization:" @@ -1159,23 +1159,23 @@ msgstr "" msgid "dashboard.select-organization-modal.external-invitations-will-be-canceled" msgstr "Pending invitations to external users will be canceled." -#: src/app/main/data/nitrate.cljs:583 +#: src/app/main/data/nitrate.cljs:595 msgid "dashboard.select-organization-modal.permission-info" msgstr "" "Here you'll find the organizations you are part of where you are allowed to " "move the team." -#: src/app/main/data/nitrate.cljs:507 +#: src/app/main/data/nitrate.cljs:519 msgid "dashboard.select-organization-modal.permission-info-add" msgstr "" "Here you'll find the organizations you are part of where you are allowed to " "create or add teams." -#: src/app/main/data/nitrate.cljs:516 +#: src/app/main/data/nitrate.cljs:528 msgid "dashboard.select-organization-modal.select" msgstr "Select an organization" -#: src/app/main/data/nitrate.cljs:514, src/app/main/data/nitrate.cljs:534, src/app/main/ui/dashboard/team_form.cljs:157 +#: src/app/main/data/nitrate.cljs:526, src/app/main/data/nitrate.cljs:546, src/app/main/ui/dashboard/team_form.cljs:157 msgid "dashboard.select-organization-modal.title" msgstr "Add team to an organization" @@ -1325,7 +1325,7 @@ msgstr "Team organization" msgid "dashboard.team-organization.add" msgstr "Add to an organization" -#: src/app/main/data/nitrate.cljs:533, src/app/main/data/nitrate.cljs:590 +#: src/app/main/data/nitrate.cljs:545, src/app/main/data/nitrate.cljs:602 msgid "dashboard.team-organization.add.no-valid-organizations" msgstr "You don't have permission to add this team to any of your organizations." @@ -1391,7 +1391,7 @@ msgstr "Unpublish Library" msgid "dashboard.update-settings" msgstr "Update settings" -#: src/app/main/data/dashboard.cljs:671 +#: src/app/main/data/dashboard.cljs:681 #, backend msgid "dashboard.user-no-longer-belong-organization" msgstr "You are no longer a member of the organization %s" @@ -1556,17 +1556,17 @@ msgstr "Cannot upload the media file." msgid "errors.character-limit-exceeded" msgstr "Character limit exceeded" -#: src/app/main/data/workspace/libraries.cljs:1691 +#: src/app/main/data/workspace/libraries.cljs:1706 msgid "errors.circular-library-reference" msgstr "Cannot add library: this would create a circular dependency" -#: src/app/main/ui/settings/integrations.cljs:93, src/app/util/clipboard.cljs:248 +#: src/app/main/ui/settings/integrations.cljs:93, src/app/main/ui/workspace/top_toolbar.cljs:305, src/app/util/clipboard.cljs:248 msgid "errors.clipboard-api-unavailable" msgstr "" "Clipboard API is unavailable. Serve Penpot over HTTPS to enable clipboard " "access" -#: src/app/main/data/workspace/clipboard.cljs:514 +#: src/app/main/data/workspace/clipboard.cljs:534 msgid "errors.clipboard-not-implemented" msgstr "Your browser cannot do this operation" @@ -1606,11 +1606,11 @@ msgstr "contact us" msgid "errors.drop-token-set-parent-to-child" msgstr "Cannot drop a parent set to an own child path." -#: src/app/main/ui/auth/register.cljs:111, src/app/main/ui/auth/verify_token.cljs:148, src/app/main/ui/settings/change_email.cljs:29 +#: src/app/main/ui/auth/register.cljs:111, src/app/main/ui/auth/verify_token.cljs:121, src/app/main/ui/settings/change_email.cljs:29 msgid "errors.email-already-exists" msgstr "Email already used" -#: src/app/main/ui/auth/verify_token.cljs:153 +#: src/app/main/ui/auth/verify_token.cljs:126 msgid "errors.email-already-validated" msgstr "Email already validated." @@ -1652,7 +1652,7 @@ msgstr "" msgid "errors.feature-not-supported" msgstr "Feature '%s' is not supported." -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:302, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:234, ../common/src/app/common/files/tokens.cljc:217, ../common/src/app/common/files/tokens.cljc:298, ../common/src/app/common/files/tokens.cljc:339, ../common/src/app/common/schema.cljc:1063 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:370, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:305, ../common/src/app/common/files/tokens.cljc:217, ../common/src/app/common/files/tokens.cljc:298, ../common/src/app/common/files/tokens.cljc:339, ../common/src/app/common/schema.cljc:1063 msgid "errors.field-max-length" msgstr "Must contain at most %s characters." @@ -1781,7 +1781,7 @@ msgstr "There was a problem leaving the organization. Please try again." msgid "errors.organization-leave.organization-owner-cannot-leave" msgstr "The organization owner can't leave the organization." -#: src/app/main/ui/auth/verify_token.cljs:125 +#: src/app/main/ui/auth/verify_token.cljs:98 msgid "errors.organization-not-found" msgstr "That organization doesn't exists" @@ -1881,7 +1881,7 @@ msgstr "The theme refers to some not existing sets: %s" msgid "errors.tokens.circular-reference" msgstr "Token has circular reference" -#: src/app/main/data/workspace/tokens/errors.cljs:116, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:245 +#: src/app/main/data/workspace/tokens/errors.cljs:116, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:316 msgid "errors.tokens.composite-line-height-needs-font-size" msgstr "Line Height depends on Font Size. Add a Font Size to get the resolved value." @@ -1990,7 +1990,7 @@ msgstr "Invalid token value. The resolved value is too large: %s" msgid "errors.tokens.opacity-range" msgstr "Opacity must be between 0 and 100% or 0 and 1 (e.g. 50% or 0.5)." -#: src/app/main/data/workspace/tokens/errors.cljs:46, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:305, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:237, ../common/src/app/common/files/tokens.cljc:235 +#: src/app/main/data/workspace/tokens/errors.cljs:46, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:373, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:308, ../common/src/app/common/files/tokens.cljc:235 msgid "errors.tokens.self-reference" msgstr "Token has self reference" @@ -2022,7 +2022,7 @@ msgstr "Invalid value: Units are not allowed." msgid "errors.unexpected-error" msgstr "An unexpected error occurred." -#: src/app/main/ui/auth/verify_token.cljs:95 +#: src/app/main/ui/auth/verify_token.cljs:68 msgid "errors.unexpected-token" msgstr "Unknown token" @@ -2209,39 +2209,39 @@ msgid_plural "files-export-modal.exporting.title" msgstr[0] "Exporting 1 file" msgstr[1] "Exporting %s files" -#: src/app/main/ui/exports/files.cljs:183 +#: src/app/main/ui/exports/files.cljs:288 msgid "files-export-modal.options.detach-libraries.message" msgstr "Linked library assets won't be included in the file." -#: src/app/main/ui/exports/files.cljs:177 +#: src/app/main/ui/exports/files.cljs:282 msgid "files-export-modal.options.detach-libraries.title" msgstr "Treat assets as basic objects" -#: src/app/main/ui/exports/files.cljs:181 +#: src/app/main/ui/exports/files.cljs:286 msgid "files-export-modal.options.include-libraries.message" msgstr "" "Files with linked libraries will be included in the export, maintaining " "their linkage. " -#: src/app/main/ui/exports/files.cljs:175 +#: src/app/main/ui/exports/files.cljs:280 msgid "files-export-modal.options.include-libraries.title" msgstr "Export file + libraries" -#: src/app/main/ui/exports/files.cljs:184 +#: src/app/main/ui/exports/files.cljs:289 msgid "files-export-modal.options.link-later.message" msgstr "" "When imported, you'll be able to link existing libraries with matching " "names." -#: src/app/main/ui/exports/files.cljs:178 +#: src/app/main/ui/exports/files.cljs:283 msgid "files-export-modal.options.link-later.title" msgstr "Link matching libraries on import" -#: src/app/main/ui/exports/files.cljs:182 +#: src/app/main/ui/exports/files.cljs:287 msgid "files-export-modal.options.merge-libraries.message" msgstr "Your file will be exported with library asset merged into the local library." -#: src/app/main/ui/exports/files.cljs:176 +#: src/app/main/ui/exports/files.cljs:281 msgid "files-export-modal.options.merge-libraries.title" msgstr "Embed library assets in the file" @@ -2802,7 +2802,7 @@ msgstr "Processing the tokens" msgid "jobs.progress.stage.typographies" msgstr "Processing the typographies" -#: src/app/main/ui/jobs/progress.cljs:33 +#: src/app/main/ui/jobs/progress.cljs:34 msgid "jobs.progress.stage.unknown" msgstr "Working" @@ -2810,7 +2810,7 @@ msgstr "Working" msgid "jobs.progress.stage.upload" msgstr "Uploading file…" -#: src/app/main/ui/dashboard/import.cljs:328, src/app/main/ui/exports/files.cljs:103 +#: src/app/main/ui/dashboard/import.cljs:326, src/app/main/ui/exports/files.cljs:117 msgid "jobs.queued" msgstr "Waiting job slot" @@ -2826,7 +2826,7 @@ msgstr "Shortcuts" msgid "labels.about-penpot" msgstr "About Penpot" -#: src/app/main/data/common.cljs:92, src/app/main/data/team.cljs:193, src/app/main/data/team.cljs:650, src/app/main/ui/dashboard/import.cljs:671, src/app/main/ui/dashboard/import.cljs:677 +#: src/app/main/data/common.cljs:92, src/app/main/data/team.cljs:193, src/app/main/data/team.cljs:650, src/app/main/ui/dashboard/import.cljs:644, src/app/main/ui/dashboard/import.cljs:650 msgid "labels.accept" msgstr "Accept" @@ -2862,7 +2862,7 @@ msgstr "All" msgid "labels.and" msgstr "and" -#: src/app/main/ui/dashboard/import.cljs:728 +#: src/app/main/ui/dashboard/import.cljs:701 msgid "labels.back" msgstr "Back" @@ -2884,7 +2884,7 @@ msgstr "Blur" msgid "labels.blur-effects" msgstr "Blur effects" -#: src/app/main/data/common.cljs:121, src/app/main/data/nitrate.cljs:518, src/app/main/data/nitrate.cljs:603, src/app/main/ui/dashboard/change_owner.cljs:67, src/app/main/ui/dashboard/change_owner.cljs:174, src/app/main/ui/dashboard/import.cljs:657, src/app/main/ui/dashboard/team.cljs:902, src/app/main/ui/dashboard/team.cljs:1411, src/app/main/ui/delete_shared.cljs:36, src/app/main/ui/exports/assets.cljs:163, src/app/main/ui/exports/files.cljs:199, src/app/main/ui/settings/import_shortcuts_diff_modal.cljs:156, src/app/main/ui/settings/integrations.cljs:220, src/app/main/ui/settings/restore_shortcuts_modal.cljs:165, src/app/main/ui/shortcuts.cljs:651, src/app/main/ui/viewer/share_link.cljs:203, src/app/main/ui/workspace/sidebar/assets/groups.cljs:178, src/app/main/ui/workspace/tokens/export/modal.cljs:43, src/app/main/ui/workspace/tokens/import/modal.cljs:268, src/app/main/ui/workspace/tokens/import_from_library.cljs:88, src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs:360, src/app/main/ui/workspace/tokens/management/forms/rename_node_modal.cljs:73, src/app/main/ui/workspace/tokens/settings/menu.cljs:104, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:243 +#: src/app/main/data/common.cljs:121, src/app/main/data/nitrate.cljs:530, src/app/main/data/nitrate.cljs:615, src/app/main/ui/dashboard/change_owner.cljs:67, src/app/main/ui/dashboard/change_owner.cljs:174, src/app/main/ui/dashboard/import.cljs:625, src/app/main/ui/dashboard/import.cljs:639, src/app/main/ui/dashboard/team.cljs:902, src/app/main/ui/dashboard/team.cljs:1411, src/app/main/ui/delete_shared.cljs:36, src/app/main/ui/exports/assets.cljs:163, src/app/main/ui/exports/files.cljs:304, src/app/main/ui/exports/files.cljs:324, src/app/main/ui/settings/import_shortcuts_diff_modal.cljs:156, src/app/main/ui/settings/integrations.cljs:220, src/app/main/ui/settings/restore_shortcuts_modal.cljs:165, src/app/main/ui/shortcuts.cljs:651, src/app/main/ui/viewer/share_link.cljs:203, src/app/main/ui/workspace/sidebar/assets/groups.cljs:178, src/app/main/ui/workspace/tokens/export/modal.cljs:43, src/app/main/ui/workspace/tokens/import/modal.cljs:268, src/app/main/ui/workspace/tokens/import_from_library.cljs:88, src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs:360, src/app/main/ui/workspace/tokens/management/forms/rename_node_modal.cljs:73, src/app/main/ui/workspace/tokens/settings/menu.cljs:104, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:243 msgid "labels.cancel" msgstr "Cancel" @@ -2900,7 +2900,7 @@ msgstr "Check for updates" msgid "labels.checking-for-updates" msgstr "Checking for updates..." -#: src/app/main/data/common.cljs:98, src/app/main/ui/alert.cljs:63, src/app/main/ui/confirm.cljs:78, src/app/main/ui/dashboard/check_updates.cljs:217, src/app/main/ui/dashboard/check_updates.cljs:256, src/app/main/ui/dashboard/check_updates.cljs:272, src/app/main/ui/dashboard/check_updates.cljs:311, src/app/main/ui/dashboard/comments.cljs:101, src/app/main/ui/dashboard/import.cljs:997, src/app/main/ui/dashboard/projects.cljs:97, src/app/main/ui/ds/layout/modal.cljs:96, src/app/main/ui/ds/product/panel_title.cljs:32, src/app/main/ui/exports/files.cljs:142, src/app/main/ui/exports/files.cljs:225, src/app/main/ui/nitrate/nitrate_code_activation_modal.cljs:82, src/app/main/ui/settings/import_shortcuts_diff_modal.cljs:93, src/app/main/ui/settings/integrations.cljs:128, src/app/main/ui/settings/integrations.cljs:247, src/app/main/ui/settings/integrations.cljs:287, src/app/main/ui/settings/integrations.cljs:329, src/app/main/ui/settings/restore_shortcuts_modal.cljs:85, src/app/main/ui/settings/subscription.cljs:407, src/app/main/ui/static.cljs:159, src/app/main/ui/static.cljs:229, src/app/main/ui/viewer/login.cljs:91, src/app/main/ui/viewer/share_link.cljs:174, src/app/main/ui/workspace/components_debugger.cljs:1102, src/app/main/ui/workspace/libraries.cljs:1155, src/app/main/ui/workspace/sidebar/layers.cljs:595, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:825, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:846, src/app/main/ui/workspace/sidebar/options/menus/variants_help_modal.cljs:43, src/app/main/ui/workspace/sidebar.cljs:265, src/app/main/ui/workspace/tokens/export.cljs:25, src/app/main/ui/workspace/tokens/import.cljs:19, src/app/main/ui/workspace/tokens/import_from_library.cljs:63, src/app/main/ui/workspace/tokens/management/forms/modals.cljs:99, src/app/main/ui/workspace/tokens/management/forms/rename_node_modal.cljs:114, src/app/main/ui/workspace/tokens/remapping_modal.cljs:93, src/app/main/ui/workspace/tokens/settings/menu.cljs:77, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:62, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:145, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:476, src/app/main/ui/workspace/webgl_unavailable_modal.cljs:56 +#: src/app/main/data/common.cljs:98, src/app/main/ui/alert.cljs:63, src/app/main/ui/confirm.cljs:78, src/app/main/ui/dashboard/check_updates.cljs:217, src/app/main/ui/dashboard/check_updates.cljs:256, src/app/main/ui/dashboard/check_updates.cljs:272, src/app/main/ui/dashboard/check_updates.cljs:311, src/app/main/ui/dashboard/comments.cljs:101, src/app/main/ui/dashboard/import.cljs:999, src/app/main/ui/dashboard/projects.cljs:97, src/app/main/ui/ds/layout/modal.cljs:96, src/app/main/ui/ds/product/panel_title.cljs:32, src/app/main/ui/exports/files.cljs:247, src/app/main/ui/exports/files.cljs:329, src/app/main/ui/nitrate/nitrate_code_activation_modal.cljs:82, src/app/main/ui/settings/import_shortcuts_diff_modal.cljs:93, src/app/main/ui/settings/integrations.cljs:128, src/app/main/ui/settings/integrations.cljs:247, src/app/main/ui/settings/integrations.cljs:287, src/app/main/ui/settings/integrations.cljs:329, src/app/main/ui/settings/restore_shortcuts_modal.cljs:85, src/app/main/ui/settings/subscription.cljs:407, src/app/main/ui/static.cljs:159, src/app/main/ui/static.cljs:229, src/app/main/ui/viewer/login.cljs:91, src/app/main/ui/viewer/share_link.cljs:174, src/app/main/ui/workspace/components_debugger.cljs:1102, src/app/main/ui/workspace/libraries.cljs:1156, src/app/main/ui/workspace/sidebar/layers.cljs:595, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:825, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:846, src/app/main/ui/workspace/sidebar/options/menus/variants_help_modal.cljs:43, src/app/main/ui/workspace/sidebar.cljs:265, src/app/main/ui/workspace/tokens/export.cljs:25, src/app/main/ui/workspace/tokens/import.cljs:19, src/app/main/ui/workspace/tokens/import_from_library.cljs:63, src/app/main/ui/workspace/tokens/management/forms/modals.cljs:99, src/app/main/ui/workspace/tokens/management/forms/rename_node_modal.cljs:114, src/app/main/ui/workspace/tokens/remapping_modal.cljs:93, src/app/main/ui/workspace/tokens/settings/menu.cljs:77, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:62, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:145, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:476, src/app/main/ui/workspace/webgl_unavailable_modal.cljs:56 msgid "labels.close" msgstr "Close" @@ -2948,7 +2948,7 @@ msgstr "Contact support" msgid "labels.contact-us" msgstr "Contact us" -#: src/app/main/ui/auth/login.cljs:208, src/app/main/ui/dashboard/deleted.cljs:46, src/app/main/ui/dashboard/deleted.cljs:281, src/app/main/ui/dashboard/file_menu.cljs:235, src/app/main/ui/dashboard/import.cljs:664, src/app/main/ui/dashboard/team.cljs:909, src/app/main/ui/exports/files.cljs:204, src/app/main/ui/settings/subscription.cljs:333, src/app/main/ui/settings/subscription.cljs:367, src/app/main/ui/settings/subscription.cljs:896 +#: src/app/main/ui/auth/login.cljs:208, src/app/main/ui/dashboard/deleted.cljs:46, src/app/main/ui/dashboard/deleted.cljs:281, src/app/main/ui/dashboard/file_menu.cljs:235, src/app/main/ui/dashboard/import.cljs:632, src/app/main/ui/dashboard/team.cljs:909, src/app/main/ui/exports/files.cljs:309, src/app/main/ui/settings/subscription.cljs:333, src/app/main/ui/settings/subscription.cljs:367, src/app/main/ui/settings/subscription.cljs:896 msgid "labels.continue" msgstr "Continue" @@ -2996,7 +2996,7 @@ msgstr "Dashboard" msgid "labels.default" msgstr "Default" -#: src/app/main/ui/dashboard/file_menu.cljs:335, src/app/main/ui/dashboard/fonts.cljs:286, src/app/main/ui/dashboard/fonts.cljs:363, src/app/main/ui/dashboard/fonts.cljs:397, src/app/main/ui/dashboard/import.cljs:332, src/app/main/ui/dashboard/project_menu.cljs:116, src/app/main/ui/dashboard/team.cljs:1445, src/app/main/ui/settings/integrations.cljs:376, src/app/main/ui/settings/profile.cljs:117, src/app/main/ui/settings/profile.cljs:128, src/app/main/ui/settings/profile.cljs:129, src/app/main/ui/workspace/sidebar/assets/common.cljs:231, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:222, src/app/main/ui/workspace/sidebar/versions.cljs:257, src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs:353, src/app/main/ui/workspace/tokens/management/node_context_menu.cljs:111, src/app/main/ui/workspace/tokens/sets/context_menu.cljs:66, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:394 +#: src/app/main/ui/dashboard/file_menu.cljs:335, src/app/main/ui/dashboard/fonts.cljs:286, src/app/main/ui/dashboard/fonts.cljs:363, src/app/main/ui/dashboard/fonts.cljs:397, src/app/main/ui/dashboard/import.cljs:303, src/app/main/ui/dashboard/project_menu.cljs:116, src/app/main/ui/dashboard/team.cljs:1445, src/app/main/ui/settings/integrations.cljs:376, src/app/main/ui/settings/profile.cljs:117, src/app/main/ui/settings/profile.cljs:128, src/app/main/ui/settings/profile.cljs:129, src/app/main/ui/workspace/sidebar/assets/common.cljs:231, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:222, src/app/main/ui/workspace/sidebar/versions.cljs:257, src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs:353, src/app/main/ui/workspace/tokens/management/node_context_menu.cljs:111, src/app/main/ui/workspace/tokens/sets/context_menu.cljs:66, src/app/main/ui/workspace/tokens/themes/create_modal.cljs:394 msgid "labels.delete" msgstr "Delete" @@ -3052,7 +3052,7 @@ msgstr "Drafts" msgid "labels.duplicate" msgstr "Duplicate" -#: src/app/main/ui/comments.cljs:1043, src/app/main/ui/dashboard/fonts.cljs:280, src/app/main/ui/dashboard/import.cljs:326, src/app/main/ui/dashboard/team.cljs:1443, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:217, src/app/main/ui/workspace/tokens/themes.cljs:52 +#: src/app/main/ui/comments.cljs:1043, src/app/main/ui/dashboard/fonts.cljs:280, src/app/main/ui/dashboard/import.cljs:297, src/app/main/ui/dashboard/team.cljs:1443, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:217, src/app/main/ui/workspace/tokens/themes.cljs:52 msgid "labels.edit" msgstr "Edit" @@ -3064,7 +3064,7 @@ msgstr "Editor" msgid "labels.empty" msgstr "Empty" -#: src/app/main/ui/dashboard/import.cljs:353, src/app/worker/import.cljs:204, src/app/worker/import.cljs:245 +#: src/app/main/data/imports.cljs:56, src/app/main/data/imports.cljs:275, src/app/main/ui/dashboard/import.cljs:322 msgid "labels.error" msgstr "Error" @@ -3208,7 +3208,7 @@ msgstr "Learning Center" msgid "labels.libraries-and-templates" msgstr "Libraries & Templates" -#: src/app/main/ui/auth/verify_token.cljs:164, src/app/main/ui/dashboard/grid.cljs:126, src/app/main/ui/dashboard/grid.cljs:142, src/app/main/ui/dashboard/import.cljs:289, src/app/main/ui/dashboard/placeholder.cljs:139, src/app/main/ui/ds/product/loader.cljs:85, src/app/main/ui/exports/files.cljs:69, src/app/main/ui/nitrate/entry.cljs:26, src/app/main/ui/static.cljs:665, src/app/main/ui/viewer.cljs:641, src/app/main/ui/workspace/sidebar/assets/file_library.cljs:263, src/app/main/ui/workspace.cljs:132 +#: src/app/main/ui/auth/verify_token.cljs:137, src/app/main/ui/dashboard/grid.cljs:126, src/app/main/ui/dashboard/grid.cljs:142, src/app/main/ui/dashboard/import.cljs:260, src/app/main/ui/dashboard/placeholder.cljs:139, src/app/main/ui/ds/product/loader.cljs:85, src/app/main/ui/exports/files.cljs:100, src/app/main/ui/nitrate/entry.cljs:26, src/app/main/ui/static.cljs:665, src/app/main/ui/viewer.cljs:641, src/app/main/ui/workspace/sidebar/assets/file_library.cljs:263, src/app/main/ui/workspace.cljs:132 msgid "labels.loading" msgstr "Loading…" @@ -3324,7 +3324,7 @@ msgstr "Old password" msgid "labels.open" msgstr "Open" -#: src/app/main/ui/comments.cljs:961, src/app/main/ui/comments.cljs:1026, src/app/main/ui/settings/integrations.cljs:400, src/app/main/ui/workspace/sidebar/options/menus/interactions.cljs:398, src/app/main/ui/workspace/sidebar/options/menus/text.cljs:590, src/app/main/ui/workspace/top_toolbar.cljs:92 +#: src/app/main/ui/comments.cljs:961, src/app/main/ui/comments.cljs:1026, src/app/main/ui/settings/integrations.cljs:400, src/app/main/ui/workspace/sidebar/options/menus/interactions.cljs:398, src/app/main/ui/workspace/sidebar/options/menus/text.cljs:590, src/app/main/ui/workspace/top_toolbar.cljs:94 msgid "labels.options" msgstr "Options" @@ -3364,7 +3364,7 @@ msgstr "Pinned Projects" msgid "labels.post" msgstr "Post" -#: src/app/main/ui/dashboard/import.cljs:698, src/app/main/ui/onboarding/questions.cljs:48, src/app/main/ui/viewer.cljs:105, src/app/main/ui/workspace/sidebar/layers.cljs:612 +#: src/app/main/ui/dashboard/import.cljs:671, src/app/main/ui/onboarding/questions.cljs:48, src/app/main/ui/viewer.cljs:105, src/app/main/ui/workspace/sidebar/layers.cljs:612 msgid "labels.previous" msgstr "Previous" @@ -3388,7 +3388,7 @@ msgstr "Projects" msgid "labels.recent" msgstr "Recent" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:209, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:179 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:243, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:216 msgid "labels.reference" msgstr "Reference" @@ -3500,7 +3500,7 @@ msgstr "Sets" msgid "labels.settings" msgstr "Settings" -#: src/app/main/ui/inspect/styles/style_box.cljs:34, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:233 +#: src/app/main/ui/inspect/styles/style_box.cljs:34, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:300 msgid "labels.shadow" msgstr "Shadow" @@ -3536,7 +3536,7 @@ msgstr "Show only your comments" msgid "labels.sketch" msgstr "Sketch" -#: src/app/main/ui/dashboard/import.cljs:703 +#: src/app/main/ui/dashboard/import.cljs:676 msgid "labels.skip" msgstr "Skip" @@ -3583,11 +3583,11 @@ msgstr "Styles" msgid "labels.svg" msgstr "SVG" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:570 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:593 msgid "labels.switch" msgstr "Switch" -#: src/app/main/ui/dashboard/organization_team_switch.cljs:562 +#: src/app/main/ui/dashboard/organization_team_switch.cljs:645 msgid "labels.team-management" msgstr "Team management" @@ -3603,7 +3603,7 @@ msgstr "Themes" msgid "labels.tutorials" msgstr "Tutorials" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:259 msgid "labels.typography" msgstr "Typography" @@ -4662,7 +4662,7 @@ msgstr "Maintenance break: we will be down for a short maintenance within 5 minu msgid "notifications.by-code.upgrade-version" msgstr "A new version is available." -#: src/app/main/ui/auth/verify_token.cljs:133, src/app/main/ui/auth/verify_token.cljs:136 +#: src/app/main/ui/auth/verify_token.cljs:106, src/app/main/ui/auth/verify_token.cljs:109 msgid "notifications.invitation-canceled" msgstr "This invitation is no longer available." @@ -4909,7 +4909,7 @@ msgstr "Reset" msgid "settings.detach" msgstr "Detach" -#: src/app/main/ui/ds/controls/numeric_input.cljs:742, src/app/main/ui/inspect/exports.cljs:170, src/app/main/ui/workspace/sidebar/options/menus/blur.cljs:299, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:192, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:222, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:243, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:261, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:278, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:297, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:299, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:345, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:502, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:1057, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:1297, src/app/main/ui/workspace/sidebar/options/menus/constraints.cljs:163, src/app/main/ui/workspace/sidebar/options/menus/constraints.cljs:174, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:225, src/app/main/ui/workspace/sidebar/options/menus/fill.cljs:223, src/app/main/ui/workspace/sidebar/options/menus/input_wrapper_tokens.cljs:27, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:244, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:388, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:399, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:425, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:435, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:519, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:553, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:586, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:620, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:755, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:793, src/app/main/ui/workspace/sidebar/options/menus/layout_item.cljs:80, src/app/main/ui/workspace/sidebar/options/menus/layout_item.cljs:86, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:189, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:197, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:572, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:595, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:606, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:634, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:647, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:656, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:667, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:688, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:699, src/app/main/ui/workspace/sidebar/options/menus/shadow.cljs:165, src/app/main/ui/workspace/sidebar/options/menus/stroke.cljs:280, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:622, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:679, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:698, src/app/main/ui/workspace/sidebar/options/rows/color_row.cljs:474, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:410, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:451 +#: src/app/main/ui/ds/controls/numeric_input.cljs:742, src/app/main/ui/inspect/exports.cljs:170, src/app/main/ui/workspace/sidebar/options/menus/blur.cljs:299, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:192, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:222, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:243, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:261, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:278, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:297, src/app/main/ui/workspace/sidebar/options/menus/border_radius.cljs:299, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:345, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:502, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:1057, src/app/main/ui/workspace/sidebar/options/menus/component.cljs:1297, src/app/main/ui/workspace/sidebar/options/menus/constraints.cljs:163, src/app/main/ui/workspace/sidebar/options/menus/constraints.cljs:174, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:225, src/app/main/ui/workspace/sidebar/options/menus/fill.cljs:223, src/app/main/ui/workspace/sidebar/options/menus/input_wrapper_tokens.cljs:27, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:245, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:388, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:399, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:425, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:435, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:519, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:553, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:586, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:620, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:755, src/app/main/ui/workspace/sidebar/options/menus/layout_container.cljs:793, src/app/main/ui/workspace/sidebar/options/menus/layout_item.cljs:80, src/app/main/ui/workspace/sidebar/options/menus/layout_item.cljs:86, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:189, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:197, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:572, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:595, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:606, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:634, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:647, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:656, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:667, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:688, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:699, src/app/main/ui/workspace/sidebar/options/menus/shadow.cljs:165, src/app/main/ui/workspace/sidebar/options/menus/stroke.cljs:280, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:622, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:679, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:698, src/app/main/ui/workspace/sidebar/options/rows/color_row.cljs:474, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:435, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:474 msgid "settings.multiple" msgstr "Mixed" @@ -6140,7 +6140,7 @@ msgstr "Search - %s - Penpot" msgid "title.dashboard.shared-libraries" msgstr "Shared Libraries - %s - Penpot" -#: src/app/main/ui/auth/verify_token.cljs:110, src/app/main/ui/auth.cljs:40 +#: src/app/main/ui/auth/verify_token.cljs:83, src/app/main/ui/auth.cljs:40 msgid "title.default" msgstr "Penpot | Full-stack design" @@ -6348,7 +6348,7 @@ msgstr "Add library" msgid "workspace.assets.box-filter-all" msgstr "All assets" -#: src/app/main/ui/dashboard/grid.cljs:156, src/app/main/ui/dashboard/grid.cljs:189, src/app/main/ui/workspace/colorpicker/libraries.cljs:364, src/app/main/ui/workspace/colorpicker/libraries.cljs:381, src/app/main/ui/workspace/colorpicker/libraries.cljs:395, src/app/main/ui/workspace/sidebar/assets/colors.cljs:512, src/app/main/ui/workspace/sidebar/assets.cljs:185 +#: src/app/main/ui/dashboard/grid.cljs:156, src/app/main/ui/dashboard/grid.cljs:189, src/app/main/ui/workspace/colorpicker/libraries.cljs:360, src/app/main/ui/workspace/colorpicker/libraries.cljs:377, src/app/main/ui/workspace/colorpicker/libraries.cljs:391, src/app/main/ui/workspace/sidebar/assets/colors.cljs:512, src/app/main/ui/workspace/sidebar/assets.cljs:185 msgid "workspace.assets.colors" msgstr "Colors" @@ -6380,7 +6380,7 @@ msgstr "Create a group" msgid "workspace.assets.create-group-hint" msgstr "Your items are going to be named automatically as \"group name / item name\"" -#: src/app/main/ui/workspace/context_menu.cljs:828, src/app/main/ui/workspace/sidebar/assets/colors.cljs:265, src/app/main/ui/workspace/sidebar/assets/components.cljs:670, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:476, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:821 +#: src/app/main/ui/workspace/context_menu.cljs:828, src/app/main/ui/workspace/sidebar/assets/colors.cljs:265, src/app/main/ui/workspace/sidebar/assets/components.cljs:670, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:529, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:821 msgid "workspace.assets.delete" msgstr "Delete" @@ -6392,7 +6392,7 @@ msgstr "Delete group" msgid "workspace.assets.delete-pages" msgstr "Delete pages" -#: src/app/main/ui/workspace/context_menu.cljs:833, src/app/main/ui/workspace/sidebar/assets/colors.cljs:262, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:472, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:817 +#: src/app/main/ui/workspace/context_menu.cljs:833, src/app/main/ui/workspace/sidebar/assets/colors.cljs:262, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:525, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:817 msgid "workspace.assets.duplicate" msgstr "Duplicate" @@ -6400,7 +6400,7 @@ msgstr "Duplicate" msgid "workspace.assets.duplicate-main" msgstr "Duplicate main" -#: src/app/main/ui/workspace/sidebar/assets/colors.cljs:258, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:467 +#: src/app/main/ui/workspace/sidebar/assets/colors.cljs:258, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:520 msgid "workspace.assets.edit" msgstr "Edit" @@ -6408,11 +6408,11 @@ msgstr "Edit" msgid "workspace.assets.filter" msgstr "Filter" -#: src/app/main/ui/workspace/colorpicker/color_tokens.cljs:194, src/app/main/ui/workspace/colorpicker/libraries.cljs:347, src/app/main/ui/workspace/sidebar/assets/components.cljs:604 +#: src/app/main/ui/workspace/colorpicker/color_tokens.cljs:194, src/app/main/ui/workspace/colorpicker/libraries.cljs:343, src/app/main/ui/workspace/sidebar/assets/components.cljs:604 msgid "workspace.assets.grid-view" msgstr "Grid view" -#: src/app/main/ui/workspace/sidebar/assets/colors.cljs:269, src/app/main/ui/workspace/sidebar/assets/components.cljs:654, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:481 +#: src/app/main/ui/workspace/sidebar/assets/colors.cljs:269, src/app/main/ui/workspace/sidebar/assets/components.cljs:654, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:534 msgid "workspace.assets.group" msgstr "Group" @@ -6420,7 +6420,7 @@ msgstr "Group" msgid "workspace.assets.group-name" msgstr "Group name" -#: src/app/main/ui/workspace/colorpicker/color_tokens.cljs:195, src/app/main/ui/workspace/colorpicker/libraries.cljs:346, src/app/main/ui/workspace/sidebar/assets/components.cljs:600 +#: src/app/main/ui/workspace/colorpicker/color_tokens.cljs:195, src/app/main/ui/workspace/colorpicker/libraries.cljs:342, src/app/main/ui/workspace/sidebar/assets/components.cljs:600 msgid "workspace.assets.list-view" msgstr "List view" @@ -6440,7 +6440,7 @@ msgstr "No assets found" msgid "workspace.assets.open-library" msgstr "Open library file" -#: src/app/main/ui/workspace/context_menu.cljs:831, src/app/main/ui/workspace/sidebar/assets/colors.cljs:254, src/app/main/ui/workspace/sidebar/assets/components.cljs:645, src/app/main/ui/workspace/sidebar/assets/groups.cljs:66, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:462 +#: src/app/main/ui/workspace/context_menu.cljs:831, src/app/main/ui/workspace/sidebar/assets/colors.cljs:254, src/app/main/ui/workspace/sidebar/assets/components.cljs:645, src/app/main/ui/workspace/sidebar/assets/groups.cljs:66, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:515 msgid "workspace.assets.rename" msgstr "Rename" @@ -6466,11 +6466,11 @@ msgstr[1] "%s components" msgid "workspace.assets.sort" msgstr "Sort" -#: src/app/main/ui/dashboard/grid.cljs:160, src/app/main/ui/dashboard/grid.cljs:216, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:423, src/app/main/ui/workspace/sidebar/assets.cljs:188 +#: src/app/main/ui/dashboard/grid.cljs:160, src/app/main/ui/dashboard/grid.cljs:216, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:479, src/app/main/ui/workspace/sidebar/assets.cljs:188 msgid "workspace.assets.typography" msgstr "Typographies" -#: src/app/main/ui/workspace/sidebar/assets/groups.cljs:87, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:431 +#: src/app/main/ui/workspace/sidebar/assets/groups.cljs:87, src/app/main/ui/workspace/sidebar/assets/typographies.cljs:304 msgid "workspace.assets.typography.add-typography" msgstr "Add typography" @@ -6494,11 +6494,11 @@ msgstr "Letter Spacing" msgid "workspace.assets.typography.line-height" msgstr "Line Height" -#: src/app/main/ui/workspace/sidebar/assets/typographies.cljs:461 +#: src/app/main/ui/workspace/sidebar/assets/typographies.cljs:297 msgid "workspace.assets.typography.multiple-texts-selected" msgstr "Select one text to create a typography style" -#: src/app/main/ui/dashboard/grid.cljs:226, src/app/main/ui/workspace/libraries.cljs:1070, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:802, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:838, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:957, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:976 +#: src/app/main/ui/dashboard/grid.cljs:226, src/app/main/ui/workspace/libraries.cljs:1071, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:802, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:838, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:957, src/app/main/ui/workspace/sidebar/options/menus/typography.cljs:976 msgid "workspace.assets.typography.sample" msgstr "Ag" @@ -6510,15 +6510,15 @@ msgstr "Text Transform" msgid "workspace.assets.ungroup" msgstr "Ungroup" -#: src/app/main/data/workspace/clipboard.cljs:1170 +#: src/app/main/data/workspace/clipboard.cljs:1206 msgid "workspace.clipboard.copying" msgstr "Copying image…" -#: src/app/main/data/workspace/clipboard.cljs:1192 +#: src/app/main/data/workspace/clipboard.cljs:1228 msgid "workspace.clipboard.image-copied" msgstr "Image copied to the clipboard" -#: src/app/main/data/workspace/clipboard.cljs:1198 +#: src/app/main/data/workspace/clipboard.cljs:1234 msgid "workspace.clipboard.image-copy-failed" msgstr "Error copying image" @@ -6728,11 +6728,11 @@ msgstr "Hide fonts palette" msgid "workspace.header.menu.lock-guides" msgstr "Lock guides" -#: src/app/main/ui/workspace/main_menu.cljs:874 +#: src/app/main/ui/workspace/main_menu.cljs:874, src/app/main/ui/workspace/top_toolbar.cljs:410 msgid "workspace.header.menu.mcp.plugin.status.connect" msgstr "Connect" -#: src/app/main/ui/workspace/main_menu.cljs:873 +#: src/app/main/ui/workspace/main_menu.cljs:873, src/app/main/ui/workspace/top_toolbar.cljs:409 msgid "workspace.header.menu.mcp.plugin.status.disconnect" msgstr "Disconnect" @@ -6962,15 +6962,15 @@ msgid_plural "workspace.libraries.colors" msgstr[0] "1 color" msgstr[1] "%s colors" -#: src/app/main/ui/workspace/colorpicker/libraries.cljs:356 +#: src/app/main/ui/workspace/colorpicker/libraries.cljs:352 msgid "workspace.libraries.colors.add-library-color" msgstr "Add color to library" -#: src/app/main/ui/workspace/color_palette.cljs:203, src/app/main/ui/workspace/colorpicker/libraries.cljs:374, src/app/main/ui/workspace/colorpicker/libraries.cljs:405 +#: src/app/main/ui/workspace/color_palette.cljs:203, src/app/main/ui/workspace/colorpicker/libraries.cljs:370, src/app/main/ui/workspace/colorpicker/libraries.cljs:401 msgid "workspace.libraries.colors.empty-palette" msgstr "There are no color styles in your library yet" -#: src/app/main/ui/workspace/colorpicker/libraries.cljs:373, src/app/main/ui/workspace/colorpicker/libraries.cljs:389 +#: src/app/main/ui/workspace/colorpicker/libraries.cljs:369, src/app/main/ui/workspace/colorpicker/libraries.cljs:385 msgid "workspace.libraries.colors.empty-recent-colors" msgstr "There are no recent colors yet" @@ -6978,11 +6978,11 @@ msgstr "There are no recent colors yet" msgid "workspace.libraries.colors.empty-typography-palette" msgstr "There are no typography styles in your library yet" -#: src/app/main/ui/workspace/color_palette_ctx_menu.cljs:97, src/app/main/ui/workspace/colorpicker/libraries.cljs:222, src/app/main/ui/workspace/text_palette_ctx_menu.cljs:49 +#: src/app/main/ui/workspace/color_palette_ctx_menu.cljs:97, src/app/main/ui/workspace/colorpicker/libraries.cljs:216, src/app/main/ui/workspace/text_palette_ctx_menu.cljs:49 msgid "workspace.libraries.colors.file-library" msgstr "File library" -#: src/app/main/ui/workspace/color_palette_ctx_menu.cljs:120, src/app/main/ui/workspace/colorpicker/libraries.cljs:221 +#: src/app/main/ui/workspace/color_palette_ctx_menu.cljs:120, src/app/main/ui/workspace/colorpicker/libraries.cljs:215 msgid "workspace.libraries.colors.recent-colors" msgstr "Recent colors" @@ -6994,7 +6994,7 @@ msgstr "RGBA" msgid "workspace.libraries.colors.save-color" msgstr "Save color style" -#: src/app/main/ui/workspace/colorpicker/libraries.cljs:338 +#: src/app/main/ui/workspace/colorpicker/libraries.cljs:334 msgid "workspace.libraries.colors.toggle-color-palette" msgstr "Toggle color palette" @@ -7050,7 +7050,7 @@ msgstr[1] "%s graphics" msgid "workspace.libraries.in-this-file" msgstr "LIBRARIES IN THIS FILE" -#: src/app/main/ui/workspace/libraries.cljs:1137, src/app/main/ui/workspace/libraries.cljs:1141, src/app/main/ui/workspace/libraries.cljs:1159 +#: src/app/main/ui/workspace/libraries.cljs:1138, src/app/main/ui/workspace/libraries.cljs:1142, src/app/main/ui/workspace/libraries.cljs:1160 msgid "workspace.libraries.libraries" msgstr "LIBRARIES" @@ -7150,11 +7150,11 @@ msgstr "Disconnect library" msgid "workspace.libraries.update" msgstr "Update" -#: src/app/main/ui/workspace/libraries.cljs:1087 +#: src/app/main/ui/workspace/libraries.cljs:1088 msgid "workspace.libraries.update.see-all-changes" msgstr "see all changes" -#: src/app/main/ui/workspace/libraries.cljs:1139, src/app/main/ui/workspace/libraries.cljs:1143 +#: src/app/main/ui/workspace/libraries.cljs:1140, src/app/main/ui/workspace/libraries.cljs:1144 msgid "workspace.libraries.updates" msgstr "UPDATES" @@ -7222,7 +7222,7 @@ msgstr "Toggle blur" msgid "workspace.options.blur-options.toggle-more-options" msgstr "Show/hide more options" -#: src/app/main/ui/exports/assets.cljs:332 +#: src/app/main/ui/exports/assets.cljs:340 msgid "workspace.options.cancel-export" msgstr "Cancel" @@ -7428,7 +7428,7 @@ msgstr "Top & Bottom" msgid "workspace.options.convert-to-typography" msgstr "Create typography style" -#: src/app/main/ui/workspace/sidebar/options.cljs:238 +#: src/app/main/ui/workspace/sidebar/options.cljs:240 msgid "workspace.options.design" msgstr "Design" @@ -7458,35 +7458,43 @@ msgstr "Remove export" msgid "workspace.options.export.suffix" msgstr "Suffix" -#: src/app/main/ui/exports/assets.cljs:267 +#: src/app/main/ui/exports/assets.cljs:274 msgid "workspace.options.exporting-busy" msgstr "Export service is busy, please try again later" -#: src/app/main/ui/exports/assets.cljs:270 +#: src/app/main/ui/exports/assets.cljs:277 msgid "workspace.options.exporting-cancelled" msgstr "Export cancelled" -#: src/app/main/ui/exports/assets.cljs:269 +#: src/app/main/ui/exports/assets.cljs:276 msgid "workspace.options.exporting-cancelling" msgstr "Cancelling..." -#: src/app/main/ui/exports/assets.cljs:272 +#: src/app/main/ui/exports/assets.cljs:279 msgid "workspace.options.exporting-complete" msgstr "Export complete" -#: src/app/main/ui/exports/assets.cljs:171, src/app/main/ui/exports/assets.cljs:273, src/app/main/ui/inspect/exports.cljs:217, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:276 +#: src/app/main/ui/exports/assets.cljs:171, src/app/main/ui/exports/assets.cljs:281, src/app/main/ui/inspect/exports.cljs:217, src/app/main/ui/workspace/sidebar/options/menus/exports.cljs:276 msgid "workspace.options.exporting-object" msgstr "Exporting…" -#: src/app/main/ui/exports/assets.cljs:268 +#: src/app/main/ui/exports/assets.cljs:275 msgid "workspace.options.exporting-object-error" msgstr "Export failed" -#: src/app/main/ui/exports/assets.cljs:274 +#: src/app/main/ui/exports/assets.cljs:282 msgid "workspace.options.exporting-object-slow" msgstr "Export unexpectedly slow" -#: src/app/main/ui/exports/assets.cljs:271 +#: src/app/main/ui/exports/assets.cljs:236 +msgid "workspace.options.exporting-packaging" +msgstr "Packing files…" + +#: src/app/main/ui/exports/assets.cljs:235 +msgid "workspace.options.exporting-preparing" +msgstr "Preparing…" + +#: src/app/main/ui/exports/assets.cljs:278 msgid "workspace.options.exporting-queued" msgstr "Waiting..." @@ -7526,7 +7534,7 @@ msgstr "Flow start" msgid "workspace.options.flows.flow-starts" msgstr "Flow starts" -#: src/app/main/ui/workspace/sidebar/options/menus/text.cljs:526 +#: src/app/main/ui/workspace/sidebar/assets/typographies.cljs:300, src/app/main/ui/workspace/sidebar/options/menus/text.cljs:526 msgid "workspace.options.font-not-available" msgstr "" "The font used by this text (%s) is no longer available. Reupload the font " @@ -7632,7 +7640,7 @@ msgstr "Toggle guide" msgid "workspace.options.height" msgstr "Height" -#: src/app/main/ui/workspace/sidebar/options.cljs:242 +#: src/app/main/ui/workspace/sidebar/options.cljs:244 msgid "workspace.options.inspect" msgstr "Inspect" @@ -7848,75 +7856,75 @@ msgstr "Interactions" msgid "workspace.options.interactions.add-interaction" msgstr "Add interaction" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:203 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:204 msgid "workspace.options.layer-options.blend-mode.color" msgstr "Color" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:192 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:193 msgid "workspace.options.layer-options.blend-mode.color-burn" msgstr "Color burn" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:195 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:196 msgid "workspace.options.layer-options.blend-mode.color-dodge" msgstr "Color dodge" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:190 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:191 msgid "workspace.options.layer-options.blend-mode.darken" msgstr "Darken" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:199 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:200 msgid "workspace.options.layer-options.blend-mode.difference" msgstr "Difference" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:200 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:201 msgid "workspace.options.layer-options.blend-mode.exclusion" msgstr "Exclusion" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:198 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:199 msgid "workspace.options.layer-options.blend-mode.hard-light" msgstr "Hard light" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:201 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:202 msgid "workspace.options.layer-options.blend-mode.hue" msgstr "Hue" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:193 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:194 msgid "workspace.options.layer-options.blend-mode.lighten" msgstr "Lighten" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:204 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:205 msgid "workspace.options.layer-options.blend-mode.luminosity" msgstr "Luminosity" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:191 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:192 msgid "workspace.options.layer-options.blend-mode.multiply" msgstr "Multiply" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:189 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:190 msgid "workspace.options.layer-options.blend-mode.normal" msgstr "Normal" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:196 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:197 msgid "workspace.options.layer-options.blend-mode.overlay" msgstr "Overlay" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:202 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:203 msgid "workspace.options.layer-options.blend-mode.saturation" msgstr "Saturation" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:194 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:195 msgid "workspace.options.layer-options.blend-mode.screen" msgstr "Screen" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:197 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:198 msgid "workspace.options.layer-options.blend-mode.soft-light" msgstr "Soft light" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:223, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:285 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:224, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:286 msgid "workspace.options.layer-options.layer-section" msgstr "Layer menu section" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:256, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:263, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:311, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:317 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:257, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:264, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:312, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:318 msgid "workspace.options.layer-options.toggle-layer" msgstr "Toggle layer visibility" @@ -7936,15 +7944,15 @@ msgstr "Min height" msgid "workspace.options.layout-item.layout-item-min-w" msgstr "Min width" -#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:245 +#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:270 msgid "workspace.options.more-colors" msgstr "More colors" -#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:227 +#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:252 msgid "workspace.options.more-lib-colors" msgstr "More library colors" -#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:268 +#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:293 msgid "workspace.options.more-token-colors" msgstr "More color tokens" @@ -7956,7 +7964,7 @@ msgstr "No matching fonts" msgid "workspace.options.no-size-preset-results" msgstr "No matching size preset" -#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:240, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:297 +#: src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:241, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:298 msgid "workspace.options.opacity" msgstr "Opacity" @@ -7964,7 +7972,7 @@ msgstr "Opacity" msgid "workspace.options.pixel-grid-color" msgstr "Pixel grid color" -#: src/app/main/ui/workspace/sidebar/options.cljs:240 +#: src/app/main/ui/workspace/sidebar/options.cljs:242 msgid "workspace.options.prototype" msgstr "Prototype" @@ -8004,7 +8012,7 @@ msgstr "Show independent radius" msgid "workspace.options.recent-fonts" msgstr "Recent" -#: src/app/main/ui/exports/assets.cljs:326 +#: src/app/main/ui/exports/assets.cljs:334 msgid "workspace.options.retry" msgstr "Retry" @@ -8024,11 +8032,11 @@ msgstr "Search size preset" msgid "workspace.options.select-a-shape" msgstr "Select a shape, board or group to drag a connection to other board." -#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:205 +#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:230 msgid "workspace.options.selection-color" msgstr "Selected colors" -#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:200 +#: src/app/main/ui/workspace/sidebar/options/menus/color_selection.cljs:225 msgid "workspace.options.selection-color.section" msgstr "Color selection section" @@ -8116,35 +8124,35 @@ msgstr "Lock ratio" msgid "workspace.options.size.unlock" msgstr "Unlock ratio" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:334 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:359 msgid "workspace.options.stroke-cap.circle-marker-short" msgstr "Circle" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:335 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:360 msgid "workspace.options.stroke-cap.diamond-marker-short" msgstr "Diamond" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:331 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:356 msgid "workspace.options.stroke-cap.line-arrow-short" msgstr "Arrow" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:329 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:354 msgid "workspace.options.stroke-cap.none" msgstr "None" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:337 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:362 msgid "workspace.options.stroke-cap.round" msgstr "Round" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:338 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:363 msgid "workspace.options.stroke-cap.square" msgstr "Square" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:333 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:358 msgid "workspace.options.stroke-cap.square-marker-short" msgstr "Rectangle" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:332 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:357 msgid "workspace.options.stroke-cap.triangle-arrow-short" msgstr "Triangle" @@ -8152,31 +8160,31 @@ msgstr "Triangle" msgid "workspace.options.stroke-color" msgstr "Stroke color" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:548 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:571 msgid "workspace.options.stroke-dash" msgstr "Dash" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:558 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:581 msgid "workspace.options.stroke-gap" msgstr "Gap" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:405, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:443 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:430, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:466 msgid "workspace.options.stroke-width" msgstr "Stroke width" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:518 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:541 msgid "workspace.options.stroke-width-bottom" msgstr "Bottom (block end)" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:531 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:554 msgid "workspace.options.stroke-width-left" msgstr "Left (inline start)" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:505 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:528 msgid "workspace.options.stroke-width-right" msgstr "Right (inline end)" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:492 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:515 msgid "workspace.options.stroke-width-top" msgstr "Top (block start)" @@ -8184,49 +8192,49 @@ msgstr "Top (block start)" msgid "workspace.options.stroke.add-stroke" msgstr "Add stroke color" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:238 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:263 msgid "workspace.options.stroke.center" msgstr "Center" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:266 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:291 msgid "workspace.options.stroke.dashed" msgstr "Dashed" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:265 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:290 msgid "workspace.options.stroke.dotted" msgstr "Dotted" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:239 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:264 msgid "workspace.options.stroke.inner" msgstr "Inside" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:267 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:292 msgid "workspace.options.stroke.mixed" msgstr "Mixed" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:240 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:265 msgid "workspace.options.stroke.outer" msgstr "Outside" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:150 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:175 msgid "workspace.options.stroke.per-side" msgstr "Stroke per side" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:149 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:174 msgid "workspace.options.stroke.per-side-disabled" msgstr "" "Strokes per side feature is only available in the new render. Switch render " "to use it." -#: src/app/main/ui/workspace/sidebar/options/menus/stroke.cljs:282, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:389 +#: src/app/main/ui/workspace/sidebar/options/menus/stroke.cljs:282, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:414 msgid "workspace.options.stroke.remove-stroke" msgstr "Remove stroke" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:264 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:289 msgid "workspace.options.stroke.solid" msgstr "Solid" -#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:384 +#: src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:409 msgid "workspace.options.stroke.toggle-stroke" msgstr "Toggle stroke" @@ -8718,7 +8726,7 @@ msgstr "Show / Hide UI" msgid "workspace.shape.menu.intersection" msgstr "Intersection" -#: src/app/main/ui/workspace/context_menu.cljs:581, src/app/main/ui/workspace/sidebar/layer_item.cljs:198, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:271, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:324 +#: src/app/main/ui/workspace/context_menu.cljs:581, src/app/main/ui/workspace/sidebar/layer_item.cljs:198, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:272, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:325 msgid "workspace.shape.menu.lock" msgstr "Lock" @@ -8810,7 +8818,7 @@ msgstr "Ungroup" msgid "workspace.shape.menu.union" msgstr "Union" -#: src/app/main/ui/workspace/context_menu.cljs:577, src/app/main/ui/workspace/sidebar/layer_item.cljs:197, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:278, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:330 +#: src/app/main/ui/workspace/context_menu.cljs:577, src/app/main/ui/workspace/sidebar/layer_item.cljs:197, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:279, src/app/main/ui/workspace/sidebar/options/menus/layer.cljs:331 msgid "workspace.shape.menu.unlock" msgstr "Unlock" @@ -8974,7 +8982,7 @@ msgstr "Base font size must be a value in pixels or unitless." msgid "workspace.tokens.change-token-source" msgstr "Change token source" -#: src/app/main/ui/workspace/tokens/management/context_menu.cljs:369, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:132 +#: src/app/main/ui/workspace/tokens/management/context_menu.cljs:369, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:152 msgid "workspace.tokens.color" msgstr "Color" @@ -9070,7 +9078,7 @@ msgstr "Preview:" msgid "workspace.tokens.export.single-file" msgstr "Single file" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:152 msgid "workspace.tokens.font-size-value-enter" msgstr "Font size or {alias}" @@ -9080,7 +9088,7 @@ msgstr "" "Error setting font weight/style. This font style does not exist in the " "current font" -#: src/app/main/ui/workspace/tokens/management/forms/form_container.cljs:60, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:137 +#: src/app/main/ui/workspace/tokens/management/forms/form_container.cljs:60, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:160 msgid "workspace.tokens.font-weight-value-enter" msgstr "Font weight (300, Bold Italic...) or an {alias}" @@ -9134,7 +9142,7 @@ msgstr "" "This set is not active. Change theme or activate this set to see changes in " "the viewport" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:245, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:196 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:312, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:266 msgid "workspace.tokens.individual-tokens" msgstr "Use individual tokens" @@ -9154,11 +9162,11 @@ msgstr "Theme" msgid "workspace.tokens.label.theme-placeholder" msgstr "Add a theme (i.e. Light)" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:176 msgid "workspace.tokens.letter-spacing-value-enter-composite" msgstr "Letter spacing or {alias}" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:168 msgid "workspace.tokens.line-height-value-enter" msgstr "Line height (multiplier, px, %) or {alias}" @@ -9226,7 +9234,7 @@ msgstr "Tokens source sets or themes have been modified" msgid "workspace.tokens.num-active-sets" msgstr "%s active sets" -#: src/app/main/ui/workspace/colorpicker.cljs:438, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:67 +#: src/app/main/ui/workspace/colorpicker.cljs:438, src/app/main/ui/workspace/sidebar/options/rows/stroke_row.cljs:112 msgid "workspace.tokens.only-first-fill-or-stroke" msgstr "Design tokens can only be applied to the first fill or stroke in the list." @@ -9250,11 +9258,11 @@ msgstr "Radius" msgid "workspace.tokens.ref-not-valid" msgstr "Reference in {%s} is not valid or is not in any active set." -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:215 msgid "workspace.tokens.reference-composite" msgstr "Enter a token typography alias" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:208 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:242 msgid "workspace.tokens.reference-composite-shadow" msgstr "Enter a token shadow alias" @@ -9326,27 +9334,27 @@ msgstr "Here you can configure the base font size, which defines the value of 1r msgid "workspace.tokens.settings" msgstr "Tokens settings" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:236 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:303 msgid "workspace.tokens.shadow-add-shadow" msgstr "Add Shadow" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:163, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:164, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:168 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:183, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:184, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:188 msgid "workspace.tokens.shadow-blur" msgstr "Blur" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:116 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:136 msgid "workspace.tokens.shadow-inset" msgstr "Inset" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:125 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:145 msgid "workspace.tokens.shadow-remove-shadow" msgstr "Remove Shadow" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:176, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:177, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:181 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:196, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:197, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:201 msgid "workspace.tokens.shadow-spread" msgstr "Spread" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:287, ../common/src/app/common/files/tokens.cljc:112 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:355, ../common/src/app/common/files/tokens.cljc:112 msgid "workspace.tokens.shadow-token-blur-value-error" msgstr "Blur value cannot be negative" @@ -9354,11 +9362,11 @@ msgstr "Blur value cannot be negative" msgid "workspace.tokens.shadow-token-spread-value-error" msgstr "Spread value cannot be negative" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:141, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:143 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:161, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:163 msgid "workspace.tokens.shadow-x" msgstr "X" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:152, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:154 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:172, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:174 msgid "workspace.tokens.shadow-y" msgstr "Y" @@ -9374,11 +9382,11 @@ msgstr "Source:" msgid "workspace.tokens.stroke-width" msgstr "Stroke Width" -#: src/app/main/ui/workspace/tokens/management/forms/form_container.cljs:54, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:161 +#: src/app/main/ui/workspace/tokens/management/forms/form_container.cljs:54, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:184 msgid "workspace.tokens.text-case-value-enter" msgstr "none | uppercase | lowercase | capitalize or {alias}" -#: src/app/main/ui/workspace/tokens/management/forms/form_container.cljs:57, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:169 +#: src/app/main/ui/workspace/tokens/management/forms/form_container.cljs:57, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:192 msgid "workspace.tokens.text-decoration-value-enter" msgstr "none | underline | strike-through or {alias}" @@ -9404,11 +9412,11 @@ msgstr "Description" msgid "workspace.tokens.token-font-family-select" msgstr "Select font family" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:144 msgid "workspace.tokens.token-font-family-value" msgstr "Font family" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:253, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:226 +#: src/app/main/ui/workspace/tokens/management/forms/typography.cljs:143 msgid "workspace.tokens.token-font-family-value-enter" msgstr "Font family or list of fonts separated by comma (,)" @@ -9416,15 +9424,15 @@ msgstr "Font family or list of fonts separated by comma (,)" msgid "workspace.tokens.token-name" msgstr "Name" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:273, ../common/src/app/common/files/tokens.cljc:156, ../common/src/app/common/files/tokens.cljc:169 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:341, ../common/src/app/common/files/tokens.cljc:156, ../common/src/app/common/files/tokens.cljc:169 msgid "workspace.tokens.token-name-duplication-validation-error" msgstr "A token already exists at the path: %s or at a prefix thereof." -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:270, ../common/src/app/common/files/tokens.cljc:153, ../common/src/app/common/files/tokens.cljc:166, ../common/src/app/common/files/tokens.cljc:289, ../common/src/app/common/files/tokens.cljc:317, ../common/src/app/common/files/tokens.cljc:330 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:338, ../common/src/app/common/files/tokens.cljc:153, ../common/src/app/common/files/tokens.cljc:166, ../common/src/app/common/files/tokens.cljc:289, ../common/src/app/common/files/tokens.cljc:317, ../common/src/app/common/files/tokens.cljc:330 msgid "workspace.tokens.token-name-length-validation-error" msgstr "Name should be at least 1 character" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:272, ../common/src/app/common/files/tokens.cljc:155, ../common/src/app/common/files/tokens.cljc:168 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:340, ../common/src/app/common/files/tokens.cljc:155, ../common/src/app/common/files/tokens.cljc:168 msgid "workspace.tokens.token-name-validation-error" msgstr "" " is not a valid token name.\n" @@ -9435,7 +9443,7 @@ msgstr "" msgid "workspace.tokens.token-value" msgstr "Value" -#: src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs:324, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:131 +#: src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs:324, src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:151 msgid "workspace.tokens.token-value-enter" msgstr "Enter a value or alias with {alias}" @@ -9457,7 +9465,7 @@ msgstr "" msgid "workspace.tokens.unknown-token-type-section" msgstr "Type '%s' is not supported (%s):" -#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:249, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:200 +#: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:316, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:270 msgid "workspace.tokens.use-reference" msgstr "Use a reference" @@ -9465,7 +9473,7 @@ msgstr "Use a reference" msgid "workspace.tokens.value-not-valid" msgstr "The value is not valid" -#: src/app/main/ui/workspace/top_toolbar.cljs:70 +#: src/app/main/ui/workspace/top_toolbar.cljs:72 msgid "workspace.toolbar.arrow" msgstr "Arrow (%s)" @@ -9481,51 +9489,51 @@ msgstr "Color Palette (%s)" msgid "workspace.toolbar.comments" msgstr "Comments (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:74 +#: src/app/main/ui/workspace/top_toolbar.cljs:76 msgid "workspace.toolbar.curve" msgstr "Curve (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:68 +#: src/app/main/ui/workspace/top_toolbar.cljs:70 msgid "workspace.toolbar.ellipse" msgstr "Ellipse (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:66 +#: src/app/main/ui/workspace/top_toolbar.cljs:68 msgid "workspace.toolbar.frame" msgstr "Board (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:73 +#: src/app/main/ui/workspace/top_toolbar.cljs:75 msgid "workspace.toolbar.image" msgstr "Image (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:413 +#: src/app/main/ui/workspace/top_toolbar.cljs:478 msgid "workspace.toolbar.label" msgstr "Drawing tools" -#: src/app/main/ui/workspace/top_toolbar.cljs:69 +#: src/app/main/ui/workspace/top_toolbar.cljs:71 msgid "workspace.toolbar.line" msgstr "Line (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:333 +#: src/app/main/ui/workspace/top_toolbar.cljs:389 msgid "workspace.toolbar.mcp" msgstr "MCP" -#: src/app/main/ui/workspace/top_toolbar.cljs:342 +#: src/app/main/ui/workspace/top_toolbar.cljs:399 msgid "workspace.toolbar.mcp-connected" msgstr "MCP connected" -#: src/app/main/ui/workspace/top_toolbar.cljs +#: src/app/main/ui/workspace/top_toolbar.cljs:405 msgid "workspace.toolbar.mcp-copy-session-id" msgstr "Copy session ID" -#: src/app/main/ui/workspace/top_toolbar.cljs +#: src/app/main/ui/workspace/top_toolbar.cljs:304 msgid "workspace.toolbar.mcp-session-copied" msgstr "Session ID copied" -#: src/app/main/ui/workspace/top_toolbar.cljs +#: src/app/main/ui/workspace/top_toolbar.cljs:401 msgid "workspace.toolbar.mcp-session-id" msgstr "Session ID: %s" -#: src/app/main/ui/workspace/top_toolbar.cljs:65, src/app/main/ui/workspace/top_toolbar.cljs:425 +#: src/app/main/ui/workspace/top_toolbar.cljs:67, src/app/main/ui/workspace/top_toolbar.cljs:490 msgid "workspace.toolbar.move" msgstr "Move (%s)" @@ -9541,15 +9549,15 @@ msgstr "Palette options" msgid "workspace.toolbar.path" msgstr "Path (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:75 +#: src/app/main/ui/workspace/top_toolbar.cljs:77 msgid "workspace.toolbar.plugins" msgstr "Plugins (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:67 +#: src/app/main/ui/workspace/top_toolbar.cljs:69 msgid "workspace.toolbar.rect" msgstr "Rectangle (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:71 +#: src/app/main/ui/workspace/top_toolbar.cljs:73 msgid "workspace.toolbar.text" msgstr "Text (%s)" @@ -9557,7 +9565,7 @@ msgstr "Text (%s)" msgid "workspace.toolbar.text-palette" msgstr "Typographies (%s)" -#: src/app/main/ui/workspace/top_toolbar.cljs:485, src/app/main/ui/workspace/top_toolbar.cljs:486 +#: src/app/main/ui/workspace/top_toolbar.cljs:552, src/app/main/ui/workspace/top_toolbar.cljs:553 msgid "workspace.toolbar.toggle-toolbar" msgstr "Toggle toolbar" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index 3da6305f9e..9ef4c87464 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -7583,6 +7583,14 @@ msgstr "Exportación lenta" msgid "workspace.options.exporting-queued" msgstr "Esperando..." +#: src/app/main/ui/exports/assets.cljs:179 +msgid "workspace.options.exporting-packaging" +msgstr "Empaquetando archivos…" + +#: src/app/main/ui/exports/assets.cljs:177 +msgid "workspace.options.exporting-preparing" +msgstr "Preparando..." + #: src/app/main/ui/workspace/sidebar/options/menus/fill.cljs:108, src/app/main/ui/workspace/sidebar/options/menus/fill.cljs:238 msgid "workspace.options.fill" msgstr "Relleno" diff --git a/render-wasm/README.md b/render-wasm/README.md index a6e0c1e25a..766e9116ab 100644 --- a/render-wasm/README.md +++ b/render-wasm/README.md @@ -83,7 +83,7 @@ Use the target cleanup commands instead of running `cargo clean` on the shared Each target writes its own generated `shared.js` (the enum discriminants the CLJS side compiles against) next to the code that imports it — respectively `frontend/src/app/render_wasm/api/shared.js` and -`exporter/src/app/wasm/shared.js`. Neither build writes to the other's paths. +`exporter/src/exporter/wasm/shared.js`. Neither build writes to the other's paths. ![Architecture overview](docs/images/architecture_schema.png) diff --git a/render-wasm/_build_env b/render-wasm/_build_env index c8d1fdd1e5..5cc3890dfa 100644 --- a/render-wasm/_build_env +++ b/render-wasm/_build_env @@ -150,7 +150,7 @@ export FRONTEND_DEST="${FRONTEND_DEST:-../frontend/resources/public/js}"; export EXPORT_DEST="${EXPORT_DEST:-../exporter/resources/wasm}"; export FRONTEND_SHARED_DEST="${FRONTEND_SHARED_DEST:-../frontend/src/app/render_wasm/api/shared.js}"; -export EXPORT_SHARED_DEST="${EXPORT_SHARED_DEST:-../exporter/src/app/wasm/shared.js}"; +export EXPORT_SHARED_DEST="${EXPORT_SHARED_DEST:-../exporter/src/exporter/wasm/shared.js}"; function clean { cargo clean;