diff --git a/backend/resources/app/templates/debug.tmpl b/backend/resources/app/templates/debug.tmpl
index 64ddd9d150..1498edaf87 100644
--- a/backend/resources/app/templates/debug.tmpl
+++ b/backend/resources/app/templates/debug.tmpl
@@ -326,6 +326,7 @@ Debug Main Page
| PROGRESS |
CMD |
BACKEND |
+ REPLICA |
NAME |
CREATED |
ENDED |
@@ -339,12 +340,13 @@ Debug Main Page
{{job.done}} / {{job.total}} |
{{job.cmd}} |
{{job.backend}} |
+ {{job.replica}} |
{{job.name}} |
{{job.created-at}} |
{{job.ended-at}} |
{% empty %}
- | No export jobs. |
+ | No export jobs. |
{% endfor %}
diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md
index bde2d9fd3d..e99e8d5cd5 100644
--- a/docs/technical-guide/configuration.md
+++ b/docs/technical-guide/configuration.md
@@ -883,6 +883,15 @@ PENPOT_INTERNAL_URI: http://penpot-frontend:8080
`http://penpot-frontend:8080` used in the docker-compose is a good default and
it is recommended to keep it unchanged.
+**Beta, new render engine (WebGL):** when you run more than one exporter, give
+each one its own name and keep it across restarts and upgrades. An exporter
+uses it to cancel only its own unfinished exports when it restarts:
+
+```bash
+# Exporter
+PENPOT_EXPORTER_REPLICA_ID: exporter-1
+```
+
### MCP
The MCP server lets AI agents read and edit Penpot files. It runs as a separate
diff --git a/exporter/README.md b/exporter/README.md
index df4d4412db..5e3986b90c 100644
--- a/exporter/README.md
+++ b/exporter/README.md
@@ -88,6 +88,7 @@ section, with a `?job-id=` filter.
| `PENPOT_EXPORTER_MAX_CONCURRENT_JOBS` | `4` | Admission limit |
| `PENPOT_EXPORTER_MAX_JOBS_PER_PROFILE`| `2` | Per-profile admission limit |
| `PENPOT_EXPORTER_QUEUE_MAX` | `64` | Queue cap; over it, `429 :queue-full` |
+| `PENPOT_EXPORTER_REPLICA_ID` | hostname | Names this replica; boot cleanup only cancels its own jobs |
| `PENPOT_WASM_WORKER_POOL_MAX` | `2` | Headless render worker threads; min 1 |
| `PENPOT_WASM_WORKER_POOL_MIN` | `1` | Workers kept warm; clamped to the max |
| `PENPOT_WASM_WORKER_IDLE_TIMEOUT` | `300` | Silence before a worker is terminated, in seconds |
diff --git a/exporter/src/app/config.cljs b/exporter/src/app/config.cljs
index 0fcc5a052d..4f94887ce0 100644
--- a/exporter/src/app/config.cljs
+++ b/exporter/src/app/config.cljs
@@ -9,6 +9,7 @@
(:require
["node:buffer" :as buffer]
["node:crypto" :as crypto]
+ ["node:os" :as os]
["node:process" :as process]
[app.common.data :as d]
[app.common.flags :as flags]
@@ -55,6 +56,7 @@
[:exporter-max-jobs-per-profile {:optional true} ::sm/int]
[:exporter-queue-max {:optional true} ::sm/int]
[:exporter-job-ttl {:optional true} ::sm/int]
+ [:exporter-replica-id {:optional true} :string]
[:wasm-worker-pool-max {:optional true} ::sm/int]
[:wasm-worker-pool-min {:optional true} ::sm/int]
[:wasm-worker-idle-timeout {:optional true} ::sm/int]
@@ -124,6 +126,12 @@
(or (c/get config :internal-uri)
(c/get config :public-uri)))
+(def replica-id
+ "Names this exporter among its siblings. It must survive a restart, since
+ the boot sweep uses it to find the jobs the previous run left behind."
+ (or (c/get config :exporter-replica-id)
+ (os/hostname)))
+
(def management-key
(let [key (or (c/get config :exporter-shared-key)
(let [secret-key (c/get config :secret-key)
diff --git a/exporter/src/app/core.cljs b/exporter/src/app/core.cljs
index a1ddab7efa..7dcf3a5b71 100644
--- a/exporter/src/app/core.cljs
+++ b/exporter/src/app/core.cljs
@@ -33,6 +33,7 @@
(l/info :msg "initializing"
:public-uri (str (cf/get :public-uri))
:internal-uri (str (cf/get-internal-uri))
+ :replica cf/replica-id
:version (:full cf/version))
(l/info :msg "headless wasm export available"
:wasm-dir wasm/artifact-dir
@@ -86,8 +87,19 @@
(fn [cause]
(js/console.error cause)))
+(defonce ^:private exiting? (atom false))
+
+(defn- exit!
+ "Cancels this process's jobs before exiting, so their records and clients
+ do not keep waiting on work that dies with it. A second signal exits at once."
+ []
+ (if (compare-and-set! exiting? false true)
+ (->> (shutdown-step "cancel-jobs" jobs/cancel-all!)
+ (p/fnly (fn [_ _] (proc/exit 0))))
+ (proc/exit 0)))
+
;; Signals are only delivered to the main thread, and `exit` in a worker would
;; take down that worker rather than the process.
(when ^boolean wt/isMainThread
- (.on proc/default "SIGTERM" (fn [] (proc/exit 0)))
- (.on proc/default "SIGINT" (fn [] (proc/exit 0))))
+ (.on proc/default "SIGTERM" exit!)
+ (.on proc/default "SIGINT" exit!))
diff --git a/exporter/src/app/jobs.cljs b/exporter/src/app/jobs.cljs
index e3497b2861..d555269a52 100644
--- a/exporter/src/app/jobs.cljs
+++ b/exporter/src/app/jobs.cljs
@@ -20,6 +20,7 @@
[app.common.logging :as l]
[app.common.time :as ct]
[app.common.uuid :as uuid]
+ [app.config :as cf]
[app.jobs.store :as store]
[app.redis :as redis]
[promesa.core :as p]))
@@ -81,6 +82,7 @@
[{:keys [profile-id cmd backend total name resource-id]} run-fn]
(let [job {:id (uuid/next)
:profile-id profile-id
+ :replica cf/replica-id
:cmd cmd
:backend backend
:state "queued"
@@ -165,7 +167,10 @@
overwrite either."
[job data]
(if-let [job (live job)]
- (persist-and-publish! (merge job data))
+ (let [job (merge job data)]
+ (l/info :hint "export job" :job-id (str (:id job)) :state (:state job)
+ :replica (:replica job))
+ (persist-and-publish! job))
(p/resolved job)))
(defn start!
@@ -208,7 +213,7 @@
:error (ex-message cause)}))
(defn- cancel-local!
- [job-id]
+ [job-id & {:as data}]
(let [k (str job-id)
rt (get @registry k)
job (:job rt)]
@@ -222,7 +227,7 @@
;; (`scheduler/drop-queued!`) releases the job, and `transition!` on a
;; released job is a no-op, so a queued job would keep claiming to be
;; queued in the store and never publish its `cancelled` update.
- (let [result (transition! job {:state "cancelled" :ended-at (now-ms)})]
+ (let [result (transition! job (merge {:state "cancelled" :ended-at (now-ms)} data))]
(doseq [f (:cancel-fns rt)]
(try
(f)
@@ -245,22 +250,40 @@
(store/request-cancel! (:id job))
job)))))))
+(defn cancel-all!
+ "Cancels every job this process holds, on shutdown. Nothing in flight survives
+ the process, and clients learn it now rather than when this replica boots
+ again, if it ever does under the same id."
+ []
+ (let [job-ids (keys @registry)]
+ (when (seq job-ids)
+ (l/info :hint "cancelling jobs on shutdown" :count (count job-ids)))
+ (->> job-ids
+ (map #(cancel-local! % :interrupted true))
+ (p/all))))
+
+(defn- abandoned?
+ "A job this replica's previous run left mid-flight. A sibling's jobs are its
+ own business: they may well be running."
+ [job]
+ (and (= cf/replica-id (:replica job))
+ (not (terminal? job))))
+
(defn- clean-abandoned!
- "Marks every job left mid-flight by a previous process as cancelled.
+ "Marks every job left mid-flight by a previous run of this replica as
+ cancelled.
A queue and its running jobs live in the memory of the process that owns
them, so nothing in flight when it died can be resumed; without this the
- record would keep claiming to be running until its TTL expires.
-
- NOTE: the store cannot tell whose jobs are whose, so with more than one
- exporter behind a load balancer this would also cancel a sibling's running
- jobs. Single-instance deployments only."
+ record would keep claiming to be running until its TTL expires. A replica
+ that never comes back under the same id leaves its records to the TTL."
[]
(->> (store/fetch-all)
(p/mcat (fn [jobs]
- (let [abandoned (remove terminal? jobs)]
+ (let [abandoned (filter abandoned? jobs)]
(when (seq abandoned)
(l/warn :hint "cancelling jobs abandoned by a previous process"
+ :replica cf/replica-id
:count (count abandoned)))
(->> abandoned
(map (fn [job]
diff --git a/exporter/src/app/jobs/scheduler.cljs b/exporter/src/app/jobs/scheduler.cljs
index 320689d90d..a986019a57 100644
--- a/exporter/src/app/jobs/scheduler.cljs
+++ b/exporter/src/app/jobs/scheduler.cljs
@@ -8,6 +8,7 @@
"Admission control for export jobs.
Limits concurrent jobs and rejects work rather than allowing an unbounded backlog.
+ Every limit is per replica: the queue lives in this process only.
Queue order is FIFO, except jobs whose profile is already at its cap are skipped,
as are headless jobs once every render worker is busy."
(:require
diff --git a/exporter/test/exporter_tests/jobs_test.cljs b/exporter/test/exporter_tests/jobs_test.cljs
index c8baa9b9fd..a92c1c1790 100644
--- a/exporter/test/exporter_tests/jobs_test.cljs
+++ b/exporter/test/exporter_tests/jobs_test.cljs
@@ -9,6 +9,7 @@
reported and swallowed, so only the in-process record is exercised."
(:require
[app.common.uuid :as uuid]
+ [app.config :as cf]
[app.jobs :as jobs]
[cljs.test :as t :include-macros true]
[promesa.core :as p]))
@@ -84,3 +85,23 @@
(t/is (= "ended" (:state ended)))
(t/is (nil? (jobs/lookup (:id job))))
(done)))))
+
+(t/deftest boot-sweep-leaves-sibling-jobs-alone
+ (let [abandoned? #'jobs/abandoned?]
+ (t/is (abandoned? {:replica cf/replica-id :state "running"}))
+ (t/is (abandoned? {:replica cf/replica-id :state "queued"}))
+ (t/is (not (abandoned? {:replica cf/replica-id :state "ended"})))
+ (t/is (not (abandoned? {:replica (str cf/replica-id "-sibling") :state "running"})))))
+
+(t/deftest shutdown-cancels-all-its-jobs
+ (t/async done
+ (p/let [running (create!)
+ queued (create!)
+ _ (jobs/start! running)
+ _ (jobs/cancel-all!)]
+ (doseq [job [running queued]]
+ (let [current (jobs/lookup (:id job))]
+ (t/is (= "cancelled" (:state current)))
+ (t/is (true? (:interrupted current))))
+ (jobs/release! (:id job)))
+ (done))))