✨ Support running several exporter replicas (#12487)

* ✨ Support running several exporter replicas

* 📚 Document one exporter per machine instead of compose replicas

* 📚 Trim exporter replica docs to the essentials

* ✨ Cancel an exporter's jobs when it shuts down

* 📚 Replace exporter replica docs with a short beta note

backend/resources/app/templates/debug.tmpl

@@ -326,6 +326,7 @@ Debug Main Page
               <th>PROGRESS</th>
               <th>CMD</th>
               <th>BACKEND</th>
+              <th>REPLICA</th>
               <th>NAME</th>
               <th>CREATED</th>
               <th>ENDED</th>
@@ -339,12 +340,13 @@ Debug Main Page
               <td>{{job.done}} / {{job.total}}</td>
               <td>{{job.cmd}}</td>
               <td>{{job.backend}}</td>
+              <td>{{job.replica}}</td>
               <td>{{job.name}}</td>
               <td>{{job.created-at}}</td>
               <td>{{job.ended-at}}</td>
             </tr>
             {% empty %}
-            <tr><td colspan="8">No export jobs.</td></tr>
+            <tr><td colspan="9">No export jobs.</td></tr>
             {% endfor %}
           </tbody>
         </table>

docs/technical-guide/configuration.md

@@ -883,6 +883,15 @@ PENPOT_INTERNAL_URI: http://penpot-frontend:8080
   `http://penpot-frontend:8080` used in the docker-compose is a good default and
   it is recommended to keep it unchanged.
 
+**Beta, new render engine (WebGL):** when you run more than one exporter, give
+each one its own name and keep it across restarts and upgrades. An exporter
+uses it to cancel only its own unfinished exports when it restarts:
+
+```bash
+# Exporter
+PENPOT_EXPORTER_REPLICA_ID: exporter-1
+```
+
 ### MCP
 
 The MCP server lets AI agents read and edit Penpot files. It runs as a separate

exporter/README.md

@@ -88,6 +88,7 @@ section, with a `?job-id=` filter.
 | `PENPOT_EXPORTER_MAX_CONCURRENT_JOBS` | `4`     | Admission limit                       |
 | `PENPOT_EXPORTER_MAX_JOBS_PER_PROFILE`| `2`     | Per-profile admission limit           |
 | `PENPOT_EXPORTER_QUEUE_MAX`           | `64`    | Queue cap; over it, `429 :queue-full` |
+| `PENPOT_EXPORTER_REPLICA_ID`          | hostname | Names this replica; boot cleanup only cancels its own jobs |
 | `PENPOT_WASM_WORKER_POOL_MAX`         | `2`     | Headless render worker threads; min 1 |
 | `PENPOT_WASM_WORKER_POOL_MIN`         | `1`     | Workers kept warm; clamped to the max |
 | `PENPOT_WASM_WORKER_IDLE_TIMEOUT`     | `300`   | Silence before a worker is terminated, in seconds |

exporter/src/app/config.cljs

@@ -9,6 +9,7 @@
   (:require
    ["node:buffer" :as buffer]
    ["node:crypto" :as crypto]
+   ["node:os" :as os]
    ["node:process" :as process]
    [app.common.data :as d]
    [app.common.flags :as flags]
@@ -55,6 +56,7 @@
    [:exporter-max-jobs-per-profile {:optional true} ::sm/int]
    [:exporter-queue-max {:optional true} ::sm/int]
    [:exporter-job-ttl {:optional true} ::sm/int]
+   [:exporter-replica-id {:optional true} :string]
    [:wasm-worker-pool-max {:optional true} ::sm/int]
    [:wasm-worker-pool-min {:optional true} ::sm/int]
    [:wasm-worker-idle-timeout {:optional true} ::sm/int]
@@ -124,6 +126,12 @@
   (or (c/get config :internal-uri)
       (c/get config :public-uri)))
 
+(def replica-id
+  "Names this exporter among its siblings. It must survive a restart, since
+  the boot sweep uses it to find the jobs the previous run left behind."
+  (or (c/get config :exporter-replica-id)
+      (os/hostname)))
+
 (def management-key
   (let [key (or (c/get config :exporter-shared-key)
                 (let [secret-key  (c/get config :secret-key)

exporter/src/app/core.cljs

@@ -33,6 +33,7 @@
       (l/info :msg "initializing"
               :public-uri (str (cf/get :public-uri))
               :internal-uri (str (cf/get-internal-uri))
+              :replica cf/replica-id
               :version (:full cf/version))
       (l/info :msg "headless wasm export available"
               :wasm-dir wasm/artifact-dir
@@ -86,8 +87,19 @@
      (fn [cause]
        (js/console.error cause)))
 
+(defonce ^:private exiting? (atom false))
+
+(defn- exit!
+  "Cancels this process's jobs before exiting, so their records and clients
+  do not keep waiting on work that dies with it. A second signal exits at once."
+  []
+  (if (compare-and-set! exiting? false true)
+    (->> (shutdown-step "cancel-jobs" jobs/cancel-all!)
+         (p/fnly (fn [_ _] (proc/exit 0))))
+    (proc/exit 0)))
+
 ;; Signals are only delivered to the main thread, and `exit` in a worker would
 ;; take down that worker rather than the process.
 (when ^boolean wt/isMainThread
-  (.on proc/default "SIGTERM" (fn [] (proc/exit 0)))
-  (.on proc/default "SIGINT" (fn [] (proc/exit 0))))
+  (.on proc/default "SIGTERM" exit!)
+  (.on proc/default "SIGINT" exit!))

exporter/src/app/jobs.cljs

@@ -20,6 +20,7 @@
    [app.common.logging :as l]
    [app.common.time :as ct]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.jobs.store :as store]
    [app.redis :as redis]
    [promesa.core :as p]))
@@ -81,6 +82,7 @@
   [{:keys [profile-id cmd backend total name resource-id]} run-fn]
   (let [job {:id (uuid/next)
              :profile-id profile-id
+             :replica cf/replica-id
              :cmd cmd
              :backend backend
              :state "queued"
@@ -165,7 +167,10 @@
   overwrite either."
   [job data]
   (if-let [job (live job)]
-    (persist-and-publish! (merge job data))
+    (let [job (merge job data)]
+      (l/info :hint "export job" :job-id (str (:id job)) :state (:state job)
+              :replica (:replica job))
+      (persist-and-publish! job))
     (p/resolved job)))
 
 (defn start!
@@ -208,7 +213,7 @@
                     :error (ex-message cause)}))
 
 (defn- cancel-local!
-  [job-id]
+  [job-id & {:as data}]
   (let [k    (str job-id)
         rt   (get @registry k)
         job  (:job rt)]
@@ -222,7 +227,7 @@
         ;; (`scheduler/drop-queued!`) releases the job, and `transition!` on a
         ;; released job is a no-op, so a queued job would keep claiming to be
         ;; queued in the store and never publish its `cancelled` update.
-        (let [result (transition! job {:state "cancelled" :ended-at (now-ms)})]
+        (let [result (transition! job (merge {:state "cancelled" :ended-at (now-ms)} data))]
           (doseq [f (:cancel-fns rt)]
             (try
               (f)
@@ -245,22 +250,40 @@
                                       (store/request-cancel! (:id job))
                                       job)))))))
 
+(defn cancel-all!
+  "Cancels every job this process holds, on shutdown. Nothing in flight survives
+  the process, and clients learn it now rather than when this replica boots
+  again, if it ever does under the same id."
+  []
+  (let [job-ids (keys @registry)]
+    (when (seq job-ids)
+      (l/info :hint "cancelling jobs on shutdown" :count (count job-ids)))
+    (->> job-ids
+         (map #(cancel-local! % :interrupted true))
+         (p/all))))
+
+(defn- abandoned?
+  "A job this replica's previous run left mid-flight. A sibling's jobs are its
+  own business: they may well be running."
+  [job]
+  (and (= cf/replica-id (:replica job))
+       (not (terminal? job))))
+
 (defn- clean-abandoned!
-  "Marks every job left mid-flight by a previous process as cancelled.
+  "Marks every job left mid-flight by a previous run of this replica as
+  cancelled.
 
   A queue and its running jobs live in the memory of the process that owns
   them, so nothing in flight when it died can be resumed; without this the
-  record would keep claiming to be running until its TTL expires.
-
-  NOTE: the store cannot tell whose jobs are whose, so with more than one
-  exporter behind a load balancer this would also cancel a sibling's running
-  jobs. Single-instance deployments only."
+  record would keep claiming to be running until its TTL expires. A replica
+  that never comes back under the same id leaves its records to the TTL."
   []
   (->> (store/fetch-all)
        (p/mcat (fn [jobs]
-                 (let [abandoned (remove terminal? jobs)]
+                 (let [abandoned (filter abandoned? jobs)]
                    (when (seq abandoned)
                      (l/warn :hint "cancelling jobs abandoned by a previous process"
+                             :replica cf/replica-id
                              :count (count abandoned)))
                    (->> abandoned
                         (map (fn [job]

exporter/src/app/jobs/scheduler.cljs

@@ -8,6 +8,7 @@
   "Admission control for export jobs.
 
   Limits concurrent jobs and rejects work rather than allowing an unbounded backlog.
+  Every limit is per replica: the queue lives in this process only.
   Queue order is FIFO, except jobs whose profile is already at its cap are skipped,
   as are headless jobs once every render worker is busy."
   (:require

exporter/test/exporter_tests/jobs_test.cljs

@@ -9,6 +9,7 @@
   reported and swallowed, so only the in-process record is exercised."
   (:require
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.jobs :as jobs]
    [cljs.test :as t :include-macros true]
    [promesa.core :as p]))
@@ -84,3 +85,23 @@
         (t/is (= "ended" (:state ended)))
         (t/is (nil? (jobs/lookup (:id job))))
         (done)))))
+
+(t/deftest boot-sweep-leaves-sibling-jobs-alone
+  (let [abandoned? #'jobs/abandoned?]
+    (t/is (abandoned? {:replica cf/replica-id :state "running"}))
+    (t/is (abandoned? {:replica cf/replica-id :state "queued"}))
+    (t/is (not (abandoned? {:replica cf/replica-id :state "ended"})))
+    (t/is (not (abandoned? {:replica (str cf/replica-id "-sibling") :state "running"})))))
+
+(t/deftest shutdown-cancels-all-its-jobs
+  (t/async done
+    (p/let [running (create!)
+            queued  (create!)
+            _       (jobs/start! running)
+            _       (jobs/cancel-all!)]
+      (doseq [job [running queued]]
+        (let [current (jobs/lookup (:id job))]
+          (t/is (= "cancelled" (:state current)))
+          (t/is (true? (:interrupted current))))
+        (jobs/release! (:id job)))
+      (done))))