penpot/backend/src/app/worker/runner.clj
Andrey Antukh 90a06e982b
♻️ Rework the audit-log consumers and the jobs substrate (#12701)
* ♻️ Rework audit-log consumers and jobs substrate

Fold the implementation, follow-up fixes, migrations and tests into one

code commit.

AI-assisted-by: gpt-6-luna

* 📚 Consolidate 2.20.0 upgrade runbooks

Keep the operational upgrade procedure and its supporting runbooks in a

separate documentation commit.

AI-assisted-by: gpt-6-luna

* ♻️ Remove the audit-log flag and derive persistence at boot

Persistence now follows the consumer flags instead of a separate switch.
app.config computes cf/audit-persist? once at boot from :nexus and
:admin-console, and every write site reads that value, so request paths
no longer check flags per event. With no consumer enabled no full rows
are written; telemetry shadows and webhook deliveries keep their own
flags. The frontend wire signal stays :audit-log as a derived value.

Update the 2.20 runbooks, memories and the devenv flags to match, and
migrate the tests that used to toggle :audit-log directly.

AI-assisted-by: hy4-flash-free

* 🐛 Accept error-reporting batches and precompute sink flags

push-audit-events now accepts crash batches with :error-reporting alone, so
the frontend signal from get-environment-data matches a real destination.
The reporters receive unhandled-exception events without storing any row.

All per-event sink checks read booleans computed once at boot: webhooks,
telemetry and error reporting join audit-persist? in app.config. The test
fixture and with-audit-config rebind all four values together, and the
tests that toggled those flags directly were migrated.

Address F2 of the review of the previous commit. F1 stays deferred to a
separate plan.

AI-assisted-by: hy4-flash-free

* 💄 Match the layout cljfmt expects in the flags set

The first comment of the varia set sat on its own line after the opening
delimiter, and cljfmt joins such a comment to the delimiter line, so the
Common Tests lint step failed on common/src/app/common/flags.cljc. Keep
the comment on the opening line, the same shape the login and email sets
already use.

Note in mem:scripts/ci that the backend lint lints ../common/src but does
not format-check it, so common has to be passed to scripts/ci as well.

AI-assisted-by: deepseek-v4-flash
2026-11-20 11:12:00 +01:00

423 lines
16 KiB
Clojure

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.worker.runner
"Job execution over the unified `job` table (impl)."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.json :as json]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.jobs :as jobs]
[app.jobs.metrics :as jobs-metrics]
[app.metrics :as mtx]
[app.msgbus :as-alias mbus]
[app.redis :as rds]
[app.worker :as wrk]
[cuerdas.core :as str]
[integrant.core :as ig]
[promesa.exec :as px])
(:import
java.lang.AutoCloseable))
(set! *warn-on-reflection* true)
(defn- claim-job
"Conditional claim: only transition pending jobs (new/scheduled/retry)
to running. Delegates to the shared jobs/claim so internal and
external workers can never drift. A cancelled, aborted or otherwise
terminal job produces
0 affected rows and is skipped without touching its state
(first-terminal-wins companion). Also predicates the payload
scheduled_at: a duplicate or stale payload is never claimed with a
timestamp that no longer matches the row."
[cfg job-id scheduled-at]
(jobs/claim cfg job-id scheduled-at))
(defn- get-exception-type
"Extract a human-readable exception type for observability."
[error]
(cond
(and (ex/exception? error) (instance? clojure.lang.ExceptionInfo error))
(or (:type (ex-data error)) :ex-info)
(instance? java.util.concurrent.TimeoutException error)
:timeout
(instance? java.sql.SQLTimeoutException error)
:timeout
:else
(keyword (str/lower (.getSimpleName (class error))))))
(defn- error-report
"Build the error stored in `job.error` out of an exception, conforming to
`app.jobs/schema:job-error`. The exception type and its own type/code are
kept as extra details for triage, on top of the three contract keys."
[error]
(let [data (ex-data error)
as-keyword (fn [v]
(cond (keyword? v) v
(string? v) (keyword v)))]
(jobs/check-job-error
{:type (or (as-keyword (:type data)) :internal)
:code (or (as-keyword (:code data)) :failed)
:hint (or (:hint data)
(when (ex/exception? error) (ex-message error))
(str error))
:ex-type (get-exception-type error)})))
;; A handler result may be the reserved completion envelope, which carries
;; the storage object the job produced. The key is reserved: a handler that
;; wants to return a business map with :resource-id in it cannot.
(defn- completion-envelope?
[result]
(and (map? result) (contains? result :resource-id)))
(defn- get-job
"Fetch the job row (params kept as raw pgobject; decoded later with the
job-def decoder)."
[cfg job-id]
(ex/try!
(some-> (db/get* cfg :job {:id job-id}))))
(defn- execute-job
[{:keys [::jobs/defs ::id ::queue ::wrk/tenant] :as cfg} job]
(try
(l/dbg :hint "start"
:name (:name job)
:job-id (str (:id job))
:tenant tenant
:queue queue
:runner-id id
:retry (:retry-num job))
(if (zero? (claim-job cfg (:id job) (:scheduled-at job)))
(l/wrn :hint "skipping job, not claimable"
:id (str (:id job))
:name (:name job)
:tenant tenant
:queue queue
:status (:status job))
(do
(let [job-def (jobs/get-job-def defs (:name job))
_ (jobs-metrics/record-claim-wait
cfg
(::jobs/name job-def)
(- (inst-ms (ct/now)) (inst-ms (:scheduled-at job))))
params (try
(->> (:params job)
(jobs/decode-params job-def)
(jobs/validate-params job-def))
(catch Throwable cause
;; Decode/validation of stored params is pure: any
;; failure here is permanent (e.g. schema tightened
;; after submit), never transient. Tag it so the
;; generic catch below fails fast instead of
;; burning max-retries.
(throw (ex-info "job params failed validation"
{:type :assertion
:code :data-validation}
cause))))
handler (::jobs/handler job-def)
context (jobs/make-context job)
tpoint (ct/tpoint)
result (binding [jobs/*job-id* (:id job)]
(try
(handler context params)
(finally
(jobs-metrics/record-execution
cfg
(:name job)
(inst-ms (tpoint)))
(jobs-metrics/record-legacy-execution
cfg
(:name job)
(inst-ms (tpoint))))))]
(l/dbg :hint "end"
:name (:name job)
:job-id (str (:id job))
:tenant tenant
:queue queue
:runner-id id
:retry (:retry-num job)
:elapsed (ct/format-duration (tpoint)))
{:status "completed"
:result result})))
(catch InterruptedException cause
(throw cause))
(catch Throwable cause
(let [edata (ex-data cause)]
(cond
;; the interrupt is the signal that the job is no longer active:
;; a normal procedure, so it is not reported as an error and it
;; never fails or retries a row that is already terminal
(= :interrupt (:type edata))
(binding [l/*context* (-> (cf/logging-context)
(assoc :params job))]
(l/inf :hint "interrupted"
:tenant tenant
:job-id (str (:job-id edata))
:status (:status edata))
{:status "interrupted"})
(and (< (:retry-num job)
(:max-retries job))
(= ::wrk/retry (:type edata)))
(cond-> {:status "retry" :error cause}
(ct/duration? (:delay edata))
(assoc :delay-ms (inst-ms (:delay edata)))
(int? (:delay edata))
(assoc :delay-ms (:delay edata))
(= ::wrk/noop (:strategy edata))
(assoc :inc-by 0))
:else
(do
(l/err :hint "unhandled exception on job"
::l/context (assoc (cf/logging-context) :params job)
:tenant tenant
:cause cause)
;; Unknown job names and invalid params never heal by retrying
;; (no rolling deploy will register them on this backend), so
;; they fail fast without burning max-retries or churning
;; modified_at.
(if (or (>= (:retry-num job) (:max-retries job))
(contains? #{:no-job-definition :data-validation}
(:code edata)))
{:status "failed" :error cause}
{:status "retry" :error cause})))))
(finally
(jobs/cleanup-throttle (:id job)))))
(defn- run-job
[{:keys [::id ::timeout ::wrk/tenant] :as cfg} job-id scheduled-at]
(loop [job (get-job cfg job-id)]
(cond
(nil? job)
(l/wrn :hint "no job found on the database"
:runner-id id
:tenant tenant
:job-id (str job-id))
(ex/exception? job)
(if (or (db/connection-error? job)
(db/serialization-error? job))
(do
(l/wrn :hint "connection error on retrieving job from database (retrying in some instants)"
:runner-id id
:tenant tenant
:cause job)
(px/sleep timeout)
(recur (get-job cfg job-id)))
(do
(l/err :hint "unhandled exception on retrieving job from database (retrying in some instants)"
:runner-id id
:tenant tenant
:cause job)
(px/sleep timeout)
(recur (get-job cfg job-id))))
(not= (inst-ms scheduled-at)
(inst-ms (:scheduled-at job)))
(l/wrn :hint "skipping job, stale payload"
:job-id (str job-id)
:runner-id id
:tenant tenant
:scheduled-at (ct/format-inst (:scheduled-at job))
:expected-scheduled-at (ct/format-inst scheduled-at))
:else
(let [result (execute-job cfg job)]
(with-meta result
{::job job})))))
(defn- run-worker-loop
[{:keys [::rds/conn ::timeout ::queue ::wrk/tenant] :as cfg}]
(letfn [(handle-job-retry [{:keys [error delay-ms inc-by] :or {inc-by 1 delay-ms 1000} :as result}]
(let [job (-> result meta ::job)
nretry (+ (:retry-num job) inc-by)
now (ct/now)
delay (->> (iterate #(* 2 %) delay-ms) (take (max 1 nretry)) (last))]
(jobs/retry-job cfg
(:id job)
nretry
(-> (ct/plus now (ct/duration {:millis delay}))
(ct/truncate :millisecond))
(error-report error)
(if (zero? inc-by) :noop :backoff))
nil))
(handle-job-failure [{:keys [error] :as result}]
(let [job (-> result meta ::job)]
(jobs/fail cfg (:id job) (error-report error))
nil))
(handle-job-completion [result]
(let [job (-> result meta ::job)
returned (:result result)
envelope? (completion-envelope? returned)]
(jobs/complete cfg
:job-id (:id job)
:result (if envelope? (:result returned) returned)
:resource-id (when envelope? (:resource-id returned)))
nil))
(decode-payload [payload]
(try
(let [payload (json/decode payload)
job-id (uuid/parse (first payload))
sched-at (ct/inst (second payload))]
(if (and (uuid? job-id)
(ct/inst? sched-at))
[job-id sched-at]
(do
(l/err :hint "received unexpected payload"
:tenant tenant
:payload payload)
nil)))
(catch Throwable cause
(l/err :hint "unable to decode payload"
::l/context (cf/logging-context)
:tenant tenant
:payload payload
:length (count payload)
:cause cause))))
(process-result [{:keys [status] :as result}]
(ex/try!
(case status
"retry" (handle-job-retry result)
"failed" (handle-job-failure result)
"completed" (handle-job-completion result)
;; the job is no longer active: there is nothing to write,
;; the row is already terminal and whoever stopped it did
"interrupted" nil
(throw (IllegalArgumentException.
(str "invalid status received: '" status "'"))))))
(run-job-loop [[job-id scheduled-at]]
(loop [result (run-job cfg job-id scheduled-at)]
(when-let [cause (some-> result process-result)]
(if (or (db/connection-error? cause)
(db/serialization-error? cause))
(do
(l/wrn :hint "database exeption on processing job result (retrying in some instants)"
:tenant tenant
:cause cause)
(px/sleep timeout)
(recur result))
(l/err :hint "unhandled exception on processing job result"
::l/context (cf/logging-context)
:tenant tenant
:cause cause)))))]
(try
(let [key (wrk/queue-key tenant queue)
[_ payload] (rds/blpop conn [key] timeout)]
(some-> payload
decode-payload
run-job-loop))
(catch InterruptedException cause
(throw cause))
(catch Exception cause
(if (rds/timeout-exception? cause)
(do
(l/err :hint "redis pop operation timeout, consider increasing redis timeout (will retry in some instants)"
::l/context (cf/logging-context)
:tenant tenant
:timeout timeout
:cause cause)
(px/sleep timeout))
(l/err :hint "unhandled exception"
::l/context (cf/logging-context)
:tenant tenant
:cause cause))))))
(defn- start-thread
[{:keys [::id ::queue ::wrk/tenant] :as cfg}]
(px/thread
{:name (str "penpot/job-runner/" id)}
(l/inf :hint "started" :id id :tenant tenant :queue queue)
(let [rconn (rds/connect cfg)]
(try
(loop [cfg (-> cfg
(assoc ::rds/conn rconn)
(assoc ::timeout (ct/duration "5s")))]
(when (px/interrupted?)
(throw (InterruptedException. "interrupted")))
(run-worker-loop cfg)
(recur cfg))
(catch InterruptedException _
(l/dbg :hint "interrupted"
:id id
:tenant tenant
:queue queue))
(catch Throwable cause
(l/err :hint "unexpected exception"
::l/context (cf/logging-context)
:id id
:tenant tenant
:queue queue
:cause cause))
(finally
(.close ^AutoCloseable rconn)
(l/inf :hint "terminated"
:id id
:tenant tenant
:queue queue))))))
(def ^:private schema:params
[:map
[::wrk/parallelism {:optional true} ::sm/int]
[::wrk/queue :keyword]
[::wrk/tenant ::sm/text]
::jobs/defs
::mtx/metrics
::db/pool
::rds/client
;; job events of a profile job are published on the profile topic
::mbus/msgbus])
(defmethod ig/assert-key ::wrk/runner
[_ params]
(assert (sm/check schema:params params)))
(defmethod ig/expand-key ::wrk/runner
[k v]
{k (merge {::wrk/parallelism 1} (d/without-nils v))})
(defmethod ig/init-key ::wrk/runner
[_ {:keys [::db/pool ::wrk/queue ::wrk/parallelism ::wrk/tenant] :as cfg}]
(let [queue (d/name queue)
cfg (assoc cfg ::queue queue)]
(if (db/read-only? pool)
(l/wrn :hint "not started (db is read-only)" :tenant tenant :queue queue :parallelism parallelism)
(doall
(->> (range parallelism)
(map #(assoc cfg ::id (str queue "/" %)))
(map start-thread))))))
(defmethod ig/halt-key! ::wrk/runner
[_ threads]
(run! px/interrupt! threads))