mirror of
https://github.com/penpot/penpot.git
synced 2026-11-22 10:20:51 +00:00
* ♻️ Rework audit-log consumers and jobs substrate Fold the implementation, follow-up fixes, migrations and tests into one code commit. AI-assisted-by: gpt-6-luna * 📚 Consolidate 2.20.0 upgrade runbooks Keep the operational upgrade procedure and its supporting runbooks in a separate documentation commit. AI-assisted-by: gpt-6-luna * ♻️ Remove the audit-log flag and derive persistence at boot Persistence now follows the consumer flags instead of a separate switch. app.config computes cf/audit-persist? once at boot from :nexus and :admin-console, and every write site reads that value, so request paths no longer check flags per event. With no consumer enabled no full rows are written; telemetry shadows and webhook deliveries keep their own flags. The frontend wire signal stays :audit-log as a derived value. Update the 2.20 runbooks, memories and the devenv flags to match, and migrate the tests that used to toggle :audit-log directly. AI-assisted-by: hy4-flash-free * 🐛 Accept error-reporting batches and precompute sink flags push-audit-events now accepts crash batches with :error-reporting alone, so the frontend signal from get-environment-data matches a real destination. The reporters receive unhandled-exception events without storing any row. All per-event sink checks read booleans computed once at boot: webhooks, telemetry and error reporting join audit-persist? in app.config. The test fixture and with-audit-config rebind all four values together, and the tests that toggled those flags directly were migrated. Address F2 of the review of the previous commit. F1 stays deferred to a separate plan. AI-assisted-by: hy4-flash-free * 💄 Match the layout cljfmt expects in the flags set The first comment of the varia set sat on its own line after the opening delimiter, and cljfmt joins such a comment to the delimiter line, so the Common Tests lint step failed on common/src/app/common/flags.cljc. Keep the comment on the opening line, the same shape the login and email sets already use. Note in mem:scripts/ci that the backend lint lints ../common/src but does not format-check it, so common has to be passed to scripts/ci as well. AI-assisted-by: deepseek-v4-flash
423 lines
16 KiB
Clojure
423 lines
16 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns app.worker.runner
|
|
"Job execution over the unified `job` table (impl)."
|
|
(:require
|
|
[app.common.data :as d]
|
|
[app.common.exceptions :as ex]
|
|
[app.common.json :as json]
|
|
[app.common.logging :as l]
|
|
[app.common.schema :as sm]
|
|
[app.common.time :as ct]
|
|
[app.common.uuid :as uuid]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[app.jobs :as jobs]
|
|
[app.jobs.metrics :as jobs-metrics]
|
|
[app.metrics :as mtx]
|
|
[app.msgbus :as-alias mbus]
|
|
[app.redis :as rds]
|
|
[app.worker :as wrk]
|
|
[cuerdas.core :as str]
|
|
[integrant.core :as ig]
|
|
[promesa.exec :as px])
|
|
(:import
|
|
java.lang.AutoCloseable))
|
|
|
|
(set! *warn-on-reflection* true)
|
|
|
|
(defn- claim-job
|
|
"Conditional claim: only transition pending jobs (new/scheduled/retry)
|
|
to running. Delegates to the shared jobs/claim so internal and
|
|
external workers can never drift. A cancelled, aborted or otherwise
|
|
terminal job produces
|
|
0 affected rows and is skipped without touching its state
|
|
(first-terminal-wins companion). Also predicates the payload
|
|
scheduled_at: a duplicate or stale payload is never claimed with a
|
|
timestamp that no longer matches the row."
|
|
[cfg job-id scheduled-at]
|
|
(jobs/claim cfg job-id scheduled-at))
|
|
|
|
(defn- get-exception-type
|
|
"Extract a human-readable exception type for observability."
|
|
[error]
|
|
(cond
|
|
(and (ex/exception? error) (instance? clojure.lang.ExceptionInfo error))
|
|
(or (:type (ex-data error)) :ex-info)
|
|
|
|
(instance? java.util.concurrent.TimeoutException error)
|
|
:timeout
|
|
|
|
(instance? java.sql.SQLTimeoutException error)
|
|
:timeout
|
|
|
|
:else
|
|
(keyword (str/lower (.getSimpleName (class error))))))
|
|
|
|
(defn- error-report
|
|
"Build the error stored in `job.error` out of an exception, conforming to
|
|
`app.jobs/schema:job-error`. The exception type and its own type/code are
|
|
kept as extra details for triage, on top of the three contract keys."
|
|
[error]
|
|
(let [data (ex-data error)
|
|
as-keyword (fn [v]
|
|
(cond (keyword? v) v
|
|
(string? v) (keyword v)))]
|
|
(jobs/check-job-error
|
|
{:type (or (as-keyword (:type data)) :internal)
|
|
:code (or (as-keyword (:code data)) :failed)
|
|
:hint (or (:hint data)
|
|
(when (ex/exception? error) (ex-message error))
|
|
(str error))
|
|
:ex-type (get-exception-type error)})))
|
|
|
|
;; A handler result may be the reserved completion envelope, which carries
|
|
;; the storage object the job produced. The key is reserved: a handler that
|
|
;; wants to return a business map with :resource-id in it cannot.
|
|
(defn- completion-envelope?
|
|
[result]
|
|
(and (map? result) (contains? result :resource-id)))
|
|
|
|
(defn- get-job
|
|
"Fetch the job row (params kept as raw pgobject; decoded later with the
|
|
job-def decoder)."
|
|
[cfg job-id]
|
|
(ex/try!
|
|
(some-> (db/get* cfg :job {:id job-id}))))
|
|
|
|
(defn- execute-job
|
|
[{:keys [::jobs/defs ::id ::queue ::wrk/tenant] :as cfg} job]
|
|
(try
|
|
(l/dbg :hint "start"
|
|
:name (:name job)
|
|
:job-id (str (:id job))
|
|
:tenant tenant
|
|
:queue queue
|
|
:runner-id id
|
|
:retry (:retry-num job))
|
|
|
|
(if (zero? (claim-job cfg (:id job) (:scheduled-at job)))
|
|
(l/wrn :hint "skipping job, not claimable"
|
|
:id (str (:id job))
|
|
:name (:name job)
|
|
:tenant tenant
|
|
:queue queue
|
|
:status (:status job))
|
|
|
|
(do
|
|
(let [job-def (jobs/get-job-def defs (:name job))
|
|
_ (jobs-metrics/record-claim-wait
|
|
cfg
|
|
(::jobs/name job-def)
|
|
(- (inst-ms (ct/now)) (inst-ms (:scheduled-at job))))
|
|
params (try
|
|
(->> (:params job)
|
|
(jobs/decode-params job-def)
|
|
(jobs/validate-params job-def))
|
|
(catch Throwable cause
|
|
;; Decode/validation of stored params is pure: any
|
|
;; failure here is permanent (e.g. schema tightened
|
|
;; after submit), never transient. Tag it so the
|
|
;; generic catch below fails fast instead of
|
|
;; burning max-retries.
|
|
(throw (ex-info "job params failed validation"
|
|
{:type :assertion
|
|
:code :data-validation}
|
|
cause))))
|
|
handler (::jobs/handler job-def)
|
|
context (jobs/make-context job)
|
|
tpoint (ct/tpoint)
|
|
result (binding [jobs/*job-id* (:id job)]
|
|
(try
|
|
(handler context params)
|
|
(finally
|
|
(jobs-metrics/record-execution
|
|
cfg
|
|
(:name job)
|
|
(inst-ms (tpoint)))
|
|
(jobs-metrics/record-legacy-execution
|
|
cfg
|
|
(:name job)
|
|
(inst-ms (tpoint))))))]
|
|
|
|
(l/dbg :hint "end"
|
|
:name (:name job)
|
|
:job-id (str (:id job))
|
|
:tenant tenant
|
|
:queue queue
|
|
:runner-id id
|
|
:retry (:retry-num job)
|
|
:elapsed (ct/format-duration (tpoint)))
|
|
|
|
{:status "completed"
|
|
:result result})))
|
|
|
|
(catch InterruptedException cause
|
|
(throw cause))
|
|
(catch Throwable cause
|
|
(let [edata (ex-data cause)]
|
|
(cond
|
|
;; the interrupt is the signal that the job is no longer active:
|
|
;; a normal procedure, so it is not reported as an error and it
|
|
;; never fails or retries a row that is already terminal
|
|
(= :interrupt (:type edata))
|
|
(binding [l/*context* (-> (cf/logging-context)
|
|
(assoc :params job))]
|
|
(l/inf :hint "interrupted"
|
|
:tenant tenant
|
|
:job-id (str (:job-id edata))
|
|
:status (:status edata))
|
|
{:status "interrupted"})
|
|
|
|
(and (< (:retry-num job)
|
|
(:max-retries job))
|
|
(= ::wrk/retry (:type edata)))
|
|
(cond-> {:status "retry" :error cause}
|
|
(ct/duration? (:delay edata))
|
|
(assoc :delay-ms (inst-ms (:delay edata)))
|
|
(int? (:delay edata))
|
|
(assoc :delay-ms (:delay edata))
|
|
|
|
(= ::wrk/noop (:strategy edata))
|
|
(assoc :inc-by 0))
|
|
|
|
:else
|
|
(do
|
|
(l/err :hint "unhandled exception on job"
|
|
::l/context (assoc (cf/logging-context) :params job)
|
|
:tenant tenant
|
|
:cause cause)
|
|
;; Unknown job names and invalid params never heal by retrying
|
|
;; (no rolling deploy will register them on this backend), so
|
|
;; they fail fast without burning max-retries or churning
|
|
;; modified_at.
|
|
(if (or (>= (:retry-num job) (:max-retries job))
|
|
(contains? #{:no-job-definition :data-validation}
|
|
(:code edata)))
|
|
{:status "failed" :error cause}
|
|
{:status "retry" :error cause})))))
|
|
(finally
|
|
(jobs/cleanup-throttle (:id job)))))
|
|
|
|
(defn- run-job
|
|
[{:keys [::id ::timeout ::wrk/tenant] :as cfg} job-id scheduled-at]
|
|
(loop [job (get-job cfg job-id)]
|
|
(cond
|
|
(nil? job)
|
|
(l/wrn :hint "no job found on the database"
|
|
:runner-id id
|
|
:tenant tenant
|
|
:job-id (str job-id))
|
|
|
|
(ex/exception? job)
|
|
(if (or (db/connection-error? job)
|
|
(db/serialization-error? job))
|
|
(do
|
|
(l/wrn :hint "connection error on retrieving job from database (retrying in some instants)"
|
|
:runner-id id
|
|
:tenant tenant
|
|
:cause job)
|
|
(px/sleep timeout)
|
|
(recur (get-job cfg job-id)))
|
|
(do
|
|
(l/err :hint "unhandled exception on retrieving job from database (retrying in some instants)"
|
|
:runner-id id
|
|
:tenant tenant
|
|
:cause job)
|
|
(px/sleep timeout)
|
|
(recur (get-job cfg job-id))))
|
|
|
|
(not= (inst-ms scheduled-at)
|
|
(inst-ms (:scheduled-at job)))
|
|
(l/wrn :hint "skipping job, stale payload"
|
|
:job-id (str job-id)
|
|
:runner-id id
|
|
:tenant tenant
|
|
:scheduled-at (ct/format-inst (:scheduled-at job))
|
|
:expected-scheduled-at (ct/format-inst scheduled-at))
|
|
|
|
:else
|
|
(let [result (execute-job cfg job)]
|
|
(with-meta result
|
|
{::job job})))))
|
|
|
|
(defn- run-worker-loop
|
|
[{:keys [::rds/conn ::timeout ::queue ::wrk/tenant] :as cfg}]
|
|
(letfn [(handle-job-retry [{:keys [error delay-ms inc-by] :or {inc-by 1 delay-ms 1000} :as result}]
|
|
(let [job (-> result meta ::job)
|
|
nretry (+ (:retry-num job) inc-by)
|
|
now (ct/now)
|
|
delay (->> (iterate #(* 2 %) delay-ms) (take (max 1 nretry)) (last))]
|
|
(jobs/retry-job cfg
|
|
(:id job)
|
|
nretry
|
|
(-> (ct/plus now (ct/duration {:millis delay}))
|
|
(ct/truncate :millisecond))
|
|
(error-report error)
|
|
(if (zero? inc-by) :noop :backoff))
|
|
nil))
|
|
|
|
(handle-job-failure [{:keys [error] :as result}]
|
|
(let [job (-> result meta ::job)]
|
|
(jobs/fail cfg (:id job) (error-report error))
|
|
nil))
|
|
|
|
(handle-job-completion [result]
|
|
(let [job (-> result meta ::job)
|
|
returned (:result result)
|
|
envelope? (completion-envelope? returned)]
|
|
(jobs/complete cfg
|
|
:job-id (:id job)
|
|
:result (if envelope? (:result returned) returned)
|
|
:resource-id (when envelope? (:resource-id returned)))
|
|
nil))
|
|
|
|
(decode-payload [payload]
|
|
(try
|
|
(let [payload (json/decode payload)
|
|
job-id (uuid/parse (first payload))
|
|
sched-at (ct/inst (second payload))]
|
|
(if (and (uuid? job-id)
|
|
(ct/inst? sched-at))
|
|
[job-id sched-at]
|
|
(do
|
|
(l/err :hint "received unexpected payload"
|
|
:tenant tenant
|
|
:payload payload)
|
|
nil)))
|
|
(catch Throwable cause
|
|
(l/err :hint "unable to decode payload"
|
|
::l/context (cf/logging-context)
|
|
:tenant tenant
|
|
:payload payload
|
|
:length (count payload)
|
|
:cause cause))))
|
|
|
|
(process-result [{:keys [status] :as result}]
|
|
(ex/try!
|
|
(case status
|
|
"retry" (handle-job-retry result)
|
|
"failed" (handle-job-failure result)
|
|
"completed" (handle-job-completion result)
|
|
;; the job is no longer active: there is nothing to write,
|
|
;; the row is already terminal and whoever stopped it did
|
|
"interrupted" nil
|
|
(throw (IllegalArgumentException.
|
|
(str "invalid status received: '" status "'"))))))
|
|
|
|
(run-job-loop [[job-id scheduled-at]]
|
|
(loop [result (run-job cfg job-id scheduled-at)]
|
|
(when-let [cause (some-> result process-result)]
|
|
(if (or (db/connection-error? cause)
|
|
(db/serialization-error? cause))
|
|
(do
|
|
(l/wrn :hint "database exeption on processing job result (retrying in some instants)"
|
|
:tenant tenant
|
|
:cause cause)
|
|
(px/sleep timeout)
|
|
(recur result))
|
|
(l/err :hint "unhandled exception on processing job result"
|
|
::l/context (cf/logging-context)
|
|
:tenant tenant
|
|
:cause cause)))))]
|
|
|
|
(try
|
|
(let [key (wrk/queue-key tenant queue)
|
|
[_ payload] (rds/blpop conn [key] timeout)]
|
|
(some-> payload
|
|
decode-payload
|
|
run-job-loop))
|
|
|
|
(catch InterruptedException cause
|
|
(throw cause))
|
|
|
|
(catch Exception cause
|
|
(if (rds/timeout-exception? cause)
|
|
(do
|
|
(l/err :hint "redis pop operation timeout, consider increasing redis timeout (will retry in some instants)"
|
|
::l/context (cf/logging-context)
|
|
:tenant tenant
|
|
:timeout timeout
|
|
:cause cause)
|
|
(px/sleep timeout))
|
|
|
|
(l/err :hint "unhandled exception"
|
|
::l/context (cf/logging-context)
|
|
:tenant tenant
|
|
:cause cause))))))
|
|
|
|
(defn- start-thread
|
|
[{:keys [::id ::queue ::wrk/tenant] :as cfg}]
|
|
(px/thread
|
|
{:name (str "penpot/job-runner/" id)}
|
|
(l/inf :hint "started" :id id :tenant tenant :queue queue)
|
|
|
|
(let [rconn (rds/connect cfg)]
|
|
(try
|
|
(loop [cfg (-> cfg
|
|
(assoc ::rds/conn rconn)
|
|
(assoc ::timeout (ct/duration "5s")))]
|
|
(when (px/interrupted?)
|
|
(throw (InterruptedException. "interrupted")))
|
|
|
|
(run-worker-loop cfg)
|
|
(recur cfg))
|
|
|
|
(catch InterruptedException _
|
|
(l/dbg :hint "interrupted"
|
|
:id id
|
|
:tenant tenant
|
|
:queue queue))
|
|
(catch Throwable cause
|
|
(l/err :hint "unexpected exception"
|
|
::l/context (cf/logging-context)
|
|
:id id
|
|
:tenant tenant
|
|
:queue queue
|
|
:cause cause))
|
|
(finally
|
|
(.close ^AutoCloseable rconn)
|
|
(l/inf :hint "terminated"
|
|
:id id
|
|
:tenant tenant
|
|
:queue queue))))))
|
|
|
|
(def ^:private schema:params
|
|
[:map
|
|
[::wrk/parallelism {:optional true} ::sm/int]
|
|
[::wrk/queue :keyword]
|
|
[::wrk/tenant ::sm/text]
|
|
::jobs/defs
|
|
::mtx/metrics
|
|
::db/pool
|
|
::rds/client
|
|
;; job events of a profile job are published on the profile topic
|
|
::mbus/msgbus])
|
|
|
|
(defmethod ig/assert-key ::wrk/runner
|
|
[_ params]
|
|
(assert (sm/check schema:params params)))
|
|
|
|
(defmethod ig/expand-key ::wrk/runner
|
|
[k v]
|
|
{k (merge {::wrk/parallelism 1} (d/without-nils v))})
|
|
|
|
(defmethod ig/init-key ::wrk/runner
|
|
[_ {:keys [::db/pool ::wrk/queue ::wrk/parallelism ::wrk/tenant] :as cfg}]
|
|
(let [queue (d/name queue)
|
|
cfg (assoc cfg ::queue queue)]
|
|
(if (db/read-only? pool)
|
|
(l/wrn :hint "not started (db is read-only)" :tenant tenant :queue queue :parallelism parallelism)
|
|
(doall
|
|
(->> (range parallelism)
|
|
(map #(assoc cfg ::id (str queue "/" %)))
|
|
(map start-thread))))))
|
|
|
|
(defmethod ig/halt-key! ::wrk/runner
|
|
[_ threads]
|
|
(run! px/interrupt! threads))
|