penpot/backend/test/backend_tests/worker_test.clj
Andrey Antukh 90a06e982b
♻️ Rework the audit-log consumers and the jobs substrate (#12701)
* ♻️ Rework audit-log consumers and jobs substrate

Fold the implementation, follow-up fixes, migrations and tests into one

code commit.

AI-assisted-by: gpt-6-luna

* 📚 Consolidate 2.20.0 upgrade runbooks

Keep the operational upgrade procedure and its supporting runbooks in a

separate documentation commit.

AI-assisted-by: gpt-6-luna

* ♻️ Remove the audit-log flag and derive persistence at boot

Persistence now follows the consumer flags instead of a separate switch.
app.config computes cf/audit-persist? once at boot from :nexus and
:admin-console, and every write site reads that value, so request paths
no longer check flags per event. With no consumer enabled no full rows
are written; telemetry shadows and webhook deliveries keep their own
flags. The frontend wire signal stays :audit-log as a derived value.

Update the 2.20 runbooks, memories and the devenv flags to match, and
migrate the tests that used to toggle :audit-log directly.

AI-assisted-by: hy4-flash-free

* 🐛 Accept error-reporting batches and precompute sink flags

push-audit-events now accepts crash batches with :error-reporting alone, so
the frontend signal from get-environment-data matches a real destination.
The reporters receive unhandled-exception events without storing any row.

All per-event sink checks read booleans computed once at boot: webhooks,
telemetry and error reporting join audit-persist? in app.config. The test
fixture and with-audit-config rebind all four values together, and the
tests that toggled those flags directly were migrated.

Address F2 of the review of the previous commit. F1 stays deferred to a
separate plan.

AI-assisted-by: hy4-flash-free

* 💄 Match the layout cljfmt expects in the flags set

The first comment of the varia set sat on its own line after the opening
delimiter, and cljfmt joins such a comment to the delimiter line, so the
Common Tests lint step failed on common/src/app/common/flags.cljc. Keep
the comment on the opening line, the same shape the login and email sets
already use.

Note in mem:scripts/ci that the backend lint lints ../common/src but does
not format-check it, so common has to be passed to scripts/ci as well.

AI-assisted-by: deepseek-v4-flash
2026-11-20 11:12:00 +01:00

116 lines
5.1 KiB
Clojure

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns backend-tests.worker-test
(:require
[app.common.schema :as sm]
[app.config :as cf]
[app.db :as db]
[app.jobs :as jobs]
[app.main :as main]
[app.metrics :as-alias mtx]
[app.msgbus :as mbus]
[app.worker :as wrk]
[app.worker.cron :as wcron]
[app.worker.dispatcher :as wdisp]
[app.worker.runner :as wrkr]
[backend-tests.helpers :as th]
[clojure.test :as t]
[integrant.core :as ig]))
;; The dispatcher's push and the runner's pop agree on the Redis list
;; only through this helper, so the test helpers use it too. That
;; keeps producer and consumer in sync by construction, but it also
;; means no other test would catch a malformed shape. This test pins
;; the physical format instead.
(t/deftest queue-key-pins-the-physical-redis-shape
(t/testing "tenant and queue compose the documented key"
(t/is (= "penpot.worker.queue:acme:default"
(wrk/queue-key "acme" :default))))
(t/testing "a string queue gives the same key as a keyword one"
(t/is (= "penpot.worker.queue:acme:default"
(wrk/queue-key "acme" "default"))))
(t/testing "hyphens travel through untouched"
(t/is (= "penpot.worker.queue:my-tenant:daily-cleanup"
(wrk/queue-key "my-tenant" :daily-cleanup)))))
;; Booting the backend asserts these integrant keys, and assert-key
;; compiles the schema first: a ref with no registered schema crashes
;; the boot with :malli.core/invalid-schema instead of asserting. The
;; component schemas lean on the shared registry (db pool, metrics,
;; redis client, msgbus, job defs), so the registry dependencies must
;; be loaded for them to compile, like the full system does.
(t/deftest worker-component-schemas-resolve-every-ref
(t/testing "cron, dispatcher and runner schemas compile"
(t/is (sm/schema? (sm/schema ::mbus/msgbus)))
(doseq [s [@#'wcron/schema:params
@#'wdisp/schema:dispatcher
@#'wrkr/schema:params]]
(t/is (sm/schema? (sm/schema s))))))
(t/deftest every-worker-consumer-component-is-known-to-the-test-system
;; The test system drops the consumer loops by key, so a new one must be
;; added there too or it drains audit_log in the background. Deriving the
;; list from the config (every worker key named `/consumer`) makes a
;; third consumer fail here, not as a flaky test elsewhere.
(let [keys (->> (keys main/worker-config)
(filter #(and (keyword? %) (= "consumer" (name %))))
(into #{}))]
(t/is (= #{:app.loggers.audit.consumers.nexus/consumer
:app.loggers.audit.consumers.admin-console/consumer}
keys)
(str "update this set and helpers/init-system's dissoc: " keys))))
(t/deftest audit-log-gc-cron-is-gated-by-its-own-flag
(t/testing "the GC entry is present only with :audit-log-gc"
(with-redefs [cf/flags #{:audit-log-gc}]
(t/is (some #(= :audit-log-gc (:job %)) (main/worker-cron-entries))))
(th/with-audit-config #{}
(t/is (not (some #(= :audit-log-gc (:job %))
(main/worker-cron-entries)))))
(with-redefs [cf/flags #{}]
(t/is (not (some #(= :audit-log-gc (:job %))
(main/worker-cron-entries)))))))
(t/deftest the-binfile-runner-is-wired
(let [cfg (get main/worker-config [:app.main/binfile ::wrk/runner])]
(t/is (some? cfg))
(t/testing "the heavy work of a user job does not share the default runner"
(t/is (= :binfile (::wrk/queue cfg)))
(t/is (= (cf/get :worker-binfile-parallelism 1) (::wrk/parallelism cfg))))
(t/testing "a job of a profile cannot store its events without a msgbus"
(t/is (contains? cfg ::mbus/msgbus)))
(t/testing "and the runner resolves jobs, metrics and the pool"
(t/is (contains? cfg ::jobs/defs))
(t/is (contains? cfg ::mtx/metrics))
(t/is (contains? cfg ::db/pool)))))
(t/deftest the-audit-consumers-are-wired
;; Each consumer is a loop in a thread: the test system drops the two
;; components by name, so a rename that leaves `helpers.clj` behind would
;; start real loops in the background. The components live in the worker
;; config only (a web instance does not drain the audit log) and their
;; `ig/init-key`/`ig/halt-key!` methods must be registered.
(doseq [[k deps] [[:app.loggers.audit.consumers.nexus/consumer
[::db/pool :app.http.client/client
:app.setup/shared-keys ::mtx/metrics]]
[:app.loggers.audit.consumers.admin-console/consumer
[::db/pool :app.nitrate/client ::mtx/metrics]]]]
(t/testing (str k)
(let [cfg (get main/worker-config k)]
(t/is (some? cfg))
(t/is (nil? (get main/system-config k)))
(t/is (some? (get-method ig/init-key k)))
(t/is (some? (get-method ig/halt-key! k)))
(t/testing "the loop resolves its dependencies"
(doseq [dep deps]
(t/is (contains? cfg dep) (str k " misses " dep))))))))