From dd6cc835ef2b97408895df6424fcba54d047024f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 12 Oct 2026 13:42:04 +0000 Subject: [PATCH] :tada: Add graceful instance drain via /readyz Add a per-process availability flag (app.health) that /readyz reports as 503 OFFLINE while offline, so the load balancer drains the instance and in-flight requests finish untouched. Operator surface is app.srepl.main/set-offline and app.srepl.main/set-online; a restart comes back online. Covered by backend-tests.health-test; HA docs updated. AI-assisted-by: muse-spark-1.3-contributor --- .serena/memories/backend/core.md | 1 + .serena/memories/prod-infra/core.md | 4 ++ backend/src/app/health.clj | 45 ++++++++++++ backend/src/app/http/debug.clj | 27 +++++--- backend/src/app/srepl/main.clj | 25 +++++++ backend/test/backend_tests/health_test.clj | 81 ++++++++++++++++++++++ docs/technical-guide/configuration.md | 10 +++ 7 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 backend/src/app/health.clj create mode 100644 backend/test/backend_tests/health_test.clj diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index ea34c6beda..896e99c60a 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -23,6 +23,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m - `app.worker`: task execution/cron plumbing. - `app.main`: Integrant system map and component wiring. - `app.config`: `PENPOT_*` env config and feature flags. +- `app.health`: per-process availability flag; `/readyz` answers 503 while offline. Flipped from `app.srepl.main` (`set-offline` / `set-online`). - `app.srepl.*`: development REPL helpers for manual backend operations (data inspection, migration helpers, one-off admin tasks). - `app.nitrate`, `app.rpc.commands.nitrate`, and `app.rpc.management.nitrate`: external Nitrate subscription/organization integration, gated by the `:nitrate` feature flag and shared-key HTTP calls. diff --git a/.serena/memories/prod-infra/core.md b/.serena/memories/prod-infra/core.md index a3a9f60de4..2af6055cf7 100644 --- a/.serena/memories/prod-infra/core.md +++ b/.serena/memories/prod-infra/core.md @@ -27,6 +27,10 @@ Two known race patterns survive multi-backend operation: Penpot in production lives with both: horizontal-scale deployments accept "exactly-once" as "essentially-once for idempotent operations." Devenv parallel instances handle it by running workers only on ws0 (see `mem:devenv/core`). +## Draining one instance + +Each backend has a per-process availability flag (`app.health`, a global atom). While offline, `/readyz` answers `503 OFFLINE` without querying the database, so the load balancer removes that instance from rotation while requests in flight still finish. Operator surface: `app.srepl.main/set-offline` / `set-online`; a restart comes back online. In-flight worker tasks are safe: tasks claimed as `scheduled` return to `new` after 5 minutes via `sql:reschedule-lost` and another instance picks them up. + ## See also - Devenv composition and the ws0-only worker placement: `mem:devenv/core`. diff --git a/backend/src/app/health.clj b/backend/src/app/health.clj new file mode 100644 index 0000000000..aa414a6348 --- /dev/null +++ b/backend/src/app/health.clj @@ -0,0 +1,45 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.health + "Instance availability state. + + A single flag tells whether this backend instance is willing to receive + traffic. The readiness endpoint (`/readyz`) reports the instance as + unavailable while the flag is off, so the load balancer removes it from + rotation. Requests already in flight are still served. + + The state is a plain global atom and not an Integrant component because + it must also exist on a bare REPL session, which is where the srepl + helpers change it." + (:require + [app.common.logging :as l])) + +(defonce state + (atom true)) + +(defn online? + "Returns true when this instance is able to receive traffic." + [] + @state) + +(defn set-offline + "Marks the instance as offline. The `/readyz` endpoint answers 503 until + `set-online` is called again. + + Use it to drain the instance from the load balancer before a restart: + the load balancer stops sending new requests and the requests in flight + finish on their own." + [] + (l/inf :hint "instance set offline" :endpoint "/readyz") + (swap! state (constantly false))) + +(defn set-online + "Marks the instance as online again, so the load balancer sends it + traffic." + [] + (l/inf :hint "instance set online" :endpoint "/readyz") + (swap! state (constantly true))) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 69a8366cdc..27b23a74a4 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -24,6 +24,7 @@ [app.config :as cf] [app.db :as db] [app.features.file-migrations :as feat.fmig] + [app.health :as health] [app.http.session :as session] [app.rpc.commands.auth :as auth] [app.rpc.commands.files-create :refer [create-file]] @@ -580,17 +581,23 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn health-handler - "Mainly a task that performs a health check." + "Mainly a task that performs a health check. When the instance is marked + as offline (see `app.health`), it reports the instance as unavailable + without querying the database, so the load balancer drains it while the + requests in flight are still served." [{:keys [::db/pool]} _] - (try - (db/exec-one! pool ["select count(*) as count from server_prop;"]) - {::yres/status 200 - ::yres/body "OK"} - (catch Throwable cause - (l/warn :hint "unable to execute query on health handler" - :cause cause) - {::yres/status 503 - ::yres/body "KO"}))) + (if (health/online?) + (try + (db/exec-one! pool ["select count(*) as count from server_prop;"]) + {::yres/status 200 + ::yres/body "OK"} + (catch Throwable cause + (l/warn :hint "unable to execute query on health handler" + :cause cause) + {::yres/status 503 + ::yres/body "KO"})) + {::yres/status 503 + ::yres/body "OFFLINE"})) (defn changelog-handler [_ _] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 6f497e9755..e72087fcfa 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -25,6 +25,7 @@ [app.db.sql :as-alias sql] [app.features.fdata :as fdata] [app.features.file-snapshots :as fsnap] + [app.health :as health] [app.http.session :as session] [app.loggers.audit :as audit] [app.msgbus :as mbus] @@ -95,6 +96,30 @@ profile (profile/get-profile-by-email conn email)] (#'auth/send-email-verification! cfg profile))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; INSTANCE AVAILABILITY +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn set-offline + "Take this instance out of the load balancer. + + The `/readyz` endpoint starts answering 503, so the load balancer + removes the instance from rotation. Requests already in flight are + still served; wait for the load balancer to notice and for the traffic + to drop before restarting the process. + + A restarted process comes back online on its own." + [] + (health/set-offline)) + +(defn set-online + "Put this instance back into the load balancer rotation. + + Only needed to undo a `set-offline` without a restart; a restarted + process starts online." + [] + (health/set-online)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; PROFILES MANAGEMENT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/test/backend_tests/health_test.clj b/backend/test/backend_tests/health_test.clj new file mode 100644 index 0000000000..c8e0c56bcc --- /dev/null +++ b/backend/test/backend_tests/health_test.clj @@ -0,0 +1,81 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.health-test + (:require + [app.common.exceptions :as ex] + [app.db :as db] + [app.health :as health] + [app.http.debug :as debug] + [app.srepl.main :as srepl] + [backend-tests.helpers :as th] + [clojure.test :as t] + [yetti.response :as-alias yres])) + +(t/use-fixtures :once th/state-init) + +(defn- restore-availability + "The availability flag is a global atom shared by the whole test JVM, so + every test leaves the instance online." + [next] + (try + (next) + (finally + (health/set-online)))) + +(t/use-fixtures :each (th/serial th/database-reset restore-availability)) + +;; ---------------------------------------------------------------- +;; Tests: app.health state +;; ---------------------------------------------------------------- + +(t/deftest instance-is-online-by-default + (t/is (true? (health/online?)))) + +(t/deftest set-offline-and-set-online-toggle-the-flag + (health/set-offline) + (t/is (false? (health/online?))) + + (health/set-online) + (t/is (true? (health/online?)))) + +;; ---------------------------------------------------------------- +;; Tests: /readyz handler +;; ---------------------------------------------------------------- + +(t/deftest readyz-reports-ok-when-instance-is-online + (let [response (debug/health-handler th/*system* {})] + (t/is (= 200 (::yres/status response))) + (t/is (= "OK" (::yres/body response))))) + +(t/deftest readyz-reports-unavailable-when-instance-is-offline + (health/set-offline) + (let [response (debug/health-handler th/*system* {})] + (t/is (= 503 (::yres/status response))) + (t/is (= "OFFLINE" (::yres/body response))))) + +(t/deftest readyz-does-not-query-the-database-when-instance-is-offline + (health/set-offline) + (with-redefs [db/exec-one! (fn [_ _] + (throw (ex-info "database unreachable" {})))] + (let [response (debug/health-handler th/*system* {})] + (t/is (= 503 (::yres/status response))) + (t/is (= "OFFLINE" (::yres/body response)))))) + +;; ---------------------------------------------------------------- +;; Tests: srepl helpers (the operator entry points) +;; ---------------------------------------------------------------- + +(t/deftest srepl-helpers-change-the-readyz-answer + (srepl/set-offline) + (let [drained (debug/health-handler th/*system* {})] + (t/is (= 503 (::yres/status drained))) + (t/is (= "OFFLINE" (::yres/body drained)))) + + (srepl/set-online) + (let [restored (debug/health-handler th/*system* {})] + (t/is (= 200 (::yres/status restored))) + (t/is (= "OK" (::yres/body restored))))) diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 1541f1d3b8..294885c897 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -447,6 +447,16 @@ In a high-availability (HA) scenario, managing the state outside of replicas is - Valkey: Penpot only needs one Valkey instance to function correctly. Due to the nature of the data it manages, replication isn't even essential. - User media storage: This should not be configured with local storage but rather with centralized storage, such as Kubernetes PVC or S3. +### Draining a backend instance + +To restart one backend without affecting ongoing clients, take it out of the +load balancer first. From the backend REPL, run `(app.srepl.main/set-offline)`. +The `/readyz` endpoint then answers `503 OFFLINE`, so the load balancer stops +sending new requests to that instance while the requests already in flight +finish normally. Wait for the traffic to drop, restart the process, and the +instance comes back online on its own. To undo the drain without a restart, +run `(app.srepl.main/set-online)`. Drain one instance at a time. + ## Backend This section enumerates the backend only configuration variables.