From 80c84a33316d495d42139f9e2693a7e1d7dd1c41 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 22 Jul 2026 14:00:35 +0200 Subject: [PATCH 01/21] :bug: Strip Authorization header when proxying asset redirects to S3 (#10777) When nginx follows a backend 307 redirect to a presigned S3 URL, it was forwarding the client's Authorization header to S3. Production S3 rejects this because it sees two auth mechanisms (presigned URL signature + Authorization header). MinIO in devenv is more lenient and ignores the extra header. Fix: add proxy_set_header Authorization "" in the @handle_redirect block. Fixes #10776 AI-assisted-by: mimo-v2.5-pro --- docker/devenv/files/nginx.conf | 1 + docker/images/files/nginx.conf.template | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index ce5566079f..3d3d70263f 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -86,6 +86,7 @@ http { set $real_mtype "$upstream_http_x_mtype"; proxy_set_header Host "$redirect_host"; + proxy_set_header Authorization ""; proxy_hide_header etag; proxy_hide_header x-amz-id-2; proxy_hide_header x-amz-request-id; diff --git a/docker/images/files/nginx.conf.template b/docker/images/files/nginx.conf.template index a27d770f82..95ddf348a1 100644 --- a/docker/images/files/nginx.conf.template +++ b/docker/images/files/nginx.conf.template @@ -94,6 +94,7 @@ http { proxy_buffering off; proxy_set_header Host "$redirect_host"; + proxy_set_header Authorization ""; proxy_hide_header etag; proxy_hide_header x-amz-id-2; proxy_hide_header x-amz-request-id; From 018d840baba28f4a009e8681e6d69f83f8f4b0f7 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 21 Jul 2026 16:41:43 +0200 Subject: [PATCH 02/21] :recycle: Refactor internal organization of system initialization Add the ability to suspend and add nrepl to the whole system AI-assisted-by: qwen3.7-plus --- backend/dev/user.clj | 10 +-- backend/scripts/_env | 1 + backend/src/app/config.clj | 2 + backend/src/app/main.clj | 97 +++++++++++++++--------- backend/src/app/srepl.clj | 118 ++++++++++++++++++++++++------ backend/src/app/srepl/binfile.clj | 6 +- backend/src/app/srepl/cli.clj | 2 +- backend/src/app/srepl/helpers.clj | 6 +- backend/src/app/srepl/main.clj | 94 ++++++++++++------------ backend/src/app/system.clj | 9 +++ 10 files changed, 227 insertions(+), 118 deletions(-) create mode 100644 backend/src/app/system.clj diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 406d0c53fd..16908f4cab 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -104,24 +104,20 @@ [] (try (main/start) - :started (catch Throwable cause (ex/print-throwable cause)))) (defn- stop [] - (main/stop) - :stopped) + (main/stop)) (defn restart [] - (stop) - (repl/refresh :after 'user/start)) + (main/restart)) (defn restart-all [] - (stop) - (repl/refresh-all :after 'user/start)) + (main/restart-all)) ;; (defn compression-bench ;; [data] diff --git a/backend/scripts/_env b/backend/scripts/_env index 58ea02ede0..d416e7bd59 100644 --- a/backend/scripts/_env +++ b/backend/scripts/_env @@ -42,6 +42,7 @@ export PENPOT_FLAGS="\ enable-smtp \ enable-prepl-server \ enable-urepl-server \ + enable-nrepl-server \ enable-rpc-climit \ enable-rpc-rlimit \ enable-quotes \ diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 3fcdfd6443..dd6904a7a2 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -253,6 +253,8 @@ [:urepl-port {:optional true} ::sm/int] [:prepl-host {:optional true} :string] [:prepl-port {:optional true} ::sm/int] + [:nrepl-host {:optional true} :string] + [:nrepl-port {:optional true} ::sm/int] [:file-data-backend {:optional true} [:enum "db" "legacy-db" "storage"]] diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index ca8fba8af1..ac42e70225 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -38,6 +38,7 @@ [app.storage.gc-deleted :as-alias sto.gc-deleted] [app.storage.gc-touched :as-alias sto.gc-touched] [app.storage.s3 :as-alias sto.s3] + [app.system :as sys] [app.util.cron] [app.worker :as-alias wrk] [app.worker.executor] @@ -45,10 +46,13 @@ [clojure.tools.namespace.repl :as repl] [cuerdas.core :as str] [integrant.core :as ig] - [nrepl.server :as nrepl] [promesa.exec :as px]) (:gen-class)) +(repl/disable-reload! (find-ns 'integrant.core)) +(repl/disable-reload! (find-ns 'app.system)) +(repl/disable-reload! (find-ns 'app.common.debug)) + (def default-metrics {:update-file-changes {::mdef/name "penpot_rpc_update_file_changes_total" @@ -444,13 +448,17 @@ ::http.client/client (ig/ref ::http.client/client) ::setup/props (ig/ref ::setup/props)} - [::srepl/urepl ::srepl/server] - {::srepl/port (cf/get :urepl-port 6062) - ::srepl/host (cf/get :urepl-host "localhost")} + ::srepl/urepl + {:port (cf/get :urepl-port 6062) + :host (cf/get :urepl-host "localhost")} - [::srepl/prepl ::srepl/server] - {::srepl/port (cf/get :prepl-port 6063) - ::srepl/host (cf/get :prepl-host "localhost")} + ::srepl/prepl + {:port (cf/get :prepl-port 6063) + :host (cf/get :prepl-host "localhost")} + + ::srepl/nrepl + {:port (cf/get :nrepl-port 6064) + :host (cf/get :nrepl-host "localhost")} ::setup/templates {} @@ -584,42 +592,70 @@ ::db/pool (ig/ref ::db/pool)}}) -(def system nil) - (defn start [] (cf/validate!) (ig/load-namespaces (merge system-config worker-config)) - (alter-var-root #'system (fn [sys] - (when sys (ig/halt! sys)) - (-> system-config - (cond-> (contains? cf/flags :backend-worker) - (merge worker-config)) - (ig/expand) - (ig/init)))) + (alter-var-root #'app.system/system + (fn [sys] + (some-> sys not-empty ig/halt!) + (-> system-config + (cond-> (contains? cf/flags :backend-worker) + (merge worker-config)) + (ig/expand) + (ig/init)))) + (l/inf :hint "welcome to penpot" :flags (str/join "," (map name cf/flags)) :worker? (contains? cf/flags :backend-worker) - :version (:full cf/version))) + :version (:full cf/version)) + :start) + +(defn resume + [] + (cf/validate!) + (ig/load-namespaces (merge system-config worker-config)) + (alter-var-root #'app.system/system + (fn [sys] + (let [config (-> system-config + (cond-> (contains? cf/flags :backend-worker) + (merge worker-config)) + (ig/expand))] + (if-let [sys (not-empty sys)] + (ig/resume config sys) + (ig/init config))))) + :resume) (defn start-custom [config] (ig/load-namespaces config) - (alter-var-root #'system (fn [sys] - (when sys (ig/halt! sys)) - (-> config - (ig/expand) - (ig/init))))) + (alter-var-root #'app.system/system + (fn [sys] + (some-> sys not-empty ig/halt!) + (-> config + (ig/expand) + (ig/init))))) (defn stop [] - (alter-var-root #'system (fn [sys] - (when sys (ig/halt! sys)) - nil))) + (alter-var-root #'app.system/system + (fn [sys] + (some-> sys not-empty ig/halt!) + {})) + :stop) + +(defn suspend + [] + (alter-var-root #'app.system/system + (fn [sys] + (some-> sys not-empty ig/suspend!) + sys)) + :suspend) + (defn restart [] - (stop) - (repl/refresh :after 'app.main/start)) + (suspend) + (repl/refresh :after 'app.main/resume)) (defn restart-all [] @@ -644,17 +680,12 @@ (if-let [sns (namespace o)] (do (require (symbol sns)) (test/test-vars [(resolve o)])) - (test/test-ns o))))) - -(repl/disable-reload! (find-ns 'integrant.core)) + (test/test-ns o))))) (defn -main [& _args] (try (let [p (promise)] - (l/inf :hint "start nrepl server" :port 6064) - (nrepl/start-server :bind "0.0.0.0" :port 6064) - (start) (deref p)) (catch Throwable cause diff --git a/backend/src/app/srepl.clj b/backend/src/app/srepl.clj index 8753efdbb1..8ac1d81e25 100644 --- a/backend/src/app/srepl.clj +++ b/backend/src/app/srepl.clj @@ -19,7 +19,8 @@ [clojure.core :as c] [clojure.core.server :as ccs] [clojure.main :as cm] - [integrant.core :as ig])) + [integrant.core :as ig] + [nrepl.server :as nrepl])) (defn- repl-init [] @@ -107,37 +108,106 @@ (finally (remove-tap tapfn)))))) -;; --- State initialization +;; --- UREPL -(defmethod ig/assert-key ::server +(defmethod ig/assert-key ::urepl [_ params] - (assert (int? (::port params)) "expected valid port") - (assert (string? (::host params)) "expected valid host")) + (assert (int? (:port params)) "expected valid port") + (assert (string? (:host params)) "expected valid host")) -(defmethod ig/expand-key ::server - [[type :as k] v] - {k (assoc v ::flag (keyword (str (name type) "-server")))}) +(defmethod ig/init-key ::urepl + [_ {:keys [:port :host] :as cfg}] + (when (contains? cf/flags :urepl-server) -(defmethod ig/init-key ::server - [[type _] {:keys [::flag ::port ::host] :as cfg}] - (when (contains? cf/flags flag) - - (l/inf :hint "initializing repl server" - :name (name type) - :port port - :host host) - - (let [accept (case type - ::prepl 'app.srepl/json-repl - ::urepl 'app.srepl/user-repl) + (l/inf :hint "init urepl server" :host host :port port) + (let [accept 'app.srepl/user-repl params {:address host :port port - :name (name type) + :name "urepl" :accept accept}] (ccs/start-server params) - (assoc params :type type)))) + "urepl"))) -(defmethod ig/halt-key! ::server +(defmethod ig/halt-key! ::urepl + [_ name] + (some-> name ccs/stop-server)) + +(defmethod ig/resume-key ::urepl + [key opts _ old-name] + (if old-name + (do + (l/inf :hint "keep urepl server") + old-name) + (ig/init-key key opts))) + +(defmethod ig/suspend-key! ::urepl + [_ name] + (l/inf :hint "keep urepl server")) + +;; --- PREPL + +(defmethod ig/assert-key ::prepl [_ params] - (some-> params :name ccs/stop-server)) + (assert (int? (:port params)) "expected valid port") + (assert (string? (:host params)) "expected valid host")) + +(defmethod ig/init-key ::prepl + [_ {:keys [:port :host] :as cfg}] + (when (contains? cf/flags :prepl-server) + + (l/inf :hint "init prepl server" :host host :port port) + (let [accept 'app.srepl/json-repl + params {:address host + :port port + :name "prepl" + :accept accept}] + + (ccs/start-server params) + "prepl"))) + + +(defmethod ig/halt-key! ::prepl + [_ name] + (some-> name ccs/stop-server)) + +(defmethod ig/resume-key ::prepl + [key opts _ old-name] + (if old-name + (do + (l/inf :hint "keep prepl server") + old-name) + (ig/init-key key opts))) + +(defmethod ig/suspend-key! ::prepl + [_ name] + (l/inf :hint "keep prepl server")) + +;; --- NREPL + +(defmethod ig/assert-key ::nrepl + [_ params] + (assert (int? (:port params)) "expected valid port") + (assert (string? (:host params)) "expected valid host")) + +(defmethod ig/init-key ::nrepl + [_ {:keys [:port :host] :as cfg}] + (when (contains? cf/flags :nrepl-server) + (l/inf :hint "init nrepl server" :host host :port port) + (nrepl/start-server :bind host :port port))) + +(defmethod ig/halt-key! ::nrepl + [_ server] + (some-> server nrepl/stop-server)) + +(defmethod ig/resume-key ::nrepl + [key opts _ old-server] + (if old-server + (do + (l/inf :hint "keep nrepl server") + old-server) + (ig/init-key key opts))) + +(defmethod ig/suspend-key! ::nrepl + [_ server] + (l/inf :hint "keep nrepl server")) diff --git a/backend/src/app/srepl/binfile.clj b/backend/src/app/srepl/binfile.clj index c1792321f7..cd5bec409e 100644 --- a/backend/src/app/srepl/binfile.clj +++ b/backend/src/app/srepl/binfile.clj @@ -8,18 +8,18 @@ (:require [app.binfile.v2 :as binfile.v2] [app.db :as db] - [app.main :as main] [app.srepl.helpers :as h] + [app.system :as sys] [cuerdas.core :as str])) (defn export-team! [team-id] (let [team-id (h/parse-uuid team-id)] - (binfile.v2/export-team! main/system team-id))) + (binfile.v2/export-team! sys/system team-id))) (defn import-team! [path & {:keys [owner rollback?] :or {rollback? true}}] - (db/tx-run! (assoc main/system ::db/rollback rollback?) + (db/tx-run! (assoc sys/system ::db/rollback rollback?) (fn [cfg] (let [team (binfile.v2/import-team! cfg path) owner (cond diff --git a/backend/src/app/srepl/cli.clj b/backend/src/app/srepl/cli.clj index 41f49a05e6..dc44047e32 100644 --- a/backend/src/app/srepl/cli.clj +++ b/backend/src/app/srepl/cli.clj @@ -29,7 +29,7 @@ (defn- get-current-system [] - (or (deref (requiring-resolve 'app.main/system)) + (or (deref (requiring-resolve 'app.system/system)) (deref (requiring-resolve 'user/system)))) (defmulti ^:private exec-command ::cmd) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 6bf9d2cdd9..658181635b 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -15,7 +15,7 @@ [app.common.time :as ct] [app.db :as db] [app.features.file-snapshots :as fsnap] - [app.main :as main])) + [app.system :as sys])) (def ^:dynamic *system* nil) @@ -37,13 +37,13 @@ (defn get-file "Get the migrated data of one file." ([id] - (get-file (or *system* main/system) id)) + (get-file (or *system* sys/system) id)) ([system id] (db/run! system bfc/get-file id))) (defn get-raw-file "Get the migrated data of one file." - ([id] (get-raw-file (or *system* main/system) id)) + ([id] (get-raw-file (or *system* sys/system) id)) ([system id] (db/run! system (fn [system] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 72792b7f5e..0268fd5d62 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -27,7 +27,6 @@ [app.features.file-snapshots :as fsnap] [app.http.session :as session] [app.loggers.audit :as audit] - [app.main :as main] [app.msgbus :as mbus] [app.rpc.commands.auth :as auth] [app.rpc.commands.files :as files] @@ -37,6 +36,7 @@ [app.rpc.commands.teams :as teams] [app.srepl.helpers :as h] [app.srepl.procs.file-repair :as procs.file-repair] + [app.system :as sys] [app.util.blob :as blob] [app.util.pointer-map :as pmap] [app.worker :as wrk] @@ -58,14 +58,14 @@ (defn print-tasks [] - (let [tasks (:app.worker/registry main/system)] + (let [tasks (:app.worker/registry sys/system)] (pp/pprint (keys tasks) :level 200))) (defn run-task! ([tname] (run-task! tname {})) ([tname params] - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task tname) (assoc ::wrk/params params))))) @@ -73,14 +73,14 @@ ([name] (schedule-task! name {})) ([name params] - (wrk/submit! (-> main/system + (wrk/submit! (-> sys/system (assoc ::wrk/task name) (assoc ::wrk/params params))))) (defn send-test-email! [destination] (assert (string? destination) "destination should be provided") - (-> main/system + (-> sys/system (assoc ::wrk/task :sendmail) (assoc ::wrk/params {:body "test email" :subject "test email" @@ -89,7 +89,7 @@ (defn resend-email-verification-email! [email] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [{:keys [::db/conn] :as cfg}] (let [email (profile/clean-email email) profile (profile/get-profile-by-email conn email)] @@ -103,7 +103,7 @@ "Mark the profile blocked and removes all the http sessiones associated with the profile-id." [email] - (some-> main/system + (some-> sys/system (db/tx-run! (fn [{:keys [::db/conn] :as system}] (when-let [profile (db/get* conn :profile @@ -117,7 +117,7 @@ "Mark the profile blocked and removes all the http sessiones associated with the profile-id." [email] - (some-> main/system + (some-> sys/system (db/tx-run! (fn [{:keys [::db/conn] :as system}] (when-let [profile (db/get* conn :profile @@ -135,7 +135,7 @@ (assert (string? email) "expected email") (assert (string? password) "expected password") - (some-> main/system + (some-> sys/system (db/tx-run! (fn [{:keys [::db/conn] :as system}] (let [password (derive-password password) @@ -156,7 +156,7 @@ :hint (str "feature '" feature "' not supported"))) (let [team-id (h/parse-uuid team-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [{:keys [::db/conn]}] (let [team (-> (db/get conn :team {:id team-id}) (update :features db/decode-pgarray #{})) @@ -175,7 +175,7 @@ :hint (str "feature '" feature "' not supported"))) (let [team-id (h/parse-uuid team-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [{:keys [::db/conn]}] (let [team (-> (db/get conn :team {:id team-id}) (update :features db/decode-pgarray #{})) @@ -216,7 +216,7 @@ :code :incorrect-level :hint (str "level '" level "' not supported"))) - (let [{:keys [::mbus/msgbus ::db/pool]} main/system + (let [{:keys [::mbus/msgbus ::db/pool]} sys/system send (fn [dest] @@ -321,7 +321,7 @@ collectable file-changes entry." [& {:keys [file-id label]}] (let [file-id (h/parse-uuid file-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [cfg] (let [file (bfc/get-file cfg file-id :realize? true)] (fsnap/create! cfg file {:label label :created-by "admin"})))))) @@ -330,7 +330,7 @@ [file-id & {:keys [label id]}] (let [file-id (h/parse-uuid file-id) snapshot-id (some-> id h/parse-uuid)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [{:keys [::db/conn] :as system}] (cond (uuid? snapshot-id) @@ -348,7 +348,7 @@ (defn list-file-snapshots! [file-id & {:as _}] (let [file-id (h/parse-uuid file-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [cfg] (->> (fsnap/get-visible-snapshots cfg file-id) (print-table [:label :id :revn :created-at :created-by])))))) @@ -356,7 +356,7 @@ (defn take-team-snapshot! [team-id & {:keys [label rollback?] :or {rollback? true}}] (let [team-id (h/parse-uuid team-id)] - (-> (assoc main/system ::db/rollback rollback?) + (-> (assoc sys/system ::db/rollback rollback?) (db/tx-run! h/take-team-snapshot! team-id label)))) (defn restore-team-snapshot! @@ -364,7 +364,7 @@ exists for all files; if is not the case, an exception is raised." [team-id label & {:keys [rollback?] :or {rollback? true}}] (let [team-id (h/parse-uuid team-id)] - (-> (assoc main/system ::db/rollback rollback?) + (-> (assoc sys/system ::db/rollback rollback?) (db/tx-run! h/restore-team-snapshot! team-id label)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -376,7 +376,7 @@ all contents of a file. Returns a list of errors." [file-id] (let [file-id (h/parse-uuid file-id)] - (db/tx-run! (assoc main/system ::db/rollback true) + (db/tx-run! (assoc sys/system ::db/rollback true) (fn [system] (let [file (bfc/get-file system file-id) libs (bfc/get-resolved-file-libraries system file)] @@ -387,7 +387,7 @@ all contents of a file. Returns a list of errors." [file-id] (let [file-id (h/parse-uuid file-id)] - (db/tx-run! (assoc main/system ::db/rollback true) + (db/tx-run! (assoc sys/system ::db/rollback true) (fn [system] (try (let [file (bfc/get-file system file-id)] @@ -405,7 +405,7 @@ (defn repair-file! "Repair the list of errors detected by validation." [file-id & {:keys [rollback?] :or {rollback? true} :as options}] - (let [system (assoc main/system ::db/rollback rollback?) + (let [system (assoc sys/system ::db/rollback rollback?) file-id (h/parse-uuid file-id) options (assoc options ::h/with-libraries? true)] (db/tx-run! system h/process-file! file-id procs.file-repair/repair-file options))) @@ -415,7 +415,7 @@ The function receives the decoded and migrated file data." [file-id update-fn & {:keys [rollback?] :or {rollback? true} :as opts}] (let [file-id (h/parse-uuid file-id)] - (db/tx-run! (assoc main/system ::db/rollback rollback?) + (db/tx-run! (assoc sys/system ::db/rollback rollback?) (fn [system] (binding [h/*system* system db/*conn* (db/get-connection system)] @@ -461,7 +461,7 @@ (when-let [[index item] (sp/ main/system + (-> sys/system (assoc ::db/rollback rollback?) (db/tx-run! (fn [system] (binding [h/*system* system @@ -506,7 +506,7 @@ (doall))] (try - (db/tx-run! main/system process-items) + (db/tx-run! sys/system process-items) ;; Await threads termination (doseq [thread threads] @@ -535,7 +535,7 @@ (defn mark-file-as-trimmed [id] (let [id (h/parse-uuid id)] - (db/tx-run! main/system (fn [cfg] + (db/tx-run! sys/system (fn [cfg] (-> (db/update! cfg :file {:has-media-trimmed true} {:id id} @@ -553,14 +553,14 @@ (let [file-id (h/parse-uuid file-id) tnow (ct/now)] - (audit/insert main/system + (audit/insert sys/system {:name "delete-file" :type "action" :props {:id file-id} :context {:triggered-by "srepl" :cause "explicit call to delete-file!"} :tracked-at tnow}) - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :file :deleted-at tnow @@ -571,7 +571,7 @@ "Mark a file and all related objects as not deleted" [file-id] (let [file-id (h/parse-uuid file-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [{:keys [::db/conn] :as system}] (when-let [file (db/get* system :file {:id file-id} @@ -593,7 +593,7 @@ (let [project-id (h/parse-uuid project-id) tnow (ct/now)] - (audit/insert main/system + (audit/insert sys/system {:name "delete-project" :type "action" :props {:id project-id} @@ -601,7 +601,7 @@ :cause "explicit call to delete-project!"} :tracked-at tnow}) - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :project :deleted-at tnow @@ -625,7 +625,7 @@ "Mark a project and all related objects as not deleted" [project-id] (let [project-id (h/parse-uuid project-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [system] (when-let [project (db/get* system :project {:id project-id} @@ -645,7 +645,7 @@ (let [team-id (h/parse-uuid team-id) tnow (ct/now)] - (audit/insert main/system + (audit/insert sys/system {:name "delete-team" :type "action" :props {:id team-id} @@ -653,7 +653,7 @@ :cause "explicit call to delete-profile!"} :tracked-at tnow}) - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :team :deleted-at tnow @@ -681,7 +681,7 @@ "Mark a team and all related objects as not deleted" [team-id] (let [team-id (h/parse-uuid team-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [system] (when-let [team (some-> (db/get* system :team {:id team-id} @@ -702,14 +702,14 @@ (let [profile-id (h/parse-uuid profile-id) tnow (ct/now)] - (audit/insert main/system + (audit/insert sys/system {:name "delete-profile" :type "action" :context {:triggered-by "srepl" :cause "explicit call to delete-profile!"} :tracked-at tnow}) - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :profile :deleted-at tnow @@ -720,7 +720,7 @@ "Mark a team and all related objects as not deleted" [profile-id] (let [profile-id (h/parse-uuid profile-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [system] (when-let [profile (some-> (db/get* system :profile {:id profile-id} @@ -793,9 +793,9 @@ (defn process-deleted-profiles-cascade [] - (->> (db/exec! main/system ["select id, deleted_at from profile where deleted_at is not null"]) + (->> (db/exec! sys/system ["select id, deleted_at from profile where deleted_at is not null"]) (run! (fn [{:keys [id deleted-at]}] - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :profile :deleted-at deleted-at @@ -803,9 +803,9 @@ (defn process-deleted-teams-cascade [] - (->> (db/exec! main/system ["select id, deleted_at from team where deleted_at is not null"]) + (->> (db/exec! sys/system ["select id, deleted_at from team where deleted_at is not null"]) (run! (fn [{:keys [id deleted-at]}] - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :team :deleted-at deleted-at @@ -813,9 +813,9 @@ (defn process-deleted-projects-cascade [] - (->> (db/exec! main/system ["select id, deleted_at from project where deleted_at is not null"]) + (->> (db/exec! sys/system ["select id, deleted_at from project where deleted_at is not null"]) (run! (fn [{:keys [id deleted-at]}] - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :project :deleted-at deleted-at @@ -823,9 +823,9 @@ (defn process-deleted-files-cascade [] - (->> (db/exec! main/system ["select id, deleted_at from file where deleted_at is not null"]) + (->> (db/exec! sys/system ["select id, deleted_at from file where deleted_at is not null"]) (run! (fn [{:keys [id deleted-at]}] - (wrk/invoke! (-> main/system + (wrk/invoke! (-> sys/system (assoc ::wrk/task :delete-object) (assoc ::wrk/params {:object :file :deleted-at deleted-at @@ -842,7 +842,7 @@ (assert (string? client-id) "expected a valid client-id") (assert (string? client-secret) "expected a valid client-secret") (assert (string? domain) "expected a valid domain") - (db/insert! main/system :sso-provider + (db/insert! sys/system :sso-provider {:id (uuid/next) :type "oidc" :client-id client-id @@ -856,7 +856,7 @@ (defn decode-session-token [token] - (session/decode-token main/system token)) + (session/decode-token sys/system token)) (defn instrument-var [var] @@ -881,7 +881,7 @@ (defn duplicate-team [team-id & {:keys [name]}] (let [team-id (h/parse-uuid team-id)] - (db/tx-run! main/system + (db/tx-run! sys/system (fn [{:keys [::db/conn] :as cfg}] (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) (let [team (-> (assoc cfg ::bfc/timestamp (ct/now)) diff --git a/backend/src/app/system.clj b/backend/src/app/system.clj new file mode 100644 index 0000000000..a424bebcf1 --- /dev/null +++ b/backend/src/app/system.clj @@ -0,0 +1,9 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.system) + +(defonce system nil) From 2344ba22a6842df6c1c96e446a130c8f49ed2a13 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 21 Jul 2026 15:47:56 +0000 Subject: [PATCH 03/21] :tada: Add error reports API and CLI tool Implement RPC methods for querying server error reports with pagination and filtering. Add CLI tool (tools/error-reports.mjs) for convenient access with table and JSON output formats. Extract profile-id from audit events and logging context for better error categorization. Build improved HREF using request path when available. AI-assisted-by: qwen3.7-plus --- .env.example | 3 + .serena/memories/scripts/error-reports.md | 228 ++++++++++++ .../resources/app/templates/error-list.tmpl | 7 +- .../app/templates/error-report.v3.tmpl | 2 +- .../app/templates/error-report.v4.tmpl | 14 +- .../app/templates/error-report.v5.tmpl | 8 +- backend/src/app/http/access_token.clj | 1 - backend/src/app/http/debug.clj | 26 +- backend/src/app/loggers/database.clj | 64 ++-- backend/src/app/main.clj | 2 +- backend/src/app/migrations.clj | 5 +- ...and-add-indexes-to-server-error-report.sql | 44 +++ backend/src/app/rpc.clj | 59 ++- backend/src/app/rpc/commands/access_token.clj | 25 ++ .../src/app/rpc/commands/error_reports.clj | 180 ++++++++++ backend/src/app/srepl.clj | 6 +- backend/src/app/srepl/main.clj | 12 +- .../backend_tests/rpc_access_tokens_test.clj | 45 +-- .../rpc_commands_error_reports_test.clj | 251 +++++++++++++ package.json | 2 + pnpm-lock.yaml | 12 + scripts/error-reports.mjs | 340 ++++++++++++++++++ 22 files changed, 1244 insertions(+), 92 deletions(-) create mode 100644 .env.example create mode 100644 .serena/memories/scripts/error-reports.md create mode 100644 backend/src/app/migrations/sql/0152-rename-version-and-add-indexes-to-server-error-report.sql create mode 100644 backend/src/app/rpc/commands/error_reports.clj create mode 100644 backend/test/backend_tests/rpc_commands_error_reports_test.clj create mode 100755 scripts/error-reports.mjs diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000..b8c352b7cd --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +# Penpot API configuration for error-reports CLI tool +PENPOT_API_URI=http://localhost:3450 +PENPOT_ACCESS_TOKEN=your-access-token-here diff --git a/.serena/memories/scripts/error-reports.md b/.serena/memories/scripts/error-reports.md new file mode 100644 index 0000000000..13ba81fece --- /dev/null +++ b/.serena/memories/scripts/error-reports.md @@ -0,0 +1,228 @@ +# Error Reports CLI Tool + +`scripts/error-reports.mjs` is a Node.js CLI tool for querying Penpot error reports via the RPC API. Provides access to error logs with filtering, pagination, and multiple output formats. + +## When to use + +- Querying error reports from the database for debugging or analysis +- Filtering errors by source, kind, tenant, or backend version +- Exporting error data in JSON or table format +- Investigating specific error reports by ID + +## Prerequisites + +- Node.js with `commander` and `dotenv` packages installed (in root `package.json`) +- Running Penpot backend with error-reports RPC endpoints +- Access token with `error-reports:read` permission + +## Configuration + +Create a `.env` file in the project root: + +```bash +PENPOT_API_URI=http://localhost:3450 +PENPOT_ACCESS_TOKEN= +``` + +Grant the required permission to your access token: + +```sql +UPDATE access_token +SET perms = ARRAY['error-reports:read']::text[], + updated_at = now() +WHERE id = ''; +``` + +## Usage + +```bash +./scripts/error-reports.mjs [options] +``` + +### Commands + +#### `list` - List error reports with pagination and filters + +```bash +./scripts/error-reports.mjs list [options] +``` + +**Options:** + +| Flag | Description | Default | +|------|-------------|---------| +| `-l, --limit ` | Max items per page (max: 200) | `50` | +| `--since ` | ISO timestamp (fetch errors before this date) | — | +| `--since-id ` | Fetch errors before this ID (cursor pagination) | — | +| `-s, --source ` | Filter by source (see source names below) | — | +| `-p, --profile-id ` | Filter by profile ID | — | +| `-k, --kind ` | Filter by kind (string) | — | +| `-t, --tenant ` | Filter by tenant (string) | — | +| `--version ` | Filter by version | — | +| `--hint ` | Filter by hint (ILIKE match) | — | +| `-a, --all` | Fetch all pages automatically | `false` | +| `-f, --format ` | Output format: `json` or `table` | `json` | +| `--env ` | Custom .env file path | `.env` | +| `-h, --help` | Show help message | — | + +#### `get` - Get a single error report by ID + +```bash +./scripts/error-reports.mjs get [options] +``` + +**Options:** + +| Flag | Description | Required | +|------|-------------|----------| +| `--id ` | Error report ID | Yes (or --error-id) | +| `--error-id ` | Error report error-id | Yes (or --id) | +| `-f, --format ` | Output format: `json` or `table` | No (default: `json`) | +| `--env ` | Custom .env file path | No (default: `.env`) | +| `-h, --help` | Show help message | No | + +## Source Names + +The `--source` filter accepts these values: + +- `logging` +- `audit-log` +- `rlimit` + +## Examples + +### List recent errors +```bash +./scripts/error-reports.mjs list --limit 10 +``` + +### Filter by source +```bash +./scripts/error-reports.mjs list --source audit-log --limit 20 +``` + +### Filter by kind +```bash +./scripts/error-reports.mjs list --kind exception-page +``` + +### Filter by tenant +```bash +./scripts/error-reports.mjs list --tenant production +``` + +### Filter by version +```bash +./scripts/error-reports.mjs list --version 2.1.0 +``` + +### Search by hint (partial match) +```bash +./scripts/error-reports.mjs list --hint "NullPointerException" +``` + +### Fetch all errors with pagination +```bash +./scripts/error-reports.mjs list --all --format json +``` + +### Get specific error by ID +```bash +./scripts/error-reports.mjs get --id 550e8400-e29b-41d4-a716-446655440000 +``` + +### Output as JSON +```bash +./scripts/error-reports.mjs list --limit 5 --format json +``` + +### Combine filters +```bash +./scripts/error-reports.mjs list --source audit-log --kind exception-page --tenant production --limit 50 +``` + +## Output Formats + +### Table (default) +Human-readable table format for terminal display: + +``` +Found 15 error reports + +ID | Created At | Source | Profile ID | Kind | Hint +-------------------------------------+---------------------+-----------+--------------------------------------+----------------+------------------ +550e8400-e29b-41d4-a716-446655440000 | 2026-01-20 10:30:00 | audit-log | e98bb95f-573d-8137-8008-252580aa456d | exception-page | Error description +abc12345-e29b-41d4-a716-446655440001 | 2026-01-20 10:29:00 | logging | - | error | Another error that is very long and ne... + +More results: use --since 2026-01-20T10:28:00Z --since-id def45678-e29b-41d4-a716-446655440002 +``` + +### JSON +Returns structured JSON with error details and pagination metadata: + +```json +{ + "items": [ + { + "id": "uuid", + "createdAt": "2026-01-20T10:30:00Z", + "source": "audit-log", + "profileId": "e98bb95f-573d-8137-8008-252580aa456d", + "kind": "exception-page", + "tenant": "production", + "version": "2.1.0", + "hint": "Error description" + } + ], + "nextSince": "2026-01-20T10:29:00Z", + "nextId": "next-uuid" +} +``` + +## Pagination + +### Manual pagination +Use `--since` and `--since-id` with values from `nextSince` and `nextId` in the response: + +```bash +./scripts/error-reports.mjs list --limit 50 +# Use nextSince and nextId from response +./scripts/error-reports.mjs list --limit 50 --since "2026-01-20T10:29:00Z" --since-id "next-uuid" +``` + +### Automatic pagination +Use `--all` to fetch all pages automatically: + +```bash +./scripts/error-reports.mjs list --all +``` + +## Key principles + +- **Authentication required** - Uses access token with `error-reports:read` permission +- **API endpoint configurable** - Set via `PENPOT_API_URI` in `.env` file +- **Table is default format** - Use `--format json` for structured JSON output +- **Pagination is automatic with --all** - Fetches all pages without manual cursor management +- **Filters are combinable** - All filter options can be used together +- **Both flag formats supported** - `--option=value` and `--option value` both work + +## Error handling + +The tool provides helpful error messages for common issues: + +- **Missing configuration**: Shows setup instructions for `.env` file +- **Authentication errors (401)**: Indicates invalid or expired token +- **Authorization errors (403)**: Indicates missing `error-reports:read` permission +- **RPC errors**: Displays error code and message from the API + +## Integration with other scripts + +- **jq**: Pipe JSON output to `jq` for further processing + ```bash + ./scripts/error-reports.mjs list --all --format json | jq '.items[] | {id, kind, hint}' + ``` +- **grep/search**: Filter output by specific patterns +- **Redirect**: Save output to files for analysis + ```bash + ./scripts/error-reports.mjs list --all --format json > errors.json + ``` diff --git a/backend/resources/app/templates/error-list.tmpl b/backend/resources/app/templates/error-list.tmpl index 043a29034e..2f6f9da735 100644 --- a/backend/resources/app/templates/error-list.tmpl +++ b/backend/resources/app/templates/error-list.tmpl @@ -10,9 +10,10 @@ penpot - error list [BACK]

Error reports (last 300)

- [BACKEND ERRORS] - [FRONTEND ERRORS] - [RLIMIT REPORTS] + [ALL ERRORS] + [BACKEND ERRORS] + [FRONTEND ERRORS] + [RLIMIT REPORTS]
diff --git a/backend/resources/app/templates/error-report.v3.tmpl b/backend/resources/app/templates/error-report.v3.tmpl index a3eddfaa86..63efeba506 100644 --- a/backend/resources/app/templates/error-report.v3.tmpl +++ b/backend/resources/app/templates/error-report.v3.tmpl @@ -6,7 +6,7 @@ Report: {{hint|abbreviate:150}} - {{id}} - Penpot Error Report (v3) {% block content %}