penpot/backend/src/app/config.clj
Álvaro Tejero Cantero 4cb0e452ff
Merge branch 'develop' into iop-vc-preview
Bring iop-vc-preview up to develop b438676d6c.

Conflicts:
- migrations.clj: develop's 0157-migrate-legacy-file-data and
  0158-del-forbidden-file-library-rels keep develop's order, and the
  branch's 0800-0808 follow them. No file names collide, so nothing
  is renumbered.
- files_update.clj: develop's save-path rewrite wins (commit-by-commit
  rejection, page-scoped validation, the file carrying ::changes and
  ::rejected). The branch's op-log save is ported onto it instead of
  the earlier [file op-changes media-index] triple:
  process-changes-and-validate attaches the media fix-up remap as
  ::media-index, and update-file* builds the op-log vector from the
  accepted changes (remapped) behind the repair changes. A branch
  save persists only when develop's saved? holds, and its audit and
  log outcome is "rejected" when the first commit is refused.
  The tpoint threading, persist-branch-file!, the public
  invalidate-caches!, and the branch audit props are kept.
- Repair and validation timing tests: take develop's calls, since
  process-changes-and-validate returns a file again.
- binfile_test.clj: keep the branch import-overwrite test and
  develop's library relation tests.
- dashboard.cljs, dashboard_test.cljs: union of requires and tests.
- viewer.cljs: develop's optional page-id plus the branch's pr-id.
- left_header.cljs: develop's raw-svg logo, recent-files href and
  project link, inside the branch's wrapper with the review badge,
  branch title and branch bar. The unused deprecated-icon require
  and nav-to-project go.
- en.po, es.po: three-way merge by msgid. Every entry of both sides
  is kept, develop's removals of a duplicate errors.save-retrying and
  of integrations.mcp-server.title.beta stand, and labels.retry
  carries develop's references plus the branch's.

Adaptations to develop:
- email.clj, files_pull_request.clj: the review request email joins
  develop's template registry and goes through send-template, as the
  comment emails it mirrors now do. The registry coverage test lists
  it, and the rollback test of the pull request commands injects its
  failure into send-template, the function the create path now calls.
- rpc_file_pull_request_test.clj: pr-flags is read at test time
  instead of at namespace load. The loaded value held the production
  defaults, which now enable the admin console (#12475), and it
  replaced th/state-init's test flags, so creating a profile called
  nitrate and failed with :nitrate-not-configured.

AI-assisted-by: claude-opus-5-5
Signed-off-by: Álvaro Tejero Cantero <alvorithm@teje.ro>
2026-11-11 17:51:20 +01:00

552 lines
21 KiB
Clojure

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.config
(:refer-clojure :exclude [get])
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.flags :as flags]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.common.version :as v]
[app.util.overrides]
[clojure.core :as c]
[clojure.java.io :as io]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[environ.core :refer [env]]
[integrant.core :as ig]))
(defmethod ig/init-key :default
[_ data]
(d/without-nils data))
(defmethod ig/expand-key :default
[k v]
{k (if (map? v)
(d/without-nils v)
v)})
(def default
{:database-uri "postgresql://postgres/penpot"
:database-username "penpot"
:database-password "penpot"
:default-blob-version 4
:rpc-rlimit-config "resources/rlimit.edn"
:rpc-climit-config "resources/climit.edn"
:auto-file-snapshot-every 5
:auto-file-snapshot-timeout "3h"
;; Branching size gates. Derived from measurement rather than taste:
;; on a 21,169-shape design system a merge costs about 6 s, so the
;; shape limit is the count at which an operation would approach a
;; 30 s timeout, halved. See `app.rpc.commands.files-branch`.
:branching-max-shapes 50000
:branching-max-pages 500
:branching-max-oplog-changes 100000
:branch-merge-policies-file "resources/app/branch-merge-policies.edn"
:public-uri "http://localhost:3449"
:host "localhost"
:tenant "default"
;; The SaaS host also sets penpotIsSaas in the browser. Keep this server
;; value in sync so Admin Console can query the same deployment type.
:is-saas false
:redis-uri "redis://redis/0"
:file-data-backend "db"
:objects-storage-backend "fs"
:objects-storage-fs-directory "assets"
:auth-token-cookie-name "auth-token"
:assets-path "/internal/assets/"
:smtp-default-reply-to "Penpot <no-reply@example.com>"
:smtp-default-from "Penpot <no-reply@example.com>"
:profile-complaint-max-age (ct/duration {:days 7})
:profile-complaint-threshold 2
:profile-bounce-max-age (ct/duration {:days 7})
:profile-bounce-threshold 10
:login-lockout-max-attempts 5
:login-lockout-window (ct/duration "15m")
:telemetry-uri "https://telemetry.penpot.app/"
:jobs-lease (ct/duration {:minutes 30})
:jobs-retention (ct/duration {:days 7})
:jobs-user-ttl (ct/duration {:days 7})
:jobs-request-timeout (ct/duration {:minutes 2})
:media-max-file-size (* 1024 1024 30) ; 30MiB
:font-max-file-size (* 1024 1024 30) ; 30MiB
:font-process-mem 512 ;; 512 MiB address space ceiling
:font-process-cpu 30 ;; 30 seconds CPU time
:font-process-timeout 60 ;; 60 seconds wall-clock
:ldap-user-query "(|(uid=:username)(mail=:username))"
:ldap-attrs-username "uid"
:ldap-attrs-email "mail"
:ldap-attrs-fullname "cn"
;; a server prop key where initial project is stored.
:initial-project-skey "initial-project"
;; time to avoid email sending after profile modification
:email-verify-threshold "15m"
:quotes-upload-sessions-per-profile 5
:quotes-upload-chunks-per-session 20
:quotes-export-jobs-per-profile 10
:quotes-import-jobs-per-profile 10
:upload-max-chunk-size (* 1024 1024 30) ; 30MiB
;; SSRF protection
:ssrf-allowed-hosts #{}
:ssrf-extra-blocked-cidrs #{}})
(def schema:tenant
"Tenant identifier: hostname-label-style, lowercase (letters, digits
and hyphens). The value is normalized to lowercase when the
configuration is read, so an operator writing `PENPOT_TENANT=ACME` and
one writing `acme` name the same tenant: the value is compared as an
exact string in the keys and rows it lands in, and one uppercase
letter would otherwise split one tenant into two.
It is interpolated into Redis keys separated by dots (msgbus
topics, rate-limit buckets, exporter job keys, which are also matched
with a glob pattern), so `.` and whitespace are rejected at startup
because one tenant would then be able to read or overwrite another's
keys.
`%`, `_` and `:` are rejected too, and the reason they were added is
gone: the job queue used to be filtered with a LIKE over a
`<tenant>:<queue>` prefix and now has a `tenant` column. They are kept
because narrowing the rule is a startup contract change with no real
tenant asking for it, and a tenant that has one would have to be
renamed. Tenants using those characters must be renamed before
upgrading."
[:re #"^[a-z0-9-]+$"])
(defn- normalize-tenant
"Lowercase the configured tenant before it is validated, so
`PENPOT_TENANT=ACME` and `acme` mean the same tenant everywhere."
[config]
(cond-> config
(string? (:tenant config)) (update :tenant str/lower)))
(def schema:config
(do #_sm/optional-keys
[:map {:title "config"}
[:flags {:optional true} [::sm/set :string]]
[:admins {:optional true} [::sm/set ::sm/email]]
[:secret-key {:optional true} :string]
[:tenant {:optional false} schema:tenant]
[:is-saas ::sm/boolean]
[:public-uri {:optional false} ::sm/uri]
[:host {:optional false} :string]
[:http-server-port {:optional true} ::sm/int]
[:http-server-host {:optional true} :string]
[:http-server-max-body-size {:optional true} ::sm/int]
[:http-server-io-threads {:optional true} ::sm/int]
[:http-server-max-worker-threads {:optional true} ::sm/int]
;; Explicit CORS allowlist used when the :cors flag is enabled.
;; Configured via PENPOT_ALLOWED_ORIGINS as a comma/whitespace
;; separated list of origins (e.g. "https://plugins.example.com").
[:allowed-origins {:optional true} [::sm/set :string]]
[:exporter-shared-key {:optional true} :string]
[:admin-console-shared-key {:optional true} :string]
[:nexus-shared-key {:optional true} :string]
[:media-processor-shared-key {:optional true} :string]
[:management-api-key {:optional true} :string]
[:telemetry-uri {:optional true} :string]
[:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
[:auto-file-snapshot-every {:optional true} ::sm/int]
[:auto-file-snapshot-timeout {:optional true} ::ct/duration]
[:branching-max-shapes {:optional true} ::sm/int]
[:branching-max-pages {:optional true} ::sm/int]
[:branching-max-oplog-changes {:optional true} ::sm/int]
;; The branch-merge policy switch file: a commented EDN map that
;; picks one alternative per policy. Re-read on mtime change, so it
;; can be edited while the backend runs. See
;; `app.rpc.commands.files-branch-policies`.
[:branch-merge-policies-file {:optional true} ::fs/path]
[:media-max-file-size {:optional true} ::sm/int]
[:font-max-file-size {:optional true} ::sm/int]
;; Font processing resource limits (PENPOT_FONT_PROCESS_*)
[:font-process-mem {:optional true} ::sm/int]
[:font-process-cpu {:optional true} ::sm/int]
[:font-process-timeout {:optional true} ::sm/int]
;; ImageMagick resource limits (PENPOT_IMAGEMAGICK_*)
[:imagemagick-thread-limit {:optional true} :string]
[:imagemagick-memory-limit {:optional true} :string]
[:imagemagick-map-limit {:optional true} :string]
[:imagemagick-area-limit {:optional true} :string]
[:imagemagick-disk-limit {:optional true} :string]
[:imagemagick-time-limit {:optional true} :string]
[:imagemagick-width-limit {:optional true} :string]
[:imagemagick-height-limit {:optional true} :string]
[:media-processing-service-uri {:optional true} ::sm/uri]
[:media-processing-service-timeout {:optional true} ::sm/int]
;; Binfile import limits (PENPOT_BINFILE_IMPORT_*)
[:binfile-import-max-binary-entry-size {:optional true} ::sm/int]
[:binfile-import-max-text-entry-size {:optional true} ::sm/int]
[:binfile-import-max-text-total-size {:optional true} ::sm/int]
[:binfile-import-max-zip-entries {:optional true} ::sm/int]
;; Max serialized size of profile props in bytes (default 2 MiB)
[:profile-props-max-size {:optional true} ::sm/int]
[:login-lockout-max-attempts {:optional true} ::sm/int]
[:login-lockout-window {:optional true} ::ct/duration]
[:deletion-delay {:optional true} ::ct/duration]
[:file-clean-delay {:optional true} ::ct/duration]
[:jobs-lease {:optional true} ::ct/duration]
[:jobs-retention {:optional true} ::ct/duration]
[:jobs-user-ttl {:optional true} ::ct/duration]
[:jobs-request-timeout {:optional true} ::ct/duration]
[:telemetry-enabled {:optional true} ::sm/boolean]
[:default-blob-version {:optional true} ::sm/int]
[:allow-demo-users {:optional true} ::sm/boolean]
[:error-report-webhook {:optional true} :string]
[:user-feedback-destination {:optional true} :string]
[:mcp-feedback-email {:optional true} ::sm/email]
[:default-rpc-rlimit {:optional true} [::sm/vec :string]]
[:rpc-rlimit-config {:optional true} ::fs/path]
[:rpc-climit-config {:optional true} ::fs/path]
[:nexus-uri {:optional true} ::sm/uri]
[:audit-log-http-handler-concurrency {:optional true} ::sm/int]
[:default-executor-parallelism {:optional true} ::sm/int] ;; REVIEW
[:worker-default-parallelism {:optional true} ::sm/int]
[:worker-webhook-parallelism {:optional true} ::sm/int]
[:worker-cron-parallelism {:optional true} ::sm/int]
[:worker-binfile-parallelism {:optional true} ::sm/int]
[:database-password {:optional true} [:maybe :string]]
[:database-uri {:optional true} ::sm/uri]
[:database-username {:optional true} [:maybe :string]]
[:database-readonly {:optional true} ::sm/boolean]
[:database-min-pool-size {:optional true} ::sm/int]
[:database-max-pool-size {:optional true} ::sm/int]
[:quotes-teams-per-profile {:optional true} ::sm/int]
[:quotes-access-tokens-per-profile {:optional true} ::sm/int]
[:quotes-projects-per-team {:optional true} ::sm/int]
[:quotes-invitations-per-team {:optional true} ::sm/int]
[:quotes-profiles-per-team {:optional true} ::sm/int]
[:quotes-files-per-project {:optional true} ::sm/int]
[:quotes-files-per-team {:optional true} ::sm/int]
[:quotes-font-variants-per-team {:optional true} ::sm/int]
[:quotes-comment-threads-per-file {:optional true} ::sm/int]
[:quotes-comments-per-file {:optional true} ::sm/int]
[:quotes-export-jobs-per-profile {:optional true} ::sm/int]
[:quotes-import-jobs-per-profile {:optional true} ::sm/int]
[:quotes-snapshots-per-file {:optional true} ::sm/int]
[:quotes-snapshots-per-team {:optional true} ::sm/int]
[:quotes-branches-per-file {:optional true} ::sm/int]
[:quotes-branches-per-team {:optional true} ::sm/int]
[:quotes-pull-requests-per-team {:optional true} ::sm/int]
[:quotes-team-access-requests-per-team {:optional true} ::sm/int]
[:quotes-team-access-requests-per-requester {:optional true} ::sm/int]
[:quotes-upload-sessions-per-profile {:optional true} ::sm/int]
[:quotes-upload-chunks-per-session {:optional true} ::sm/int]
[:upload-max-chunk-size {:optional true} ::sm/int]
[:quotes-media-storage-bytes-per-team {:optional true} ::sm/int]
[:auth-token-cookie-name {:optional true} :string]
[:auth-token-cookie-max-age {:optional true} ::ct/duration]
[:auth-token-cookie-max-age-absolute {:optional true} ::ct/duration]
[:registration-domain-whitelist {:optional true} [::sm/set :string]]
[:email-verify-threshold {:optional true} ::ct/duration]
[:github-client-id {:optional true} :string]
[:github-client-secret {:optional true} :string]
[:gitlab-base-uri {:optional true} :string]
[:gitlab-client-id {:optional true} :string]
[:gitlab-client-secret {:optional true} :string]
[:google-client-id {:optional true} :string]
[:google-client-secret {:optional true} :string]
[:oidc-client-id {:optional true} :string]
[:oidc-user-info-source {:optional true} [:enum "auto" "userinfo" "token"]]
[:oidc-client-secret {:optional true} :string]
[:oidc-base-uri {:optional true} :string]
[:oidc-token-uri {:optional true} :string]
[:oidc-auth-uri {:optional true} :string]
[:oidc-user-uri {:optional true} :string]
[:oidc-jwks-uri {:optional true} :string]
[:oidc-scopes {:optional true} [::sm/set :string]]
[:oidc-roles {:optional true} [::sm/set :string]]
[:oidc-roles-attr {:optional true} :string]
[:oidc-email-attr {:optional true} :string]
[:oidc-name-attr {:optional true} :string]
[:ldap-attrs-email {:optional true} :string]
[:ldap-attrs-fullname {:optional true} :string]
[:ldap-attrs-username {:optional true} :string]
[:ldap-base-dn {:optional true} :string]
[:ldap-bind-dn {:optional true} :string]
[:ldap-bind-password {:optional true} :string]
[:ldap-host {:optional true} :string]
[:ldap-port {:optional true} ::sm/int]
[:ldap-ssl {:optional true} ::sm/boolean]
[:ldap-starttls {:optional true} ::sm/boolean]
[:ldap-user-query {:optional true} :string]
[:profile-bounce-max-age {:optional true} ::ct/duration]
[:profile-bounce-threshold {:optional true} ::sm/int]
[:profile-complaint-max-age {:optional true} ::ct/duration]
[:profile-complaint-threshold {:optional true} ::sm/int]
[:redis-uri {:optional true} ::sm/uri]
[:email-domain-blacklist {:optional true} ::fs/path]
[:email-domain-whitelist {:optional true} ::fs/path]
[:smtp-default-from {:optional true} :string]
[:smtp-default-reply-to {:optional true} :string]
[:smtp-host {:optional true} :string]
[:smtp-password {:optional true} [:maybe :string]]
[:smtp-port {:optional true} ::sm/int]
[:smtp-ssl {:optional true} ::sm/boolean]
[:smtp-tls {:optional true} ::sm/boolean]
[:smtp-username {:optional true} [:maybe :string]]
[:urepl-host {:optional true} :string]
[:urepl-port {:optional true} ::sm/int]
[:prepl-host {:optional true} :string]
[:prepl-port {:optional true} ::sm/int]
[:nrepl-host {:optional true} :string]
[:nrepl-port {:optional true} ::sm/int]
[:file-data-backend {:optional true} [:enum "db" "legacy-db" "storage"]]
[:media-directory {:optional true} :string] ;; REVIEW
[:media-uri {:optional true} :string]
[:assets-path {:optional true} :string]
[:netty-io-threads {:optional true} ::sm/int]
[:admin-console-uri {:optional true} ::sm/uri]
;; DEPRECATED
[:assets-storage-backend {:optional true} :keyword]
[:storage-assets-fs-directory {:optional true} :string]
[:storage-assets-s3-bucket {:optional true} :string]
[:storage-assets-s3-region {:optional true} :keyword]
[:storage-assets-s3-endpoint {:optional true} ::sm/uri]
[:objects-storage-backend {:optional true} :keyword]
[:objects-storage-fs-directory {:optional true} :string]
[:objects-storage-s3-bucket {:optional true} :string]
[:objects-storage-s3-region {:optional true} :keyword]
[:objects-storage-s3-endpoint {:optional true} ::sm/uri]
;; Write storage_object.metadata as plain JSON instead of
;; Transit-JSON. Unset by default (Phase 1: keep writing Transit).
[:storage-metadata-as-json {:optional true} ::sm/boolean]
;; SSRF protection
[:ssrf-allowed-hosts {:optional true} [::sm/set :string]]
[:ssrf-extra-blocked-cidrs {:optional true} [::sm/set :string]]]))
(defn telemetry-excluded-host?
"Returns true when the given host belongs to the official SaaS
instances, where telemetry must be fully disabled."
[host]
(let [host (some-> host (str/lower) (str/trim))]
(and (string? host)
(not (str/blank? host))
(or (= host "penpot.dev")
(= host "penpot.app")
(str/ends-with? host ".penpot.dev")
(str/ends-with? host ".penpot.app")))))
(defn- parse-flags
[config]
(let [public-uri (c/get config :public-uri)
public-uri (some-> public-uri (u/uri))
extra-flags (cond-> #{}
;; When public-uri is http (non-localhost), disable secure cookies
(and public-uri
(= (:scheme public-uri) "http")
(not= (:host public-uri) "localhost"))
(conj :disable-secure-session-cookies)
;; When telemetry-enabled config is true, add :telemetry flag
(true? (c/get config :telemetry-enabled))
(conj :enable-telemetry))]
(flags/parse flags/default extra-flags (:flags config))))
(defn read-env
[prefix]
(let [prefix (str prefix "-")
len (count prefix)]
(reduce-kv
(fn [acc k v]
(cond-> acc
(str/starts-with? (name k) prefix)
(assoc (keyword (subs (name k) len)) v)))
{}
env)))
(def decode-config
(sm/decoder schema:config sm/string-transformer))
(def validate-config
(sm/validator schema:config))
(def explain-config
(sm/explainer schema:config))
(defn read-config
"Reads the configuration from environment variables and decodes all
known values."
[& {:keys [prefix default] :or {prefix "penpot"}}]
(->> (read-env prefix)
(merge default)
(normalize-tenant)
(decode-config)))
(def version
(v/parse (or (some-> (io/resource "version.txt")
(slurp)
(str/trim))
"%version%")))
(defonce ^:dynamic config (read-config :default default))
(defonce ^:dynamic flags (parse-flags config))
(defn validate!
"Validate the currently loaded configuration data."
[& {:keys [exit-on-error?] :or {exit-on-error? true}}]
(if (validate-config config)
true
(let [explain (explain-config config)]
(println "Error on validating configuration:")
(sm/pretty-explain explain
:variant ::sm/schemaless-explain
:message "Configuration Validation Error")
(flush)
(if exit-on-error?
(System/exit -1)
(ex/raise :type :validation
:code :config-validaton
::sm/explain explain)))))
(defn get-deletion-delay
[]
(or (c/get config :deletion-delay)
(ct/duration {:days 7})))
(defn get-file-clean-delay
[]
(or (c/get config :file-clean-delay)
(ct/duration {:days 2})))
(defn join-uri
"Join path segments onto a base URI, preserving a potential subpath
(same semantics as the frontend config). The base is normalized with
a trailing slash; segments must not start with `/` (a leading slash
would resolve against the host root and drop the subpath)."
[base & segments]
(assert (not (some #(str/starts-with? % "/") segments))
"URI segments must be relative (no leading slash)")
(str (apply u/join (u/ensure-path-slash base) segments)))
(defn get-public-uri
"Canonical public URI builder: `join-uri` over the configured
:public-uri. With no segments, returns the normalized base."
[& segments]
(apply join-uri (c/get config :public-uri) segments))
(defn get-jobs-lease
"Max time a job can run without touching modified_at (heartbeat or
progress) before the dispatcher marks it as `aborted` (system-side
terminal, never retried, reported with an error log)."
[]
(or (c/get config :jobs-lease)
(ct/duration {:minutes 30})))
(defn get-jobs-request-timeout
"Default timeout for the ephemeral request! calls (waiting for the
reply-key blpop); can be overridden per call. Any override is applied
by raising the pooled connection command timeout for the duration of
the call, which the pool restores on return."
[]
(or (c/get config :jobs-request-timeout)
(ct/duration {:minutes 2})))
(defn get-jobs-retention
"How long terminal (completed/failed/cancelled/aborted) internal job rows are
kept before the jobs GC deletes them. The legacy `task` table is not
touched by the jobs GC: while both versions run in parallel it is
cleaned by the legacy `tasks-gc` of that version."
[]
(or (c/get config :jobs-retention)
(ct/duration {:days 7})))
(defn get-jobs-user-ttl
"How long a user-facing job, and the artifact it owns, is kept before
the jobs GC deletes the row and marks the object for the storage GC.
Not the same as `:jobs-retention`, which only sweeps terminal internal
rows."
[]
(or (c/get config :jobs-user-ttl)
(ct/duration {:days 7})))
(defn get
"A configuration getter. Helps code be more testable."
([key]
(c/get config key))
([key default]
(c/get config key default)))
(defn telemetry-excluded?
"Returns true when telemetry must be fully disabled because the
public-uri host points to an official instance (penpot.dev or
penpot.app). When true, no telemetry data is collected or sent,
not even the limited newsletter report."
[]
(let [host (some-> (c/get config :public-uri) (u/uri) :host)]
(telemetry-excluded-host? host)))
(defn logging-context
[]
{:backend/version (:full version)})
;; Set value for all new threads bindings.
(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))