mirror of
https://github.com/penpot/penpot.git
synced 2026-11-22 10:20:51 +00:00
* ✨ Ship telemetry shadows with a cursor consumer Send anonymized audit events through a dedicated telemetry consumer, including when Nexus or admin-console also persist full events. Keep the legacy payload and cadence of 10,000 events per chunk and three hours between passes. Nexus skips shadows to avoid archiving duplicates. Move shadow deletion to the audit GC. Delete full rows and shadows along independent cursor paths, and expire shadows after seven days even when they have not been sent. Run the deletion paths in parallel with bounded batches, separate time budgets and independent failure handling. Replace the shared consumer catalog with transport definitions and one generic pass runner. Keep the legacy telemetry report and newsletter in the scheduled task, and update the tests, memories and rollout runbooks. Addresses F1 of the audit-log consumer review. AI-assisted-by: deepseek-v4-flash AI-assisted-by: gpt-6.1-terra * 🐛 Address audit log consumer review findings Enable audit log GC by default and document how to pause it during rollout or before registering a consumer that needs a complete replay. Run GC paths in order on the runner thread. Count each committed batch and preserve partial totals on ordinary failures while propagating interruption. Skip full-row payloads in telemetry queries and decoding. Keep the age limit for local retention only, and document the shared pass contract. AI-assisted-by: gpt-6.2-sol * 🐛 Close audit log consumer review findings Verify telemetry counts processed and skipped events in a mixed batch. Clarify how to pause GC, the scope of the replay race, and what the retention warning counts. Remove private Nexus setup from the public upgrade runbook. AI-assisted-by: gpt-6.2-sol * 📚 Simplify the audit log guide Consolidate the guidance on event trust, privacy and delivery. Keep examples that explain why the design works and link to code and runbooks for details. Clarify replay limits and distinguish age-based deletion from missed delivery. AI-assisted-by: gpt-6.2-sol
287 lines
10 KiB
Clojure
287 lines
10 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns app.tasks.telemetry
|
|
"A task that is responsible to collect anonymous statistical
|
|
information about the current instance and send it to the telemetry
|
|
server."
|
|
(:require
|
|
[app.common.data :as d]
|
|
[app.common.exceptions :as ex]
|
|
[app.common.logging :as l]
|
|
[app.common.schema :as sm]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[app.http.client :as http]
|
|
[app.jobs :as jobs]
|
|
[app.main :as-alias main]
|
|
[app.setup :as-alias setup]
|
|
[app.util.json :as json]
|
|
[integrant.core :as ig]
|
|
[promesa.exec :as px]))
|
|
|
|
(defn- get-subscriptions
|
|
[cfg]
|
|
(let [sql "SELECT email FROM profile where props->>'~:newsletter-updates' = 'true'"]
|
|
(db/run! cfg (fn [{:keys [::db/conn]}]
|
|
(->> (db/exec! conn [sql])
|
|
(into [] (map :email))
|
|
(not-empty))))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; LEGACY DATA COLLECTION
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(defn- get-num-teams
|
|
[conn]
|
|
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM team"]) :count))
|
|
|
|
(defn- get-num-projects
|
|
[conn]
|
|
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM project"]) :count))
|
|
|
|
(defn- get-num-files
|
|
[conn]
|
|
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM file"]) :count))
|
|
|
|
(def ^:private sql:num-file-changes
|
|
"SELECT count(*) AS count
|
|
FROM file_change
|
|
WHERE created_at < date_trunc('day', now()) + '24 hours'::interval
|
|
AND created_at > date_trunc('day', now())")
|
|
|
|
(defn- get-num-file-changes
|
|
[conn]
|
|
(-> (db/exec-one! conn [sql:num-file-changes]) :count))
|
|
|
|
(def ^:private sql:num-touched-files
|
|
"SELECT count(distinct file_id) AS count
|
|
FROM file_change
|
|
WHERE created_at < date_trunc('day', now()) + '24 hours'::interval
|
|
AND created_at > date_trunc('day', now())")
|
|
|
|
(defn- get-num-touched-files
|
|
[conn]
|
|
(-> (db/exec-one! conn [sql:num-touched-files]) :count))
|
|
|
|
(defn- get-num-users
|
|
[conn]
|
|
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM profile"]) :count))
|
|
|
|
(defn- get-num-fonts
|
|
[conn]
|
|
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM team_font_variant"]) :count))
|
|
|
|
(defn- get-num-comments
|
|
[conn]
|
|
(-> (db/exec-one! conn ["SELECT count(*) AS count FROM comment"]) :count))
|
|
|
|
(def sql:team-averages
|
|
"with projects_by_team AS (
|
|
SELECT t.id, count(p.id) AS num_projects
|
|
FROM team AS t
|
|
LEFT JOIN project AS p ON (p.team_id = t.id)
|
|
GROUP BY 1
|
|
), files_by_project AS (
|
|
SELECT p.id, count(f.id) AS num_files
|
|
FROM project AS p
|
|
LEFT JOIN file AS f ON (f.project_id = p.id)
|
|
GROUP BY 1
|
|
), comment_threads_by_file AS (
|
|
SELECT f.id, count(ct.id) AS num_comment_threads
|
|
FROM file AS f
|
|
LEFT JOIN comment_thread AS ct ON (ct.file_id = f.id)
|
|
GROUP BY 1
|
|
), users_by_team AS (
|
|
SELECT t.id, count(tp.profile_id) AS num_users
|
|
FROM team AS t
|
|
LEFT JOIN team_profile_rel AS tp ON(tp.team_id = t.id)
|
|
GROUP BY 1
|
|
)
|
|
SELECT (SELECT avg(num_projects)::integer FROM projects_by_team) AS avg_projects_on_team,
|
|
(SELECT max(num_projects)::integer FROM projects_by_team) AS max_projects_on_team,
|
|
(SELECT avg(num_files)::integer FROM files_by_project) AS avg_files_on_project,
|
|
(SELECT max(num_files)::integer FROM files_by_project) AS max_files_on_project,
|
|
(SELECT avg(num_comment_threads)::integer FROM comment_threads_by_file) AS avg_comment_threads_on_file,
|
|
(SELECT max(num_comment_threads)::integer FROM comment_threads_by_file) AS max_comment_threads_on_file,
|
|
(SELECT avg(num_users)::integer FROM users_by_team) AS avg_users_on_team,
|
|
(SELECT max(num_users)::integer FROM users_by_team) AS max_users_on_team")
|
|
|
|
(defn- get-team-averages
|
|
[conn]
|
|
(->> [sql:team-averages]
|
|
(db/exec-one! conn)))
|
|
|
|
(defn- get-email-domains
|
|
[conn]
|
|
(let [sql "SELECT DISTINCT split_part(email, '@', 2) AS domain FROM profile ORDER BY 1"]
|
|
(->> (db/exec! conn [sql])
|
|
(mapv :domain))))
|
|
|
|
(defn- get-enabled-auth-providers
|
|
[conn]
|
|
(let [sql (str "SELECT auth_backend AS backend, count(*) AS total "
|
|
" FROM profile GROUP BY 1")
|
|
rows (db/exec! conn [sql])]
|
|
(->> rows
|
|
(map (fn [{:keys [backend total]}]
|
|
(let [backend (or backend "penpot")]
|
|
[(keyword (str "auth-backend-" backend))
|
|
total])))
|
|
(into {}))))
|
|
|
|
(defn- get-jvm-stats
|
|
[]
|
|
(let [^Runtime runtime (Runtime/getRuntime)]
|
|
{:jvm-heap-current (.totalMemory runtime)
|
|
:jvm-heap-max (.maxMemory runtime)
|
|
:jvm-cpus (.availableProcessors runtime)
|
|
:os-arch (System/getProperty "os.arch")
|
|
:os-name (System/getProperty "os.name")
|
|
:os-version (System/getProperty "os.version")
|
|
:user-tz (System/getProperty "user.timezone")}))
|
|
|
|
(def ^:private sql:get-counters
|
|
"SELECT name, count(*) AS count
|
|
FROM audit_log
|
|
WHERE source LIKE 'telemetry:%'
|
|
AND created_at >= date_trunc('day', now())
|
|
AND created_at < date_trunc('day', now()) + interval '1 day'
|
|
GROUP BY 1
|
|
ORDER BY 2 DESC")
|
|
|
|
(defn- get-action-counters
|
|
"The event counters of the legacy report: an aggregate of the telemetry
|
|
shadows that are still in the buffer when the report runs. The number
|
|
already depended on the flags (the whole day when no full-row consumer
|
|
persisted, only since the previous run otherwise); now the consumer ships
|
|
the shadows and the GC deletes them, so it reflects what the buffer still
|
|
holds. The per-event data reaches Nexus as `telemetry-events` from the
|
|
consumer; this report does not carry it."
|
|
[conn]
|
|
(let [counters (->> (db/exec! conn [sql:get-counters])
|
|
(d/index-by (comp keyword :name) :count))
|
|
total (reduce + 0 (vals counters))]
|
|
{:total-accomulated-events total
|
|
:event-counters counters}))
|
|
|
|
(defn- get-legacy-stats
|
|
[{:keys [::db/conn]}]
|
|
(let [referer (if (cf/get :telemetry-with-taiga)
|
|
"taiga"
|
|
(cf/get :telemetry-referer))]
|
|
(-> {:referer referer
|
|
:public-uri (str (cf/get :public-uri))
|
|
:total-teams (get-num-teams conn)
|
|
:total-projects (get-num-projects conn)
|
|
:total-files (get-num-files conn)
|
|
:total-users (get-num-users conn)
|
|
:total-fonts (get-num-fonts conn)
|
|
:total-comments (get-num-comments conn)
|
|
:total-file-changes (get-num-file-changes conn)
|
|
:total-touched-files (get-num-touched-files conn)
|
|
:email-domains (get-email-domains conn)}
|
|
(merge
|
|
(get-team-averages conn)
|
|
(get-jvm-stats)
|
|
(get-enabled-auth-providers conn)
|
|
(get-action-counters conn))
|
|
(d/without-nils))))
|
|
|
|
(defn- make-legacy-request
|
|
[cfg data]
|
|
(let [request {:method :post
|
|
:uri (cf/get :telemetry-uri)
|
|
:headers {"content-type" "application/json"}
|
|
:body (json/encode-str data)}
|
|
response (http/req cfg request {:skip-ssrf-check? true})]
|
|
(when (> (:status response) 206)
|
|
(ex/raise :type :internal
|
|
:code :invalid-response
|
|
:response-status (:status response)
|
|
:response-body (:body response)))))
|
|
|
|
(defn- send-legacy-data
|
|
[{:keys [::setup/props] :as cfg} stats subs]
|
|
(let [data (cond-> {:type :telemetry-legacy-report
|
|
:version (:full cf/version)
|
|
:instance-id (:instance-id props)}
|
|
(some? stats)
|
|
(assoc :stats stats)
|
|
|
|
(seq subs)
|
|
(assoc :subscriptions subs))]
|
|
|
|
(make-legacy-request cfg data)))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; TASK ENTRY POINT
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(declare execute-telemetry)
|
|
|
|
(defmethod ig/assert-key ::job-def
|
|
[_ params]
|
|
(assert (http/client? (::http/client params)) "expected a valid http client")
|
|
(assert (db/pool? (::db/pool params)) "expected a valid database pool")
|
|
(assert (some? (::setup/props params)) "expected setup props to be available"))
|
|
|
|
(def schema:telemetry-params
|
|
[:map
|
|
[:send? {:optional true} :boolean]
|
|
[:enabled? {:optional true} :boolean]])
|
|
|
|
(defmethod ig/init-key ::job-def
|
|
[_ cfg]
|
|
{::jobs/name :telemetry
|
|
::jobs/schema schema:telemetry-params
|
|
::jobs/handler
|
|
(fn [_context params]
|
|
(execute-telemetry cfg params))
|
|
::jobs/decoder (sm/decoder schema:telemetry-params sm/json-transformer)
|
|
::jobs/validator (sm/validator schema:telemetry-params)})
|
|
|
|
(defn execute-telemetry
|
|
"Plain job handler: send periodic telemetry data."
|
|
[cfg params]
|
|
(let [send? (get params :send? true)
|
|
enabled? (or (get params :enabled? false)
|
|
cf/telemetry-enabled?)
|
|
;; Deferred so the query runs only when a report is
|
|
;; actually going to be sent.
|
|
subs (delay (get-subscriptions cfg))]
|
|
|
|
(jobs/heartbeat cfg)
|
|
|
|
;; If we have telemetry enabled, then proceed the normal
|
|
;; operation sending legacy report
|
|
|
|
(if enabled?
|
|
(when send?
|
|
;; Randomize start time to avoid thundering herd when multiple
|
|
;; instances restart at the same time.
|
|
(px/sleep (rand-int 10000))
|
|
|
|
(try
|
|
(let [stats (db/run! cfg get-legacy-stats)]
|
|
(send-legacy-data cfg stats @subs))
|
|
(catch Exception cause
|
|
(l/wrn :hint "unable to send legacy report"
|
|
:cause cause))))
|
|
|
|
;; If we have telemetry disabled, but there are users that are
|
|
;; explicitly checked the newsletter subscription on the
|
|
;; onboarding dialog or the profile section, then proceed to
|
|
;; send a limited telemetry data, that consists in the list of
|
|
;; subscribed emails and the running penpot version.
|
|
;; Official instances (penpot.dev / penpot.app) are excluded:
|
|
;; they must never send subscriber emails to the telemetry
|
|
;; endpoint (which is ourselves).
|
|
(when (and (not (cf/telemetry-excluded?)) send? @subs)
|
|
(px/sleep (rand-int 10000))
|
|
(ex/ignoring
|
|
(send-legacy-data cfg nil @subs))))))
|