mirror of
https://github.com/penpot/penpot.git
synced 2026-06-01 21:20:18 +00:00
* 🎉 Add telemetry anonymous event collection Rewrite the audit logging subsystem to support three operating modes and add anonymous telemetry event collection: Modes: - A (audit-log only): events persisted with full context - B (audit-log + telemetry): same as A, plus events are collected for telemetry shipping - C (telemetry-only): events stored anonymously with PII stripped, telemetry flag active, audit-log flag inactive Audit system refactoring (app.loggers.audit): - Replace qualified map keys (::audit/name etc.) with plain keywords - Rename submit! -> submit, insert! -> insert, prepare-event -> prepare-rpc-event - Add submit* as a lower-level public API - Add process-event dispatch function that handles all three modes and webhooks in a single tx-run! - Add :id to event schema (auto-generated if omitted) - Add filter-telemetry-props: anonymises event props per event type. Keeps UUID/boolean/number values; for login/identify events preserves lang, auth-backend, email-domain; for navigate events preserves route, file-id, team-id, page-id; instance-start trigger passes through. - Add filter-telemetry-context: retains only safe context keys. Backend: version, initiator, client-version, client-user-agent. Frontend: browser, os, locale, screen metrics, event-origin. - Timestamps truncated to day precision via ct/truncate for telemetry storage - PII stripped: props emptied, ip-addr zeroed, session-linking and access-token fields removed from context Config (app.config): - Derive :enable-telemetry flag from telemetry-enabled config option Email utilities (app.email): - Add email/clean and email/get-domain helper functions for domain extraction from email addresses Setup (app.setup): - Emit instance-start trigger event at system startup - Simplify handle-instance-id (remove read-only check) RPC layer (app.rpc): - wrap-audit now activates when :telemetry flag is set - Add :request-id to RPC params context for event correlation RPC commands (management, teams_invitations, verify_token, OIDC auth, webhooks): migrate all audit call sites to use the new plain-key API SREPL (app.srepl.main): - Migrate all audit/insert! calls to audit/insert with plain keys Telemetry task (app.tasks.telemetry): - Restructure legacy report into make-legacy-request; distinguish payload type as :telemetry-legacy-report - Add collect-and-send-audit-events: loop fetching up to 10,000 rows per iteration, encodes and sends each page, deletes on success, stops immediately on failure for retry - Add send-event-batch: POSTs fressian+zstd batch (base64 via blob/encode-str) to the telemetry endpoint with instance-id per event - Add gc-telemetry-events: enforces 100,000-row safety cap by dropping oldest rows first - Add delete-sent-events: deletes successfully shipped rows by id Blob utilities (app.util.blob): - Add encode-str/decode-str: combine fressian+zstd encoding with URL- safe base64 for JSON-safe string transport Database: - Add migration 0145: index on audit_log (source, created_at ASC) for efficient telemetry batch collection queries Frontend: - Always initialize event system regardless of :audit-log flag - Defer auth events (signin identify) to after profile is set - Refactor event subsystem for telemetry support Tests (21 test vars, 94 assertions in tasks-telemetry-test): - Cover all code paths: disabled/enabled telemetry, no-events no-op, happy-path batch send and delete, failure retention, payload anonymity, context stripping, timestamp day precision, batch encoding round-trip, multi-page iteration, GC cap enforcement, partial failure handling - blob encode-str/decode-str round-trip tests (14 test vars) - RPC audit integration tests (5 test vars) Signed-off-by: Andrey Antukh <niwi@niwi.nz> * 📎 Add pr feedback changes --------- Signed-off-by: Andrey Antukh <niwi@niwi.nz>
153 lines
4.3 KiB
Clojure
153 lines
4.3 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns app.main
|
|
(:require
|
|
[app.common.data.macros :as dm]
|
|
[app.common.logging :as log]
|
|
[app.common.time :as ct]
|
|
[app.common.transit :as t]
|
|
[app.common.types.objects-map]
|
|
[app.config :as cf]
|
|
[app.main.data.auth :as da]
|
|
[app.main.data.event :as ev]
|
|
[app.main.data.profile :as dp]
|
|
[app.main.data.websocket :as ws]
|
|
[app.main.errors]
|
|
[app.main.features :as feat]
|
|
[app.main.rasterizer :as thr]
|
|
[app.main.store :as st]
|
|
[app.main.ui :as ui]
|
|
[app.main.ui.alert]
|
|
[app.main.ui.confirm]
|
|
[app.main.ui.css-cursors :as cur]
|
|
[app.main.ui.delete-shared]
|
|
[app.main.ui.routes :as rt]
|
|
[app.main.worker :as mw]
|
|
[app.plugins :as plugins]
|
|
[app.util.dom :as dom]
|
|
[app.util.i18n :as i18n]
|
|
[beicon.v2.core :as rx]
|
|
[cuerdas.core :as str]
|
|
[debug]
|
|
[features]
|
|
[potok.v2.core :as ptk]
|
|
[rumext.v2 :as mf]))
|
|
|
|
(log/setup! {:app :info})
|
|
(log/set-level! :debug)
|
|
|
|
(when (= :browser cf/target)
|
|
(log/inf :version (:full cf/version)
|
|
:asserts *assert*
|
|
:build-date cf/build-date
|
|
:public-uri (dm/str cf/public-uri)
|
|
:session-id (str cf/session-id))
|
|
(log/inf :hint "enabled flags" :flags (str/join " " (map name cf/flags))))
|
|
|
|
(declare reinit)
|
|
|
|
(defonce app-root
|
|
(let [el (dom/get-element "app")]
|
|
(mf/create-root el)))
|
|
|
|
(defn init-ui
|
|
[]
|
|
(mf/render! app-root (mf/element ui/app)))
|
|
|
|
(defn- initialize-rasterizer
|
|
[]
|
|
(ptk/reify ::initialize-rasterizer
|
|
ptk/EffectEvent
|
|
(effect [_ _ _]
|
|
;; The rasterizer is used for the dashboard thumbnails
|
|
(thr/init!))))
|
|
|
|
(defn initialize
|
|
[]
|
|
(ptk/reify ::initialize
|
|
ptk/UpdateEvent
|
|
(update [_ state]
|
|
(assoc state :session-id cf/session-id))
|
|
|
|
ptk/WatchEvent
|
|
(watch [_ _ stream]
|
|
(rx/merge
|
|
(rx/of (ev/initialize)
|
|
(dp/refresh-profile))
|
|
|
|
;; Watch for profile deletion events
|
|
(->> stream
|
|
(rx/filter dp/profile-deleted-event?)
|
|
(rx/map da/logged-out))
|
|
|
|
;; Once profile is fetched, initialize all penpot application
|
|
;; routes
|
|
(->> stream
|
|
(rx/filter dp/profile-fetched?)
|
|
(rx/take 1)
|
|
(rx/map #(rt/init-routes)))
|
|
|
|
;; Once profile fetched and the current user is authenticated,
|
|
;; proceed to initialize the websockets connection.
|
|
(->> stream
|
|
(rx/filter dp/profile-fetched?)
|
|
(rx/map deref)
|
|
(rx/filter dp/is-authenticated?)
|
|
(rx/take 1)
|
|
(rx/map #(ws/initialize)))
|
|
|
|
(->> stream
|
|
(rx/filter (ptk/type? ::feat/initialize))
|
|
(rx/take 1)
|
|
(rx/map #(initialize-rasterizer)))))))
|
|
|
|
(defn ^:export init
|
|
[options]
|
|
;; WORKAROUND: we set this really not usefull property for signal a
|
|
;; sideffect and prevent GCC remove it. We need it because we need
|
|
;; to populate the Date prototype with transit related properties
|
|
;; before SES hardning is applied on loading MCP plugin
|
|
(unchecked-set js/globalThis "penpotStartDate"
|
|
(-> (ct/now)
|
|
(t/encode-str)
|
|
(t/decode-str)))
|
|
|
|
;; Before initializing anything, check if the browser has loaded
|
|
;; stale JS from a previous deployment. If so, do a hard reload so
|
|
;; the browser fetches fresh assets matching the current index.html.
|
|
(if (cf/stale-build?)
|
|
(cf/throttled-reload
|
|
:reason (dm/str "stale JS: compiled=" cf/compiled-version-tag
|
|
" expected=" cf/version-tag))
|
|
(do
|
|
(some-> (unchecked-get options "defaultTranslations")
|
|
(i18n/set-default-translations))
|
|
(mw/init!)
|
|
(i18n/init)
|
|
(cur/init-styles)
|
|
|
|
(init-ui)
|
|
(st/emit! (plugins/initialize)
|
|
(initialize)))))
|
|
|
|
(defn ^:export reinit
|
|
([]
|
|
(reinit false))
|
|
([hard?]
|
|
;; The hard flag will force to unmount the whole UI and will redraw every component
|
|
(when hard?
|
|
(mf/unmount! app-root)
|
|
(set! app-root (mf/create-root (dom/get-element "app"))))
|
|
(st/emit! (ev/initialize))
|
|
(init-ui)))
|
|
|
|
(defn ^:dev/after-load after-load
|
|
[]
|
|
(reinit))
|
|
|
|
(set! (.-stackTraceLimit js/Error) 50)
|