mirror of
https://github.com/penpot/penpot.git
synced 2026-05-13 03:53:51 +00:00
7d4be33d4f
* 🎉 Add telemetry anonymous event collection Rewrite the audit logging subsystem to support three operating modes and add anonymous telemetry event collection: Modes: - A (audit-log only): events persisted with full context - B (audit-log + telemetry): same as A, plus events are collected for telemetry shipping - C (telemetry-only): events stored anonymously with PII stripped, telemetry flag active, audit-log flag inactive Audit system refactoring (app.loggers.audit): - Replace qualified map keys (::audit/name etc.) with plain keywords - Rename submit! -> submit, insert! -> insert, prepare-event -> prepare-rpc-event - Add submit* as a lower-level public API - Add process-event dispatch function that handles all three modes and webhooks in a single tx-run! - Add :id to event schema (auto-generated if omitted) - Add filter-telemetry-props: anonymises event props per event type. Keeps UUID/boolean/number values; for login/identify events preserves lang, auth-backend, email-domain; for navigate events preserves route, file-id, team-id, page-id; instance-start trigger passes through. - Add filter-telemetry-context: retains only safe context keys. Backend: version, initiator, client-version, client-user-agent. Frontend: browser, os, locale, screen metrics, event-origin. - Timestamps truncated to day precision via ct/truncate for telemetry storage - PII stripped: props emptied, ip-addr zeroed, session-linking and access-token fields removed from context Config (app.config): - Derive :enable-telemetry flag from telemetry-enabled config option Email utilities (app.email): - Add email/clean and email/get-domain helper functions for domain extraction from email addresses Setup (app.setup): - Emit instance-start trigger event at system startup - Simplify handle-instance-id (remove read-only check) RPC layer (app.rpc): - wrap-audit now activates when :telemetry flag is set - Add :request-id to RPC params context for event correlation RPC commands (management, teams_invitations, verify_token, OIDC auth, webhooks): migrate all audit call sites to use the new plain-key API SREPL (app.srepl.main): - Migrate all audit/insert! calls to audit/insert with plain keys Telemetry task (app.tasks.telemetry): - Restructure legacy report into make-legacy-request; distinguish payload type as :telemetry-legacy-report - Add collect-and-send-audit-events: loop fetching up to 10,000 rows per iteration, encodes and sends each page, deletes on success, stops immediately on failure for retry - Add send-event-batch: POSTs fressian+zstd batch (base64 via blob/encode-str) to the telemetry endpoint with instance-id per event - Add gc-telemetry-events: enforces 100,000-row safety cap by dropping oldest rows first - Add delete-sent-events: deletes successfully shipped rows by id Blob utilities (app.util.blob): - Add encode-str/decode-str: combine fressian+zstd encoding with URL- safe base64 for JSON-safe string transport Database: - Add migration 0145: index on audit_log (source, created_at ASC) for efficient telemetry batch collection queries Frontend: - Always initialize event system regardless of :audit-log flag - Defer auth events (signin identify) to after profile is set - Refactor event subsystem for telemetry support Tests (21 test vars, 94 assertions in tasks-telemetry-test): - Cover all code paths: disabled/enabled telemetry, no-events no-op, happy-path batch send and delete, failure retention, payload anonymity, context stripping, timestamp day precision, batch encoding round-trip, multi-page iteration, GC cap enforcement, partial failure handling - blob encode-str/decode-str round-trip tests (14 test vars) - RPC audit integration tests (5 test vars) Signed-off-by: Andrey Antukh <niwi@niwi.nz> * 📎 Add pr feedback changes --------- Signed-off-by: Andrey Antukh <niwi@niwi.nz>
107 lines
3.9 KiB
Clojure
107 lines
3.9 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns backend-tests.util-blob-test
|
|
(:require
|
|
[app.util.blob :as blob]
|
|
[clojure.string :as str]
|
|
[clojure.test :as t]))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; encode-str / decode-str round-trip
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(t/deftest encode-str-roundtrip-empty-map
|
|
(let [data {}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-empty-vector
|
|
(let [data []]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-nil
|
|
(let [data nil]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-simple-map
|
|
(let [data {:name "penpot" :version 42}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-nested-structure
|
|
(let [data {:users [{:name "Alice" :tags #{"admin" "active"}}
|
|
{:name "Bob" :tags #{"user"}}]
|
|
:config {:debug false :timeout 3000}}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-vector-of-maps
|
|
(let [data [{:name "navigate" :type "action" :source "telemetry"}
|
|
{:name "create-file" :type "action" :source "telemetry"}]]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-keywords-and-strings
|
|
(let [data {:keyword/value :foo
|
|
:string/value "hello world"
|
|
:boolean/value true
|
|
:nil/value nil}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-numeric-types
|
|
(let [data {:int 42
|
|
:neg -7
|
|
:zero 0
|
|
:big 9999999999}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; URL-safe encoding properties
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(t/deftest encode-str-url-safe-no-unsafe-chars
|
|
;; URL-safe base64 must not contain +, /, or padding =
|
|
(let [data {:a (apply str (repeat 100 "x"))
|
|
:b (range 200)
|
|
:c {"key" "value with special chars: @#$%^&*()"}}
|
|
encoded (blob/encode-str data)]
|
|
(t/is (not (str/includes? encoded "+")))
|
|
(t/is (not (str/includes? encoded "/")))
|
|
(t/is (not (str/includes? encoded "=")))))
|
|
|
|
(t/deftest encode-str-url-safe-roundtrip-after-encoding
|
|
;; Ensure the URL-safe encoding still round-trips correctly
|
|
(let [data {:payload (vec (range 500))
|
|
:nested {:a {:b {:c "deep"}}}}
|
|
encoded (blob/encode-str data)
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; version-specific encoding
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(t/deftest encode-str-with-version-4
|
|
(let [data {:events [{:name "click"} {:name "scroll"}]}
|
|
encoded (blob/encode-str data {:version 4})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
(t/deftest encode-str-with-version-5
|
|
(let [data {:events [{:name "click"} {:name "scroll"}]}
|
|
encoded (blob/encode-str data {:version 5})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
(t/deftest encode-str-with-version-1
|
|
(let [data {:simple "data"}
|
|
encoded (blob/encode-str data {:version 1})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
(t/deftest encode-str-with-version-3
|
|
(let [data {:simple "data"}
|
|
encoded (blob/encode-str data {:version 3})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|