From 6bd1af4f4a686f38ad41def85a4b73af9a88e6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Tejero=20Cantero?= Date: Fri, 7 Aug 2026 01:43:21 +0200 Subject: [PATCH] :sparkles: Record the applied transforms in GraphMeta Ingestion is a partial port, so a graph this backend writes has had only some of the pipeline applied. Rather than have a reader infer which from a build version, the build writes it down: `GraphMeta.transforms` names every transform id applied, projection-time denormalizations included, and the parity consumer computes the complement and runs only that. The ids cross a language boundary as data, so they are kebab-case strings rather than keywords and must stay byte-identical to the consumer's own list. This stays off `graph-backend`. Nothing in this repository reads the column, and an unread column is weight a reviewer is right to question. AI-assisted-by: mixed models --- backend/src/app/graph/ingest.clj | 8 +++--- backend/src/app/graph/meta.clj | 42 +++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/backend/src/app/graph/ingest.clj b/backend/src/app/graph/ingest.clj index af0644ee4d..7dea98f5cc 100644 --- a/backend/src/app/graph/ingest.clj +++ b/backend/src/app/graph/ingest.clj @@ -59,9 +59,11 @@ (graph.arrow/load-projection! conn {:nodes nodes :edges edges} allocator) (ladybug/exec-on-connection! conn ["CHECKPOINT;"]) (let [transforms (projection.transforms/apply-transforms! system conn data file)] - ;; Written last: its presence doubles as the build-complete marker. - (graph.meta/write! conn {:file-id file-id - :revn (:revn file)}) + ;; Written last: its presence doubles as the build-complete marker, + ;; and it is what the parity consumer reads to know what is left. + (graph.meta/write! conn {:file-id file-id + :revn (:revn file) + :transform-ids (:ids transforms)}) {:file-id file-id :revn (:revn file) :name (or (:name data) (:name file)) diff --git a/backend/src/app/graph/meta.clj b/backend/src/app/graph/meta.clj index 129babd1fa..0e7ed1cbb4 100644 --- a/backend/src/app/graph/meta.clj +++ b/backend/src/app/graph/meta.clj @@ -5,13 +5,20 @@ ;; Copyright (c) KALEIDOS INC Sucursal en España SL (ns app.graph.meta - "`GraphMeta`: the graph's own account of who built it and from what. + "`GraphMeta`: the graph's own account of who built it, from what, and what + it did. A projected graph is a cache of a file at a revision, built by a known schema. The row records both, so a reader can decide whether to reuse the database or rebuild it: a `schema_version` that no longer matches the registry, or a `source_revn` behind the file's, means the cache is stale. + Ingestion is a partial port, so a graph can arrive with any subset of the + pipeline applied. `transforms` names what this build did, and the parity + consumer computes the complement and runs only that in Python. The ids cross + a language boundary as data, so they are kebab-case strings rather than + keywords and must stay byte-identical to the consumer's own list. + The row is written *last* in a build, so its presence also marks the build complete. @@ -21,7 +28,8 @@ (:require [app.common.time :as ct] [app.graph.ladybug :as ladybug] - [app.graph.schema.nodes :as nodes]) + [app.graph.schema.nodes :as nodes] + [clojure.string :as str]) (:import com.ladybugdb.Connection)) @@ -41,12 +49,37 @@ "`producer_version` STRING, " "`schema_version` STRING, " "`source_revn` INT64, " + "`transforms` STRING[], " "`built_at` TIMESTAMP, " "PRIMARY KEY (`source_file_id`));")) +(def projection-transforms + "Transform ids this backend satisfies while *projecting*, before any + transform pass runs. + + A reader cares whether the result is in the graph, not how it got there. The + consumer derives `IsChildOf` from persisted `shapes` arrays in a later pass; + `app.graph.project.document/project-shape-ids` emits the edges during the + tree walk. Same id, same observable graph. The two denormalizations are the + same case." + ["add-document" + "link-contained-shapes" + "denormalize-page-id" + "denormalize-component-id"]) + +(defn- format-transforms + [ids] + (str "[" (->> (sort (distinct ids)) + (map ladybug/format-string) + (str/join ", ")) + "]")) + (defn write! - "Record what this build produced for `file-id`." - [^Connection conn {:keys [file-id revn]}] + "Record what this build produced for `file-id`. + + `transform-ids` are the ids applied *on top of* `projection-transforms`, so + a caller only names what its transform pass did." + [^Connection conn {:keys [file-id revn transform-ids]}] (ladybug/exec-on-connection! conn [ddl]) (ladybug/exec-on-connection! conn @@ -55,5 +88,6 @@ "m.producer_version = " (ladybug/format-string (or (System/getenv "PENPOT_BUILD") "devenv")) ", " "m.schema_version = " (ladybug/format-string nodes/schema-version) ", " "m.source_revn = " (ladybug/format-int (or revn 0)) ", " + "m.transforms = " (format-transforms (concat projection-transforms transform-ids)) ", " "m.built_at = " (ladybug/format-timestamp (ct/now)) ";")]))