diff --git a/backend/src/app/graph/debug.clj b/backend/src/app/graph/debug.clj index 5debdd8a6e..0f6146a534 100644 --- a/backend/src/app/graph/debug.clj +++ b/backend/src/app/graph/debug.clj @@ -315,8 +315,9 @@ primitive, so the transfer goes through Ladybug's `EXPORT DATABASE` (Parquet per table) into a fresh on-disk database via `IMPORT DATABASE`. - Note the round-trip drops table comments; beadpot resolves such tables by - name (`beadpot.graph.context/_adopt_node_table`), so this is not load-bearing. + Note the round trip drops table comments. Nothing in the graph is addressed + by a table comment: every table is resolved by name, so the loss costs + nothing. Returns the path of the written database, or nil when no session is loaded. The caller owns the file and must delete it once streamed." diff --git a/backend/src/app/graph/ladybug.clj b/backend/src/app/graph/ladybug.clj index ddf8936759..386af1c5e2 100644 --- a/backend/src/app/graph/ladybug.clj +++ b/backend/src/app/graph/ladybug.clj @@ -71,7 +71,7 @@ (str "json('" (escape-cypher-string (json/encode v)) "')")) (defn format-timestamp - "Ladybug TIMESTAMP literal (beadpot: `timestamp('…')`)." + "Ladybug TIMESTAMP literal of the form `timestamp('')`." [v] (let [s (cond (instance? java.time.Instant v) diff --git a/backend/src/app/graph/project/document.clj b/backend/src/app/graph/project/document.clj index d859c64594..e6fbd3f2c1 100644 --- a/backend/src/app/graph/project/document.clj +++ b/backend/src/app/graph/project/document.clj @@ -10,16 +10,13 @@ Projects Document, Page, Component, the full shape tree (skipping the root frame), and `IsChildOf` edges from shapes/pages/components to their parent. - Two of beadpot's denormalizing transforms happen here rather than in a later - pass, because the walk already has the answer in hand: + Two denormalizations happen here rather than in a later pass, because the + walk already has both answers in hand and a post-ingest statement would have + to rediscover them: - - `page-id` on every shape (beadpot `DenormalizePageId`), from the page the - walk is currently in; - - `component-id` propagated from an instance head down to its descendants - (beadpot `DenormalizeComponentId`), from the head context the walk carries. - - Both ids are declared in `app.graph.meta/projection-transforms`, so beadpot - reads the graph's own provenance and does not redo them." + - `page-id` on every shape, from the page the walk is currently in; + - `component-id` propagated from an instance head down to its descendants, + from the head context the walk carries." (:require [app.common.logging :as l] [app.common.uuid :as uuid] @@ -73,9 +70,8 @@ (defn descend-component-ctx "The component context to pass to `shape`'s children. - Mirrors beadpot `DenormalizeComponentId`, whose recursive match stops at the - nearest ancestor Frame carrying a `component-id` and treats any intermediate - node that carries one as a barrier: + Inheritance stops at the nearest ancestor Frame carrying a `component-id`, + and any intermediate shape that carries one is a barrier: - a Frame with its own `component-id` becomes the new context (it is an instance head, and its descendants belong to *it*, not to an outer head); diff --git a/backend/src/app/graph/project/transforms.clj b/backend/src/app/graph/project/transforms.clj index 3735814b66..4ef97328ec 100644 --- a/backend/src/app/graph/project/transforms.clj +++ b/backend/src/app/graph/project/transforms.clj @@ -5,17 +5,12 @@ ;; Copyright (c) KALEIDOS INC Sucursal en España SL (ns app.graph.project.transforms - "Derived graph links, ported from beadpot's post-projection transforms. + "Derived graph links: edges a reader could compute from the projected + columns, materialized once at build time so a query does not have to. - Each entry in `registry` carries the shared transform id - (`app.graph.meta`), so a build records exactly what it produced and beadpot - runs only the complement in Python. Adding a transform here is therefore the - whole port step: nothing else has to be told about it. - - The Cypher mirrors beadpot's, which is written through its query builder but - reduces to the same statements — the parity harness diffs the resulting - graphs, so a semantic drift shows up as a differing edge set rather than as - a differing query." + Each entry in `registry` names the transform, the relationship it produces, + and the function that produces it, so adding one is a single entry and + nothing else has to be told about it." (:require [app.common.logging :as l] [app.graph.ladybug :as ladybug] @@ -32,8 +27,7 @@ (defn- link-component-instances! "`IsInstanceOf` from Frame instance heads to their Component. - beadpot `graph/transform/assets.py::LinkComponentInstances`. Every head is - linked, the main instance and any copy root alike. + Every head is linked, the main instance and any copy root alike. `component-file` is what makes a head a head here, not `component-id` alone. `app.common.types.component/instance-of?` requires both, and the projection @@ -54,18 +48,17 @@ "One statement per (from, to) shape-table pair. Ladybug cannot create a relationship bound by multiple node labels in a - single `MERGE` — inherited from Kùzu, which it forks (upstream issue - kuzudb/kuzu#5841). beadpot loops over label pairs for the same reason; the - loop is a dialect constraint, not a modelling choice." + single `MERGE`, a constraint inherited from Kùzu, which it forks (upstream + issue kuzudb/kuzu#5841). The loop over label pairs is that dialect + constraint, not a modelling choice." [f] (for [from nodes/shape-tables to nodes/shape-tables] (f from to))) (defn- link-shape-refs! - "`RefersTo` from an instance shape to its homologue in the main instance. - - beadpot `graph/transform/assets.py::LinkShapeRefs`, driven by `shape-ref`." + "`RefersTo` from an instance shape to its homologue in the main instance, + driven by `shape-ref`." [^Connection conn] (reduce (fn [total statement] (+ total (run-scalar! conn statement))) @@ -86,11 +79,14 @@ (defn- link-swap-slots! "`FillsSwapSlot` from a swapped-in shape to the slot it replaces. - beadpot `graph/transform/swap_slots.py::LinkSwapSlots`. Penpot records a - component sub-shape swap as a `swap-slot-` entry in the *replacing* - shape's `touched` set, where `` is the replaced slot shape from the - master. The entries are then stripped from `touched`, mirroring - `app.common.types.component/normal-touched-groups`." + Penpot records a component sub-shape swap as a `swap-slot-` entry in + the *replacing* shape's `touched` set, where `` names the replaced + slot shape in the main instance. The entries are then stripped from + `touched`, as `app.common.types.component/normal-touched-groups` does, so a + reader of `touched` sees design edits rather than swap bookkeeping. + + Stripping makes this the one transform that writes a column another + transform could read. Anything reading `touched` has to run before it." [^Connection conn] (let [linked (reduce @@ -120,10 +116,12 @@ linked)) (def registry - "Every transform this backend applies, in application order. + "Every transform this backend applies. - `:id` is the shared vocabulary with beadpot (`app.graph.meta`); `:rel` names - what the transform produces, for the ingest report." + `:id` names the transform in the ingest report and the log. `:rel` names + the relationship it produces. The three registered here read disjoint + columns, so the vector order is not load-bearing. The one ordering + constraint that exists is stated on `link-swap-slots!`." [{:id "link-component-instances" :rel :IsInstanceOf :run link-component-instances!} {:id "link-shape-refs" :rel :RefersTo :run link-shape-refs!} {:id "link-swap-slots" :rel :FillsSwapSlot :run link-swap-slots!}]) @@ -131,9 +129,8 @@ (defn apply-transforms! "Apply every registered transform to an already loaded graph. - Returns `{:ids [...] :counts {...} :transforms n}`; `:ids` is what the build - records in `GraphMeta`, so beadpot subtracts exactly this set from its own - pipeline." + Returns `{:ids [...] :counts {...} :transforms n}`, where `:ids` names what + ran and `:counts` gives the edges each one produced." [_system ^Connection conn _data _file] (reduce (fn [acc {:keys [id rel run]}] diff --git a/backend/src/app/graph/schema/contract.clj b/backend/src/app/graph/schema/contract.clj index 798f4ae460..684a12dd2e 100644 --- a/backend/src/app/graph/schema/contract.clj +++ b/backend/src/app/graph/schema/contract.clj @@ -5,60 +5,53 @@ ;; Copyright (c) KALEIDOS INC Sucursal en España SL (ns app.graph.schema.contract - "The graph-schema contract shared with beadpot. + "Deliberate choices in Penpot's graph schema, recorded as data. - Penpot is authoritative on the *design model*: what a shape is, what a - component means, which attributes exist. beadpot is authoritative on the - *graph schema*: the table and column names, and the Ladybug types, that - downstream consumers (ML graph mappings, featurization) read. A graph this - backend writes must therefore be indistinguishable, to those consumers, from - one beadpot's Python pipeline writes. + Penpot must pick a spelling and a type for every graph column. A Ladybug + column gets both once, at table creation, and neither widens afterwards. The + choices are therefore worth making deliberately and worth recording. - Everything that could drift between the two lives here, as data: + Three of them live here: - - `column-name` maps a Penpot key to its beadpot column. The rule is - snake_case of the key; `renames` records every exception. - - `dropped-keys` names Penpot keys that deliberately have no column. + - `column-name` maps a Penpot key to its column. The rule is snake_case of + the key, and `renames` records every exception. + - `dropped-keys` and `per-table-dropped` name Penpot keys that deliberately + get no column. - `type-overrides` pins the Ladybug type where the Malli-derived one - (`app.graph.schema.types`) differs from beadpot's and the difference is - load-bearing for a consumer. + (`app.graph.schema.types`) is coarser than the column deserves. - Each entry carries its reason. A new divergence must be added here, which is - the point: `backend_tests.graph_contract_test` walks the checked-in beadpot - manifest (`resources/app/graph/beadpot-schema.json`, produced by - `bp graph schema export`) and fails on anything this namespace does not - account for. Schema drift becomes a failing test with a precise message - instead of a silently renamed column in a training set." + Each entry carries its reason. A divergence from the default rule is then a + diff to review rather than a silent rename." (:require [app.common.json :as json] [clojure.string :as str])) (def ^:private renames - "Penpot key → beadpot column name, where the two differ. + "Penpot key to column name, where the column is not snake_case of the key. Keyed by the Penpot key alone: no shape type gives one of these a second meaning, so a per-table map would only add ceremony." - {;; beadpot names the discriminant after the table (`BooleanNode`), not - ;; after Penpot's `:bool` shape type. + {;; `bool` collides with the Ladybug type name, so the column is named after + ;; the table (`Boolean`) rather than after Penpot's `:bool` shape type. :bool-type "boolean_type" - ;; beadpot keeps the wire name `component-root` out of the graph because - ;; the column records what the *file* saved, which can lag what the shape - ;; tree implies — `saved_` marks it as the stored value, not a derivation. + ;; The column records what the file saved, which can lag what the shape + ;; tree implies. The `saved_` prefix marks it as the stored value rather + ;; than a derivation. :component-root "saved_component_root" - ;; Penpot stores a list under a singular key; beadpot pluralizes it. + ;; The value is a list, so the plural is accurate. :shadow "shadows" - ;; beadpot spells out the revision number. + ;; The column spells the revision number out. :revn "revision"}) (def dropped-keys - "Penpot keys projected by the Malli registry that get no beadpot column. + "Penpot keys projected by the Malli registry that get no column. - Dropping is the right call only when the column would be dead weight - downstream; anything a consumer might learn from belongs in beadpot instead - (see `pending-beadpot-columns`)." + Dropping is right only when the column would be dead weight for every reader + of the graph. A key a reader might learn from belongs in `unprojected-keys` + instead." {:deleted-at "Only non-nil for a soft-deleted file, and a deleted file is never ingested." @@ -68,37 +61,35 @@ :pixel-grid-opacity "Viewer chrome, as above."}) -(def pending-beadpot-columns - "Penpot keys that *should* become beadpot columns but do not exist there yet. +(def unprojected-keys + "Penpot keys that should become graph columns and do not have one yet. - Distinct from `dropped-keys` on purpose: these are a debt beadpot owes, - not a decision to discard data. The contract test reports them separately so - a new upstream attribute cannot be quietly buried in the drop list." + Distinct from `dropped-keys` on purpose: these are a debt the projection + owes, not a decision to discard data. Keeping the two apart means a new + upstream attribute cannot be quietly buried in the drop list." {:background-blur - "Landed upstream behind a default-on flag; beadpot has no field for it yet."}) + "Landed upstream behind a default-on flag. No column for it yet."}) (def ^:private per-table-dropped "Keys dropped only on certain tables. `:grids` is the standing case: Penpot's shape schema admits it on every - shape, but only a Frame ever carries one, and beadpot models it on Frame - alone. Emitting an always-null column on ten other tables would widen every - multi-table scan for nothing." + shape, but only a Frame ever carries one. Emitting an always-null column on + ten other tables would widen every multi-table scan for nothing." {:grids #{"Boolean" "Circle" "Group" "Image" "Path" "Rectangle" "SVGRaw" "Text"}}) (def type-overrides - "Ladybug column type per beadpot column name, where beadpot's differs. + "Ladybug column type per column name, where the derived type is too coarse. `app.graph.schema.types` derives a type from the Malli schema, which is the - right default but coarser than beadpot in places: a Malli `:map` becomes - `JSON`, where beadpot may use a native Ladybug MAP or a fixed-size array - that a consumer can read as a tensor without parsing. + right default but coarser than the column deserves in places: a Malli `:map` + becomes `JSON`, where a native Ladybug MAP or a fixed-size array lets a + consumer read a tensor row without parsing. Only load-bearing divergences are pinned here, in the order they became - load-bearing; the rest are reported by the contract test and closed by - moving the whole DDL onto the beadpot manifest." - {;; `LinkAppliedTokens` (beadpot) reads this with `map_keys` / - ;; `map_extract`; as JSON the transform cannot run at all. + load-bearing." + {;; Must be a native MAP: a JSON blob cannot be indexed by key in Cypher, so + ;; `map_keys` and `map_extract` cannot reach a single token at all. "applied_tokens" "MAP(STRING, STRING)" ;; `grc/schema:rect` is an inline `:and` over a map, not the registered @@ -109,38 +100,38 @@ "selrect" "DOUBLE[4]" ;; The SVG provenance attributes are typed `:map` in the shape schema on - ;; purpose — legacy files hold them as plain maps rather than as `::grc/rect` - ;; and `::gmt/matrix` records, and a tighter *schema* would reject those files - ;; (`app.common.types.shape/schema:shape-generic-attrs`). A tighter *column* - ;; is free: `app.graph.schema.values/coerce` reads either form. + ;; purpose. Legacy files hold them as plain maps rather than as + ;; `::grc/rect` and `::gmt/matrix` records, and a tighter *schema* would + ;; reject those files + ;; (`app.common.types.shape/schema:shape-generic-attrs`). A tighter + ;; *column* is free: `app.graph.schema.values/coerce` reads either form. "svg_viewbox" "DOUBLE[4]" "svg_transform" "DOUBLE[6]" - ;; `:fills` is an `:or` — the packed `app.common.types.fills` value or a - ;; plain vector of fill maps — so the schema alone cannot say it is a + ;; `:fills` is an `:or` over the packed `app.common.types.fills` value and + ;; a plain vector of fill maps, so the schema alone cannot say it is a ;; collection. It always is one, and a fill has enough optional shape - ;; (solid, gradient, image) that JSON per element is the honest element type. + ;; (solid, gradient, image) that JSON per element is the honest element + ;; type. "fills" "JSON[]"}) (def ^:private map-key-fns "How to render the *keys* of a MAP column, per column. - Column names are snake_case because they are graph schema; the keys inside a - MAP are not — they are values, and beadpot models them as whatever it parsed - from the wire. `applied_tokens` is keyed by shape attribute in the camelCase - form Penpot's own JSON encoder produces (`app.common.json/write-camel-key`), - which is what beadpot's `AppliedTokenKey` holds and what - `UsesToken.for_property` therefore carries: `strokeWidth`, not + A column name is schema, so it is snake_case. The keys inside a MAP are + values, so they keep the spelling their producer used. `applied_tokens` is + keyed by shape attribute in the camelCase form + `app.common.json/write-camel-key` produces: `strokeWidth`, not `stroke-width`." {"applied_tokens" json/write-camel-key}) (defn map-key-fn - "Key renderer for a MAP column; `name` unless the column says otherwise." + "Key renderer for a MAP column. `name` unless the column says otherwise." [column] (get map-key-fns column name)) (defn column-name - "The beadpot column name for Penpot key `k`. + "The graph column name for Penpot key `k`. Default: snake_case of the key. `renames` overrides." [k] @@ -154,6 +145,6 @@ (contains? (get per-table-dropped k #{}) table))) (defn ladybug-type - "beadpot's Ladybug type for `column`, or `fallback` when it agrees." + "The pinned Ladybug type for `column`, or `fallback` when nothing is pinned." [column fallback] (get type-overrides column fallback)) diff --git a/backend/src/app/graph/schema/nodes.clj b/backend/src/app/graph/schema/nodes.clj index e45c32c6b9..5946fa5c8d 100644 --- a/backend/src/app/graph/schema/nodes.clj +++ b/backend/src/app/graph/schema/nodes.clj @@ -11,10 +11,10 @@ options (`:drop`, optional `:extra`). Derived artifacts — Ladybug DDL, Arrow fields, validation, type dispatch — all flow from that. - Column *names* and *types* are not ours to choose: they are the - contract downstream consumers read, and beadpot owns it. Every - divergence between a Penpot key and its column lives in - `app.graph.schema.contract`." + This registry is the single source of the graph schema. A Ladybug column + gets its name and its type once, at table creation, and there is no + widening afterwards. Every divergence between a Penpot key and its column + is recorded in `app.graph.schema.contract`." (:require [app.common.exceptions :as ex] [app.common.schema :as sm] @@ -31,7 +31,6 @@ (def schema-version "penpot-graph-slice-4") -;; beadpot/graph/schemas.py drop_fields (def ^:private document-projection {:source ctf/schema:file :drop [:data] @@ -154,7 +153,7 @@ (nth entry 1))) (defn column-name - "beadpot column name for projected key `k` on `table`." + "Graph column name for projected key `k` on `table`." [_table k] (contract/column-name k)) @@ -170,8 +169,8 @@ (defn column-keys "Projected column keys for `table`, in registry order. - Keys the beadpot contract drops on this table are omitted, so the column - order, the Arrow batch and the DDL cannot disagree about what exists." + Keys the contract drops on this table are omitted, so the column order, the + Arrow batch, and the DDL cannot disagree about what exists." [table] (into [] (comp (map first) @@ -256,7 +255,7 @@ table)) (defn cypher-property-key - "Backtick-wrapped beadpot column name for inline Cypher literals." + "Backtick-wrapped column name for inline Cypher literals." [table k] (str "`" (column-name table k) "`")) @@ -306,7 +305,7 @@ ", `position` INT64);")) (defn is-instance-of-ddl - "Frame instance heads → Component (beadpot `IsInstanceOf`)." + "Frame instance heads → Component." [] "CREATE REL TABLE `IsInstanceOf` (FROM `Frame` TO `Component`);") @@ -315,8 +314,7 @@ Created up-front rather than on demand: the bulk loader must never race on lazy table creation, and a consumer can then tell \"this producer cannot - emit that pair\" from \"this document happens to have none\" (beadpot - `graph.manifest/REL_FAMILIES`)." + emit that pair\" from \"this document happens to have none\"." [rel props] (str "CREATE REL TABLE `" rel "` (" (str/join ", " (for [from shape-tables @@ -326,13 +324,13 @@ ");")) (defn refers-to-ddl - "Instance shape → its homologue in the component main instance - (beadpot `RefersTo`, from `shape-ref`)." + "Instance shape → its homologue in the component main instance, resolved + from `shape-ref`." [] (shape-to-shape-rel-ddl "RefersTo" nil)) (defn fills-swap-slot-ddl - "Swapped-in shape → the slot shape it replaces (beadpot `FillsSwapSlot`)." + "Swapped-in shape → the slot shape it replaces." [] (shape-to-shape-rel-ddl "FillsSwapSlot" ["`slot_id` UUID"])) diff --git a/backend/src/app/graph/schema/projection.clj b/backend/src/app/graph/schema/projection.clj index ab994638a2..4d0e969329 100644 --- a/backend/src/app/graph/schema/projection.clj +++ b/backend/src/app/graph/schema/projection.clj @@ -7,8 +7,8 @@ (ns app.graph.schema.projection "Derive Ladybug node column schemas from Penpot Malli sources. - Same model as beadpot's `drop_fields`: start from the canonical schema - and remove keys that must not become graph columns." + Start from the canonical schema and remove the keys that must not become + graph columns." (:require [app.common.exceptions :as ex] [app.common.schema :as sm] @@ -71,7 +71,7 @@ "Build a graph node schema from canonical Malli `source`. Options: - - `:drop` - keys removed from the source (beadpot `drop_fields`) + - `:drop` - keys removed from the source - `:extra` - optional extra `[:map ...]` merged on top" [source {:keys [drop extra]}] (let [projected (select-projected-keys source drop)] diff --git a/backend/src/app/graph/schema/types.clj b/backend/src/app/graph/schema/types.clj index 73c21e572c..a905d92dcc 100644 --- a/backend/src/app/graph/schema/types.clj +++ b/backend/src/app/graph/schema/types.clj @@ -28,9 +28,9 @@ more than one shape (a `:multi`, an `:or`, an optional-keyed map) becomes `JSON`, because a Ladybug column cannot be two types. - Every encoding here has a matching value formatter in `app.graph.ladybug`; - the two must move together, and `bp graph schema diff` is what catches it - when they do not." + Every encoding here has a matching value formatter in `app.graph.ladybug`. + The two must move together: a column type with no case there falls back to + guessing the literal from the runtime value." (:require [app.common.logging :as l] [app.common.schema :as sm] diff --git a/backend/src/app/graph/schema/values.clj b/backend/src/app/graph/schema/values.clj index 2a618d5c02..e93b74f499 100644 --- a/backend/src/app/graph/schema/values.clj +++ b/backend/src/app/graph/schema/values.clj @@ -120,7 +120,7 @@ :else nil)) (defn- packed-color - "`#RRGGBB` as `0xRRGGBBAA`, the form beadpot's `ColorValue` packs. + "`#RRGGBB` as the packed integer `0xRRGGBBAA`. Alpha defaults to opaque: the column holds a colour, and any opacity Penpot keeps alongside it is a separate attribute." diff --git a/backend/src/app/graph/sync.clj b/backend/src/app/graph/sync.clj index 2720902912..a3820c5ccc 100644 --- a/backend/src/app/graph/sync.clj +++ b/backend/src/app/graph/sync.clj @@ -118,9 +118,9 @@ ;; `:add-obj` children inherit — it is the shape's effective ;; component-id, which loses the barrier case of a *non-Frame* ;; carrying its own `component-id` (indistinguishable once - ;; denormalized). Cold projection, which beadpot diffs against, - ;; keeps the distinction; only a graph synced across such a - ;; shape can drift, and a Reload rebuilds it. + ;; denormalized). Cold projection keeps the distinction. Only a + ;; graph synced across such a shape can drift, and a Reload + ;; rebuilds it. :component-ctx (:component-id attrs) :page-id (or (:page-id attrs) (resolve-page-id shape-id parents pages))}])) @@ -178,7 +178,7 @@ "CREATE (s)-[:IsChildOf {position: " (ladybug/format-int position) "}]->(p);")) (defn- create-instance-of-statement - "Link a Frame instance head to its Component (beadpot `IsInstanceOf`). + "Link a Frame instance head to its Component. No-op when the Component is absent (e.g. library component not ingested)." [frame-id component-id] @@ -247,9 +247,11 @@ (defn- set-document-revision-statement - "The column is `revision`, not `revn` — beadpot spells the revision number out - (`schema.contract`). Named through `cypher-property-key` so the two cannot - disagree again." + "Set the Document's revision number. + + `app.graph.schema.contract` names the column `revision`, not `revn`. The name + is produced by `nodes/cypher-property-key`, so this statement and the DDL + cannot disagree." [doc-id revn] (str "MATCH (d:Document {id: " (ladybug/format-uuid doc-id) "}) " "SET d." (nodes/cypher-property-key "Document" :revn) " = "