From e9f889d929784b3469b58ff0dde2676d543a687d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Tejero=20Cantero?= Date: Sun, 26 Jul 2026 09:57:34 +0200 Subject: [PATCH] :sparkles: Type graph columns as tightly as Ladybug allows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ladybug is schema-first and strongly typed: a property key gets its type at table-creation time and there is no widening later. That makes the Malli to Ladybug mapping the whole of the graph's typing, and it was leaving a lot on the table: a transform stored as `STRING`, a rect as `JSON`, a set of feature flags as a single `STRING`. A column typed `DOUBLE[4]` is four numbers a consumer reads as a tensor row; the same value as JSON is text somebody has to parse and trust. `app.graph.schema.types` now maps, in order: scalars; Penpot value types whose layout is fixed even though Malli only sees a map or a string (`::gmt/matrix` to `DOUBLE[6]`, `::gpt/point` to `DOUBLE[2]`, `::grc/rect` to `DOUBLE[4]`, `::clr/hex-color` to `UINT32`); then structure, with collections to `T[]`, `:map-of` to `MAP(k, v)`, and a closed map of scalars to a `STRUCT`. JSON is the fallback of last resort, for schemas that genuinely admit more than one shape. Two defects fell out. `::sm/set` was unmapped, so `features` and `migrations` were single strings rather than `STRING[]`, and `::sm/one-of`, how Penpot spells a closed set of keywords, was unmapped too, so `blend-mode`, `grow-type`, the constraints and every `layout-*` were mistyped. A tight column is only worth having if the writer fills it in that shape, so `app.graph.schema.values` shapes a value for its type: a matrix record into six doubles, a hex colour into a packed integer, a map into a struct's fields. Both writers go through it, so the bulk load and the incremental sync cannot disagree. What that required: - STRUCT field names must be backticked in the DDL *and* in every literal, because a grid cell has a field named `column`. The catalog reports them bare. - A struct literal's type is its field list, so every declared field must appear, and an absent one needs `cast(NULL, '')`. A bare NULL is typed STRING and changes the struct's type. - `STRUCT(…)[]` starts with `STRUCT(` but is a list, so the list check comes first. - Nested lists cannot be rendered with `str`: Clojure's `[1 2]` is space-separated and Ladybug reads it as a one-element array. Three more corrections in the same area: - `project-attrs` used truthiness where it meant `some?`, so `opacity 0` and `blocked false` projected as absent. - Set-valued columns are written sorted. A set has no order, so the column varied between builds of the same file, which is precisely what stops two builds being diffable. - An empty collection is written as `[]` rather than skipped. A shape with no fills has none; NULL would say "unknown". Renamed the `kuzu-*` helpers to `ladybug-*`: Kùzu is deprecated and Ladybug substitutes it, so a name bearing the engine should bear this one. The one remaining mention cites the upstream issue Ladybug inherits. AI-assisted-by: mixed models --- backend/src/app/graph/bulk.clj | 81 +++++--- backend/src/app/graph/ladybug.clj | 160 +++++++++------- backend/src/app/graph/project/transforms.clj | 5 +- backend/src/app/graph/schema/contract.clj | 16 +- backend/src/app/graph/schema/nodes.clj | 25 ++- backend/src/app/graph/schema/types.clj | 134 ++++++++++++- backend/src/app/graph/schema/values.clj | 188 +++++++++++++++++++ 7 files changed, 495 insertions(+), 114 deletions(-) create mode 100644 backend/src/app/graph/schema/values.clj diff --git a/backend/src/app/graph/bulk.clj b/backend/src/app/graph/bulk.clj index f72db17b42..df51a87492 100644 --- a/backend/src/app/graph/bulk.clj +++ b/backend/src/app/graph/bulk.clj @@ -13,6 +13,7 @@ [app.common.json :as json] [app.graph.ladybug :as ladybug] [app.graph.schema.nodes :as nodes] + [app.graph.schema.values :as values] [clojure.java.io :as io] [clojure.string :as str] [datoteka.fs :as fs]) @@ -54,8 +55,8 @@ (coll? v) (csv-escape-string (json/encode v)) :else (csv-escape-string (str v)))) -(defn- kuzu-list-element - "Format one element of a Ladybug LIST column for CSV COPY. Kuzu parses +(defn- ladybug-list-element + "Format one element of a Ladybug LIST column for CSV COPY. Ladybug parses the (CSV-unquoted) field as a list literal: bare values for UUID/number elements, single-quoted strings (backslash-escaped) for STRING/JSON." [elem-type v] @@ -79,15 +80,34 @@ :else (str v))) -(defn- kuzu-list-cell - "CSV cell for a LIST-typed column (`UUID[]`, `STRING[]`, `JSON[]`, …). - JSON-encoding the collection (as `csv-cell` does) is wrong here: Kuzu +(defn- ladybug-literal + "A coerced value as the list literal Ladybug's CSV reader expects. + + Recursive, because the types are: `points` is a `DOUBLE[2][]`, and rendering + its elements with `str` would emit Clojure's space-separated `[1 2]`, which + Ladybug reads as a one-element array." + [ladybug-type v] + (if-let [[_ element] (and (string? ladybug-type) + (re-matches #"(.+?)\[\d*\]$" ladybug-type))] + (str "[" (str/join "," (map #(ladybug-literal element %) + (if (coll? v) (seq v) [v]))) + "]") + (ladybug-list-element ladybug-type v))) + +(defn- ladybug-list-cell + "CSV cell for a LIST-typed column (`UUID[]`, `DOUBLE[4]`, `DOUBLE[2][]`, …). + JSON-encoding the collection (as `csv-cell` does) is wrong here: Ladybug expects its own list literal, e.g. `[id1,id2]` with bare elements." [ladybug-type v] - (let [elem-type (subs ladybug-type 0 (- (count ladybug-type) 2)) - elems (if (coll? v) (seq v) [v])] - (csv-escape-string - (str "[" (str/join "," (map #(kuzu-list-element elem-type %) elems)) "]")))) + (csv-escape-string (ladybug-literal ladybug-type v))) + +(defn- compound-type? + "A MAP or STRUCT, but not a *list* of them — a list is handled elementwise." + [ladybug-type] + (and (string? ladybug-type) + (not (ladybug/list-type? ladybug-type)) + (or (str/starts-with? ladybug-type "MAP(") + (str/starts-with? ladybug-type "STRUCT(")))) (defn- csv-representable? "Can a value of `ladybug-type` survive a CSV round-trip? @@ -106,13 +126,16 @@ writer, this is where the line falls." [ladybug-type] (cond - (not (string? ladybug-type)) true - (ladybug/map-type? ladybug-type) false - (str/starts-with? ladybug-type "STRUCT") false - (str/ends-with? ladybug-type "[]") - (not (contains? #{"STRING" "JSON"} - (subs ladybug-type 0 (- (count ladybug-type) 2)))) - :else true)) + (not (string? ladybug-type)) true + (compound-type? ladybug-type) false + :else + (if-let [[_ element] (re-matches #"(.+?)\[\d*\]$" ladybug-type)] + ;; A list is safe only if its elements are: no delimiter can occur inside + ;; a UUID or a number, and a nested list or struct reintroduces the + ;; problem one level down. + (and (not (contains? #{"STRING" "JSON"} element)) + (csv-representable? element)) + true))) (defn- defer-to-cypher? "Must this value be written after the COPY rather than in the CSV? @@ -125,16 +148,20 @@ (defn- csv-typed-cell [ladybug-type v] - (cond - ;; Written after the COPY, through Cypher — see `fixup-statements`. - (defer-to-cypher? ladybug-type v) "" + ;; Shape the value for its column first (a matrix into six doubles, a hex + ;; colour into a packed integer); the CSV cell then only has to escape plain + ;; data. Same first step as the Cypher path, so the two agree. + (let [v (values/coerce ladybug-type v)] + (cond + ;; Written after the COPY, through Cypher — see `fixup-statements`. + (defer-to-cypher? ladybug-type v) "" - (and (some? v) - (string? ladybug-type) - (str/ends-with? ladybug-type "[]")) - (kuzu-list-cell ladybug-type v) + (and (some? v) + (string? ladybug-type) + (re-matches #".+\[\d*\]$" ladybug-type)) + (ladybug-list-cell ladybug-type v) - :else (csv-cell v))) + :else (csv-cell v)))) (defn- fixup-statements "Cypher to set the values the CSV had to leave empty. @@ -145,12 +172,14 @@ [table rows] (let [columns (nodes/column-keys table)] (for [row rows + ;; An empty collection is a value, not an absence: a shape with no + ;; fills has `[]`, and writing NULL instead would tell a consumer the + ;; attribute is unknown rather than empty. :let [sets (for [k columns :let [v (get row k) t (nodes/column-ladybug-type table k)] :when (some? v) - :when (defer-to-cypher? t v) - :when (or (not (coll? v)) (seq v))] + :when (defer-to-cypher? t v)] (str "n." (nodes/cypher-property-key table k) " = " (nodes/format-column-value table k v)))] :when (seq sets)] diff --git a/backend/src/app/graph/ladybug.clj b/backend/src/app/graph/ladybug.clj index a9f19838bc..d9779eecfb 100644 --- a/backend/src/app/graph/ladybug.clj +++ b/backend/src/app/graph/ladybug.clj @@ -11,6 +11,7 @@ (:require [app.common.exceptions :as ex] [app.common.json :as json] + [app.graph.schema.values :as values] [clojure.string :as str] [datoteka.fs :as fs]) (:import @@ -100,82 +101,111 @@ (coll? v) (format-json v) :else (format-string (str v)))) -(defn- format-list-element - "Format one element of a Cypher LIST literal for typed `elem-type`." - [elem-type v] - (case elem-type - "UUID" (format-uuid v) - ;; `name` for keywords, so a `:touched` entry reads `swap-slot-…` and not - ;; `:swap-slot-…` — see `app.graph.bulk/kuzu-list-element`. - "STRING" (format-string (if (keyword? v) (name v) (str v))) - "JSON" (format-json v) - "INT64" (format-int v) - "DOUBLE" (format-number v) - "BOOLEAN" (if v "true" "false") - "TIMESTAMP" (format-timestamp v) - (format-value v))) - -(defn- format-list - "Cypher LIST literal for Ladybug LIST columns (`UUID[]`, `STRING[]`, …). - - Must not use `json(...)`: assigning a JSON value to `UUID[]` yields - `Conversion exception: Invalid UUID` (e.g. Frame.`shapes` on component - instantiate via sync)." - [ladybug-type v] - (let [elem-type (subs ladybug-type 0 (- (count ladybug-type) 2)) - elems (if (coll? v) (seq v) [v])] - (str "[" - (str/join ", " (map #(format-list-element elem-type %) elems)) - "]"))) - -(defn format-map - "Cypher literal for a `MAP(STRING, STRING)` column. - - Ladybug's CSV reader parses map literals (`{k=v, …}`) with no escape - mechanism: a comma inside a value ends the entry, and quotes are kept as part - of the string. Nothing user-authored — a design-token name, say — survives - that round-trip, so map columns are written through Cypher instead, where - `map/2` takes two properly escaped lists (`app.graph.bulk`). - - `key-fn` renders each key; the caller supplies it because the right form is - a property of the column, not of this function - (`app.graph.schema.contract/map-key-fn`)." - ([m] (format-map m name)) - ([m key-fn] - (let [entries (seq m)] - (str "map([" (str/join ", " (map #(format-string (key-fn (key %))) entries)) "], " - "[" (str/join ", " (map #(format-string (str (val %))) entries)) "])")))) - (defn map-type? "Is `ladybug-type` a MAP column? Those cannot be bulk-loaded from CSV." [ladybug-type] - (and (string? ladybug-type) (str/starts-with? ladybug-type "MAP("))) + (and (string? ladybug-type) + (str/starts-with? ladybug-type "MAP(") + (not (str/ends-with? ladybug-type "]")))) + +(defn list-type? + "Is this a list or fixed-size array type? Checked before MAP and STRUCT, + since `STRUCT(…)[]` starts with `STRUCT(` but is a list of them." + [ladybug-type] + (and (string? ladybug-type) + (some? (re-matches #".+\[\d*\]$" ladybug-type)))) + +(defn struct-type? + [ladybug-type] + (and (string? ladybug-type) + (str/starts-with? ladybug-type "STRUCT(") + (not (list-type? ladybug-type)))) + +(declare format-typed-value) + +(defn- format-typed-list + "Cypher LIST literal, elements formatted by the element type. + + Handles `T[]` and the fixed-size `T[n]` alike: the size constrains the column, + not the literal." + [ladybug-type v] + (let [element (second (re-matches #"(.+?)\[\d*\]$" ladybug-type)) + elems (if (or (sequential? v) (set? v)) (seq v) [v])] + (str "[" (str/join ", " (map #(format-typed-value element %) elems)) "]"))) + +(defn- format-struct + "Cypher STRUCT literal, `{field: value, …}`. + + *Every* declared field is emitted, NULL where the value has none: a struct + literal's type is its field list, so omitting a field yields a different type + and Ladybug refuses the implicit cast (`STRUCT(m2 DOUBLE, m4 DOUBLE)` cannot + be assigned to `STRUCT(m1 …, m2 …, m3 …, m4 …)`). Penpot's layout margins are + exactly that case — a shape sets only the sides it overrides." + [ladybug-type v] + (let [fields (values/struct-fields ladybug-type)] + (str "{" + (str/join ", " + (for [[field field-type] fields + :let [fv (get v field)]] + ;; Backticked for the same reason as in the DDL: a field + ;; named `column` is a keyword and will not parse bare. + ;; A bare NULL is typed STRING, which changes the struct's + ;; type as surely as omitting the field would, so absent + ;; fields get a NULL cast to their declared type. + (str "`" field "`: " + (if (nil? fv) + (str "cast(NULL, '" field-type "')") + (format-typed-value field-type fv))))) + "}"))) (defn format-typed-value - [ladybug-type v] - (cond - (nil? v) - "NULL" + "Cypher literal for `v` in a column of `ladybug-type`. - (map-type? ladybug-type) - (format-map v) + Recursive over the type language, because the types are: a + `MAP(UUID, STRUCT(…))` needs its keys, its fields and each field's own type + honoured. `app.graph.schema.values/coerce` shapes the value first — turning a + matrix record into six doubles, a hex colour into a packed integer — so this + function only has to escape plain data. - (= ladybug-type "JSON") - (format-json v) + `map-key-fn` renders the keys of a `MAP(STRING, …)`; the caller supplies it + because the right form is a property of the column, not of this function + (`app.graph.schema.contract/map-key-fn`)." + ([ladybug-type v] (format-typed-value ladybug-type v nil)) + ([ladybug-type v map-key-fn] + (let [v (values/coerce ladybug-type v)] + (cond + (nil? v) + "NULL" - ;; Coerce string ids from transit edge-cases into UUID literals. - (= ladybug-type "UUID") - (format-uuid v) + (list-type? ladybug-type) + (format-typed-list ladybug-type v) - (= ladybug-type "TIMESTAMP") - (format-timestamp v) + (map-type? ladybug-type) + (let [[key-type value-type] (values/map-types ladybug-type) + entries (seq v) + format-key (if (and map-key-fn (= "STRING" key-type)) + #(format-string (map-key-fn (key %))) + #(format-typed-value key-type (key %)))] + (str "map([" (str/join ", " (map format-key entries)) + "], [" + (str/join ", " (map #(format-typed-value value-type (val %)) entries)) + "])")) - (and (string? ladybug-type) - (str/ends-with? ladybug-type "[]")) - (format-list ladybug-type v) + (struct-type? ladybug-type) + (format-struct ladybug-type v) - :else - (format-value v))) + (= ladybug-type "JSON") + (format-json v) + + ;; Coerce string ids from transit edge-cases into UUID literals. + (= ladybug-type "UUID") + (format-uuid v) + + (= ladybug-type "TIMESTAMP") + (format-timestamp v) + + :else + (format-value v))))) (defn- ensure-semicolon [statement] diff --git a/backend/src/app/graph/project/transforms.clj b/backend/src/app/graph/project/transforms.clj index a49d17f89c..3735814b66 100644 --- a/backend/src/app/graph/project/transforms.clj +++ b/backend/src/app/graph/project/transforms.clj @@ -54,8 +54,9 @@ "One statement per (from, to) shape-table pair. Ladybug cannot create a relationship bound by multiple node labels in a - single `MERGE` (kuzudb/kuzu#5841), which is why beadpot loops over label - pairs too; the loop is a dialect constraint, not a modelling choice." + single `MERGE` — inherited from Kùzu, which it forks (upstream issue + kuzudb/kuzu#5841). beadpot loops over label pairs for the same reason; the + loop is a dialect constraint, not a modelling choice." [f] (for [from nodes/shape-tables to nodes/shape-tables] diff --git a/backend/src/app/graph/schema/contract.clj b/backend/src/app/graph/schema/contract.clj index 0b20b7f592..7d2c74aebd 100644 --- a/backend/src/app/graph/schema/contract.clj +++ b/backend/src/app/graph/schema/contract.clj @@ -99,7 +99,21 @@ moving the whole DDL onto the beadpot manifest." {;; `LinkAppliedTokens` (beadpot) reads this with `map_keys` / ;; `map_extract`; as JSON the transform cannot run at all. - "applied_tokens" "MAP(STRING, STRING)"}) + "applied_tokens" "MAP(STRING, STRING)" + + ;; `grc/schema:rect` is an inline `:and` over a map, not the registered + ;; `::grc/rect`, so `app.graph.schema.types` cannot recognize it by type. + ;; Four doubles rather than the eight-field struct: `x1`/`y1`/`x2`/`y2` are + ;; derivable from `x`/`y`/`width`/`height`, and a fixed-size array is a + ;; tensor row a consumer reads without parsing. + "selrect" "DOUBLE[4]" + "svg_viewbox" "DOUBLE[4]" + + ;; `:fills` is an `:or` — the packed `app.common.types.fills` value or a + ;; plain vector of fill maps — so the schema alone cannot say it is a + ;; collection. It always is one, and a fill has enough optional shape + ;; (solid, gradient, image) that JSON per element is the honest element type. + "fills" "JSON[]"}) (def ^:private map-key-fns "How to render the *keys* of a MAP column, per column. diff --git a/backend/src/app/graph/schema/nodes.clj b/backend/src/app/graph/schema/nodes.clj index 2fde3cb23d..b80cb1ae99 100644 --- a/backend/src/app/graph/schema/nodes.clj +++ b/backend/src/app/graph/schema/nodes.clj @@ -198,9 +198,14 @@ :cause e)))))) (defn- get-projected-attr + "The attribute under `k`, keyword or string key. + + `if-some`, not `or`: `false` and `0` are values, and falling through on them + is how `opacity 0` became `nil` and then the column default." [attrs k] - (or (get attrs k) - (when (keyword? k) (get attrs (name k))))) + (if-some [v (get attrs k)] + v + (when (keyword? k) (get attrs (name k))))) (defn- raise-empty-projection! [table attrs] @@ -213,10 +218,13 @@ (defn project-attrs "Select and validate the projected columns for `table` from `attrs`." [table attrs] + ;; `some?`, not truthiness: `false` and `0` are values. Dropping them sent + ;; `opacity 0` to the column default of 1.0 — a fully transparent shape + ;; projected as opaque. (let [projected (into {} (keep (fn [k] - (when-let [v (get-projected-attr attrs k)] - [k v])) + (let [v (get-projected-attr attrs k)] + (when (some? v) [k v]))) (column-keys table)))] (when (empty? projected) (raise-empty-projection! table attrs)) @@ -242,12 +250,9 @@ by the bulk loader's post-COPY fixups and by the incremental sync alike, so the two cannot disagree about a value's shape." [table k v] - (let [ladybug-type (column-ladybug-type table k)] - (if (ladybug/map-type? ladybug-type) - (if (nil? v) - "NULL" - (ladybug/format-map v (contract/map-key-fn (column-name table k)))) - (ladybug/format-typed-value ladybug-type v)))) + (ladybug/format-typed-value (column-ladybug-type table k) + v + (contract/map-key-fn (column-name table k)))) (defn- create-node-table-ddl [{:keys [table pk]}] diff --git a/backend/src/app/graph/schema/types.clj b/backend/src/app/graph/schema/types.clj index 75a5a12c69..73c21e572c 100644 --- a/backend/src/app/graph/schema/types.clj +++ b/backend/src/app/graph/schema/types.clj @@ -7,11 +7,35 @@ (ns app.graph.schema.types "Map Malli schemas to Ladybug column types. - Analogue of beadpot's `get_ladybug_type` (util/ladybug.py)." + Ladybug is schema-first and strongly typed: every property key gets its type + at table-creation time, and there is no widening later. That makes this + mapping the whole of the graph's typing, and it is worth being tight — a + column typed `DOUBLE[4]` is four numbers a consumer reads as a tensor row, + where the same value as `JSON` is text somebody has to parse and trust. So + JSON is the fallback of last resort, taken only where the Malli schema + genuinely admits shapes no single column can hold. + + Three groups, in the order the mapping tries them: + + 1. **Scalars** (`base-type->ladybug`) — the leaf Malli types. + 2. **Registered composites** (`custom-type->ladybug`) — Penpot's own value + types whose *layout* is fixed even though Malli only sees a map or a + string: a matrix is six doubles, a point two, a rect four, a hex colour + one packed integer. These are named explicitly because the tight encoding + is a modelling decision, not something derivable from the schema. + 3. **Structure** — collections become `T[]`, `:map-of` becomes `MAP(k, v)`, + and a closed map of scalars becomes a `STRUCT`. Anything that could be + more than one shape (a `:multi`, an `:or`, an optional-keyed map) becomes + `JSON`, because a Ladybug column cannot be two types. + + Every encoding here has a matching value formatter in `app.graph.ladybug`; + the two must move together, and `bp graph schema diff` is what catches it + when they do not." (:require [app.common.logging :as l] [app.common.schema :as sm] [app.common.time :as ct] + [clojure.string :as str] [malli.core :as m])) (def ^:private malli-opts sm/default-options) @@ -34,27 +58,117 @@ :keyword "STRING" :inst "TIMESTAMP"}) +(def ^:private custom-type->ladybug + "Penpot value types with a fixed layout Malli does not express. + + Fixed-size arrays are the point of each: they are dense, they need no + parsing, and a consumer can read a whole column as a tensor. + + - `::gmt/matrix` — the affine transform, `[a b c d e f]`. + - `::gpt/point` — `[x y]`. + - `::grc/rect` — `[x y width height]`. `x1`/`y1`/`x2`/`y2` are dropped: they + are derivable from those four, and carrying them would double the column. + - `::clr/hex-color` — `#RRGGBB` packed as `0xRRGGBBAA`, so colours compare + and group without string handling." + {:app.common.geom.matrix/matrix "DOUBLE[6]" + :app.common.geom.point/point "DOUBLE[2]" + :app.common.geom.rect/rect "DOUBLE[4]" + :app.common.types.color/hex-color "UINT32"}) + +(def ^:private collection-types + #{:vector :sequential :set ::sm/vec ::sm/set ::sm/coll}) + +(def ^:private string-collection-types + "Registered collection schemas whose element type is not in `children`." + {::sm/set-of-strings "STRING[]" + ::sm/set-of-keywords "STRING[]" + ::sm/set-of-uuid "UUID[]" + ::sm/vec-of-uuid "UUID[]"}) + (defn- normalize-schema + "Resolve refs, but stop at a schema this namespace maps explicitly. + + Order matters: `::grc/rect` derefs to an `:and` over a map, and following + that would lose the fixed-size-array encoding." [schema] (let [s (sm/schema schema)] - (if (m/-ref-schema? s) + (if (and (m/-ref-schema? s) + (not (contains? custom-type->ladybug (m/type s))) + (not (contains? string-collection-types (m/type s)))) (recur (m/deref s malli-opts)) s))) +(declare ladybug-type) + +(defn- entry-child + "The value schema of a Malli map entry (`[k s]` or `[k props s]`)." + [entry] + (if (> (count entry) 2) (nth entry 2) (nth entry 1))) + +(defn- entry-optional? + [entry] + (and (> (count entry) 2) + (:optional (nth entry 1)))) + +(defn- struct-type + "`STRUCT(...)` for a closed map of scalars, or nil when JSON is the honest answer. + + A struct is a fixed layout: every field present, every field a single type. + An optional key would make the column's shape depend on the row, and a nested + collection or map makes it recursive — Ladybug allows nesting, but a consumer + reading such a column gains nothing over JSON, so the line is drawn at + scalars." + [s] + (let [entries (m/entries s malli-opts)] + (when (and (seq entries) + (not-any? entry-optional? entries)) + (let [fields (for [entry entries + :let [t (ladybug-type (entry-child entry))]] + (when (and t + (not= "JSON" t) + (not (str/includes? t "("))) + ;; snake_case like a column name, and always + ;; backtick-quoted: a grid cell has a field called + ;; `column`, which is a Ladybug keyword, and an unquoted + ;; one fails to parse in the DDL *and* in every literal. + ;; The catalog reports them unquoted. + (str "`" (str/replace (name (key entry)) "-" "_") "` " t)))] + (when (every? some? fields) + (str "STRUCT(" (str/join ", " fields) ")")))))) + (defn ladybug-type "Return the Ladybug column type for a Malli child schema." [schema] (let [s (normalize-schema schema) t (m/type s)] (or (base-type->ladybug t) + (custom-type->ladybug t) + (string-collection-types t) + (when (contains? collection-types t) + (when-let [child (first (m/children s malli-opts))] + (str (ladybug-type child) "[]"))) (case t (:maybe :and) (ladybug-type (first (m/children s malli-opts))) - (:vector :sequential :set) - (str (ladybug-type (first (m/children s malli-opts))) "[]") - :enum "STRING" - (:map :map-of) "JSON" - (do - (l/wrn :hint "unmapped malli type for ladybug column, defaulting to STRING" - :malli-type t) - "STRING"))))) + ;; `::sm/one-of` is how Penpot spells a closed set of keywords — + ;; `:blend-mode`, `:grow-type`, every `:layout-*`. One keyword, one + ;; string. + (:enum ::sm/one-of) "STRING" + + :map-of + (let [[key-schema value-schema] (m/children s malli-opts)] + (str "MAP(" (ladybug-type key-schema) ", " + (ladybug-type value-schema) ")")) + + :map (or (struct-type s) "JSON") + + ;; A schema we do not recognize. If it has no children it is a leaf — + ;; one of Penpot's registered keyword or enum schemas, say — and a + ;; string holds it exactly. If it has children it is a composite whose + ;; shape we cannot pin down, and JSON is the honest answer. + (if (empty? (m/children s malli-opts)) + "STRING" + (do + (l/wrn :hint "unmapped composite malli type, defaulting to JSON" + :malli-type t) + "JSON")))))) diff --git a/backend/src/app/graph/schema/values.clj b/backend/src/app/graph/schema/values.clj new file mode 100644 index 0000000000..20ef3f86e8 --- /dev/null +++ b/backend/src/app/graph/schema/values.clj @@ -0,0 +1,188 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.schema.values + "Shape a Penpot value into the plain data its Ladybug column type wants. + + Ladybug is strongly typed, and `app.graph.schema.types` maps Penpot's Malli + schemas onto types as tight as it can — a matrix is `DOUBLE[6]`, a rect + `DOUBLE[4]`, a colour `UINT32`, a closed map a `STRUCT`. A tight column is + only worth having if the writer actually fills it in that shape, which is + what this namespace does: it turns records and maps into the numbers, vectors + and plain maps the type names. + + It deliberately stops there. Escaping — Cypher literals, CSV cells — belongs + to the writer (`app.graph.ladybug`, `app.graph.bulk`), so that shaping a + value and serializing it are separate concerns and each has one home. + + The type language is the Ladybug one, read recursively: `T[]`, `T[n]`, + `MAP(k, v)`, `STRUCT(name t, …)`. Anything else is passed through." + (:require + [app.common.geom.matrix :as gmt] + [app.common.geom.point :as gpt] + [app.common.types.color :as clr] + [clojure.string :as str])) + +(defn- split-args + "Split a comma-separated type argument list, respecting nesting. + + `\"UUID, STRUCT(a INT64, b INT64)\"` → `[\"UUID\" \"STRUCT(a INT64, b INT64)\"]`." + [s] + (loop [chars (seq s) depth 0 current (StringBuilder.) out []] + (if-let [c (first chars)] + (cond + (and (= c \,) (zero? depth)) + (recur (rest chars) depth (StringBuilder.) (conj out (str/trim (str current)))) + + (or (= c \() (= c \[)) + (recur (rest chars) (inc depth) (.append current c) out) + + (or (= c \)) (= c \])) + (recur (rest chars) (dec depth) (.append current c) out) + + :else + (recur (rest chars) depth (.append current c) out)) + (let [last-arg (str/trim (str current))] + (cond-> out (seq last-arg) (conj last-arg)))))) + +(defn- parse-list + "`[element-type]` when `t` is a list or fixed-size array type, else nil. + + `DOUBLE[]` and `DOUBLE[4]` are both lists of doubles as far as shaping goes; + the size only matters to the DDL." + [t] + (when-let [[_ element] (re-matches #"(.+?)\[\d*\]$" t)] + [element])) + +(defn- parse-map + "`[key-type value-type]` when `t` is a MAP type, else nil." + [t] + (when-let [[_ args] (re-matches #"MAP\((.*)\)$" t)] + (let [[k v] (split-args args)] + (when (and k v) [k v])))) + +(defn- parse-struct + "`[[field-name field-type] …]` when `t` is a STRUCT type, else nil. + + Field names arrive backtick-quoted (see `app.graph.schema.types`); the + quoting is syntax, so it is stripped here and re-applied by the writer." + [t] + (when-let [[_ args] (re-matches #"STRUCT\((.*)\)$" t)] + (for [arg (split-args args) + :let [idx (str/index-of arg " ")] + :when idx] + [(str/replace (subs arg 0 idx) "`" "") + (str/trim (subs arg (inc idx)))]))) + +(def ^:private struct-field-keys + "Field name → the Penpot keys that may hold it. + + A STRUCT field name is the snake_case of the Penpot key, but a value arrives + with its original key, and some arrive from JSON with the string form. Both + are tried before giving up." + (memoize + (fn [field] + [(keyword (str/replace field "_" "-")) + (keyword field) + field + (str/replace field "_" "-")]))) + +(defn- struct-field + [value field] + (some (fn [k] (when (contains? value k) (get value k))) + (struct-field-keys field))) + +(defn- fixed-vector + "`v` as a plain vector of numbers, for a `DOUBLE[n]` column. + + Records come first because they are what a realized snapshot holds; the map + forms are what a JSON round-trip leaves behind." + [v] + (cond + (gmt/matrix? v) [(:a v) (:b v) (:c v) (:d v) (:e v) (:f v)] + (gpt/point? v) [(:x v) (:y v)] + + ;; A rect: four of the eight fields, the rest being derivable. + (and (map? v) (contains? v :width) (contains? v :height)) + [(:x v) (:y v) (:width v) (:height v)] + + (and (map? v) (contains? v :x) (contains? v :y)) + [(:x v) (:y v)] + + (and (map? v) (contains? v :a) (contains? v :f)) + [(:a v) (:b v) (:c v) (:d v) (:e v) (:f v)] + + (sequential? v) (vec v) + :else nil)) + +(defn- packed-color + "`#RRGGBB` as `0xRRGGBBAA`, the form beadpot's `ColorValue` packs. + + Alpha defaults to opaque: the column holds a colour, and any opacity Penpot + keeps alongside it is a separate attribute." + [v] + (cond + (integer? v) v + (and (string? v) (clr/valid-hex-color? v)) + (let [rgb (Long/parseLong (subs v 1) 16)] + (bit-or (bit-shift-left rgb 8) 0xFF)) + :else nil)) + +(def struct-fields + "`[[field-name field-type] …]` for a STRUCT type, memoized. + + Public because the writers need the same field list to emit a literal." + (memoize (fn [ladybug-type] (vec (parse-struct ladybug-type))))) + +(def map-types + "`[key-type value-type]` for a MAP type, memoized." + (memoize (fn [ladybug-type] (parse-map ladybug-type)))) + +(declare coerce) + +(defn- coerce-struct + [fields v] + (when (map? v) + (into {} + (keep (fn [[field field-type]] + (when-some [fv (struct-field v field)] + [field (coerce field-type fv)]))) + fields))) + +(defn coerce + "`v` as the plain data a column of `ladybug-type` holds. + + Returns `nil` when the value cannot be shaped that way, which callers treat + as \"write NULL\" — a wrong shape in a strongly typed column fails the whole + load, so declining is better than guessing." + [ladybug-type v] + (cond + (nil? v) nil + (not (string? ladybug-type)) v + + (= "UINT32" ladybug-type) (packed-color v) + + ;; Fixed-size numeric arrays are records: matrix, point, rect. + (re-matches #"DOUBLE\[\d+\]" ladybug-type) (fixed-vector v) + + :else + (if-let [[element] (parse-list ladybug-type)] + (when (or (sequential? v) (set? v)) + ;; A set has no order, so its column would otherwise vary between + ;; builds of the same file. Sorting makes it deterministic — which is + ;; what lets two builds be diffed at all, and what a stable golden + ;; needs. Sequential values keep their order: for `shapes` and + ;; `points`, the order *is* the content. + (let [elements (mapv #(coerce element %) v)] + (if (set? v) (vec (sort-by str elements)) elements))) + (if-let [[key-type value-type] (parse-map ladybug-type)] + (when (map? v) + (into {} + (map (fn [[k mv]] [(coerce key-type k) (coerce value-type mv)])) + v)) + (if-let [fields (seq (parse-struct ladybug-type))] + (coerce-struct fields v) + v)))))