diff --git a/backend/resources/app/templates/debug.tmpl b/backend/resources/app/templates/debug.tmpl index 44097517fd..32e0732411 100644 --- a/backend/resources/app/templates/debug.tmpl +++ b/backend/resources/app/templates/debug.tmpl @@ -233,6 +233,7 @@ Debug Main Page
+ Open graph console
diff --git a/backend/resources/app/templates/graph-console.tmpl b/backend/resources/app/templates/graph-console.tmpl new file mode 100644 index 0000000000..4baf9dae1f --- /dev/null +++ b/backend/resources/app/templates/graph-console.tmpl @@ -0,0 +1,108 @@ +{% extends "app/templates/base.tmpl" %} + +{% block title %} +Graph Console +{% endblock %} + +{% block content %} + +
+
+

← Back to debug

+ +
+ Load graph in memory + + Projects the Penpot file into an in-memory Ladybug database for this + admin session. Loading a new file replaces the previous one. + +
+
+ +
+
+ +
+
+ {% if session %} +
+
+ +
+
+ {% endif %} +
+ + {% if session %} +
+ Loaded session + +

+ File: {{session.name}} ({{session.file-id}})
+ Revision: {{session.revn}}
+ Schema: {{session.schema-version}}
+ Loaded at: {{session.loaded-at}} +

+ {% if session.projection.stats %} +

+ Projection: + documents={{session.projection.stats.documents}}, + pages={{session.projection.stats.pages}}, + shapes={{session.projection.stats.shapes}} +

+ {% endif %} + +
+ +
+ Cypher query +
+
+ +
+
+ +
+
+
+ {% endif %} + + {% if error %} +
+ Error + 
{{error}}
+
+ {% endif %} + + {% if query-result %} +
+ Results ({{query-result.row-count}} rows{% if query-result.truncated? %}, truncated{% endif %}) + + + + {% for column in query-result.columns %} + + {% endfor %} + + + + {% for row in query-result.rows %} + + {% for cell in row %} + + {% endfor %} + + {% endfor %} + +
{{column}}
{{cell}}
+
+ {% endif %} +
+
+{% endblock %} diff --git a/backend/src/app/graph/bulk.clj b/backend/src/app/graph/bulk.clj index 1e057ccb3c..9d24cb5dcc 100644 --- a/backend/src/app/graph/bulk.clj +++ b/backend/src/app/graph/bulk.clj @@ -72,8 +72,8 @@ (.write w "from,to,position\n") (doseq [{:keys [from-id to-id position]} edges] (.write w (str (csv-cell from-id) "," - (csv-cell to-id) "," - (csv-cell position) "\n"))))) + (csv-cell to-id) "," + (csv-cell position) "\n"))))) (defn- delete-tree! [path] diff --git a/backend/src/app/graph/debug.clj b/backend/src/app/graph/debug.clj new file mode 100644 index 0000000000..b1e6d32416 --- /dev/null +++ b/backend/src/app/graph/debug.clj @@ -0,0 +1,117 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.debug + "In-memory Ladybug sessions for the debug graph console." + (:require + [app.common.exceptions :as ex] + [app.common.time :as ct] + [app.graph.ingest :as graph.ingest] + [app.graph.ladybug :as ladybug] + [clojure.string :as str]) + (:import + com.ladybugdb.Connection + com.ladybugdb.Database)) + +(set! *warn-on-reflection* true) + +(def default-query + "MATCH (n:Document) RETURN n.id AS id, n.name AS name;") + +(defonce ^:private sessions + (atom {})) + +(defn- session-key + [profile-id] + (str profile-id)) + +(defn- destroy-session! + [{:keys [conn db]}] + (when conn + (ex/ignoring (.close ^Connection conn))) + (when db + (ex/ignoring (.close ^Database db)))) + +(defn- format-cell + [value] + (cond + (nil? value) "NULL" + (string? value) value + :else (str value))) + +(defn- format-query-result + [{:keys [columns rows truncated?]}] + {:columns (mapv str columns) + :rows (mapv (fn [row] + (mapv format-cell row)) + rows) + :truncated? truncated? + :row-count (count rows)}) + +(defn session-info + "Return a public view of the current session for `profile-id`, if any." + [profile-id] + (when-let [{:keys [file-id meta loaded-at]} (get @sessions (session-key profile-id))] + {:file-id file-id + :name (:name meta) + :revn (:revn meta) + :schema-version (:schema-version meta) + :projection (:projection meta) + :loaded-at (ct/format-inst loaded-at :iso)})) + +(defn unload-session! + "Close and discard the in-memory graph for `profile-id`." + [profile-id] + (when-let [session (get @sessions (session-key profile-id))] + (destroy-session! session)) + (swap! sessions dissoc (session-key profile-id))) + +(defn load-session! + "Ingest `file-id` into a new in-memory Ladybug database for `profile-id`." + [cfg profile-id file-id] + (unload-session! profile-id) + (let [^Database db (Database.) + ^Connection conn (Connection. db)] + (.setQueryTimeout conn 0) + (try + (let [meta (graph.ingest/ingest-on-connection! cfg conn file-id + :db-path ":memory:" + :skip-stats? true + :skip-validation? true)] + (swap! sessions assoc (session-key profile-id) + {:db db + :conn conn + :file-id file-id + :meta meta + :loaded-at (ct/now)}) + meta) + (catch Throwable cause + (destroy-session! {:conn conn :db db}) + (throw cause))))) + +(defn query-session! + "Run `statement` against the in-memory graph for `profile-id`." + [profile-id statement] + (when (str/blank? statement) + (ex/raise :type :validation + :code :missing-query + :hint "cypher query is required")) + (if-let [{:keys [conn]} (get @sessions (session-key profile-id))] + (-> (ladybug/query-on-connection! conn statement) + format-query-result) + (ex/raise :type :not-found + :code :graph-session-not-loaded + :hint "load a file graph before running queries"))) + +(defn console-context + "Build template data for the graph debug console page." + [profile-id & {:keys [query query-result error message]}] + {:session (session-info profile-id) + :query (or query default-query) + :query-result query-result + :error error + :message message + :default-query default-query}) diff --git a/backend/src/app/graph/ingest.clj b/backend/src/app/graph/ingest.clj index 6fedba50c5..c17cc5320f 100644 --- a/backend/src/app/graph/ingest.clj +++ b/backend/src/app/graph/ingest.clj @@ -18,14 +18,14 @@ [app.graph.project.transforms :as project.transforms] [app.graph.schema :as schema] [app.graph.stats :as stats] - [app.srepl.helpers :as h])) + [app.srepl.helpers :as h]) + (:import + com.ladybugdb.Connection)) -(defn ingest-file! - [system file-id & {:keys [db-path reset-db? skip-stats?] - :or {reset-db? true}}] +(defn- fetch-file! + [system file-id] (let [file-id (h/parse-uuid file-id) - file (db/run! system #(bfc/get-file % file-id :realize? true)) - db-path (or db-path (ladybug/db-path-for-file file-id))] + file (db/run! system #(bfc/get-file % file-id :realize? true))] (when-not file (ex/raise :type :not-found :code :file-not-found @@ -35,30 +35,48 @@ :code :file-without-data :hint "file has no data to project" :file-id (str file-id))) - (ctf/check-file-data (:data file)) - (when reset-db? - (ladybug/reset-db-path! db-path)) + [file-id file])) + +(defn ingest-on-connection! + "Project `file-id` into an already open Ladybug `conn`." + [system ^Connection conn file-id & {:keys [db-path skip-stats? skip-validation?] + :or {skip-stats? true}}] + (let [[file-id file] (fetch-file! system file-id) + db-path (or db-path (ladybug/db-path-for-file file-id)) + data (:data file)] + (when-not skip-validation? + (ctf/check-file-data data)) (l/inf :hint "graph ingest" :file-id (str file-id) :revn (:revn file) :db-path db-path :schema schema/schema-version) - (let [data (:data file) - ddl (schema/ddl-statements) + (let [ddl (schema/ddl-statements) {:keys [nodes edges stats]} (project.document/projection-data data file) - staging-path (bulk/staging-dir db-path file-id)] - (ladybug/with-connection! db-path - (fn [conn] - (ladybug/exec-on-connection! conn ddl) - (bulk/load-projection! conn {:nodes nodes :edges edges} staging-path) - (ladybug/exec-on-connection! conn ["CHECKPOINT;"]) - {:file-id file-id - :revn (:revn file) - :name (or (:name data) (:name file)) - :db-path db-path - :schema-version schema/schema-version - :projection {:stats stats} - :transforms (project.transforms/apply-transforms! system db-path data file) - :stats (when-not skip-stats? - (stats/summarize-connection conn))}))))) + staging-path (bulk/staging-dir db-path file-id)] + (ladybug/exec-on-connection! conn ddl) + (bulk/load-projection! conn {:nodes nodes :edges edges} staging-path) + (ladybug/exec-on-connection! conn ["CHECKPOINT;"]) + {:file-id file-id + :revn (:revn file) + :name (or (:name data) (:name file)) + :db-path db-path + :schema-version schema/schema-version + :projection {:stats stats} + :transforms (project.transforms/apply-transforms! system db-path data file) + :stats (when-not skip-stats? + (stats/summarize-connection conn))}))) + +(defn ingest-file! + [system file-id & {:keys [db-path reset-db? skip-stats? skip-validation?] + :or {reset-db? true}}] + (let [db-path (or db-path (ladybug/db-path-for-file (h/parse-uuid file-id)))] + (when reset-db? + (ladybug/reset-db-path! db-path)) + (ladybug/with-connection! db-path + (fn [conn] + (ingest-on-connection! system conn file-id + :db-path db-path + :skip-stats? skip-stats? + :skip-validation? skip-validation?))))) diff --git a/backend/src/app/graph/ladybug.clj b/backend/src/app/graph/ladybug.clj index 62648ff112..533971aff7 100644 --- a/backend/src/app/graph/ladybug.clj +++ b/backend/src/app/graph/ladybug.clj @@ -92,6 +92,46 @@ :statement statement :err err)))) +(defn- query-columns + [^QueryResult result] + (let [ncols (.getNumColumns result)] + (vec (for [i (range ncols)] + (.getColumnName result (long i)))))) + +(defn- query-row + [^FlatTuple tuple ncols] + (vec (for [i (range ncols)] + (with-open [^Value value (.getValue tuple (long i))] + (value->clj value))))) + +(def ^:private default-query-max-rows 200) + +(defn- read-query-rows + [^QueryResult result ncols max-rows] + (loop [rows [] n 0] + (if (and (< n max-rows) (.hasNext result)) + (let [row (with-open [^FlatTuple tuple (.getNext result)] + (query-row tuple ncols))] + (recur (conj rows row) (inc n))) + rows))) + +(defn query-on-connection! + "Execute a Cypher query on `conn` and return tabular results. + + Returns `{:columns [...] :rows [[...] ...] :truncated? bool}`." + [^Connection conn statement & {:keys [max-rows] + :or {max-rows default-query-max-rows}}] + (let [cypher (ensure-semicolon statement)] + (with-open [^QueryResult result (.query conn cypher)] + (check-success! result cypher) + (let [ncols (long (.getNumColumns result)) + columns (query-columns result) + rows (read-query-rows result ncols max-rows) + total (long (.getNumTuples result))] + {:columns columns + :rows rows + :truncated? (and (pos? total) (> total (count rows)))})))) + (def ^:private default-query-timeout-ms "0 disables query timeout (recommended for bulk COPY ingest)." 0) diff --git a/backend/src/app/graph/stats.clj b/backend/src/app/graph/stats.clj index dee05ce2f3..a4ed69dbcd 100644 --- a/backend/src/app/graph/stats.clj +++ b/backend/src/app/graph/stats.clj @@ -30,8 +30,8 @@ "RETURN count(n) AS " name "_c;"))]) schema/node-tables)) :edges {:IsChildOf (count-on-connection - conn - "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}}) + conn + "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}}) (defn summarize "Return node/edge counts from the graph database." diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index f9b40370c5..b0816f2b34 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -21,6 +21,7 @@ [app.config :as cf] [app.db :as db] [app.features.file-migrations :as feat.fmig] + [app.graph.debug :as graph.debug] [app.graph.ingest :as graph.ingest] [app.http.session :as session] [app.rpc.commands.auth :as auth] @@ -350,6 +351,52 @@ ::yres/headers {"content-type" "application/octet-stream" "content-disposition" (str "attachment; filename=" file-id ".lbug")}}))) +(defn- graph-console-response + [profile-id data] + {::yres/status 200 + ::yres/headers {"content-type" "text/html; charset=utf-8" + "x-robots-tag" "noindex"} + ::yres/body (-> (io/resource "app/templates/graph-console.tmpl") + (tmpl/render (assoc data :version (:full cf/version))))}) + +(defn graph-console-handler + [_cfg {:keys [::session/profile-id]}] + (graph-console-response profile-id + (graph.debug/console-context profile-id))) + +(defn graph-load-handler + [cfg {:keys [params ::session/profile-id]}] + (let [file-id (some-> (:file-id params) parse-uuid)] + (when-not file-id + (ex/raise :type :validation + :code :missing-arguments + :hint "missing file-id")) + (graph.debug/load-session! cfg profile-id file-id) + {::yres/status 302 + ::yres/headers {"location" "/dbg/graph"}})) + +(defn graph-unload-handler + [_cfg {:keys [::session/profile-id]}] + (graph.debug/unload-session! profile-id) + {::yres/status 302 + ::yres/headers {"location" "/dbg/graph"}}) + +(defn graph-query-handler + [_cfg {:keys [params ::session/profile-id]}] + (let [query (:query params)] + (try + (let [result (graph.debug/query-session! profile-id query)] + (graph-console-response profile-id + (graph.debug/console-context profile-id + :query query + :query-result result))) + (catch Throwable e + (graph-console-response profile-id + (graph.debug/console-context profile-id + :query query + :error (or (:hint (ex-data e)) + (ex-message e)))))))) + (defn import-handler [{:keys [::db/pool] :as cfg} {:keys [params ::session/profile-id] :as request}] (when-not (contains? params :file) @@ -587,6 +634,7 @@ ["" {:handler (partial index-handler cfg)}] ["/health" {:handler (partial health-handler cfg)}] ["/changelog" {:handler (partial changelog-handler cfg)}] + ["/graph" {:handler (partial graph-console-handler cfg)}] ["/error/:id" {:handler (partial error-handler cfg)}] ["/error" {:handler (partial error-list-handler cfg)}] ["/actions" {:middleware [[errors]]} @@ -598,6 +646,9 @@ {:handler (partial handle-team-features cfg)}] ["/file-export" {:handler (partial export-handler cfg)}] ["/graph-export" {:handler (partial graph-export-handler cfg)}] + ["/graph-load" {:handler (partial graph-load-handler cfg)}] + ["/graph-query" {:handler (partial graph-query-handler cfg)}] + ["/graph-unload" {:handler (partial graph-unload-handler cfg)}] ["/file-import" {:handler (partial import-handler cfg)}] ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]])