diff --git a/.serena/.gitignore b/.serena/.gitignore index 2e510aff58..964f4226bb 100644 --- a/.serena/.gitignore +++ b/.serena/.gitignore @@ -1,2 +1,3 @@ /cache /project.local.yml +/diagrams/out/ diff --git a/.serena/diagrams/architecture-flow.mmd b/.serena/diagrams/architecture-flow.mmd new file mode 100644 index 0000000000..ee18ed9b3b --- /dev/null +++ b/.serena/diagrams/architecture-flow.mmd @@ -0,0 +1,60 @@ +%% Penpot runtime architecture: client, backend, services, workers, export. +flowchart TB + user(["User
web browser"]) + + subgraph client["Client · browser"] + fe["Frontend SPA
ClojureScript + React"] + ww["Web Worker
background computation"] + wasm["render-wasm
Rust / Skia → WASM"] + plugins["Plugin runtime
sandboxed"] + end + + subgraph server["Backend · JVM / Clojure"] + rpc["RPC API
/api/rpc/query · mutation"] + ws["Notifications
WebSocket"] + http["HTTP routes
assets · media · export"] + dispatcher["Dispatcher
claims task rows"] + runner["Runners
execute tasks"] + end + + subgraph data["Stateful services"] + pg[("PostgreSQL
durable store + task queue")] + redis[("Redis / Valkey
pub/sub · cache · rate limit")] + obj[("Object storage
S3 or filesystem")] + end + + subgraph export["Export pipeline"] + exporter["Exporter
ClojureScript / Node"] + chrome["Headless Chrome
Playwright"] + end + + media["media-processor
sharp · FontForge"] + mail["SMTP mailer"] + + user --> fe + fe <--> ww + fe --> wasm + fe --> plugins + fe -->|"HTTPS · Transit"| rpc + fe <-->|"WebSocket"| ws + fe -->|"HTTPS"| http + fe -->|"HTTPS export"| exporter + + rpc -->|"SQL"| pg + rpc -->|"enqueue task"| pg + rpc --> redis + rpc --> obj + ws <--> redis + http --> obj + + pg -->|"FOR UPDATE SKIP LOCKED"| dispatcher + dispatcher -->|"RPUSH claimed ids"| redis + redis -->|"BLPOP"| runner + runner --> pg + runner --> obj + runner --> mail + runner --> media + media --> obj + + exporter --> chrome + chrome -->|"loads app"| fe diff --git a/.serena/diagrams/file-edit-flow.mmd b/.serena/diagrams/file-edit-flow.mmd new file mode 100644 index 0000000000..42c56299ab --- /dev/null +++ b/.serena/diagrams/file-edit-flow.mmd @@ -0,0 +1,19 @@ +%% Collaborative edit flow: one user's change reaches the other clients. +sequenceDiagram + autonumber + participant U as User / browser + participant FE as Frontend SPA + participant RPC as Backend RPC + participant PG as PostgreSQL + participant MB as Redis pub/sub + participant WS as WebSocket + participant O as Other clients + + U->>FE: edits a shape + FE->>RPC: mutation update-file (Transit) + RPC->>PG: persist change, bump revision + RPC->>MB: publish file-change on file topic + RPC-->>FE: mutation result + MB-->>WS: deliver file-change + WS-->>O: push change over WebSocket + O->>O: apply change locally diff --git a/.serena/memories/architecture/core.md b/.serena/memories/architecture/core.md new file mode 100644 index 0000000000..d70ae43dcf --- /dev/null +++ b/.serena/memories/architecture/core.md @@ -0,0 +1,31 @@ +# Architecture Diagrams + +Canonical, versioned diagrams of the Penpot runtime and its flows, meant for humans and agents. + +## Format and location + +- Source of truth: `.serena/diagrams/**/*.mmd`, one Mermaid diagram per file; plain text, git-versioned, diff-friendly, and readable by an LLM without rendering. +- Mermaid is the chosen format because LLMs generate it reliably and it renders in a browser and in HTML without a build step. +- `architecture-flow.mmd` is the runtime topology: browser client, JVM backend, stateful services, task-worker model, and export pipeline. +- `file-edit-flow.mmd` is the collaborative edit sequence: RPC mutation → PostgreSQL → Redis pub/sub → WebSocket → peer clients. +- The rendered HTML index is generated, not versioned: `.serena/diagrams/out/` is gitignored. + +## Rendering + +- HTML index: `scripts/render-diagrams` writes `.serena/diagrams/out/index.html`; Mermaid is imported from a pinned jsDelivr CDN, so opening the page needs network. +- SVG or PNG: `scripts/render-diagrams --format svg` (or `png`) calls `mmdc` from `@mermaid-js/mermaid-cli`; install it with `pnpm add -D @mermaid-js/mermaid-cli`. +- Render a single diagram with `scripts/render-diagrams --diagram architecture-flow`. +- Tests for the renderer: `python3 scripts/test_render_diagrams.py`. + +## Content sources + +- Module graph and cross-module dependencies: `mem:critical-info` (Dependency graph). +- Runtime services and the task-queue versus Pub/Sub topology that constrains horizontal scaling: `mem:prod-infra/core`. +- Backend internals, RPC, notifications, and storage backends: `mem:backend/core` and `mem:backend/storage`. +- The prose architecture pages, in C4 with PlantUML, live in `docs/technical-guide/developer/architecture/`; keep the Mermaid flow views consistent with them instead of duplicating their content. + +## Maintenance invariants + +- Update the affected `.mmd` in the same change that alters the topology it shows: a new module, a new service, or a changed flow. +- One concern per file; name files by topic with a `.mmd` suffix; keep node labels quoted so punctuation does not break the parser. +- Do not put diagram source inline in memories; reference the file so the graph stays small and the diagram stays renderable. diff --git a/.serena/memories/critical-info.md b/.serena/memories/critical-info.md index f0cec128dc..982466db2a 100644 --- a/.serena/memories/critical-info.md +++ b/.serena/memories/critical-info.md @@ -63,6 +63,10 @@ The memory is structured in a way that you can get the critical information abou `frontend -> common`, `backend -> common`, `exporter -> common`, and `frontend -> render-wasm`. Changes in `common` can affect frontend, backend, exporter, file migrations, and design-library behavior; validate across consumers when semantics change. +# Architecture diagrams + +- Versioned Mermaid diagrams of the runtime topology and the collaborative edit flow, plus how to render them to HTML or SVG/PNG: `mem:architecture/core`. + # Working with Penpot designs - Before automating or inspecting Penpot designs through the Plugin API, call the Penpot MCP `high_level_overview` tool. diff --git a/scripts/render-diagrams b/scripts/render-diagrams new file mode 100755 index 0000000000..8154c737b3 --- /dev/null +++ b/scripts/render-diagrams @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Render the versioned Mermaid architecture diagrams. + +Reads ``*.mmd`` sources from a directory (``.serena/diagrams`` by +default) and writes a single browsable HTML index. With ``--format +svg`` or ``--format png`` it shells out to ``mmdc`` from +``@mermaid-js/mermaid-cli`` instead. + +Examples:: + + scripts/render-diagrams + scripts/render-diagrams --format svg + scripts/render-diagrams --diagram architecture-flow +""" + +from __future__ import annotations + +import argparse +import html +import shutil +import subprocess +import sys +from pathlib import Path + +DEFAULT_SRC = Path(".serena/diagrams") +DEFAULT_MERMAID_VERSION = "11" +SOURCE_SUFFIXES = (".mmd", ".mermaid") + + +class RenderError(RuntimeError): + """Raised when a diagram cannot be rendered.""" + + +def discover_diagrams(src: Path, out: Path | None = None) -> list[Path]: + """Return the diagram sources under ``src``, sorted by path. + + ``out``, when given, is skipped so a previous render is never read + back as a source. + """ + if not src.is_dir(): + return [] + + ignored = out.resolve() if out is not None else None + found: list[Path] = [] + for path in src.rglob("*"): + if not path.is_file() or path.suffix.lower() not in SOURCE_SUFFIXES: + continue + if ignored is not None and ignored in path.resolve().parents: + continue + found.append(path) + return sorted(found) + + +def filter_diagrams(diagrams: list[Path], names: list[str] | None) -> list[Path]: + """Keep only the diagrams whose file stem is in ``names``.""" + if not names: + return diagrams + wanted = set(names) + return [path for path in diagrams if path.stem in wanted] + + +def build_html( + diagrams: list[Path], + src: Path, + mermaid_version: str = DEFAULT_MERMAID_VERSION, +) -> str: + """Build a self-contained HTML index embedding every diagram source.""" + sections = [] + for path in diagrams: + source = path.read_text(encoding="utf-8").strip() + name = path.relative_to(src).as_posix() + sections.append( + f'
\n' + f"

{html.escape(name)}

\n" + f'
{html.escape(source)}
\n' + f"
" + ) + + body = "\n".join(sections) if sections else "

No diagrams found.

" + return f""" + + + + +Penpot architecture diagrams + + + +

Penpot architecture diagrams

+
+{body} +
+ + + +""" + + +def render( + diagrams: list[Path], + src: Path, + out: Path, + mode: str = "html", + mermaid_version: str = DEFAULT_MERMAID_VERSION, +) -> list[Path]: + """Render ``diagrams`` into ``out`` and return the produced files.""" + out.mkdir(parents=True, exist_ok=True) + + if mode == "html": + index = out / "index.html" + index.write_text( + build_html(diagrams, src, mermaid_version), encoding="utf-8" + ) + return [index] + + if shutil.which("mmdc") is None: + raise RenderError( + "mmdc not found. Install it with " + "'pnpm add -D @mermaid-js/mermaid-cli', or use " + "--format html." + ) + + produced = [] + for path in diagrams: + stem = path.relative_to(src).with_suffix("").as_posix().replace("/", "_") + target = out / f"{stem}.{mode}" + subprocess.run( + ["mmdc", "-i", str(path), "-o", str(target)], + check=True, + ) + produced.append(target) + return produced + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--src", + type=Path, + default=DEFAULT_SRC, + help=f"directory with .mmd sources (default: {DEFAULT_SRC})", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="output directory (default: /out)", + ) + parser.add_argument( + "--format", + choices=("html", "svg", "png"), + default="html", + help="output format (default: html)", + ) + parser.add_argument( + "--diagram", + action="append", + default=None, + metavar="NAME", + help="render only this source stem (repeatable)", + ) + parser.add_argument( + "--mermaid-version", + default=DEFAULT_MERMAID_VERSION, + help=f"CDN Mermaid version for HTML (default: {DEFAULT_MERMAID_VERSION})", + ) + args = parser.parse_args(argv) + + out = args.out if args.out is not None else args.src / "out" + diagrams = filter_diagrams(discover_diagrams(args.src, out), args.diagram) + + if not diagrams: + print(f"No diagrams found in {args.src}", file=sys.stderr) + return 1 + + try: + produced = render( + diagrams, args.src, out, args.format, args.mermaid_version + ) + except (RenderError, subprocess.CalledProcessError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + for path in produced: + print(path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_render_diagrams.py b/scripts/test_render_diagrams.py new file mode 100644 index 0000000000..d671c2b544 --- /dev/null +++ b/scripts/test_render_diagrams.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Tests for scripts/render-diagrams. + +Run with: + + python3 scripts/test_render_diagrams.py +""" + +import importlib.machinery +import importlib.util +import io +import pathlib +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from unittest.mock import patch + +# Loading scripts/render-diagrams should not emit scripts/__pycache__/. +sys.dont_write_bytecode = True + +SCRIPT_PATH = pathlib.Path(__file__).resolve().parent / "render-diagrams" + + +def load_render_diagrams(): + """Load scripts/render-diagrams as a module without running its CLI.""" + loader = importlib.machinery.SourceFileLoader( + "render_diagrams_helper", str(SCRIPT_PATH) + ) + spec = importlib.util.spec_from_loader("render_diagrams_helper", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +rd = load_render_diagrams() + + +class DiscoverDiagramsTests(unittest.TestCase): + def test_finds_and_sorts_known_suffixes(self): + with tempfile.TemporaryDirectory() as tmp: + src = pathlib.Path(tmp) + (src / "b.mmd").write_text("graph LR", encoding="utf-8") + (src / "a.mermaid").write_text("graph LR", encoding="utf-8") + (src / "notes.txt").write_text("ignore me", encoding="utf-8") + + found = rd.discover_diagrams(src) + + self.assertEqual([p.name for p in found], ["a.mermaid", "b.mmd"]) + + def test_skips_output_directory(self): + with tempfile.TemporaryDirectory() as tmp: + src = pathlib.Path(tmp) + (src / "real.mmd").write_text("graph LR", encoding="utf-8") + out = src / "out" + out.mkdir() + (out / "stale.mmd").write_text("graph LR", encoding="utf-8") + + found = rd.discover_diagrams(src, out) + + self.assertEqual([p.name for p in found], ["real.mmd"]) + + def test_missing_directory_returns_empty(self): + self.assertEqual(rd.discover_diagrams(pathlib.Path("/no/such/dir")), []) + + +class FilterDiagramsTests(unittest.TestCase): + def test_none_keeps_everything(self): + diagrams = [pathlib.Path("a.mmd"), pathlib.Path("b.mmd")] + self.assertEqual(rd.filter_diagrams(diagrams, None), diagrams) + + def test_keeps_only_requested_stems(self): + a = pathlib.Path("a.mmd") + b = pathlib.Path("b.mmd") + self.assertEqual(rd.filter_diagrams([a, b], ["b"]), [b]) + + +class BuildHtmlTests(unittest.TestCase): + def test_embeds_source_and_mermaid_version(self): + with tempfile.TemporaryDirectory() as tmp: + src = pathlib.Path(tmp) + diagram = src / "flow.mmd" + diagram.write_text("flowchart LR\n a --> b\n", encoding="utf-8") + + page = rd.build_html([diagram], src, mermaid_version="11") + + self.assertIn('class="mermaid"', page) + self.assertIn("flow.mmd", page) + self.assertIn("a --> b", page) + self.assertIn("mermaid@11/dist/mermaid.esm.min.mjs", page) + + def test_reports_when_empty(self): + page = rd.build_html([], pathlib.Path(".")) + self.assertIn("No diagrams found", page) + + +class RenderTests(unittest.TestCase): + def test_html_writes_index(self): + with tempfile.TemporaryDirectory() as tmp: + src = pathlib.Path(tmp) / "src" + src.mkdir() + (src / "flow.mmd").write_text("graph LR", encoding="utf-8") + out = pathlib.Path(tmp) / "out" + + produced = rd.render( + rd.discover_diagrams(src, out), src, out, mode="html" + ) + + self.assertEqual([p.name for p in produced], ["index.html"]) + self.assertTrue(produced[0].read_text(encoding="utf-8").startswith("")) + + def test_svg_without_mmdc_raises(self): + with tempfile.TemporaryDirectory() as tmp: + src = pathlib.Path(tmp) / "src" + src.mkdir() + (src / "flow.mmd").write_text("graph LR", encoding="utf-8") + + with patch.object(rd.shutil, "which", return_value=None): + with self.assertRaises(rd.RenderError): + rd.render( + rd.discover_diagrams(src), + src, + pathlib.Path(tmp) / "out", + mode="svg", + ) + + +class MainTests(unittest.TestCase): + def test_missing_sources_exit_code(self): + with tempfile.TemporaryDirectory() as tmp: + stderr = io.StringIO() + with redirect_stderr(stderr): + code = rd.main(["--src", str(pathlib.Path(tmp) / "nope")]) + + self.assertEqual(code, 1) + self.assertIn("No diagrams found", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main()