penpot/frontend/scripts/_helpers.js

import proc from "node:child_process";
import crypto from "node:crypto";
import fs from "node:fs/promises";
import ph from "node:path";
import os from "node:os";
import url from "node:url";

import * as marked from "marked";
import SVGSpriter from "svg-sprite";
import Watcher from "watcher";
import gettext from "gettext-parser";
import l from "lodash";
import log from "fancy-log";
import mustache from "mustache";
import pLimit from "p-limit";
import ppt from "pretty-time";
import wpool from "workerpool";

import { buildFontsPreviewSprite } from "./build-fonts-preview.js";

function getCoreCount() {
  return os.cpus().length;
}

export const dirname = url.fileURLToPath(new URL(".", import.meta.url));

export function startWorker() {
  return wpool.pool(dirname + "/_worker.js", {
    maxWorkers: getCoreCount(),
  });
}

export const IS_DEBUG = process.env.NODE_ENV !== "production";
export const BUILD_DATE = process.env.BUILD_DATE || new Date().toString();
export const BUILD_TS = process.env.BUILD_TS || Date.now();
export const VERSION = process.env.VERSION || "develop";
export const VERSION_TAG = process.env.VERSION_TAG || VERSION;

async function findFiles(basePath, predicate, options = {}) {
  predicate =
    predicate ??
    function () {
      return true;
    };

  let files = await fs.readdir(basePath, {
    recursive: options.recursive ?? false,
  });
  files = files.map((path) => ph.join(basePath, path));

  return files;
}

function syncDirs(originPath, destPath, excludes = []) {
  const excludeArgs = excludes.map((p) => `--exclude=${p}`).join(" ");
  const command = `rsync -ar --delete ${excludeArgs} ${originPath} ${destPath}`;

  return new Promise((resolve, reject) => {
    proc.exec(command, (cause, stdout) => {
      if (cause) {
        reject(cause);
      } else {
        resolve();
      }
    });
  });
}

export function isSassFile(path) {
  return path.endsWith(".scss");
}

export function isSvgFile(path) {
  return path.endsWith(".svg");
}

export function isJsFile(path) {
  return path.endsWith(".js");
}

export async function compileSass(worker, path, options) {
  path = ph.resolve(path);

  // log.info("compile:", path);
  return worker.exec("compileSass", [path, options]);
}

export async function compileSassDebug(worker) {
  const result = await compileSass(worker, "resources/styles/debug.scss", {});
  return `${result.css}\n`;
}

export async function compileSassStorybook(worker) {
  const limitFn = pLimit(4);
  const sourceDir = ph.join("src", "app", "main", "ui", "ds");

  const dsFiles = (await fs.readdir(sourceDir, { recursive: true }))
    .filter(isSassFile)
    .map((filename) => ph.join(sourceDir, filename));
  const procs = [compileSass(worker, "resources/styles/main-default.scss", {})];

  for (let path of dsFiles) {
    const proc = limitFn(() => compileSass(worker, path, { modules: true }));
    procs.push(proc);
  }

  const result = await Promise.all(procs);
  return result.reduce(
    (acc, item) => {
      acc.index[item.outputPath] = item.css;
      acc.items.push(item.outputPath);
      return acc;
    },
    { index: {}, items: [] },
  );
}

export async function compileSassAll(worker) {
  const limitFn = pLimit(4);
  const sourceDir = "src";

  const isDesignSystemFile = (path) => {
    return path.startsWith("app/main/ui/ds/");
  };

  const isOldComponentSystemFile = (path) => {
    return path.startsWith("app/main/ui/components/");
  };

  let files = (await fs.readdir(sourceDir, { recursive: true })).filter(
    isSassFile,
  );

  const appFiles = files
    .filter((path) => !isDesignSystemFile(path))
    .filter((path) => !isOldComponentSystemFile(path))
    .map((path) => ph.join(sourceDir, path));

  const dsFiles = files
    .filter(isDesignSystemFile)
    .map((path) => ph.join(sourceDir, path));

  const oldComponentsFiles = files
    .filter(isOldComponentSystemFile)
    .map((path) => ph.join(sourceDir, path));

  const procs = [compileSass(worker, "resources/styles/main-default.scss", {})];

  for (let path of [...oldComponentsFiles, ...dsFiles, ...appFiles]) {
    const proc = limitFn(() => compileSass(worker, path, { modules: true }));
    procs.push(proc);
  }

  const result = await Promise.all(procs);

  return result.reduce(
    (acc, item) => {
      acc.index[item.outputPath] = item.css;
      acc.items.push(item.outputPath);
      return acc;
    },
    { index: {}, items: [] },
  );
}

export function concatSass(data) {
  const output = [];

  for (let path of data.items) {
    output.push(data.index[path]);
  }

  return output.join("\n");
}

export async function watch(baseDir, predicate, callback) {
  predicate = predicate ?? (() => true);

  const watcher = new Watcher(baseDir, {
    persistent: true,
    recursive: true,
    debounce: 500,
  });

  watcher.on("change", (path) => {
    if (predicate(path)) {
      callback(path);
    }
  });

  watcher.on("error", (cause) => {
    console.log("WATCHER ERROR", cause);
  });
}

export async function ensureDirectories() {
  await fs.mkdir("./resources/public/js/worker/", { recursive: true });
  await fs.mkdir("./resources/public/css/", { recursive: true });
}

async function readManifestFile(resource) {
  const manifestPath = "resources/public/" + resource;
  let content = await fs.readFile(manifestPath, { encoding: "utf8" });
  return JSON.parse(content);
}

async function generateManifest() {
  const index = {
    app_main: "./js/main.js",
    render_main: "./js/render.js",
    rasterizer_main: "./js/rasterizer.js",

    config: "./js/config.js?version=" + VERSION_TAG,
    config_render: "./js/config-render.js?version=" + VERSION_TAG,
    polyfills: "./js/polyfills.js?version=" + VERSION_TAG,
    libs: "./js/libs.js?version=" + VERSION_TAG,
    default_translations: "./js/translation.en.js?version=" + VERSION_TAG,

    importmap: JSON.stringify({
      imports: {
        "./js/shared.js": "./js/shared.js?version=" + VERSION_TAG,
        "./js/main.js": "./js/main.js?version=" + VERSION_TAG,
        "./js/render.js": "./js/render.js?version=" + VERSION_TAG,
        "./js/render-wasm.js": "./js/render-wasm.js?version=" + VERSION_TAG,
        "./js/rasterizer.js": "./js/rasterizer.js?version=" + VERSION_TAG,
        "./js/main-dashboard.js":
          "./js/main-dashboard.js?version=" + VERSION_TAG,
        "./js/main-auth.js": "./js/main-auth.js?version=" + VERSION_TAG,
        "./js/main-viewer.js": "./js/main-viewer.js?version=" + VERSION_TAG,
        "./js/main-settings.js": "./js/main-settings.js?version=" + VERSION_TAG,
        "./js/main-workspace.js":
          "./js/main-workspace.js?version=" + VERSION_TAG,
        "./js/util-highlight.js":
          "./js/util-highlight.js?version=" + VERSION_TAG,
      },
    }),
  };

  return index;
}

async function renderTemplate(path, context = {}, partials = {}) {
  const content = await fs.readFile(path, { encoding: "utf-8" });

  context = Object.assign({}, context, {
    isDebug: IS_DEBUG,
    version: VERSION,
    version_tag: VERSION_TAG,
    build_date: BUILD_DATE,
    build_ts: BUILD_TS,
  });

  return mustache.render(content, context, partials);
}

const markedOptions = {
  renderer: {
    link(token) {
      if (token.href === "mailto") {
        return `<a href="mailto:${token.text}">${token.text}</a>`;
      } else {
        let target = "_blank";

        if (token.text.endsWith("|target:self")) {
          const index = token.text.indexOf("|target:self");
          token.text = token.text.substring(0, index);
          target = "_self";
        }

        const href = token.href;
        const text = token.text;
        return `<a href="${href}" target="${target}">${text}</a>`;
      }
    },
  },
};

marked.use(markedOptions);

export async function compileTranslations() {
  const outputDir = "resources/public/js/";
  await fs.mkdir(outputDir, { recursive: true });

  const langs = [
    "ar",
    "ca",
    "de",
    "el",
    "en",
    "eu",
    "it",
    "es",
    "fa",
    "fr",
    "fr_CA",
    "he",
    "sr",
    "nb_NO",
    "pl",
    "pt_BR",
    "ro",
    "id",
    "ru",
    "tr",
    "hi",
    "zh_CN",
    "zh_Hant",
    "hr",
    "gl",
    "pt_PT",
    "cs",
    "fo",
    "ko",
    "lv",
    "nl",
    // this happens when file does not matches correct
    // iso code for the language.
    ["ja_jp", "jpn_JP"],
    ["uk", "ukr_UA"],
    "ha",
  ];

  for (let lang of langs) {
    const result = {};

    let filename = `${lang}.po`;
    if (l.isArray(lang)) {
      filename = `${lang[1]}.po`;
      lang = lang[0];
    }

    const content = await fs.readFile(`./translations/${filename}`, {
      encoding: "utf-8",
    });

    lang = lang.toLowerCase();

    const data = gettext.po.parse(content, "utf-8");
    const trdata = data.translations[""];

    for (let key of Object.keys(trdata)) {
      if (key === "") continue;
      const comments = trdata[key].comments || {};
      const isMarkdown = l.includes(comments.flag, "markdown");

      const msgs = trdata[key].msgstr;
      if (msgs.length === 1) {
        let message = msgs[0];
        if (isMarkdown) {
          message = marked.parseInline(message);
        }

        result[key] = message;
      } else {
        result[key] = msgs.map((item) => {
          if (isMarkdown) {
            return marked.parseInline(item);
          } else {
            return item;
          }
        });
      }
    }

    const esm = `export default ${JSON.stringify(result, null, 0)};\n`;
    const outputFile = ph.join(outputDir, "translation." + lang + ".js");
    await fs.writeFile(outputFile, esm);
  }
}

async function generateSvgSprite(files, prefix) {
  const spriter = new SVGSpriter({
    mode: {
      symbol: { inline: true },
    },
  });

  for (let path of files) {
    const name = `${prefix}${ph.basename(path)}`;
    const content = await fs.readFile(path, { encoding: "utf-8" });
    spriter.add(name, name, content);
  }

  const { result } = await spriter.compileAsync();
  const resource = result.symbol.sprite;
  return resource.contents;
}

async function generateSvgSprites() {
  await fs.mkdir("resources/public/images/sprites/symbol/", {
    recursive: true,
  });

  const icons = await findFiles("resources/images/icons/", isSvgFile);
  const iconsSprite = await generateSvgSprite(icons, "icon-");
  await fs.writeFile(
    "resources/public/images/sprites/symbol/icons.svg",
    iconsSprite,
  );

  const cursors = await findFiles("resources/images/cursors/", isSvgFile);
  const cursorsSprite = await generateSvgSprite(cursors, "cursor-");
  await fs.writeFile(
    "resources/public/images/sprites/symbol/cursors.svg",
    cursorsSprite,
  );

  const assets = await findFiles("resources/images/assets/", isSvgFile);
  const assetsSprite = await generateSvgSprite(assets, "asset-");
  await fs.writeFile(
    "resources/public/images/sprites/assets.svg",
    assetsSprite,
  );
}

// Collect the CSP hashes of the inline scripts of a rendered template into
// the given set. The hash covers the exact bytes between the script tags, so
// it has to be computed on the rendered output and never on the mustache
// source. Scripts carrying a src attribute are external and are covered by
// 'self' instead.
function collectCspHashes(html, hashes) {
  const pattern = /<script\b(?![^>]*\bsrc=)[^>]*>([\s\S]*?)<\/script>/gi;

  for (const match of html.matchAll(pattern)) {
    const digest = crypto
      .createHash("sha256")
      .update(match[1], "utf8")
      .digest("base64");
    hashes.add(`'sha256-${digest}'`);
  }

  return hashes;
}

async function generateTemplates() {
  await fs.mkdir("./resources/public/", { recursive: true });

  const manifest = await generateManifest();
  let content;

  // Every template written into resources/public/ is served by the frontend
  // container under the same Content Security Policy, so all of them have to
  // contribute their hashes. The storybook previews are excluded because they
  // are not served by that container.
  const cspHashes = new Set();

  const iconsSprite = await fs.readFile(
    "resources/public/images/sprites/symbol/icons.svg",
    "utf8",
  );
  const cursorsSprite = await fs.readFile(
    "resources/public/images/sprites/symbol/cursors.svg",
    "utf8",
  );
  const assetsSprite = await fs.readFile(
    "resources/public/images/sprites/assets.svg",
    "utf-8",
  );
  const partials = {
    "../public/images/sprites/symbol/icons.svg": iconsSprite,
    "../public/images/sprites/symbol/cursors.svg": cursorsSprite,
    "../public/images/sprites/assets.svg": assetsSprite,
  };

  const context = {
    manifest: manifest,
  };

  content = await renderTemplate(
    "resources/templates/index.mustache",
    context,
    partials,
  );

  await fs.writeFile("./resources/public/index.html", content);
  collectCspHashes(content, cspHashes);

  content = await renderTemplate(
    "resources/templates/challenge.mustache",
    context,
    partials,
  );
  await fs.writeFile("./resources/public/challenge.html", content);
  collectCspHashes(content, cspHashes);

  content = await renderTemplate(
    "resources/templates/preview-body.mustache",
    context,
    partials,
  );
  await fs.writeFile("./.storybook/preview-body.html", content);

  content = await renderTemplate(
    "resources/templates/preview-head.mustache",
    context,
    partials,
  );
  await fs.writeFile("./.storybook/preview-head.html", content);

  content = await renderTemplate(
    "resources/templates/render.mustache",
    context,
  );

  await fs.writeFile("./resources/public/render.html", content);
  collectCspHashes(content, cspHashes);

  content = await renderTemplate(
    "resources/templates/rasterizer.mustache",
    context,
  );

  await fs.writeFile("./resources/public/rasterizer.html", content);
  collectCspHashes(content, cspHashes);

  await fs.writeFile(
    "./resources/public/csp-script-hashes.txt",
    [...cspHashes].join(" ") + "\n",
  );
}

export async function compileStorybookStyles() {
  const worker = startWorker();
  const start = process.hrtime();

  log.info("init: compile storybook styles");
  let result = await compileSassStorybook(worker);
  result = concatSass(result);

  await fs.mkdir("./resources/public/css", { recursive: true });
  await fs.writeFile("./resources/public/css/ds.css", result);

  const end = process.hrtime(start);
  log.info("done: compile storybook styles", `(${ppt(end)})`);
  worker.terminate();
}

export async function compileStyles() {
  const worker = startWorker();
  const start = process.hrtime();

  log.info("init: compile styles");

  let result = await compileSassAll(worker);
  result = concatSass(result);

  await fs.mkdir("./resources/public/css", { recursive: true });
  await fs.writeFile("./resources/public/css/main.css", result);

  if (IS_DEBUG) {
    let debugCSS = await compileSassDebug(worker);
    await fs.writeFile("./resources/public/css/debug.css", debugCSS);
  }

  const end = process.hrtime(start);
  log.info("done: compile styles", `(${ppt(end)})`);
  worker.terminate();
}

export async function compileSvgSprites() {
  const start = process.hrtime();
  log.info("init: compile svgsprite");
  let error = false;

  try {
    await generateSvgSprites();
  } catch (cause) {
    error = cause;
  }

  const end = process.hrtime(start);

  if (error) {
    log.error("error: compile svgsprite", `(${ppt(end)})`);
    console.error(error);
  } else {
    log.info("done: compile svgsprite", `(${ppt(end)})`);
  }
}

export async function compileFontsPreviewSprite() {
  const start = process.hrtime();
  log.info("init: compile fonts preview sprite");
  let error = false;
  let result;

  try {
    result = await buildFontsPreviewSprite();
  } catch (cause) {
    error = cause;
  }

  const end = process.hrtime(start);

  if (error) {
    log.error("error: compile fonts preview sprite", `(${ppt(end)})`);
    console.error(error);
  } else if (result.skipped) {
    log.info(
      "done: compile fonts preview sprite (up-to-date, skipped)",
      `(${ppt(end)})`,
    );
  } else {
    log.info(
      `done: compile fonts preview sprite (${result.ok} ok, ${result.failed} fallback)`,
      `(${ppt(end)})`,
    );
  }
}

export async function compileTemplates() {
  const start = process.hrtime();
  let error = false;
  log.info("init: compile templates");

  try {
    await generateTemplates();
  } catch (cause) {
    error = cause;
  }

  const end = process.hrtime(start);

  if (error) {
    log.error("error: compile templates", `(${ppt(end)})`);
    console.error(error);
  } else {
    log.info("done: compile templates", `(${ppt(end)})`);
  }
}

export async function compilePolyfills() {
  const start = process.hrtime();
  log.info("init: compile polyfills");

  const files = await findFiles("resources/polyfills/", isJsFile);
  let result = [];
  for (let path of files) {
    const content = await fs.readFile(path, { encoding: "utf-8" });
    result.push(content);
  }

  await fs.mkdir("./resources/public/js", { recursive: true });
  fs.writeFile("resources/public/js/polyfills.js", result.join("\n"));

  const end = process.hrtime(start);
  log.info("done: compile polyfills", `(${ppt(end)})`);
}

export async function copyAssets() {
  const start = process.hrtime();
  log.info("init: copy assets");

  await syncDirs("resources/images/", "resources/public/images/");
  // The font preview sprite is generated into public/fonts/ (not committed), so
  // exclude it from --delete to keep it across builds (see compileFontsPreviewSprite).
  await syncDirs("resources/fonts/", "resources/public/fonts/", [
    "fonts-preview-sprite.svg",
  ]);

  const end = process.hrtime(start);
  log.info("done: copy assets", `(${ppt(end)})`);
}

export async function copyWasmPlayground() {
  const start = process.hrtime();
  log.info("init: copy wasm playground");

  await syncDirs(
    "resources/wasm-playground/",
    "resources/public/wasm-playground/",
  );

  const end = process.hrtime(start);
  log.info("done: copy wasm playground", `(${ppt(end)})`);
}