memos/web/scripts/generate-tag-unicode-data.mjs

import { createHash } from "node:crypto";
import { writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";

const UNICODE_RELEASE = "17.0.0";
const sources = {
  derived: `https://www.unicode.org/Public/${UNICODE_RELEASE}/ucd/DerivedCoreProperties.txt`,
  emoji: `https://www.unicode.org/Public/${UNICODE_RELEASE}/emoji/emoji-test.txt`,
  unicodeData: `https://www.unicode.org/Public/${UNICODE_RELEASE}/ucd/UnicodeData.txt`,
};

const fetchText = async (url) => {
  const response = await fetch(url);
  if (!response.ok) throw new Error(`Failed to fetch ${url}: ${response.status}`);
  return response.text();
};

const parseRange = (value) => {
  const [from, to = from] = value.split("..").map((part) => Number.parseInt(part, 16));
  return [from, to];
};

const mergeRanges = (ranges) => {
  const merged = [];
  for (const [from, to] of ranges.sort((left, right) => left[0] - right[0])) {
    const previous = merged.at(-1);
    if (previous && from <= previous[1] + 1) previous[1] = Math.max(previous[1], to);
    else merged.push([from, to]);
  }
  return merged;
};

const parseDerivedProperty = (source, property) =>
  mergeRanges(
    source
      .split("\n")
      .map((line) => line.match(/^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*([A-Za-z_]+)/))
      .filter((match) => match?.[2] === property)
      .map((match) => parseRange(match[1])),
  );

const parseCombiningMarkRanges = (source) => {
  const ranges = [];
  let pending;
  for (const line of source.split("\n")) {
    if (!line) continue;
    const fields = line.split(";");
    const codePoint = Number.parseInt(fields[0], 16);
    const name = fields[1];
    const category = fields[2];
    if (name.endsWith(", First>")) {
      pending = { codePoint, category };
    } else if (name.endsWith(", Last>") && pending) {
      if (pending.category === "Mn" || pending.category === "Mc") ranges.push([pending.codePoint, codePoint]);
      pending = undefined;
    } else if (category === "Mn" || category === "Mc") {
      ranges.push([codePoint, codePoint]);
    }
  }
  return mergeRanges(ranges);
};

const parseEmoji = (source) =>
  source
    .split("\n")
    .map((line) => line.match(/^([0-9A-F ]+)\s*;\s*fully-qualified\b/))
    .filter(Boolean)
    .map((match) =>
      String.fromCodePoint(
        ...match[1]
          .trim()
          .split(/\s+/)
          .map((part) => Number.parseInt(part, 16)),
      ),
    );

const formatRanges = (name, ranges) => {
  const values = ranges.flatMap(([from, to]) => [`0x${from.toString(16)}`, `0x${to.toString(16)}`]);
  const lines = [];
  for (let index = 0; index < values.length; index += 12) lines.push(`  ${values.slice(index, index + 12).join(", ")},`);
  return `export const ${name}: readonly number[] = [\n${lines.join("\n")}\n];`;
};

const formatEmoji = (emoji) => {
  const lines = [];
  for (let index = 0; index < emoji.length; index += 12) {
    lines.push(
      `  ${emoji
        .slice(index, index + 12)
        .map((value) => JSON.stringify(value))
        .join(", ")},`,
    );
  }
  return `export const FULLY_QUALIFIED_EMOJI: readonly string[] = [\n${lines.join("\n")}\n];`;
};

const unicodeDataDigest = (xidContinue, defaultIgnorable, combiningMarks, emoji) => {
  const canonical = [];
  const addRanges = (name, ranges) => {
    for (const [from, to] of ranges) canonical.push(`${name}:${from.toString(16)}-${to.toString(16)}\n`);
  };
  addRanges("xid", xidContinue);
  addRanges("ignorable", defaultIgnorable);
  addRanges("combining", combiningMarks);
  canonical.push(
    ...emoji.map((value) => `emoji:${[...value].map((character) => character.codePointAt(0).toString(16)).join(",")}\n`).sort(),
  );
  return createHash("sha256").update(canonical.join("")).digest("hex");
};

const [derived, emojiTest, unicodeData] = await Promise.all(Object.values(sources).map(fetchText));
const xidContinue = parseDerivedProperty(derived, "XID_Continue");
const defaultIgnorable = parseDerivedProperty(derived, "Default_Ignorable_Code_Point");
const combiningMarks = parseCombiningMarkRanges(unicodeData);
const emoji = parseEmoji(emojiTest);
const output = `// Generated by scripts/generate-tag-unicode-data.mjs. Do not edit.\n// Unicode and Emoji ${UNICODE_RELEASE} data used by Memos tag recognition.\n// Unicode data SHA-256: ${unicodeDataDigest(xidContinue, defaultIgnorable, combiningMarks, emoji)}\n// biome-ignore-all format: generated data is kept in compact source order\n\n${formatRanges(
  "XID_CONTINUE_RANGES",
  xidContinue,
)}\n\n${formatRanges("DEFAULT_IGNORABLE_RANGES", defaultIgnorable)}\n\n${formatRanges("COMBINING_MARK_RANGES", combiningMarks)}\n\n${formatEmoji(emoji)}\n`;

const outputURL = new URL("../src/utils/tag-unicode-data.ts", import.meta.url);
await writeFile(fileURLToPath(outputURL), output);