fix(editor): support backslash-escaping literal #tags
Memos layers #tag syntax on top of CommonMark, but there was no way to write a literal "#NAS" — it always became a clickable tag (issue #6035). Handle escapes lexically, the way TipTap/prosemirror-markdown do, with no "escaped tag" node in the document model: - Parse: marked's built-in backslash escape already turns "\#NAS" into plain text; the read-only renderer (remark-tag) now honors the same escape by lexing from the original source slice and falling back to the prior value-based parse when it can't faithfully reconstruct the text. - Serialize: a tag-aware Markdown extension (tagMarkdown.ts) backslash- escapes a "#" that would re-parse into a tag, skipping links and code where tags never form. - The editor's #tag lexing moves to the canonical @tiptap/markdown markdownTokenizer; tag-in-link skipping moves to a tree pass so the editor and renderer agree. The capped tag-run grammar is consolidated into utils/tag-grammar.ts (TAG_RUN), shared by the tokenizer, the serialize-escape, and the renderer so they can't drift. README documents why the markdown manager is worked around in three places.
This commit is contained in:
parent
20c19ef82d
commit
5f7e038aa7
9 changed files with 369 additions and 108 deletions
|
|
@ -1,93 +1,19 @@
|
||||||
import type { MarkdownToken } from "@tiptap/core";
|
import type { MarkdownToken } from "@tiptap/core";
|
||||||
import { InputRule, Mark, mergeAttributes } from "@tiptap/core";
|
import { InputRule, Mark, mergeAttributes } from "@tiptap/core";
|
||||||
import type { TokenizerThis, Tokens } from "marked";
|
|
||||||
import { marked } from "marked";
|
|
||||||
import { tagStyles } from "@/lib/markdownStyles";
|
import { tagStyles } from "@/lib/markdownStyles";
|
||||||
import { MAX_TAG_LENGTH, TAG_CHAR_CLASS } from "@/utils/tag-grammar";
|
import { TAG_RUN } from "@/utils/tag-grammar";
|
||||||
|
|
||||||
// Default tag pill, shared with the read-only view (MemoContent/Tag.tsx).
|
// Default tag pill, shared with the read-only view (MemoContent/Tag.tsx).
|
||||||
// Computed once — renderHTML runs on every view update.
|
// Computed once — renderHTML runs on every view update.
|
||||||
const TAG_CLASS = `${tagStyles.base} ${tagStyles.defaultColor}`;
|
const TAG_CLASS = `${tagStyles.base} ${tagStyles.defaultColor}`;
|
||||||
|
|
||||||
// Built from the shared tag grammar (@/utils/tag-grammar) so the editor's
|
// Built from the shared TAG_RUN (@/utils/tag-grammar) so the editor's input
|
||||||
// tokenizer/input rule can't drift from the read-only renderer's lexer
|
// rule and tokenizer can't drift from the serialize-escape or the read-only
|
||||||
// (web/src/utils/remark-plugins/remark-tag.ts).
|
// renderer's lexer (web/src/utils/remark-plugins/remark-tag.ts). The capped-run
|
||||||
const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})\\s$`, "u");
|
// lookahead in TAG_RUN also makes an over-long run decline to match — no
|
||||||
const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})`, "u");
|
// separate length check needed.
|
||||||
// Tests the REMAINDER of the source (not a single code unit) so astral-plane
|
const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_RUN})\\s$`, "u");
|
||||||
// tag characters (emoji et al.) are seen whole, not as lone surrogates.
|
const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_RUN})`, "u");
|
||||||
const TAG_CHAR_AHEAD = new RegExp(`^${TAG_CHAR_CLASS}`, "u");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Tag tokenizer, registered DIRECTLY on the global marked singleton instead of
|
|
||||||
* through `markdownTokenizer`. Two reasons:
|
|
||||||
*
|
|
||||||
* 1. `@tiptap/markdown`'s MarkdownManager wraps `markdownTokenizer.tokenize`
|
|
||||||
* in `tokenizer(src, tokens) { ... tokenize(src, tokens, helper) }` — the
|
|
||||||
* wrapper receives marked's TokenizerThis (with `lexer.state.inLink`) but
|
|
||||||
* does NOT forward it, so a manager-registered tokenizer can never know it
|
|
||||||
* is inside a link label. Registered natively, marked invokes us with
|
|
||||||
* `this.lexer` bound and we can decline inside `[label](url)` the same way
|
|
||||||
* remark-tag skips link nodes.
|
|
||||||
* 2. The manager defaults to the global `marked` export (`markedInstance =
|
|
||||||
* options?.marked ?? marked`) and `web` resolves the exact same marked
|
|
||||||
* module instance as `@tiptap/markdown` does, so this registration is
|
|
||||||
* visible to every lexer the manager creates. Module scope + idempotent:
|
|
||||||
* registered exactly once per page load (the manager's own per-Editor
|
|
||||||
* `marked.use` calls are the accumulation hazard documented in
|
|
||||||
* markdownCodec.ts; this adds a single registration, ever).
|
|
||||||
*
|
|
||||||
* The Tag mark below still declares `markdownTokenName: "memoTag"` +
|
|
||||||
* `parseMarkdown`, which is all the manager needs to route the token.
|
|
||||||
*/
|
|
||||||
function tokenizeTag(this: TokenizerThis, src: string): Tokens.Generic | undefined {
|
|
||||||
// remark-tag skips link nodes entirely; marked sets `state.inLink` while
|
|
||||||
// tokenizing link/reflink labels, so declining here keeps `[see #x](url)`
|
|
||||||
// a plain link label instead of tearing the tag out of it.
|
|
||||||
if (this.lexer?.state?.inLink) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
const match = TAG_TOKEN_RULE.exec(src);
|
|
||||||
if (!match) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
const rest = src.slice(match[0].length);
|
|
||||||
// `#a#b`: decline when the run is directly followed by another `#`, so the
|
|
||||||
// FIRST run stays plain text. NOT full remark parity — remark-tag treats
|
|
||||||
// `#a#b` as two tags and `##x` as all-text, while here `#a#b` becomes text
|
|
||||||
// "#a" + tag "b" and `##x` becomes text "#" + tag "x". The divergence is
|
|
||||||
// visual-only in the editor: both shapes serialize back byte-identically
|
|
||||||
// (the surrounding text nodes re-emit their literal characters).
|
|
||||||
if (rest.startsWith("#")) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
// Runs longer than 100 tag characters are not tags at all in remark-tag
|
|
||||||
// (the whole run stays plain text) — decline instead of splitting the run
|
|
||||||
// into a 100-char tag plus leftover text.
|
|
||||||
if (TAG_CHAR_AHEAD.test(rest)) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
return { type: "memoTag", raw: match[0], text: match[0], tag: match[1] };
|
|
||||||
}
|
|
||||||
|
|
||||||
let tagTokenizerRegistered = false;
|
|
||||||
function registerTagTokenizer() {
|
|
||||||
if (tagTokenizerRegistered) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
tagTokenizerRegistered = true;
|
|
||||||
marked.use({
|
|
||||||
extensions: [
|
|
||||||
{
|
|
||||||
name: "memoTag",
|
|
||||||
level: "inline",
|
|
||||||
start: (src: string) => src.indexOf("#"),
|
|
||||||
tokenizer: tokenizeTag,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
registerTagTokenizer();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mark for memos `#tags`: styled in the editor, serialized back to `#tag`
|
* Mark for memos `#tags`: styled in the editor, serialized back to `#tag`
|
||||||
|
|
@ -99,8 +25,20 @@ registerTagTokenizer();
|
||||||
* `applyMarkToContent` only attaches enclosing marks to text nodes, so an
|
* `applyMarkToContent` only attaches enclosing marks to text nodes, so an
|
||||||
* atom inside bold silently drops the bold delimiters.
|
* atom inside bold silently drops the bold delimiters.
|
||||||
*
|
*
|
||||||
* Parsed live while typing (input rule) and from markdown (the native marked
|
* Parsed live while typing (the input rule) and from markdown (the
|
||||||
* tokenizer above).
|
* `markdownTokenizer` below — the canonical @tiptap/markdown extension point,
|
||||||
|
* https://tiptap.dev/docs/editor/markdown/advanced-usage/custom-tokenizer).
|
||||||
|
*
|
||||||
|
* Two consequences of going through the manager (vs. registering on `marked`
|
||||||
|
* directly) are handled in tagMarkdown.ts:
|
||||||
|
* - The tokenizer can't see whether it sits inside a link label — the manager
|
||||||
|
* forwards only `{ inlineTokens, blockTokens }`, not the lexer's `inLink`
|
||||||
|
* state — so tag-in-link skipping is a tree pass there, mirroring how the
|
||||||
|
* read-only renderer (remark-tag) skips link nodes.
|
||||||
|
* - The manager re-registers this tokenizer onto the global `marked` on every
|
||||||
|
* Editor construction (the accumulation noted in markdownCodec.ts). The
|
||||||
|
* impact is sub-millisecond for memo-sized content and the test codec is a
|
||||||
|
* singleton; accepted as the cost of the canonical API.
|
||||||
*/
|
*/
|
||||||
export const Tag = Mark.create({
|
export const Tag = Mark.create({
|
||||||
name: "tag",
|
name: "tag",
|
||||||
|
|
@ -154,6 +92,26 @@ export const Tag = Mark.create({
|
||||||
];
|
];
|
||||||
},
|
},
|
||||||
|
|
||||||
|
markdownTokenizer: {
|
||||||
|
name: "memoTag",
|
||||||
|
level: "inline",
|
||||||
|
start: (src: string) => src.indexOf("#"),
|
||||||
|
tokenize: (src: string) => {
|
||||||
|
const match = TAG_TOKEN_RULE.exec(src);
|
||||||
|
if (!match) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
// `#a#b`: decline when the run is directly followed by another `#`, so the
|
||||||
|
// FIRST run stays plain text "#a" + tag "b" (`##x` likewise → text "#" +
|
||||||
|
// tag "x"). The serialize-escape then escapes the tag-shaped literal "#a",
|
||||||
|
// so `#a#b` round-trips as `\#a#b` — doc-equivalent and stable thereafter.
|
||||||
|
// (An over-long run already declines: TAG_RUN's lookahead fails to match.)
|
||||||
|
if (src.slice(match[0].length).startsWith("#")) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return { type: "memoTag", raw: match[0], tag: match[1] };
|
||||||
|
},
|
||||||
|
},
|
||||||
markdownTokenName: "memoTag",
|
markdownTokenName: "memoTag",
|
||||||
parseMarkdown: (token, helpers) => {
|
parseMarkdown: (token, helpers) => {
|
||||||
const t = token as MarkdownToken & { tag?: string };
|
const t = token as MarkdownToken & { tag?: string };
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
import { type AnyExtension, mergeAttributes } from "@tiptap/core";
|
import { type AnyExtension, mergeAttributes } from "@tiptap/core";
|
||||||
import { Heading } from "@tiptap/extension-heading";
|
import { Heading } from "@tiptap/extension-heading";
|
||||||
import { TaskItem, TaskList } from "@tiptap/extension-list";
|
import { TaskItem, TaskList } from "@tiptap/extension-list";
|
||||||
import { Markdown } from "@tiptap/markdown";
|
|
||||||
import StarterKit from "@tiptap/starter-kit";
|
import StarterKit from "@tiptap/starter-kit";
|
||||||
import { type HeadingLevel, headingClass, markdownStyles } from "@/lib/markdownStyles";
|
import { type HeadingLevel, headingClass, markdownStyles } from "@/lib/markdownStyles";
|
||||||
import { preservedExtensions } from "./PreservedBlock";
|
import { preservedExtensions } from "./PreservedBlock";
|
||||||
import { Tag } from "./Tag";
|
import { Tag } from "./Tag";
|
||||||
|
import { TagAwareMarkdown } from "./tagMarkdown";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* StarterKit's Heading is bundled and cannot vary classes by level via static
|
* StarterKit's Heading is bundled and cannot vary classes by level via static
|
||||||
|
|
@ -46,7 +46,7 @@ export function buildExtensions(): AnyExtension[] {
|
||||||
StyledHeading.configure({ levels: [1, 2, 3, 4, 5, 6] }),
|
StyledHeading.configure({ levels: [1, 2, 3, 4, 5, 6] }),
|
||||||
TaskList,
|
TaskList,
|
||||||
TaskItem.configure({ nested: true }),
|
TaskItem.configure({ nested: true }),
|
||||||
Markdown,
|
TagAwareMarkdown,
|
||||||
...preservedExtensions,
|
...preservedExtensions,
|
||||||
Tag,
|
Tag,
|
||||||
];
|
];
|
||||||
|
|
|
||||||
113
web/src/components/MemoEditor/Editor/tagMarkdown.ts
Normal file
113
web/src/components/MemoEditor/Editor/tagMarkdown.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
import type { JSONContent } from "@tiptap/core";
|
||||||
|
import { Markdown } from "@tiptap/markdown";
|
||||||
|
import type { Schema } from "@tiptap/pm/model";
|
||||||
|
import { TAG_RUN } from "@/utils/tag-grammar";
|
||||||
|
|
||||||
|
// A `#` begins a memo tag when a capped TAG_RUN follows it — the same grammar
|
||||||
|
// the editor tokenizer (Tag.ts) matches, so escape and parse can't disagree.
|
||||||
|
// The lookahead matches only the `#`, leaving the run itself untouched.
|
||||||
|
const TAG_HASH = new RegExp(`#(?=${TAG_RUN})`, "gu");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Backslash-escape a `#` that would otherwise re-parse into a `#tag`.
|
||||||
|
*
|
||||||
|
* memos layers hashtag syntax on top of CommonMark, so — exactly as
|
||||||
|
* prosemirror-markdown escapes inline syntax characters like `*` and `_` —
|
||||||
|
* plain text that looks like a tag must be escaped on serialize, or it silently
|
||||||
|
* becomes a tag on the next parse.
|
||||||
|
*/
|
||||||
|
export function escapeTagHashes(text: string): string {
|
||||||
|
return text.replace(TAG_HASH, "\\#");
|
||||||
|
}
|
||||||
|
|
||||||
|
type JsonMark = string | { type?: string };
|
||||||
|
type JsonNode = { type?: string; marks?: JsonMark[] };
|
||||||
|
type PatchableManager = {
|
||||||
|
encodeTextForMarkdown?: (text: string, node: JsonNode, parentNode?: JsonNode) => string;
|
||||||
|
parse?: (markdown: string) => JSONContent;
|
||||||
|
};
|
||||||
|
|
||||||
|
const markName = (mark: JsonMark): string => (typeof mark === "string" ? mark : (mark.type ?? ""));
|
||||||
|
const hasMark = (node: JsonNode, name: string): boolean => (node.marks ?? []).some((mark) => markName(mark) === name);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A `#tag` only forms in plain inline prose. The tokenizer declines inside
|
||||||
|
* links, and `code: true` marks/nodes (the Tag mark itself, inline code,
|
||||||
|
* preserved spans/blocks, code blocks) serialize verbatim — so escaping a `#`
|
||||||
|
* in any of those contexts would be wrong (and would corrupt the byte-identical
|
||||||
|
* round-trips those constructs rely on).
|
||||||
|
*/
|
||||||
|
function escapesTagsHere(schema: Schema, node: JsonNode, parentNode?: JsonNode): boolean {
|
||||||
|
for (const mark of node.marks ?? []) {
|
||||||
|
const name = markName(mark);
|
||||||
|
if (name === "link") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (schema.marks[name]?.spec.code) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const parentType = parentNode?.type;
|
||||||
|
if (parentType && schema.nodes[parentType]?.spec.code) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Strip the `tag` mark from any text that also carries a `link` mark.
|
||||||
|
*
|
||||||
|
* The `markdownTokenizer` in Tag.ts cannot tell it is inside a link label — the
|
||||||
|
* manager forwards only `{ inlineTokens, blockTokens }`, never the lexer's
|
||||||
|
* `inLink` state — so `[see #x](url)` parses with `#x` tagged. We undo that as a
|
||||||
|
* tree pass, the same way the read-only renderer (remark-tag) skips link nodes,
|
||||||
|
* keeping the editor and renderer in agreement: a `#` in a link label is link
|
||||||
|
* text, never a tag pill.
|
||||||
|
*/
|
||||||
|
function stripTagInsideLinks(node: JSONContent): JSONContent {
|
||||||
|
if (Array.isArray(node.marks) && hasMark(node, "link")) {
|
||||||
|
node.marks = node.marks.filter((mark) => markName(mark) !== "tag");
|
||||||
|
}
|
||||||
|
node.content?.forEach(stripTagInsideLinks);
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The `markdownTokenizer` in Tag.ts handles `#tag` the canonical
|
||||||
|
* @tiptap/markdown way, but two memos-specific concerns sit above the tokenizer
|
||||||
|
* and have no public hook, so we compose them onto the manager here:
|
||||||
|
*
|
||||||
|
* - serialize: escape a tag-shaped `#` in plain prose ({@link escapeTagHashes})
|
||||||
|
* so a literal `#NAS` can't silently re-parse into a tag;
|
||||||
|
* - parse: skip tags inside link labels ({@link stripTagInsideLinks}).
|
||||||
|
*
|
||||||
|
* Escapes stay a purely lexical concern — the document model carries no
|
||||||
|
* "escaped tag" node or mark. A literal `#NAS` is just text: parsing strips a
|
||||||
|
* leading `\` (marked's built-in escape) and serializing adds it back, the same
|
||||||
|
* way TipTap handles every other escapable character.
|
||||||
|
*/
|
||||||
|
export const TagAwareMarkdown = Markdown.extend({
|
||||||
|
onBeforeCreate(props) {
|
||||||
|
this.parent?.(props);
|
||||||
|
const { editor } = this;
|
||||||
|
const manager = editor.markdown as unknown as PatchableManager | undefined;
|
||||||
|
if (!manager) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const encode = manager.encodeTextForMarkdown;
|
||||||
|
if (typeof encode === "function") {
|
||||||
|
const bound = encode.bind(manager);
|
||||||
|
manager.encodeTextForMarkdown = (text, node, parentNode) => {
|
||||||
|
const encoded = bound(text, node, parentNode);
|
||||||
|
return escapesTagsHere(editor.schema, node, parentNode) ? escapeTagHashes(encoded) : encoded;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const parse = manager.parse;
|
||||||
|
if (typeof parse === "function") {
|
||||||
|
const bound = parse.bind(manager);
|
||||||
|
manager.parse = (markdown) => stripTagInsideLinks(bound(markdown));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
@ -45,7 +45,8 @@ MemoEditor/
|
||||||
│ ├── extensions.ts # Canonical schema-relevant extension set (shared with codec)
|
│ ├── extensions.ts # Canonical schema-relevant extension set (shared with codec)
|
||||||
│ ├── markdownCodec.ts # Headless parse/serialize helpers (singleton editor)
|
│ ├── markdownCodec.ts # Headless parse/serialize helpers (singleton editor)
|
||||||
│ ├── PreservedBlock.ts # Byte-for-byte preservation of tables, math, raw HTML
|
│ ├── PreservedBlock.ts # Byte-for-byte preservation of tables, math, raw HTML
|
||||||
│ ├── Tag.ts # Memos #tag mark
|
│ ├── Tag.ts # Memos #tag mark (markdownTokenizer + input rule)
|
||||||
|
│ ├── tagMarkdown.ts # Tag-aware Markdown: # escape on serialize, link-skip on parse
|
||||||
│ ├── TagSuggestion.ts # # popup for WYSIWYG mode
|
│ ├── TagSuggestion.ts # # popup for WYSIWYG mode
|
||||||
│ └── suggestionMenu.tsx # Shared suggestion popup renderer (used by TagSuggestion)
|
│ └── suggestionMenu.tsx # Shared suggestion popup renderer (used by TagSuggestion)
|
||||||
├── Toolbar/ # Toolbar sub-components (InsertMenu, VisibilitySelector)
|
├── Toolbar/ # Toolbar sub-components (InsertMenu, VisibilitySelector)
|
||||||
|
|
@ -85,7 +86,17 @@ Mode switching is a markdown handoff: because both editors write into `state.con
|
||||||
|
|
||||||
`PreservedBlock.ts` handles syntax the WYSIWYG editor does not model richly: tables, `$$math$$`, and raw HTML are captured at parse time with their raw markdown source, shown as editable monospace literal text, and re-emitted byte-for-byte on serialize.
|
`PreservedBlock.ts` handles syntax the WYSIWYG editor does not model richly: tables, `$$math$$`, and raw HTML are captured at parse time with their raw markdown source, shown as editable monospace literal text, and re-emitted byte-for-byte on serialize.
|
||||||
|
|
||||||
`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans.
|
`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans. Its `#tag` lexing is a `markdownTokenizer` (the canonical `@tiptap/markdown` extension point); `tagMarkdown.ts` adds the two things that tokenizer can't do — on serialize it backslash-escapes a literal `#` that would otherwise re-parse into a tag (escapes are lexical, so there is no "escaped tag" node), and on parse it strips the tag mark from text inside link labels. The `#tag` grammar itself lives once in `utils/tag-grammar.ts` (`TAG_RUN`), shared by the tokenizer, the serialize-escape, and the read-only `remark-tag` renderer so they can't drift.
|
||||||
|
|
||||||
|
### Why the markdown manager is worked around in several places
|
||||||
|
|
||||||
|
`@tiptap/markdown` (3.26.0) exposes no public, per-instance hook for custom tokenizers or for text escaping, and it registers each extension's tokenizer onto the **global** `marked` singleton on every `new Editor()` — registrations it never removes. That one limitation is the reason for three otherwise-surprising choices, each documented in detail at its call site:
|
||||||
|
|
||||||
|
- **`markdownCodec.ts` keeps a single editor instance** — re-creating editors would leak tokenizer registrations onto global `marked` and measurably degrade parse time.
|
||||||
|
- **`PreservedBlock.ts` registers its tokenizers on `marked` once at module scope** (idempotent) instead of via the per-extension `markdownTokenizer`, to avoid that per-construction accumulation.
|
||||||
|
- **`tagMarkdown.ts` composes onto the manager in `onBeforeCreate`** (escape, link-skip) because the relevant manager methods are `private` with no public seam.
|
||||||
|
|
||||||
|
`Tag.ts` deliberately uses the canonical `markdownTokenizer` API and accepts the per-construction re-registration as its cost; `PreservedBlock.ts` refuses it for its seven tokenizers. The asymmetry is intentional — collapse both onto one path if upstream ever ships a public per-instance tokenizer hook.
|
||||||
|
|
||||||
### Suggestions
|
### Suggestions
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,64 @@
|
||||||
import type { Root, Text } from "mdast";
|
import type { Root, Text } from "mdast";
|
||||||
import type { Node as UnistNode } from "unist";
|
import type { Position, Node as UnistNode } from "unist";
|
||||||
import type { TagNode, TagNodeData } from "@/types/markdown";
|
import type { TagNode, TagNodeData } from "@/types/markdown";
|
||||||
import { isTagChar, MAX_TAG_LENGTH } from "@/utils/tag-grammar";
|
import { isTagChar, MAX_TAG_LENGTH } from "@/utils/tag-grammar";
|
||||||
|
|
||||||
function parseTagsFromText(text: string): Array<{ type: "text"; value: string } | { type: "tag"; value: string }> {
|
type Segment = { type: "text"; value: string } | { type: "tag"; value: string };
|
||||||
const segments: Array<{ type: "text"; value: string } | { type: "tag"; value: string }> = [];
|
|
||||||
|
|
||||||
const chars = [...text];
|
// CommonMark "ASCII punctuation": the only characters a leading backslash can
|
||||||
|
// escape. A backslash before anything else is a literal backslash.
|
||||||
|
function isAsciiPunctuation(char: string): boolean {
|
||||||
|
if (char.length !== 1) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const code = char.charCodeAt(0);
|
||||||
|
return (
|
||||||
|
(code >= 0x21 && code <= 0x2f) || // ! " # $ % & ' ( ) * + , - . /
|
||||||
|
(code >= 0x3a && code <= 0x40) || // : ; < = > ? @
|
||||||
|
(code >= 0x5b && code <= 0x60) || // [ \ ] ^ _ `
|
||||||
|
(code >= 0x7b && code <= 0x7e) // { | } ~
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply CommonMark backslash-unescaping to a raw source slice, tracking which
|
||||||
|
* resulting characters came from an escape. `\#` yields a `#` flagged escaped,
|
||||||
|
* so the tag lexer can tell a deliberately-escaped hash from a real tag.
|
||||||
|
* Returns code points (not UTF-16 units) so astral characters stay intact.
|
||||||
|
*/
|
||||||
|
function unescapeBackslashes(source: string): { chars: string[]; escaped: boolean[] } {
|
||||||
|
const codePoints = [...source];
|
||||||
|
const chars: string[] = [];
|
||||||
|
const escaped: boolean[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < codePoints.length; i++) {
|
||||||
|
if (codePoints[i] === "\\" && i + 1 < codePoints.length && isAsciiPunctuation(codePoints[i + 1])) {
|
||||||
|
chars.push(codePoints[i + 1]);
|
||||||
|
escaped.push(true);
|
||||||
|
i++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
chars.push(codePoints[i]);
|
||||||
|
escaped.push(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { chars, escaped };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split a run of characters into text/tag segments. `escaped[i]` marks a
|
||||||
|
* character that came from a backslash escape: an escaped `#` can never start a
|
||||||
|
* tag (so `\#NAS` stays literal text), matching the backend goldmark parser and
|
||||||
|
* the editor's marked tokenizer, both of which honor the escape natively.
|
||||||
|
*/
|
||||||
|
function parseSegments(chars: string[], escaped: boolean[]): Segment[] {
|
||||||
|
const segments: Segment[] = [];
|
||||||
let i = 0;
|
let i = 0;
|
||||||
|
|
||||||
while (i < chars.length) {
|
while (i < chars.length) {
|
||||||
if (chars[i] === "#" && i + 1 < chars.length && isTagChar(chars[i + 1])) {
|
if (chars[i] === "#" && !escaped[i] && i + 1 < chars.length && isTagChar(chars[i + 1])) {
|
||||||
const prevChar = i > 0 ? chars[i - 1] : "";
|
const prevChar = i > 0 ? chars[i - 1] : "";
|
||||||
const nextChar = i + 1 < chars.length ? chars[i + 1] : "";
|
const nextChar = chars[i + 1];
|
||||||
|
|
||||||
if (prevChar === "#" || nextChar === "#" || nextChar === " ") {
|
if (prevChar === "#" || nextChar === "#" || nextChar === " ") {
|
||||||
segments.push({ type: "text", value: chars[i] });
|
segments.push({ type: "text", value: chars[i] });
|
||||||
|
|
@ -35,8 +81,9 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Consume a plain-text run up to the next tag-eligible (non-escaped) hash.
|
||||||
let j = i + 1;
|
let j = i + 1;
|
||||||
while (j < chars.length && chars[j] !== "#") {
|
while (j < chars.length && !(chars[j] === "#" && !escaped[j])) {
|
||||||
j++;
|
j++;
|
||||||
}
|
}
|
||||||
segments.push({ type: "text", value: chars.slice(i, j).join("") });
|
segments.push({ type: "text", value: chars.slice(i, j).join("") });
|
||||||
|
|
@ -46,6 +93,34 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string }
|
||||||
return segments;
|
return segments;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Segment a text node, preferring the original source slice (where escapes are
|
||||||
|
* still visible) over the post-escape `value`. The source-derived parse is only
|
||||||
|
* trusted when it reconstructs `value` byte-for-byte; otherwise — entity
|
||||||
|
* references, missing positions, upstream rewrites — we fall back to the
|
||||||
|
* value-based parse, which is exactly the pre-escape-support behavior. This
|
||||||
|
* makes escape support strictly additive: it never changes a node it can't
|
||||||
|
* faithfully account for.
|
||||||
|
*/
|
||||||
|
function segmentsForTextNode(value: string, position: Position | undefined, source: string): Segment[] {
|
||||||
|
const startOffset = position?.start?.offset;
|
||||||
|
const endOffset = position?.end?.offset;
|
||||||
|
|
||||||
|
if (source && startOffset != null && endOffset != null) {
|
||||||
|
const slice = source.slice(startOffset, endOffset);
|
||||||
|
const { chars, escaped } = unescapeBackslashes(slice);
|
||||||
|
if (chars.join("") === value) {
|
||||||
|
return parseSegments(chars, escaped);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const chars = [...value];
|
||||||
|
return parseSegments(
|
||||||
|
chars,
|
||||||
|
chars.map(() => false),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function createTagNode(tagValue: string): TagNode {
|
function createTagNode(tagValue: string): TagNode {
|
||||||
const data: TagNodeData = {
|
const data: TagNodeData = {
|
||||||
hName: "span",
|
hName: "span",
|
||||||
|
|
@ -73,14 +148,14 @@ function isLinkNode(node: UnistNode): boolean {
|
||||||
return node.type === "link" || node.type === "linkReference";
|
return node.type === "link" || node.type === "linkReference";
|
||||||
}
|
}
|
||||||
|
|
||||||
function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void {
|
function transformTagTextNodes(parent: ParentNode, insideLink: boolean, source: string): void {
|
||||||
for (let index = 0; index < parent.children.length; index++) {
|
for (let index = 0; index < parent.children.length; index++) {
|
||||||
const child = parent.children[index];
|
const child = parent.children[index];
|
||||||
const childInsideLink = insideLink || isLinkNode(child);
|
const childInsideLink = insideLink || isLinkNode(child);
|
||||||
|
|
||||||
if (child.type === "text" && !childInsideLink) {
|
if (child.type === "text" && !childInsideLink) {
|
||||||
const textNode = child as Text;
|
const textNode = child as Text;
|
||||||
const segments = parseTagsFromText(textNode.value);
|
const segments = segmentsForTextNode(textNode.value, textNode.position, source);
|
||||||
|
|
||||||
if (segments.every((seg) => seg.type === "text")) {
|
if (segments.every((seg) => seg.type === "text")) {
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -102,13 +177,16 @@ function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isParentNode(child)) {
|
if (isParentNode(child)) {
|
||||||
transformTagTextNodes(child, childInsideLink);
|
transformTagTextNodes(child, childInsideLink, source);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type VFileLike = { value?: string | Uint8Array };
|
||||||
|
|
||||||
export const remarkTag = () => {
|
export const remarkTag = () => {
|
||||||
return (tree: Root) => {
|
return (tree: Root, file: VFileLike) => {
|
||||||
transformTagTextNodes(tree as ParentNode, false);
|
const source = typeof file?.value === "string" ? file.value : "";
|
||||||
|
transformTagTextNodes(tree as ParentNode, false, source);
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
/**
|
/**
|
||||||
* The single source of truth for memos' `#tag` lexing grammar, shared by the
|
* The single source of truth for memos' `#tag` lexing grammar, shared by the
|
||||||
* editor tokenizer (components/MemoEditor/Editor/Tag.ts) and the read-only
|
* editor tokenizer + serialize-escape (components/MemoEditor/Editor/Tag.ts and
|
||||||
* renderer (utils/remark-plugins/remark-tag.ts) so the two can't drift.
|
* tagMarkdown.ts) and the read-only renderer (utils/remark-plugins/remark-tag.ts)
|
||||||
|
* so they can't drift.
|
||||||
*
|
*
|
||||||
* A tag character is any Unicode letter, number, or symbol, plus `_ - / &`.
|
* A tag character is any Unicode letter, number, or symbol, plus `_ - / &`.
|
||||||
* A tag run is capped at MAX_TAG_LENGTH characters.
|
* A tag run is capped at MAX_TAG_LENGTH characters.
|
||||||
|
|
@ -10,6 +11,18 @@ export const TAG_CHAR_CLASS = "[\\p{L}\\p{N}\\p{S}_\\-/&]";
|
||||||
|
|
||||||
export const MAX_TAG_LENGTH = 100;
|
export const MAX_TAG_LENGTH = 100;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Regex source for a *capped* tag run: 1..MAX_TAG_LENGTH tag characters,
|
||||||
|
* refusing to match when a (MAX+1)-th tag character would follow (an over-long
|
||||||
|
* run is not a tag). Embed with the `u` flag.
|
||||||
|
*
|
||||||
|
* The two halves of the round-trip build on this same source so they can't
|
||||||
|
* disagree on what counts as a tag: the editor's input rule + tokenizer match
|
||||||
|
* `#(${TAG_RUN})`, and the serialize-escape (tagMarkdown.ts) escapes a `#`
|
||||||
|
* followed by `(?=${TAG_RUN})`.
|
||||||
|
*/
|
||||||
|
export const TAG_RUN = `${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}}(?!${TAG_CHAR_CLASS})`;
|
||||||
|
|
||||||
// Matches exactly one tag character. The `u` flag makes the class match whole
|
// Matches exactly one tag character. The `u` flag makes the class match whole
|
||||||
// code points, so astral-plane symbols (emoji et al.) are tested intact rather
|
// code points, so astral-plane symbols (emoji et al.) are tested intact rather
|
||||||
// than as lone surrogates.
|
// than as lone surrogates.
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,43 @@ describe("tag suggestion insertion", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("tag escaping", () => {
|
||||||
|
// Drives the ProseMirror input-rules plugin the way real typing does:
|
||||||
|
// someProp("handleTextInput", …) is exactly what the view calls on keystroke.
|
||||||
|
function typeText(editor: EditorInstance, text: string) {
|
||||||
|
const { from } = editor.state.selection;
|
||||||
|
editor.view.someProp("handleTextInput", (handler) => handler(editor.view, from, from, text));
|
||||||
|
}
|
||||||
|
|
||||||
|
it("undoing the tag autoformat leaves a durable literal #tag, not a pill", () => {
|
||||||
|
const { ref, editor } = setup("");
|
||||||
|
act(() => {
|
||||||
|
editor.view.dispatch(editor.state.tr.insertText("#NAS", 1));
|
||||||
|
});
|
||||||
|
act(() => typeText(editor, " ")); // tag input rule fires → pill
|
||||||
|
act(() => {
|
||||||
|
editor.commands.undoInputRule(); // the standard "I didn't want that" gesture
|
||||||
|
});
|
||||||
|
|
||||||
|
// Back to plain text — and the serializer escapes it so it can't re-tag.
|
||||||
|
expect(ref.current?.getMarkdown()?.trim()).toBe("\\#NAS");
|
||||||
|
const para = editor.getJSON().content?.[0];
|
||||||
|
expect(para?.content?.some((n) => (n.marks ?? []).some((m) => m.type === "tag"))).toBeFalsy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a plain #tag still becomes a tag", () => {
|
||||||
|
const { ref, editor } = setup("");
|
||||||
|
act(() => {
|
||||||
|
editor.view.dispatch(editor.state.tr.insertText("#work", 1));
|
||||||
|
});
|
||||||
|
act(() => typeText(editor, " "));
|
||||||
|
|
||||||
|
expect(ref.current?.getMarkdown()?.trim()).toBe("#work");
|
||||||
|
const para = editor.getJSON().content?.[0];
|
||||||
|
expect(para?.content?.[0]).toMatchObject({ text: "#work", marks: [{ type: "tag" }] });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("external content sync", () => {
|
describe("external content sync", () => {
|
||||||
it("a trim-equal echo of the editor's own output does not reset the document", () => {
|
it("a trim-equal echo of the editor's own output does not reset the document", () => {
|
||||||
const { ref, rerender } = setupRerenderable("hello");
|
const { ref, rerender } = setupRerenderable("hello");
|
||||||
|
|
|
||||||
|
|
@ -96,9 +96,13 @@ describe("Tag mark", () => {
|
||||||
expect(roundTripMarkdown(input).trim()).toBe(input);
|
expect(roundTripMarkdown(input).trim()).toBe(input);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("round-trips ##x and #a#b byte-identically (visual divergence documented in Tag.ts)", () => {
|
it("round-trips ##x unchanged and normalizes #a#b to \\#a#b (conservative # escaping)", () => {
|
||||||
|
// "##x" → text "#" + tag "x"; the lone "#" is not tag-shaped, so untouched.
|
||||||
expect(roundTripMarkdown("##x").trim()).toBe("##x");
|
expect(roundTripMarkdown("##x").trim()).toBe("##x");
|
||||||
expect(roundTripMarkdown("#a#b").trim()).toBe("#a#b");
|
// "#a#b" → text "#a" + tag "b"; the plain "#a" is tag-shaped, so the
|
||||||
|
// serializer escapes it to keep it literal. Doc-equivalent, stable after.
|
||||||
|
expect(roundTripMarkdown("#a#b").trim()).toBe("\\#a#b");
|
||||||
|
expect(roundTripMarkdown("\\#a#b").trim()).toBe("\\#a#b");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("treats a run longer than 100 tag characters as plain text", () => {
|
it("treats a run longer than 100 tag characters as plain text", () => {
|
||||||
|
|
@ -109,9 +113,29 @@ describe("Tag mark", () => {
|
||||||
expect(textNodes.some((n) => hasTagMark(n))).toBe(false);
|
expect(textNodes.some((n) => hasTagMark(n))).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("documents known gap: backslash-escaped \\# degrades after one cycle (upstream escape dropping)", () => {
|
it("keeps a backslash-escaped \\#tag literal and durable across round-trips", () => {
|
||||||
const out = roundTripMarkdown("\\#escaped");
|
const input = "\\#NAS is my server";
|
||||||
expect(out.trim()).toBe("#escaped"); // upstream drops the escape (pre-existing); re-parse then sees a tag
|
|
||||||
expect(roundTripMarkdown(out)).toBe(out); // at least stable from then on
|
// Escapes are lexical: `\#NAS` parses to ordinary text — no tag mark, no
|
||||||
|
// bespoke "escaped tag" node — the `\` is simply consumed.
|
||||||
|
const children = firstParagraphChildren(input);
|
||||||
|
expect(children.some(hasTagMark)).toBe(false);
|
||||||
|
expect(children.map((n) => n.text ?? "").join("")).toBe("#NAS is my server");
|
||||||
|
|
||||||
|
// The serializer re-escapes the tag-shaped `#`, so it never degrades into a
|
||||||
|
// tag...
|
||||||
|
const once = roundTripMarkdown(input).trim();
|
||||||
|
expect(once).toBe("\\#NAS is my server");
|
||||||
|
// ...and stays stable on every subsequent cycle.
|
||||||
|
expect(roundTripMarkdown(once).trim()).toBe(once);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("escapes only the literal tag while still tagging a real one beside it", () => {
|
||||||
|
const input = "\\#NAS and #real";
|
||||||
|
expect(roundTripMarkdown(input).trim()).toBe("\\#NAS and #real");
|
||||||
|
|
||||||
|
const tagged = firstParagraphChildren(input).filter(hasTagMark);
|
||||||
|
expect(tagged).toHaveLength(1);
|
||||||
|
expect(tagged[0]).toMatchObject({ text: "#real", marks: [{ type: "tag", attrs: { tag: "real" } }] });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -51,4 +51,31 @@ describe("remarkTag", () => {
|
||||||
expect(html).toContain('data-tag="urgent"');
|
expect(html).toContain('data-tag="urgent"');
|
||||||
expect(html).toContain('data-tag="later"');
|
expect(html).toContain('data-tag="later"');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not turn a backslash-escaped \\#tag into a tag, but still tags an unescaped one", () => {
|
||||||
|
const html = renderMarkdown("\\#NAS is my server and a #real tag");
|
||||||
|
|
||||||
|
// Escaped: rendered as the literal text "#NAS", never a tag pill.
|
||||||
|
expect(html).not.toContain('data-tag="NAS"');
|
||||||
|
expect(html).toContain("#NAS");
|
||||||
|
// Unescaped neighbour is unaffected.
|
||||||
|
expect(html).toContain('data-tag="real"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("escapes only the marked hash when escaped and unescaped tags share a node", () => {
|
||||||
|
const html = renderMarkdown("\\#first then #second");
|
||||||
|
|
||||||
|
expect(html).not.toContain('data-tag="first"');
|
||||||
|
expect(html).toContain("#first");
|
||||||
|
expect(html).toContain('data-tag="second"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still tags a hash that shares a text node with an entity reference", () => {
|
||||||
|
// The source slice ("...&...") differs from the decoded value, so the
|
||||||
|
// escape-aware path bows out and the tag is detected the original way.
|
||||||
|
const html = renderMarkdown("Tom & Jerry #cartoon");
|
||||||
|
|
||||||
|
expect(html).toContain('data-tag="cartoon"');
|
||||||
|
expect(html).toContain("Tom & Jerry");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue