diff --git a/web/src/components/MemoEditor/Editor/Tag.ts b/web/src/components/MemoEditor/Editor/Tag.ts index 12734d03..478cabc9 100644 --- a/web/src/components/MemoEditor/Editor/Tag.ts +++ b/web/src/components/MemoEditor/Editor/Tag.ts @@ -1,93 +1,19 @@ import type { MarkdownToken } from "@tiptap/core"; import { InputRule, Mark, mergeAttributes } from "@tiptap/core"; -import type { TokenizerThis, Tokens } from "marked"; -import { marked } from "marked"; import { tagStyles } from "@/lib/markdownStyles"; -import { MAX_TAG_LENGTH, TAG_CHAR_CLASS } from "@/utils/tag-grammar"; +import { TAG_RUN } from "@/utils/tag-grammar"; // Default tag pill, shared with the read-only view (MemoContent/Tag.tsx). // Computed once — renderHTML runs on every view update. const TAG_CLASS = `${tagStyles.base} ${tagStyles.defaultColor}`; -// Built from the shared tag grammar (@/utils/tag-grammar) so the editor's -// tokenizer/input rule can't drift from the read-only renderer's lexer -// (web/src/utils/remark-plugins/remark-tag.ts). -const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})\\s$`, "u"); -const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})`, "u"); -// Tests the REMAINDER of the source (not a single code unit) so astral-plane -// tag characters (emoji et al.) are seen whole, not as lone surrogates. -const TAG_CHAR_AHEAD = new RegExp(`^${TAG_CHAR_CLASS}`, "u"); - -/** - * Tag tokenizer, registered DIRECTLY on the global marked singleton instead of - * through `markdownTokenizer`. Two reasons: - * - * 1. `@tiptap/markdown`'s MarkdownManager wraps `markdownTokenizer.tokenize` - * in `tokenizer(src, tokens) { ... tokenize(src, tokens, helper) }` — the - * wrapper receives marked's TokenizerThis (with `lexer.state.inLink`) but - * does NOT forward it, so a manager-registered tokenizer can never know it - * is inside a link label. Registered natively, marked invokes us with - * `this.lexer` bound and we can decline inside `[label](url)` the same way - * remark-tag skips link nodes. - * 2. The manager defaults to the global `marked` export (`markedInstance = - * options?.marked ?? marked`) and `web` resolves the exact same marked - * module instance as `@tiptap/markdown` does, so this registration is - * visible to every lexer the manager creates. Module scope + idempotent: - * registered exactly once per page load (the manager's own per-Editor - * `marked.use` calls are the accumulation hazard documented in - * markdownCodec.ts; this adds a single registration, ever). - * - * The Tag mark below still declares `markdownTokenName: "memoTag"` + - * `parseMarkdown`, which is all the manager needs to route the token. - */ -function tokenizeTag(this: TokenizerThis, src: string): Tokens.Generic | undefined { - // remark-tag skips link nodes entirely; marked sets `state.inLink` while - // tokenizing link/reflink labels, so declining here keeps `[see #x](url)` - // a plain link label instead of tearing the tag out of it. - if (this.lexer?.state?.inLink) { - return undefined; - } - const match = TAG_TOKEN_RULE.exec(src); - if (!match) { - return undefined; - } - const rest = src.slice(match[0].length); - // `#a#b`: decline when the run is directly followed by another `#`, so the - // FIRST run stays plain text. NOT full remark parity — remark-tag treats - // `#a#b` as two tags and `##x` as all-text, while here `#a#b` becomes text - // "#a" + tag "b" and `##x` becomes text "#" + tag "x". The divergence is - // visual-only in the editor: both shapes serialize back byte-identically - // (the surrounding text nodes re-emit their literal characters). - if (rest.startsWith("#")) { - return undefined; - } - // Runs longer than 100 tag characters are not tags at all in remark-tag - // (the whole run stays plain text) — decline instead of splitting the run - // into a 100-char tag plus leftover text. - if (TAG_CHAR_AHEAD.test(rest)) { - return undefined; - } - return { type: "memoTag", raw: match[0], text: match[0], tag: match[1] }; -} - -let tagTokenizerRegistered = false; -function registerTagTokenizer() { - if (tagTokenizerRegistered) { - return; - } - tagTokenizerRegistered = true; - marked.use({ - extensions: [ - { - name: "memoTag", - level: "inline", - start: (src: string) => src.indexOf("#"), - tokenizer: tokenizeTag, - }, - ], - }); -} -registerTagTokenizer(); +// Built from the shared TAG_RUN (@/utils/tag-grammar) so the editor's input +// rule and tokenizer can't drift from the serialize-escape or the read-only +// renderer's lexer (web/src/utils/remark-plugins/remark-tag.ts). The capped-run +// lookahead in TAG_RUN also makes an over-long run decline to match — no +// separate length check needed. +const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_RUN})\\s$`, "u"); +const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_RUN})`, "u"); /** * Mark for memos `#tags`: styled in the editor, serialized back to `#tag` @@ -99,8 +25,20 @@ registerTagTokenizer(); * `applyMarkToContent` only attaches enclosing marks to text nodes, so an * atom inside bold silently drops the bold delimiters. * - * Parsed live while typing (input rule) and from markdown (the native marked - * tokenizer above). + * Parsed live while typing (the input rule) and from markdown (the + * `markdownTokenizer` below — the canonical @tiptap/markdown extension point, + * https://tiptap.dev/docs/editor/markdown/advanced-usage/custom-tokenizer). + * + * Two consequences of going through the manager (vs. registering on `marked` + * directly) are handled in tagMarkdown.ts: + * - The tokenizer can't see whether it sits inside a link label — the manager + * forwards only `{ inlineTokens, blockTokens }`, not the lexer's `inLink` + * state — so tag-in-link skipping is a tree pass there, mirroring how the + * read-only renderer (remark-tag) skips link nodes. + * - The manager re-registers this tokenizer onto the global `marked` on every + * Editor construction (the accumulation noted in markdownCodec.ts). The + * impact is sub-millisecond for memo-sized content and the test codec is a + * singleton; accepted as the cost of the canonical API. */ export const Tag = Mark.create({ name: "tag", @@ -154,6 +92,26 @@ export const Tag = Mark.create({ ]; }, + markdownTokenizer: { + name: "memoTag", + level: "inline", + start: (src: string) => src.indexOf("#"), + tokenize: (src: string) => { + const match = TAG_TOKEN_RULE.exec(src); + if (!match) { + return undefined; + } + // `#a#b`: decline when the run is directly followed by another `#`, so the + // FIRST run stays plain text "#a" + tag "b" (`##x` likewise → text "#" + + // tag "x"). The serialize-escape then escapes the tag-shaped literal "#a", + // so `#a#b` round-trips as `\#a#b` — doc-equivalent and stable thereafter. + // (An over-long run already declines: TAG_RUN's lookahead fails to match.) + if (src.slice(match[0].length).startsWith("#")) { + return undefined; + } + return { type: "memoTag", raw: match[0], tag: match[1] }; + }, + }, markdownTokenName: "memoTag", parseMarkdown: (token, helpers) => { const t = token as MarkdownToken & { tag?: string }; diff --git a/web/src/components/MemoEditor/Editor/extensions.ts b/web/src/components/MemoEditor/Editor/extensions.ts index ff8b32e3..fa941a08 100644 --- a/web/src/components/MemoEditor/Editor/extensions.ts +++ b/web/src/components/MemoEditor/Editor/extensions.ts @@ -1,11 +1,11 @@ import { type AnyExtension, mergeAttributes } from "@tiptap/core"; import { Heading } from "@tiptap/extension-heading"; import { TaskItem, TaskList } from "@tiptap/extension-list"; -import { Markdown } from "@tiptap/markdown"; import StarterKit from "@tiptap/starter-kit"; import { type HeadingLevel, headingClass, markdownStyles } from "@/lib/markdownStyles"; import { preservedExtensions } from "./PreservedBlock"; import { Tag } from "./Tag"; +import { TagAwareMarkdown } from "./tagMarkdown"; /** * StarterKit's Heading is bundled and cannot vary classes by level via static @@ -46,7 +46,7 @@ export function buildExtensions(): AnyExtension[] { StyledHeading.configure({ levels: [1, 2, 3, 4, 5, 6] }), TaskList, TaskItem.configure({ nested: true }), - Markdown, + TagAwareMarkdown, ...preservedExtensions, Tag, ]; diff --git a/web/src/components/MemoEditor/Editor/tagMarkdown.ts b/web/src/components/MemoEditor/Editor/tagMarkdown.ts new file mode 100644 index 00000000..eedfe463 --- /dev/null +++ b/web/src/components/MemoEditor/Editor/tagMarkdown.ts @@ -0,0 +1,113 @@ +import type { JSONContent } from "@tiptap/core"; +import { Markdown } from "@tiptap/markdown"; +import type { Schema } from "@tiptap/pm/model"; +import { TAG_RUN } from "@/utils/tag-grammar"; + +// A `#` begins a memo tag when a capped TAG_RUN follows it — the same grammar +// the editor tokenizer (Tag.ts) matches, so escape and parse can't disagree. +// The lookahead matches only the `#`, leaving the run itself untouched. +const TAG_HASH = new RegExp(`#(?=${TAG_RUN})`, "gu"); + +/** + * Backslash-escape a `#` that would otherwise re-parse into a `#tag`. + * + * memos layers hashtag syntax on top of CommonMark, so — exactly as + * prosemirror-markdown escapes inline syntax characters like `*` and `_` — + * plain text that looks like a tag must be escaped on serialize, or it silently + * becomes a tag on the next parse. + */ +export function escapeTagHashes(text: string): string { + return text.replace(TAG_HASH, "\\#"); +} + +type JsonMark = string | { type?: string }; +type JsonNode = { type?: string; marks?: JsonMark[] }; +type PatchableManager = { + encodeTextForMarkdown?: (text: string, node: JsonNode, parentNode?: JsonNode) => string; + parse?: (markdown: string) => JSONContent; +}; + +const markName = (mark: JsonMark): string => (typeof mark === "string" ? mark : (mark.type ?? "")); +const hasMark = (node: JsonNode, name: string): boolean => (node.marks ?? []).some((mark) => markName(mark) === name); + +/** + * A `#tag` only forms in plain inline prose. The tokenizer declines inside + * links, and `code: true` marks/nodes (the Tag mark itself, inline code, + * preserved spans/blocks, code blocks) serialize verbatim — so escaping a `#` + * in any of those contexts would be wrong (and would corrupt the byte-identical + * round-trips those constructs rely on). + */ +function escapesTagsHere(schema: Schema, node: JsonNode, parentNode?: JsonNode): boolean { + for (const mark of node.marks ?? []) { + const name = markName(mark); + if (name === "link") { + return false; + } + if (schema.marks[name]?.spec.code) { + return false; + } + } + const parentType = parentNode?.type; + if (parentType && schema.nodes[parentType]?.spec.code) { + return false; + } + return true; +} + +/** + * Strip the `tag` mark from any text that also carries a `link` mark. + * + * The `markdownTokenizer` in Tag.ts cannot tell it is inside a link label — the + * manager forwards only `{ inlineTokens, blockTokens }`, never the lexer's + * `inLink` state — so `[see #x](url)` parses with `#x` tagged. We undo that as a + * tree pass, the same way the read-only renderer (remark-tag) skips link nodes, + * keeping the editor and renderer in agreement: a `#` in a link label is link + * text, never a tag pill. + */ +function stripTagInsideLinks(node: JSONContent): JSONContent { + if (Array.isArray(node.marks) && hasMark(node, "link")) { + node.marks = node.marks.filter((mark) => markName(mark) !== "tag"); + } + node.content?.forEach(stripTagInsideLinks); + return node; +} + +/** + * The `markdownTokenizer` in Tag.ts handles `#tag` the canonical + * @tiptap/markdown way, but two memos-specific concerns sit above the tokenizer + * and have no public hook, so we compose them onto the manager here: + * + * - serialize: escape a tag-shaped `#` in plain prose ({@link escapeTagHashes}) + * so a literal `#NAS` can't silently re-parse into a tag; + * - parse: skip tags inside link labels ({@link stripTagInsideLinks}). + * + * Escapes stay a purely lexical concern — the document model carries no + * "escaped tag" node or mark. A literal `#NAS` is just text: parsing strips a + * leading `\` (marked's built-in escape) and serializing adds it back, the same + * way TipTap handles every other escapable character. + */ +export const TagAwareMarkdown = Markdown.extend({ + onBeforeCreate(props) { + this.parent?.(props); + const { editor } = this; + const manager = editor.markdown as unknown as PatchableManager | undefined; + if (!manager) { + return; + } + + const encode = manager.encodeTextForMarkdown; + if (typeof encode === "function") { + const bound = encode.bind(manager); + manager.encodeTextForMarkdown = (text, node, parentNode) => { + const encoded = bound(text, node, parentNode); + return escapesTagsHere(editor.schema, node, parentNode) ? escapeTagHashes(encoded) : encoded; + }; + } + + const parse = manager.parse; + if (typeof parse === "function") { + const bound = parse.bind(manager); + manager.parse = (markdown) => stripTagInsideLinks(bound(markdown)); + } + }, +}); diff --git a/web/src/components/MemoEditor/README.md b/web/src/components/MemoEditor/README.md index 2b2a40ad..afbd46a5 100644 --- a/web/src/components/MemoEditor/README.md +++ b/web/src/components/MemoEditor/README.md @@ -45,7 +45,8 @@ MemoEditor/ │ ├── extensions.ts # Canonical schema-relevant extension set (shared with codec) │ ├── markdownCodec.ts # Headless parse/serialize helpers (singleton editor) │ ├── PreservedBlock.ts # Byte-for-byte preservation of tables, math, raw HTML -│ ├── Tag.ts # Memos #tag mark +│ ├── Tag.ts # Memos #tag mark (markdownTokenizer + input rule) +│ ├── tagMarkdown.ts # Tag-aware Markdown: # escape on serialize, link-skip on parse │ ├── TagSuggestion.ts # # popup for WYSIWYG mode │ └── suggestionMenu.tsx # Shared suggestion popup renderer (used by TagSuggestion) ├── Toolbar/ # Toolbar sub-components (InsertMenu, VisibilitySelector) @@ -85,7 +86,17 @@ Mode switching is a markdown handoff: because both editors write into `state.con `PreservedBlock.ts` handles syntax the WYSIWYG editor does not model richly: tables, `$$math$$`, and raw HTML are captured at parse time with their raw markdown source, shown as editable monospace literal text, and re-emitted byte-for-byte on serialize. -`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans. +`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans. Its `#tag` lexing is a `markdownTokenizer` (the canonical `@tiptap/markdown` extension point); `tagMarkdown.ts` adds the two things that tokenizer can't do — on serialize it backslash-escapes a literal `#` that would otherwise re-parse into a tag (escapes are lexical, so there is no "escaped tag" node), and on parse it strips the tag mark from text inside link labels. The `#tag` grammar itself lives once in `utils/tag-grammar.ts` (`TAG_RUN`), shared by the tokenizer, the serialize-escape, and the read-only `remark-tag` renderer so they can't drift. + +### Why the markdown manager is worked around in several places + +`@tiptap/markdown` (3.26.0) exposes no public, per-instance hook for custom tokenizers or for text escaping, and it registers each extension's tokenizer onto the **global** `marked` singleton on every `new Editor()` — registrations it never removes. That one limitation is the reason for three otherwise-surprising choices, each documented in detail at its call site: + +- **`markdownCodec.ts` keeps a single editor instance** — re-creating editors would leak tokenizer registrations onto global `marked` and measurably degrade parse time. +- **`PreservedBlock.ts` registers its tokenizers on `marked` once at module scope** (idempotent) instead of via the per-extension `markdownTokenizer`, to avoid that per-construction accumulation. +- **`tagMarkdown.ts` composes onto the manager in `onBeforeCreate`** (escape, link-skip) because the relevant manager methods are `private` with no public seam. + +`Tag.ts` deliberately uses the canonical `markdownTokenizer` API and accepts the per-construction re-registration as its cost; `PreservedBlock.ts` refuses it for its seven tokenizers. The asymmetry is intentional — collapse both onto one path if upstream ever ships a public per-instance tokenizer hook. ### Suggestions diff --git a/web/src/utils/remark-plugins/remark-tag.ts b/web/src/utils/remark-plugins/remark-tag.ts index e2184718..0271f177 100644 --- a/web/src/utils/remark-plugins/remark-tag.ts +++ b/web/src/utils/remark-plugins/remark-tag.ts @@ -1,18 +1,64 @@ import type { Root, Text } from "mdast"; -import type { Node as UnistNode } from "unist"; +import type { Position, Node as UnistNode } from "unist"; import type { TagNode, TagNodeData } from "@/types/markdown"; import { isTagChar, MAX_TAG_LENGTH } from "@/utils/tag-grammar"; -function parseTagsFromText(text: string): Array<{ type: "text"; value: string } | { type: "tag"; value: string }> { - const segments: Array<{ type: "text"; value: string } | { type: "tag"; value: string }> = []; +type Segment = { type: "text"; value: string } | { type: "tag"; value: string }; - const chars = [...text]; +// CommonMark "ASCII punctuation": the only characters a leading backslash can +// escape. A backslash before anything else is a literal backslash. +function isAsciiPunctuation(char: string): boolean { + if (char.length !== 1) { + return false; + } + const code = char.charCodeAt(0); + return ( + (code >= 0x21 && code <= 0x2f) || // ! " # $ % & ' ( ) * + , - . / + (code >= 0x3a && code <= 0x40) || // : ; < = > ? @ + (code >= 0x5b && code <= 0x60) || // [ \ ] ^ _ ` + (code >= 0x7b && code <= 0x7e) // { | } ~ + ); +} + +/** + * Apply CommonMark backslash-unescaping to a raw source slice, tracking which + * resulting characters came from an escape. `\#` yields a `#` flagged escaped, + * so the tag lexer can tell a deliberately-escaped hash from a real tag. + * Returns code points (not UTF-16 units) so astral characters stay intact. + */ +function unescapeBackslashes(source: string): { chars: string[]; escaped: boolean[] } { + const codePoints = [...source]; + const chars: string[] = []; + const escaped: boolean[] = []; + + for (let i = 0; i < codePoints.length; i++) { + if (codePoints[i] === "\\" && i + 1 < codePoints.length && isAsciiPunctuation(codePoints[i + 1])) { + chars.push(codePoints[i + 1]); + escaped.push(true); + i++; + continue; + } + chars.push(codePoints[i]); + escaped.push(false); + } + + return { chars, escaped }; +} + +/** + * Split a run of characters into text/tag segments. `escaped[i]` marks a + * character that came from a backslash escape: an escaped `#` can never start a + * tag (so `\#NAS` stays literal text), matching the backend goldmark parser and + * the editor's marked tokenizer, both of which honor the escape natively. + */ +function parseSegments(chars: string[], escaped: boolean[]): Segment[] { + const segments: Segment[] = []; let i = 0; while (i < chars.length) { - if (chars[i] === "#" && i + 1 < chars.length && isTagChar(chars[i + 1])) { + if (chars[i] === "#" && !escaped[i] && i + 1 < chars.length && isTagChar(chars[i + 1])) { const prevChar = i > 0 ? chars[i - 1] : ""; - const nextChar = i + 1 < chars.length ? chars[i + 1] : ""; + const nextChar = chars[i + 1]; if (prevChar === "#" || nextChar === "#" || nextChar === " ") { segments.push({ type: "text", value: chars[i] }); @@ -35,8 +81,9 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string } } } + // Consume a plain-text run up to the next tag-eligible (non-escaped) hash. let j = i + 1; - while (j < chars.length && chars[j] !== "#") { + while (j < chars.length && !(chars[j] === "#" && !escaped[j])) { j++; } segments.push({ type: "text", value: chars.slice(i, j).join("") }); @@ -46,6 +93,34 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string } return segments; } +/** + * Segment a text node, preferring the original source slice (where escapes are + * still visible) over the post-escape `value`. The source-derived parse is only + * trusted when it reconstructs `value` byte-for-byte; otherwise — entity + * references, missing positions, upstream rewrites — we fall back to the + * value-based parse, which is exactly the pre-escape-support behavior. This + * makes escape support strictly additive: it never changes a node it can't + * faithfully account for. + */ +function segmentsForTextNode(value: string, position: Position | undefined, source: string): Segment[] { + const startOffset = position?.start?.offset; + const endOffset = position?.end?.offset; + + if (source && startOffset != null && endOffset != null) { + const slice = source.slice(startOffset, endOffset); + const { chars, escaped } = unescapeBackslashes(slice); + if (chars.join("") === value) { + return parseSegments(chars, escaped); + } + } + + const chars = [...value]; + return parseSegments( + chars, + chars.map(() => false), + ); +} + function createTagNode(tagValue: string): TagNode { const data: TagNodeData = { hName: "span", @@ -73,14 +148,14 @@ function isLinkNode(node: UnistNode): boolean { return node.type === "link" || node.type === "linkReference"; } -function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void { +function transformTagTextNodes(parent: ParentNode, insideLink: boolean, source: string): void { for (let index = 0; index < parent.children.length; index++) { const child = parent.children[index]; const childInsideLink = insideLink || isLinkNode(child); if (child.type === "text" && !childInsideLink) { const textNode = child as Text; - const segments = parseTagsFromText(textNode.value); + const segments = segmentsForTextNode(textNode.value, textNode.position, source); if (segments.every((seg) => seg.type === "text")) { continue; @@ -102,13 +177,16 @@ function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void { } if (isParentNode(child)) { - transformTagTextNodes(child, childInsideLink); + transformTagTextNodes(child, childInsideLink, source); } } } +type VFileLike = { value?: string | Uint8Array }; + export const remarkTag = () => { - return (tree: Root) => { - transformTagTextNodes(tree as ParentNode, false); + return (tree: Root, file: VFileLike) => { + const source = typeof file?.value === "string" ? file.value : ""; + transformTagTextNodes(tree as ParentNode, false, source); }; }; diff --git a/web/src/utils/tag-grammar.ts b/web/src/utils/tag-grammar.ts index b35d16f8..dcd81974 100644 --- a/web/src/utils/tag-grammar.ts +++ b/web/src/utils/tag-grammar.ts @@ -1,7 +1,8 @@ /** * The single source of truth for memos' `#tag` lexing grammar, shared by the - * editor tokenizer (components/MemoEditor/Editor/Tag.ts) and the read-only - * renderer (utils/remark-plugins/remark-tag.ts) so the two can't drift. + * editor tokenizer + serialize-escape (components/MemoEditor/Editor/Tag.ts and + * tagMarkdown.ts) and the read-only renderer (utils/remark-plugins/remark-tag.ts) + * so they can't drift. * * A tag character is any Unicode letter, number, or symbol, plus `_ - / &`. * A tag run is capped at MAX_TAG_LENGTH characters. @@ -10,6 +11,18 @@ export const TAG_CHAR_CLASS = "[\\p{L}\\p{N}\\p{S}_\\-/&]"; export const MAX_TAG_LENGTH = 100; +/** + * Regex source for a *capped* tag run: 1..MAX_TAG_LENGTH tag characters, + * refusing to match when a (MAX+1)-th tag character would follow (an over-long + * run is not a tag). Embed with the `u` flag. + * + * The two halves of the round-trip build on this same source so they can't + * disagree on what counts as a tag: the editor's input rule + tokenizer match + * `#(${TAG_RUN})`, and the serialize-escape (tagMarkdown.ts) escapes a `#` + * followed by `(?=${TAG_RUN})`. + */ +export const TAG_RUN = `${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}}(?!${TAG_CHAR_CLASS})`; + // Matches exactly one tag character. The `u` flag makes the class match whole // code points, so astral-plane symbols (emoji et al.) are tested intact rather // than as lone surrogates. diff --git a/web/tests/editor-controller.test.tsx b/web/tests/editor-controller.test.tsx index 4769854b..bbe1ca61 100644 --- a/web/tests/editor-controller.test.tsx +++ b/web/tests/editor-controller.test.tsx @@ -105,6 +105,43 @@ describe("tag suggestion insertion", () => { }); }); +describe("tag escaping", () => { + // Drives the ProseMirror input-rules plugin the way real typing does: + // someProp("handleTextInput", …) is exactly what the view calls on keystroke. + function typeText(editor: EditorInstance, text: string) { + const { from } = editor.state.selection; + editor.view.someProp("handleTextInput", (handler) => handler(editor.view, from, from, text)); + } + + it("undoing the tag autoformat leaves a durable literal #tag, not a pill", () => { + const { ref, editor } = setup(""); + act(() => { + editor.view.dispatch(editor.state.tr.insertText("#NAS", 1)); + }); + act(() => typeText(editor, " ")); // tag input rule fires → pill + act(() => { + editor.commands.undoInputRule(); // the standard "I didn't want that" gesture + }); + + // Back to plain text — and the serializer escapes it so it can't re-tag. + expect(ref.current?.getMarkdown()?.trim()).toBe("\\#NAS"); + const para = editor.getJSON().content?.[0]; + expect(para?.content?.some((n) => (n.marks ?? []).some((m) => m.type === "tag"))).toBeFalsy(); + }); + + it("a plain #tag still becomes a tag", () => { + const { ref, editor } = setup(""); + act(() => { + editor.view.dispatch(editor.state.tr.insertText("#work", 1)); + }); + act(() => typeText(editor, " ")); + + expect(ref.current?.getMarkdown()?.trim()).toBe("#work"); + const para = editor.getJSON().content?.[0]; + expect(para?.content?.[0]).toMatchObject({ text: "#work", marks: [{ type: "tag" }] }); + }); +}); + describe("external content sync", () => { it("a trim-equal echo of the editor's own output does not reset the document", () => { const { ref, rerender } = setupRerenderable("hello"); diff --git a/web/tests/editor-tag.test.ts b/web/tests/editor-tag.test.ts index e23e1cd7..5b7e9392 100644 --- a/web/tests/editor-tag.test.ts +++ b/web/tests/editor-tag.test.ts @@ -96,9 +96,13 @@ describe("Tag mark", () => { expect(roundTripMarkdown(input).trim()).toBe(input); }); - it("round-trips ##x and #a#b byte-identically (visual divergence documented in Tag.ts)", () => { + it("round-trips ##x unchanged and normalizes #a#b to \\#a#b (conservative # escaping)", () => { + // "##x" → text "#" + tag "x"; the lone "#" is not tag-shaped, so untouched. expect(roundTripMarkdown("##x").trim()).toBe("##x"); - expect(roundTripMarkdown("#a#b").trim()).toBe("#a#b"); + // "#a#b" → text "#a" + tag "b"; the plain "#a" is tag-shaped, so the + // serializer escapes it to keep it literal. Doc-equivalent, stable after. + expect(roundTripMarkdown("#a#b").trim()).toBe("\\#a#b"); + expect(roundTripMarkdown("\\#a#b").trim()).toBe("\\#a#b"); }); it("treats a run longer than 100 tag characters as plain text", () => { @@ -109,9 +113,29 @@ describe("Tag mark", () => { expect(textNodes.some((n) => hasTagMark(n))).toBe(false); }); - it("documents known gap: backslash-escaped \\# degrades after one cycle (upstream escape dropping)", () => { - const out = roundTripMarkdown("\\#escaped"); - expect(out.trim()).toBe("#escaped"); // upstream drops the escape (pre-existing); re-parse then sees a tag - expect(roundTripMarkdown(out)).toBe(out); // at least stable from then on + it("keeps a backslash-escaped \\#tag literal and durable across round-trips", () => { + const input = "\\#NAS is my server"; + + // Escapes are lexical: `\#NAS` parses to ordinary text — no tag mark, no + // bespoke "escaped tag" node — the `\` is simply consumed. + const children = firstParagraphChildren(input); + expect(children.some(hasTagMark)).toBe(false); + expect(children.map((n) => n.text ?? "").join("")).toBe("#NAS is my server"); + + // The serializer re-escapes the tag-shaped `#`, so it never degrades into a + // tag... + const once = roundTripMarkdown(input).trim(); + expect(once).toBe("\\#NAS is my server"); + // ...and stays stable on every subsequent cycle. + expect(roundTripMarkdown(once).trim()).toBe(once); + }); + + it("escapes only the literal tag while still tagging a real one beside it", () => { + const input = "\\#NAS and #real"; + expect(roundTripMarkdown(input).trim()).toBe("\\#NAS and #real"); + + const tagged = firstParagraphChildren(input).filter(hasTagMark); + expect(tagged).toHaveLength(1); + expect(tagged[0]).toMatchObject({ text: "#real", marks: [{ type: "tag", attrs: { tag: "real" } }] }); }); }); diff --git a/web/tests/remark-tag.test.tsx b/web/tests/remark-tag.test.tsx index c4d26c90..9b89e8f1 100644 --- a/web/tests/remark-tag.test.tsx +++ b/web/tests/remark-tag.test.tsx @@ -51,4 +51,31 @@ describe("remarkTag", () => { expect(html).toContain('data-tag="urgent"'); expect(html).toContain('data-tag="later"'); }); + + it("does not turn a backslash-escaped \\#tag into a tag, but still tags an unescaped one", () => { + const html = renderMarkdown("\\#NAS is my server and a #real tag"); + + // Escaped: rendered as the literal text "#NAS", never a tag pill. + expect(html).not.toContain('data-tag="NAS"'); + expect(html).toContain("#NAS"); + // Unescaped neighbour is unaffected. + expect(html).toContain('data-tag="real"'); + }); + + it("escapes only the marked hash when escaped and unescaped tags share a node", () => { + const html = renderMarkdown("\\#first then #second"); + + expect(html).not.toContain('data-tag="first"'); + expect(html).toContain("#first"); + expect(html).toContain('data-tag="second"'); + }); + + it("still tags a hash that shares a text node with an entity reference", () => { + // The source slice ("...&...") differs from the decoded value, so the + // escape-aware path bows out and the tag is detected the original way. + const html = renderMarkdown("Tom & Jerry #cartoon"); + + expect(html).toContain('data-tag="cartoon"'); + expect(html).toContain("Tom & Jerry"); + }); });