fix(editor): support backslash-escaping literal #tags
Memos layers #tag syntax on top of CommonMark, but there was no way to write a literal "#NAS" — it always became a clickable tag (issue #6035). Handle escapes lexically, the way TipTap/prosemirror-markdown do, with no "escaped tag" node in the document model: - Parse: marked's built-in backslash escape already turns "\#NAS" into plain text; the read-only renderer (remark-tag) now honors the same escape by lexing from the original source slice and falling back to the prior value-based parse when it can't faithfully reconstruct the text. - Serialize: a tag-aware Markdown extension (tagMarkdown.ts) backslash- escapes a "#" that would re-parse into a tag, skipping links and code where tags never form. - The editor's #tag lexing moves to the canonical @tiptap/markdown markdownTokenizer; tag-in-link skipping moves to a tree pass so the editor and renderer agree. The capped tag-run grammar is consolidated into utils/tag-grammar.ts (TAG_RUN), shared by the tokenizer, the serialize-escape, and the renderer so they can't drift. README documents why the markdown manager is worked around in three places.
This commit is contained in:
boojack
parent
20c19ef82d
commit
5f7e038aa7
9 changed files with 369 additions and 108 deletions
|
|
@ -1,93 +1,19 @@
|
|||
import type { MarkdownToken } from "@tiptap/core";
|
||||
import { InputRule, Mark, mergeAttributes } from "@tiptap/core";
|
||||
import type { TokenizerThis, Tokens } from "marked";
|
||||
import { marked } from "marked";
|
||||
import { tagStyles } from "@/lib/markdownStyles";
|
||||
import { MAX_TAG_LENGTH, TAG_CHAR_CLASS } from "@/utils/tag-grammar";
|
||||
import { TAG_RUN } from "@/utils/tag-grammar";
|
||||
|
||||
// Default tag pill, shared with the read-only view (MemoContent/Tag.tsx).
|
||||
// Computed once — renderHTML runs on every view update.
|
||||
const TAG_CLASS = `${tagStyles.base} ${tagStyles.defaultColor}`;
|
||||
|
||||
// Built from the shared tag grammar (@/utils/tag-grammar) so the editor's
|
||||
// tokenizer/input rule can't drift from the read-only renderer's lexer
|
||||
// (web/src/utils/remark-plugins/remark-tag.ts).
|
||||
const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})\\s$`, "u");
|
||||
const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})`, "u");
|
||||
// Tests the REMAINDER of the source (not a single code unit) so astral-plane
|
||||
// tag characters (emoji et al.) are seen whole, not as lone surrogates.
|
||||
const TAG_CHAR_AHEAD = new RegExp(`^${TAG_CHAR_CLASS}`, "u");
|
||||
|
||||
/**
|
||||
* Tag tokenizer, registered DIRECTLY on the global marked singleton instead of
|
||||
* through `markdownTokenizer`. Two reasons:
|
||||
*
|
||||
* 1. `@tiptap/markdown`'s MarkdownManager wraps `markdownTokenizer.tokenize`
|
||||
* in `tokenizer(src, tokens) { ... tokenize(src, tokens, helper) }` — the
|
||||
* wrapper receives marked's TokenizerThis (with `lexer.state.inLink`) but
|
||||
* does NOT forward it, so a manager-registered tokenizer can never know it
|
||||
* is inside a link label. Registered natively, marked invokes us with
|
||||
* `this.lexer` bound and we can decline inside `[label](url)` the same way
|
||||
* remark-tag skips link nodes.
|
||||
* 2. The manager defaults to the global `marked` export (`markedInstance =
|
||||
* options?.marked ?? marked`) and `web` resolves the exact same marked
|
||||
* module instance as `@tiptap/markdown` does, so this registration is
|
||||
* visible to every lexer the manager creates. Module scope + idempotent:
|
||||
* registered exactly once per page load (the manager's own per-Editor
|
||||
* `marked.use` calls are the accumulation hazard documented in
|
||||
* markdownCodec.ts; this adds a single registration, ever).
|
||||
*
|
||||
* The Tag mark below still declares `markdownTokenName: "memoTag"` +
|
||||
* `parseMarkdown`, which is all the manager needs to route the token.
|
||||
*/
|
||||
function tokenizeTag(this: TokenizerThis, src: string): Tokens.Generic | undefined {
|
||||
// remark-tag skips link nodes entirely; marked sets `state.inLink` while
|
||||
// tokenizing link/reflink labels, so declining here keeps `[see #x](url)`
|
||||
// a plain link label instead of tearing the tag out of it.
|
||||
if (this.lexer?.state?.inLink) {
|
||||
return undefined;
|
||||
}
|
||||
const match = TAG_TOKEN_RULE.exec(src);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
const rest = src.slice(match[0].length);
|
||||
// `#a#b`: decline when the run is directly followed by another `#`, so the
|
||||
// FIRST run stays plain text. NOT full remark parity — remark-tag treats
|
||||
// `#a#b` as two tags and `##x` as all-text, while here `#a#b` becomes text
|
||||
// "#a" + tag "b" and `##x` becomes text "#" + tag "x". The divergence is
|
||||
// visual-only in the editor: both shapes serialize back byte-identically
|
||||
// (the surrounding text nodes re-emit their literal characters).
|
||||
if (rest.startsWith("#")) {
|
||||
return undefined;
|
||||
}
|
||||
// Runs longer than 100 tag characters are not tags at all in remark-tag
|
||||
// (the whole run stays plain text) — decline instead of splitting the run
|
||||
// into a 100-char tag plus leftover text.
|
||||
if (TAG_CHAR_AHEAD.test(rest)) {
|
||||
return undefined;
|
||||
}
|
||||
return { type: "memoTag", raw: match[0], text: match[0], tag: match[1] };
|
||||
}
|
||||
|
||||
let tagTokenizerRegistered = false;
|
||||
function registerTagTokenizer() {
|
||||
if (tagTokenizerRegistered) {
|
||||
return;
|
||||
}
|
||||
tagTokenizerRegistered = true;
|
||||
marked.use({
|
||||
extensions: [
|
||||
{
|
||||
name: "memoTag",
|
||||
level: "inline",
|
||||
start: (src: string) => src.indexOf("#"),
|
||||
tokenizer: tokenizeTag,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
registerTagTokenizer();
|
||||
// Built from the shared TAG_RUN (@/utils/tag-grammar) so the editor's input
|
||||
// rule and tokenizer can't drift from the serialize-escape or the read-only
|
||||
// renderer's lexer (web/src/utils/remark-plugins/remark-tag.ts). The capped-run
|
||||
// lookahead in TAG_RUN also makes an over-long run decline to match — no
|
||||
// separate length check needed.
|
||||
const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_RUN})\\s$`, "u");
|
||||
const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_RUN})`, "u");
|
||||
|
||||
/**
|
||||
* Mark for memos `#tags`: styled in the editor, serialized back to `#tag`
|
||||
|
|
@ -99,8 +25,20 @@ registerTagTokenizer();
|
|||
* `applyMarkToContent` only attaches enclosing marks to text nodes, so an
|
||||
* atom inside bold silently drops the bold delimiters.
|
||||
*
|
||||
* Parsed live while typing (input rule) and from markdown (the native marked
|
||||
* tokenizer above).
|
||||
* Parsed live while typing (the input rule) and from markdown (the
|
||||
* `markdownTokenizer` below — the canonical @tiptap/markdown extension point,
|
||||
* https://tiptap.dev/docs/editor/markdown/advanced-usage/custom-tokenizer).
|
||||
*
|
||||
* Two consequences of going through the manager (vs. registering on `marked`
|
||||
* directly) are handled in tagMarkdown.ts:
|
||||
* - The tokenizer can't see whether it sits inside a link label — the manager
|
||||
* forwards only `{ inlineTokens, blockTokens }`, not the lexer's `inLink`
|
||||
* state — so tag-in-link skipping is a tree pass there, mirroring how the
|
||||
* read-only renderer (remark-tag) skips link nodes.
|
||||
* - The manager re-registers this tokenizer onto the global `marked` on every
|
||||
* Editor construction (the accumulation noted in markdownCodec.ts). The
|
||||
* impact is sub-millisecond for memo-sized content and the test codec is a
|
||||
* singleton; accepted as the cost of the canonical API.
|
||||
*/
|
||||
export const Tag = Mark.create({
|
||||
name: "tag",
|
||||
|
|
@ -154,6 +92,26 @@ export const Tag = Mark.create({
|
|||
];
|
||||
},
|
||||
|
||||
markdownTokenizer: {
|
||||
name: "memoTag",
|
||||
level: "inline",
|
||||
start: (src: string) => src.indexOf("#"),
|
||||
tokenize: (src: string) => {
|
||||
const match = TAG_TOKEN_RULE.exec(src);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
// `#a#b`: decline when the run is directly followed by another `#`, so the
|
||||
// FIRST run stays plain text "#a" + tag "b" (`##x` likewise → text "#" +
|
||||
// tag "x"). The serialize-escape then escapes the tag-shaped literal "#a",
|
||||
// so `#a#b` round-trips as `\#a#b` — doc-equivalent and stable thereafter.
|
||||
// (An over-long run already declines: TAG_RUN's lookahead fails to match.)
|
||||
if (src.slice(match[0].length).startsWith("#")) {
|
||||
return undefined;
|
||||
}
|
||||
return { type: "memoTag", raw: match[0], tag: match[1] };
|
||||
},
|
||||
},
|
||||
markdownTokenName: "memoTag",
|
||||
parseMarkdown: (token, helpers) => {
|
||||
const t = token as MarkdownToken & { tag?: string };
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { type AnyExtension, mergeAttributes } from "@tiptap/core";
|
||||
import { Heading } from "@tiptap/extension-heading";
|
||||
import { TaskItem, TaskList } from "@tiptap/extension-list";
|
||||
import { Markdown } from "@tiptap/markdown";
|
||||
import StarterKit from "@tiptap/starter-kit";
|
||||
import { type HeadingLevel, headingClass, markdownStyles } from "@/lib/markdownStyles";
|
||||
import { preservedExtensions } from "./PreservedBlock";
|
||||
import { Tag } from "./Tag";
|
||||
import { TagAwareMarkdown } from "./tagMarkdown";
|
||||
|
||||
/**
|
||||
* StarterKit's Heading is bundled and cannot vary classes by level via static
|
||||
|
|
@ -46,7 +46,7 @@ export function buildExtensions(): AnyExtension[] {
|
|||
StyledHeading.configure({ levels: [1, 2, 3, 4, 5, 6] }),
|
||||
TaskList,
|
||||
TaskItem.configure({ nested: true }),
|
||||
Markdown,
|
||||
TagAwareMarkdown,
|
||||
...preservedExtensions,
|
||||
Tag,
|
||||
];
|
||||
|
|
|
|||
113
web/src/components/MemoEditor/Editor/tagMarkdown.ts
Normal file
113
web/src/components/MemoEditor/Editor/tagMarkdown.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import type { JSONContent } from "@tiptap/core";
|
||||
import { Markdown } from "@tiptap/markdown";
|
||||
import type { Schema } from "@tiptap/pm/model";
|
||||
import { TAG_RUN } from "@/utils/tag-grammar";
|
||||
|
||||
// A `#` begins a memo tag when a capped TAG_RUN follows it — the same grammar
|
||||
// the editor tokenizer (Tag.ts) matches, so escape and parse can't disagree.
|
||||
// The lookahead matches only the `#`, leaving the run itself untouched.
|
||||
const TAG_HASH = new RegExp(`#(?=${TAG_RUN})`, "gu");
|
||||
|
||||
/**
|
||||
* Backslash-escape a `#` that would otherwise re-parse into a `#tag`.
|
||||
*
|
||||
* memos layers hashtag syntax on top of CommonMark, so — exactly as
|
||||
* prosemirror-markdown escapes inline syntax characters like `*` and `_` —
|
||||
* plain text that looks like a tag must be escaped on serialize, or it silently
|
||||
* becomes a tag on the next parse.
|
||||
*/
|
||||
export function escapeTagHashes(text: string): string {
|
||||
return text.replace(TAG_HASH, "\\#");
|
||||
}
|
||||
|
||||
type JsonMark = string | { type?: string };
|
||||
type JsonNode = { type?: string; marks?: JsonMark[] };
|
||||
type PatchableManager = {
|
||||
encodeTextForMarkdown?: (text: string, node: JsonNode, parentNode?: JsonNode) => string;
|
||||
parse?: (markdown: string) => JSONContent;
|
||||
};
|
||||
|
||||
const markName = (mark: JsonMark): string => (typeof mark === "string" ? mark : (mark.type ?? ""));
|
||||
const hasMark = (node: JsonNode, name: string): boolean => (node.marks ?? []).some((mark) => markName(mark) === name);
|
||||
|
||||
/**
|
||||
* A `#tag` only forms in plain inline prose. The tokenizer declines inside
|
||||
* links, and `code: true` marks/nodes (the Tag mark itself, inline code,
|
||||
* preserved spans/blocks, code blocks) serialize verbatim — so escaping a `#`
|
||||
* in any of those contexts would be wrong (and would corrupt the byte-identical
|
||||
* round-trips those constructs rely on).
|
||||
*/
|
||||
function escapesTagsHere(schema: Schema, node: JsonNode, parentNode?: JsonNode): boolean {
|
||||
for (const mark of node.marks ?? []) {
|
||||
const name = markName(mark);
|
||||
if (name === "link") {
|
||||
return false;
|
||||
}
|
||||
if (schema.marks[name]?.spec.code) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const parentType = parentNode?.type;
|
||||
if (parentType && schema.nodes[parentType]?.spec.code) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip the `tag` mark from any text that also carries a `link` mark.
|
||||
*
|
||||
* The `markdownTokenizer` in Tag.ts cannot tell it is inside a link label — the
|
||||
* manager forwards only `{ inlineTokens, blockTokens }`, never the lexer's
|
||||
* `inLink` state — so `[see #x](url)` parses with `#x` tagged. We undo that as a
|
||||
* tree pass, the same way the read-only renderer (remark-tag) skips link nodes,
|
||||
* keeping the editor and renderer in agreement: a `#` in a link label is link
|
||||
* text, never a tag pill.
|
||||
*/
|
||||
function stripTagInsideLinks(node: JSONContent): JSONContent {
|
||||
if (Array.isArray(node.marks) && hasMark(node, "link")) {
|
||||
node.marks = node.marks.filter((mark) => markName(mark) !== "tag");
|
||||
}
|
||||
node.content?.forEach(stripTagInsideLinks);
|
||||
return node;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `markdownTokenizer` in Tag.ts handles `#tag` the canonical
|
||||
* @tiptap/markdown way, but two memos-specific concerns sit above the tokenizer
|
||||
* and have no public hook, so we compose them onto the manager here:
|
||||
*
|
||||
* - serialize: escape a tag-shaped `#` in plain prose ({@link escapeTagHashes})
|
||||
* so a literal `#NAS` can't silently re-parse into a tag;
|
||||
* - parse: skip tags inside link labels ({@link stripTagInsideLinks}).
|
||||
*
|
||||
* Escapes stay a purely lexical concern — the document model carries no
|
||||
* "escaped tag" node or mark. A literal `#NAS` is just text: parsing strips a
|
||||
* leading `\` (marked's built-in escape) and serializing adds it back, the same
|
||||
* way TipTap handles every other escapable character.
|
||||
*/
|
||||
export const TagAwareMarkdown = Markdown.extend({
|
||||
onBeforeCreate(props) {
|
||||
this.parent?.(props);
|
||||
const { editor } = this;
|
||||
const manager = editor.markdown as unknown as PatchableManager | undefined;
|
||||
if (!manager) {
|
||||
return;
|
||||
}
|
||||
|
||||
const encode = manager.encodeTextForMarkdown;
|
||||
if (typeof encode === "function") {
|
||||
const bound = encode.bind(manager);
|
||||
manager.encodeTextForMarkdown = (text, node, parentNode) => {
|
||||
const encoded = bound(text, node, parentNode);
|
||||
return escapesTagsHere(editor.schema, node, parentNode) ? escapeTagHashes(encoded) : encoded;
|
||||
};
|
||||
}
|
||||
|
||||
const parse = manager.parse;
|
||||
if (typeof parse === "function") {
|
||||
const bound = parse.bind(manager);
|
||||
manager.parse = (markdown) => stripTagInsideLinks(bound(markdown));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
|
@ -45,7 +45,8 @@ MemoEditor/
|
|||
│ ├── extensions.ts # Canonical schema-relevant extension set (shared with codec)
|
||||
│ ├── markdownCodec.ts # Headless parse/serialize helpers (singleton editor)
|
||||
│ ├── PreservedBlock.ts # Byte-for-byte preservation of tables, math, raw HTML
|
||||
│ ├── Tag.ts # Memos #tag mark
|
||||
│ ├── Tag.ts # Memos #tag mark (markdownTokenizer + input rule)
|
||||
│ ├── tagMarkdown.ts # Tag-aware Markdown: # escape on serialize, link-skip on parse
|
||||
│ ├── TagSuggestion.ts # # popup for WYSIWYG mode
|
||||
│ └── suggestionMenu.tsx # Shared suggestion popup renderer (used by TagSuggestion)
|
||||
├── Toolbar/ # Toolbar sub-components (InsertMenu, VisibilitySelector)
|
||||
|
|
@ -85,7 +86,17 @@ Mode switching is a markdown handoff: because both editors write into `state.con
|
|||
|
||||
`PreservedBlock.ts` handles syntax the WYSIWYG editor does not model richly: tables, `$$math$$`, and raw HTML are captured at parse time with their raw markdown source, shown as editable monospace literal text, and re-emitted byte-for-byte on serialize.
|
||||
|
||||
`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans.
|
||||
`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans. Its `#tag` lexing is a `markdownTokenizer` (the canonical `@tiptap/markdown` extension point); `tagMarkdown.ts` adds the two things that tokenizer can't do — on serialize it backslash-escapes a literal `#` that would otherwise re-parse into a tag (escapes are lexical, so there is no "escaped tag" node), and on parse it strips the tag mark from text inside link labels. The `#tag` grammar itself lives once in `utils/tag-grammar.ts` (`TAG_RUN`), shared by the tokenizer, the serialize-escape, and the read-only `remark-tag` renderer so they can't drift.
|
||||
|
||||
### Why the markdown manager is worked around in several places
|
||||
|
||||
`@tiptap/markdown` (3.26.0) exposes no public, per-instance hook for custom tokenizers or for text escaping, and it registers each extension's tokenizer onto the **global** `marked` singleton on every `new Editor()` — registrations it never removes. That one limitation is the reason for three otherwise-surprising choices, each documented in detail at its call site:
|
||||
|
||||
- **`markdownCodec.ts` keeps a single editor instance** — re-creating editors would leak tokenizer registrations onto global `marked` and measurably degrade parse time.
|
||||
- **`PreservedBlock.ts` registers its tokenizers on `marked` once at module scope** (idempotent) instead of via the per-extension `markdownTokenizer`, to avoid that per-construction accumulation.
|
||||
- **`tagMarkdown.ts` composes onto the manager in `onBeforeCreate`** (escape, link-skip) because the relevant manager methods are `private` with no public seam.
|
||||
|
||||
`Tag.ts` deliberately uses the canonical `markdownTokenizer` API and accepts the per-construction re-registration as its cost; `PreservedBlock.ts` refuses it for its seven tokenizers. The asymmetry is intentional — collapse both onto one path if upstream ever ships a public per-instance tokenizer hook.
|
||||
|
||||
### Suggestions
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,64 @@
|
|||
import type { Root, Text } from "mdast";
|
||||
import type { Node as UnistNode } from "unist";
|
||||
import type { Position, Node as UnistNode } from "unist";
|
||||
import type { TagNode, TagNodeData } from "@/types/markdown";
|
||||
import { isTagChar, MAX_TAG_LENGTH } from "@/utils/tag-grammar";
|
||||
|
||||
function parseTagsFromText(text: string): Array<{ type: "text"; value: string } | { type: "tag"; value: string }> {
|
||||
const segments: Array<{ type: "text"; value: string } | { type: "tag"; value: string }> = [];
|
||||
type Segment = { type: "text"; value: string } | { type: "tag"; value: string };
|
||||
|
||||
const chars = [...text];
|
||||
// CommonMark "ASCII punctuation": the only characters a leading backslash can
|
||||
// escape. A backslash before anything else is a literal backslash.
|
||||
function isAsciiPunctuation(char: string): boolean {
|
||||
if (char.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
const code = char.charCodeAt(0);
|
||||
return (
|
||||
(code >= 0x21 && code <= 0x2f) || // ! " # $ % & ' ( ) * + , - . /
|
||||
(code >= 0x3a && code <= 0x40) || // : ; < = > ? @
|
||||
(code >= 0x5b && code <= 0x60) || // [ \ ] ^ _ `
|
||||
(code >= 0x7b && code <= 0x7e) // { | } ~
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply CommonMark backslash-unescaping to a raw source slice, tracking which
|
||||
* resulting characters came from an escape. `\#` yields a `#` flagged escaped,
|
||||
* so the tag lexer can tell a deliberately-escaped hash from a real tag.
|
||||
* Returns code points (not UTF-16 units) so astral characters stay intact.
|
||||
*/
|
||||
function unescapeBackslashes(source: string): { chars: string[]; escaped: boolean[] } {
|
||||
const codePoints = [...source];
|
||||
const chars: string[] = [];
|
||||
const escaped: boolean[] = [];
|
||||
|
||||
for (let i = 0; i < codePoints.length; i++) {
|
||||
if (codePoints[i] === "\\" && i + 1 < codePoints.length && isAsciiPunctuation(codePoints[i + 1])) {
|
||||
chars.push(codePoints[i + 1]);
|
||||
escaped.push(true);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
chars.push(codePoints[i]);
|
||||
escaped.push(false);
|
||||
}
|
||||
|
||||
return { chars, escaped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a run of characters into text/tag segments. `escaped[i]` marks a
|
||||
* character that came from a backslash escape: an escaped `#` can never start a
|
||||
* tag (so `\#NAS` stays literal text), matching the backend goldmark parser and
|
||||
* the editor's marked tokenizer, both of which honor the escape natively.
|
||||
*/
|
||||
function parseSegments(chars: string[], escaped: boolean[]): Segment[] {
|
||||
const segments: Segment[] = [];
|
||||
let i = 0;
|
||||
|
||||
while (i < chars.length) {
|
||||
if (chars[i] === "#" && i + 1 < chars.length && isTagChar(chars[i + 1])) {
|
||||
if (chars[i] === "#" && !escaped[i] && i + 1 < chars.length && isTagChar(chars[i + 1])) {
|
||||
const prevChar = i > 0 ? chars[i - 1] : "";
|
||||
const nextChar = i + 1 < chars.length ? chars[i + 1] : "";
|
||||
const nextChar = chars[i + 1];
|
||||
|
||||
if (prevChar === "#" || nextChar === "#" || nextChar === " ") {
|
||||
segments.push({ type: "text", value: chars[i] });
|
||||
|
|
@ -35,8 +81,9 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string }
|
|||
}
|
||||
}
|
||||
|
||||
// Consume a plain-text run up to the next tag-eligible (non-escaped) hash.
|
||||
let j = i + 1;
|
||||
while (j < chars.length && chars[j] !== "#") {
|
||||
while (j < chars.length && !(chars[j] === "#" && !escaped[j])) {
|
||||
j++;
|
||||
}
|
||||
segments.push({ type: "text", value: chars.slice(i, j).join("") });
|
||||
|
|
@ -46,6 +93,34 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string }
|
|||
return segments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Segment a text node, preferring the original source slice (where escapes are
|
||||
* still visible) over the post-escape `value`. The source-derived parse is only
|
||||
* trusted when it reconstructs `value` byte-for-byte; otherwise — entity
|
||||
* references, missing positions, upstream rewrites — we fall back to the
|
||||
* value-based parse, which is exactly the pre-escape-support behavior. This
|
||||
* makes escape support strictly additive: it never changes a node it can't
|
||||
* faithfully account for.
|
||||
*/
|
||||
function segmentsForTextNode(value: string, position: Position | undefined, source: string): Segment[] {
|
||||
const startOffset = position?.start?.offset;
|
||||
const endOffset = position?.end?.offset;
|
||||
|
||||
if (source && startOffset != null && endOffset != null) {
|
||||
const slice = source.slice(startOffset, endOffset);
|
||||
const { chars, escaped } = unescapeBackslashes(slice);
|
||||
if (chars.join("") === value) {
|
||||
return parseSegments(chars, escaped);
|
||||
}
|
||||
}
|
||||
|
||||
const chars = [...value];
|
||||
return parseSegments(
|
||||
chars,
|
||||
chars.map(() => false),
|
||||
);
|
||||
}
|
||||
|
||||
function createTagNode(tagValue: string): TagNode {
|
||||
const data: TagNodeData = {
|
||||
hName: "span",
|
||||
|
|
@ -73,14 +148,14 @@ function isLinkNode(node: UnistNode): boolean {
|
|||
return node.type === "link" || node.type === "linkReference";
|
||||
}
|
||||
|
||||
function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void {
|
||||
function transformTagTextNodes(parent: ParentNode, insideLink: boolean, source: string): void {
|
||||
for (let index = 0; index < parent.children.length; index++) {
|
||||
const child = parent.children[index];
|
||||
const childInsideLink = insideLink || isLinkNode(child);
|
||||
|
||||
if (child.type === "text" && !childInsideLink) {
|
||||
const textNode = child as Text;
|
||||
const segments = parseTagsFromText(textNode.value);
|
||||
const segments = segmentsForTextNode(textNode.value, textNode.position, source);
|
||||
|
||||
if (segments.every((seg) => seg.type === "text")) {
|
||||
continue;
|
||||
|
|
@ -102,13 +177,16 @@ function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void {
|
|||
}
|
||||
|
||||
if (isParentNode(child)) {
|
||||
transformTagTextNodes(child, childInsideLink);
|
||||
transformTagTextNodes(child, childInsideLink, source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type VFileLike = { value?: string | Uint8Array };
|
||||
|
||||
export const remarkTag = () => {
|
||||
return (tree: Root) => {
|
||||
transformTagTextNodes(tree as ParentNode, false);
|
||||
return (tree: Root, file: VFileLike) => {
|
||||
const source = typeof file?.value === "string" ? file.value : "";
|
||||
transformTagTextNodes(tree as ParentNode, false, source);
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
/**
|
||||
* The single source of truth for memos' `#tag` lexing grammar, shared by the
|
||||
* editor tokenizer (components/MemoEditor/Editor/Tag.ts) and the read-only
|
||||
* renderer (utils/remark-plugins/remark-tag.ts) so the two can't drift.
|
||||
* editor tokenizer + serialize-escape (components/MemoEditor/Editor/Tag.ts and
|
||||
* tagMarkdown.ts) and the read-only renderer (utils/remark-plugins/remark-tag.ts)
|
||||
* so they can't drift.
|
||||
*
|
||||
* A tag character is any Unicode letter, number, or symbol, plus `_ - / &`.
|
||||
* A tag run is capped at MAX_TAG_LENGTH characters.
|
||||
|
|
@ -10,6 +11,18 @@ export const TAG_CHAR_CLASS = "[\\p{L}\\p{N}\\p{S}_\\-/&]";
|
|||
|
||||
export const MAX_TAG_LENGTH = 100;
|
||||
|
||||
/**
|
||||
* Regex source for a *capped* tag run: 1..MAX_TAG_LENGTH tag characters,
|
||||
* refusing to match when a (MAX+1)-th tag character would follow (an over-long
|
||||
* run is not a tag). Embed with the `u` flag.
|
||||
*
|
||||
* The two halves of the round-trip build on this same source so they can't
|
||||
* disagree on what counts as a tag: the editor's input rule + tokenizer match
|
||||
* `#(${TAG_RUN})`, and the serialize-escape (tagMarkdown.ts) escapes a `#`
|
||||
* followed by `(?=${TAG_RUN})`.
|
||||
*/
|
||||
export const TAG_RUN = `${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}}(?!${TAG_CHAR_CLASS})`;
|
||||
|
||||
// Matches exactly one tag character. The `u` flag makes the class match whole
|
||||
// code points, so astral-plane symbols (emoji et al.) are tested intact rather
|
||||
// than as lone surrogates.
|
||||
|
|
|
|||
|
|
@ -105,6 +105,43 @@ describe("tag suggestion insertion", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("tag escaping", () => {
|
||||
// Drives the ProseMirror input-rules plugin the way real typing does:
|
||||
// someProp("handleTextInput", …) is exactly what the view calls on keystroke.
|
||||
function typeText(editor: EditorInstance, text: string) {
|
||||
const { from } = editor.state.selection;
|
||||
editor.view.someProp("handleTextInput", (handler) => handler(editor.view, from, from, text));
|
||||
}
|
||||
|
||||
it("undoing the tag autoformat leaves a durable literal #tag, not a pill", () => {
|
||||
const { ref, editor } = setup("");
|
||||
act(() => {
|
||||
editor.view.dispatch(editor.state.tr.insertText("#NAS", 1));
|
||||
});
|
||||
act(() => typeText(editor, " ")); // tag input rule fires → pill
|
||||
act(() => {
|
||||
editor.commands.undoInputRule(); // the standard "I didn't want that" gesture
|
||||
});
|
||||
|
||||
// Back to plain text — and the serializer escapes it so it can't re-tag.
|
||||
expect(ref.current?.getMarkdown()?.trim()).toBe("\\#NAS");
|
||||
const para = editor.getJSON().content?.[0];
|
||||
expect(para?.content?.some((n) => (n.marks ?? []).some((m) => m.type === "tag"))).toBeFalsy();
|
||||
});
|
||||
|
||||
it("a plain #tag still becomes a tag", () => {
|
||||
const { ref, editor } = setup("");
|
||||
act(() => {
|
||||
editor.view.dispatch(editor.state.tr.insertText("#work", 1));
|
||||
});
|
||||
act(() => typeText(editor, " "));
|
||||
|
||||
expect(ref.current?.getMarkdown()?.trim()).toBe("#work");
|
||||
const para = editor.getJSON().content?.[0];
|
||||
expect(para?.content?.[0]).toMatchObject({ text: "#work", marks: [{ type: "tag" }] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("external content sync", () => {
|
||||
it("a trim-equal echo of the editor's own output does not reset the document", () => {
|
||||
const { ref, rerender } = setupRerenderable("hello");
|
||||
|
|
|
|||
|
|
@ -96,9 +96,13 @@ describe("Tag mark", () => {
|
|||
expect(roundTripMarkdown(input).trim()).toBe(input);
|
||||
});
|
||||
|
||||
it("round-trips ##x and #a#b byte-identically (visual divergence documented in Tag.ts)", () => {
|
||||
it("round-trips ##x unchanged and normalizes #a#b to \\#a#b (conservative # escaping)", () => {
|
||||
// "##x" → text "#" + tag "x"; the lone "#" is not tag-shaped, so untouched.
|
||||
expect(roundTripMarkdown("##x").trim()).toBe("##x");
|
||||
expect(roundTripMarkdown("#a#b").trim()).toBe("#a#b");
|
||||
// "#a#b" → text "#a" + tag "b"; the plain "#a" is tag-shaped, so the
|
||||
// serializer escapes it to keep it literal. Doc-equivalent, stable after.
|
||||
expect(roundTripMarkdown("#a#b").trim()).toBe("\\#a#b");
|
||||
expect(roundTripMarkdown("\\#a#b").trim()).toBe("\\#a#b");
|
||||
});
|
||||
|
||||
it("treats a run longer than 100 tag characters as plain text", () => {
|
||||
|
|
@ -109,9 +113,29 @@ describe("Tag mark", () => {
|
|||
expect(textNodes.some((n) => hasTagMark(n))).toBe(false);
|
||||
});
|
||||
|
||||
it("documents known gap: backslash-escaped \\# degrades after one cycle (upstream escape dropping)", () => {
|
||||
const out = roundTripMarkdown("\\#escaped");
|
||||
expect(out.trim()).toBe("#escaped"); // upstream drops the escape (pre-existing); re-parse then sees a tag
|
||||
expect(roundTripMarkdown(out)).toBe(out); // at least stable from then on
|
||||
it("keeps a backslash-escaped \\#tag literal and durable across round-trips", () => {
|
||||
const input = "\\#NAS is my server";
|
||||
|
||||
// Escapes are lexical: `\#NAS` parses to ordinary text — no tag mark, no
|
||||
// bespoke "escaped tag" node — the `\` is simply consumed.
|
||||
const children = firstParagraphChildren(input);
|
||||
expect(children.some(hasTagMark)).toBe(false);
|
||||
expect(children.map((n) => n.text ?? "").join("")).toBe("#NAS is my server");
|
||||
|
||||
// The serializer re-escapes the tag-shaped `#`, so it never degrades into a
|
||||
// tag...
|
||||
const once = roundTripMarkdown(input).trim();
|
||||
expect(once).toBe("\\#NAS is my server");
|
||||
// ...and stays stable on every subsequent cycle.
|
||||
expect(roundTripMarkdown(once).trim()).toBe(once);
|
||||
});
|
||||
|
||||
it("escapes only the literal tag while still tagging a real one beside it", () => {
|
||||
const input = "\\#NAS and #real";
|
||||
expect(roundTripMarkdown(input).trim()).toBe("\\#NAS and #real");
|
||||
|
||||
const tagged = firstParagraphChildren(input).filter(hasTagMark);
|
||||
expect(tagged).toHaveLength(1);
|
||||
expect(tagged[0]).toMatchObject({ text: "#real", marks: [{ type: "tag", attrs: { tag: "real" } }] });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -51,4 +51,31 @@ describe("remarkTag", () => {
|
|||
expect(html).toContain('data-tag="urgent"');
|
||||
expect(html).toContain('data-tag="later"');
|
||||
});
|
||||
|
||||
it("does not turn a backslash-escaped \\#tag into a tag, but still tags an unescaped one", () => {
|
||||
const html = renderMarkdown("\\#NAS is my server and a #real tag");
|
||||
|
||||
// Escaped: rendered as the literal text "#NAS", never a tag pill.
|
||||
expect(html).not.toContain('data-tag="NAS"');
|
||||
expect(html).toContain("#NAS");
|
||||
// Unescaped neighbour is unaffected.
|
||||
expect(html).toContain('data-tag="real"');
|
||||
});
|
||||
|
||||
it("escapes only the marked hash when escaped and unescaped tags share a node", () => {
|
||||
const html = renderMarkdown("\\#first then #second");
|
||||
|
||||
expect(html).not.toContain('data-tag="first"');
|
||||
expect(html).toContain("#first");
|
||||
expect(html).toContain('data-tag="second"');
|
||||
});
|
||||
|
||||
it("still tags a hash that shares a text node with an entity reference", () => {
|
||||
// The source slice ("...&...") differs from the decoded value, so the
|
||||
// escape-aware path bows out and the tag is detected the original way.
|
||||
const html = renderMarkdown("Tom & Jerry #cartoon");
|
||||
|
||||
expect(html).toContain('data-tag="cartoon"');
|
||||
expect(html).toContain("Tom & Jerry");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue