memos/internal/markdown/extensions/gfm.go

package extensions

import (
	"bytes"
	"regexp"
	"unicode"
	"unicode/utf8"

	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	goldmarkextension "github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"

	mast "github.com/usememos/memos/internal/markdown/ast"
	mparser "github.com/usememos/memos/internal/markdown/parser"
)

const gfmDomainPattern = `(?:[\p{L}\p{N}_-]+\.)*[\p{L}\p{N}-]+\.[\p{L}\p{N}-]+`

var (
	gfmURLPattern = regexp.MustCompile(`^https?://` + gfmDomainPattern + `[^<\x09-\x0d\p{Z}]*`)
	gfmWWWPattern = regexp.MustCompile(`^www\.` + gfmDomainPattern + `[^<\x09-\x0d\p{Z}]*`)
	// Extended emails are resolved after emphasis instead of by the linkify parser.
	gfmNoEmail = regexp.MustCompile(`a^`)
)

type gfmLinkify struct{}

// NewGFMLinkify returns a linkifier configured for written GFM 0.29.
func NewGFMLinkify() goldmark.Extender {
	return &gfmLinkify{}
}

func (*gfmLinkify) Extend(markdown goldmark.Markdown) {
	markdown.Parser().AddOptions(
		parser.WithInlineParsers(util.Prioritized(newGFMLinkifyParser(), 999)),
		parser.WithASTTransformers(
			util.Prioritized(&gfmLinkifyASTTransformer{}, 900),
			util.Prioritized(&gfmEmailASTTransformer{}, 950),
		),
	)
}

func newGFMLinkifyParser() parser.InlineParser {
	return goldmarkextension.NewLinkifyParser(
		goldmarkextension.WithLinkifyAllowedProtocols([]string{"http:", "https:"}),
		goldmarkextension.WithLinkifyURLRegexp(gfmURLPattern),
		goldmarkextension.WithLinkifyWWWRegexp(gfmWWWPattern),
		// Extended emails are resolved after emphasis delimiters so unmatched
		// underscores are consolidated into their surrounding text node first.
		goldmarkextension.WithLinkifyEmailRegexp(gfmNoEmail),
	)
}

type gfmLinkifyASTTransformer struct{}

func (*gfmLinkifyASTTransformer) Transform(document *ast.Document, reader text.Reader, _ parser.Context) {
	var invalid []*ast.AutoLink
	_ = ast.Walk(document, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
		link, ok := node.(*ast.AutoLink)
		if !entering || !ok || link.AutoLinkType != ast.AutoLinkURL {
			return ast.WalkContinue, nil
		}
		label := link.Label(reader.Source())
		start, found := autoLinkLabelStart(link, reader.Source(), label)
		if !found || isAngleAutoLink(reader.Source(), start, len(label)) {
			return ast.WalkContinue, nil
		}
		if !hasValidGFMDomain(label) {
			invalid = append(invalid, link)
		}
		return ast.WalkContinue, nil
	})

	for _, link := range invalid {
		parent := link.Parent()
		if parent == nil {
			continue
		}
		label := link.Label(reader.Source())
		start, found := autoLinkLabelStart(link, reader.Source(), label)
		if !found {
			continue
		}
		parent.ReplaceChild(parent, link, ast.NewTextSegment(text.NewSegment(start, start+len(label))))
	}

	replaceInitialBOMURL(document, reader.Source())
	replaceUnparsedGFMURLs(document, reader.Source())
}

type inlineNodeMatch struct {
	start int
	end   int
	node  ast.Node
}

func eligibleLiteralTextNodes(document *ast.Document) []*ast.Text {
	var nodes []*ast.Text
	_ = ast.Walk(document, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
		textNode, ok := node.(*ast.Text)
		if entering && ok && isEligibleLiteralText(textNode) {
			nodes = append(nodes, textNode)
		}
		return ast.WalkContinue, nil
	})
	return nodes
}

func replaceInitialBOMURL(document *ast.Document, source []byte) {
	const bomLength = len("\uFEFF")
	if !bytes.HasPrefix(source, []byte("\uFEFF")) {
		return
	}

	linkParser := newGFMLinkifyParser()
	for _, textNode := range eligibleLiteralTextNodes(document) {
		segment := textNode.Segment
		if segment.Start > bomLength || segment.Stop <= bomLength || textNode.Parent() == nil {
			continue
		}

		urlReader := text.NewReader(source[:segment.Stop])
		urlReader.Advance(bomLength)
		link, ok := linkParser.Parse(textNode.Parent(), urlReader, parser.NewContext()).(*ast.AutoLink)
		if !ok || link.AutoLinkType != ast.AutoLinkURL {
			return
		}
		label := link.Label(source)
		if !hasValidGFMDomain(label) {
			return
		}
		end := bomLength + len(label)
		link.SetPos(bomLength)
		replaceInlineMatches(textNode, []inlineNodeMatch{{start: bomLength, end: end, node: link}})
		return
	}
}

func replaceUnparsedGFMURLs(document *ast.Document, source []byte) {
	linkParser := newGFMLinkifyParser()
	textNodes := eligibleLiteralTextNodes(document)
	mergeAdjacentLiteralText(textNodes, source)
	for _, textNode := range textNodes {
		if textNode.Parent() == nil {
			continue
		}

		segment := textNode.Segment
		var matches []inlineNodeMatch
		for offset := segment.Start; offset < segment.Stop; offset++ {
			if !hasWrittenGFMURLBoundary(source, offset) {
				continue
			}

			urlReader := text.NewReader(source[:segment.Stop])
			urlReader.Advance(offset)
			link, ok := linkParser.Parse(textNode.Parent(), urlReader, parser.NewContext()).(*ast.AutoLink)
			if !ok || link.AutoLinkType != ast.AutoLinkURL {
				continue
			}
			label := link.Label(source)
			if len(label) == 0 || offset+len(label) > segment.Stop ||
				!bytes.HasPrefix(source[offset:segment.Stop], label) || !hasValidGFMDomain(label) {
				continue
			}

			start := offset
			end := start + len(label)
			link.SetPos(start)
			matches = append(matches, inlineNodeMatch{start: start, end: end, node: link})
			offset = end - 1
		}
		replaceInlineMatches(textNode, matches)
	}
}

func hasWrittenGFMURLBoundary(source []byte, offset int) bool {
	if offset == 0 {
		return true
	}
	previous := source[offset-1]
	return previous == ' ' || previous >= '\t' && previous <= '\r' ||
		previous == '*' || previous == '_' || previous == '~' || previous == '('
}

func autoLinkLabelStart(link *ast.AutoLink, source, label []byte) (int, bool) {
	start := link.Pos()
	if start >= 0 && start <= len(source)-len(label) && bytes.HasPrefix(source[start:], label) {
		return start, true
	}
	// Goldmark gives a written link triggered by preceding punctuation that
	// punctuation's position, although it is not part of the label.
	start++
	if start >= 0 && start <= len(source)-len(label) && bytes.HasPrefix(source[start:], label) {
		return start, true
	}
	return 0, false
}

func isAngleAutoLink(source []byte, labelStart, labelLength int) bool {
	return labelStart > 0 && labelStart+labelLength < len(source) &&
		source[labelStart-1] == '<' && source[labelStart+labelLength] == '>'
}

func hasValidGFMDomain(label []byte) bool {
	domainStart := 0
	switch {
	case bytes.HasPrefix(label, []byte("http://")):
		domainStart = len("http://")
	case bytes.HasPrefix(label, []byte("https://")):
		domainStart = len("https://")
	case bytes.HasPrefix(label, []byte("www.")):
		domainStart = len("www.")
	default:
		return false
	}

	domainEnd := domainStart
	for domainEnd < len(label) {
		r, size := utf8.DecodeRune(label[domainEnd:])
		if (r == utf8.RuneError && size == 1) || !isGFMDomainRune(r) {
			break
		}
		domainEnd += size
	}
	segments := bytes.Split(label[domainStart:domainEnd], []byte{'.'})
	if len(segments) < 2 {
		return false
	}
	return !bytes.ContainsRune(segments[len(segments)-2], '_') && !bytes.ContainsRune(segments[len(segments)-1], '_')
}

func isGFMDomainRune(value rune) bool {
	return unicode.IsLetter(value) || unicode.IsNumber(value) || value == '_' || value == '-' || value == '.'
}

type gfmEmailASTTransformer struct{}

func (*gfmEmailASTTransformer) Transform(document *ast.Document, reader text.Reader, _ parser.Context) {
	source := reader.Source()
	textNodes := eligibleLiteralTextNodes(document)
	mergeAdjacentLiteralText(textNodes, source)
	for _, textNode := range textNodes {
		if textNode.Parent() != nil {
			replaceGFMEmailsInText(textNode, source)
		}
	}
}

func replaceGFMEmailsInText(textNode *ast.Text, source []byte) {
	segment := textNode.Segment
	emailMatches := mparser.FindGFMEmailMatches(source[segment.Start:segment.Stop])
	matches := make([]inlineNodeMatch, 0, len(emailMatches))
	for _, match := range emailMatches {
		start := segment.Start + match.Start
		end := segment.Start + match.End
		emailNode := &mast.GFMEmailNode{
			Source:  append([]byte(nil), source[start:end]...),
			Address: append([]byte(nil), match.Address...),
		}
		emailNode.SetPos(start)
		matches = append(matches, inlineNodeMatch{
			start: start,
			end:   end,
			node:  emailNode,
		})
	}
	replaceInlineMatches(textNode, matches)
}

func replaceInlineMatches(textNode *ast.Text, matches []inlineNodeMatch) {
	if len(matches) == 0 {
		return
	}
	segment := textNode.Segment
	parent := textNode.Parent()
	cursor := segment.Start
	padding := segment.Padding
	for _, match := range matches {
		if match.start > cursor || padding > 0 {
			insertSplitTextBefore(parent, textNode, textNode, text.NewSegmentPadding(cursor, match.start, padding), false)
		}
		parent.InsertBefore(parent, textNode, match.node)
		cursor = match.end
		padding = 0
	}

	if cursor < segment.Stop || textNode.SoftLineBreak() || textNode.HardLineBreak() {
		insertSplitTextBefore(parent, textNode, textNode, text.NewSegment(cursor, segment.Stop), true)
	}
	parent.RemoveChild(parent, textNode)
}