feat(link-preview): support standards-based metadata (#6206)
This commit is contained in:
parent
f3a6bfa8d9
commit
16cd3107bf
7 changed files with 1269 additions and 215 deletions
|
|
@ -8,16 +8,29 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"golang.org/x/net/html"
|
"golang.org/x/net/html"
|
||||||
"golang.org/x/net/html/atom"
|
"golang.org/x/sync/semaphore"
|
||||||
|
"golang.org/x/sync/singleflight"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrInternalIP indicates that a link preview target resolves to a disallowed internal address.
|
||||||
var ErrInternalIP = errors.New("internal IP addresses are not allowed")
|
var ErrInternalIP = errors.New("internal IP addresses are not allowed")
|
||||||
|
|
||||||
const maxHTMLMetaBytes = 512 * 1024
|
const (
|
||||||
|
defaultLinkPreviewUserAgent = "MemosBot/1.0 (+https://usememos.com)"
|
||||||
|
|
||||||
|
maxHTMLMetaBytes = 512 * 1024
|
||||||
|
maxOEmbedBytes = 128 * 1024
|
||||||
|
maxCacheEntries = 1000
|
||||||
|
maxConcurrentFetches = 8
|
||||||
|
fetchTimeout = 5 * time.Second
|
||||||
|
successCacheTTL = 24 * time.Hour
|
||||||
|
failureCacheTTL = time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
lookupIPAddr = net.DefaultResolver.LookupIPAddr
|
lookupIPAddr = net.DefaultResolver.LookupIPAddr
|
||||||
|
|
@ -25,7 +38,6 @@ var (
|
||||||
Timeout: 30 * time.Second,
|
Timeout: 30 * time.Second,
|
||||||
KeepAlive: 30 * time.Second,
|
KeepAlive: 30 * time.Second,
|
||||||
}).DialContext
|
}).DialContext
|
||||||
httpClient = newHTTPClient()
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func newHTTPClient() *http.Client {
|
func newHTTPClient() *http.Client {
|
||||||
|
|
@ -35,7 +47,7 @@ func newHTTPClient() *http.Client {
|
||||||
|
|
||||||
return &http.Client{
|
return &http.Client{
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
Timeout: 5 * time.Second,
|
Timeout: fetchTimeout,
|
||||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||||
if err := validateURL(req.URL.String()); err != nil {
|
if err := validateURL(req.URL.String()); err != nil {
|
||||||
return errors.Wrap(err, "redirect to internal IP")
|
return errors.Wrap(err, "redirect to internal IP")
|
||||||
|
|
@ -127,101 +139,272 @@ func validateURL(urlStr string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HTMLMeta contains metadata used to build a link preview.
|
||||||
type HTMLMeta struct {
|
type HTMLMeta struct {
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Image string `json:"image"`
|
Image string `json:"image"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {
|
type cacheEntry struct {
|
||||||
if err := validateURL(urlStr); err != nil {
|
meta *HTMLMeta
|
||||||
return nil, err
|
err error
|
||||||
}
|
expiresAt time.Time
|
||||||
|
storedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
response, err := httpClient.Get(urlStr)
|
// HTMLMetaFetcher fetches and caches standards-based link preview metadata.
|
||||||
|
type HTMLMetaFetcher struct {
|
||||||
|
client *http.Client
|
||||||
|
semaphore *semaphore.Weighted
|
||||||
|
group singleflight.Group
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
cacheMu sync.Mutex
|
||||||
|
cache map[string]cacheEntry
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHTMLMetaFetcher creates a link preview fetcher with an SSRF-safe HTTP client.
|
||||||
|
func NewHTMLMetaFetcher() *HTMLMetaFetcher {
|
||||||
|
return &HTMLMetaFetcher{
|
||||||
|
client: newHTTPClient(),
|
||||||
|
semaphore: semaphore.NewWeighted(maxConcurrentFetches),
|
||||||
|
now: time.Now,
|
||||||
|
cache: make(map[string]cacheEntry),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get fetches metadata for a URL.
|
||||||
|
func (f *HTMLMetaFetcher) Get(ctx context.Context, urlStr string) (*HTMLMeta, error) {
|
||||||
|
key, err := normalizeURL(urlStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if meta, ok, err := f.getCached(key); ok {
|
||||||
|
return meta, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resultChannel := f.group.DoChan(key, func() (any, error) {
|
||||||
|
if meta, ok, err := f.getCached(key); ok {
|
||||||
|
return meta, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The flight is shared by every coalesced waiter, so one caller's
|
||||||
|
// cancellation must not abort it for the remaining waiters. The single
|
||||||
|
// deadline covers both queueing and outbound requests.
|
||||||
|
flightContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), fetchTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := f.semaphore.Acquire(flightContext, 1); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.semaphore.Release(1)
|
||||||
|
|
||||||
|
meta, err := f.fetch(flightContext, key)
|
||||||
|
if err == nil {
|
||||||
|
f.setCached(key, meta, nil, successCacheTTL)
|
||||||
|
} else {
|
||||||
|
f.setCached(key, nil, err, failureCacheTTL)
|
||||||
|
}
|
||||||
|
return meta, err
|
||||||
|
})
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case result := <-resultChannel:
|
||||||
|
if result.Err != nil {
|
||||||
|
return nil, result.Err
|
||||||
|
}
|
||||||
|
meta, ok := result.Val.(*HTMLMeta)
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("invalid link metadata result")
|
||||||
|
}
|
||||||
|
return cloneHTMLMeta(meta), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *HTMLMetaFetcher) fetch(ctx context.Context, urlStr string) (*HTMLMeta, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to create link preview request")
|
||||||
|
}
|
||||||
|
setRequestHeaders(request, "text/html, application/xhtml+xml;q=0.9, */*;q=0.1")
|
||||||
|
|
||||||
|
response, err := f.client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to fetch link preview")
|
||||||
|
}
|
||||||
defer response.Body.Close()
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
||||||
|
return nil, errors.Errorf("unexpected HTTP status: %s", response.Status)
|
||||||
|
}
|
||||||
|
|
||||||
mediatype, err := getMediatype(response)
|
mediaType, err := getMediatype(response)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if mediatype != "text/html" {
|
if mediaType != "text/html" && mediaType != "application/xhtml+xml" {
|
||||||
return nil, errors.New("not a HTML page")
|
return nil, errors.New("not a HTML page")
|
||||||
}
|
}
|
||||||
|
|
||||||
htmlMeta := extractHTMLMeta(io.LimitReader(response.Body, maxHTMLMetaBytes))
|
document, err := html.Parse(io.LimitReader(response.Body, maxHTMLMetaBytes))
|
||||||
enrichSiteMeta(response.Request.URL, htmlMeta)
|
if err != nil {
|
||||||
return htmlMeta, nil
|
return nil, errors.Wrap(err, "failed to parse HTML")
|
||||||
|
}
|
||||||
|
pageURL, err := finalResponseURL(response, urlStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
sources, oEmbedEndpoint := extractDocumentMetadata(document)
|
||||||
|
baseURL := resolveDocumentBase(pageURL, sources.baseHref)
|
||||||
|
var oEmbed metadataSource
|
||||||
|
if endpoint := resolveHTTPURL(baseURL, oEmbedEndpoint); endpoint != "" {
|
||||||
|
// oEmbed is optional enrichment, so its failure must not discard the
|
||||||
|
// metadata already extracted from the HTML document.
|
||||||
|
oEmbed, _ = f.fetchOEmbed(ctx, endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := mergeMetadata(baseURL, oEmbed, sources.openGraph, sources.twitter, sources.jsonLD, sources.standard, siteImageSource(pageURL), sources.semantic)
|
||||||
|
return meta, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractHTMLMeta(resp io.Reader) *HTMLMeta {
|
func (f *HTMLMetaFetcher) fetchOEmbed(ctx context.Context, endpoint string) (metadataSource, error) {
|
||||||
tokenizer := html.NewTokenizer(resp)
|
if err := validateURL(endpoint); err != nil {
|
||||||
htmlMeta := new(HTMLMeta)
|
return metadataSource{}, err
|
||||||
|
}
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return metadataSource{}, errors.Wrap(err, "failed to create oEmbed request")
|
||||||
|
}
|
||||||
|
setRequestHeaders(request, "application/json+oembed, application/json;q=0.9")
|
||||||
|
|
||||||
for {
|
response, err := f.client.Do(request)
|
||||||
tokenType := tokenizer.Next()
|
if err != nil {
|
||||||
if tokenType == html.ErrorToken {
|
return metadataSource{}, errors.Wrap(err, "failed to fetch oEmbed endpoint")
|
||||||
break
|
}
|
||||||
} else if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
|
defer response.Body.Close()
|
||||||
token := tokenizer.Token()
|
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
||||||
if token.DataAtom == atom.Body {
|
return metadataSource{}, errors.Errorf("unexpected oEmbed HTTP status: %s", response.Status)
|
||||||
break
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if token.DataAtom == atom.Title {
|
mediaType, err := getMediatype(response)
|
||||||
tokenizer.Next()
|
if err != nil {
|
||||||
token := tokenizer.Token()
|
return metadataSource{}, err
|
||||||
htmlMeta.Title = token.Data
|
}
|
||||||
} else if token.DataAtom == atom.Meta {
|
if mediaType != "application/json" && mediaType != "application/json+oembed" && !strings.HasSuffix(mediaType, "+json") {
|
||||||
ogTitle, ok := extractMetaProperty(token, "og:title")
|
return metadataSource{}, errors.New("oEmbed response is not JSON")
|
||||||
if ok {
|
|
||||||
htmlMeta.Title = ogTitle
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ogDescription, ok := extractMetaProperty(token, "og:description")
|
data, err := io.ReadAll(io.LimitReader(response.Body, maxOEmbedBytes+1))
|
||||||
if ok {
|
if err != nil {
|
||||||
htmlMeta.Description = ogDescription
|
return metadataSource{}, errors.Wrap(err, "failed to read oEmbed response")
|
||||||
|
}
|
||||||
|
if len(data) > maxOEmbedBytes {
|
||||||
|
return metadataSource{}, errors.New("oEmbed response exceeds size limit")
|
||||||
}
|
}
|
||||||
|
|
||||||
ogImage, ok := extractMetaProperty(token, "og:image")
|
baseURL, err := finalResponseURL(response, endpoint)
|
||||||
if ok {
|
if err != nil {
|
||||||
htmlMeta.Image = ogImage
|
return metadataSource{}, err
|
||||||
}
|
}
|
||||||
|
return extractOEmbedMetadata(data, baseURL)
|
||||||
description, ok := extractMetaProperty(token, "description")
|
|
||||||
if ok && htmlMeta.Description == "" {
|
|
||||||
htmlMeta.Description = description
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return htmlMeta
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractMetaProperty(token html.Token, prop string) (content string, ok bool) {
|
func setRequestHeaders(request *http.Request, accept string) {
|
||||||
content, ok = "", false
|
request.Header.Set("User-Agent", defaultLinkPreviewUserAgent)
|
||||||
for _, attr := range token.Attr {
|
request.Header.Set("Accept", accept)
|
||||||
if (attr.Key == "property" || attr.Key == "name") && strings.EqualFold(attr.Val, prop) {
|
|
||||||
ok = true
|
|
||||||
}
|
|
||||||
if attr.Key == "content" {
|
|
||||||
content = attr.Val
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return content, ok
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func enrichSiteMeta(url *url.URL, meta *HTMLMeta) {
|
func (f *HTMLMetaFetcher) getCached(key string) (*HTMLMeta, bool, error) {
|
||||||
if url.Hostname() == "www.youtube.com" {
|
f.cacheMu.Lock()
|
||||||
if url.Path == "/watch" {
|
defer f.cacheMu.Unlock()
|
||||||
vid := url.Query().Get("v")
|
|
||||||
if vid != "" {
|
entry, ok := f.cache[key]
|
||||||
meta.Image = fmt.Sprintf("https://img.youtube.com/vi/%s/mqdefault.jpg", vid)
|
if !ok {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if !f.now().Before(entry.expiresAt) {
|
||||||
|
delete(f.cache, key)
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
return cloneHTMLMeta(entry.meta), true, entry.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *HTMLMetaFetcher) setCached(key string, meta *HTMLMeta, err error, ttl time.Duration) {
|
||||||
|
f.cacheMu.Lock()
|
||||||
|
defer f.cacheMu.Unlock()
|
||||||
|
|
||||||
|
now := f.now()
|
||||||
|
if _, exists := f.cache[key]; !exists && len(f.cache) >= maxCacheEntries {
|
||||||
|
var oldestKey string
|
||||||
|
var oldestTime time.Time
|
||||||
|
for candidateKey, entry := range f.cache {
|
||||||
|
if oldestKey == "" || entry.storedAt.Before(oldestTime) {
|
||||||
|
oldestKey = candidateKey
|
||||||
|
oldestTime = entry.storedAt
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
delete(f.cache, oldestKey)
|
||||||
|
}
|
||||||
|
f.cache[key] = cacheEntry{
|
||||||
|
meta: cloneHTMLMeta(meta),
|
||||||
|
err: err,
|
||||||
|
expiresAt: now.Add(ttl),
|
||||||
|
storedAt: now,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeURL(urlStr string) (string, error) {
|
||||||
|
parsed, err := url.Parse(strings.TrimSpace(urlStr))
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("invalid URL format")
|
||||||
|
}
|
||||||
|
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||||
|
parsed.Host = strings.ToLower(parsed.Host)
|
||||||
|
parsed.Fragment = ""
|
||||||
|
if parsed.Path == "" {
|
||||||
|
parsed.Path = "/"
|
||||||
|
}
|
||||||
|
if (parsed.Scheme == "http" && parsed.Port() == "80") || (parsed.Scheme == "https" && parsed.Port() == "443") {
|
||||||
|
parsed.Host = parsed.Hostname()
|
||||||
|
if strings.Contains(parsed.Host, ":") {
|
||||||
|
parsed.Host = "[" + parsed.Host + "]"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := validateURL(parsed.String()); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func finalResponseURL(response *http.Response, fallback string) (*url.URL, error) {
|
||||||
|
if response.Request != nil && response.Request.URL != nil {
|
||||||
|
return response.Request.URL, nil
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(fallback)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "invalid response URL")
|
||||||
|
}
|
||||||
|
return parsed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneHTMLMeta(meta *HTMLMeta) *HTMLMeta {
|
||||||
|
if meta == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *meta
|
||||||
|
return ©
|
||||||
|
}
|
||||||
|
|
||||||
|
// siteImageSource supplies deterministic site-specific thumbnails that outrank
|
||||||
|
// the low-confidence semantic <img> fallback but lose to metadata the site
|
||||||
|
// itself declares.
|
||||||
|
func siteImageSource(pageURL *url.URL) metadataSource {
|
||||||
|
if pageURL.Hostname() == "www.youtube.com" && pageURL.Path == "/watch" {
|
||||||
|
if videoID := pageURL.Query().Get("v"); videoID != "" {
|
||||||
|
return metadataSource{image: fmt.Sprintf("https://img.youtube.com/vi/%s/mqdefault.jpg", url.PathEscape(videoID))}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return metadataSource{}
|
||||||
|
}
|
||||||
|
|
|
||||||
446
internal/httpgetter/html_meta_extract.go
Normal file
446
internal/httpgetter/html_meta_extract.go
Normal file
|
|
@ -0,0 +1,446 @@
|
||||||
|
package httpgetter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"mime"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxTitleRunes = 200
|
||||||
|
maxDescriptionRunes = 300
|
||||||
|
minSemanticTextRunes = 40
|
||||||
|
maxImageURLBytes = 2048
|
||||||
|
)
|
||||||
|
|
||||||
|
type metadataSource struct {
|
||||||
|
title string
|
||||||
|
description string
|
||||||
|
image string
|
||||||
|
}
|
||||||
|
|
||||||
|
type documentMetadata struct {
|
||||||
|
openGraph metadataSource
|
||||||
|
twitter metadataSource
|
||||||
|
jsonLD metadataSource
|
||||||
|
standard metadataSource
|
||||||
|
semantic metadataSource
|
||||||
|
baseHref string
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractDocumentMetadata(document *html.Node) (documentMetadata, string) {
|
||||||
|
var sources documentMetadata
|
||||||
|
var oEmbedEndpoint string
|
||||||
|
var jsonLDScripts []string
|
||||||
|
|
||||||
|
walkNodes(document, func(node *html.Node) {
|
||||||
|
if node.Type != html.ElementNode {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch strings.ToLower(node.Data) {
|
||||||
|
case "title":
|
||||||
|
// Foreign-content titles (inline SVG/MathML accessibility labels)
|
||||||
|
// are not the document title.
|
||||||
|
if node.Namespace == "" {
|
||||||
|
setIfEmpty(&sources.standard.title, nodeText(node))
|
||||||
|
}
|
||||||
|
case "base":
|
||||||
|
setIfEmpty(&sources.baseHref, firstAttribute(node, "href"))
|
||||||
|
case "meta":
|
||||||
|
name := strings.ToLower(strings.TrimSpace(firstAttribute(node, "property", "name")))
|
||||||
|
content := firstAttribute(node, "content")
|
||||||
|
switch name {
|
||||||
|
case "og:title":
|
||||||
|
setIfEmpty(&sources.openGraph.title, content)
|
||||||
|
case "og:description":
|
||||||
|
setIfEmpty(&sources.openGraph.description, content)
|
||||||
|
case "og:image", "og:image:url", "og:image:secure_url":
|
||||||
|
setImageIfEmpty(&sources.openGraph.image, content)
|
||||||
|
case "twitter:title":
|
||||||
|
setIfEmpty(&sources.twitter.title, content)
|
||||||
|
case "twitter:description":
|
||||||
|
setIfEmpty(&sources.twitter.description, content)
|
||||||
|
case "twitter:image", "twitter:image:src":
|
||||||
|
setImageIfEmpty(&sources.twitter.image, content)
|
||||||
|
case "description":
|
||||||
|
setIfEmpty(&sources.standard.description, content)
|
||||||
|
case "title", "name", "headline":
|
||||||
|
setIfEmpty(&sources.standard.title, content)
|
||||||
|
case "image":
|
||||||
|
setImageIfEmpty(&sources.standard.image, content)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
case "link":
|
||||||
|
rels := strings.Fields(strings.ToLower(firstAttribute(node, "rel")))
|
||||||
|
if containsString(rels, "image_src") {
|
||||||
|
setImageIfEmpty(&sources.standard.image, firstAttribute(node, "href"))
|
||||||
|
}
|
||||||
|
if oEmbedEndpoint == "" && containsString(rels, "alternate") {
|
||||||
|
mediaType := firstAttribute(node, "type")
|
||||||
|
if isJSONOEmbedMediaType(mediaType) {
|
||||||
|
oEmbedEndpoint = firstAttribute(node, "href")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "script":
|
||||||
|
if isJSONLDMediaType(firstAttribute(node, "type")) {
|
||||||
|
jsonLDScripts = append(jsonLDScripts, rawNodeText(node))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
sources.jsonLD = extractJSONLDMetadata(jsonLDScripts)
|
||||||
|
sources.semantic = extractSemanticMetadata(document)
|
||||||
|
return sources, oEmbedEndpoint
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeMetadata(pageURL *url.URL, sources ...metadataSource) *HTMLMeta {
|
||||||
|
meta := &HTMLMeta{}
|
||||||
|
for _, source := range sources {
|
||||||
|
setIfEmpty(&meta.Title, source.title)
|
||||||
|
setIfEmpty(&meta.Description, source.description)
|
||||||
|
if meta.Image == "" {
|
||||||
|
meta.Image = resolveHTTPURL(pageURL, source.image)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
meta.Title = truncateText(normalizeWhitespace(meta.Title), maxTitleRunes)
|
||||||
|
meta.Description = truncateText(normalizeWhitespace(meta.Description), maxDescriptionRunes)
|
||||||
|
// A truncated URL would be broken, so oversized image URLs are dropped
|
||||||
|
// rather than shortened.
|
||||||
|
if len(meta.Image) > maxImageURLBytes {
|
||||||
|
meta.Image = ""
|
||||||
|
}
|
||||||
|
return meta
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractOEmbedMetadata(data []byte, baseURL *url.URL) (metadataSource, error) {
|
||||||
|
var payload struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
ThumbnailURL string `json:"thumbnail_url"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
if err := decoder.Decode(&payload); err != nil {
|
||||||
|
return metadataSource{}, errors.Wrap(err, "failed to decode oEmbed response")
|
||||||
|
}
|
||||||
|
image := resolveHTTPURL(baseURL, payload.ThumbnailURL)
|
||||||
|
if image == "" && strings.EqualFold(payload.Type, "photo") {
|
||||||
|
image = resolveHTTPURL(baseURL, payload.URL)
|
||||||
|
}
|
||||||
|
return metadataSource{
|
||||||
|
title: payload.Title,
|
||||||
|
description: payload.Description,
|
||||||
|
image: image,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractJSONLDMetadata(scripts []string) metadataSource {
|
||||||
|
var best metadataSource
|
||||||
|
bestScore := -1
|
||||||
|
for _, script := range scripts {
|
||||||
|
var value any
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(script))
|
||||||
|
if err := decoder.Decode(&value); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
walkJSONLD(value, func(object map[string]any) {
|
||||||
|
candidate := metadataSource{
|
||||||
|
title: firstJSONString(object["headline"], object["name"]),
|
||||||
|
description: firstJSONString(object["description"]),
|
||||||
|
image: jsonLDImage(object["image"]),
|
||||||
|
}
|
||||||
|
if candidate.image == "" {
|
||||||
|
candidate.image = jsonLDImage(object["thumbnailUrl"])
|
||||||
|
}
|
||||||
|
if candidate == (metadataSource{}) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
score := jsonLDTypeScore(object["@type"])
|
||||||
|
if candidate.title != "" {
|
||||||
|
score++
|
||||||
|
}
|
||||||
|
if candidate.description != "" {
|
||||||
|
score++
|
||||||
|
}
|
||||||
|
if candidate.image != "" {
|
||||||
|
score++
|
||||||
|
}
|
||||||
|
if score > bestScore {
|
||||||
|
best = candidate
|
||||||
|
bestScore = score
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|
||||||
|
func walkJSONLD(value any, visit func(map[string]any)) {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case []any:
|
||||||
|
for _, item := range typed {
|
||||||
|
walkJSONLD(item, visit)
|
||||||
|
}
|
||||||
|
case map[string]any:
|
||||||
|
visit(typed)
|
||||||
|
for _, nested := range typed {
|
||||||
|
walkJSONLD(nested, visit)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonLDTypeScore(value any) int {
|
||||||
|
types := jsonLDStrings(value)
|
||||||
|
for _, value := range types {
|
||||||
|
switch strings.ToLower(value) {
|
||||||
|
case "article", "newsarticle", "blogposting", "socialmediaposting", "product", "videoobject", "movie", "book", "event", "recipe":
|
||||||
|
return 20
|
||||||
|
case "creativework", "webpage", "profilepage", "aboutpage", "collectionpage":
|
||||||
|
return 10
|
||||||
|
case "breadcrumblist", "itemlist":
|
||||||
|
return -10
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonLDImage(value any) string {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case string:
|
||||||
|
if isPotentialHTTPURL(typed) {
|
||||||
|
return typed
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, item := range typed {
|
||||||
|
if image := jsonLDImage(item); image != "" {
|
||||||
|
return image
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case map[string]any:
|
||||||
|
// "@id" is deliberately excluded: it is a node reference (often the
|
||||||
|
// page's own URL plus a fragment), not an image location.
|
||||||
|
for _, candidate := range []any{typed["url"], typed["contentUrl"]} {
|
||||||
|
if image := jsonLDImage(candidate); image != "" {
|
||||||
|
return image
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstJSONString(values ...any) string {
|
||||||
|
for _, value := range values {
|
||||||
|
for _, candidate := range jsonLDStrings(value) {
|
||||||
|
if strings.TrimSpace(candidate) != "" {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonLDStrings(value any) []string {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case string:
|
||||||
|
return []string{typed}
|
||||||
|
case []any:
|
||||||
|
var result []string
|
||||||
|
for _, item := range typed {
|
||||||
|
result = append(result, jsonLDStrings(item)...)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractSemanticMetadata(document *html.Node) metadataSource {
|
||||||
|
root := firstElement(document, "main", "article")
|
||||||
|
if root == nil {
|
||||||
|
root = firstElement(document, "body")
|
||||||
|
}
|
||||||
|
if root == nil {
|
||||||
|
return metadataSource{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var source metadataSource
|
||||||
|
walkNodes(root, func(node *html.Node) {
|
||||||
|
if node.Type != html.ElementNode {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch strings.ToLower(node.Data) {
|
||||||
|
case "h1":
|
||||||
|
setIfEmpty(&source.title, nodeText(node))
|
||||||
|
case "p":
|
||||||
|
text := normalizeWhitespace(nodeText(node))
|
||||||
|
if source.description == "" && utf8.RuneCountInString(text) >= minSemanticTextRunes {
|
||||||
|
source.description = text
|
||||||
|
}
|
||||||
|
case "img":
|
||||||
|
setImageIfEmpty(&source.image, firstAttribute(node, "src"))
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return source
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveHTTPURL(baseURL *url.URL, candidate string) string {
|
||||||
|
candidate = strings.TrimSpace(candidate)
|
||||||
|
if candidate == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(candidate)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if baseURL != nil {
|
||||||
|
parsed = baseURL.ResolveReference(parsed)
|
||||||
|
}
|
||||||
|
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if parsed.Hostname() == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return parsed.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func walkNodes(node *html.Node, visit func(*html.Node)) {
|
||||||
|
if node == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
visit(node)
|
||||||
|
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||||
|
walkNodes(child, visit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstElement prefers earlier names over document order: all matches for
|
||||||
|
// names[0] outrank any match for names[1].
|
||||||
|
func firstElement(node *html.Node, names ...string) *html.Node {
|
||||||
|
for _, name := range names {
|
||||||
|
var result *html.Node
|
||||||
|
walkNodes(node, func(candidate *html.Node) {
|
||||||
|
if result != nil || candidate.Type != html.ElementNode {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.EqualFold(candidate.Data, name) {
|
||||||
|
result = candidate
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if result != nil {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstAttribute(node *html.Node, names ...string) string {
|
||||||
|
for _, name := range names {
|
||||||
|
for _, attribute := range node.Attr {
|
||||||
|
if strings.EqualFold(attribute.Key, name) {
|
||||||
|
return attribute.Val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func rawNodeText(node *html.Node) string {
|
||||||
|
var builder strings.Builder
|
||||||
|
walkNodes(node, func(candidate *html.Node) {
|
||||||
|
if candidate.Type == html.TextNode {
|
||||||
|
builder.WriteString(candidate.Data)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return builder.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func nodeText(node *html.Node) string {
|
||||||
|
return normalizeWhitespace(rawNodeText(node))
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeWhitespace(value string) string {
|
||||||
|
return strings.Join(strings.FieldsFunc(value, unicode.IsSpace), " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateText(value string, maximum int) string {
|
||||||
|
runes := []rune(value)
|
||||||
|
if len(runes) <= maximum {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(runes[:maximum-1])) + "…"
|
||||||
|
}
|
||||||
|
|
||||||
|
func setIfEmpty(target *string, value string) {
|
||||||
|
if *target == "" && strings.TrimSpace(value) != "" {
|
||||||
|
*target = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setImageIfEmpty(target *string, value string) {
|
||||||
|
if *target == "" && isPotentialHTTPURL(value) {
|
||||||
|
*target = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isPotentialHTTPURL(value string) bool {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(value)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if parsed.Scheme == "" {
|
||||||
|
return !strings.HasPrefix(value, "//") || parsed.Hostname() != ""
|
||||||
|
}
|
||||||
|
return (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Hostname() != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsString(values []string, expected string) bool {
|
||||||
|
for _, value := range values {
|
||||||
|
if value == expected {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJSONOEmbedMediaType(value string) bool {
|
||||||
|
mediaType, _, err := mime.ParseMediaType(value)
|
||||||
|
return err == nil && strings.EqualFold(mediaType, "application/json+oembed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJSONLDMediaType(value string) bool {
|
||||||
|
mediaType, _, err := mime.ParseMediaType(value)
|
||||||
|
return err == nil && strings.EqualFold(mediaType, "application/ld+json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveDocumentBase applies the document's <base href>, falling back to the
|
||||||
|
// final response URL when the base is missing or not an absolute http(s) URL.
|
||||||
|
func resolveDocumentBase(pageURL *url.URL, baseHref string) *url.URL {
|
||||||
|
baseHref = strings.TrimSpace(baseHref)
|
||||||
|
if baseHref == "" {
|
||||||
|
return pageURL
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(baseHref)
|
||||||
|
if err != nil {
|
||||||
|
return pageURL
|
||||||
|
}
|
||||||
|
resolved := pageURL.ResolveReference(parsed)
|
||||||
|
if (resolved.Scheme != "http" && resolved.Scheme != "https") || resolved.Hostname() == "" {
|
||||||
|
return pageURL
|
||||||
|
}
|
||||||
|
return resolved
|
||||||
|
}
|
||||||
|
|
@ -3,13 +3,20 @@ package httpgetter
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/net/html"
|
||||||
)
|
)
|
||||||
|
|
||||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
@ -18,125 +25,452 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
return f(req)
|
return f(req)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetHTMLMeta(t *testing.T) {
|
func TestExtractDocumentMetadataPrecedenceAndFallbacks(t *testing.T) {
|
||||||
originalHTTPClient := httpClient
|
tests := []struct {
|
||||||
t.Cleanup(func() {
|
name string
|
||||||
httpClient = originalHTTPClient
|
markup string
|
||||||
})
|
expected HTMLMeta
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Open Graph wins field by field",
|
||||||
|
markup: `<html><head>
|
||||||
|
<title>Standard title</title>
|
||||||
|
<meta name="description" content="Standard description">
|
||||||
|
<meta property="og:title" content="Open Graph title">
|
||||||
|
<meta name="twitter:title" content="Twitter title">
|
||||||
|
<meta name="twitter:description" content="Twitter description">
|
||||||
|
<meta name="twitter:image" content="/twitter.png">
|
||||||
|
</head></html>`,
|
||||||
|
expected: HTMLMeta{
|
||||||
|
Title: "Open Graph title",
|
||||||
|
Description: "Twitter description",
|
||||||
|
Image: "https://example.com/twitter.png",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "JSON-LD graph and image object",
|
||||||
|
markup: `<html><head><script type="application/ld+json">{
|
||||||
|
"@graph": [
|
||||||
|
{"@type":"Organization","name":"Publisher"},
|
||||||
|
{"@type":"NewsArticle","headline":"JSON-LD title","description":"JSON-LD description","image":{"contentUrl":"images/cover.jpg"}}
|
||||||
|
]
|
||||||
|
}</script></head></html>`,
|
||||||
|
expected: HTMLMeta{
|
||||||
|
Title: "JSON-LD title",
|
||||||
|
Description: "JSON-LD description",
|
||||||
|
Image: "https://example.com/posts/images/cover.jpg",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested JSON-LD main entity",
|
||||||
|
markup: `<html><head><script type="application/ld+json">{
|
||||||
|
"@type":"WebPage",
|
||||||
|
"mainEntity":{
|
||||||
|
"@type":"Article",
|
||||||
|
"headline":"Nested article title",
|
||||||
|
"description":"Nested article description",
|
||||||
|
"image":"/nested.png"
|
||||||
|
}
|
||||||
|
}</script></head></html>`,
|
||||||
|
expected: HTMLMeta{
|
||||||
|
Title: "Nested article title",
|
||||||
|
Description: "Nested article description",
|
||||||
|
Image: "https://example.com/nested.png",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standard metadata",
|
||||||
|
markup: `<html><head><title>Standard title</title>
|
||||||
|
<meta NAME="Description" content="Standard description">
|
||||||
|
<link rel="IMAGE_SRC" href="/standard.png">
|
||||||
|
</head></html>`,
|
||||||
|
expected: HTMLMeta{
|
||||||
|
Title: "Standard title",
|
||||||
|
Description: "Standard description",
|
||||||
|
Image: "https://example.com/standard.png",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "semantic main content",
|
||||||
|
markup: `<html><body><nav><h1>Navigation title</h1></nav><main>
|
||||||
|
<h1>Semantic title</h1>
|
||||||
|
<p>This paragraph is deliberately long enough to become the semantic preview description.</p>
|
||||||
|
<img src="./semantic.png">
|
||||||
|
</main></body></html>`,
|
||||||
|
expected: HTMLMeta{
|
||||||
|
Title: "Semantic title",
|
||||||
|
Description: "This paragraph is deliberately long enough to become the semantic preview description.",
|
||||||
|
Image: "https://example.com/posts/semantic.png",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
httpClient = &http.Client{
|
for _, test := range tests {
|
||||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
require.Equal(t, "http://93.184.216.34/article", req.URL.String())
|
meta := extractTestMetadata(t, test.markup)
|
||||||
return &http.Response{
|
require.Equal(t, test.expected, *meta)
|
||||||
StatusCode: http.StatusOK,
|
})
|
||||||
Header: http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
|
}
|
||||||
Body: io.NopCloser(strings.NewReader(`<!doctype html>
|
}
|
||||||
<html>
|
|
||||||
<head>
|
func TestExtractJSONLDImageForms(t *testing.T) {
|
||||||
<title>Fallback title</title>
|
tests := []struct {
|
||||||
<meta name="description" content="Fallback description">
|
name string
|
||||||
|
image string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "string", image: `"/one.png"`, want: "https://example.com/one.png"},
|
||||||
|
{name: "array", image: `["data:image/png;base64,abc", "/two.png"]`, want: "https://example.com/two.png"},
|
||||||
|
{name: "url object", image: `{"url":"/three.png"}`, want: "https://example.com/three.png"},
|
||||||
|
{name: "content URL object", image: `{"url":"javascript:alert(1)","contentUrl":"/four.png"}`, want: "https://example.com/four.png"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
markup := `<script type="application/ld+json">{"@type":"Article","headline":"Title","image":` + test.image + `}</script>`
|
||||||
|
meta := extractTestMetadata(t, markup)
|
||||||
|
require.Equal(t, test.want, meta.Image)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractDocumentMetadataNormalizesAndTruncatesText(t *testing.T) {
|
||||||
|
markup := `<html><head><meta property="og:title" content=" ` + strings.Repeat("A", maxTitleRunes+25) + ` title ">
|
||||||
|
<meta property="og:description" content="` + strings.Repeat("界", maxDescriptionRunes+25) + `"></head></html>`
|
||||||
|
meta := extractTestMetadata(t, markup)
|
||||||
|
|
||||||
|
require.Len(t, []rune(meta.Title), maxTitleRunes)
|
||||||
|
require.NotContains(t, meta.Title, " ")
|
||||||
|
require.True(t, strings.HasSuffix(meta.Title, "…"))
|
||||||
|
require.Len(t, []rune(meta.Description), maxDescriptionRunes)
|
||||||
|
require.True(t, strings.HasSuffix(meta.Description, "…"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractDocumentMetadataSkipsUnsafeImages(t *testing.T) {
|
||||||
|
markup := `<html><head>
|
||||||
|
<meta property="og:image" content="javascript:alert(1)">
|
||||||
|
<meta property="og:image" content="/safe.png">
|
||||||
|
</head></html>`
|
||||||
|
meta := extractTestMetadata(t, markup)
|
||||||
|
require.Equal(t, "https://example.com/safe.png", meta.Image)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherOEmbed(t *testing.T) {
|
||||||
|
var requests atomic.Int32
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
requests.Add(1)
|
||||||
|
switch req.URL.Path {
|
||||||
|
case "/post":
|
||||||
|
return htmlResponse(req, `<html><head>
|
||||||
|
<link rel="alternate" type="Application/JSON+OEMBED; charset=utf-8" href="/oembed">
|
||||||
<meta property="og:title" content="Open Graph title">
|
<meta property="og:title" content="Open Graph title">
|
||||||
<meta property="og:description" content="Open Graph description">
|
<meta property="og:description" content="Open Graph description">
|
||||||
<meta property="og:image" content="https://example.com/cover.png">
|
<meta property="og:image" content="/og.png">
|
||||||
</head>
|
</head></html>`), nil
|
||||||
<body>ignored</body>
|
case "/oembed":
|
||||||
</html>`)),
|
assert.Contains(t, req.Header.Get("Accept"), "application/json+oembed")
|
||||||
Request: req,
|
return response(req, http.StatusOK, "application/json+oembed", `{
|
||||||
}, nil
|
"type":"rich","title":"oEmbed title","description":"oEmbed description",
|
||||||
}),
|
"thumbnail_url":"/oembed.png","html":"<script>must not be returned</script>"
|
||||||
|
}`), nil
|
||||||
|
default:
|
||||||
|
return nil, errors.New("unexpected request path: " + req.URL.Path)
|
||||||
}
|
}
|
||||||
|
}))
|
||||||
|
|
||||||
metadata, err := GetHTMLMeta("http://93.184.216.34/article")
|
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/post")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, HTMLMeta{
|
require.Equal(t, &HTMLMeta{
|
||||||
Title: "Open Graph title",
|
Title: "oEmbed title",
|
||||||
Description: "Open Graph description",
|
Description: "oEmbed description",
|
||||||
Image: "https://example.com/cover.png",
|
Image: "http://93.184.216.34/oembed.png",
|
||||||
}, *metadata)
|
}, meta)
|
||||||
|
require.EqualValues(t, 2, requests.Load())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetHTMLMetaWithNameOnly(t *testing.T) {
|
func TestHTMLMetaFetcherIgnoresInvalidOEmbed(t *testing.T) {
|
||||||
originalHTTPClient := httpClient
|
tests := []struct {
|
||||||
t.Cleanup(func() {
|
name string
|
||||||
httpClient = originalHTTPClient
|
endpoint string
|
||||||
|
contentType string
|
||||||
|
body string
|
||||||
|
wantCalls int32
|
||||||
|
}{
|
||||||
|
{name: "unsupported content type", endpoint: "/oembed", contentType: "text/html", body: `<p>not JSON</p>`, wantCalls: 2},
|
||||||
|
{name: "oversized response", endpoint: "/oembed", contentType: "application/json", body: strings.Repeat("x", maxOEmbedBytes+1), wantCalls: 2},
|
||||||
|
{name: "unsafe endpoint", endpoint: "http://127.0.0.1/oembed", contentType: "application/json", body: `{}`, wantCalls: 1},
|
||||||
|
{name: "XML discovery", endpoint: "/oembed", contentType: "application/json", body: `{}`, wantCalls: 1},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
call := calls.Add(1)
|
||||||
|
if call == 1 {
|
||||||
|
discoveryType := "application/json+oembed"
|
||||||
|
if test.name == "XML discovery" {
|
||||||
|
discoveryType = "text/xml+oembed"
|
||||||
|
}
|
||||||
|
return htmlResponse(req, `<link rel="alternate" type="`+discoveryType+`" href="`+test.endpoint+`">
|
||||||
|
<meta property="og:title" content="Fallback title">`), nil
|
||||||
|
}
|
||||||
|
return response(req, http.StatusOK, test.contentType, test.body), nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/post")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "Fallback title", meta.Title)
|
||||||
|
require.Equal(t, test.wantCalls, calls.Load())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherBotAwareSPA(t *testing.T) {
|
||||||
|
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
assert.Contains(t, req.Header.Get("Accept"), "text/html")
|
||||||
|
if req.Header.Get("User-Agent") == defaultLinkPreviewUserAgent {
|
||||||
|
return htmlResponse(req, `<meta property="og:title" content="Rich SPA title">
|
||||||
|
<meta property="og:description" content="Prerendered for MemosBot">`), nil
|
||||||
|
}
|
||||||
|
return htmlResponse(req, `<title>Application shell</title><div id="root"></div>`), nil
|
||||||
})
|
})
|
||||||
|
|
||||||
httpClient = &http.Client{
|
fetcher := newTestFetcher(transport)
|
||||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/spa")
|
||||||
require.Equal(t, "http://93.184.216.34/blog", req.URL.String())
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: http.StatusOK,
|
|
||||||
Header: http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
|
|
||||||
Body: io.NopCloser(strings.NewReader(`<!doctype html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Sample Page</title>
|
|
||||||
<meta name="description" content="This description should appear in the link preview.">
|
|
||||||
</head>
|
|
||||||
<body>Hello</body>
|
|
||||||
</html>`)),
|
|
||||||
Request: req,
|
|
||||||
}, nil
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata, err := GetHTMLMeta("http://93.184.216.34/blog")
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, HTMLMeta{
|
require.Equal(t, "Rich SPA title", meta.Title)
|
||||||
Title: "Sample Page",
|
require.Equal(t, "Prerendered for MemosBot", meta.Description)
|
||||||
Description: "This description should appear in the link preview.",
|
|
||||||
Image: "",
|
|
||||||
}, *metadata)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetHTMLMetaWithNameCaseInsensitive(t *testing.T) {
|
func TestHTMLMetaFetcherDefaultHeadersAndDeadline(t *testing.T) {
|
||||||
originalHTTPClient := httpClient
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
t.Cleanup(func() {
|
assert.Equal(t, defaultLinkPreviewUserAgent, req.Header.Get("User-Agent"))
|
||||||
httpClient = originalHTTPClient
|
deadline, ok := req.Context().Deadline()
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.LessOrEqual(t, time.Until(deadline), fetchTimeout)
|
||||||
|
return htmlResponse(req, `<title>Title</title>`), nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
_, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, fetchTimeout, newHTTPClient().Timeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiteImageSourceEscapesYouTubeVideoID(t *testing.T) {
|
||||||
|
pageURL, err := url.Parse("https://www.youtube.com/watch?v=video%2Fsegment%3Fvariant")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
source := siteImageSource(pageURL)
|
||||||
|
require.Equal(t, "https://img.youtube.com/vi/video%2Fsegment%3Fvariant/mqdefault.jpg", source.image)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherResolvesAgainstFinalResponseURL(t *testing.T) {
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
finalRequest := req.Clone(req.Context())
|
||||||
|
finalRequest.URL, _ = url.Parse("https://cdn.example.com/final/path/")
|
||||||
|
return htmlResponse(finalRequest, `<meta property="og:image" content="cover.png">`), nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/original")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "https://cdn.example.com/final/path/cover.png", meta.Image)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherValidatesResponse(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
contentType string
|
||||||
|
}{
|
||||||
|
{name: "non-2xx", status: http.StatusNotFound, contentType: "text/html"},
|
||||||
|
{name: "unsupported content type", status: http.StatusOK, contentType: "application/pdf"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
return response(req, test.status, test.contentType, "body"), nil
|
||||||
|
}))
|
||||||
|
_, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")
|
||||||
|
require.Error(t, err)
|
||||||
})
|
})
|
||||||
|
|
||||||
httpClient = &http.Client{
|
|
||||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
require.Equal(t, "http://93.184.216.34/blog", req.URL.String())
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: http.StatusOK,
|
|
||||||
Header: http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
|
|
||||||
Body: io.NopCloser(strings.NewReader(`<!doctype html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Sample Page</title>
|
|
||||||
<meta name="Description" content="Case insensitive description match.">
|
|
||||||
</head>
|
|
||||||
<body>Hello</body>
|
|
||||||
</html>`)),
|
|
||||||
Request: req,
|
|
||||||
}, nil
|
|
||||||
}),
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
metadata, err := GetHTMLMeta("http://93.184.216.34/blog")
|
func TestHTMLMetaFetcherBoundsHTMLResponse(t *testing.T) {
|
||||||
|
body := `<title>Bounded title</title>` + strings.Repeat("x", maxHTMLMetaBytes) + `<meta property="og:title" content="Outside limit">`
|
||||||
|
reader := &countingReadCloser{Reader: strings.NewReader(body)}
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
result := response(req, http.StatusOK, "text/html", "")
|
||||||
|
result.Body = reader
|
||||||
|
return result, nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, HTMLMeta{
|
require.Equal(t, "Bounded title", meta.Title)
|
||||||
Title: "Sample Page",
|
require.LessOrEqual(t, reader.bytesRead, int64(maxHTMLMetaBytes))
|
||||||
Description: "Case insensitive description match.",
|
|
||||||
Image: "",
|
|
||||||
}, *metadata)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetHTMLMetaForInternal(t *testing.T) {
|
func TestHTMLMetaFetcherContextCancellation(t *testing.T) {
|
||||||
// test for internal IP
|
requestStarted := make(chan struct{})
|
||||||
if _, err := GetHTMLMeta("http://192.168.0.1"); !errors.Is(err, ErrInternalIP) {
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
t.Errorf("Expected error for internal IP, got %v", err)
|
close(requestStarted)
|
||||||
}
|
<-req.Context().Done()
|
||||||
|
return nil, req.Context().Err()
|
||||||
|
}))
|
||||||
|
|
||||||
// test for resolved internal IP
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
if _, err := GetHTMLMeta("http://localhost"); !errors.Is(err, ErrInternalIP) {
|
result := make(chan error, 1)
|
||||||
t.Errorf("Expected error for resolved internal IP, got %v", err)
|
go func() {
|
||||||
}
|
_, err := fetcher.Get(ctx, "http://93.184.216.34/page")
|
||||||
|
result <- err
|
||||||
|
}()
|
||||||
|
<-requestStarted
|
||||||
|
cancel()
|
||||||
|
require.ErrorIs(t, <-result, context.Canceled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHTTPClientHasTimeout(t *testing.T) {
|
func TestHTMLMetaFetcherCache(t *testing.T) {
|
||||||
require.NotZero(t, httpClient.Timeout)
|
var now = time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
|
||||||
|
var successCalls atomic.Int32
|
||||||
|
successFetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
successCalls.Add(1)
|
||||||
|
return htmlResponse(req, `<title>Cached title</title>`), nil
|
||||||
|
}))
|
||||||
|
successFetcher.now = func() time.Time { return now }
|
||||||
|
|
||||||
|
first, err := successFetcher.Get(context.Background(), "HTTP://93.184.216.34:80/page#one")
|
||||||
|
require.NoError(t, err)
|
||||||
|
first.Title = "mutated"
|
||||||
|
second, err := successFetcher.Get(context.Background(), "http://93.184.216.34/page#two")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "Cached title", second.Title)
|
||||||
|
require.EqualValues(t, 1, successCalls.Load())
|
||||||
|
|
||||||
|
now = now.Add(successCacheTTL + time.Second)
|
||||||
|
_, err = successFetcher.Get(context.Background(), "http://93.184.216.34/page")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.EqualValues(t, 2, successCalls.Load())
|
||||||
|
|
||||||
|
var failureCalls atomic.Int32
|
||||||
|
failureFetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
failureCalls.Add(1)
|
||||||
|
return response(req, http.StatusBadGateway, "text/html", "failure"), nil
|
||||||
|
}))
|
||||||
|
failureFetcher.now = func() time.Time { return now }
|
||||||
|
for range 2 {
|
||||||
|
_, err := failureFetcher.Get(context.Background(), "http://93.184.216.34/failure")
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
require.EqualValues(t, 1, failureCalls.Load())
|
||||||
|
now = now.Add(failureCacheTTL + time.Second)
|
||||||
|
_, err = failureFetcher.Get(context.Background(), "http://93.184.216.34/failure")
|
||||||
|
require.Error(t, err)
|
||||||
|
require.EqualValues(t, 2, failureCalls.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherBoundsCache(t *testing.T) {
|
||||||
|
fetcher := NewHTMLMetaFetcher()
|
||||||
|
now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
|
||||||
|
fetcher.now = func() time.Time { return now }
|
||||||
|
for index := range maxCacheEntries + 1 {
|
||||||
|
fetcher.setCached(fmt.Sprintf("https://example.com/%d", index), &HTMLMeta{Title: "title"}, nil, successCacheTTL)
|
||||||
|
now = now.Add(time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Len(t, fetcher.cache, maxCacheEntries)
|
||||||
|
_, exists := fetcher.cache["https://example.com/0"]
|
||||||
|
require.False(t, exists)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherCoalescesRequests(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
started := make(chan struct{})
|
||||||
|
release := make(chan struct{})
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
close(started)
|
||||||
|
}
|
||||||
|
<-release
|
||||||
|
return htmlResponse(req, `<title>Shared title</title>`), nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
const requestCount = 20
|
||||||
|
var waitGroup sync.WaitGroup
|
||||||
|
errorsChannel := make(chan error, requestCount)
|
||||||
|
for range requestCount {
|
||||||
|
waitGroup.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer waitGroup.Done()
|
||||||
|
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/shared")
|
||||||
|
if err == nil && meta.Title != "Shared title" {
|
||||||
|
err = errors.New("unexpected metadata")
|
||||||
|
}
|
||||||
|
errorsChannel <- err
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
<-started
|
||||||
|
close(release)
|
||||||
|
waitGroup.Wait()
|
||||||
|
close(errorsChannel)
|
||||||
|
for err := range errorsChannel {
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
require.EqualValues(t, 1, calls.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherLimitsConcurrency(t *testing.T) {
|
||||||
|
var active atomic.Int32
|
||||||
|
var maximum atomic.Int32
|
||||||
|
entered := make(chan struct{}, 12)
|
||||||
|
release := make(chan struct{})
|
||||||
|
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
current := active.Add(1)
|
||||||
|
for {
|
||||||
|
previous := maximum.Load()
|
||||||
|
if current <= previous || maximum.CompareAndSwap(previous, current) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entered <- struct{}{}
|
||||||
|
<-release
|
||||||
|
active.Add(-1)
|
||||||
|
return htmlResponse(req, `<title>Title</title>`), nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
const requestCount = 12
|
||||||
|
var waitGroup sync.WaitGroup
|
||||||
|
for index := range requestCount {
|
||||||
|
waitGroup.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer waitGroup.Done()
|
||||||
|
_, _ = fetcher.Get(context.Background(), "http://93.184.216.34/page?id="+string(rune('a'+index)))
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
for range maxConcurrentFetches {
|
||||||
|
<-entered
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-entered:
|
||||||
|
t.Fatal("more than eight uncached fetches started concurrently")
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
waitGroup.Wait()
|
||||||
|
require.EqualValues(t, maxConcurrentFetches, maximum.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTMLMetaFetcherRejectsUnsafeURLsAndRedirects(t *testing.T) {
|
||||||
|
fetcher := NewHTMLMetaFetcher()
|
||||||
|
_, err := fetcher.Get(context.Background(), "http://192.168.0.1/page")
|
||||||
|
require.ErrorIs(t, err, ErrInternalIP)
|
||||||
|
|
||||||
|
redirect, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/private", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
err = newHTTPClient().CheckRedirect(redirect, nil)
|
||||||
|
require.ErrorIs(t, err, ErrInternalIP)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSecureDialContextRejectsResolvedInternalIP(t *testing.T) {
|
func TestSecureDialContextRejectsResolvedInternalIP(t *testing.T) {
|
||||||
|
|
@ -187,3 +521,48 @@ func TestSecureDialContextDialsResolvedIP(t *testing.T) {
|
||||||
require.NotNil(t, conn)
|
require.NotNil(t, conn)
|
||||||
require.Equal(t, "93.184.216.34:80", dialedAddress)
|
require.Equal(t, "93.184.216.34:80", dialedAddress)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func extractTestMetadata(t *testing.T, markup string) *HTMLMeta {
|
||||||
|
t.Helper()
|
||||||
|
document, err := html.Parse(strings.NewReader(markup))
|
||||||
|
require.NoError(t, err)
|
||||||
|
sources, _ := extractDocumentMetadata(document)
|
||||||
|
pageURL, err := url.Parse("https://example.com/posts/page")
|
||||||
|
require.NoError(t, err)
|
||||||
|
return mergeMetadata(pageURL, metadataSource{}, sources.openGraph, sources.twitter, sources.jsonLD, sources.standard, sources.semantic)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestFetcher(transport http.RoundTripper) *HTMLMetaFetcher {
|
||||||
|
fetcher := NewHTMLMetaFetcher()
|
||||||
|
fetcher.client = &http.Client{Transport: transport}
|
||||||
|
return fetcher
|
||||||
|
}
|
||||||
|
|
||||||
|
func htmlResponse(request *http.Request, body string) *http.Response {
|
||||||
|
return response(request, http.StatusOK, "text/html; charset=utf-8", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func response(request *http.Request, statusCode int, contentType, body string) *http.Response {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: statusCode,
|
||||||
|
Status: http.StatusText(statusCode),
|
||||||
|
Header: http.Header{"Content-Type": []string{contentType}},
|
||||||
|
Body: io.NopCloser(strings.NewReader(body)),
|
||||||
|
Request: request,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingReadCloser struct {
|
||||||
|
*strings.Reader
|
||||||
|
bytesRead int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *countingReadCloser) Read(buffer []byte) (int, error) {
|
||||||
|
count, err := reader.Reader.Read(buffer)
|
||||||
|
reader.bytesRead += int64(count)
|
||||||
|
return count, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*countingReadCloser) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
|
|
@ -13,22 +14,29 @@ import (
|
||||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGetLinkMetadata(t *testing.T) {
|
type fakeLinkMetadataFetcher func(context.Context, string) (*httpgetter.HTMLMeta, error)
|
||||||
originalFetchHTMLMeta := fetchHTMLMeta
|
|
||||||
t.Cleanup(func() {
|
|
||||||
fetchHTMLMeta = originalFetchHTMLMeta
|
|
||||||
})
|
|
||||||
|
|
||||||
fetchHTMLMeta = func(url string) (*httpgetter.HTMLMeta, error) {
|
type linkMetadataContextKey struct{}
|
||||||
|
|
||||||
|
func (fetch fakeLinkMetadataFetcher) Get(ctx context.Context, url string) (*httpgetter.HTMLMeta, error) {
|
||||||
|
return fetch(ctx, url)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetLinkMetadata(t *testing.T) {
|
||||||
|
requestContext := context.WithValue(context.Background(), linkMetadataContextKey{}, "context value")
|
||||||
|
service := &APIV1Service{
|
||||||
|
linkMetadataFetcher: fakeLinkMetadataFetcher(func(ctx context.Context, url string) (*httpgetter.HTMLMeta, error) {
|
||||||
|
require.Same(t, requestContext, ctx)
|
||||||
require.Equal(t, "https://example.com/article", url)
|
require.Equal(t, "https://example.com/article", url)
|
||||||
return &httpgetter.HTMLMeta{
|
return &httpgetter.HTMLMeta{
|
||||||
Title: "Example title",
|
Title: "Example title",
|
||||||
Description: "Example description",
|
Description: "Example description",
|
||||||
Image: "https://example.com/cover.png",
|
Image: "https://example.com/cover.png",
|
||||||
}, nil
|
}, nil
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata, err := (&APIV1Service{}).GetLinkMetadata(context.Background(), &v1pb.GetLinkMetadataRequest{
|
metadata, err := service.GetLinkMetadata(requestContext, &v1pb.GetLinkMetadataRequest{
|
||||||
Url: "https://example.com/article",
|
Url: "https://example.com/article",
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
@ -44,31 +52,41 @@ func TestGetLinkMetadataEmptyURL(t *testing.T) {
|
||||||
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetLinkMetadataFetchError(t *testing.T) {
|
||||||
|
service := &APIV1Service{
|
||||||
|
linkMetadataFetcher: fakeLinkMetadataFetcher(func(context.Context, string) (*httpgetter.HTMLMeta, error) {
|
||||||
|
return nil, errors.New("fetch failed")
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
_, err := service.GetLinkMetadata(context.Background(), &v1pb.GetLinkMetadataRequest{Url: "https://example.com"})
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
||||||
|
require.Contains(t, err.Error(), "failed to fetch link metadata")
|
||||||
|
}
|
||||||
|
|
||||||
func TestGetLinkMetadataInternalURL(t *testing.T) {
|
func TestGetLinkMetadataInternalURL(t *testing.T) {
|
||||||
_, err := (&APIV1Service{}).GetLinkMetadata(context.Background(), &v1pb.GetLinkMetadataRequest{
|
service := &APIV1Service{linkMetadataFetcher: httpgetter.NewHTMLMetaFetcher()}
|
||||||
|
_, err := service.GetLinkMetadata(context.Background(), &v1pb.GetLinkMetadataRequest{
|
||||||
Url: "http://192.168.0.1",
|
Url: "http://192.168.0.1",
|
||||||
})
|
})
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBatchGetLinkMetadata(t *testing.T) {
|
func TestBatchGetLinkMetadataPreservesOrder(t *testing.T) {
|
||||||
originalFetchHTMLMeta := fetchHTMLMeta
|
|
||||||
t.Cleanup(func() {
|
|
||||||
fetchHTMLMeta = originalFetchHTMLMeta
|
|
||||||
})
|
|
||||||
|
|
||||||
var fetchedURLs []string
|
var fetchedURLs []string
|
||||||
fetchHTMLMeta = func(url string) (*httpgetter.HTMLMeta, error) {
|
service := &APIV1Service{
|
||||||
|
linkMetadataFetcher: fakeLinkMetadataFetcher(func(_ context.Context, url string) (*httpgetter.HTMLMeta, error) {
|
||||||
fetchedURLs = append(fetchedURLs, url)
|
fetchedURLs = append(fetchedURLs, url)
|
||||||
return &httpgetter.HTMLMeta{
|
return &httpgetter.HTMLMeta{
|
||||||
Title: fmt.Sprintf("Title for %s", url),
|
Title: fmt.Sprintf("Title for %s", url),
|
||||||
Description: fmt.Sprintf("Description for %s", url),
|
Description: fmt.Sprintf("Description for %s", url),
|
||||||
Image: fmt.Sprintf("%s/cover.png", url),
|
Image: fmt.Sprintf("%s/cover.png", url),
|
||||||
}, nil
|
}, nil
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
response, err := (&APIV1Service{}).BatchGetLinkMetadata(context.Background(), &v1pb.BatchGetLinkMetadataRequest{
|
response, err := service.BatchGetLinkMetadata(context.Background(), &v1pb.BatchGetLinkMetadataRequest{
|
||||||
Urls: []string{
|
Urls: []string{
|
||||||
"https://example.com/one",
|
"https://example.com/one",
|
||||||
"https://example.com/two",
|
"https://example.com/two",
|
||||||
|
|
@ -83,6 +101,26 @@ func TestBatchGetLinkMetadata(t *testing.T) {
|
||||||
require.Equal(t, "Title for https://example.com/two", response.LinkMetadata[1].Title)
|
require.Equal(t, "Title for https://example.com/two", response.LinkMetadata[1].Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBatchGetLinkMetadataStopsOnError(t *testing.T) {
|
||||||
|
var fetchedURLs []string
|
||||||
|
service := &APIV1Service{
|
||||||
|
linkMetadataFetcher: fakeLinkMetadataFetcher(func(_ context.Context, url string) (*httpgetter.HTMLMeta, error) {
|
||||||
|
fetchedURLs = append(fetchedURLs, url)
|
||||||
|
if url == "https://example.com/two" {
|
||||||
|
return nil, errors.New("fetch failed")
|
||||||
|
}
|
||||||
|
return &httpgetter.HTMLMeta{Title: url}, nil
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := service.BatchGetLinkMetadata(context.Background(), &v1pb.BatchGetLinkMetadataRequest{
|
||||||
|
Urls: []string{"https://example.com/one", "https://example.com/two", "https://example.com/three"},
|
||||||
|
})
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
||||||
|
require.Equal(t, []string{"https://example.com/one", "https://example.com/two"}, fetchedURLs)
|
||||||
|
}
|
||||||
|
|
||||||
func TestBatchGetLinkMetadataEmptyURLs(t *testing.T) {
|
func TestBatchGetLinkMetadataEmptyURLs(t *testing.T) {
|
||||||
_, err := (&APIV1Service{}).BatchGetLinkMetadata(context.Background(), &v1pb.BatchGetLinkMetadataRequest{})
|
_, err := (&APIV1Service{}).BatchGetLinkMetadata(context.Background(), &v1pb.BatchGetLinkMetadataRequest{})
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ import (
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
"google.golang.org/protobuf/types/known/emptypb"
|
"google.golang.org/protobuf/types/known/emptypb"
|
||||||
|
|
||||||
"github.com/usememos/memos/internal/httpgetter"
|
|
||||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||||
storepb "github.com/usememos/memos/proto/gen/store"
|
storepb "github.com/usememos/memos/proto/gen/store"
|
||||||
"github.com/usememos/memos/server/access"
|
"github.com/usememos/memos/server/access"
|
||||||
|
|
@ -28,8 +27,6 @@ type suppressSSEKey struct{}
|
||||||
|
|
||||||
const maxBatchGetLinkMetadata = 10
|
const maxBatchGetLinkMetadata = 10
|
||||||
|
|
||||||
var fetchHTMLMeta = httpgetter.GetHTMLMeta
|
|
||||||
|
|
||||||
func withSuppressSSE(ctx context.Context) context.Context {
|
func withSuppressSSE(ctx context.Context) context.Context {
|
||||||
return context.WithValue(ctx, suppressSSEKey{}, true)
|
return context.WithValue(ctx, suppressSSEKey{}, true)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,16 +7,21 @@ import (
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
|
|
||||||
|
"github.com/usememos/memos/internal/httpgetter"
|
||||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type linkMetadataFetcher interface {
|
||||||
|
Get(context.Context, string) (*httpgetter.HTMLMeta, error)
|
||||||
|
}
|
||||||
|
|
||||||
// GetLinkMetadata gets metadata for a link.
|
// GetLinkMetadata gets metadata for a link.
|
||||||
func (*APIV1Service) GetLinkMetadata(_ context.Context, request *v1pb.GetLinkMetadataRequest) (*v1pb.LinkMetadata, error) {
|
func (s *APIV1Service) GetLinkMetadata(ctx context.Context, request *v1pb.GetLinkMetadataRequest) (*v1pb.LinkMetadata, error) {
|
||||||
return getLinkMetadata(request.GetUrl())
|
return s.buildLinkMetadata(ctx, request.GetUrl())
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchGetLinkMetadata gets metadata for links.
|
// BatchGetLinkMetadata gets metadata for links.
|
||||||
func (*APIV1Service) BatchGetLinkMetadata(_ context.Context, request *v1pb.BatchGetLinkMetadataRequest) (*v1pb.BatchGetLinkMetadataResponse, error) {
|
func (s *APIV1Service) BatchGetLinkMetadata(ctx context.Context, request *v1pb.BatchGetLinkMetadataRequest) (*v1pb.BatchGetLinkMetadataResponse, error) {
|
||||||
if len(request.Urls) == 0 {
|
if len(request.Urls) == 0 {
|
||||||
return nil, status.Errorf(codes.InvalidArgument, "urls are required")
|
return nil, status.Errorf(codes.InvalidArgument, "urls are required")
|
||||||
}
|
}
|
||||||
|
|
@ -26,7 +31,7 @@ func (*APIV1Service) BatchGetLinkMetadata(_ context.Context, request *v1pb.Batch
|
||||||
|
|
||||||
linkMetadata := make([]*v1pb.LinkMetadata, 0, len(request.Urls))
|
linkMetadata := make([]*v1pb.LinkMetadata, 0, len(request.Urls))
|
||||||
for _, url := range request.Urls {
|
for _, url := range request.Urls {
|
||||||
metadata, err := getLinkMetadata(url)
|
metadata, err := s.buildLinkMetadata(ctx, url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -38,12 +43,12 @@ func (*APIV1Service) BatchGetLinkMetadata(_ context.Context, request *v1pb.Batch
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getLinkMetadata(inputURL string) (*v1pb.LinkMetadata, error) {
|
func (s *APIV1Service) buildLinkMetadata(ctx context.Context, inputURL string) (*v1pb.LinkMetadata, error) {
|
||||||
url := strings.TrimSpace(inputURL)
|
url := strings.TrimSpace(inputURL)
|
||||||
if url == "" {
|
if url == "" {
|
||||||
return nil, status.Errorf(codes.InvalidArgument, "url is required")
|
return nil, status.Errorf(codes.InvalidArgument, "url is required")
|
||||||
}
|
}
|
||||||
htmlMeta, err := fetchHTMLMeta(url)
|
htmlMeta, err := s.linkMetadataFetcher.Get(ctx, url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, status.Errorf(codes.InvalidArgument, "failed to fetch link metadata: %v", err)
|
return nil, status.Errorf(codes.InvalidArgument, "failed to fetch link metadata: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"golang.org/x/sync/semaphore"
|
"golang.org/x/sync/semaphore"
|
||||||
"google.golang.org/protobuf/encoding/protojson"
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
|
||||||
|
"github.com/usememos/memos/internal/httpgetter"
|
||||||
"github.com/usememos/memos/internal/markdown"
|
"github.com/usememos/memos/internal/markdown"
|
||||||
"github.com/usememos/memos/internal/profile"
|
"github.com/usememos/memos/internal/profile"
|
||||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||||
|
|
@ -47,14 +48,17 @@ type APIV1Service struct {
|
||||||
|
|
||||||
// instanceStatsCache memoizes GetInstanceStats results for instanceStatsCacheTTL.
|
// instanceStatsCache memoizes GetInstanceStats results for instanceStatsCacheTTL.
|
||||||
instanceStatsCache instanceStatsCache
|
instanceStatsCache instanceStatsCache
|
||||||
|
|
||||||
|
linkMetadataFetcher linkMetadataFetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewAPIV1Service creates an API v1 service with its shared dependencies.
|
||||||
func NewAPIV1Service(secret string, profile *profile.Profile, store *store.Store) *APIV1Service {
|
func NewAPIV1Service(secret string, profile *profile.Profile, store *store.Store) *APIV1Service {
|
||||||
markdownService := markdown.NewService(
|
markdownService := markdown.NewService(
|
||||||
markdown.WithTagExtension(),
|
markdown.WithTagExtension(),
|
||||||
markdown.WithMentionExtension(),
|
markdown.WithMentionExtension(),
|
||||||
)
|
)
|
||||||
return &APIV1Service{
|
service := &APIV1Service{
|
||||||
Secret: secret,
|
Secret: secret,
|
||||||
Profile: profile,
|
Profile: profile,
|
||||||
Store: store,
|
Store: store,
|
||||||
|
|
@ -64,6 +68,8 @@ func NewAPIV1Service(secret string, profile *profile.Profile, store *store.Store
|
||||||
thumbnailSemaphore: semaphore.NewWeighted(3), // Limit to 3 concurrent thumbnail generations
|
thumbnailSemaphore: semaphore.NewWeighted(3), // Limit to 3 concurrent thumbnail generations
|
||||||
imageProcessingSemaphore: semaphore.NewWeighted(2),
|
imageProcessingSemaphore: semaphore.NewWeighted(2),
|
||||||
}
|
}
|
||||||
|
service.linkMetadataFetcher = httpgetter.NewHTMLMetaFetcher()
|
||||||
|
return service
|
||||||
}
|
}
|
||||||
|
|
||||||
// newGatewayMarshaler mirrors grpc-gateway's default JSON marshaler with one
|
// newGatewayMarshaler mirrors grpc-gateway's default JSON marshaler with one
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue