package httpgetter import ( "bytes" "encoding/json" "mime" "net/url" "strings" "unicode" "unicode/utf8" "github.com/pkg/errors" "golang.org/x/net/html" ) const ( maxTitleRunes = 200 maxDescriptionRunes = 300 minSemanticTextRunes = 40 maxImageURLBytes = 2048 ) type metadataSource struct { title string description string image string } type documentMetadata struct { openGraph metadataSource twitter metadataSource jsonLD metadataSource standard metadataSource semantic metadataSource baseHref string } func extractDocumentMetadata(document *html.Node) (documentMetadata, string) { var sources documentMetadata var oEmbedEndpoint string var jsonLDScripts []string walkNodes(document, func(node *html.Node) { if node.Type != html.ElementNode { return } switch strings.ToLower(node.Data) { case "title": // Foreign-content titles (inline SVG/MathML accessibility labels) // are not the document title. if node.Namespace == "" { setIfEmpty(&sources.standard.title, nodeText(node)) } case "base": setIfEmpty(&sources.baseHref, firstAttribute(node, "href")) case "meta": name := strings.ToLower(strings.TrimSpace(firstAttribute(node, "property", "name"))) content := firstAttribute(node, "content") switch name { case "og:title": setIfEmpty(&sources.openGraph.title, content) case "og:description": setIfEmpty(&sources.openGraph.description, content) case "og:image", "og:image:url", "og:image:secure_url": setImageIfEmpty(&sources.openGraph.image, content) case "twitter:title": setIfEmpty(&sources.twitter.title, content) case "twitter:description": setIfEmpty(&sources.twitter.description, content) case "twitter:image", "twitter:image:src": setImageIfEmpty(&sources.twitter.image, content) case "description": setIfEmpty(&sources.standard.description, content) case "title", "name", "headline": setIfEmpty(&sources.standard.title, content) case "image": setImageIfEmpty(&sources.standard.image, content) default: } case "link": rels := strings.Fields(strings.ToLower(firstAttribute(node, "rel"))) if containsString(rels, "image_src") { setImageIfEmpty(&sources.standard.image, firstAttribute(node, "href")) } if oEmbedEndpoint == "" && containsString(rels, "alternate") { mediaType := firstAttribute(node, "type") if isJSONOEmbedMediaType(mediaType) { oEmbedEndpoint = firstAttribute(node, "href") } } case "script": if isJSONLDMediaType(firstAttribute(node, "type")) { jsonLDScripts = append(jsonLDScripts, rawNodeText(node)) } default: } }) sources.jsonLD = extractJSONLDMetadata(jsonLDScripts) sources.semantic = extractSemanticMetadata(document) return sources, oEmbedEndpoint } func mergeMetadata(pageURL *url.URL, sources ...metadataSource) *HTMLMeta { meta := &HTMLMeta{} for _, source := range sources { setIfEmpty(&meta.Title, source.title) setIfEmpty(&meta.Description, source.description) if meta.Image == "" { meta.Image = resolveHTTPURL(pageURL, source.image) } } meta.Title = truncateText(normalizeWhitespace(meta.Title), maxTitleRunes) meta.Description = truncateText(normalizeWhitespace(meta.Description), maxDescriptionRunes) // A truncated URL would be broken, so oversized image URLs are dropped // rather than shortened. if len(meta.Image) > maxImageURLBytes { meta.Image = "" } return meta } func extractOEmbedMetadata(data []byte, baseURL *url.URL) (metadataSource, error) { var payload struct { Type string `json:"type"` Title string `json:"title"` Description string `json:"description"` ThumbnailURL string `json:"thumbnail_url"` URL string `json:"url"` } decoder := json.NewDecoder(bytes.NewReader(data)) if err := decoder.Decode(&payload); err != nil { return metadataSource{}, errors.Wrap(err, "failed to decode oEmbed response") } image := resolveHTTPURL(baseURL, payload.ThumbnailURL) if image == "" && strings.EqualFold(payload.Type, "photo") { image = resolveHTTPURL(baseURL, payload.URL) } return metadataSource{ title: payload.Title, description: payload.Description, image: image, }, nil } func extractJSONLDMetadata(scripts []string) metadataSource { var best metadataSource bestScore := -1 for _, script := range scripts { var value any decoder := json.NewDecoder(strings.NewReader(script)) if err := decoder.Decode(&value); err != nil { continue } walkJSONLD(value, func(object map[string]any) { candidate := metadataSource{ title: firstJSONString(object["headline"], object["name"]), description: firstJSONString(object["description"]), image: jsonLDImage(object["image"]), } if candidate.image == "" { candidate.image = jsonLDImage(object["thumbnailUrl"]) } if candidate == (metadataSource{}) { return } score := jsonLDTypeScore(object["@type"]) if candidate.title != "" { score++ } if candidate.description != "" { score++ } if candidate.image != "" { score++ } if score > bestScore { best = candidate bestScore = score } }) } return best } func walkJSONLD(value any, visit func(map[string]any)) { switch typed := value.(type) { case []any: for _, item := range typed { walkJSONLD(item, visit) } case map[string]any: visit(typed) for _, nested := range typed { walkJSONLD(nested, visit) } default: } } func jsonLDTypeScore(value any) int { types := jsonLDStrings(value) for _, value := range types { switch strings.ToLower(value) { case "article", "newsarticle", "blogposting", "socialmediaposting", "product", "videoobject", "movie", "book", "event", "recipe": return 20 case "creativework", "webpage", "profilepage", "aboutpage", "collectionpage": return 10 case "breadcrumblist", "itemlist": return -10 } } return 0 } func jsonLDImage(value any) string { switch typed := value.(type) { case string: if isPotentialHTTPURL(typed) { return typed } case []any: for _, item := range typed { if image := jsonLDImage(item); image != "" { return image } } case map[string]any: // "@id" is deliberately excluded: it is a node reference (often the // page's own URL plus a fragment), not an image location. for _, candidate := range []any{typed["url"], typed["contentUrl"]} { if image := jsonLDImage(candidate); image != "" { return image } } default: } return "" } func firstJSONString(values ...any) string { for _, value := range values { for _, candidate := range jsonLDStrings(value) { if strings.TrimSpace(candidate) != "" { return candidate } } } return "" } func jsonLDStrings(value any) []string { switch typed := value.(type) { case string: return []string{typed} case []any: var result []string for _, item := range typed { result = append(result, jsonLDStrings(item)...) } return result } return nil } func extractSemanticMetadata(document *html.Node) metadataSource { root := firstElement(document, "main", "article") if root == nil { root = firstElement(document, "body") } if root == nil { return metadataSource{} } var source metadataSource walkNodes(root, func(node *html.Node) { if node.Type != html.ElementNode { return } switch strings.ToLower(node.Data) { case "h1": setIfEmpty(&source.title, nodeText(node)) case "p": text := normalizeWhitespace(nodeText(node)) if source.description == "" && utf8.RuneCountInString(text) >= minSemanticTextRunes { source.description = text } case "img": setImageIfEmpty(&source.image, firstAttribute(node, "src")) default: } }) return source } func resolveHTTPURL(baseURL *url.URL, candidate string) string { candidate = strings.TrimSpace(candidate) if candidate == "" { return "" } parsed, err := url.Parse(candidate) if err != nil { return "" } if baseURL != nil { parsed = baseURL.ResolveReference(parsed) } if parsed.Scheme != "http" && parsed.Scheme != "https" { return "" } if parsed.Hostname() == "" { return "" } return parsed.String() } func walkNodes(node *html.Node, visit func(*html.Node)) { if node == nil { return } visit(node) for child := node.FirstChild; child != nil; child = child.NextSibling { walkNodes(child, visit) } } // firstElement prefers earlier names over document order: all matches for // names[0] outrank any match for names[1]. func firstElement(node *html.Node, names ...string) *html.Node { for _, name := range names { var result *html.Node walkNodes(node, func(candidate *html.Node) { if result != nil || candidate.Type != html.ElementNode { return } if strings.EqualFold(candidate.Data, name) { result = candidate } }) if result != nil { return result } } return nil } func firstAttribute(node *html.Node, names ...string) string { for _, name := range names { for _, attribute := range node.Attr { if strings.EqualFold(attribute.Key, name) { return attribute.Val } } } return "" } func rawNodeText(node *html.Node) string { var builder strings.Builder walkNodes(node, func(candidate *html.Node) { if candidate.Type == html.TextNode { builder.WriteString(candidate.Data) } }) return builder.String() } func nodeText(node *html.Node) string { return normalizeWhitespace(rawNodeText(node)) } func normalizeWhitespace(value string) string { return strings.Join(strings.FieldsFunc(value, unicode.IsSpace), " ") } func truncateText(value string, maximum int) string { runes := []rune(value) if len(runes) <= maximum { return value } return strings.TrimSpace(string(runes[:maximum-1])) + "…" } func setIfEmpty(target *string, value string) { if *target == "" && strings.TrimSpace(value) != "" { *target = value } } func setImageIfEmpty(target *string, value string) { if *target == "" && isPotentialHTTPURL(value) { *target = value } } func isPotentialHTTPURL(value string) bool { value = strings.TrimSpace(value) if value == "" { return false } parsed, err := url.Parse(value) if err != nil { return false } if parsed.Scheme == "" { return !strings.HasPrefix(value, "//") || parsed.Hostname() != "" } return (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Hostname() != "" } func containsString(values []string, expected string) bool { for _, value := range values { if value == expected { return true } } return false } func isJSONOEmbedMediaType(value string) bool { mediaType, _, err := mime.ParseMediaType(value) return err == nil && strings.EqualFold(mediaType, "application/json+oembed") } func isJSONLDMediaType(value string) bool { mediaType, _, err := mime.ParseMediaType(value) return err == nil && strings.EqualFold(mediaType, "application/ld+json") } // resolveDocumentBase applies the document's , falling back to the // final response URL when the base is missing or not an absolute http(s) URL. func resolveDocumentBase(pageURL *url.URL, baseHref string) *url.URL { baseHref = strings.TrimSpace(baseHref) if baseHref == "" { return pageURL } parsed, err := url.Parse(baseHref) if err != nil { return pageURL } resolved := pageURL.ResolveReference(parsed) if (resolved.Scheme != "http" && resolved.Scheme != "https") || resolved.Hostname() == "" { return pageURL } return resolved }