// Package htmlutil provides helper functions for parsing HTML with // golang.org/x/net/html and extracting values by Selector descriptors. package htmlutil import ( "net/url" "regexp" "strings" "github.com/libnovel/backend/internal/scraper" "golang.org/x/net/html" ) // ResolveURL returns an absolute URL. If href is already absolute it is // returned unchanged. Otherwise it is resolved against base. func ResolveURL(base, href string) string { if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { return href } b, err := url.Parse(base) if err != nil { return base + href } ref, err := url.Parse(href) if err != nil { return base + href } return b.ResolveReference(ref).String() } // ParseHTML parses raw HTML and returns the root node. func ParseHTML(raw string) (*html.Node, error) { return html.Parse(strings.NewReader(raw)) } // selectorMatches reports whether node n matches sel. func selectorMatches(n *html.Node, sel scraper.Selector) bool { if n.Type != html.ElementNode { return false } if sel.Tag != "" && n.Data != sel.Tag { return false } if sel.ID != "" { for _, a := range n.Attr { if a.Key == "id" && a.Val == sel.ID { goto checkClass } } return false } checkClass: if sel.Class != "" { for _, a := range n.Attr { if a.Key == "class" { for _, cls := range strings.Fields(a.Val) { if cls == sel.Class { goto matched } } } } return false } matched: return true } // AttrVal returns the value of attribute key from node n. func AttrVal(n *html.Node, key string) string { for _, a := range n.Attr { if a.Key == key { return a.Val } } return "" } // TextContent returns the concatenated text content of all descendant text nodes. func TextContent(n *html.Node) string { var sb strings.Builder var walk func(*html.Node) walk = func(cur *html.Node) { if cur.Type == html.TextNode { sb.WriteString(cur.Data) } for c := cur.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(n) return strings.TrimSpace(sb.String()) } // FindFirst returns the first node matching sel within root. func FindFirst(root *html.Node, sel scraper.Selector) *html.Node { var found *html.Node var walk func(*html.Node) bool walk = func(n *html.Node) bool { if selectorMatches(n, sel) { found = n return true } for c := n.FirstChild; c != nil; c = c.NextSibling { if walk(c) { return true } } return false } walk(root) return found } // FindAll returns all nodes matching sel within root. func FindAll(root *html.Node, sel scraper.Selector) []*html.Node { var results []*html.Node var walk func(*html.Node) walk = func(n *html.Node) { if selectorMatches(n, sel) { results = append(results, n) } for c := n.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(root) return results } // ExtractText extracts a string value from node n using sel. // If sel.Attr is set the attribute value is returned; otherwise the inner text. func ExtractText(n *html.Node, sel scraper.Selector) string { if sel.Attr != "" { return AttrVal(n, sel.Attr) } return TextContent(n) } // ExtractFirst locates the first match in root and returns its text/attr value. func ExtractFirst(root *html.Node, sel scraper.Selector) string { n := FindFirst(root, sel) if n == nil { return "" } return ExtractText(n, sel) } // ExtractAll locates all matches in root and returns their text/attr values. func ExtractAll(root *html.Node, sel scraper.Selector) []string { nodes := FindAll(root, sel) out := make([]string, 0, len(nodes)) for _, n := range nodes { if v := ExtractText(n, sel); v != "" { out = append(out, v) } } return out } // NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown // representation suitable for chapter storage. func NodeToMarkdown(n *html.Node) string { var sb strings.Builder nodeToMD(n, &sb) out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n") return strings.TrimSpace(out) } var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`) var blockElements = map[string]bool{ "p": true, "div": true, "br": true, "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "li": true, "blockquote": true, "pre": true, "hr": true, } func nodeToMD(n *html.Node, sb *strings.Builder) { switch n.Type { case html.TextNode: sb.WriteString(n.Data) case html.ElementNode: tag := n.Data switch tag { case "br": sb.WriteString("\n") case "hr": sb.WriteString("\n---\n") case "h1", "h2", "h3", "h4", "h5", "h6": level := int(tag[1] - '0') sb.WriteString("\n" + strings.Repeat("#", level) + " ") for c := n.FirstChild; c != nil; c = c.NextSibling { nodeToMD(c, sb) } sb.WriteString("\n\n") return case "p", "div", "blockquote": sb.WriteString("\n") for c := n.FirstChild; c != nil; c = c.NextSibling { nodeToMD(c, sb) } sb.WriteString("\n") return case "em", "i": sb.WriteString("*") for c := n.FirstChild; c != nil; c = c.NextSibling { nodeToMD(c, sb) } sb.WriteString("*") return case "strong", "b": sb.WriteString("**") for c := n.FirstChild; c != nil; c = c.NextSibling { nodeToMD(c, sb) } sb.WriteString("**") return case "script", "style", "noscript": return // drop } for c := n.FirstChild; c != nil; c = c.NextSibling { nodeToMD(c, sb) } if blockElements[tag] { sb.WriteString("\n") } } }