libnovel/backend/internal/novelfire/htmlutil/htmlutil.go

// Package htmlutil provides helper functions for parsing HTML with
// golang.org/x/net/html and extracting values by Selector descriptors.
package htmlutil

import (
	"net/url"
	"regexp"
	"strings"

	"github.com/libnovel/backend/internal/scraper"
	"golang.org/x/net/html"
)

// ResolveURL returns an absolute URL. If href is already absolute it is
// returned unchanged. Otherwise it is resolved against base.
func ResolveURL(base, href string) string {
	if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
		return href
	}
	b, err := url.Parse(base)
	if err != nil {
		return base + href
	}
	ref, err := url.Parse(href)
	if err != nil {
		return base + href
	}
	return b.ResolveReference(ref).String()
}

// ParseHTML parses raw HTML and returns the root node.
func ParseHTML(raw string) (*html.Node, error) {
	return html.Parse(strings.NewReader(raw))
}

// selectorMatches reports whether node n matches sel.
func selectorMatches(n *html.Node, sel scraper.Selector) bool {
	if n.Type != html.ElementNode {
		return false
	}
	if sel.Tag != "" && n.Data != sel.Tag {
		return false
	}
	if sel.ID != "" {
		for _, a := range n.Attr {
			if a.Key == "id" && a.Val == sel.ID {
				goto checkClass
			}
		}
		return false
	}
checkClass:
	if sel.Class != "" {
		for _, a := range n.Attr {
			if a.Key == "class" {
				for _, cls := range strings.Fields(a.Val) {
					if cls == sel.Class {
						goto matched
					}
				}
			}
		}
		return false
	}
matched:
	return true
}

// AttrVal returns the value of attribute key from node n.
func AttrVal(n *html.Node, key string) string {
	for _, a := range n.Attr {
		if a.Key == key {
			return a.Val
		}
	}
	return ""
}

// TextContent returns the concatenated text content of all descendant text nodes.
func TextContent(n *html.Node) string {
	var sb strings.Builder
	var walk func(*html.Node)
	walk = func(cur *html.Node) {
		if cur.Type == html.TextNode {
			sb.WriteString(cur.Data)
		}
		for c := cur.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(n)
	return strings.TrimSpace(sb.String())
}

// FindFirst returns the first node matching sel within root.
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
	var found *html.Node
	var walk func(*html.Node) bool
	walk = func(n *html.Node) bool {
		if selectorMatches(n, sel) {
			found = n
			return true
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if walk(c) {
				return true
			}
		}
		return false
	}
	walk(root)
	return found
}

// FindAll returns all nodes matching sel within root.
func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
	var results []*html.Node
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if selectorMatches(n, sel) {
			results = append(results, n)
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(root)
	return results
}

// ExtractText extracts a string value from node n using sel.
// If sel.Attr is set the attribute value is returned; otherwise the inner text.
func ExtractText(n *html.Node, sel scraper.Selector) string {
	if sel.Attr != "" {
		return AttrVal(n, sel.Attr)
	}
	return TextContent(n)
}

// ExtractFirst locates the first match in root and returns its text/attr value.
func ExtractFirst(root *html.Node, sel scraper.Selector) string {
	n := FindFirst(root, sel)
	if n == nil {
		return ""
	}
	return ExtractText(n, sel)
}

// ExtractAll locates all matches in root and returns their text/attr values.
func ExtractAll(root *html.Node, sel scraper.Selector) []string {
	nodes := FindAll(root, sel)
	out := make([]string, 0, len(nodes))
	for _, n := range nodes {
		if v := ExtractText(n, sel); v != "" {
			out = append(out, v)
		}
	}
	return out
}

// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
// representation suitable for chapter storage.
func NodeToMarkdown(n *html.Node) string {
	var sb strings.Builder
	nodeToMD(n, &sb)
	out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n")
	return strings.TrimSpace(out)
}

var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`)

var blockElements = map[string]bool{
	"p": true, "div": true, "br": true, "h1": true, "h2": true,
	"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
	"blockquote": true, "pre": true, "hr": true,
}

func nodeToMD(n *html.Node, sb *strings.Builder) {
	switch n.Type {
	case html.TextNode:
		sb.WriteString(n.Data)
	case html.ElementNode:
		tag := n.Data
		switch tag {
		case "br":
			sb.WriteString("\n")
		case "hr":
			sb.WriteString("\n---\n")
		case "h1", "h2", "h3", "h4", "h5", "h6":
			level := int(tag[1] - '0')
			sb.WriteString("\n" + strings.Repeat("#", level) + " ")
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				nodeToMD(c, sb)
			}
			sb.WriteString("\n\n")
			return
		case "p", "div", "blockquote":
			sb.WriteString("\n")
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				nodeToMD(c, sb)
			}
			sb.WriteString("\n")
			return
		case "em", "i":
			sb.WriteString("*")
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				nodeToMD(c, sb)
			}
			sb.WriteString("*")
			return
		case "strong", "b":
			sb.WriteString("**")
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				nodeToMD(c, sb)
			}
			sb.WriteString("**")
			return
		case "script", "style", "noscript":
			return // drop
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			nodeToMD(c, sb)
		}
		if blockElements[tag] {
			sb.WriteString("\n")
		}
	}
}