feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
228
v3/backend/internal/novelfire/htmlutil/htmlutil.go
Normal file
228
v3/backend/internal/novelfire/htmlutil/htmlutil.go
Normal file
@@ -0,0 +1,228 @@
|
||||
// Package htmlutil provides helper functions for parsing HTML with
|
||||
// golang.org/x/net/html and extracting values by Selector descriptors.
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// ResolveURL returns an absolute URL. If href is already absolute it is
|
||||
// returned unchanged. Otherwise it is resolved against base.
|
||||
func ResolveURL(base, href string) string {
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
return b.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
// ParseHTML parses raw HTML and returns the root node.
|
||||
func ParseHTML(raw string) (*html.Node, error) {
|
||||
return html.Parse(strings.NewReader(raw))
|
||||
}
|
||||
|
||||
// selectorMatches reports whether node n matches sel.
|
||||
func selectorMatches(n *html.Node, sel scraper.Selector) bool {
|
||||
if n.Type != html.ElementNode {
|
||||
return false
|
||||
}
|
||||
if sel.Tag != "" && n.Data != sel.Tag {
|
||||
return false
|
||||
}
|
||||
if sel.ID != "" {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "id" && a.Val == sel.ID {
|
||||
goto checkClass
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
checkClass:
|
||||
if sel.Class != "" {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "class" {
|
||||
for _, cls := range strings.Fields(a.Val) {
|
||||
if cls == sel.Class {
|
||||
goto matched
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
matched:
|
||||
return true
|
||||
}
|
||||
|
||||
// AttrVal returns the value of attribute key from node n.
|
||||
func AttrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TextContent returns the concatenated text content of all descendant text nodes.
|
||||
func TextContent(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(cur *html.Node) {
|
||||
if cur.Type == html.TextNode {
|
||||
sb.WriteString(cur.Data)
|
||||
}
|
||||
for c := cur.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// FindFirst returns the first node matching sel within root.
|
||||
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
|
||||
var found *html.Node
|
||||
var walk func(*html.Node) bool
|
||||
walk = func(n *html.Node) bool {
|
||||
if selectorMatches(n, sel) {
|
||||
found = n
|
||||
return true
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if walk(c) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
walk(root)
|
||||
return found
|
||||
}
|
||||
|
||||
// FindAll returns all nodes matching sel within root.
|
||||
func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
|
||||
var results []*html.Node
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if selectorMatches(n, sel) {
|
||||
results = append(results, n)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
return results
|
||||
}
|
||||
|
||||
// ExtractText extracts a string value from node n using sel.
|
||||
// If sel.Attr is set the attribute value is returned; otherwise the inner text.
|
||||
func ExtractText(n *html.Node, sel scraper.Selector) string {
|
||||
if sel.Attr != "" {
|
||||
return AttrVal(n, sel.Attr)
|
||||
}
|
||||
return TextContent(n)
|
||||
}
|
||||
|
||||
// ExtractFirst locates the first match in root and returns its text/attr value.
|
||||
func ExtractFirst(root *html.Node, sel scraper.Selector) string {
|
||||
n := FindFirst(root, sel)
|
||||
if n == nil {
|
||||
return ""
|
||||
}
|
||||
return ExtractText(n, sel)
|
||||
}
|
||||
|
||||
// ExtractAll locates all matches in root and returns their text/attr values.
|
||||
func ExtractAll(root *html.Node, sel scraper.Selector) []string {
|
||||
nodes := FindAll(root, sel)
|
||||
out := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
if v := ExtractText(n, sel); v != "" {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
|
||||
// representation suitable for chapter storage.
|
||||
func NodeToMarkdown(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
nodeToMD(n, &sb)
|
||||
out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n")
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
|
||||
var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`)
|
||||
|
||||
var blockElements = map[string]bool{
|
||||
"p": true, "div": true, "br": true, "h1": true, "h2": true,
|
||||
"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
|
||||
"blockquote": true, "pre": true, "hr": true,
|
||||
}
|
||||
|
||||
func nodeToMD(n *html.Node, sb *strings.Builder) {
|
||||
switch n.Type {
|
||||
case html.TextNode:
|
||||
sb.WriteString(n.Data)
|
||||
case html.ElementNode:
|
||||
tag := n.Data
|
||||
switch tag {
|
||||
case "br":
|
||||
sb.WriteString("\n")
|
||||
case "hr":
|
||||
sb.WriteString("\n---\n")
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6":
|
||||
level := int(tag[1] - '0')
|
||||
sb.WriteString("\n" + strings.Repeat("#", level) + " ")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("\n\n")
|
||||
return
|
||||
case "p", "div", "blockquote":
|
||||
sb.WriteString("\n")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
return
|
||||
case "em", "i":
|
||||
sb.WriteString("*")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("*")
|
||||
return
|
||||
case "strong", "b":
|
||||
sb.WriteString("**")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("**")
|
||||
return
|
||||
case "script", "style", "noscript":
|
||||
return // drop
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
if blockElements[tag] {
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user