refactor: audit, split server.go, add unit tests, and fix latent bugs
- Remove dead code: browser cdp/content_scrape strategies, writer package, printUsage, downloadAndStoreCoverCLI in main.go - Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination hard cap removed, splitChapterTitle off-by-one in date extraction - Split server.go (~1700 lines) into focused handler files: handlers_audio, handlers_browse, handlers_progress, handlers_ranking, handlers_scrape - Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go to consolidate duplicate helpers - Flatten deeply nested conditionals: voices() early-return guards, ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead strings.Cut line, splitChapterTitle double-nested unit/suffix loop - Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases), orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle (22 cases); all pass with go build/vet/test clean
This commit is contained in:
Admin
@@ -3,6 +3,7 @@
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -10,6 +11,24 @@ import (
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// ResolveURL returns an absolute URL. If href is already absolute it is
|
||||
// returned unchanged. Otherwise it is resolved against base using standard
|
||||
// URL resolution (handles relative paths, absolute paths, etc.).
|
||||
func ResolveURL(base, href string) string {
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
return b.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
// ParseHTML parses raw HTML and returns the root node.
|
||||
func ParseHTML(raw string) (*html.Node, error) {
|
||||
return html.Parse(strings.NewReader(raw))
|
||||
@@ -48,8 +67,8 @@ matched:
|
||||
return true
|
||||
}
|
||||
|
||||
// attrVal returns the value of attribute key from node n.
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
// AttrVal returns the value of attribute key from node n.
|
||||
func AttrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
@@ -58,8 +77,11 @@ func attrVal(n *html.Node, key string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// textContent returns the concatenated text content of all descendant text nodes.
|
||||
func textContent(n *html.Node) string {
|
||||
// attrVal is an unexported alias kept for internal use within this package.
|
||||
func attrVal(n *html.Node, key string) string { return AttrVal(n, key) }
|
||||
|
||||
// TextContent returns the concatenated text content of all descendant text nodes.
|
||||
func TextContent(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(cur *html.Node) {
|
||||
@@ -74,6 +96,9 @@ func textContent(n *html.Node) string {
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// textContent is an unexported alias kept for internal use within this package.
|
||||
func textContent(n *html.Node) string { return TextContent(n) }
|
||||
|
||||
// FindFirst returns the first node matching sel within root.
|
||||
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
|
||||
var found *html.Node
|
||||
|
||||
221
scraper/internal/scraper/htmlutil/htmlutil_test.go
Normal file
221
scraper/internal/scraper/htmlutil/htmlutil_test.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// ── ResolveURL ────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestResolveURL(t *testing.T) {
|
||||
cases := []struct{ base, href, want string }{
|
||||
// Already absolute → unchanged.
|
||||
{"https://example.com", "https://other.com/page", "https://other.com/page"},
|
||||
{"https://example.com", "http://other.com/page", "http://other.com/page"},
|
||||
// Absolute path.
|
||||
{"https://example.com", "/book/slug", "https://example.com/book/slug"},
|
||||
// Relative path.
|
||||
{"https://example.com/genre/all", "page?p=2", "https://example.com/genre/page?p=2"},
|
||||
// Empty href → base itself.
|
||||
{"https://example.com", "", "https://example.com"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := ResolveURL(c.base, c.href)
|
||||
if got != c.want {
|
||||
t.Errorf("ResolveURL(%q, %q) = %q, want %q", c.base, c.href, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── AttrVal ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestAttrVal(t *testing.T) {
|
||||
root, err := ParseHTML(`<html><body><a href="/book/slug" class="link">text</a></body></html>`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := FindFirst(root, scraper.Selector{Tag: "a"})
|
||||
if a == nil {
|
||||
t.Fatal("expected to find <a>")
|
||||
}
|
||||
if got := AttrVal(a, "href"); got != "/book/slug" {
|
||||
t.Errorf("AttrVal href = %q, want %q", got, "/book/slug")
|
||||
}
|
||||
if got := AttrVal(a, "class"); got != "link" {
|
||||
t.Errorf("AttrVal class = %q, want %q", got, "link")
|
||||
}
|
||||
if got := AttrVal(a, "missing"); got != "" {
|
||||
t.Errorf("AttrVal missing = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TextContent ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestTextContent(t *testing.T) {
|
||||
root, err := ParseHTML(`<html><body><p>Hello <b>world</b></p></body></html>`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := FindFirst(root, scraper.Selector{Tag: "p"})
|
||||
if p == nil {
|
||||
t.Fatal("expected to find <p>")
|
||||
}
|
||||
if got := TextContent(p); got != "Hello world" {
|
||||
t.Errorf("TextContent = %q, want %q", got, "Hello world")
|
||||
}
|
||||
}
|
||||
|
||||
// ── FindFirst / FindAll ───────────────────────────────────────────────────────
|
||||
|
||||
func TestFindFirst_ByTag(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><h1>Title</h1><h2>Sub</h2></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find <h1>")
|
||||
}
|
||||
if TextContent(n) != "Title" {
|
||||
t.Errorf("h1 text = %q, want %q", TextContent(n), "Title")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_ByClass(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><span class="author foo">JR</span></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find span.author")
|
||||
}
|
||||
if TextContent(n) != "JR" {
|
||||
t.Errorf("author text = %q, want %q", TextContent(n), "JR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_ByID(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>text</p></div></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find #content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_NoMatch(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><p>nothing</p></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if n != nil {
|
||||
t.Errorf("expected nil for missing tag, got %v", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindAll_Multiple(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body>
|
||||
<li class="novel-item">A</li>
|
||||
<li class="novel-item">B</li>
|
||||
<li class="other">C</li>
|
||||
</body></html>`)
|
||||
nodes := FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item"})
|
||||
if len(nodes) != 2 {
|
||||
t.Errorf("FindAll novel-item = %d, want 2", len(nodes))
|
||||
}
|
||||
}
|
||||
|
||||
// ── ExtractFirst / ExtractAll ─────────────────────────────────────────────────
|
||||
|
||||
func TestExtractFirst_TextNode(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><h1 class="novel-title">Shadow Slave</h1></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
||||
if got != "Shadow Slave" {
|
||||
t.Errorf("ExtractFirst title = %q, want %q", got, "Shadow Slave")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFirst_AttrNode(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><img src="/covers/slug.jpg"></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "img", Attr: "src"})
|
||||
if got != "/covers/slug.jpg" {
|
||||
t.Errorf("ExtractFirst img src = %q, want %q", got, "/covers/slug.jpg")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFirst_Missing(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if got != "" {
|
||||
t.Errorf("ExtractFirst missing = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAll_Genres(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body>
|
||||
<div class="genres">
|
||||
<a href="/genre/action">Action</a>
|
||||
<a href="/genre/fantasy">Fantasy</a>
|
||||
</div>
|
||||
</body></html>`)
|
||||
genresNode := FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
||||
if genresNode == nil {
|
||||
t.Fatal("expected genres div")
|
||||
}
|
||||
genres := ExtractAll(genresNode, scraper.Selector{Tag: "a"})
|
||||
if len(genres) != 2 {
|
||||
t.Fatalf("genres = %v, want 2", genres)
|
||||
}
|
||||
if genres[0] != "Action" || genres[1] != "Fantasy" {
|
||||
t.Errorf("genres = %v, want [Action Fantasy]", genres)
|
||||
}
|
||||
}
|
||||
|
||||
// ── NodeToMarkdown ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestNodeToMarkdown_Paragraphs(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content">
|
||||
<p>First paragraph.</p>
|
||||
<p>Second paragraph.</p>
|
||||
</div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if container == nil {
|
||||
t.Fatal("missing #content")
|
||||
}
|
||||
md := NodeToMarkdown(container)
|
||||
if md == "" {
|
||||
t.Fatal("NodeToMarkdown returned empty string")
|
||||
}
|
||||
for _, want := range []string{"First paragraph", "Second paragraph"} {
|
||||
if !strings.Contains(md, want) {
|
||||
t.Errorf("NodeToMarkdown missing %q in:\n%s", want, md)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_Bold(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>He was <strong>very</strong> strong.</p></div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
if !strings.Contains(md, "**very**") {
|
||||
t.Errorf("NodeToMarkdown should wrap <strong> in **, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_ScriptStripped(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>Good</p><script>alert(1)</script></div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
if strings.Contains(md, "alert") {
|
||||
t.Errorf("NodeToMarkdown should strip <script> content, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_CollapseBlankLines(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content">
|
||||
<p>A</p>
|
||||
<p></p>
|
||||
<p></p>
|
||||
<p>B</p>
|
||||
</div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
// Should not have more than one consecutive blank line.
|
||||
if strings.Contains(md, "\n\n\n") {
|
||||
t.Errorf("NodeToMarkdown should collapse triple newlines, got:\n%q", md)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user