fb6b364382
- Remove dead code: browser cdp/content_scrape strategies, writer package, printUsage, downloadAndStoreCoverCLI in main.go - Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination hard cap removed, splitChapterTitle off-by-one in date extraction - Split server.go (~1700 lines) into focused handler files: handlers_audio, handlers_browse, handlers_progress, handlers_ranking, handlers_scrape - Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go to consolidate duplicate helpers - Flatten deeply nested conditionals: voices() early-return guards, ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead strings.Cut line, splitChapterTitle double-nested unit/suffix loop - Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases), orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle (22 cases); all pass with go build/vet/test clean
222 lines
7.6 KiB
Go
222 lines
7.6 KiB
Go
package htmlutil
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
)
|
|
|
|
// ── ResolveURL ────────────────────────────────────────────────────────────────
|
|
|
|
func TestResolveURL(t *testing.T) {
|
|
cases := []struct{ base, href, want string }{
|
|
// Already absolute → unchanged.
|
|
{"https://example.com", "https://other.com/page", "https://other.com/page"},
|
|
{"https://example.com", "http://other.com/page", "http://other.com/page"},
|
|
// Absolute path.
|
|
{"https://example.com", "/book/slug", "https://example.com/book/slug"},
|
|
// Relative path.
|
|
{"https://example.com/genre/all", "page?p=2", "https://example.com/genre/page?p=2"},
|
|
// Empty href → base itself.
|
|
{"https://example.com", "", "https://example.com"},
|
|
}
|
|
for _, c := range cases {
|
|
got := ResolveURL(c.base, c.href)
|
|
if got != c.want {
|
|
t.Errorf("ResolveURL(%q, %q) = %q, want %q", c.base, c.href, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── AttrVal ───────────────────────────────────────────────────────────────────
|
|
|
|
func TestAttrVal(t *testing.T) {
|
|
root, err := ParseHTML(`<html><body><a href="/book/slug" class="link">text</a></body></html>`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := FindFirst(root, scraper.Selector{Tag: "a"})
|
|
if a == nil {
|
|
t.Fatal("expected to find <a>")
|
|
}
|
|
if got := AttrVal(a, "href"); got != "/book/slug" {
|
|
t.Errorf("AttrVal href = %q, want %q", got, "/book/slug")
|
|
}
|
|
if got := AttrVal(a, "class"); got != "link" {
|
|
t.Errorf("AttrVal class = %q, want %q", got, "link")
|
|
}
|
|
if got := AttrVal(a, "missing"); got != "" {
|
|
t.Errorf("AttrVal missing = %q, want empty", got)
|
|
}
|
|
}
|
|
|
|
// ── TextContent ───────────────────────────────────────────────────────────────
|
|
|
|
func TestTextContent(t *testing.T) {
|
|
root, err := ParseHTML(`<html><body><p>Hello <b>world</b></p></body></html>`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := FindFirst(root, scraper.Selector{Tag: "p"})
|
|
if p == nil {
|
|
t.Fatal("expected to find <p>")
|
|
}
|
|
if got := TextContent(p); got != "Hello world" {
|
|
t.Errorf("TextContent = %q, want %q", got, "Hello world")
|
|
}
|
|
}
|
|
|
|
// ── FindFirst / FindAll ───────────────────────────────────────────────────────
|
|
|
|
func TestFindFirst_ByTag(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><h1>Title</h1><h2>Sub</h2></body></html>`)
|
|
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
|
if n == nil {
|
|
t.Fatal("expected to find <h1>")
|
|
}
|
|
if TextContent(n) != "Title" {
|
|
t.Errorf("h1 text = %q, want %q", TextContent(n), "Title")
|
|
}
|
|
}
|
|
|
|
func TestFindFirst_ByClass(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><span class="author foo">JR</span></body></html>`)
|
|
n := FindFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
|
if n == nil {
|
|
t.Fatal("expected to find span.author")
|
|
}
|
|
if TextContent(n) != "JR" {
|
|
t.Errorf("author text = %q, want %q", TextContent(n), "JR")
|
|
}
|
|
}
|
|
|
|
func TestFindFirst_ByID(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><div id="content"><p>text</p></div></body></html>`)
|
|
n := FindFirst(root, scraper.Selector{ID: "content"})
|
|
if n == nil {
|
|
t.Fatal("expected to find #content")
|
|
}
|
|
}
|
|
|
|
func TestFindFirst_NoMatch(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><p>nothing</p></body></html>`)
|
|
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
|
if n != nil {
|
|
t.Errorf("expected nil for missing tag, got %v", n)
|
|
}
|
|
}
|
|
|
|
func TestFindAll_Multiple(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body>
|
|
<li class="novel-item">A</li>
|
|
<li class="novel-item">B</li>
|
|
<li class="other">C</li>
|
|
</body></html>`)
|
|
nodes := FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item"})
|
|
if len(nodes) != 2 {
|
|
t.Errorf("FindAll novel-item = %d, want 2", len(nodes))
|
|
}
|
|
}
|
|
|
|
// ── ExtractFirst / ExtractAll ─────────────────────────────────────────────────
|
|
|
|
func TestExtractFirst_TextNode(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><h1 class="novel-title">Shadow Slave</h1></body></html>`)
|
|
got := ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
|
if got != "Shadow Slave" {
|
|
t.Errorf("ExtractFirst title = %q, want %q", got, "Shadow Slave")
|
|
}
|
|
}
|
|
|
|
func TestExtractFirst_AttrNode(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><img src="/covers/slug.jpg"></body></html>`)
|
|
got := ExtractFirst(root, scraper.Selector{Tag: "img", Attr: "src"})
|
|
if got != "/covers/slug.jpg" {
|
|
t.Errorf("ExtractFirst img src = %q, want %q", got, "/covers/slug.jpg")
|
|
}
|
|
}
|
|
|
|
func TestExtractFirst_Missing(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body></body></html>`)
|
|
got := ExtractFirst(root, scraper.Selector{Tag: "h1"})
|
|
if got != "" {
|
|
t.Errorf("ExtractFirst missing = %q, want empty", got)
|
|
}
|
|
}
|
|
|
|
func TestExtractAll_Genres(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body>
|
|
<div class="genres">
|
|
<a href="/genre/action">Action</a>
|
|
<a href="/genre/fantasy">Fantasy</a>
|
|
</div>
|
|
</body></html>`)
|
|
genresNode := FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
|
if genresNode == nil {
|
|
t.Fatal("expected genres div")
|
|
}
|
|
genres := ExtractAll(genresNode, scraper.Selector{Tag: "a"})
|
|
if len(genres) != 2 {
|
|
t.Fatalf("genres = %v, want 2", genres)
|
|
}
|
|
if genres[0] != "Action" || genres[1] != "Fantasy" {
|
|
t.Errorf("genres = %v, want [Action Fantasy]", genres)
|
|
}
|
|
}
|
|
|
|
// ── NodeToMarkdown ────────────────────────────────────────────────────────────
|
|
|
|
func TestNodeToMarkdown_Paragraphs(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><div id="content">
|
|
<p>First paragraph.</p>
|
|
<p>Second paragraph.</p>
|
|
</div></body></html>`)
|
|
container := FindFirst(root, scraper.Selector{ID: "content"})
|
|
if container == nil {
|
|
t.Fatal("missing #content")
|
|
}
|
|
md := NodeToMarkdown(container)
|
|
if md == "" {
|
|
t.Fatal("NodeToMarkdown returned empty string")
|
|
}
|
|
for _, want := range []string{"First paragraph", "Second paragraph"} {
|
|
if !strings.Contains(md, want) {
|
|
t.Errorf("NodeToMarkdown missing %q in:\n%s", want, md)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNodeToMarkdown_Bold(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><div id="content"><p>He was <strong>very</strong> strong.</p></div></body></html>`)
|
|
container := FindFirst(root, scraper.Selector{ID: "content"})
|
|
md := NodeToMarkdown(container)
|
|
if !strings.Contains(md, "**very**") {
|
|
t.Errorf("NodeToMarkdown should wrap <strong> in **, got:\n%s", md)
|
|
}
|
|
}
|
|
|
|
func TestNodeToMarkdown_ScriptStripped(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><div id="content"><p>Good</p><script>alert(1)</script></div></body></html>`)
|
|
container := FindFirst(root, scraper.Selector{ID: "content"})
|
|
md := NodeToMarkdown(container)
|
|
if strings.Contains(md, "alert") {
|
|
t.Errorf("NodeToMarkdown should strip <script> content, got:\n%s", md)
|
|
}
|
|
}
|
|
|
|
func TestNodeToMarkdown_CollapseBlankLines(t *testing.T) {
|
|
root, _ := ParseHTML(`<html><body><div id="content">
|
|
<p>A</p>
|
|
<p></p>
|
|
<p></p>
|
|
<p>B</p>
|
|
</div></body></html>`)
|
|
container := FindFirst(root, scraper.Selector{ID: "content"})
|
|
md := NodeToMarkdown(container)
|
|
// Should not have more than one consecutive blank line.
|
|
if strings.Contains(md, "\n\n\n") {
|
|
t.Errorf("NodeToMarkdown should collapse triple newlines, got:\n%q", md)
|
|
}
|
|
}
|