801928aadf
Some checks failed
CI / Backend (push) Failing after 11s
CI / UI (push) Successful in 48s
Release / Test backend (push) Successful in 50s
Release / Check ui (push) Successful in 55s
CI / UI (pull_request) Successful in 37s
Release / Docker / caddy (push) Successful in 47s
CI / Backend (pull_request) Successful in 46s
Release / Docker / runner (push) Successful in 2m37s
Release / Docker / ui (push) Successful in 2m42s
Release / Docker / backend (push) Successful in 3m13s
Release / Gitea Release (push) Failing after 2s
novelfire.net changed its book page structure. Old selectors produced empty status and null genres for every book, causing all Meilisearch filters to return zero results. Old → new: - status: <span class="status"> → <strong class="ongoing|completed|hiatus"> (text lowercased for consistent index values) - genres: <div class="genres"> <a> → <div class="categories"> <a class="property-item"> (text lowercased for consistent index values) Adds TestParseMetadataSelectors to guard against future regressions.
181 lines
4.9 KiB
Go
181 lines
4.9 KiB
Go
package novelfire
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"testing"
|
|
)
|
|
|
|
func TestSlugFromURL(t *testing.T) {
|
|
cases := []struct {
|
|
url string
|
|
want string
|
|
}{
|
|
{"https://novelfire.net/book/shadow-slave", "shadow-slave"},
|
|
{"https://novelfire.net/book/a-dragon-against-the-whole-world", "a-dragon-against-the-whole-world"},
|
|
{"https://novelfire.net/book/foo/chapter-1", "foo"},
|
|
{"https://novelfire.net/", ""},
|
|
{"not-a-url", "not-a-url"},
|
|
}
|
|
for _, c := range cases {
|
|
got := slugFromURL(c.url)
|
|
if got != c.want {
|
|
t.Errorf("slugFromURL(%q) = %q, want %q", c.url, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestChapterNumberFromURL(t *testing.T) {
|
|
cases := []struct {
|
|
url string
|
|
want int
|
|
}{
|
|
{"https://novelfire.net/book/shadow-slave/chapter-42", 42},
|
|
{"https://novelfire.net/book/shadow-slave/chapter-1000", 1000},
|
|
{"https://novelfire.net/book/shadow-slave/chap-7", 7},
|
|
{"https://novelfire.net/book/shadow-slave/ch-3", 3},
|
|
{"https://novelfire.net/book/shadow-slave/42", 42},
|
|
{"https://novelfire.net/book/shadow-slave/no-number-here", 0},
|
|
{"not-a-url", 0},
|
|
}
|
|
for _, c := range cases {
|
|
got := chapterNumberFromURL(c.url)
|
|
if got != c.want {
|
|
t.Errorf("chapterNumberFromURL(%q) = %d, want %d", c.url, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseChapterCount(t *testing.T) {
|
|
cases := []struct {
|
|
in string
|
|
want int
|
|
}{
|
|
{"123 Chapters", 123},
|
|
{"1,234 Chapters", 1234},
|
|
{"0", 0},
|
|
{"", 0},
|
|
{"500", 500},
|
|
}
|
|
for _, c := range cases {
|
|
got := parseChapterCount(c.in)
|
|
if got != c.want {
|
|
t.Errorf("parseChapterCount(%q) = %d, want %d", c.in, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryGet_ContextCancellation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // cancel immediately
|
|
|
|
stub := newStubClient()
|
|
stub.setError("https://example.com/page", context.Canceled)
|
|
|
|
_, err := retryGet(ctx, nil, stub, "https://example.com/page", 3, 0)
|
|
if err == nil {
|
|
t.Fatal("expected error on cancelled context")
|
|
}
|
|
}
|
|
|
|
func TestRetryGet_EventualSuccess(t *testing.T) {
|
|
stub := newStubClient()
|
|
calls := 0
|
|
stub.setFn("https://example.com/page", func() (string, error) {
|
|
calls++
|
|
if calls < 3 {
|
|
return "", context.DeadlineExceeded
|
|
}
|
|
return "<html>ok</html>", nil
|
|
})
|
|
|
|
got, err := retryGet(context.Background(), nil, stub, "https://example.com/page", 5, 0)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got != "<html>ok</html>" {
|
|
t.Errorf("got %q, want html", got)
|
|
}
|
|
if calls != 3 {
|
|
t.Errorf("expected 3 calls, got %d", calls)
|
|
}
|
|
}
|
|
|
|
// TestParseMetadataSelectors verifies that the status and genres selectors
|
|
// match the current novelfire.net HTML structure.
|
|
func TestParseMetadataSelectors(t *testing.T) {
|
|
// Minimal HTML reproducing the relevant novelfire.net book page structure.
|
|
const html = `<!DOCTYPE html>
|
|
<html><body>
|
|
<h1 class="novel-title">Shadow Slave</h1>
|
|
<span class="author">Guiltythree</span>
|
|
<figure class="cover"><img src="https://cdn.example.com/cover.jpg"></figure>
|
|
<div class="header-stats">
|
|
<span><strong>123</strong><small>Chapters</small></span>
|
|
<span> <strong class="ongoing">Ongoing</strong> <small>Status</small></span>
|
|
</div>
|
|
<div class="categories">
|
|
<h4>Genres</h4>
|
|
<ul>
|
|
<li><a href="/genre-fantasy/..." class="property-item">Fantasy</a></li>
|
|
<li><a href="/genre-action/..." class="property-item">Action</a></li>
|
|
<li><a href="/genre-adventure/..." class="property-item">Adventure</a></li>
|
|
</ul>
|
|
</div>
|
|
<span class="chapter-count">123 Chapters</span>
|
|
</body></html>`
|
|
|
|
stub := newStubClient()
|
|
stub.setFn("https://novelfire.net/book/shadow-slave", func() (string, error) {
|
|
return html, nil
|
|
})
|
|
|
|
s := &Scraper{client: stub, log: slog.Default()}
|
|
meta, err := s.ScrapeMetadata(t.Context(), "https://novelfire.net/book/shadow-slave")
|
|
if err != nil {
|
|
t.Fatalf("ScrapeMetadata: %v", err)
|
|
}
|
|
|
|
if meta.Status != "ongoing" {
|
|
t.Errorf("status = %q, want %q", meta.Status, "ongoing")
|
|
}
|
|
|
|
wantGenres := []string{"fantasy", "action", "adventure"}
|
|
if len(meta.Genres) != len(wantGenres) {
|
|
t.Fatalf("genres = %v, want %v", meta.Genres, wantGenres)
|
|
}
|
|
for i, g := range meta.Genres {
|
|
if g != wantGenres[i] {
|
|
t.Errorf("genres[%d] = %q, want %q", i, g, wantGenres[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── minimal stub client for tests ─────────────────────────────────────────────
|
|
|
|
type stubClient struct {
|
|
errors map[string]error
|
|
fns map[string]func() (string, error)
|
|
}
|
|
|
|
func newStubClient() *stubClient {
|
|
return &stubClient{
|
|
errors: make(map[string]error),
|
|
fns: make(map[string]func() (string, error)),
|
|
}
|
|
}
|
|
|
|
func (s *stubClient) setError(u string, err error) { s.errors[u] = err }
|
|
|
|
func (s *stubClient) setFn(u string, fn func() (string, error)) { s.fns[u] = fn }
|
|
|
|
func (s *stubClient) GetContent(_ context.Context, pageURL string) (string, error) {
|
|
if fn, ok := s.fns[pageURL]; ok {
|
|
return fn()
|
|
}
|
|
if err, ok := s.errors[pageURL]; ok {
|
|
return "", err
|
|
}
|
|
return "", context.DeadlineExceeded
|
|
}
|