fix(scraper): update status and genres selectors for current novelfire.net HTML

novelfire.net changed its book page structure. Old selectors produced empty status and null genres for every book, causing all Meilisearch filters to return zero results. Old → new: - status: <span class="status"> → <strong class="ongoing|completed|hiatus"> (text lowercased for consistent index values) - genres: <div class="genres"> <a> → <div class="categories"> <a class="property-item"> (text lowercased for consistent index values) Adds TestParseMetadataSelectors to guard against future regressions.
2026-03-28 22:54:35 +05:00
parent 040072c3f5
commit 801928aadf
2 changed files with 69 additions and 4 deletions
--- a/backend/internal/novelfire/scraper.go
+++ b/backend/internal/novelfire/scraper.go
@@ -178,12 +178,26 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
 		}
 	}

-	status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
+	// Status: novelfire renders <strong class="ongoing">Ongoing</strong> (or
+	// "completed", "hiatus") inside the .header-stats block. We take the text
+	// content and lowercase it so the index value is always canonical lowercase.
+	var status string
+	for _, cls := range []string{"ongoing", "completed", "hiatus"} {
+		if v := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "strong", Class: cls}); v != "" {
+			status = strings.ToLower(strings.TrimSpace(v))
+			break
+		}
+	}

-	genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
+	// Genres: novelfire renders <div class="categories"><ul><li><a class="property-item">Genre</a>
+	// Each <a class="property-item"> is one genre tag. Lowercase for index consistency.
 	var genres []string
-	if genresNode != nil {
-		genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
+	if categoriesNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "categories"}); categoriesNode != nil {
+		for _, v := range htmlutil.ExtractAll(categoriesNode, scraper.Selector{Tag: "a", Class: "property-item", Multiple: true}) {
+			if v != "" {
+				genres = append(genres, strings.ToLower(strings.TrimSpace(v)))
+			}
+		}
 	}

 	summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
--- a/backend/internal/novelfire/scraper_test.go
+++ b/backend/internal/novelfire/scraper_test.go
@@ -2,6 +2,7 @@ package novelfire

 import (
 	"context"
+	"log/slog"
 	"testing"
 )

@@ -100,6 +101,56 @@ func TestRetryGet_EventualSuccess(t *testing.T) {
 	}
 }

+// TestParseMetadataSelectors verifies that the status and genres selectors
+// match the current novelfire.net HTML structure.
+func TestParseMetadataSelectors(t *testing.T) {
+	// Minimal HTML reproducing the relevant novelfire.net book page structure.
+	const html = `<!DOCTYPE html>
+<html><body>
+<h1 class="novel-title">Shadow Slave</h1>
+<span class="author">Guiltythree</span>
+<figure class="cover"><img src="https://cdn.example.com/cover.jpg"></figure>
+<div class="header-stats">
+  <span><strong>123</strong><small>Chapters</small></span>
+  <span> <strong class="ongoing">Ongoing</strong> <small>Status</small></span>
+</div>
+<div class="categories">
+  <h4>Genres</h4>
+  <ul>
+    <li><a href="/genre-fantasy/..." class="property-item">Fantasy</a></li>
+    <li><a href="/genre-action/..." class="property-item">Action</a></li>
+    <li><a href="/genre-adventure/..." class="property-item">Adventure</a></li>
+  </ul>
+</div>
+<span class="chapter-count">123 Chapters</span>
+</body></html>`
+
+	stub := newStubClient()
+	stub.setFn("https://novelfire.net/book/shadow-slave", func() (string, error) {
+		return html, nil
+	})
+
+	s := &Scraper{client: stub, log: slog.Default()}
+	meta, err := s.ScrapeMetadata(t.Context(), "https://novelfire.net/book/shadow-slave")
+	if err != nil {
+		t.Fatalf("ScrapeMetadata: %v", err)
+	}
+
+	if meta.Status != "ongoing" {
+		t.Errorf("status = %q, want %q", meta.Status, "ongoing")
+	}
+
+	wantGenres := []string{"fantasy", "action", "adventure"}
+	if len(meta.Genres) != len(wantGenres) {
+		t.Fatalf("genres = %v, want %v", meta.Genres, wantGenres)
+	}
+	for i, g := range meta.Genres {
+		if g != wantGenres[i] {
+			t.Errorf("genres[%d] = %q, want %q", i, g, wantGenres[i])
+		}
+	}
+}
+
 // ── minimal stub client for tests ─────────────────────────────────────────────

 type stubClient struct {