fix: correct ScrapeRanking DOM selectors for real novelfire.net HTML

The previous selectors were based on a hypothetical structure that does not match the actual site. The real novelfire.net popular listing uses: <li class="novel-item"> (not <div>) <a href="/book/slug" title="Title"> <figure class="novel-cover"><img data-src="/path.jpg"></figure> <h4 class="novel-title text2row">Title</h4> </a> </li> And pagination uses <a rel="next"> (not <a class="next">). Changes: - ScrapeRanking: use li.novel-item, h4.novel-title, figure.novel-cover, img[data-src]; strip base64 placeholder covers - hasNextPageLink(): new helper walking all <a> nodes for rel="next" - Import golang.org/x/net/html for Node.Attr access - Test fixtures rewritten to match real structure (li/h4/rel=next) - Status and genres removed from ranking items (not present on listing page) Verified end-to-end: ranking.json written with 24 items on first fetch
2026-03-01 21:54:11 +05:00
parent 9837c89075
commit ca33f8c3cf
2 changed files with 92 additions and 66 deletions
--- a/scraper/internal/novelfire/ranking_test.go
+++ b/scraper/internal/novelfire/ranking_test.go
@@ -14,46 +14,51 @@ import (

 // rankingPage1HTML is a realistic mock of the popular genre listing page
 // (novelfire.net/genre-all/sort-popular/status-all/all-novel?page=1).
-// It contains two novel-item cards and a "next" link for pagination tests.
+// It uses the real novelfire.net DOM: <li class="novel-item"> cards with
+// <h4 class="novel-title"> and a rel="next" pagination link.
 func rankingPage1HTML() string {
 	return `<!DOCTYPE html>
 <html><body>
-<div class="list-novel">
-  <div class="novel-item">
-    <figure class="cover"><img src="/covers/iron-throne.jpg"></figure>
-    <div class="item-body">
-      <h3 class="novel-title"><a href="/book/the-iron-throne">The Iron Throne</a></h3>
-      <span class="status">Ongoing</span>
-      <div class="genres"><a>Fantasy</a><a>Action</a></div>
-    </div>
-  </div>
-  <div class="novel-item">
-    <figure class="cover"><img src="/covers/shadow-mage.jpg"></figure>
-    <div class="item-body">
-      <h3 class="novel-title"><a href="/book/shadow-mage">Shadow Mage</a></h3>
-      <span class="status">Completed</span>
-      <div class="genres"><a>Magic</a></div>
-    </div>
-  </div>
-</div>
-<a class="next" href="/genre-all/sort-popular/status-all/all-novel?page=2">Next</a>
+<ul class="list-novel">
+  <li class="novel-item">
+    <a title="The Iron Throne" href="/book/the-iron-throne">
+      <figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/iron-throne.jpg" alt="The Iron Throne"></figure>
+      <h4 class="novel-title text2row">The Iron Throne</h4>
+    </a>
+    <div class="novel-stats"><i class="icon-book-open"></i> 500 Chapters</div>
+  </li>
+  <li class="novel-item">
+    <a title="Shadow Mage" href="/book/shadow-mage">
+      <figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/shadow-mage.jpg" alt="Shadow Mage"></figure>
+      <h4 class="novel-title text2row">Shadow Mage</h4>
+    </a>
+    <div class="novel-stats"><i class="icon-book-open"></i> 200 Chapters</div>
+  </li>
+</ul>
+<ul class="pagination">
+  <li class="page-item active"><span class="page-link">1</span></li>
+  <li class="page-item"><a class="page-link" href="/genre-all/sort-popular/status-all/all-novel?page=2" rel="next" aria-label="Next">›</a></li>
+</ul>
 </body></html>`
 }

 func rankingPage2HTML() string {
 	return `<!DOCTYPE html>
 <html><body>
-<div class="list-novel">
-  <div class="novel-item">
-    <figure class="cover"><img src="/covers/void-hunter.jpg"></figure>
-    <div class="item-body">
-      <h3 class="novel-title"><a href="/book/void-hunter">Void Hunter</a></h3>
-      <span class="status">Ongoing</span>
-      <div class="genres"><a>Sci-Fi</a></div>
-    </div>
-  </div>
-</div>
-<!-- no .next link → last page -->
+<ul class="list-novel">
+  <li class="novel-item">
+    <a title="Void Hunter" href="/book/void-hunter">
+      <figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/void-hunter.jpg" alt="Void Hunter"></figure>
+      <h4 class="novel-title text2row">Void Hunter</h4>
+    </a>
+    <div class="novel-stats"><i class="icon-book-open"></i> 100 Chapters</div>
+  </li>
+</ul>
+<!-- no rel="next" link → last page -->
+<ul class="pagination">
+  <li class="page-item"><a class="page-link" href="?page=1" rel="prev">‹</a></li>
+  <li class="page-item active"><span class="page-link">2</span></li>
+</ul>
 </body></html>`
 }

--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -21,6 +21,7 @@ import (
 	"github.com/libnovel/scraper/internal/browser"
 	"github.com/libnovel/scraper/internal/scraper"
 	"github.com/libnovel/scraper/internal/scraper/htmlutil"
+	"golang.org/x/net/html"
 )

 const (
@@ -367,6 +368,22 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra

 // ─── RankingProvider ───────────────────────────────────────────────────────────

+// hasNextPageLink returns true if the HTML document contains a pagination link
+// with rel="next". novelfire.net uses:
+//
+//	<a class="page-link" href="...?page=N" rel="next" ...>
+func hasNextPageLink(root *html.Node) bool {
+	links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
+	for _, a := range links {
+		for _, attr := range a.Attr {
+			if attr.Key == "rel" && attr.Val == "next" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // ScrapeRanking pages through up to maxPages pages of the popular-novels genre
 // listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel).
 // Pages are fetched one at a time, strictly sequentially.
@@ -430,58 +447,62 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
 				return
 			}

-			// Genre listing uses div.novel-item cards (same structure as catalogue).
-			cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
+			// Real novelfire.net popular listing structure:
+			//   <li class="novel-item">
+			//     <a href="/book/slug" title="Title">
+			//       <figure class="novel-cover"><img data-src="..."></figure>
+			//       <h4 class="novel-title text2row">Title</h4>
+			//     </a>
+			//   </li>
+			cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
 			if len(cards) == 0 {
 				s.log.Debug("no novel cards found, stopping pagination", "page", page)
 				break
 			}

 			for _, card := range cards {
-				// Cover: <figure class="cover"><img src="..." or data-src="...">
+				// The outer <a> carries the href and title attribute.
+				linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
+				if linkNode == nil {
+					continue
+				}
+				href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
+				bookURL := resolveURL(baseURL, href)
+				if bookURL == "" {
+					continue
+				}
+
+				// Title: prefer <h4 class="novel-title"> text; fall back to <a title="...">
+				title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
+				if title == "" {
+					title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
+				}
+				if title == "" {
+					continue
+				}
+
+				// Cover: <figure class="novel-cover"><img data-src="...">
 				var cover string
-				if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
-					cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
+				if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
+					cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
 					if cover == "" {
-						cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
+						cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
+					}
+					// Filter out base64 placeholder images.
+					if strings.HasPrefix(cover, "data:") {
+						cover = ""
 					}
 					if cover != "" && !strings.HasPrefix(cover, "http") {
 						cover = baseURL + cover
 					}
 				}

-				// Title and URL: <h3 class="novel-title"><a href="/book/slug">Title</a></h3>
-				titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
-				var title, bookURL string
-				if titleNode != nil {
-					linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
-					if linkNode != nil {
-						title = htmlutil.ExtractText(linkNode, scraper.Selector{})
-						href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
-						bookURL = resolveURL(baseURL, href)
-					}
-				}
-				if title == "" || bookURL == "" {
-					continue
-				}
-
-				// Status: <span class="status">Ongoing</span>
-				status := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "span", Class: "status"}))
-
-				// Genres: <div class="genres"><a>Genre</a>...
-				var genres []string
-				if genresNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "div", Class: "genres"}); genresNode != nil {
-					genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
-				}
-
 				slug := slugFromURL(bookURL)

 				meta := scraper.BookMeta{
 					Slug:      slug,
 					Title:     title,
 					Cover:     cover,
-					Status:    status,
-					Genres:    genres,
 					SourceURL: bookURL,
 					Ranking:   rank,
 				}
@@ -494,9 +515,9 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
 				}
 			}

-			// Stop if no next-page link exists (natural end of listing).
-			nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
-			if nextHref == "" {
+			// Stop if no next-page link exists.
+			// The real pagination uses <a rel="next" ...> inside .pagination.
+			if !hasNextPageLink(root) {
 				s.log.Debug("no next-page link found, stopping pagination", "page", page)
 				break
 			}