From d14644238f56afbb26b435d7fbce299a94cf979b Mon Sep 17 00:00:00 2001
From: Admin <admin@kalekber.cc>
Date: Wed, 4 Mar 2026 16:54:00 +0500
Subject: [PATCH] fix(scraper): fix ScrapeCatalogue selectors, zero-byte cache
 skip, and wire ScrapeRanking into runAsync

- ScrapeCatalogue: use li.novel-item (was div), extract href from outer <a> and title from h4.novel-title (was h3), detect next page via rel=next (was class=next)
- handleBrowse/triggerBrowseSnapshot: treat zero-byte MinIO cache entries as misses, skip storing empty SingleFile output
- runAsync: after a successful full-catalogue run, call ScrapeRanking and upsert each result into PocketBase ranking collection
---
 scraper/internal/novelfire/scraper.go | 45 ++++++++++++++++++++-------
 scraper/internal/server/server.go     | 37 +++++++++++++++++++++-
 2 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go
index 10b55f1..720752d 100644
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -142,24 +142,28 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 				return
 			}
 
-			// Extract novel cards: <div class="novel-item">
-			cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
+			// Extract novel cards: <li class="novel-item">
+			//   <a href="/book/slug" title="Title">
+			//     <figure class="novel-cover"><img data-src="..."></figure>
+			//     <h4 class="novel-title text2row">Title</h4>
+			//   </a>
+			cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
 			if len(cards) == 0 {
 				s.log.Warn("no novel cards found, stopping pagination", "page", page)
 				return
 			}
 
 			for _, card := range cards {
-				// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
-				titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
+				// The outer <a> carries the href; <h4 class="novel-title"> has the title text.
+				linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
+				titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
 
 				var title, href string
+				if linkNode != nil {
+					href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
+				}
 				if titleNode != nil {
-					linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
-					if linkNode != nil {
-						title = htmlutil.ExtractText(linkNode, scraper.Selector{})
-						href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
-					}
+					title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
 				}
 				if href == "" || title == "" {
 					continue
@@ -173,8 +177,27 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 				}
 			}
 
-			// Find next page link: <a class="next" href="...">
-			nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
+			// Find next page link: <a rel="next" href="..."> (same structure as ranking pages)
+			if !hasNextPageLink(root) {
+				break
+			}
+			nextHref := ""
+			for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
+				isNext := false
+				for _, attr := range a.Attr {
+					if attr.Key == "rel" && attr.Val == "next" {
+						isNext = true
+					}
+				}
+				if isNext {
+					for _, attr := range a.Attr {
+						if attr.Key == "href" {
+							nextHref = attr.Val
+						}
+					}
+					break
+				}
+			}
 			if nextHref == "" {
 				break
 			}
diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go
index 9e6e173..f461106 100644
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -829,6 +829,36 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
 		o := orchestrator.New(cfg, s.novel, s.log, s.store)
 		runErr := o.Run(ctx)
 
+		// After a successful full-catalogue run, refresh the ranking list.
+		if runErr == nil && cfg.SingleBookURL == "" {
+			s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
+			rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
+			defer rankCancel()
+			rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
+			rank := 1
+			for meta := range rankEntries {
+				item := storage.RankingItem{
+					Rank:      rank,
+					Slug:      meta.Slug,
+					Title:     meta.Title,
+					Author:    meta.Author,
+					Cover:     meta.Cover,
+					Status:    meta.Status,
+					Genres:    meta.Genres,
+					SourceURL: meta.SourceURL,
+				}
+				if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
+					s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
+				}
+				rank++
+			}
+			if rerr := <-rankErrs; rerr != nil {
+				s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
+			} else {
+				s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
+			}
+		}
+
 		// Determine final status.
 		finalStatus := "done"
 		errMsg := ""
@@ -938,7 +968,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 
 	// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
 	cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
-	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
+	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
 		novels, hasNext := parseBrowsePage(strings.NewReader(html))
 		s.log.Debug("browse: served from cache", "key", cacheKey)
 		w.Header().Set("Content-Type", "application/json")
@@ -1062,6 +1092,11 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
 				"key", cacheKey, "err", readErr)
 			return
 		}
+		if len(htmlBytes) == 0 {
+			s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache",
+				"key", cacheKey)
+			return
+		}
 
 		// Store the HTML snapshot.
 		if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {