diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go
index 2edd35e..a1fbd04 100644
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -365,25 +365,29 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
// ─── RankingProvider ───────────────────────────────────────────────────────────
-func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <-chan error) {
- entries := make(chan scraper.BookMeta, 64)
+// ScrapeRanking pages through up to maxPages ranking pages on novelfire.net/ranking.
+// Pages are fetched one at a time, strictly sequentially: the next page is only
+// requested after every entry from the current page has been sent to the channel.
+// maxPages <= 0 means "fetch all pages until no more are found".
+func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
+ entries := make(chan scraper.BookMeta, 32)
errs := make(chan error, 16)
go func() {
defer close(entries)
defer close(errs)
- pageURL := baseURL + rankingPath
rank := 1
- for pageURL != "" {
+ for page := 1; maxPages <= 0 || page <= maxPages; page++ {
select {
case <-ctx.Done():
return
default:
}
- s.log.Info("scraping ranking page", "url", pageURL)
+ pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
+ s.log.Info("scraping ranking page", "page", page, "url", pageURL)
// Always use the urlClient for the ranking page — it requires JS rendering.
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
@@ -394,23 +398,29 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
BestAttempt: true,
})
if err != nil {
- s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
- errs <- fmt.Errorf("ranking page: %w", err)
+ s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
+ errs <- fmt.Errorf("ranking page %d: %w", page, err)
return
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
- errs <- fmt.Errorf("ranking page parse: %w", err)
+ errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
return
}
rankList := htmlutil.FindFirst(root, scraper.Selector{Class: "rank-novels"})
if rankList == nil {
+ s.log.Debug("rank-novels container not found, stopping pagination", "page", page)
break
}
items := htmlutil.FindAll(rankList, scraper.Selector{Tag: "li", Class: "novel-item"})
+ if len(items) == 0 {
+ s.log.Debug("no ranking items on page, stopping pagination", "page", page)
+ break
+ }
+
for _, item := range items {
// Cover:
var cover string
@@ -436,7 +446,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
// Status: Ongoing/Completed
status := htmlutil.ExtractFirst(item, scraper.Selector{Tag: "span", Class: "status"})
- // Genres:
Genre1Genre2...
+ // Genres:
Genre1...
var genres []string
categoriesNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "div", Class: "categories"})
if categoriesNode != nil {
@@ -463,8 +473,12 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
}
}
- // Next page - ranking pages use different pagination, just get first page for now
- break
+ // Stop if no next-page link exists (natural end of ranking list).
+ nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
+ if nextHref == "" {
+ s.log.Debug("no next-page link found, stopping pagination", "page", page)
+ break
+ }
}
}()
diff --git a/scraper/internal/scraper/interfaces.go b/scraper/internal/scraper/interfaces.go
index d98ec6a..7799476 100644
--- a/scraper/internal/scraper/interfaces.go
+++ b/scraper/internal/scraper/interfaces.go
@@ -112,9 +112,12 @@ type ChapterTextProvider interface {
// RankingProvider can enumerate novels from a ranking page.
type RankingProvider interface {
- // ScrapeRanking pages through the ranking list, sending BookMeta values
- // (with basic info like title, cover, genres, status, sourceURL) to the returned channel.
- ScrapeRanking(ctx context.Context) (<-chan BookMeta, <-chan error)
+ // ScrapeRanking pages through up to maxPages ranking pages, sending BookMeta
+ // values (with basic info like title, cover, genres, status, sourceURL) to
+ // the returned channel. Pages are fetched sequentially and lazily: the next
+ // page is only requested once all entries from the current page have been
+ // sent. maxPages <= 0 means "all pages".
+ ScrapeRanking(ctx context.Context, maxPages int) (<-chan BookMeta, <-chan error)
}
// NovelScraper is the full interface that a concrete novel source must implement.
diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go
index 4bf24e2..a56df51 100644
--- a/scraper/internal/server/ui.go
+++ b/scraper/internal/server/ui.go
@@ -222,13 +222,21 @@ const rankingTmpl = `
class="text-sm px-3 py-1.5 rounded-lg bg-zinc-700 hover:bg-zinc-600 text-white inline-flex items-center gap-1">
View Markdown
-
+ class="flex items-center gap-2">
+
+
+
@@ -353,7 +361,18 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
// immediately returns a polling badge. The browser polls /ui/ranking/status
// until the job finishes, then follows an HX-Redirect back to /ranking.
+//
+// Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means
+// fetch all pages; otherwise at most that many pages are scraped.
func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
+ _ = r.ParseForm()
+ maxPages := 0
+ if p := strings.TrimSpace(r.FormValue("pages")); p != "" {
+ if n, err := strconv.Atoi(p); err == nil && n > 0 {
+ maxPages = n
+ }
+ }
+
s.mu.Lock()
if s.rankingRunning {
s.mu.Unlock()
@@ -370,10 +389,15 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
s.mu.Unlock()
}()
- ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+ // Allow ~90 s per page; minimum 120 s for a single page.
+ timeout := 120 * time.Second
+ if maxPages > 1 {
+ timeout = time.Duration(maxPages) * 90 * time.Second
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
- rankingCh, errCh := s.novel.ScrapeRanking(ctx)
+ rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages)
var rankingItems []writer.RankingItem
for {