feat: paginated ranking scrape with lazy page fetching

- ScrapeRanking now accepts a maxPages int parameter (0 = all pages).
  Each page is fetched strictly sequentially; the next page is only
  requested after every entry from the current page has been sent,
  so there is no pre-fetching or look-ahead.
  Pagination stops automatically when no next-page link is present
  or when the rank-novels container is absent/empty.

- The ranking URL pattern follows the existing catalogue convention:
  /ranking?page=N (next-page link detection as the stop condition).

- Server: handleRankingRefresh reads an optional 'pages' form field
  and passes it to ScrapeRanking. Timeout scales at 90 s/page.

- UI: Refresh Rankings button is now a small form with a numeric
  'Pages' input (default 1), letting the user choose how many pages
  to pull in one refresh without touching the server config.
This commit is contained in:
Admin
2026-03-01 16:54:52 +05:00
parent e9f880f7f7
commit 1469e49190
3 changed files with 61 additions and 20 deletions

View File

@@ -365,25 +365,29 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
// ─── RankingProvider ───────────────────────────────────────────────────────────
func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <-chan error) {
entries := make(chan scraper.BookMeta, 64)
// ScrapeRanking pages through up to maxPages ranking pages on novelfire.net/ranking.
// Pages are fetched one at a time, strictly sequentially: the next page is only
// requested after every entry from the current page has been sent to the channel.
// maxPages <= 0 means "fetch all pages until no more are found".
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
entries := make(chan scraper.BookMeta, 32)
errs := make(chan error, 16)
go func() {
defer close(entries)
defer close(errs)
pageURL := baseURL + rankingPath
rank := 1
for pageURL != "" {
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
select {
case <-ctx.Done():
return
default:
}
s.log.Info("scraping ranking page", "url", pageURL)
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
s.log.Info("scraping ranking page", "page", page, "url", pageURL)
// Always use the urlClient for the ranking page — it requires JS rendering.
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
@@ -394,23 +398,29 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
BestAttempt: true,
})
if err != nil {
s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
errs <- fmt.Errorf("ranking page: %w", err)
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
errs <- fmt.Errorf("ranking page %d: %w", page, err)
return
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
errs <- fmt.Errorf("ranking page parse: %w", err)
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
return
}
rankList := htmlutil.FindFirst(root, scraper.Selector{Class: "rank-novels"})
if rankList == nil {
s.log.Debug("rank-novels container not found, stopping pagination", "page", page)
break
}
items := htmlutil.FindAll(rankList, scraper.Selector{Tag: "li", Class: "novel-item"})
if len(items) == 0 {
s.log.Debug("no ranking items on page, stopping pagination", "page", page)
break
}
for _, item := range items {
// Cover: <figure class="cover"><a href="/book/slug"><img data-src="..."></a></figure>
var cover string
@@ -436,7 +446,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
// Status: <span class="status"> Ongoing/Completed </span>
status := htmlutil.ExtractFirst(item, scraper.Selector{Tag: "span", Class: "status"})
// Genres: <div class="categories"><div class="scroll"><span>Genre1</span><span>Genre2</span>...</div></div>
// Genres: <div class="categories"><div class="scroll"><span>Genre1</span>...</div></div>
var genres []string
categoriesNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "div", Class: "categories"})
if categoriesNode != nil {
@@ -463,8 +473,12 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
}
}
// Next page - ranking pages use different pagination, just get first page for now
break
// Stop if no next-page link exists (natural end of ranking list).
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
if nextHref == "" {
s.log.Debug("no next-page link found, stopping pagination", "page", page)
break
}
}
}()