feat: paginated ranking scrape with lazy page fetching
- ScrapeRanking now accepts a maxPages int parameter (0 = all pages). Each page is fetched strictly sequentially; the next page is only requested after every entry from the current page has been sent, so there is no pre-fetching or look-ahead. Pagination stops automatically when no next-page link is present or when the rank-novels container is absent/empty. - The ranking URL pattern follows the existing catalogue convention: /ranking?page=N (next-page link detection as the stop condition). - Server: handleRankingRefresh reads an optional 'pages' form field and passes it to ScrapeRanking. Timeout scales at 90 s/page. - UI: Refresh Rankings button is now a small form with a numeric 'Pages' input (default 1), letting the user choose how many pages to pull in one refresh without touching the server config.
This commit is contained in:
@@ -365,25 +365,29 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
|
||||
|
||||
// ─── RankingProvider ───────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <-chan error) {
|
||||
entries := make(chan scraper.BookMeta, 64)
|
||||
// ScrapeRanking pages through up to maxPages ranking pages on novelfire.net/ranking.
|
||||
// Pages are fetched one at a time, strictly sequentially: the next page is only
|
||||
// requested after every entry from the current page has been sent to the channel.
|
||||
// maxPages <= 0 means "fetch all pages until no more are found".
|
||||
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
entries := make(chan scraper.BookMeta, 32)
|
||||
errs := make(chan error, 16)
|
||||
|
||||
go func() {
|
||||
defer close(entries)
|
||||
defer close(errs)
|
||||
|
||||
pageURL := baseURL + rankingPath
|
||||
rank := 1
|
||||
|
||||
for pageURL != "" {
|
||||
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
s.log.Info("scraping ranking page", "url", pageURL)
|
||||
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
||||
s.log.Info("scraping ranking page", "page", page, "url", pageURL)
|
||||
|
||||
// Always use the urlClient for the ranking page — it requires JS rendering.
|
||||
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
||||
@@ -394,23 +398,29 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
|
||||
BestAttempt: true,
|
||||
})
|
||||
if err != nil {
|
||||
s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
|
||||
errs <- fmt.Errorf("ranking page: %w", err)
|
||||
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
|
||||
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("ranking page parse: %w", err)
|
||||
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
rankList := htmlutil.FindFirst(root, scraper.Selector{Class: "rank-novels"})
|
||||
if rankList == nil {
|
||||
s.log.Debug("rank-novels container not found, stopping pagination", "page", page)
|
||||
break
|
||||
}
|
||||
|
||||
items := htmlutil.FindAll(rankList, scraper.Selector{Tag: "li", Class: "novel-item"})
|
||||
if len(items) == 0 {
|
||||
s.log.Debug("no ranking items on page, stopping pagination", "page", page)
|
||||
break
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
// Cover: <figure class="cover"><a href="/book/slug"><img data-src="..."></a></figure>
|
||||
var cover string
|
||||
@@ -436,7 +446,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
|
||||
// Status: <span class="status"> Ongoing/Completed </span>
|
||||
status := htmlutil.ExtractFirst(item, scraper.Selector{Tag: "span", Class: "status"})
|
||||
|
||||
// Genres: <div class="categories"><div class="scroll"><span>Genre1</span><span>Genre2</span>...</div></div>
|
||||
// Genres: <div class="categories"><div class="scroll"><span>Genre1</span>...</div></div>
|
||||
var genres []string
|
||||
categoriesNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "div", Class: "categories"})
|
||||
if categoriesNode != nil {
|
||||
@@ -463,8 +473,12 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
|
||||
}
|
||||
}
|
||||
|
||||
// Next page - ranking pages use different pagination, just get first page for now
|
||||
break
|
||||
// Stop if no next-page link exists (natural end of ranking list).
|
||||
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
||||
if nextHref == "" {
|
||||
s.log.Debug("no next-page link found, stopping pagination", "page", page)
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user