diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index 2edd35e..a1fbd04 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -365,25 +365,29 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra // ─── RankingProvider ─────────────────────────────────────────────────────────── -func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <-chan error) { - entries := make(chan scraper.BookMeta, 64) +// ScrapeRanking pages through up to maxPages ranking pages on novelfire.net/ranking. +// Pages are fetched one at a time, strictly sequentially: the next page is only +// requested after every entry from the current page has been sent to the channel. +// maxPages <= 0 means "fetch all pages until no more are found". +func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) { + entries := make(chan scraper.BookMeta, 32) errs := make(chan error, 16) go func() { defer close(entries) defer close(errs) - pageURL := baseURL + rankingPath rank := 1 - for pageURL != "" { + for page := 1; maxPages <= 0 || page <= maxPages; page++ { select { case <-ctx.Done(): return default: } - s.log.Info("scraping ranking page", "url", pageURL) + pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page) + s.log.Info("scraping ranking page", "page", page, "url", pageURL) // Always use the urlClient for the ranking page — it requires JS rendering. raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ @@ -394,23 +398,29 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, < BestAttempt: true, }) if err != nil { - s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err) - errs <- fmt.Errorf("ranking page: %w", err) + s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err) + errs <- fmt.Errorf("ranking page %d: %w", page, err) return } root, err := htmlutil.ParseHTML(raw) if err != nil { - errs <- fmt.Errorf("ranking page parse: %w", err) + errs <- fmt.Errorf("ranking page %d parse: %w", page, err) return } rankList := htmlutil.FindFirst(root, scraper.Selector{Class: "rank-novels"}) if rankList == nil { + s.log.Debug("rank-novels container not found, stopping pagination", "page", page) break } items := htmlutil.FindAll(rankList, scraper.Selector{Tag: "li", Class: "novel-item"}) + if len(items) == 0 { + s.log.Debug("no ranking items on page, stopping pagination", "page", page) + break + } + for _, item := range items { // Cover:
var cover string @@ -436,7 +446,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, < // Status: Ongoing/Completed status := htmlutil.ExtractFirst(item, scraper.Selector{Tag: "span", Class: "status"}) - // Genres:
Genre1Genre2...
+ // Genres:
Genre1...
var genres []string categoriesNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "div", Class: "categories"}) if categoriesNode != nil { @@ -463,8 +473,12 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, < } } - // Next page - ranking pages use different pagination, just get first page for now - break + // Stop if no next-page link exists (natural end of ranking list). + nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"}) + if nextHref == "" { + s.log.Debug("no next-page link found, stopping pagination", "page", page) + break + } } }() diff --git a/scraper/internal/scraper/interfaces.go b/scraper/internal/scraper/interfaces.go index d98ec6a..7799476 100644 --- a/scraper/internal/scraper/interfaces.go +++ b/scraper/internal/scraper/interfaces.go @@ -112,9 +112,12 @@ type ChapterTextProvider interface { // RankingProvider can enumerate novels from a ranking page. type RankingProvider interface { - // ScrapeRanking pages through the ranking list, sending BookMeta values - // (with basic info like title, cover, genres, status, sourceURL) to the returned channel. - ScrapeRanking(ctx context.Context) (<-chan BookMeta, <-chan error) + // ScrapeRanking pages through up to maxPages ranking pages, sending BookMeta + // values (with basic info like title, cover, genres, status, sourceURL) to + // the returned channel. Pages are fetched sequentially and lazily: the next + // page is only requested once all entries from the current page have been + // sent. maxPages <= 0 means "all pages". + ScrapeRanking(ctx context.Context, maxPages int) (<-chan BookMeta, <-chan error) } // NovelScraper is the full interface that a concrete novel source must implement. diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index 4bf24e2..a56df51 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -222,13 +222,21 @@ const rankingTmpl = ` class="text-sm px-3 py-1.5 rounded-lg bg-zinc-700 hover:bg-zinc-600 text-white inline-flex items-center gap-1"> View Markdown - + class="flex items-center gap-2"> + + +
@@ -353,7 +361,18 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { // handleRankingRefresh starts an async scrape of novelfire.net/ranking and // immediately returns a polling badge. The browser polls /ui/ranking/status // until the job finishes, then follows an HX-Redirect back to /ranking. +// +// Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means +// fetch all pages; otherwise at most that many pages are scraped. func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + maxPages := 0 + if p := strings.TrimSpace(r.FormValue("pages")); p != "" { + if n, err := strconv.Atoi(p); err == nil && n > 0 { + maxPages = n + } + } + s.mu.Lock() if s.rankingRunning { s.mu.Unlock() @@ -370,10 +389,15 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { s.mu.Unlock() }() - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + // Allow ~90 s per page; minimum 120 s for a single page. + timeout := 120 * time.Second + if maxPages > 1 { + timeout = time.Duration(maxPages) * 90 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - rankingCh, errCh := s.novel.ScrapeRanking(ctx) + rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) var rankingItems []writer.RankingItem for {