feat: paginated ranking scrape with lazy page fetching

- ScrapeRanking now accepts a maxPages int parameter (0 = all pages).
  Each page is fetched strictly sequentially; the next page is only
  requested after every entry from the current page has been sent,
  so there is no pre-fetching or look-ahead.
  Pagination stops automatically when no next-page link is present
  or when the rank-novels container is absent/empty.

- The ranking URL pattern follows the existing catalogue convention:
  /ranking?page=N (next-page link detection as the stop condition).

- Server: handleRankingRefresh reads an optional 'pages' form field
  and passes it to ScrapeRanking. Timeout scales at 90 s/page.

- UI: Refresh Rankings button is now a small form with a numeric
  'Pages' input (default 1), letting the user choose how many pages
  to pull in one refresh without touching the server config.
This commit is contained in:
Admin
2026-03-01 16:54:52 +05:00
parent e9f880f7f7
commit 1469e49190
3 changed files with 61 additions and 20 deletions

View File

@@ -222,13 +222,21 @@ const rankingTmpl = `
class="text-sm px-3 py-1.5 rounded-lg bg-zinc-700 hover:bg-zinc-600 text-white inline-flex items-center gap-1">
View Markdown
</a>
<button
<form
hx-post="/ranking/refresh"
hx-target="#ranking-refresh-status"
hx-swap="innerHTML"
class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
Refresh Rankings
</button>
class="flex items-center gap-2">
<label class="flex items-center gap-1.5">
<span class="text-xs text-zinc-400 whitespace-nowrap">Pages</span>
<input type="number" name="pages" value="1" min="1"
class="w-16 rounded-lg bg-zinc-800 border border-zinc-700 px-2 py-1 text-sm text-zinc-100 text-center focus:outline-none focus:border-amber-500 transition-colors" />
</label>
<button type="submit"
class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
Refresh Rankings
</button>
</form>
</div>
</div>
<div id="ranking-refresh-status" class="mt-2"></div>
@@ -353,7 +361,18 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
// immediately returns a polling badge. The browser polls /ui/ranking/status
// until the job finishes, then follows an HX-Redirect back to /ranking.
//
// Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means
// fetch all pages; otherwise at most that many pages are scraped.
func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
_ = r.ParseForm()
maxPages := 0
if p := strings.TrimSpace(r.FormValue("pages")); p != "" {
if n, err := strconv.Atoi(p); err == nil && n > 0 {
maxPages = n
}
}
s.mu.Lock()
if s.rankingRunning {
s.mu.Unlock()
@@ -370,10 +389,15 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
s.mu.Unlock()
}()
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
// Allow ~90 s per page; minimum 120 s for a single page.
timeout := 120 * time.Second
if maxPages > 1 {
timeout = time.Duration(maxPages) * 90 * time.Second
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
rankingCh, errCh := s.novel.ScrapeRanking(ctx)
rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages)
var rankingItems []writer.RankingItem
for {