feat: paginated ranking scrape with lazy page fetching
- ScrapeRanking now accepts a maxPages int parameter (0 = all pages). Each page is fetched strictly sequentially; the next page is only requested after every entry from the current page has been sent, so there is no pre-fetching or look-ahead. Pagination stops automatically when no next-page link is present or when the rank-novels container is absent/empty. - The ranking URL pattern follows the existing catalogue convention: /ranking?page=N (next-page link detection as the stop condition). - Server: handleRankingRefresh reads an optional 'pages' form field and passes it to ScrapeRanking. Timeout scales at 90 s/page. - UI: Refresh Rankings button is now a small form with a numeric 'Pages' input (default 1), letting the user choose how many pages to pull in one refresh without touching the server config.
This commit is contained in:
@@ -222,13 +222,21 @@ const rankingTmpl = `
|
||||
class="text-sm px-3 py-1.5 rounded-lg bg-zinc-700 hover:bg-zinc-600 text-white inline-flex items-center gap-1">
|
||||
View Markdown
|
||||
</a>
|
||||
<button
|
||||
<form
|
||||
hx-post="/ranking/refresh"
|
||||
hx-target="#ranking-refresh-status"
|
||||
hx-swap="innerHTML"
|
||||
class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
|
||||
Refresh Rankings
|
||||
</button>
|
||||
class="flex items-center gap-2">
|
||||
<label class="flex items-center gap-1.5">
|
||||
<span class="text-xs text-zinc-400 whitespace-nowrap">Pages</span>
|
||||
<input type="number" name="pages" value="1" min="1"
|
||||
class="w-16 rounded-lg bg-zinc-800 border border-zinc-700 px-2 py-1 text-sm text-zinc-100 text-center focus:outline-none focus:border-amber-500 transition-colors" />
|
||||
</label>
|
||||
<button type="submit"
|
||||
class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
|
||||
Refresh Rankings
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div id="ranking-refresh-status" class="mt-2"></div>
|
||||
@@ -353,7 +361,18 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
|
||||
// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
|
||||
// immediately returns a polling badge. The browser polls /ui/ranking/status
|
||||
// until the job finishes, then follows an HX-Redirect back to /ranking.
|
||||
//
|
||||
// Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means
|
||||
// fetch all pages; otherwise at most that many pages are scraped.
|
||||
func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
maxPages := 0
|
||||
if p := strings.TrimSpace(r.FormValue("pages")); p != "" {
|
||||
if n, err := strconv.Atoi(p); err == nil && n > 0 {
|
||||
maxPages = n
|
||||
}
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if s.rankingRunning {
|
||||
s.mu.Unlock()
|
||||
@@ -370,10 +389,15 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
// Allow ~90 s per page; minimum 120 s for a single page.
|
||||
timeout := 120 * time.Second
|
||||
if maxPages > 1 {
|
||||
timeout = time.Duration(maxPages) * 90 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
rankingCh, errCh := s.novel.ScrapeRanking(ctx)
|
||||
rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages)
|
||||
|
||||
var rankingItems []writer.RankingItem
|
||||
for {
|
||||
|
||||
Reference in New Issue
Block a user