fix: make ranking refresh async and use JS-rendered client for ScrapeRanking

Two bugs caused the 'Refresh Rankings' button to silently fail in production:

1. ScrapeRanking was using the plain HTTP client (s.client) instead of the
   browserless content client (s.urlClient). The /ranking page requires
   JavaScript rendering, so a plain fetch returned HTML without any novel
   entries. Now uses s.urlClient so the page is fully rendered before scraping.

2. handleRankingRefresh was synchronous, holding the HTTP connection open for
   up to 60 s while scraping. Reverse proxies and HTMX timeouts closed the
   connection before the scrape finished. Rewritten to the same async pattern
   used for book scraping: POST /ranking/refresh returns immediately with a
   polling badge; the browser polls GET /ui/ranking/status every 3 s; when
   the goroutine finishes the status endpoint sends HX-Redirect to /ranking.

scraper/internal/novelfire/scraper.go

@@ -385,23 +385,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
 
 			s.log.Info("scraping ranking page", "url", pageURL)
 
-			// Use WaitFor only for browser-based strategies
+			// Always use the urlClient for the ranking page — it requires JS rendering.
-			var raw string
+			raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
-			var err error
-			if s.client.Strategy() == browser.StrategyDirect {
-				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
-					URL:                 pageURL,
-					RejectResourceTypes: rejectResourceTypes,
-				})
-			} else {
-				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
 				URL:                 pageURL,
 				WaitFor:             &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
 				RejectResourceTypes: rejectResourceTypes,
 				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
 				BestAttempt:         true,
 			})
-			}
 			if err != nil {
 				s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
 				errs <- fmt.Errorf("ranking page: %w", err)

scraper/internal/server/server.go

@@ -31,6 +31,7 @@ type Server struct {
 	writer         *writer.Writer
 	mu             sync.Mutex
 	running        bool
+	rankingRunning bool
 	kokoroURL      string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
 	kokoroVoice    string // default voice, e.g. af_bella
 }
@@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
 	mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
 	mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
 	mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
+	mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
 	// Plain-text chapter content for browser-side TTS
 	mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
 

scraper/internal/server/ui.go

@@ -224,16 +224,14 @@ const rankingTmpl = `
       </a>
       <button
         hx-post="/ranking/refresh"
-        hx-target="#main-content"
+        hx-target="#ranking-refresh-status"
         hx-swap="innerHTML"
-        hx-indicator="#refresh-spinner"
-        hx-push-url="/ranking"
         class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
-        <span id="refresh-spinner" class="htmx-indicator animate-spin">&#8635;</span>
         Refresh Rankings
       </button>
     </div>
   </div>
+  <div id="ranking-refresh-status" class="mt-2"></div>
 
   <!-- Book grid -->
   <div class="grid gap-4 sm:grid-cols-2 mt-8">
@@ -352,10 +350,27 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
 	s.respond(w, r, "Rankings", buf.String())
 }
 
-// handleRankingRefresh triggers a live scrape of novelfire.net/ranking,
+// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
-// persists the result to ranking.md, then re-renders the ranking page.
+// immediately returns a polling badge. The browser polls /ui/ranking/status
+// until the job finishes, then follows an HX-Redirect back to /ranking.
 func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
-	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
+	s.mu.Lock()
+	if s.rankingRunning {
+		s.mu.Unlock()
+		renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…"))
+		return
+	}
+	s.rankingRunning = true
+	s.mu.Unlock()
+
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			s.rankingRunning = false
+			s.mu.Unlock()
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
 		defer cancel()
 
 		rankingCh, errCh := s.novel.ScrapeRanking(ctx)
@@ -387,7 +402,6 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
 					s.log.Error("ranking scrape error", "err", err)
 				}
 			}
-
 			if rankingCh == nil && errCh == nil {
 				break
 			}
@@ -398,19 +412,45 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
 				s.log.Error("failed to save ranking", "err", err)
 			}
 		}
+	}()
 
-	cachedAt := ""
+	renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
-	if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
-		cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
 }
 
-	t := template.Must(template.New("ranking").Parse(rankingTmpl))
+// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs.
-	var buf bytes.Buffer
+// While running it returns a self-replacing badge; when done it issues an
-	_ = t.Execute(&buf, struct {
+// HX-Redirect so the browser navigates to /ranking.
-		Books    interface{}
+func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) {
-		CachedAt string
+	s.mu.Lock()
-	}{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt})
+	running := s.rankingRunning
-	s.respond(w, r, "Rankings", buf.String())
+	s.mu.Unlock()
+
+	if running {
+		renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
+		return
+	}
+	// Job done — redirect the HTMX request to the ranking page.
+	w.Header().Set("HX-Redirect", "/ranking")
+	w.WriteHeader(http.StatusOK)
+}
+
+// rankingStatusHTML returns a self-replacing polling badge for the ranking
+// refresh job. state is "running" or "done".
+func rankingStatusHTML(state, msg string) string {
+	var colour, dot, poll string
+	switch state {
+	case "running":
+		colour = "text-amber-300 bg-amber-950 border-amber-800"
+		dot = `<span class="inline-block w-2 h-2 rounded-full bg-amber-400 animate-pulse mr-2"></span>`
+		poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"`
+	default:
+		colour = "text-green-300 bg-green-950 border-green-800"
+		dot = `<span class="inline-block w-2 h-2 rounded-full bg-green-400 mr-2"></span>`
+	}
+	return fmt.Sprintf(
+		`<div class="flex items-center text-sm px-3 py-2 rounded-lg border %s" %s>%s%s</div>`,
+		colour, poll, dot, template.HTMLEscapeString(msg),
+	)
 }
 
 // ─── GET /ranking/view — view ranking markdown ─────────────────────────────────