fix: make ranking refresh async and use JS-rendered client for ScrapeRanking

Two bugs caused the 'Refresh Rankings' button to silently fail in production:

1. ScrapeRanking was using the plain HTTP client (s.client) instead of the
   browserless content client (s.urlClient). The /ranking page requires
   JavaScript rendering, so a plain fetch returned HTML without any novel
   entries. Now uses s.urlClient so the page is fully rendered before scraping.

2. handleRankingRefresh was synchronous, holding the HTTP connection open for
   up to 60 s while scraping. Reverse proxies and HTMX timeouts closed the
   connection before the scrape finished. Rewritten to the same async pattern
   used for book scraping: POST /ranking/refresh returns immediately with a
   polling badge; the browser polls GET /ui/ranking/status every 3 s; when
   the goroutine finishes the status endpoint sends HX-Redirect to /ranking.

scraper/internal/novelfire/scraper.go

@@ -385,23 +385,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
 
 			s.log.Info("scraping ranking page", "url", pageURL)
 
-			// Use WaitFor only for browser-based strategies
+			// Always use the urlClient for the ranking page — it requires JS rendering.
-			var raw string
+			raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
-			var err error
+				URL:                 pageURL,
-			if s.client.Strategy() == browser.StrategyDirect {
+				WaitFor:             &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
-				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
+				RejectResourceTypes: rejectResourceTypes,
-					URL:                 pageURL,
+				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
-					RejectResourceTypes: rejectResourceTypes,
+				BestAttempt:         true,
-				})
+			})
-			} else {
-				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
-					URL:                 pageURL,
-					WaitFor:             &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
-					RejectResourceTypes: rejectResourceTypes,
-					GotoOptions:         &browser.GotoOptions{Timeout: 60000},
-					BestAttempt:         true,
-				})
-			}
 			if err != nil {
 				s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
 				errs <- fmt.Errorf("ranking page: %w", err)

scraper/internal/server/server.go

@@ -24,15 +24,16 @@ import (
 
 // Server wraps an HTTP mux with the scraping endpoints.
 type Server struct {
-	addr        string
+	addr           string
-	oCfg        orchestrator.Config
+	oCfg           orchestrator.Config
-	novel       scraper.NovelScraper
+	novel          scraper.NovelScraper
-	log         *slog.Logger
+	log            *slog.Logger
-	writer      *writer.Writer
+	writer         *writer.Writer
-	mu          sync.Mutex
+	mu             sync.Mutex
-	running     bool
+	running        bool
-	kokoroURL   string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
+	rankingRunning bool
-	kokoroVoice string // default voice, e.g. af_bella
+	kokoroURL      string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
+	kokoroVoice    string // default voice, e.g. af_bella
 }
 
 // New creates a new Server.
@@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
 	mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
 	mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
 	mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
+	mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
 	// Plain-text chapter content for browser-side TTS
 	mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
 

scraper/internal/server/ui.go

@@ -224,16 +224,14 @@ const rankingTmpl = `
       </a>
       <button
         hx-post="/ranking/refresh"
-        hx-target="#main-content"
+        hx-target="#ranking-refresh-status"
         hx-swap="innerHTML"
-        hx-indicator="#refresh-spinner"
-        hx-push-url="/ranking"
         class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
-        <span id="refresh-spinner" class="htmx-indicator animate-spin">&#8635;</span>
         Refresh Rankings
       </button>
     </div>
   </div>
+  <div id="ranking-refresh-status" class="mt-2"></div>
 
   <!-- Book grid -->
   <div class="grid gap-4 sm:grid-cols-2 mt-8">
@@ -352,65 +350,107 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
 	s.respond(w, r, "Rankings", buf.String())
 }
 
-// handleRankingRefresh triggers a live scrape of novelfire.net/ranking,
+// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
-// persists the result to ranking.md, then re-renders the ranking page.
+// immediately returns a polling badge. The browser polls /ui/ranking/status
+// until the job finishes, then follows an HX-Redirect back to /ranking.
 func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
-	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
+	s.mu.Lock()
-	defer cancel()
+	if s.rankingRunning {
+		s.mu.Unlock()
+		renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…"))
+		return
+	}
+	s.rankingRunning = true
+	s.mu.Unlock()
 
-	rankingCh, errCh := s.novel.ScrapeRanking(ctx)
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			s.rankingRunning = false
+			s.mu.Unlock()
+		}()
 
-	var rankingItems []writer.RankingItem
+		ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	for {
+		defer cancel()
-		select {
+
-		case meta, ok := <-rankingCh:
+		rankingCh, errCh := s.novel.ScrapeRanking(ctx)
-			if !ok {
+
-				rankingCh = nil
+		var rankingItems []writer.RankingItem
-				continue
+		for {
+			select {
+			case meta, ok := <-rankingCh:
+				if !ok {
+					rankingCh = nil
+					continue
+				}
+				rankingItems = append(rankingItems, writer.RankingItem{
+					Rank:      meta.Ranking,
+					Slug:      meta.Slug,
+					Title:     meta.Title,
+					Author:    meta.Author,
+					Cover:     meta.Cover,
+					Status:    meta.Status,
+					Genres:    meta.Genres,
+					SourceURL: meta.SourceURL,
+				})
+			case err, ok := <-errCh:
+				if !ok {
+					errCh = nil
+					continue
+				}
+				if err != nil {
+					s.log.Error("ranking scrape error", "err", err)
+				}
 			}
-			rankingItems = append(rankingItems, writer.RankingItem{
+			if rankingCh == nil && errCh == nil {
-				Rank:      meta.Ranking,
+				break
-				Slug:      meta.Slug,
-				Title:     meta.Title,
-				Author:    meta.Author,
-				Cover:     meta.Cover,
-				Status:    meta.Status,
-				Genres:    meta.Genres,
-				SourceURL: meta.SourceURL,
-			})
-		case err, ok := <-errCh:
-			if !ok {
-				errCh = nil
-				continue
-			}
-			if err != nil {
-				s.log.Error("ranking scrape error", "err", err)
 			}
 		}
 
-		if rankingCh == nil && errCh == nil {
+		if len(rankingItems) > 0 {
-			break
+			if err := s.writer.WriteRanking(rankingItems); err != nil {
+				s.log.Error("failed to save ranking", "err", err)
+			}
 		}
-	}
+	}()
 
-	if len(rankingItems) > 0 {
+	renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
-		if err := s.writer.WriteRanking(rankingItems); err != nil {
+}
-			s.log.Error("failed to save ranking", "err", err)
-		}
-	}
 
-	cachedAt := ""
+// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs.
-	if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
+// While running it returns a self-replacing badge; when done it issues an
-		cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
+// HX-Redirect so the browser navigates to /ranking.
-	}
+func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	running := s.rankingRunning
+	s.mu.Unlock()
 
-	t := template.Must(template.New("ranking").Parse(rankingTmpl))
+	if running {
-	var buf bytes.Buffer
+		renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
-	_ = t.Execute(&buf, struct {
+		return
-		Books    interface{}
+	}
-		CachedAt string
+	// Job done — redirect the HTMX request to the ranking page.
-	}{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt})
+	w.Header().Set("HX-Redirect", "/ranking")
-	s.respond(w, r, "Rankings", buf.String())
+	w.WriteHeader(http.StatusOK)
+}
+
+// rankingStatusHTML returns a self-replacing polling badge for the ranking
+// refresh job. state is "running" or "done".
+func rankingStatusHTML(state, msg string) string {
+	var colour, dot, poll string
+	switch state {
+	case "running":
+		colour = "text-amber-300 bg-amber-950 border-amber-800"
+		dot = `<span class="inline-block w-2 h-2 rounded-full bg-amber-400 animate-pulse mr-2"></span>`
+		poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"`
+	default:
+		colour = "text-green-300 bg-green-950 border-green-800"
+		dot = `<span class="inline-block w-2 h-2 rounded-full bg-green-400 mr-2"></span>`
+	}
+	return fmt.Sprintf(
+		`<div class="flex items-center text-sm px-3 py-2 rounded-lg border %s" %s>%s%s</div>`,
+		colour, poll, dot, template.HTMLEscapeString(msg),
+	)
 }
 
 // ─── GET /ranking/view — view ranking markdown ─────────────────────────────────