fix: make ranking refresh async and use JS-rendered client for ScrapeRanking

Two bugs caused the 'Refresh Rankings' button to silently fail in production: 1. ScrapeRanking was using the plain HTTP client (s.client) instead of the browserless content client (s.urlClient). The /ranking page requires JavaScript rendering, so a plain fetch returned HTML without any novel entries. Now uses s.urlClient so the page is fully rendered before scraping. 2. handleRankingRefresh was synchronous, holding the HTTP connection open for up to 60 s while scraping. Reverse proxies and HTMX timeouts closed the connection before the scrape finished. Rewritten to the same async pattern used for book scraping: POST /ranking/refresh returns immediately with a polling badge; the browser polls GET /ui/ranking/status every 3 s; when the goroutine finishes the status endpoint sends HX-Redirect to /ranking.
2026-03-01 16:52:03 +05:00
parent bcdef02997
commit e9f880f7f7
3 changed files with 111 additions and 78 deletions
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -385,23 +385,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <

 			s.log.Info("scraping ranking page", "url", pageURL)

-			// Use WaitFor only for browser-based strategies
-			var raw string
-			var err error
-			if s.client.Strategy() == browser.StrategyDirect {
-				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
-					URL:                 pageURL,
-					RejectResourceTypes: rejectResourceTypes,
-				})
-			} else {
-				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
-					URL:                 pageURL,
-					WaitFor:             &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
-					RejectResourceTypes: rejectResourceTypes,
-					GotoOptions:         &browser.GotoOptions{Timeout: 60000},
-					BestAttempt:         true,
-				})
-			}
+			// Always use the urlClient for the ranking page — it requires JS rendering.
+			raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
+				URL:                 pageURL,
+				WaitFor:             &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
+				RejectResourceTypes: rejectResourceTypes,
+				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+				BestAttempt:         true,
+			})
 			if err != nil {
 				s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
 				errs <- fmt.Errorf("ranking page: %w", err)
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -24,15 +24,16 @@ import (

 // Server wraps an HTTP mux with the scraping endpoints.
 type Server struct {
-	addr        string
-	oCfg        orchestrator.Config
-	novel       scraper.NovelScraper
-	log         *slog.Logger
-	writer      *writer.Writer
-	mu          sync.Mutex
-	running     bool
-	kokoroURL   string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
-	kokoroVoice string // default voice, e.g. af_bella
+	addr           string
+	oCfg           orchestrator.Config
+	novel          scraper.NovelScraper
+	log            *slog.Logger
+	writer         *writer.Writer
+	mu             sync.Mutex
+	running        bool
+	rankingRunning bool
+	kokoroURL      string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
+	kokoroVoice    string // default voice, e.g. af_bella
 }

 // New creates a new Server.
@@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
 	mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
 	mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
 	mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
+	mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
 	// Plain-text chapter content for browser-side TTS
 	mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)

--- a/scraper/internal/server/ui.go
+++ b/scraper/internal/server/ui.go
@@ -224,16 +224,14 @@ const rankingTmpl = `
      </a>
      <button
        hx-post="/ranking/refresh"
-        hx-target="#main-content"
+        hx-target="#ranking-refresh-status"
        hx-swap="innerHTML"
-        hx-indicator="#refresh-spinner"
-        hx-push-url="/ranking"
        class="text-sm px-3 py-1.5 rounded-lg bg-amber-700 hover:bg-amber-600 text-white inline-flex items-center gap-2">
-        <span id="refresh-spinner" class="htmx-indicator animate-spin">&#8635;</span>
        Refresh Rankings
      </button>
    </div>
  </div>
+  <div id="ranking-refresh-status" class="mt-2"></div>

  <!-- Book grid -->
  <div class="grid gap-4 sm:grid-cols-2 mt-8">
@@ -352,65 +350,107 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
 	s.respond(w, r, "Rankings", buf.String())
 }

-// handleRankingRefresh triggers a live scrape of novelfire.net/ranking,
-// persists the result to ranking.md, then re-renders the ranking page.
+// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
+// immediately returns a polling badge. The browser polls /ui/ranking/status
+// until the job finishes, then follows an HX-Redirect back to /ranking.
 func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
-	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
-	defer cancel()
+	s.mu.Lock()
+	if s.rankingRunning {
+		s.mu.Unlock()
+		renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…"))
+		return
+	}
+	s.rankingRunning = true
+	s.mu.Unlock()

-	rankingCh, errCh := s.novel.ScrapeRanking(ctx)
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			s.rankingRunning = false
+			s.mu.Unlock()
+		}()

-	var rankingItems []writer.RankingItem
-	for {
-		select {
-		case meta, ok := <-rankingCh:
-			if !ok {
-				rankingCh = nil
-				continue
+		ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+		defer cancel()
+
+		rankingCh, errCh := s.novel.ScrapeRanking(ctx)
+
+		var rankingItems []writer.RankingItem
+		for {
+			select {
+			case meta, ok := <-rankingCh:
+				if !ok {
+					rankingCh = nil
+					continue
+				}
+				rankingItems = append(rankingItems, writer.RankingItem{
+					Rank:      meta.Ranking,
+					Slug:      meta.Slug,
+					Title:     meta.Title,
+					Author:    meta.Author,
+					Cover:     meta.Cover,
+					Status:    meta.Status,
+					Genres:    meta.Genres,
+					SourceURL: meta.SourceURL,
+				})
+			case err, ok := <-errCh:
+				if !ok {
+					errCh = nil
+					continue
+				}
+				if err != nil {
+					s.log.Error("ranking scrape error", "err", err)
+				}
 			}
-			rankingItems = append(rankingItems, writer.RankingItem{
-				Rank:      meta.Ranking,
-				Slug:      meta.Slug,
-				Title:     meta.Title,
-				Author:    meta.Author,
-				Cover:     meta.Cover,
-				Status:    meta.Status,
-				Genres:    meta.Genres,
-				SourceURL: meta.SourceURL,
-			})
-		case err, ok := <-errCh:
-			if !ok {
-				errCh = nil
-				continue
-			}
-			if err != nil {
-				s.log.Error("ranking scrape error", "err", err)
+			if rankingCh == nil && errCh == nil {
+				break
 			}
 		}

-		if rankingCh == nil && errCh == nil {
-			break
+		if len(rankingItems) > 0 {
+			if err := s.writer.WriteRanking(rankingItems); err != nil {
+				s.log.Error("failed to save ranking", "err", err)
+			}
 		}
-	}
+	}()

-	if len(rankingItems) > 0 {
-		if err := s.writer.WriteRanking(rankingItems); err != nil {
-			s.log.Error("failed to save ranking", "err", err)
-		}
-	}
+	renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
+}

-	cachedAt := ""
-	if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
-		cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
-	}
+// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs.
+// While running it returns a self-replacing badge; when done it issues an
+// HX-Redirect so the browser navigates to /ranking.
+func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	running := s.rankingRunning
+	s.mu.Unlock()

-	t := template.Must(template.New("ranking").Parse(rankingTmpl))
-	var buf bytes.Buffer
-	_ = t.Execute(&buf, struct {
-		Books    interface{}
-		CachedAt string
-	}{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt})
-	s.respond(w, r, "Rankings", buf.String())
+	if running {
+		renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
+		return
+	}
+	// Job done — redirect the HTMX request to the ranking page.
+	w.Header().Set("HX-Redirect", "/ranking")
+	w.WriteHeader(http.StatusOK)
+}
+
+// rankingStatusHTML returns a self-replacing polling badge for the ranking
+// refresh job. state is "running" or "done".
+func rankingStatusHTML(state, msg string) string {
+	var colour, dot, poll string
+	switch state {
+	case "running":
+		colour = "text-amber-300 bg-amber-950 border-amber-800"
+		dot = `<span class="inline-block w-2 h-2 rounded-full bg-amber-400 animate-pulse mr-2"></span>`
+		poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"`
+	default:
+		colour = "text-green-300 bg-green-950 border-green-800"
+		dot = `<span class="inline-block w-2 h-2 rounded-full bg-green-400 mr-2"></span>`
+	}
+	return fmt.Sprintf(
+		`<div class="flex items-center text-sm px-3 py-2 rounded-lg border %s" %s>%s%s</div>`,
+		colour, poll, dot, template.HTMLEscapeString(msg),
+	)
 }

 // ─── GET /ranking/view — view ranking markdown ─────────────────────────────────