diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go
index eb33f0e..2edd35e 100644
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -385,23 +385,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <
s.log.Info("scraping ranking page", "url", pageURL)
- // Use WaitFor only for browser-based strategies
- var raw string
- var err error
- if s.client.Strategy() == browser.StrategyDirect {
- raw, err = s.client.GetContent(ctx, browser.ContentRequest{
- URL: pageURL,
- RejectResourceTypes: rejectResourceTypes,
- })
- } else {
- raw, err = s.client.GetContent(ctx, browser.ContentRequest{
- URL: pageURL,
- WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
- RejectResourceTypes: rejectResourceTypes,
- GotoOptions: &browser.GotoOptions{Timeout: 60000},
- BestAttempt: true,
- })
- }
+ // Always use the urlClient for the ranking page — it requires JS rendering.
+ raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
+ URL: pageURL,
+ WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
+ RejectResourceTypes: rejectResourceTypes,
+ GotoOptions: &browser.GotoOptions{Timeout: 60000},
+ BestAttempt: true,
+ })
if err != nil {
s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
errs <- fmt.Errorf("ranking page: %w", err)
diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go
index 7f45390..48f48d5 100644
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -24,15 +24,16 @@ import (
// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
- addr string
- oCfg orchestrator.Config
- novel scraper.NovelScraper
- log *slog.Logger
- writer *writer.Writer
- mu sync.Mutex
- running bool
- kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
- kokoroVoice string // default voice, e.g. af_bella
+ addr string
+ oCfg orchestrator.Config
+ novel scraper.NovelScraper
+ log *slog.Logger
+ writer *writer.Writer
+ mu sync.Mutex
+ running bool
+ rankingRunning bool
+ kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
+ kokoroVoice string // default voice, e.g. af_bella
}
// New creates a new Server.
@@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
+ mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
// Plain-text chapter content for browser-side TTS
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go
index f49a31b..4bf24e2 100644
--- a/scraper/internal/server/ui.go
+++ b/scraper/internal/server/ui.go
@@ -224,16 +224,14 @@ const rankingTmpl = `
+
@@ -352,65 +350,107 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
s.respond(w, r, "Rankings", buf.String())
}
-// handleRankingRefresh triggers a live scrape of novelfire.net/ranking,
-// persists the result to ranking.md, then re-renders the ranking page.
+// handleRankingRefresh starts an async scrape of novelfire.net/ranking and
+// immediately returns a polling badge. The browser polls /ui/ranking/status
+// until the job finishes, then follows an HX-Redirect back to /ranking.
func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
- ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
- defer cancel()
+ s.mu.Lock()
+ if s.rankingRunning {
+ s.mu.Unlock()
+ renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…"))
+ return
+ }
+ s.rankingRunning = true
+ s.mu.Unlock()
- rankingCh, errCh := s.novel.ScrapeRanking(ctx)
+ go func() {
+ defer func() {
+ s.mu.Lock()
+ s.rankingRunning = false
+ s.mu.Unlock()
+ }()
- var rankingItems []writer.RankingItem
- for {
- select {
- case meta, ok := <-rankingCh:
- if !ok {
- rankingCh = nil
- continue
+ ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+ defer cancel()
+
+ rankingCh, errCh := s.novel.ScrapeRanking(ctx)
+
+ var rankingItems []writer.RankingItem
+ for {
+ select {
+ case meta, ok := <-rankingCh:
+ if !ok {
+ rankingCh = nil
+ continue
+ }
+ rankingItems = append(rankingItems, writer.RankingItem{
+ Rank: meta.Ranking,
+ Slug: meta.Slug,
+ Title: meta.Title,
+ Author: meta.Author,
+ Cover: meta.Cover,
+ Status: meta.Status,
+ Genres: meta.Genres,
+ SourceURL: meta.SourceURL,
+ })
+ case err, ok := <-errCh:
+ if !ok {
+ errCh = nil
+ continue
+ }
+ if err != nil {
+ s.log.Error("ranking scrape error", "err", err)
+ }
}
- rankingItems = append(rankingItems, writer.RankingItem{
- Rank: meta.Ranking,
- Slug: meta.Slug,
- Title: meta.Title,
- Author: meta.Author,
- Cover: meta.Cover,
- Status: meta.Status,
- Genres: meta.Genres,
- SourceURL: meta.SourceURL,
- })
- case err, ok := <-errCh:
- if !ok {
- errCh = nil
- continue
- }
- if err != nil {
- s.log.Error("ranking scrape error", "err", err)
+ if rankingCh == nil && errCh == nil {
+ break
}
}
- if rankingCh == nil && errCh == nil {
- break
+ if len(rankingItems) > 0 {
+ if err := s.writer.WriteRanking(rankingItems); err != nil {
+ s.log.Error("failed to save ranking", "err", err)
+ }
}
- }
+ }()
- if len(rankingItems) > 0 {
- if err := s.writer.WriteRanking(rankingItems); err != nil {
- s.log.Error("failed to save ranking", "err", err)
- }
- }
+ renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
+}
- cachedAt := ""
- if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
- cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
- }
+// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs.
+// While running it returns a self-replacing badge; when done it issues an
+// HX-Redirect so the browser navigates to /ranking.
+func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) {
+ s.mu.Lock()
+ running := s.rankingRunning
+ s.mu.Unlock()
- t := template.Must(template.New("ranking").Parse(rankingTmpl))
- var buf bytes.Buffer
- _ = t.Execute(&buf, struct {
- Books interface{}
- CachedAt string
- }{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt})
- s.respond(w, r, "Rankings", buf.String())
+ if running {
+ renderFragment(w, rankingStatusHTML("running", "Fetching rankings…"))
+ return
+ }
+ // Job done — redirect the HTMX request to the ranking page.
+ w.Header().Set("HX-Redirect", "/ranking")
+ w.WriteHeader(http.StatusOK)
+}
+
+// rankingStatusHTML returns a self-replacing polling badge for the ranking
+// refresh job. state is "running" or "done".
+func rankingStatusHTML(state, msg string) string {
+ var colour, dot, poll string
+ switch state {
+ case "running":
+ colour = "text-amber-300 bg-amber-950 border-amber-800"
+ dot = `
`
+ poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"`
+ default:
+ colour = "text-green-300 bg-green-950 border-green-800"
+ dot = `
`
+ }
+ return fmt.Sprintf(
+ `
%s%s
`,
+ colour, poll, dot, template.HTMLEscapeString(msg),
+ )
}
// ─── GET /ranking/view — view ranking markdown ─────────────────────────────────