fix: make ranking refresh async and use JS-rendered client for ScrapeRanking

Two bugs caused the 'Refresh Rankings' button to silently fail in production:

1. ScrapeRanking was using the plain HTTP client (s.client) instead of the
   browserless content client (s.urlClient). The /ranking page requires
   JavaScript rendering, so a plain fetch returned HTML without any novel
   entries. Now uses s.urlClient so the page is fully rendered before scraping.

2. handleRankingRefresh was synchronous, holding the HTTP connection open for
   up to 60 s while scraping. Reverse proxies and HTMX timeouts closed the
   connection before the scrape finished. Rewritten to the same async pattern
   used for book scraping: POST /ranking/refresh returns immediately with a
   polling badge; the browser polls GET /ui/ranking/status every 3 s; when
   the goroutine finishes the status endpoint sends HX-Redirect to /ranking.
This commit is contained in:
Admin
2026-03-01 16:52:03 +05:00
parent bcdef02997
commit e9f880f7f7
3 changed files with 111 additions and 78 deletions

View File

@@ -24,15 +24,16 @@ import (
// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
mu sync.Mutex
running bool
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
kokoroVoice string // default voice, e.g. af_bella
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
mu sync.Mutex
running bool
rankingRunning bool
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
kokoroVoice string // default voice, e.g. af_bella
}
// New creates a new Server.
@@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
// Plain-text chapter content for browser-side TTS
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)