From e9f880f7f7a9647f587788ac1d5df67a920abbbc Mon Sep 17 00:00:00 2001 From: Admin Date: Sun, 1 Mar 2026 16:52:03 +0500 Subject: [PATCH] fix: make ranking refresh async and use JS-rendered client for ScrapeRanking Two bugs caused the 'Refresh Rankings' button to silently fail in production: 1. ScrapeRanking was using the plain HTTP client (s.client) instead of the browserless content client (s.urlClient). The /ranking page requires JavaScript rendering, so a plain fetch returned HTML without any novel entries. Now uses s.urlClient so the page is fully rendered before scraping. 2. handleRankingRefresh was synchronous, holding the HTTP connection open for up to 60 s while scraping. Reverse proxies and HTMX timeouts closed the connection before the scrape finished. Rewritten to the same async pattern used for book scraping: POST /ranking/refresh returns immediately with a polling badge; the browser polls GET /ui/ranking/status every 3 s; when the goroutine finishes the status endpoint sends HX-Redirect to /ranking. --- scraper/internal/novelfire/scraper.go | 25 ++--- scraper/internal/server/server.go | 20 ++-- scraper/internal/server/ui.go | 144 ++++++++++++++++---------- 3 files changed, 111 insertions(+), 78 deletions(-) diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index eb33f0e..2edd35e 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -385,23 +385,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, < s.log.Info("scraping ranking page", "url", pageURL) - // Use WaitFor only for browser-based strategies - var raw string - var err error - if s.client.Strategy() == browser.StrategyDirect { - raw, err = s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - RejectResourceTypes: rejectResourceTypes, - }) - } else { - raw, err = s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, - BestAttempt: true, - }) - } + // Always use the urlClient for the ranking page — it requires JS rendering. + raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ + URL: pageURL, + WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000}, + RejectResourceTypes: rejectResourceTypes, + GotoOptions: &browser.GotoOptions{Timeout: 60000}, + BestAttempt: true, + }) if err != nil { s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err) errs <- fmt.Errorf("ranking page: %w", err) diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 7f45390..48f48d5 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -24,15 +24,16 @@ import ( // Server wraps an HTTP mux with the scraping endpoints. type Server struct { - addr string - oCfg orchestrator.Config - novel scraper.NovelScraper - log *slog.Logger - writer *writer.Writer - mu sync.Mutex - running bool - kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 - kokoroVoice string // default voice, e.g. af_bella + addr string + oCfg orchestrator.Config + novel scraper.NovelScraper + log *slog.Logger + writer *writer.Writer + mu sync.Mutex + running bool + rankingRunning bool + kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 + kokoroVoice string // default voice, e.g. af_bella } // New creates a new Server. @@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage) mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook) mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus) + mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) // Plain-text chapter content for browser-side TTS mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText) diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index f49a31b..4bf24e2 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -224,16 +224,14 @@ const rankingTmpl = ` +
@@ -352,65 +350,107 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { s.respond(w, r, "Rankings", buf.String()) } -// handleRankingRefresh triggers a live scrape of novelfire.net/ranking, -// persists the result to ranking.md, then re-renders the ranking page. +// handleRankingRefresh starts an async scrape of novelfire.net/ranking and +// immediately returns a polling badge. The browser polls /ui/ranking/status +// until the job finishes, then follows an HX-Redirect back to /ranking. func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) - defer cancel() + s.mu.Lock() + if s.rankingRunning { + s.mu.Unlock() + renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…")) + return + } + s.rankingRunning = true + s.mu.Unlock() - rankingCh, errCh := s.novel.ScrapeRanking(ctx) + go func() { + defer func() { + s.mu.Lock() + s.rankingRunning = false + s.mu.Unlock() + }() - var rankingItems []writer.RankingItem - for { - select { - case meta, ok := <-rankingCh: - if !ok { - rankingCh = nil - continue + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + rankingCh, errCh := s.novel.ScrapeRanking(ctx) + + var rankingItems []writer.RankingItem + for { + select { + case meta, ok := <-rankingCh: + if !ok { + rankingCh = nil + continue + } + rankingItems = append(rankingItems, writer.RankingItem{ + Rank: meta.Ranking, + Slug: meta.Slug, + Title: meta.Title, + Author: meta.Author, + Cover: meta.Cover, + Status: meta.Status, + Genres: meta.Genres, + SourceURL: meta.SourceURL, + }) + case err, ok := <-errCh: + if !ok { + errCh = nil + continue + } + if err != nil { + s.log.Error("ranking scrape error", "err", err) + } } - rankingItems = append(rankingItems, writer.RankingItem{ - Rank: meta.Ranking, - Slug: meta.Slug, - Title: meta.Title, - Author: meta.Author, - Cover: meta.Cover, - Status: meta.Status, - Genres: meta.Genres, - SourceURL: meta.SourceURL, - }) - case err, ok := <-errCh: - if !ok { - errCh = nil - continue - } - if err != nil { - s.log.Error("ranking scrape error", "err", err) + if rankingCh == nil && errCh == nil { + break } } - if rankingCh == nil && errCh == nil { - break + if len(rankingItems) > 0 { + if err := s.writer.WriteRanking(rankingItems); err != nil { + s.log.Error("failed to save ranking", "err", err) + } } - } + }() - if len(rankingItems) > 0 { - if err := s.writer.WriteRanking(rankingItems); err != nil { - s.log.Error("failed to save ranking", "err", err) - } - } + renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) +} - cachedAt := "" - if info, statErr := s.writer.RankingFileInfo(); statErr == nil { - cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") - } +// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs. +// While running it returns a self-replacing badge; when done it issues an +// HX-Redirect so the browser navigates to /ranking. +func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + running := s.rankingRunning + s.mu.Unlock() - t := template.Must(template.New("ranking").Parse(rankingTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Books interface{} - CachedAt string - }{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt}) - s.respond(w, r, "Rankings", buf.String()) + if running { + renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) + return + } + // Job done — redirect the HTMX request to the ranking page. + w.Header().Set("HX-Redirect", "/ranking") + w.WriteHeader(http.StatusOK) +} + +// rankingStatusHTML returns a self-replacing polling badge for the ranking +// refresh job. state is "running" or "done". +func rankingStatusHTML(state, msg string) string { + var colour, dot, poll string + switch state { + case "running": + colour = "text-amber-300 bg-amber-950 border-amber-800" + dot = `` + poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` + default: + colour = "text-green-300 bg-green-950 border-green-800" + dot = `` + } + return fmt.Sprintf( + `
%s%s
`, + colour, poll, dot, template.HTMLEscapeString(msg), + ) } // ─── GET /ranking/view — view ranking markdown ─────────────────────────────────