diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index eb33f0e..2edd35e 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -385,23 +385,14 @@ func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, < s.log.Info("scraping ranking page", "url", pageURL) - // Use WaitFor only for browser-based strategies - var raw string - var err error - if s.client.Strategy() == browser.StrategyDirect { - raw, err = s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - RejectResourceTypes: rejectResourceTypes, - }) - } else { - raw, err = s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, - BestAttempt: true, - }) - } + // Always use the urlClient for the ranking page — it requires JS rendering. + raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ + URL: pageURL, + WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000}, + RejectResourceTypes: rejectResourceTypes, + GotoOptions: &browser.GotoOptions{Timeout: 60000}, + BestAttempt: true, + }) if err != nil { s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err) errs <- fmt.Errorf("ranking page: %w", err) diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 7f45390..48f48d5 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -24,15 +24,16 @@ import ( // Server wraps an HTTP mux with the scraping endpoints. type Server struct { - addr string - oCfg orchestrator.Config - novel scraper.NovelScraper - log *slog.Logger - writer *writer.Writer - mu sync.Mutex - running bool - kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 - kokoroVoice string // default voice, e.g. af_bella + addr string + oCfg orchestrator.Config + novel scraper.NovelScraper + log *slog.Logger + writer *writer.Writer + mu sync.Mutex + running bool + rankingRunning bool + kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 + kokoroVoice string // default voice, e.g. af_bella } // New creates a new Server. @@ -65,6 +66,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage) mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook) mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus) + mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) // Plain-text chapter content for browser-side TTS mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText) diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index f49a31b..4bf24e2 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -224,16 +224,14 @@ const rankingTmpl = ` +
@@ -352,65 +350,107 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { s.respond(w, r, "Rankings", buf.String()) } -// handleRankingRefresh triggers a live scrape of novelfire.net/ranking, -// persists the result to ranking.md, then re-renders the ranking page. +// handleRankingRefresh starts an async scrape of novelfire.net/ranking and +// immediately returns a polling badge. The browser polls /ui/ranking/status +// until the job finishes, then follows an HX-Redirect back to /ranking. func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) - defer cancel() + s.mu.Lock() + if s.rankingRunning { + s.mu.Unlock() + renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…")) + return + } + s.rankingRunning = true + s.mu.Unlock() - rankingCh, errCh := s.novel.ScrapeRanking(ctx) + go func() { + defer func() { + s.mu.Lock() + s.rankingRunning = false + s.mu.Unlock() + }() - var rankingItems []writer.RankingItem - for { - select { - case meta, ok := <-rankingCh: - if !ok { - rankingCh = nil - continue + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + rankingCh, errCh := s.novel.ScrapeRanking(ctx) + + var rankingItems []writer.RankingItem + for { + select { + case meta, ok := <-rankingCh: + if !ok { + rankingCh = nil + continue + } + rankingItems = append(rankingItems, writer.RankingItem{ + Rank: meta.Ranking, + Slug: meta.Slug, + Title: meta.Title, + Author: meta.Author, + Cover: meta.Cover, + Status: meta.Status, + Genres: meta.Genres, + SourceURL: meta.SourceURL, + }) + case err, ok := <-errCh: + if !ok { + errCh = nil + continue + } + if err != nil { + s.log.Error("ranking scrape error", "err", err) + } } - rankingItems = append(rankingItems, writer.RankingItem{ - Rank: meta.Ranking, - Slug: meta.Slug, - Title: meta.Title, - Author: meta.Author, - Cover: meta.Cover, - Status: meta.Status, - Genres: meta.Genres, - SourceURL: meta.SourceURL, - }) - case err, ok := <-errCh: - if !ok { - errCh = nil - continue - } - if err != nil { - s.log.Error("ranking scrape error", "err", err) + if rankingCh == nil && errCh == nil { + break } } - if rankingCh == nil && errCh == nil { - break + if len(rankingItems) > 0 { + if err := s.writer.WriteRanking(rankingItems); err != nil { + s.log.Error("failed to save ranking", "err", err) + } } - } + }() - if len(rankingItems) > 0 { - if err := s.writer.WriteRanking(rankingItems); err != nil { - s.log.Error("failed to save ranking", "err", err) - } - } + renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) +} - cachedAt := "" - if info, statErr := s.writer.RankingFileInfo(); statErr == nil { - cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") - } +// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs. +// While running it returns a self-replacing badge; when done it issues an +// HX-Redirect so the browser navigates to /ranking. +func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + running := s.rankingRunning + s.mu.Unlock() - t := template.Must(template.New("ranking").Parse(rankingTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Books interface{} - CachedAt string - }{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt}) - s.respond(w, r, "Rankings", buf.String()) + if running { + renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) + return + } + // Job done — redirect the HTMX request to the ranking page. + w.Header().Set("HX-Redirect", "/ranking") + w.WriteHeader(http.StatusOK) +} + +// rankingStatusHTML returns a self-replacing polling badge for the ranking +// refresh job. state is "running" or "done". +func rankingStatusHTML(state, msg string) string { + var colour, dot, poll string + switch state { + case "running": + colour = "text-amber-300 bg-amber-950 border-amber-800" + dot = `` + poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` + default: + colour = "text-green-300 bg-green-950 border-green-800" + dot = `` + } + return fmt.Sprintf( + `
%s%s
`, + colour, poll, dot, template.HTMLEscapeString(msg), + ) } // ─── GET /ranking/view — view ranking markdown ─────────────────────────────────