fix(scraper): replace browserless with direct HTTP and fix presign 404 race

- Fix audio presign 404: MinIO upload is now synchronous before response is sent,
  eliminating the race where presign was called before the file landed in MinIO
- Replace all Browserless usage with direct HTTP client across catalogue, metadata,
  ranking, and browse — novelfire.net pages are server-rendered and don't need a
  headless browser; direct is faster and more reliable
- Harden handleBrowse with 3-attempt retry loop, proper backoff, and full
  browser-like headers to reduce 502s from novelfire.net bot detection
- Remove Browserless env vars (BROWSERLESS_URL/TOKEN/STRATEGY) from main.go;
  add SCRAPER_TIMEOUT as a single timeout knob
- Clean up now-dead rejectResourceTypes var and Browserless-specific WaitFor/
  RejectResourceTypes/GotoOptions fields from scraper calls

scraper/cmd/scraper/main.go

@@ -78,26 +78,16 @@ func run(log *slog.Logger) error {
 
 	cmd := strings.ToLower(args[0])
 
-	browserCfg := browser.Config{
-		BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"),
-		Token:   envOr("BROWSERLESS_TOKEN", ""),
-	}
-	browserCfg.MaxConcurrent = 5
-	if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
+	// All scraping uses direct HTTP — novelfire.net pages are server-rendered
+	// and do not require a headless browser. A direct HTTP client is faster,
+	// more reliable, and has no Browserless dependency.
+	directCfg := browser.Config{MaxConcurrent: 5}
+	if s := os.Getenv("SCRAPER_TIMEOUT"); s != "" {
 		if n, err := strconv.Atoi(s); err == nil && n > 0 {
-			browserCfg.MaxConcurrent = n
+			directCfg.Timeout = time.Duration(n) * time.Second
 		}
 	}
-	if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
-		if n, err := strconv.Atoi(s); err == nil && n > 0 {
-			browserCfg.Timeout = time.Duration(n) * time.Second
-		}
-	}
-
-	strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
-	bc := newBrowserClient(strategy, browserCfg)
-	// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
-	directClient := browser.NewDirectHTTPClient(browserCfg)
+	directClient := browser.NewDirectHTTPClient(directCfg)
 
 	// ── Storage backends ────────────────────────────────────────────────────
 	minioCfg := storage.MinioConfig{
@@ -125,7 +115,7 @@ func run(log *slog.Logger) error {
 		return fmt.Errorf("storage init failed: %w", err)
 	}
 
-	nf := novelfire.New(bc, log, directClient, directClient, store)
+	nf := novelfire.New(directClient, log, directClient, directClient, store)
 
 	workers := 0
 	if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -149,9 +139,8 @@ func run(log *slog.Logger) error {
 			oCfg.SingleBookURL = args[2]
 		}
 		log.Info("starting one-shot scrape",
-			"strategy", strategy,
+			"strategy", "direct",
 			"workers", workers,
-			"max_concurrent", browserCfg.MaxConcurrent,
 			"single_book", oCfg.SingleBookURL,
 			"pocketbase_url", pbCfg.BaseURL,
 			"pocketbase_email", pbCfg.AdminEmail,
@@ -191,9 +180,8 @@ func run(log *slog.Logger) error {
 		kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
 		log.Info("starting HTTP server",
 			"addr", addr,
-			"strategy", strategy,
+			"strategy", "direct",
 			"workers", workers,
-			"max_concurrent", browserCfg.MaxConcurrent,
 			"kokoro_url", kokoroURL,
 			"kokoro_voice", kokoroVoice,
 			"pocketbase_url", pbCfg.BaseURL,
@@ -533,19 +521,6 @@ func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageU
 	log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
 }
 
-func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
-	switch strategy {
-	case browser.StrategyScrape:
-		return browser.NewScrapeClient(cfg)
-	case browser.StrategyCDP:
-		return browser.NewCDPClient(cfg)
-	case browser.StrategyDirect:
-		return browser.NewDirectHTTPClient(cfg)
-	default:
-		return browser.NewContentClient(cfg)
-	}
-}
-
 func envOr(key, fallback string) string {
 	if v := os.Getenv(key); v != "" {
 		return v

scraper/internal/novelfire/scraper.go

@@ -30,25 +30,6 @@ const (
 	rankingPath   = "/genre-all/sort-popular/status-all/all-novel"
 )
 
-// rejectResourceTypes lists Browserless resource types to block on every request.
-// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
-// Everything else is safe to drop for HTML-only scraping.
-var rejectResourceTypes = []string{
-	"cspviolationreport",
-	"eventsource",
-	"fedcm",
-	"font",
-	"image",
-	"manifest",
-	"media",
-	"other",
-	"ping",
-	"signedexchange",
-	"stylesheet",
-	"texttrack",
-	"websocket",
-}
-
 // RankingStore is the subset of storage.Store consumed by ScrapeRanking.
 type RankingStore interface {
 	WriteRankingItem(ctx context.Context, item scraper.RankingItem) error
@@ -56,19 +37,19 @@ type RankingStore interface {
 }
 
 // Scraper is the novelfire.net implementation of scraper.NovelScraper.
-// It uses the /content strategy by default (rendered HTML via Browserless).
+// It uses direct HTTP requests (no headless browser required).
 type Scraper struct {
 	client        browser.BrowserClient
-	urlClient     browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
-	chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed)
+	urlClient     browser.BrowserClient // used for chapter list pagination
+	chapterClient browser.BrowserClient // used for chapter text fetching
 	rankingStore  RankingStore
 	log           *slog.Logger
 }
 
 // New returns a new novelfire Scraper.
-// client is used for catalogue/metadata/ranking fetching (Browserless).
+// client is used for catalogue/metadata/ranking fetching (direct HTTP).
 // urlClient is used for chapter list pagination; falls back to client if nil.
-// chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil.
+// chapterClient is used for chapter text fetching; falls back to client if nil.
 // rankingStore is optional; pass nil to disable freshness checks and per-item persistence.
 func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
 	if log == nil {
@@ -108,18 +89,9 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 			}
 
 			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
-			s.log.Debug("catalogue page fetch starting",
-				"page", page,
-				"payload_url", pageURL,
-				"payload_wait_selector", ".novel-item",
-				"payload_wait_selector_timeout_ms", 5000,
-			)
 
 			html, err := s.client.GetContent(ctx, browser.ContentRequest{
-				URL:                 pageURL,
-				WaitFor:             &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
-				RejectResourceTypes: rejectResourceTypes,
-				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+				URL: pageURL,
 			})
 			if err != nil {
 				s.log.Debug("catalogue page fetch failed",
@@ -212,17 +184,10 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 // ─── MetadataProvider ────────────────────────────────────────────────────────
 
 func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
-	s.log.Debug("metadata fetch starting",
-		"payload_url", bookURL,
-		"payload_wait_selector", ".novel-title",
-		"payload_wait_selector_timeout_ms", 5000,
-	)
+	s.log.Debug("metadata fetch starting", "url", bookURL)
 
 	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
-		URL:                 bookURL,
-		WaitFor:             &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
-		RejectResourceTypes: rejectResourceTypes,
-		GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+		URL: bookURL,
 	})
 	if err != nil {
 		s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
@@ -498,10 +463,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
 
 			s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
 			raw, err := s.client.GetContent(ctx, browser.ContentRequest{
-				URL:                 pageURL,
-				WaitFor:             &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
-				RejectResourceTypes: rejectResourceTypes,
-				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+				URL: pageURL,
 			})
 			if err != nil {
 				s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)

scraper/internal/server/server.go

@@ -511,23 +511,19 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
 		s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
 	}
 
-	// Download generated audio from Kokoro and persist to MinIO so that
-	// presigned URLs for the audio object are accessible.
-	go func() {
-		audioData, dlErr := s.downloadFromKokoro(context.Background(), filename)
-		if dlErr != nil {
-			s.log.Warn("audio MinIO upload skipped: kokoro download failed",
-				"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
-			return
-		}
-		minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
-		if putErr := s.store.PutAudio(context.Background(), minioKey, audioData); putErr != nil {
-			s.log.Warn("audio MinIO upload failed",
-				"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
-		} else {
-			s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
-		}
-	}()
+	// Download generated audio from Kokoro and persist to MinIO synchronously
+	// so that the presigned URL returned to the client is immediately valid.
+	minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
+	audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
+	if dlErr != nil {
+		s.log.Warn("audio MinIO upload skipped: kokoro download failed",
+			"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
+	} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
+		s.log.Warn("audio MinIO upload failed",
+			"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
+	} else {
+		s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
+	}
 
 	s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
 	s.writeAudioResponse(w, slug, n, voice, speed, filename)
@@ -963,7 +959,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		pageNum = 1
 	}
 
-	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
 	defer cancel()
 
 	// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
@@ -981,33 +977,62 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// ── Live fallback: fetch from novelfire.net ───────────────────────────
+	// ── Live fallback: direct fetch from novelfire.net ───────────────────
 	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
 	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
 		novelFireBase, genre, sortBy, status, novelType, page)
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
-	if err != nil {
-		http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
+	var novels []NovelListing
+	var hasNext bool
+	var fetchErr error
+	for attempt := 1; attempt <= 3; attempt++ {
+		if attempt > 1 {
+			select {
+			case <-ctx.Done():
+				http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
+				return
+			case <-time.After(time.Duration(attempt) * time.Second):
+			}
+		}
+
+		var req *http.Request
+		req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
+		if fetchErr != nil {
+			http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
+			return
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+		req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+		req.Header.Set("Cache-Control", "no-cache")
+		req.Header.Set("Pragma", "no-cache")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			fetchErr = err
+			s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+			fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
+			s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
+			continue
+		}
+
+		novels, hasNext = parseBrowsePage(resp.Body)
+		resp.Body.Close()
+		fetchErr = nil
+		break
+	}
+	if fetchErr != nil {
+		s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
+		http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
 		return
 	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
-	req.Header.Set("Accept", "text/html,application/xhtml+xml")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		s.log.Error("browse fetch failed", "url", targetURL, "err", err)
-		http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway)
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway)
-		return
-	}
-
-	novels, hasNext := parseBrowsePage(resp.Body)
 
 	// ── Background: populate MinIO cache via SingleFile ───────────────────
 	// Fire-and-forget: capture the JS-rendered page with SingleFile, store