fix(scraper): replace browserless with direct HTTP and fix presign 404 race

- Fix audio presign 404: MinIO upload is now synchronous before response is sent, eliminating the race where presign was called before the file landed in MinIO - Replace all Browserless usage with direct HTTP client across catalogue, metadata, ranking, and browse — novelfire.net pages are server-rendered and don't need a headless browser; direct is faster and more reliable - Harden handleBrowse with 3-attempt retry loop, proper backoff, and full browser-like headers to reduce 502s from novelfire.net bot detection - Remove Browserless env vars (BROWSERLESS_URL/TOKEN/STRATEGY) from main.go; add SCRAPER_TIMEOUT as a single timeout knob - Clean up now-dead rejectResourceTypes var and Browserless-specific WaitFor/ RejectResourceTypes/GotoOptions fields from scraper calls
2026-03-04 17:32:18 +05:00
parent 0402c408e4
commit cf0c0dfaaf
3 changed files with 83 additions and 121 deletions
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -30,25 +30,6 @@ const (
 	rankingPath   = "/genre-all/sort-popular/status-all/all-novel"
 )

-// rejectResourceTypes lists Browserless resource types to block on every request.
-// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
-// Everything else is safe to drop for HTML-only scraping.
-var rejectResourceTypes = []string{
-	"cspviolationreport",
-	"eventsource",
-	"fedcm",
-	"font",
-	"image",
-	"manifest",
-	"media",
-	"other",
-	"ping",
-	"signedexchange",
-	"stylesheet",
-	"texttrack",
-	"websocket",
-}
-
 // RankingStore is the subset of storage.Store consumed by ScrapeRanking.
 type RankingStore interface {
 	WriteRankingItem(ctx context.Context, item scraper.RankingItem) error
@@ -56,19 +37,19 @@ type RankingStore interface {
 }

 // Scraper is the novelfire.net implementation of scraper.NovelScraper.
-// It uses the /content strategy by default (rendered HTML via Browserless).
+// It uses direct HTTP requests (no headless browser required).
 type Scraper struct {
 	client        browser.BrowserClient
-	urlClient     browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
-	chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed)
+	urlClient     browser.BrowserClient // used for chapter list pagination
+	chapterClient browser.BrowserClient // used for chapter text fetching
 	rankingStore  RankingStore
 	log           *slog.Logger
 }

 // New returns a new novelfire Scraper.
-// client is used for catalogue/metadata/ranking fetching (Browserless).
+// client is used for catalogue/metadata/ranking fetching (direct HTTP).
 // urlClient is used for chapter list pagination; falls back to client if nil.
-// chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil.
+// chapterClient is used for chapter text fetching; falls back to client if nil.
 // rankingStore is optional; pass nil to disable freshness checks and per-item persistence.
 func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
 	if log == nil {
@@ -108,18 +89,9 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 			}

 			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
-			s.log.Debug("catalogue page fetch starting",
-				"page", page,
-				"payload_url", pageURL,
-				"payload_wait_selector", ".novel-item",
-				"payload_wait_selector_timeout_ms", 5000,
-			)

 			html, err := s.client.GetContent(ctx, browser.ContentRequest{
-				URL:                 pageURL,
-				WaitFor:             &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
-				RejectResourceTypes: rejectResourceTypes,
-				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+				URL: pageURL,
 			})
 			if err != nil {
 				s.log.Debug("catalogue page fetch failed",
@@ -212,17 +184,10 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 // ─── MetadataProvider ────────────────────────────────────────────────────────

 func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
-	s.log.Debug("metadata fetch starting",
-		"payload_url", bookURL,
-		"payload_wait_selector", ".novel-title",
-		"payload_wait_selector_timeout_ms", 5000,
-	)
+	s.log.Debug("metadata fetch starting", "url", bookURL)

 	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
-		URL:                 bookURL,
-		WaitFor:             &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
-		RejectResourceTypes: rejectResourceTypes,
-		GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+		URL: bookURL,
 	})
 	if err != nil {
 		s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
@@ -498,10 +463,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap

 			s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
 			raw, err := s.client.GetContent(ctx, browser.ContentRequest{
-				URL:                 pageURL,
-				WaitFor:             &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
-				RejectResourceTypes: rejectResourceTypes,
-				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+				URL: pageURL,
 			})
 			if err != nil {
 				s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -511,23 +511,19 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
 		s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
 	}

-	// Download generated audio from Kokoro and persist to MinIO so that
-	// presigned URLs for the audio object are accessible.
-	go func() {
-		audioData, dlErr := s.downloadFromKokoro(context.Background(), filename)
-		if dlErr != nil {
-			s.log.Warn("audio MinIO upload skipped: kokoro download failed",
-				"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
-			return
-		}
-		minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
-		if putErr := s.store.PutAudio(context.Background(), minioKey, audioData); putErr != nil {
-			s.log.Warn("audio MinIO upload failed",
-				"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
-		} else {
-			s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
-		}
-	}()
+	// Download generated audio from Kokoro and persist to MinIO synchronously
+	// so that the presigned URL returned to the client is immediately valid.
+	minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
+	audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
+	if dlErr != nil {
+		s.log.Warn("audio MinIO upload skipped: kokoro download failed",
+			"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
+	} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
+		s.log.Warn("audio MinIO upload failed",
+			"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
+	} else {
+		s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
+	}

 	s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
 	s.writeAudioResponse(w, slug, n, voice, speed, filename)
@@ -963,7 +959,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		pageNum = 1
 	}

-	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
 	defer cancel()

 	// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
@@ -981,33 +977,62 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		return
 	}

-	// ── Live fallback: fetch from novelfire.net ───────────────────────────
+	// ── Live fallback: direct fetch from novelfire.net ───────────────────
 	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
 	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
 		novelFireBase, genre, sortBy, status, novelType, page)

-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
-	if err != nil {
-		http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
+	var novels []NovelListing
+	var hasNext bool
+	var fetchErr error
+	for attempt := 1; attempt <= 3; attempt++ {
+		if attempt > 1 {
+			select {
+			case <-ctx.Done():
+				http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
+				return
+			case <-time.After(time.Duration(attempt) * time.Second):
+			}
+		}
+
+		var req *http.Request
+		req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
+		if fetchErr != nil {
+			http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
+			return
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+		req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+		req.Header.Set("Cache-Control", "no-cache")
+		req.Header.Set("Pragma", "no-cache")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			fetchErr = err
+			s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+			fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
+			s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
+			continue
+		}
+
+		novels, hasNext = parseBrowsePage(resp.Body)
+		resp.Body.Close()
+		fetchErr = nil
+		break
+	}
+	if fetchErr != nil {
+		s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
+		http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
 		return
 	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
-	req.Header.Set("Accept", "text/html,application/xhtml+xml")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		s.log.Error("browse fetch failed", "url", targetURL, "err", err)
-		http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway)
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway)
-		return
-	}
-
-	novels, hasNext := parseBrowsePage(resp.Body)

 	// ── Background: populate MinIO cache via SingleFile ───────────────────
 	// Fire-and-forget: capture the JS-rendered page with SingleFile, store