perf(scraper): switch chapter list to direct HTTP, remove Browserless dependency

novelfire.net chapter-list pages (/chapters?page=N) are server-rendered —
verified via curl. Switch urlClient to NewDirectHTTPClient alongside the
existing chapterClient. Remove BROWSERLESS_URL_STRATEGY env var and clean
up the now-irrelevant WaitFor/GotoOptions fields from both ScrapeChapterList
and ScrapeChapterListPage ContentRequests.

.env.example

@@ -39,10 +39,6 @@ ERROR_ALERT_URL=
 # Which Browserless strategy the scraper uses: content | scrape | cdp | direct
 BROWSERLESS_STRATEGY=direct
 
-# Strategy for URL retrieval (chapter list). Uses browserless content strategy by default.
-# Set to direct to use plain HTTP, or content/scrape/cdp for browserless.
-BROWSERLESS_URL_STRATEGY=content
-
 # ── Scraper ───────────────────────────────────────────────────────────────────
 # Chapter worker goroutines (0 = NumCPU inside the container)
 SCRAPER_WORKERS=0

docker-compose.yml

@@ -119,10 +119,9 @@ services:
     environment:
       BROWSERLESS_URL: "http://browserless:${BROWSERLESS_PORT:-3030}"
       BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}"
-      # content | scrape | cdp | direct — swap to test different strategies.
+      # content | scrape | cdp | direct — used for catalogue, metadata, ranking.
+      # Chapter list and chapter text always use direct HTTP regardless of this setting.
       BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-direct}"
-      # Strategy for URL retrieval (chapter list). Default: content (browserless)
-      BROWSERLESS_URL_STRATEGY: "${BROWSERLESS_URL_STRATEGY:-content}"
       # 0 → defaults to NumCPU inside the container.
       SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}"
       SCRAPER_STATIC_ROOT: "/app/static/books"

scraper/cmd/scraper/main.go

@@ -12,7 +12,7 @@
 //
 //	BROWSERLESS_URL             Browserless base URL           (default: http://localhost:3030)
 //	BROWSERLESS_TOKEN           Browserless API token          (default: "")
-//	BROWSERLESS_STRATEGY        content | scrape | cdp | direct (default: direct; chapters always use direct HTTP)
+//	BROWSERLESS_STRATEGY        content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP)
 //	BROWSERLESS_MAX_CONCURRENT  Max simultaneous browser sessions  (default: 5)
 //	SCRAPER_WORKERS             Chapter goroutine count        (default: NumCPU)
 //	SCRAPER_HTTP_ADDR           HTTP listen address            (default: :8080)
@@ -92,11 +92,9 @@ func run(log *slog.Logger) error {
 	}
 
 	strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
-	urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
 	bc := newBrowserClient(strategy, browserCfg)
-	urlClient := newBrowserClient(urlStrategy, browserCfg)
+	// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
-	// Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless.
+	directClient := browser.NewDirectHTTPClient(browserCfg)
-	chapterClient := browser.NewDirectHTTPClient(browserCfg)
 
 	// ── Storage backends ────────────────────────────────────────────────────
 	minioCfg := storage.MinioConfig{
@@ -121,7 +119,7 @@ func run(log *slog.Logger) error {
 		return fmt.Errorf("storage init failed: %w", err)
 	}
 
-	nf := novelfire.New(bc, log, urlClient, chapterClient, store)
+	nf := novelfire.New(bc, log, directClient, directClient, store)
 
 	workers := 0
 	if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -228,8 +226,7 @@ Commands:
 Environment variables:
   BROWSERLESS_URL             Browserless base URL      (default: http://localhost:3030)
   BROWSERLESS_TOKEN           API token                 (default: "")
-  BROWSERLESS_STRATEGY        content|scrape|cdp|direct (default: direct)
+  BROWSERLESS_STRATEGY        content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP)
-  BROWSERLESS_URL_STRATEGY    Strategy for URL retrieval (default: content)
   BROWSERLESS_MAX_CONCURRENT  Max simultaneous sessions (default: 5)
   BROWSERLESS_TIMEOUT         HTTP request timeout sec  (default: 90)
   SCRAPER_WORKERS             Chapter goroutines        (default: NumCPU = %d)

scraper/internal/novelfire/scraper.go

@@ -283,24 +283,11 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
 		s.log.Debug("chapter list fetch starting",
 			"page", page,
 			"payload_url", pageURL,
-			"payload_wait_selector", ".chapter-list",
-			"payload_wait_selector_timeout_ms", 15000,
-			"payload_wait_timeout_ms", 2000,
 			"strategy", s.urlClient.Strategy(),
 		)
 
 		raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
 			URL: pageURL,
-			// Wait up to 15 s for the chapter list container to appear in the DOM.
-			WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
-			// After the selector is found, wait an additional 2 s for any
-			// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
-			WaitForTimeout:      2000,
-			RejectResourceTypes: rejectResourceTypes,
-			GotoOptions:         &browser.GotoOptions{Timeout: 60000},
-			// Do NOT use BestAttempt — we want a complete page or a clear error,
-			// not silently partial HTML that looks like "no more chapters".
-			BestAttempt: false,
 		})
 		if err != nil {
 			s.log.Debug("chapter list fetch failed",
@@ -386,11 +373,7 @@ func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]
 	s.log.Info("scraping chapter list page (single)", "url", pageURL)
 
 	raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
-		URL:                 pageURL,
+		URL: pageURL,
-		WaitFor:             &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
-		WaitForTimeout:      2000,
-		RejectResourceTypes: rejectResourceTypes,
-		GotoOptions:         &browser.GotoOptions{Timeout: 60000},
 	})
 	if err != nil {
 		return nil, fmt.Errorf("chapter list page fetch: %w", err)