diff --git a/.env.example b/.env.example index d9b883f..09c9db6 100644 --- a/.env.example +++ b/.env.example @@ -39,10 +39,6 @@ ERROR_ALERT_URL= # Which Browserless strategy the scraper uses: content | scrape | cdp | direct BROWSERLESS_STRATEGY=direct -# Strategy for URL retrieval (chapter list). Uses browserless content strategy by default. -# Set to direct to use plain HTTP, or content/scrape/cdp for browserless. -BROWSERLESS_URL_STRATEGY=content - # ── Scraper ─────────────────────────────────────────────────────────────────── # Chapter worker goroutines (0 = NumCPU inside the container) SCRAPER_WORKERS=0 diff --git a/docker-compose.yml b/docker-compose.yml index e466290..9cd9888 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,10 +119,9 @@ services: environment: BROWSERLESS_URL: "http://browserless:${BROWSERLESS_PORT:-3030}" BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}" - # content | scrape | cdp | direct — swap to test different strategies. + # content | scrape | cdp | direct — used for catalogue, metadata, ranking. + # Chapter list and chapter text always use direct HTTP regardless of this setting. BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-direct}" - # Strategy for URL retrieval (chapter list). Default: content (browserless) - BROWSERLESS_URL_STRATEGY: "${BROWSERLESS_URL_STRATEGY:-content}" # 0 → defaults to NumCPU inside the container. SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}" SCRAPER_STATIC_ROOT: "/app/static/books" diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index 7839ebd..113319e 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -12,7 +12,7 @@ // // BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) // BROWSERLESS_TOKEN Browserless API token (default: "") -// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapters always use direct HTTP) +// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP) // BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5) // SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) // SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) @@ -92,11 +92,9 @@ func run(log *slog.Logger) error { } strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect)))) - urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent)))) bc := newBrowserClient(strategy, browserCfg) - urlClient := newBrowserClient(urlStrategy, browserCfg) - // Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless. - chapterClient := browser.NewDirectHTTPClient(browserCfg) + // Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both. + directClient := browser.NewDirectHTTPClient(browserCfg) // ── Storage backends ──────────────────────────────────────────────────── minioCfg := storage.MinioConfig{ @@ -121,7 +119,7 @@ func run(log *slog.Logger) error { return fmt.Errorf("storage init failed: %w", err) } - nf := novelfire.New(bc, log, urlClient, chapterClient, store) + nf := novelfire.New(bc, log, directClient, directClient, store) workers := 0 if s := os.Getenv("SCRAPER_WORKERS"); s != "" { @@ -228,8 +226,7 @@ Commands: Environment variables: BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) BROWSERLESS_TOKEN API token (default: "") - BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct) - BROWSERLESS_URL_STRATEGY Strategy for URL retrieval (default: content) + BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP) BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5) BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90) SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d) diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index 5ed356a..cbdc2c1 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -283,24 +283,11 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra s.log.Debug("chapter list fetch starting", "page", page, "payload_url", pageURL, - "payload_wait_selector", ".chapter-list", - "payload_wait_selector_timeout_ms", 15000, - "payload_wait_timeout_ms", 2000, "strategy", s.urlClient.Strategy(), ) raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ URL: pageURL, - // Wait up to 15 s for the chapter list container to appear in the DOM. - WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000}, - // After the selector is found, wait an additional 2 s for any - // deferred JS rendering (lazy-loaded links, infinite-scroll hydration). - WaitForTimeout: 2000, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, - // Do NOT use BestAttempt — we want a complete page or a clear error, - // not silently partial HTML that looks like "no more chapters". - BestAttempt: false, }) if err != nil { s.log.Debug("chapter list fetch failed", @@ -386,11 +373,7 @@ func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([] s.log.Info("scraping chapter list page (single)", "url", pageURL) raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000}, - WaitForTimeout: 2000, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, + URL: pageURL, }) if err != nil { return nil, fmt.Errorf("chapter list page fetch: %w", err)