perf(scraper): use direct HTTP for chapter text fetching, bypass Browserless

novelfire.net chapter content is server-rendered, so Browserless is not
needed. Add a dedicated chapterClient (always StrategyDirect) to Scraper
and use it in ScrapeChapterText, removing the now-irrelevant WaitFor /
RejectResourceTypes / GotoOptions fields from the ContentRequest.
This commit is contained in:
Admin
2026-03-03 20:40:01 +05:00
parent d89cefe975
commit cff0c78b4f
7 changed files with 25 additions and 22 deletions

View File

@@ -12,7 +12,7 @@
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapters always use direct HTTP)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
@@ -95,6 +95,8 @@ func run(log *slog.Logger) error {
urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
// Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless.
chapterClient := browser.NewDirectHTTPClient(browserCfg)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
@@ -119,7 +121,7 @@ func run(log *slog.Logger) error {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, store)
nf := novelfire.New(bc, log, urlClient, chapterClient, store)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {