perf(scraper): use direct HTTP for chapter text fetching, bypass Browserless
novelfire.net chapter content is server-rendered, so Browserless is not needed. Add a dedicated chapterClient (always StrategyDirect) to Scraper and use it in ScrapeChapterText, removing the now-irrelevant WaitFor / RejectResourceTypes / GotoOptions fields from the ContentRequest.
This commit is contained in:
@@ -12,7 +12,7 @@
|
||||
//
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapters always use direct HTTP)
|
||||
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
@@ -95,6 +95,8 @@ func run(log *slog.Logger) error {
|
||||
urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
|
||||
bc := newBrowserClient(strategy, browserCfg)
|
||||
urlClient := newBrowserClient(urlStrategy, browserCfg)
|
||||
// Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless.
|
||||
chapterClient := browser.NewDirectHTTPClient(browserCfg)
|
||||
|
||||
// ── Storage backends ────────────────────────────────────────────────────
|
||||
minioCfg := storage.MinioConfig{
|
||||
@@ -119,7 +121,7 @@ func run(log *slog.Logger) error {
|
||||
return fmt.Errorf("storage init failed: %w", err)
|
||||
}
|
||||
|
||||
nf := novelfire.New(bc, log, urlClient, store)
|
||||
nf := novelfire.New(bc, log, urlClient, chapterClient, store)
|
||||
|
||||
workers := 0
|
||||
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
||||
|
||||
Reference in New Issue
Block a user