kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

perf(scraper): switch chapter list to direct HTTP, remove Browserless dependency

Browse Source 
novelfire.net chapter-list pages (/chapters?page=N) are server-rendered —
verified via curl. Switch urlClient to NewDirectHTTPClient alongside the
existing chapterClient. Remove BROWSERLESS_URL_STRATEGY env var and clean
up the now-irrelevant WaitFor/GotoOptions fields from both ScrapeChapterList
and ScrapeChapterListPage ContentRequests.
This commit is contained in:
Admin
2026-03-03 20:46:22 +05:00
parent cff0c78b4f
commit cfd893d24b
4 changed files with 8 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scraper/cmd/scraper/main.go
View File

@@ -12,7 +12,7 @@
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapters always use direct HTTP)
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
@@ -92,11 +92,9 @@ func run(log *slog.Logger) error {
}
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
// Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless.
chapterClient := browser.NewDirectHTTPClient(browserCfg)
// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
directClient := browser.NewDirectHTTPClient(browserCfg)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
@@ -121,7 +119,7 @@ func run(log *slog.Logger) error {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, chapterClient, store)
nf := novelfire.New(bc, log, directClient, directClient, store)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -228,8 +226,7 @@ Commands:
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct)
BROWSERLESS_URL_STRATEGY Strategy for URL retrieval (default: content)
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP)
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)