kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

perf(scraper): switch chapter list to direct HTTP, remove Browserless dependency

Browse Source 
novelfire.net chapter-list pages (/chapters?page=N) are server-rendered —
verified via curl. Switch urlClient to NewDirectHTTPClient alongside the
existing chapterClient. Remove BROWSERLESS_URL_STRATEGY env var and clean
up the now-irrelevant WaitFor/GotoOptions fields from both ScrapeChapterList
and ScrapeChapterListPage ContentRequests.
This commit is contained in:
Admin
2026-03-03 20:46:22 +05:00
parent cff0c78b4f
commit cfd893d24b
4 changed files with 8 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
scraper/internal/novelfire/scraper.go
View File

@@ -283,24 +283,11 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
s.log.Debug("chapter list fetch starting",
"page", page,
"payload_url", pageURL,
"payload_wait_selector", ".chapter-list",
"payload_wait_selector_timeout_ms", 15000,
"payload_wait_timeout_ms", 2000,
"strategy", s.urlClient.Strategy(),
)
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
// Wait up to 15 s for the chapter list container to appear in the DOM.
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
// After the selector is found, wait an additional 2 s for any
// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
WaitForTimeout: 2000,
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
// Do NOT use BestAttempt — we want a complete page or a clear error,
// not silently partial HTML that looks like "no more chapters".
BestAttempt: false,
})
if err != nil {
s.log.Debug("chapter list fetch failed",
@@ -386,11 +373,7 @@ func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]
s.log.Info("scraping chapter list page (single)", "url", pageURL)
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
WaitForTimeout: 2000,
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
URL: pageURL,
})
if err != nil {
return nil, fmt.Errorf("chapter list page fetch: %w", err)