kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

perf(scraper): use direct HTTP for chapter text fetching, bypass Browserless

Browse Source 
novelfire.net chapter content is server-rendered, so Browserless is not
needed. Add a dedicated chapterClient (always StrategyDirect) to Scraper
and use it in ScrapeChapterText, removing the now-irrelevant WaitFor /
RejectResourceTypes / GotoOptions fields from the ContentRequest.
This commit is contained in:
Admin
2026-03-03 20:40:01 +05:00
parent d89cefe975
commit cff0c78b4f
7 changed files with 25 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scraper/internal/novelfire/scraper_test.go
View File

@@ -62,12 +62,12 @@ func (c *pagedStubClient) CDPSession(_ context.Context, _ string, _ browser.CDPS
// ── helpers ───────────────────────────────────────────────────────────────────
func newScraper(html string) *Scraper {
return New(&stubClient{html: html}, nil, &stubClient{html: html}, nil)
return New(&stubClient{html: html}, nil, &stubClient{html: html}, nil, nil)
}
func newPagedScraper(pages ...string) *Scraper {
urlClient := &pagedStubClient{pages: pages}
return New(&stubClient{}, nil, urlClient, nil)
return New(&stubClient{}, nil, urlClient, nil, nil)
}
// ── ScrapeChapterText ─────────────────────────────────────────────────────────