kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

perf(scraper): use direct HTTP for chapter text fetching, bypass Browserless

Browse Source 
novelfire.net chapter content is server-rendered, so Browserless is not
needed. Add a dedicated chapterClient (always StrategyDirect) to Scraper
and use it in ScrapeChapterText, removing the now-irrelevant WaitFor /
RejectResourceTypes / GotoOptions fields from the ContentRequest.
This commit is contained in:
Admin
2026-03-03 20:40:01 +05:00
parent d89cefe975
commit cff0c78b4f
7 changed files with 25 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
scraper/internal/novelfire/scraper.go
View File

@@ -58,24 +58,29 @@ type RankingStore interface {
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
// It uses the /content strategy by default (rendered HTML via Browserless).
type Scraper struct {
client browser.BrowserClient
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
rankingStore RankingStore
log *slog.Logger
client browser.BrowserClient
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed)
rankingStore RankingStore
log *slog.Logger
}
// New returns a new novelfire Scraper.
// client is used for content fetching, urlClient is used for URL retrieval (chapter list).
// If urlClient is nil, client will be used for both.
// client is used for catalogue/metadata/ranking fetching (Browserless).
// urlClient is used for chapter list pagination; falls back to client if nil.
// chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil.
// rankingStore is optional; pass nil to disable freshness checks and per-item persistence.
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
if log == nil {
log = slog.Default()
}
if urlClient == nil {
urlClient = client
}
return &Scraper{client: client, urlClient: urlClient, rankingStore: rankingStore, log: log}
if chapterClient == nil {
chapterClient = client
}
return &Scraper{client: client, urlClient: urlClient, chapterClient: chapterClient, rankingStore: rankingStore, log: log}
}
// SourceName implements NovelScraper.
@@ -651,12 +656,8 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef)
"payload_wait_selector_timeout_ms", 5000,
)
raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{
URL: ref.URL,
WaitFor: &browser.WaitForSelector{Selector: "#content", Timeout: 5000},
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
BestAttempt: true,
raw, err := retryGetContent(ctx, s.log, s.chapterClient, browser.ContentRequest{
URL: ref.URL,
}, 9, 6*time.Second)
if err != nil {
s.log.Debug("chapter text fetch failed",