diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index 953025b..7839ebd 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -12,7 +12,7 @@ // // BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) // BROWSERLESS_TOKEN Browserless API token (default: "") -// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) +// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapters always use direct HTTP) // BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5) // SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) // SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) @@ -95,6 +95,8 @@ func run(log *slog.Logger) error { urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent)))) bc := newBrowserClient(strategy, browserCfg) urlClient := newBrowserClient(urlStrategy, browserCfg) + // Chapter text is server-rendered on novelfire.net — direct HTTP is faster and avoids Browserless. + chapterClient := browser.NewDirectHTTPClient(browserCfg) // ── Storage backends ──────────────────────────────────────────────────── minioCfg := storage.MinioConfig{ @@ -119,7 +121,7 @@ func run(log *slog.Logger) error { return fmt.Errorf("storage init failed: %w", err) } - nf := novelfire.New(bc, log, urlClient, store) + nf := novelfire.New(bc, log, urlClient, chapterClient, store) workers := 0 if s := os.Getenv("SCRAPER_WORKERS"); s != "" { diff --git a/scraper/internal/e2e/e2e_test.go b/scraper/internal/e2e/e2e_test.go index b4df064..1210ed9 100644 --- a/scraper/internal/e2e/e2e_test.go +++ b/scraper/internal/e2e/e2e_test.go @@ -121,7 +121,7 @@ func newE2EFixture(t *testing.T) *e2eFixture { MaxConcurrent: 2, }) log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) - sc := novelfire.New(directClient, log, urlClient, nil) + sc := novelfire.New(directClient, log, urlClient, directClient, nil) return &e2eFixture{ sc: sc, diff --git a/scraper/internal/novelfire/integration_test.go b/scraper/internal/novelfire/integration_test.go index d4dace4..39f821a 100644 --- a/scraper/internal/novelfire/integration_test.go +++ b/scraper/internal/novelfire/integration_test.go @@ -53,7 +53,7 @@ func newIntegrationScraper(t *testing.T) *Scraper { MaxConcurrent: 1, }) log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) - return New(client, log, client, nil) + return New(client, log, client, nil, nil) } // ── Metadata ────────────────────────────────────────────────────────────────── diff --git a/scraper/internal/novelfire/ranking_test.go b/scraper/internal/novelfire/ranking_test.go index 2127ae9..1f6c13b 100644 --- a/scraper/internal/novelfire/ranking_test.go +++ b/scraper/internal/novelfire/ranking_test.go @@ -114,7 +114,7 @@ func TestScrapeRanking_MultiPage(t *testing.T) { // Use pagedStubClient for s.client so each GetContent call returns the // next page. ScrapeRanking now calls s.client directly. urlClient := &pagedStubClient{pages: []string{rankingPage1HTML(), rankingPage2HTML()}} - s := New(urlClient, nil, nil, nil) // nil cache — no disk I/O in tests + s := New(urlClient, nil, nil, nil, nil) // nil cache — no disk I/O in tests entryCh, errCh := s.ScrapeRanking(context.Background(), 0) // 0 = all pages entries := drainRanking(t, entryCh, errCh) diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index ca80732..5ed356a 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -58,24 +58,29 @@ type RankingStore interface { // Scraper is the novelfire.net implementation of scraper.NovelScraper. // It uses the /content strategy by default (rendered HTML via Browserless). type Scraper struct { - client browser.BrowserClient - urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy) - rankingStore RankingStore - log *slog.Logger + client browser.BrowserClient + urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy) + chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed) + rankingStore RankingStore + log *slog.Logger } // New returns a new novelfire Scraper. -// client is used for content fetching, urlClient is used for URL retrieval (chapter list). -// If urlClient is nil, client will be used for both. +// client is used for catalogue/metadata/ranking fetching (Browserless). +// urlClient is used for chapter list pagination; falls back to client if nil. +// chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil. // rankingStore is optional; pass nil to disable freshness checks and per-item persistence. -func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, rankingStore RankingStore) *Scraper { +func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper { if log == nil { log = slog.Default() } if urlClient == nil { urlClient = client } - return &Scraper{client: client, urlClient: urlClient, rankingStore: rankingStore, log: log} + if chapterClient == nil { + chapterClient = client + } + return &Scraper{client: client, urlClient: urlClient, chapterClient: chapterClient, rankingStore: rankingStore, log: log} } // SourceName implements NovelScraper. @@ -651,12 +656,8 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) "payload_wait_selector_timeout_ms", 5000, ) - raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{ - URL: ref.URL, - WaitFor: &browser.WaitForSelector{Selector: "#content", Timeout: 5000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, - BestAttempt: true, + raw, err := retryGetContent(ctx, s.log, s.chapterClient, browser.ContentRequest{ + URL: ref.URL, }, 9, 6*time.Second) if err != nil { s.log.Debug("chapter text fetch failed", diff --git a/scraper/internal/novelfire/scraper_test.go b/scraper/internal/novelfire/scraper_test.go index 1422586..e629d38 100644 --- a/scraper/internal/novelfire/scraper_test.go +++ b/scraper/internal/novelfire/scraper_test.go @@ -62,12 +62,12 @@ func (c *pagedStubClient) CDPSession(_ context.Context, _ string, _ browser.CDPS // ── helpers ─────────────────────────────────────────────────────────────────── func newScraper(html string) *Scraper { - return New(&stubClient{html: html}, nil, &stubClient{html: html}, nil) + return New(&stubClient{html: html}, nil, &stubClient{html: html}, nil, nil) } func newPagedScraper(pages ...string) *Scraper { urlClient := &pagedStubClient{pages: pages} - return New(&stubClient{}, nil, urlClient, nil) + return New(&stubClient{}, nil, urlClient, nil, nil) } // ── ScrapeChapterText ───────────────────────────────────────────────────────── diff --git a/scraper/internal/storage/scrape_integration_test.go b/scraper/internal/storage/scrape_integration_test.go index 95db0c6..515875a 100644 --- a/scraper/internal/storage/scrape_integration_test.go +++ b/scraper/internal/storage/scrape_integration_test.go @@ -57,7 +57,7 @@ func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) { MaxConcurrent: 1, }) log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) - sc := novelfire.New(client, log, client, nil) + sc := novelfire.New(client, log, client, nil, nil) hs := newTestHybridStore(t) return sc, hs }