diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index cf2ed18..60faab3 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -78,26 +78,16 @@ func run(log *slog.Logger) error { cmd := strings.ToLower(args[0]) - browserCfg := browser.Config{ - BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"), - Token: envOr("BROWSERLESS_TOKEN", ""), - } - browserCfg.MaxConcurrent = 5 - if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" { + // All scraping uses direct HTTP — novelfire.net pages are server-rendered + // and do not require a headless browser. A direct HTTP client is faster, + // more reliable, and has no Browserless dependency. + directCfg := browser.Config{MaxConcurrent: 5} + if s := os.Getenv("SCRAPER_TIMEOUT"); s != "" { if n, err := strconv.Atoi(s); err == nil && n > 0 { - browserCfg.MaxConcurrent = n + directCfg.Timeout = time.Duration(n) * time.Second } } - if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" { - if n, err := strconv.Atoi(s); err == nil && n > 0 { - browserCfg.Timeout = time.Duration(n) * time.Second - } - } - - strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect)))) - bc := newBrowserClient(strategy, browserCfg) - // Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both. - directClient := browser.NewDirectHTTPClient(browserCfg) + directClient := browser.NewDirectHTTPClient(directCfg) // ── Storage backends ──────────────────────────────────────────────────── minioCfg := storage.MinioConfig{ @@ -125,7 +115,7 @@ func run(log *slog.Logger) error { return fmt.Errorf("storage init failed: %w", err) } - nf := novelfire.New(bc, log, directClient, directClient, store) + nf := novelfire.New(directClient, log, directClient, directClient, store) workers := 0 if s := os.Getenv("SCRAPER_WORKERS"); s != "" { @@ -149,9 +139,8 @@ func run(log *slog.Logger) error { oCfg.SingleBookURL = args[2] } log.Info("starting one-shot scrape", - "strategy", strategy, + "strategy", "direct", "workers", workers, - "max_concurrent", browserCfg.MaxConcurrent, "single_book", oCfg.SingleBookURL, "pocketbase_url", pbCfg.BaseURL, "pocketbase_email", pbCfg.AdminEmail, @@ -191,9 +180,8 @@ func run(log *slog.Logger) error { kokoroVoice := envOr("KOKORO_VOICE", "af_bella") log.Info("starting HTTP server", "addr", addr, - "strategy", strategy, + "strategy", "direct", "workers", workers, - "max_concurrent", browserCfg.MaxConcurrent, "kokoro_url", kokoroURL, "kokoro_voice", kokoroVoice, "pocketbase_url", pbCfg.BaseURL, @@ -533,19 +521,6 @@ func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageU log.Debug("save-browse: cover stored", "key", key, "bytes", len(data)) } -func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient { - switch strategy { - case browser.StrategyScrape: - return browser.NewScrapeClient(cfg) - case browser.StrategyCDP: - return browser.NewCDPClient(cfg) - case browser.StrategyDirect: - return browser.NewDirectHTTPClient(cfg) - default: - return browser.NewContentClient(cfg) - } -} - func envOr(key, fallback string) string { if v := os.Getenv(key); v != "" { return v diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index 720752d..5be55b0 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -30,25 +30,6 @@ const ( rankingPath = "/genre-all/sort-popular/status-all/all-novel" ) -// rejectResourceTypes lists Browserless resource types to block on every request. -// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls). -// Everything else is safe to drop for HTML-only scraping. -var rejectResourceTypes = []string{ - "cspviolationreport", - "eventsource", - "fedcm", - "font", - "image", - "manifest", - "media", - "other", - "ping", - "signedexchange", - "stylesheet", - "texttrack", - "websocket", -} - // RankingStore is the subset of storage.Store consumed by ScrapeRanking. type RankingStore interface { WriteRankingItem(ctx context.Context, item scraper.RankingItem) error @@ -56,19 +37,19 @@ type RankingStore interface { } // Scraper is the novelfire.net implementation of scraper.NovelScraper. -// It uses the /content strategy by default (rendered HTML via Browserless). +// It uses direct HTTP requests (no headless browser required). type Scraper struct { client browser.BrowserClient - urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy) - chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed) + urlClient browser.BrowserClient // used for chapter list pagination + chapterClient browser.BrowserClient // used for chapter text fetching rankingStore RankingStore log *slog.Logger } // New returns a new novelfire Scraper. -// client is used for catalogue/metadata/ranking fetching (Browserless). +// client is used for catalogue/metadata/ranking fetching (direct HTTP). // urlClient is used for chapter list pagination; falls back to client if nil. -// chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil. +// chapterClient is used for chapter text fetching; falls back to client if nil. // rankingStore is optional; pass nil to disable freshness checks and per-item persistence. func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper { if log == nil { @@ -108,18 +89,9 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue } s.log.Info("scraping catalogue page", "page", page, "url", pageURL) - s.log.Debug("catalogue page fetch starting", - "page", page, - "payload_url", pageURL, - "payload_wait_selector", ".novel-item", - "payload_wait_selector_timeout_ms", 5000, - ) html, err := s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, + URL: pageURL, }) if err != nil { s.log.Debug("catalogue page fetch failed", @@ -212,17 +184,10 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue // ─── MetadataProvider ──────────────────────────────────────────────────────── func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) { - s.log.Debug("metadata fetch starting", - "payload_url", bookURL, - "payload_wait_selector", ".novel-title", - "payload_wait_selector_timeout_ms", 5000, - ) + s.log.Debug("metadata fetch starting", "url", bookURL) raw, err := s.client.GetContent(ctx, browser.ContentRequest{ - URL: bookURL, - WaitFor: &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, + URL: bookURL, }) if err != nil { s.log.Debug("metadata fetch failed", "url", bookURL, "err", err) @@ -498,10 +463,7 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap s.log.Info("scraping popular ranking page", "page", page, "url", pageURL) raw, err := s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, + URL: pageURL, }) if err != nil { s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err) diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index f461106..483f945 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -511,23 +511,19 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err) } - // Download generated audio from Kokoro and persist to MinIO so that - // presigned URLs for the audio object are accessible. - go func() { - audioData, dlErr := s.downloadFromKokoro(context.Background(), filename) - if dlErr != nil { - s.log.Warn("audio MinIO upload skipped: kokoro download failed", - "slug", slug, "chapter", n, "filename", filename, "err", dlErr) - return - } - minioKey := s.store.AudioObjectKey(slug, n, voice, speed) - if putErr := s.store.PutAudio(context.Background(), minioKey, audioData); putErr != nil { - s.log.Warn("audio MinIO upload failed", - "slug", slug, "chapter", n, "key", minioKey, "err", putErr) - } else { - s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey) - } - }() + // Download generated audio from Kokoro and persist to MinIO synchronously + // so that the presigned URL returned to the client is immediately valid. + minioKey := s.store.AudioObjectKey(slug, n, voice, speed) + audioData, dlErr := s.downloadFromKokoro(r.Context(), filename) + if dlErr != nil { + s.log.Warn("audio MinIO upload skipped: kokoro download failed", + "slug", slug, "chapter", n, "filename", filename, "err", dlErr) + } else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil { + s.log.Warn("audio MinIO upload failed", + "slug", slug, "chapter", n, "key", minioKey, "err", putErr) + } else { + s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey) + } s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) s.writeAudioResponse(w, slug, n, voice, speed, filename) @@ -963,7 +959,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { pageNum = 1 } - ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second) defer cancel() // ── Cache-first: try MinIO snapshot (new key layout) ───────────────── @@ -981,33 +977,62 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { return } - // ── Live fallback: fetch from novelfire.net ─────────────────────────── + // ── Live fallback: direct fetch from novelfire.net ─────────────────── // Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page} targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s", novelFireBase, genre, sortBy, status, novelType, page) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) - if err != nil { - http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError) + var novels []NovelListing + var hasNext bool + var fetchErr error + for attempt := 1; attempt <= 3; attempt++ { + if attempt > 1 { + select { + case <-ctx.Done(): + http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) + return + case <-time.After(time.Duration(attempt) * time.Second): + } + } + + var req *http.Request + req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) + if fetchErr != nil { + http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError) + return + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Accept-Encoding", "gzip, deflate, br") + req.Header.Set("Cache-Control", "no-cache") + req.Header.Set("Pragma", "no-cache") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + fetchErr = err + s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err) + continue + } + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode) + s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode) + continue + } + + novels, hasNext = parseBrowsePage(resp.Body) + resp.Body.Close() + fetchErr = nil + break + } + if fetchErr != nil { + s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr) + http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway) return } - req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)") - req.Header.Set("Accept", "text/html,application/xhtml+xml") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - s.log.Error("browse fetch failed", "url", targetURL, "err", err) - http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway) - return - } - - novels, hasNext := parseBrowsePage(resp.Body) // ── Background: populate MinIO cache via SingleFile ─────────────────── // Fire-and-forget: capture the JS-rendered page with SingleFile, store