// Package novelfire provides a NovelScraper implementation for novelfire.net. // // Site structure (as of 2025): // // Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N // Book page : https://novelfire.net/book/{slug} // Chapters : https://novelfire.net/book/{slug}/chapters?page=N // Chapter : https://novelfire.net/book/{slug}/{chapter-slug} package novelfire import ( "context" "fmt" "log/slog" "net/url" "path" "strconv" "strings" "time" "github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/scraper/htmlutil" "golang.org/x/net/html" ) const ( baseURL = "https://novelfire.net" cataloguePath = "/genre-all/sort-new/status-all/all-novel" rankingPath = "/genre-all/sort-popular/status-all/all-novel" ) // rejectResourceTypes lists Browserless resource types to block on every request. // We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls). // Everything else is safe to drop for HTML-only scraping. var rejectResourceTypes = []string{ "cspviolationreport", "eventsource", "fedcm", "font", "image", "manifest", "media", "other", "ping", "signedexchange", "stylesheet", "texttrack", "websocket", } // RankingStore is the subset of storage.Store consumed by ScrapeRanking. type RankingStore interface { WriteRankingItem(ctx context.Context, item scraper.RankingItem) error RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) } // Scraper is the novelfire.net implementation of scraper.NovelScraper. // It uses the /content strategy by default (rendered HTML via Browserless). type Scraper struct { client browser.BrowserClient urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy) chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed) rankingStore RankingStore log *slog.Logger } // New returns a new novelfire Scraper. // client is used for catalogue/metadata/ranking fetching (Browserless). // urlClient is used for chapter list pagination; falls back to client if nil. // chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil. // rankingStore is optional; pass nil to disable freshness checks and per-item persistence. func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper { if log == nil { log = slog.Default() } if urlClient == nil { urlClient = client } if chapterClient == nil { chapterClient = client } return &Scraper{client: client, urlClient: urlClient, chapterClient: chapterClient, rankingStore: rankingStore, log: log} } // SourceName implements NovelScraper. func (s *Scraper) SourceName() string { return "novelfire.net" } // ─── CatalogueProvider ─────────────────────────────────────────────────────── // ScrapeCatalogue streams all CatalogueEntry values across all pages. func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) { entries := make(chan scraper.CatalogueEntry, 64) errs := make(chan error, 16) go func() { defer close(entries) defer close(errs) pageURL := baseURL + cataloguePath page := 1 for pageURL != "" { select { case <-ctx.Done(): return default: } s.log.Info("scraping catalogue page", "page", page, "url", pageURL) s.log.Debug("catalogue page fetch starting", "page", page, "payload_url", pageURL, "payload_wait_selector", ".novel-item", "payload_wait_selector_timeout_ms", 5000, ) html, err := s.client.GetContent(ctx, browser.ContentRequest{ URL: pageURL, WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000}, RejectResourceTypes: rejectResourceTypes, GotoOptions: &browser.GotoOptions{Timeout: 60000}, }) if err != nil { s.log.Debug("catalogue page fetch failed", "page", page, "url", pageURL, "err", err, ) errs <- fmt.Errorf("catalogue page %d: %w", page, err) return } s.log.Debug("catalogue page fetch completed", "page", page, "url", pageURL, "response_bytes", len(html), ) root, err := htmlutil.ParseHTML(html) if err != nil { errs <- fmt.Errorf("catalogue page %d parse: %w", page, err) return } // Extract novel cards:
...