// Package novelfire provides a NovelScraper implementation for novelfire.net. // // Site structure (as of 2025): // // Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N // Book page : https://novelfire.net/book/{slug} // Chapters : https://novelfire.net/book/{slug}/chapters?page=N // Chapter : https://novelfire.net/book/{slug}/{chapter-slug} package novelfire import ( "context" "fmt" "log/slog" "net/url" "path" "strconv" "strings" "time" "github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/scraper/htmlutil" "golang.org/x/net/html" ) const ( baseURL = "https://novelfire.net" cataloguePath = "/genre-all/sort-new/status-all/all-novel" rankingPath = "/genre-all/sort-popular/status-all/all-novel" ) // rejectResourceTypes lists Browserless resource types to block on every request. // We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls). // Everything else is safe to drop for HTML-only scraping. var rejectResourceTypes = []string{ "cspviolationreport", "eventsource", "fedcm", "font", "image", "manifest", "media", "other", "ping", "signedexchange", "stylesheet", "texttrack", "websocket", } // RankingStore is the subset of storage.Store consumed by ScrapeRanking. type RankingStore interface { WriteRankingItem(ctx context.Context, item scraper.RankingItem) error RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) } // Scraper is the novelfire.net implementation of scraper.NovelScraper. // It uses the /content strategy by default (rendered HTML via Browserless). type Scraper struct { client browser.BrowserClient urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy) chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed) rankingStore RankingStore log *slog.Logger } // New returns a new novelfire Scraper. // client is used for catalogue/metadata/ranking fetching (Browserless). // urlClient is used for chapter list pagination; falls back to client if nil. // chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil. // rankingStore is optional; pass nil to disable freshness checks and per-item persistence. func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper { if log == nil { log = slog.Default() } if urlClient == nil { urlClient = client } if chapterClient == nil { chapterClient = client } return &Scraper{client: client, urlClient: urlClient, chapterClient: chapterClient, rankingStore: rankingStore, log: log} } // SourceName implements NovelScraper. func (s *Scraper) SourceName() string { return "novelfire.net" } // ─── CatalogueProvider ─────────────────────────────────────────────────────── // ScrapeCatalogue streams all CatalogueEntry values across all pages. func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) { entries := make(chan scraper.CatalogueEntry, 64) errs := make(chan error, 16) go func() { defer close(entries) defer close(errs) pageURL := baseURL + cataloguePath page := 1 for pageURL != "" { select { case <-ctx.Done(): return default: } s.log.Info("scraping catalogue page", "page", page, "url", pageURL) s.log.Debug("catalogue page fetch starting", "page", page, "payload_url", pageURL, "payload_wait_selector", ".novel-item", "payload_wait_selector_timeout_ms", 5000, ) html, err := s.client.GetContent(ctx, browser.ContentRequest{ URL: pageURL, WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000}, RejectResourceTypes: rejectResourceTypes, GotoOptions: &browser.GotoOptions{Timeout: 60000}, }) if err != nil { s.log.Debug("catalogue page fetch failed", "page", page, "url", pageURL, "err", err, ) errs <- fmt.Errorf("catalogue page %d: %w", page, err) return } s.log.Debug("catalogue page fetch completed", "page", page, "url", pageURL, "response_bytes", len(html), ) root, err := htmlutil.ParseHTML(html) if err != nil { errs <- fmt.Errorf("catalogue page %d parse: %w", page, err) return } // Extract novel cards:
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}) if len(cards) == 0 { s.log.Warn("no novel cards found, stopping pagination", "page", page) return } for _, card := range cards { // Title:

Title titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"}) var title, href string if titleNode != nil { linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"}) if linkNode != nil { title = htmlutil.ExtractText(linkNode, scraper.Selector{}) href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) } } if href == "" || title == "" { continue } bookURL := resolveURL(baseURL, href) select { case <-ctx.Done(): return case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}: } } // Find next page link: Author Name author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"}) //
var cover string if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil { cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"}) if cover != "" && !strings.HasPrefix(cover, "http") { cover = baseURL + cover } } // Ongoing status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"}) // Genres: all tags inside
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"}) var genres []string if genresNode != nil { genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true}) } //

...

summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"}) // 123 Chapters totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"}) totalChapters := parseChapterCount(totalStr) slug := slugFromURL(bookURL) meta := scraper.BookMeta{ Slug: slug, Title: title, Author: author, Cover: cover, Status: status, Genres: genres, Summary: summary, TotalChapters: totalChapters, SourceURL: bookURL, } s.log.Debug("metadata parsed", "url", bookURL, "slug", meta.Slug, "title", meta.Title, "author", meta.Author, "status", meta.Status, "genres", meta.Genres, "total_chapters", meta.TotalChapters, ) return meta, nil } // ─── ChapterListProvider ───────────────────────────────────────────────────── func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) { var refs []scraper.ChapterRef // Chapter list URL: {bookURL}/chapters?page=N baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters" page := 1 for { select { case <-ctx.Done(): return refs, ctx.Err() default: } pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) s.log.Info("scraping chapter list", "page", page, "url", pageURL) s.log.Debug("chapter list fetch starting", "page", page, "payload_url", pageURL, "strategy", s.urlClient.Strategy(), ) raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ URL: pageURL, }) if err != nil { s.log.Debug("chapter list fetch failed", "page", page, "url", pageURL, "err", err, ) return refs, fmt.Errorf("chapter list page %d: %w", page, err) } s.log.Debug("chapter list fetch completed", "page", page, "url", pageURL, "response_bytes", len(raw), ) root, err := htmlutil.ParseHTML(raw) if err != nil { return refs, fmt.Errorf("chapter list page %d parse: %w", page, err) } chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"}) if chapterList == nil { // No chapter list container on this page — we've gone past the last page. s.log.Debug("chapter list container not found, stopping pagination", "page", page) break } // Each chapter row:
  • Title
  • items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"}) s.log.Debug("chapter list page parsed", "page", page, "url", pageURL, "chapters_on_page", len(items), "total_refs_so_far", len(refs), ) // Zero items on this page means we've gone past the last page. if len(items) == 0 { s.log.Debug("no chapters on page, stopping pagination", "page", page) break } for _, item := range items { linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"}) if linkNode == nil { continue } href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"}) chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) if href == "" { continue } chURL := resolveURL(baseURL, href) num := chapterNumberFromURL(chURL) if num <= 0 { // Fall back to position if the URL has no parseable number. num = len(refs) + 1 s.log.Warn("chapter number not parseable from URL, falling back to position", "url", chURL, "position", num, ) } refs = append(refs, scraper.ChapterRef{ Number: num, Title: strings.TrimSpace(chTitle), URL: chURL, }) } page++ } return refs, nil } // ScrapeChapterListPage fetches and parses a single chapter-list page URL and // returns all chapter refs found on that page without following pagination. // pageURL should be the full URL including query params, e.g.: // // https://novelfire.net/book/shadow-slave/chapters?page=1 func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]scraper.ChapterRef, error) { s.log.Info("scraping chapter list page (single)", "url", pageURL) raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ URL: pageURL, }) if err != nil { return nil, fmt.Errorf("chapter list page fetch: %w", err) } root, err := htmlutil.ParseHTML(raw) if err != nil { return nil, fmt.Errorf("chapter list page parse: %w", err) } chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"}) if chapterList == nil { return nil, fmt.Errorf("chapter list container not found in %s", pageURL) } items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"}) var refs []scraper.ChapterRef for _, item := range items { linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"}) if linkNode == nil { continue } href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"}) chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) if href == "" { continue } chURL := resolveURL(baseURL, href) num := chapterNumberFromURL(chURL) if num <= 0 { num = len(refs) + 1 } refs = append(refs, scraper.ChapterRef{ Number: num, Title: strings.TrimSpace(chTitle), URL: chURL, }) } return refs, nil } // ─── RankingProvider ─────────────────────────────────────────────────────────── // hasNextPageLink returns true if the HTML document contains a pagination link // with rel="next". novelfire.net uses: // //
  • //
    //

    Title

    //
    //
  • cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) if len(cards) == 0 { s.log.Debug("no novel cards found, stopping pagination", "page", page) break } for _, card := range cards { // The outer carries the href and title attribute. linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"}) if linkNode == nil { continue } href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) bookURL := resolveURL(baseURL, href) if bookURL == "" { continue } // Title: prefer

    text; fall back to title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})) if title == "" { title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"})) } if title == "" { continue } // Cover:
    var cover string if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil { cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) if cover == "" { cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) } // Filter out base64 placeholder images. if strings.HasPrefix(cover, "data:") { cover = "" } if cover != "" && !strings.HasPrefix(cover, "http") { cover = baseURL + cover } } bookSlug := slugFromURL(bookURL) meta := scraper.BookMeta{ Slug: bookSlug, Title: title, Cover: cover, SourceURL: bookURL, Ranking: rank, } rank++ // Persist item to store immediately. if s.rankingStore != nil { item := scraper.RankingItem{ Rank: meta.Ranking, Slug: meta.Slug, Title: meta.Title, Cover: meta.Cover, SourceURL: meta.SourceURL, } if werr := s.rankingStore.WriteRankingItem(ctx, item); werr != nil { s.log.Warn("ranking item write failed", "slug", meta.Slug, "err", werr) } } select { case <-ctx.Done(): return case entries <- meta: } } // Stop if no next-page link exists. // The real pagination uses