// Package novelfire provides a NovelScraper implementation for novelfire.net. // // Site structure (as of 2025): // // Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N // Book page : https://novelfire.net/book/{slug} // Chapters : https://novelfire.net/book/{slug}/chapters?page=N // Chapter : https://novelfire.net/book/{slug}/{chapter-slug} package novelfire import ( "context" "fmt" "log/slog" "net/url" "strconv" "strings" "github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/scraper/htmlutil" ) const ( baseURL = "https://novelfire.net" cataloguePath = "/genre-all/sort-new/status-all/all-novel" ) // Scraper is the novelfire.net implementation of scraper.NovelScraper. // It uses the /content strategy by default (rendered HTML via Browserless). type Scraper struct { client browser.BrowserClient log *slog.Logger } // New returns a new novelfire Scraper. func New(client browser.BrowserClient, log *slog.Logger) *Scraper { if log == nil { log = slog.Default() } return &Scraper{client: client, log: log} } // SourceName implements NovelScraper. func (s *Scraper) SourceName() string { return "novelfire.net" } // ─── CatalogueProvider ─────────────────────────────────────────────────────── func (s *Scraper) CatalogueURL() string { return baseURL + cataloguePath } func (s *Scraper) EntriesSelector() scraper.Selector { // Each novel card:
return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true} } func (s *Scraper) NextPageSelector() scraper.Selector { //

Title titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"} titleNode := htmlutil.FindFirst(card, titleSel) var title, href string if titleNode != nil { linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"}) if linkNode != nil { title = htmlutil.ExtractText(linkNode, scraper.Selector{}) href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) } } if href == "" || title == "" { continue } // Resolve relative URL. bookURL := resolveURL(baseURL, href) select { case <-ctx.Done(): return case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}: } } // Find next page link. nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector()) if nextHref == "" { break } pageURL = resolveURL(baseURL, nextHref) page++ } }() return entries, errs } // ─── MetadataProvider ──────────────────────────────────────────────────────── func (s *Scraper) MetadataSelectors() map[string]scraper.Selector { return map[string]scraper.Selector{ //

Title

"title": {Tag: "h1", Class: "novel-title"}, // Author Name "author": {Tag: "span", Class: "author"}, // "cover": {Tag: "img", Class: "cover", Attr: "src"}, // Ongoing "status": {Tag: "span", Class: "status"}, //
Tag1Tag2
"genres": {Tag: "div", Class: "genres", Multiple: true}, //

...

"summary": {Tag: "div", Class: "summary"}, // 123 Chapters "total_chapters": {Tag: "span", Class: "chapter-count"}, } } func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) { raw, err := s.client.GetContent(ctx, browser.ContentRequest{ URL: bookURL, WaitFor: ".novel-title", WaitForTimeout: 10000, RejectResources: true, }) if err != nil { return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err) } root, err := htmlutil.ParseHTML(raw) if err != nil { return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err) } sels := s.MetadataSelectors() title := htmlutil.ExtractFirst(root, sels["title"]) author := htmlutil.ExtractFirst(root, sels["author"]) cover := htmlutil.ExtractFirst(root, sels["cover"]) status := htmlutil.ExtractFirst(root, sels["status"]) // Genres: all tags inside the genres div. genresNode := htmlutil.FindFirst(root, sels["genres"]) var genres []string if genresNode != nil { genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true}) } summary := htmlutil.ExtractFirst(root, sels["summary"]) totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"]) totalChapters := parseChapterCount(totalStr) // Derive slug from URL. slug := slugFromURL(bookURL) return scraper.BookMeta{ Slug: slug, Title: title, Author: author, Cover: cover, Status: status, Genres: genres, Summary: summary, TotalChapters: totalChapters, SourceURL: bookURL, }, nil } // ─── ChapterListProvider ───────────────────────────────────────────────────── func (s *Scraper) ChaptersURL(bookURL string) string { return strings.TrimRight(bookURL, "/") + "/chapters" } func (s *Scraper) ChapterEntrySelector() scraper.Selector { //
  • Chapter 1: Title
  • return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true} } func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) { var refs []scraper.ChapterRef pageURL := s.ChaptersURL(bookURL) page := 1 for pageURL != "" { select { case <-ctx.Done(): return refs, ctx.Err() default: } s.log.Info("scraping chapter list", "page", page, "url", pageURL) raw, err := s.client.GetContent(ctx, browser.ContentRequest{ URL: pageURL, WaitFor: ".chapter-item", WaitForTimeout: 10000, RejectResources: true, }) if err != nil { return refs, fmt.Errorf("chapter list page %d: %w", page, err) } root, err := htmlutil.ParseHTML(raw) if err != nil { return refs, fmt.Errorf("chapter list page %d parse: %w", page, err) } items := htmlutil.FindAll(root, s.ChapterEntrySelector()) for _, item := range items { linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"}) if linkNode == nil { continue } href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) if href == "" { continue } chURL := resolveURL(baseURL, href) num := len(refs) + 1 refs = append(refs, scraper.ChapterRef{ Number: num, Title: strings.TrimSpace(chTitle), URL: chURL, }) } // Next page. nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector()) if nextHref == "" { break } pageURL = resolveURL(baseURL, nextHref) page++ } return refs, nil } // ─── ChapterTextProvider ───────────────────────────────────────────────────── func (s *Scraper) ChapterTextSelector() scraper.Selector { //
    or
    return scraper.Selector{Tag: "div", ID: "chapter-container"} } func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) { raw, err := s.client.GetContent(ctx, browser.ContentRequest{ URL: ref.URL, WaitFor: "#chapter-container", WaitForTimeout: 15000, RejectResources: true, }) if err != nil { return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err) } root, err := htmlutil.ParseHTML(raw) if err != nil { return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err) } container := htmlutil.FindFirst(root, s.ChapterTextSelector()) if container == nil { // Fallback: try class-based selector. container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"}) } if container == nil { return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL) } text := htmlutil.NodeToMarkdown(container) return scraper.Chapter{ Ref: ref, Text: text, }, nil } // ─── helpers ───────────────────────────────────────────────────────────────── func resolveURL(base, href string) string { if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { return href } b, err := url.Parse(base) if err != nil { return base + href } ref, err := url.Parse(href) if err != nil { return base + href } return b.ResolveReference(ref).String() } func slugFromURL(bookURL string) string { u, err := url.Parse(bookURL) if err != nil { return bookURL } parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) >= 2 && parts[0] == "book" { return parts[1] } if len(parts) > 0 { return parts[len(parts)-1] } return "" } func parseChapterCount(s string) int { // Formats: "123 Chapters", "1,234 Chapters", "123" s = strings.ReplaceAll(s, ",", "") fields := strings.Fields(s) if len(fields) == 0 { return 0 } n, _ := strconv.Atoi(fields[0]) return n }