feat: initial commit

2026-02-26 12:56:25 +05:00
commit d68ea71239
15 changed files with 1893 additions and 0 deletions
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -0,0 +1,372 @@
+// Package novelfire provides a NovelScraper implementation for novelfire.net.
+//
+// Site structure (as of 2025):
+//
+//	Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
+//	Book page : https://novelfire.net/book/{slug}
+//	Chapters  : https://novelfire.net/book/{slug}/chapters?page=N
+//	Chapter   : https://novelfire.net/book/{slug}/{chapter-slug}
+package novelfire
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/libnovel/scraper/internal/browser"
+	"github.com/libnovel/scraper/internal/scraper"
+	"github.com/libnovel/scraper/internal/scraper/htmlutil"
+)
+
+const (
+	baseURL       = "https://novelfire.net"
+	cataloguePath = "/genre-all/sort-new/status-all/all-novel"
+)
+
+// Scraper is the novelfire.net implementation of scraper.NovelScraper.
+// It uses the /content strategy by default (rendered HTML via Browserless).
+type Scraper struct {
+	client browser.BrowserClient
+	log    *slog.Logger
+}
+
+// New returns a new novelfire Scraper.
+func New(client browser.BrowserClient, log *slog.Logger) *Scraper {
+	if log == nil {
+		log = slog.Default()
+	}
+	return &Scraper{client: client, log: log}
+}
+
+// SourceName implements NovelScraper.
+func (s *Scraper) SourceName() string { return "novelfire.net" }
+
+// ─── CatalogueProvider ───────────────────────────────────────────────────────
+
+func (s *Scraper) CatalogueURL() string {
+	return baseURL + cataloguePath
+}
+
+func (s *Scraper) EntriesSelector() scraper.Selector {
+	// Each novel card: <div class="novel-item">
+	return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}
+}
+
+func (s *Scraper) NextPageSelector() scraper.Selector {
+	// <a class="next" href="...">
+	return scraper.Selector{Tag: "a", Class: "next", Attr: "href"}
+}
+
+// ScrapeCatalogue streams all CatalogueEntry values across all pages.
+func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
+	entries := make(chan scraper.CatalogueEntry, 64)
+	errs := make(chan error, 16)
+
+	go func() {
+		defer close(entries)
+		defer close(errs)
+
+		pageURL := s.CatalogueURL()
+		page := 1
+
+		for pageURL != "" {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
+			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
+
+			html, err := s.client.GetContent(ctx, browser.ContentRequest{
+				URL:             pageURL,
+				WaitFor:         ".novel-item",
+				WaitForTimeout:  10000,
+				RejectResources: true,
+			})
+			if err != nil {
+				errs <- fmt.Errorf("catalogue page %d: %w", page, err)
+				return
+			}
+
+			root, err := htmlutil.ParseHTML(html)
+			if err != nil {
+				errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
+				return
+			}
+
+			// Extract novel cards.
+			cards := htmlutil.FindAll(root, s.EntriesSelector())
+			if len(cards) == 0 {
+				s.log.Warn("no novel cards found, stopping pagination", "page", page)
+				return
+			}
+
+			for _, card := range cards {
+				// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
+				titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"}
+				titleNode := htmlutil.FindFirst(card, titleSel)
+
+				var title, href string
+				if titleNode != nil {
+					linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
+					if linkNode != nil {
+						title = htmlutil.ExtractText(linkNode, scraper.Selector{})
+						href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
+					}
+				}
+				if href == "" || title == "" {
+					continue
+				}
+
+				// Resolve relative URL.
+				bookURL := resolveURL(baseURL, href)
+				select {
+				case <-ctx.Done():
+					return
+				case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
+				}
+			}
+
+			// Find next page link.
+			nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
+			if nextHref == "" {
+				break
+			}
+			pageURL = resolveURL(baseURL, nextHref)
+			page++
+		}
+	}()
+
+	return entries, errs
+}
+
+// ─── MetadataProvider ────────────────────────────────────────────────────────
+
+func (s *Scraper) MetadataSelectors() map[string]scraper.Selector {
+	return map[string]scraper.Selector{
+		// <h1 class="novel-title">Title</h1>
+		"title": {Tag: "h1", Class: "novel-title"},
+		// <span class="author"><a>Author Name</a></span>
+		"author": {Tag: "span", Class: "author"},
+		// <img class="cover" src="...">
+		"cover": {Tag: "img", Class: "cover", Attr: "src"},
+		// <span class="status">Ongoing</span>
+		"status": {Tag: "span", Class: "status"},
+		// <div class="genres"><a>Tag1</a><a>Tag2</a>…</div>
+		"genres": {Tag: "div", Class: "genres", Multiple: true},
+		// <div class="summary"><p>...</p></div>
+		"summary": {Tag: "div", Class: "summary"},
+		// <span class="chapter-count">123 Chapters</span>
+		"total_chapters": {Tag: "span", Class: "chapter-count"},
+	}
+}
+
+func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
+	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
+		URL:             bookURL,
+		WaitFor:         ".novel-title",
+		WaitForTimeout:  10000,
+		RejectResources: true,
+	})
+	if err != nil {
+		return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
+	}
+
+	root, err := htmlutil.ParseHTML(raw)
+	if err != nil {
+		return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
+	}
+
+	sels := s.MetadataSelectors()
+
+	title := htmlutil.ExtractFirst(root, sels["title"])
+	author := htmlutil.ExtractFirst(root, sels["author"])
+	cover := htmlutil.ExtractFirst(root, sels["cover"])
+	status := htmlutil.ExtractFirst(root, sels["status"])
+
+	// Genres: all <a> tags inside the genres div.
+	genresNode := htmlutil.FindFirst(root, sels["genres"])
+	var genres []string
+	if genresNode != nil {
+		genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
+	}
+
+	summary := htmlutil.ExtractFirst(root, sels["summary"])
+
+	totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"])
+	totalChapters := parseChapterCount(totalStr)
+
+	// Derive slug from URL.
+	slug := slugFromURL(bookURL)
+
+	return scraper.BookMeta{
+		Slug:          slug,
+		Title:         title,
+		Author:        author,
+		Cover:         cover,
+		Status:        status,
+		Genres:        genres,
+		Summary:       summary,
+		TotalChapters: totalChapters,
+		SourceURL:     bookURL,
+	}, nil
+}
+
+// ─── ChapterListProvider ─────────────────────────────────────────────────────
+
+func (s *Scraper) ChaptersURL(bookURL string) string {
+	return strings.TrimRight(bookURL, "/") + "/chapters"
+}
+
+func (s *Scraper) ChapterEntrySelector() scraper.Selector {
+	// <li class="chapter-item"><a href="/book/slug/chapter-1">Chapter 1: Title</a></li>
+	return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true}
+}
+
+func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
+	var refs []scraper.ChapterRef
+	pageURL := s.ChaptersURL(bookURL)
+	page := 1
+
+	for pageURL != "" {
+		select {
+		case <-ctx.Done():
+			return refs, ctx.Err()
+		default:
+		}
+
+		s.log.Info("scraping chapter list", "page", page, "url", pageURL)
+
+		raw, err := s.client.GetContent(ctx, browser.ContentRequest{
+			URL:             pageURL,
+			WaitFor:         ".chapter-item",
+			WaitForTimeout:  10000,
+			RejectResources: true,
+		})
+		if err != nil {
+			return refs, fmt.Errorf("chapter list page %d: %w", page, err)
+		}
+
+		root, err := htmlutil.ParseHTML(raw)
+		if err != nil {
+			return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
+		}
+
+		items := htmlutil.FindAll(root, s.ChapterEntrySelector())
+		for _, item := range items {
+			linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"})
+			if linkNode == nil {
+				continue
+			}
+			href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
+			chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
+			if href == "" {
+				continue
+			}
+			chURL := resolveURL(baseURL, href)
+			num := len(refs) + 1
+			refs = append(refs, scraper.ChapterRef{
+				Number: num,
+				Title:  strings.TrimSpace(chTitle),
+				URL:    chURL,
+			})
+		}
+
+		// Next page.
+		nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
+		if nextHref == "" {
+			break
+		}
+		pageURL = resolveURL(baseURL, nextHref)
+		page++
+	}
+
+	return refs, nil
+}
+
+// ─── ChapterTextProvider ─────────────────────────────────────────────────────
+
+func (s *Scraper) ChapterTextSelector() scraper.Selector {
+	// <div id="chapter-container"> or <div class="chapter-content">
+	return scraper.Selector{Tag: "div", ID: "chapter-container"}
+}
+
+func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
+	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
+		URL:             ref.URL,
+		WaitFor:         "#chapter-container",
+		WaitForTimeout:  15000,
+		RejectResources: true,
+	})
+	if err != nil {
+		return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
+	}
+
+	root, err := htmlutil.ParseHTML(raw)
+	if err != nil {
+		return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
+	}
+
+	container := htmlutil.FindFirst(root, s.ChapterTextSelector())
+	if container == nil {
+		// Fallback: try class-based selector.
+		container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"})
+	}
+	if container == nil {
+		return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL)
+	}
+
+	text := htmlutil.NodeToMarkdown(container)
+
+	return scraper.Chapter{
+		Ref:  ref,
+		Text: text,
+	}, nil
+}
+
+// ─── helpers ─────────────────────────────────────────────────────────────────
+
+func resolveURL(base, href string) string {
+	if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
+		return href
+	}
+	b, err := url.Parse(base)
+	if err != nil {
+		return base + href
+	}
+	ref, err := url.Parse(href)
+	if err != nil {
+		return base + href
+	}
+	return b.ResolveReference(ref).String()
+}
+
+func slugFromURL(bookURL string) string {
+	u, err := url.Parse(bookURL)
+	if err != nil {
+		return bookURL
+	}
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(parts) >= 2 && parts[0] == "book" {
+		return parts[1]
+	}
+	if len(parts) > 0 {
+		return parts[len(parts)-1]
+	}
+	return ""
+}
+
+func parseChapterCount(s string) int {
+	// Formats: "123 Chapters", "1,234 Chapters", "123"
+	s = strings.ReplaceAll(s, ",", "")
+	fields := strings.Fields(s)
+	if len(fields) == 0 {
+		return 0
+	}
+	n, _ := strconv.Atoi(fields[0])
+	return n
+}