libnovel/backend/internal/novelfire/scraper.go

// Package novelfire provides a NovelScraper implementation for novelfire.net.
//
// Site structure (as of 2025):
//
//	Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
//	Book page : https://novelfire.net/book/{slug}
//	Chapters  : https://novelfire.net/book/{slug}/chapters?page=N
//	Chapter   : https://novelfire.net/book/{slug}/{chapter-slug}
package novelfire

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"math/rand"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/libnovel/backend/internal/browser"
	"github.com/libnovel/backend/internal/domain"
	"github.com/libnovel/backend/internal/novelfire/htmlutil"
	"github.com/libnovel/backend/internal/scraper"
	"golang.org/x/net/html"
)

const (
	baseURL       = "https://novelfire.net"
	cataloguePath = "/genre-all/sort-new/status-all/all-novel"
	rankingPath   = "/genre-all/sort-popular/status-all/all-novel"
)

// Scraper is the novelfire.net implementation of scraper.NovelScraper.
type Scraper struct {
	client browser.Client
	log    *slog.Logger
}

// Compile-time interface check.
var _ scraper.NovelScraper = (*Scraper)(nil)

// New returns a new novelfire Scraper backed by client.
func New(client browser.Client, log *slog.Logger) *Scraper {
	if log == nil {
		log = slog.Default()
	}
	return &Scraper{client: client, log: log}
}

// SourceName implements NovelScraper.
func (s *Scraper) SourceName() string { return "novelfire.net" }

// ── CatalogueProvider ─────────────────────────────────────────────────────────

// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
// Each page fetch uses retryGet with 429-aware exponential backoff.
// A small inter-page delay (cataloguePageDelay) is inserted between requests to
// avoid hammering the server when paging through hundreds of catalogue pages.
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
	entries := make(chan domain.CatalogueEntry, 64)
	errs := make(chan error, 16)

	go func() {
		defer close(entries)
		defer close(errs)

		pageURL := baseURL + cataloguePath
		page := 1

		for pageURL != "" {
			select {
			case <-ctx.Done():
				return
			default:
			}

			// Polite inter-page delay — skipped on the very first page.
			if page > 1 {
				jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond
				select {
				case <-ctx.Done():
					return
				case <-time.After(jitter):
				}
			}

			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
			raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second)
			if err != nil {
				errs <- fmt.Errorf("catalogue page %d: %w", page, err)
				return
			}

			root, err := htmlutil.ParseHTML(raw)
			if err != nil {
				errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
				return
			}

			cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
			if len(cards) == 0 {
				s.log.Warn("no novel cards found, stopping pagination", "page", page)
				return
			}

			for _, card := range cards {
				linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
				titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})

				var title, href string
				if linkNode != nil {
					href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
				}
				if titleNode != nil {
					title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
				}
				if href == "" || title == "" {
					continue
				}

				bookURL := resolveURL(baseURL, href)
				select {
				case <-ctx.Done():
					return
				case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}:
				}
			}

			if !hasNextPageLink(root) {
				break
			}
			nextHref := ""
			for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
				if htmlutil.AttrVal(a, "rel") == "next" {
					nextHref = htmlutil.AttrVal(a, "href")
					break
				}
			}
			if nextHref == "" {
				break
			}
			pageURL = resolveURL(baseURL, nextHref)
			page++
		}
	}()

	return entries, errs
}

// ── MetadataProvider ──────────────────────────────────────────────────────────

// ScrapeMetadata fetches and parses book metadata from the book's landing page.
// Uses retryGet with 429-aware exponential backoff (up to 9 attempts).
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
	s.log.Debug("metadata fetch starting", "url", bookURL)

	raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second)
	if err != nil {
		return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
	}

	root, err := htmlutil.ParseHTML(raw)
	if err != nil {
		return domain.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
	}

	title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
	author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})

	var cover string
	if fig := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
		cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
		if cover != "" && !strings.HasPrefix(cover, "http") {
			cover = baseURL + cover
		}
	}

	// Status: novelfire renders <strong class="ongoing">Ongoing</strong> (or
	// "completed", "hiatus") inside the .header-stats block. We take the text
	// content and lowercase it so the index value is always canonical lowercase.
	var status string
	for _, cls := range []string{"ongoing", "completed", "hiatus"} {
		if v := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "strong", Class: cls}); v != "" {
			status = strings.ToLower(strings.TrimSpace(v))
			break
		}
	}

	// Genres: novelfire renders <div class="categories"><ul><li><a class="property-item">Genre</a>
	// Each <a class="property-item"> is one genre tag. Lowercase for index consistency.
	var genres []string
	if categoriesNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "categories"}); categoriesNode != nil {
		for _, v := range htmlutil.ExtractAll(categoriesNode, scraper.Selector{Tag: "a", Class: "property-item", Multiple: true}) {
			if v != "" {
				genres = append(genres, strings.ToLower(strings.TrimSpace(v)))
			}
		}
	}

	summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
	totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
	totalChapters := parseChapterCount(totalStr)

	slug := slugFromURL(bookURL)

	meta := domain.BookMeta{
		Slug:          slug,
		Title:         title,
		Author:        author,
		Cover:         cover,
		Status:        status,
		Genres:        genres,
		Summary:       summary,
		TotalChapters: totalChapters,
		SourceURL:     bookURL,
	}
	s.log.Debug("metadata parsed", "slug", meta.Slug, "title", meta.Title)
	return meta, nil
}

// ── ChapterListProvider ───────────────────────────────────────────────────────

// ScrapeChapterList returns chapter references for a book, ordered ascending.
// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been
// collected — use this for range scrapes so we don't paginate 100 pages just
// to discover refs we'll never scrape. upTo == 0 fetches all pages.
// Each page fetch uses retryGet with 429-aware exponential backoff.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) {
	var refs []domain.ChapterRef
	baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
	page := 1

	for {
		select {
		case <-ctx.Done():
			return refs, ctx.Err()
		default:
		}

		pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
		s.log.Info("scraping chapter list", "page", page, "url", pageURL)

		raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
		if err != nil {
			return refs, fmt.Errorf("chapter list page %d: %w", page, err)
		}

		root, err := htmlutil.ParseHTML(raw)
		if err != nil {
			return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
		}

		chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
		if chapterList == nil {
			s.log.Debug("chapter list container not found, stopping pagination", "page", page)
			break
		}

		items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
		if len(items) == 0 {
			break
		}

		for _, item := range items {
			linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
			if linkNode == nil {
				continue
			}
			href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
			chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
			if href == "" {
				continue
			}
			chURL := resolveURL(baseURL, href)
			num := chapterNumberFromURL(chURL)
			if num <= 0 {
				num = len(refs) + 1
				s.log.Warn("chapter number not parseable from URL, falling back to position",
					"url", chURL, "position", num)
			}
			refs = append(refs, domain.ChapterRef{
				Number: num,
				Title:  strings.TrimSpace(chTitle),
				URL:    chURL,
			})
		}

		// Early-stop: if we have seen at least upTo chapter numbers, we have
		// enough refs to cover the requested range — no need to paginate further.
		if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo {
			s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs))
			break
		}

		page++
	}

	return refs, nil
}

// ── ChapterTextProvider ───────────────────────────────────────────────────────

// ScrapeChapterText fetches and parses a single chapter page.
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
	s.log.Debug("chapter text fetch starting", "chapter", ref.Number, "url", ref.URL)

	raw, err := retryGet(ctx, s.log, s.client, ref.URL, 9, 6*time.Second)
	if err != nil {
		return domain.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
	}

	root, err := htmlutil.ParseHTML(raw)
	if err != nil {
		return domain.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
	}

	container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
	if container == nil {
		return domain.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
	}

	text := htmlutil.NodeToMarkdown(container)

	s.log.Debug("chapter text parsed", "chapter", ref.Number, "text_bytes", len(text))

	return domain.Chapter{Ref: ref, Text: text}, nil
}

// ── RankingProvider ───────────────────────────────────────────────────────────

// ScrapeRanking pages through up to maxPages pages of the popular-novels listing.
// maxPages <= 0 means all pages. The caller decides whether to persist items.
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
	entries := make(chan domain.BookMeta, 32)
	errs := make(chan error, 16)

	go func() {
		defer close(entries)
		defer close(errs)

		rank := 1

		for page := 1; maxPages <= 0 || page <= maxPages; page++ {
			select {
			case <-ctx.Done():
				return
			default:
			}

			pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
			s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)

			raw, err := s.client.GetContent(ctx, pageURL)
			if err != nil {
				errs <- fmt.Errorf("ranking page %d: %w", page, err)
				return
			}

			root, err := htmlutil.ParseHTML(raw)
			if err != nil {
				errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
				return
			}

			cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
			if len(cards) == 0 {
				break
			}

			for _, card := range cards {
				linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
				if linkNode == nil {
					continue
				}
				href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
				bookURL := resolveURL(baseURL, href)
				if bookURL == "" {
					continue
				}

				title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
				if title == "" {
					title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
				}
				if title == "" {
					continue
				}

				var cover string
				if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
					cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
					if cover == "" {
						cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
					}
					if strings.HasPrefix(cover, "data:") {
						cover = ""
					}
					if cover != "" && !strings.HasPrefix(cover, "http") {
						cover = baseURL + cover
					}
				}

				meta := domain.BookMeta{
					Slug:      slugFromURL(bookURL),
					Title:     title,
					Cover:     cover,
					SourceURL: bookURL,
					Ranking:   rank,
				}
				rank++

				select {
				case <-ctx.Done():
					return
				case entries <- meta:
				}
			}

			if !hasNextPageLink(root) {
				break
			}
		}
	}()

	return entries, errs
}

// ── helpers ───────────────────────────────────────────────────────────────────

func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) }

func hasNextPageLink(root *html.Node) bool {
	links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
	for _, a := range links {
		for _, attr := range a.Attr {
			if attr.Key == "rel" && attr.Val == "next" {
				return true
			}
		}
	}
	return false
}

func slugFromURL(bookURL string) string {
	u, err := url.Parse(bookURL)
	if err != nil {
		return bookURL
	}
	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
	if len(parts) >= 2 && parts[0] == "book" {
		return parts[1]
	}
	if len(parts) > 0 {
		return parts[len(parts)-1]
	}
	return ""
}

func parseChapterCount(s string) int {
	s = strings.ReplaceAll(s, ",", "")
	fields := strings.Fields(s)
	if len(fields) == 0 {
		return 0
	}
	n, _ := strconv.Atoi(fields[0])
	return n
}

func chapterNumberFromURL(chapterURL string) int {
	u, err := url.Parse(chapterURL)
	if err != nil {
		return 0
	}
	seg := path.Base(u.Path)
	seg = strings.TrimPrefix(seg, "chapter-")
	seg = strings.TrimPrefix(seg, "chap-")
	seg = strings.TrimPrefix(seg, "ch-")
	digits := strings.FieldsFunc(seg, func(r rune) bool {
		return r < '0' || r > '9'
	})
	if len(digits) == 0 {
		return 0
	}
	n, _ := strconv.Atoi(digits[0])
	return n
}

// retryGet calls client.GetContent up to maxAttempts times with exponential backoff.
// If the server returns 429 (ErrRateLimit), the suggested Retry-After delay is used
// instead of the geometric backoff delay.
func retryGet(
	ctx context.Context,
	log *slog.Logger,
	client browser.Client,
	pageURL string,
	maxAttempts int,
	baseDelay time.Duration,
) (string, error) {
	var lastErr error
	delay := baseDelay
	for attempt := 1; attempt <= maxAttempts; attempt++ {
		raw, err := client.GetContent(ctx, pageURL)
		if err == nil {
			return raw, nil
		}
		lastErr = err
		if ctx.Err() != nil {
			return "", err
		}
		if attempt < maxAttempts {
			// If the server is rate-limiting us, honour its Retry-After delay.
			waitFor := delay
			var rlErr *browser.RateLimitError
			if errors.As(err, &rlErr) {
				waitFor = rlErr.RetryAfter
				if log != nil {
					log.Warn("rate limited, backing off",
						"url", pageURL, "attempt", attempt, "retry_in", waitFor)
				}
			} else {
				if log != nil {
					log.Warn("fetch failed, retrying",
						"url", pageURL, "attempt", attempt, "retry_in", delay, "err", err)
				}
				delay *= 2
			}
			select {
			case <-ctx.Done():
				return "", ctx.Err()
			case <-time.After(waitFor):
			}
		}
	}
	return "", lastErr
}