libnovel/scraper/internal/scraper/interfaces.go

// Package scraper defines the core interfaces and domain types for the libnovel
// scraping system. Each novel source implements these interfaces; the orchestrator
// wires them together without knowing anything about the concrete provider.
package scraper

import "context"

// ─── Domain types ────────────────────────────────────────────────────────────

// BookMeta carries all bibliographic information about a novel.
type BookMeta struct {
	// Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world".
	Slug string `yaml:"slug"`
	// Title is the human-readable novel title.
	Title string `yaml:"title"`
	// Author of the novel.
	Author string `yaml:"author"`
	// Cover is an absolute URL to the cover image.
	Cover string `yaml:"cover,omitempty"`
	// Status is e.g. "Ongoing", "Completed".
	Status string `yaml:"status,omitempty"`
	// Genres is a list of genre tags.
	Genres []string `yaml:"genres,omitempty"`
	// Summary is the full description/synopsis text.
	Summary string `yaml:"summary,omitempty"`
	// TotalChapters is the total number of chapters known at scrape time.
	TotalChapters int `yaml:"total_chapters,omitempty"`
	// SourceURL is the canonical URL of the book's landing page.
	SourceURL string `yaml:"source_url"`
	// Ranking is the rank number from ranking pages.
	Ranking int `yaml:"ranking,omitempty"`
}

// CatalogueEntry is a lightweight reference returned by CatalogueProvider.
type CatalogueEntry struct {
	// Title is the novel title as shown in the catalogue listing.
	Title string
	// URL is the canonical landing-page URL of the novel.
	URL string
}

// ChapterRef is a reference to a single chapter returned by ChapterListProvider.
type ChapterRef struct {
	// Number is the 1-based chapter index within the book.
	Number int
	// Title is the chapter display title.
	Title string
	// URL is the full URL of the chapter page.
	URL string
	// Volume is an optional volume number (0 means no volume grouping).
	Volume int
}

// Chapter contains the fully-extracted text of a single chapter.
type Chapter struct {
	Ref ChapterRef
	// Text is the plain / lightly-formatted chapter body (Markdown).
	Text string
}

// ─── Scraping selector descriptors ───────────────────────────────────────────

// Selector describes how to locate an element in an HTML document.
// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set
// they are combined (AND semantics).
type Selector struct {
	// Tag is the HTML element name, e.g. "div", "p", "h1".
	Tag string
	// Class is one CSS class name (without the leading dot).
	Class string
	// ID is the element id attribute (without the leading #).
	ID string
	// Attr is an optional attribute name whose value should be extracted
	// instead of the text content (e.g. "href", "src").
	Attr string
	// Multiple indicates that all matching elements should be collected,
	// not just the first one.
	Multiple bool
}

// ─── Provider interfaces ──────────────────────────────────────────────────────

// CatalogueProvider can enumerate every novel available on a source site.
// It handles pagination transparently and streams CatalogueEntry values.
type CatalogueProvider interface {
	// ScrapeCatalogue pages through the entire catalogue, sending
	// CatalogueEntry values to the returned channel. The channel is closed
	// when all pages have been scraped or ctx is cancelled.
	// Errors are surfaced via the error channel; a non-nil error does not
	// necessarily terminate scraping.
	ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error)
}

// MetadataProvider can extract structured book metadata from a novel's landing page.
type MetadataProvider interface {
	// ScrapeMetadata fetches and parses the metadata for the book at bookURL.
	ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error)
}

// ChapterListProvider can enumerate all chapters of a book from the chapter-list page.
type ChapterListProvider interface {
	// ScrapeChapterList returns all chapter references for a book, ordered
	// by chapter number ascending.
	ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error)
}

// ChapterTextProvider can extract the readable text from a single chapter page.
type ChapterTextProvider interface {
	// ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown.
	ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error)
}

// RankingProvider can enumerate novels from a ranking page.
type RankingProvider interface {
	// ScrapeRanking pages through up to maxPages ranking pages, sending BookMeta
	// values (with basic info like title, cover, genres, status, sourceURL) to
	// the returned channel. Pages are fetched sequentially and lazily: the next
	// page is only requested once all entries from the current page have been
	// sent. maxPages <= 0 means "all pages".
	ScrapeRanking(ctx context.Context, maxPages int) (<-chan BookMeta, <-chan error)
}

// RankingPageCacher persists and retrieves raw HTML for individual ranking pages.
// Implementations (e.g. writer.Writer) store files on disk so that a
// subsequent ScrapeRanking call can serve cached HTML without a network round-trip.
type RankingPageCacher interface {
	// WriteRankingPageCache stores the raw HTML string for the given page number.
	WriteRankingPageCache(page int, html string) error
	// ReadRankingPageCache returns the cached HTML for page, or ("", nil) on a miss.
	ReadRankingPageCache(page int) (string, error)
}

// NovelScraper is the full interface that a concrete novel source must implement.
// It composes all four provider interfaces.
type NovelScraper interface {
	CatalogueProvider
	MetadataProvider
	ChapterListProvider
	ChapterTextProvider
	RankingProvider

	// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
	SourceName() string
}