// Package scraper defines the core interfaces and domain types for the libnovel // scraping system. Each novel source implements these interfaces; the orchestrator // wires them together without knowing anything about the concrete provider. package scraper import ( "context" "time" ) // ─── Domain types ──────────────────────────────────────────────────────────── // BookMeta carries all bibliographic information about a novel. type BookMeta struct { // Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world". Slug string `yaml:"slug"` // Title is the human-readable novel title. Title string `yaml:"title"` // Author of the novel. Author string `yaml:"author"` // Cover is an absolute URL to the cover image. Cover string `yaml:"cover,omitempty"` // Status is e.g. "Ongoing", "Completed". Status string `yaml:"status,omitempty"` // Genres is a list of genre tags. Genres []string `yaml:"genres,omitempty"` // Summary is the full description/synopsis text. Summary string `yaml:"summary,omitempty"` // TotalChapters is the total number of chapters known at scrape time. TotalChapters int `yaml:"total_chapters,omitempty"` // SourceURL is the canonical URL of the book's landing page. SourceURL string `yaml:"source_url"` // Ranking is the rank number from ranking pages. Ranking int `yaml:"ranking,omitempty"` } // CatalogueEntry is a lightweight reference returned by CatalogueProvider. type CatalogueEntry struct { // Title is the novel title as shown in the catalogue listing. Title string // URL is the canonical landing-page URL of the novel. URL string } // ChapterRef is a reference to a single chapter returned by ChapterListProvider. type ChapterRef struct { // Number is the 1-based chapter index within the book. Number int // Title is the chapter display title. Title string // URL is the full URL of the chapter page. URL string // Volume is an optional volume number (0 means no volume grouping). Volume int } // Chapter contains the fully-extracted text of a single chapter. type Chapter struct { Ref ChapterRef // Text is the plain / lightly-formatted chapter body (Markdown). Text string } // RankingItem represents a single entry in the novel ranking list. type RankingItem struct { Rank int `json:"rank"` Slug string `json:"slug"` Title string `json:"title"` Author string `json:"author,omitempty"` Cover string `json:"cover,omitempty"` Status string `json:"status,omitempty"` Genres []string `json:"genres,omitempty"` SourceURL string `json:"source_url,omitempty"` Updated time.Time `json:"updated,omitempty"` } // ─── Scraping selector descriptors ─────────────────────────────────────────── // Selector describes how to locate an element in an HTML document. // Exactly one of Tag, Class, or ID should be non-empty; when multiple are set // they are combined (AND semantics). type Selector struct { // Tag is the HTML element name, e.g. "div", "p", "h1". Tag string // Class is one CSS class name (without the leading dot). Class string // ID is the element id attribute (without the leading #). ID string // Attr is an optional attribute name whose value should be extracted // instead of the text content (e.g. "href", "src"). Attr string // Multiple indicates that all matching elements should be collected, // not just the first one. Multiple bool } // ─── Provider interfaces ────────────────────────────────────────────────────── // CatalogueProvider can enumerate every novel available on a source site. // It handles pagination transparently and streams CatalogueEntry values. type CatalogueProvider interface { // ScrapeCatalogue pages through the entire catalogue, sending // CatalogueEntry values to the returned channel. The channel is closed // when all pages have been scraped or ctx is cancelled. // Errors are surfaced via the error channel; a non-nil error does not // necessarily terminate scraping. ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error) } // MetadataProvider can extract structured book metadata from a novel's landing page. type MetadataProvider interface { // ScrapeMetadata fetches and parses the metadata for the book at bookURL. ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error) } // ChapterListProvider can enumerate all chapters of a book from the chapter-list page. type ChapterListProvider interface { // ScrapeChapterList returns all chapter references for a book, ordered // by chapter number ascending. ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error) } // ChapterTextProvider can extract the readable text from a single chapter page. type ChapterTextProvider interface { // ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown. ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error) } // RankingProvider can enumerate novels from a ranking page. type RankingProvider interface { // ScrapeRanking pages through up to maxPages ranking pages, sending BookMeta // values (with basic info like title, cover, genres, status, sourceURL) to // the returned channel. Pages are fetched sequentially and lazily: the next // page is only requested once all entries from the current page have been // sent. maxPages <= 0 means "all pages". ScrapeRanking(ctx context.Context, maxPages int) (<-chan BookMeta, <-chan error) } // NovelScraper is the full interface that a concrete novel source must implement. // It composes all four provider interfaces. type NovelScraper interface { CatalogueProvider MetadataProvider ChapterListProvider ChapterTextProvider RankingProvider // SourceName returns the human-readable name of this scraper, e.g. "novelfire.net". SourceName() string }