feat: initial commit

scraper/internal/scraper/interfaces.go

@@ -0,0 +1,149 @@
+// Package scraper defines the core interfaces and domain types for the libnovel
+// scraping system. Each novel source implements these interfaces; the orchestrator
+// wires them together without knowing anything about the concrete provider.
+package scraper
+
+import "context"
+
+// ─── Domain types ────────────────────────────────────────────────────────────
+
+// BookMeta carries all bibliographic information about a novel.
+type BookMeta struct {
+	// Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world".
+	Slug string `yaml:"slug"`
+	// Title is the human-readable novel title.
+	Title string `yaml:"title"`
+	// Author of the novel.
+	Author string `yaml:"author"`
+	// Cover is an absolute URL to the cover image.
+	Cover string `yaml:"cover,omitempty"`
+	// Status is e.g. "Ongoing", "Completed".
+	Status string `yaml:"status,omitempty"`
+	// Genres is a list of genre tags.
+	Genres []string `yaml:"genres,omitempty"`
+	// Summary is the full description/synopsis text.
+	Summary string `yaml:"summary,omitempty"`
+	// TotalChapters is the total number of chapters known at scrape time.
+	TotalChapters int `yaml:"total_chapters,omitempty"`
+	// SourceURL is the canonical URL of the book's landing page.
+	SourceURL string `yaml:"source_url"`
+}
+
+// CatalogueEntry is a lightweight reference returned by CatalogueProvider.
+type CatalogueEntry struct {
+	// Title is the novel title as shown in the catalogue listing.
+	Title string
+	// URL is the canonical landing-page URL of the novel.
+	URL string
+}
+
+// ChapterRef is a reference to a single chapter returned by ChapterListProvider.
+type ChapterRef struct {
+	// Number is the 1-based chapter index within the book.
+	Number int
+	// Title is the chapter display title.
+	Title string
+	// URL is the full URL of the chapter page.
+	URL string
+	// Volume is an optional volume number (0 means no volume grouping).
+	Volume int
+}
+
+// Chapter contains the fully-extracted text of a single chapter.
+type Chapter struct {
+	Ref ChapterRef
+	// Text is the plain / lightly-formatted chapter body (Markdown).
+	Text string
+}
+
+// ─── Scraping selector descriptors ───────────────────────────────────────────
+
+// Selector describes how to locate an element in an HTML document.
+// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set
+// they are combined (AND semantics).
+type Selector struct {
+	// Tag is the HTML element name, e.g. "div", "p", "h1".
+	Tag string
+	// Class is one CSS class name (without the leading dot).
+	Class string
+	// ID is the element id attribute (without the leading #).
+	ID string
+	// Attr is an optional attribute name whose value should be extracted
+	// instead of the text content (e.g. "href", "src").
+	Attr string
+	// Multiple indicates that all matching elements should be collected,
+	// not just the first one.
+	Multiple bool
+}
+
+// ─── Provider interfaces ──────────────────────────────────────────────────────
+
+// CatalogueProvider can enumerate every novel available on a source site.
+// It handles pagination transparently and streams CatalogueEntry values.
+type CatalogueProvider interface {
+	// CatalogueURL returns the root URL of the catalogue listing.
+	CatalogueURL() string
+
+	// EntriesSelector returns the selector that matches each novel card / row
+	// in the catalogue listing page.
+	EntriesSelector() Selector
+
+	// NextPageSelector returns the selector for the "next page" link.
+	// If the current page has no next page the implementation must return
+	// ("", nil) from ScrapeNextPage.
+	NextPageSelector() Selector
+
+	// ScrapeCatalogue pages through the entire catalogue, sending
+	// CatalogueEntry values to the returned channel. The channel is closed
+	// when all pages have been scraped or ctx is cancelled.
+	// Errors are surfaced via the error channel; a non-nil error does not
+	// necessarily terminate scraping.
+	ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error)
+}
+
+// MetadataProvider can extract structured book metadata from a novel's landing page.
+type MetadataProvider interface {
+	// MetadataSelectors returns a map of field name → Selector used to
+	// locate each metadata element on the book page.
+	// Required keys: "title", "author".
+	// Optional keys: "cover", "status", "genres", "summary", "total_chapters".
+	MetadataSelectors() map[string]Selector
+
+	// ScrapeMetadata fetches and parses the metadata for the book at bookURL.
+	ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error)
+}
+
+// ChapterListProvider can enumerate all chapters of a book from the chapter-list page.
+type ChapterListProvider interface {
+	// ChaptersURL derives the chapter-list URL from a book landing-page URL.
+	ChaptersURL(bookURL string) string
+
+	// ChapterEntrySelector returns the selector that matches each chapter row
+	// in the chapter list page.
+	ChapterEntrySelector() Selector
+
+	// ScrapeChapterList returns all chapter references for a book, ordered
+	// by chapter number ascending.
+	ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error)
+}
+
+// ChapterTextProvider can extract the readable text from a single chapter page.
+type ChapterTextProvider interface {
+	// ChapterTextSelector returns the selector that wraps the chapter body.
+	ChapterTextSelector() Selector
+
+	// ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown.
+	ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error)
+}
+
+// NovelScraper is the full interface that a concrete novel source must implement.
+// It composes all four provider interfaces.
+type NovelScraper interface {
+	CatalogueProvider
+	MetadataProvider
+	ChapterListProvider
+	ChapterTextProvider
+
+	// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
+	SourceName() string
+}