feat: add exponential backoff, some UI elements to see the resut of a scrape

scraper/internal/scraper/htmlutil/htmlutil.go

@@ -3,6 +3,7 @@
 package htmlutil
 
 import (
+	"regexp"
 	"strings"
 
 	"github.com/libnovel/scraper/internal/scraper"
@@ -150,13 +151,20 @@ func InnerHTML(n *html.Node) string {
 
 // NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
 // representation suitable for chapter storage. Block elements become newlines;
-// inline elements are inlined.
+// inline elements are inlined. Runs of more than one blank line are collapsed
+// to a single blank line.
 func NodeToMarkdown(n *html.Node) string {
 	var sb strings.Builder
 	nodeToMD(n, &sb)
-	return strings.TrimSpace(sb.String())
+	// Collapse 3+ consecutive newlines (i.e. more than one blank line) to 2.
+	out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n")
+	return strings.TrimSpace(out)
 }
 
+// multiBlankLine matches three or more consecutive newline characters
+// (any mix of \n and surrounding whitespace-only lines).
+var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`)
+
 var blockElements = map[string]bool{
 	"p": true, "div": true, "br": true, "h1": true, "h2": true,
 	"h3": true, "h4": true, "h5": true, "h6": true, "li": true,

scraper/internal/scraper/interfaces.go

@@ -81,18 +81,6 @@ type Selector struct {
 // CatalogueProvider can enumerate every novel available on a source site.
 // It handles pagination transparently and streams CatalogueEntry values.
 type CatalogueProvider interface {
-	// CatalogueURL returns the root URL of the catalogue listing.
-	CatalogueURL() string
-
-	// EntriesSelector returns the selector that matches each novel card / row
-	// in the catalogue listing page.
-	EntriesSelector() Selector
-
-	// NextPageSelector returns the selector for the "next page" link.
-	// If the current page has no next page the implementation must return
-	// ("", nil) from ScrapeNextPage.
-	NextPageSelector() Selector
-
 	// ScrapeCatalogue pages through the entire catalogue, sending
 	// CatalogueEntry values to the returned channel. The channel is closed
 	// when all pages have been scraped or ctx is cancelled.
@@ -103,25 +91,12 @@ type CatalogueProvider interface {
 
 // MetadataProvider can extract structured book metadata from a novel's landing page.
 type MetadataProvider interface {
-	// MetadataSelectors returns a map of field name → Selector used to
-	// locate each metadata element on the book page.
-	// Required keys: "title", "author".
-	// Optional keys: "cover", "status", "genres", "summary", "total_chapters".
-	MetadataSelectors() map[string]Selector
-
 	// ScrapeMetadata fetches and parses the metadata for the book at bookURL.
 	ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error)
 }
 
 // ChapterListProvider can enumerate all chapters of a book from the chapter-list page.
 type ChapterListProvider interface {
-	// ChaptersURL derives the chapter-list URL from a book landing-page URL.
-	ChaptersURL(bookURL string) string
-
-	// ChapterEntrySelector returns the selector that matches each chapter row
-	// in the chapter list page.
-	ChapterEntrySelector() Selector
-
 	// ScrapeChapterList returns all chapter references for a book, ordered
 	// by chapter number ascending.
 	ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error)
@@ -129,9 +104,6 @@ type ChapterListProvider interface {
 
 // ChapterTextProvider can extract the readable text from a single chapter page.
 type ChapterTextProvider interface {
-	// ChapterTextSelector returns the selector that wraps the chapter body.
-	ChapterTextSelector() Selector
-
 	// ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown.
 	ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error)
 }