kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat: add exponential backoff, some UI elements to see the resut of a scrape

Browse Source
This commit is contained in:
Admin
2026-02-26 18:51:32 +05:00
parent d68ea71239
commit e6e6f7dc4d
12 changed files with 462 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
scraper/internal/novelfire/scraper.go
View File

@@ -15,6 +15,7 @@ import (
"net/url"
"strconv"
"strings"
"time"
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/scraper"
@@ -26,6 +27,27 @@ const (
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
)
// rejectResourceTypes lists Browserless resource types to block on every request.
// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
// Everything else is safe to drop for HTML-only scraping.
var rejectResourceTypes = []string{
"cspviolationreport",
"eventsource",
"fedcm",
"font",
"image",
"manifest",
"media",
"other",
"ping",
"prefetch",
"preflight",
"signedexchange",
"stylesheet",
"texttrack",
"websocket",
}
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
// It uses the /content strategy by default (rendered HTML via Browserless).
type Scraper struct {
@@ -46,20 +68,6 @@ func (s *Scraper) SourceName() string { return "novelfire.net" }
// ─── CatalogueProvider ───────────────────────────────────────────────────────
func (s *Scraper) CatalogueURL() string {
return baseURL + cataloguePath
}
func (s *Scraper) EntriesSelector() scraper.Selector {
// Each novel card: <div class="novel-item">
return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}
}
func (s *Scraper) NextPageSelector() scraper.Selector {
// <a class="next" href="...">
return scraper.Selector{Tag: "a", Class: "next", Attr: "href"}
}
// ScrapeCatalogue streams all CatalogueEntry values across all pages.
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
entries := make(chan scraper.CatalogueEntry, 64)
@@ -69,7 +77,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
defer close(entries)
defer close(errs)
pageURL := s.CatalogueURL()
pageURL := baseURL + cataloguePath
page := 1
for pageURL != "" {
@@ -80,17 +88,34 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
}
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
s.log.Debug("catalogue page fetch starting",
"page", page,
"payload_url", pageURL,
"payload_wait_selector", ".novel-item",
"payload_wait_selector_timeout_ms", 10000,
"payload_wait_for_timeout_ms", 10000,
)
html, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: ".novel-item",
WaitForTimeout: 10000,
RejectResources: true,
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 10000},
WaitForTimeout: 10000,
RejectResourceTypes: rejectResourceTypes,
})
if err != nil {
s.log.Debug("catalogue page fetch failed",
"page", page,
"url", pageURL,
"err", err,
)
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
return
}
s.log.Debug("catalogue page fetch completed",
"page", page,
"url", pageURL,
"response_bytes", len(html),
)
root, err := htmlutil.ParseHTML(html)
if err != nil {
@@ -98,8 +123,8 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
return
}
// Extract novel cards.
cards := htmlutil.FindAll(root, s.EntriesSelector())
// Extract novel cards: <div class="novel-item">
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
if len(cards) == 0 {
s.log.Warn("no novel cards found, stopping pagination", "page", page)
return
@@ -107,8 +132,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
for _, card := range cards {
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"}
titleNode := htmlutil.FindFirst(card, titleSel)
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
var title, href string
if titleNode != nil {
@@ -122,7 +146,6 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
continue
}
// Resolve relative URL.
bookURL := resolveURL(baseURL, href)
select {
case <-ctx.Done():
@@ -131,8 +154,8 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
}
}
// Find next page link.
nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
// Find next page link: <a class="next" href="...">
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
if nextHref == "" {
break
}
@@ -146,64 +169,56 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
// ─── MetadataProvider ────────────────────────────────────────────────────────
func (s *Scraper) MetadataSelectors() map[string]scraper.Selector {
return map[string]scraper.Selector{
// <h1 class="novel-title">Title</h1>
"title": {Tag: "h1", Class: "novel-title"},
// <span class="author"><a>Author Name</a></span>
"author": {Tag: "span", Class: "author"},
// <img class="cover" src="...">
"cover": {Tag: "img", Class: "cover", Attr: "src"},
// <span class="status">Ongoing</span>
"status": {Tag: "span", Class: "status"},
// <div class="genres"><a>Tag1</a><a>Tag2</a>…</div>
"genres": {Tag: "div", Class: "genres", Multiple: true},
// <div class="summary"><p>...</p></div>
"summary": {Tag: "div", Class: "summary"},
// <span class="chapter-count">123 Chapters</span>
"total_chapters": {Tag: "span", Class: "chapter-count"},
}
}
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
s.log.Debug("metadata fetch starting",
"payload_url", bookURL,
"payload_wait_selector", ".novel-title",
"payload_wait_selector_timeout_ms", 10000,
"payload_wait_for_timeout_ms", 10000,
)
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: bookURL,
WaitFor: ".novel-title",
WaitForTimeout: 10000,
RejectResources: true,
URL: bookURL,
WaitFor: &browser.WaitForSelector{Selector: ".novel-title", Timeout: 10000},
WaitForTimeout: 10000,
RejectResourceTypes: rejectResourceTypes,
})
if err != nil {
s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
}
s.log.Debug("metadata fetch completed", "url", bookURL, "response_bytes", len(raw))
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
}
sels := s.MetadataSelectors()
// <h1 class="novel-title">Title</h1>
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
// <span class="author"><a>Author Name</a></span>
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
// <img class="cover" src="...">
cover := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "img", Class: "cover", Attr: "src"})
// <span class="status">Ongoing</span>
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
title := htmlutil.ExtractFirst(root, sels["title"])
author := htmlutil.ExtractFirst(root, sels["author"])
cover := htmlutil.ExtractFirst(root, sels["cover"])
status := htmlutil.ExtractFirst(root, sels["status"])
// Genres: all <a> tags inside the genres div.
genresNode := htmlutil.FindFirst(root, sels["genres"])
// Genres: all <a> tags inside <div class="genres">
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
var genres []string
if genresNode != nil {
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
}
summary := htmlutil.ExtractFirst(root, sels["summary"])
totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"])
// <div class="summary"><p>...</p></div>
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
// <span class="chapter-count">123 Chapters</span>
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
totalChapters := parseChapterCount(totalStr)
// Derive slug from URL.
slug := slugFromURL(bookURL)
return scraper.BookMeta{
meta := scraper.BookMeta{
Slug: slug,
Title: title,
Author: author,
@@ -213,23 +228,25 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.B
Summary: summary,
TotalChapters: totalChapters,
SourceURL: bookURL,
}, nil
}
s.log.Debug("metadata parsed",
"url", bookURL,
"slug", meta.Slug,
"title", meta.Title,
"author", meta.Author,
"status", meta.Status,
"genres", meta.Genres,
"total_chapters", meta.TotalChapters,
)
return meta, nil
}
// ─── ChapterListProvider ─────────────────────────────────────────────────────
func (s *Scraper) ChaptersURL(bookURL string) string {
return strings.TrimRight(bookURL, "/") + "/chapters"
}
func (s *Scraper) ChapterEntrySelector() scraper.Selector {
// <li class="chapter-item"><a href="/book/slug/chapter-1">Chapter 1: Title</a></li>
return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true}
}
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
var refs []scraper.ChapterRef
pageURL := s.ChaptersURL(bookURL)
// Chapter list URL: {bookURL}/chapters
pageURL := strings.TrimRight(bookURL, "/") + "/chapters"
page := 1
for pageURL != "" {
@@ -241,28 +258,51 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
s.log.Debug("chapter list fetch starting",
"page", page,
"payload_url", pageURL,
"payload_wait_selector", ".chapter-list",
"payload_wait_selector_timeout_ms", 10000,
"payload_wait_for_timeout_ms", 10000,
)
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: ".chapter-item",
WaitForTimeout: 10000,
RejectResources: true,
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 10000},
WaitForTimeout: 10000,
RejectResourceTypes: rejectResourceTypes,
})
if err != nil {
s.log.Debug("chapter list fetch failed",
"page", page,
"url", pageURL,
"err", err,
)
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
}
s.log.Debug("chapter list fetch completed",
"page", page,
"url", pageURL,
"response_bytes", len(raw),
)
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
}
items := htmlutil.FindAll(root, s.ChapterEntrySelector())
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
if chapterList == nil {
break
}
// Each chapter row: <li class="chapter-item"><a href="...">Title</a></li>
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
for _, item := range items {
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"})
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
if linkNode == nil {
continue
}
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
if href == "" {
continue
@@ -276,8 +316,15 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
})
}
// Next page.
nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
s.log.Debug("chapter list page parsed",
"page", page,
"url", pageURL,
"chapters_on_page", len(items),
"total_refs_so_far", len(refs),
)
// Next page: <a class="next" href="...">
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
if nextHref == "" {
break
}
@@ -290,38 +337,105 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
// ─── ChapterTextProvider ─────────────────────────────────────────────────────
func (s *Scraper) ChapterTextSelector() scraper.Selector {
// <div id="chapter-container"> or <div class="chapter-content">
return scraper.Selector{Tag: "div", ID: "chapter-container"}
// retryGetContent calls client.GetContent up to maxAttempts times, backing off
// exponentially between retries. Only errors that look like transient Browserless
// 5xx responses (navigation timeouts, etc.) are retried; context cancellation and
// permanent errors are returned immediately.
func retryGetContent(
ctx context.Context,
log *slog.Logger,
client browser.BrowserClient,
req browser.ContentRequest,
maxAttempts int,
baseDelay time.Duration,
) (string, error) {
var lastErr error
delay := baseDelay
for attempt := 1; attempt <= maxAttempts; attempt++ {
html, err := client.GetContent(ctx, req)
if err == nil {
return html, nil
}
lastErr = err
// Stop immediately on context cancellation.
if ctx.Err() != nil {
return "", err
}
// Only retry on Browserless 5xx responses.
if !strings.Contains(err.Error(), "unexpected status 5") {
return "", err
}
if attempt < maxAttempts {
log.Warn("chapter fetch failed, retrying",
"url", req.URL,
"attempt", attempt,
"max_attempts", maxAttempts,
"retry_in", delay,
"err", err,
)
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(delay):
}
delay *= 2
}
}
return "", lastErr
}
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: ref.URL,
WaitFor: "#chapter-container",
WaitForTimeout: 15000,
RejectResources: true,
})
s.log.Debug("chapter text fetch starting",
"chapter", ref.Number,
"title", ref.Title,
"payload_url", ref.URL,
"payload_wait_selector", "#content",
"payload_wait_selector_timeout_ms", 75000,
"payload_wait_for_timeout_ms", 75000,
)
raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{
URL: ref.URL,
WaitFor: &browser.WaitForSelector{Selector: "#content", Timeout: 75000},
WaitForTimeout: 75000,
RejectResourceTypes: rejectResourceTypes,
}, 9, 6*time.Second)
if err != nil {
s.log.Debug("chapter text fetch failed",
"chapter", ref.Number,
"url", ref.URL,
"err", err,
)
return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
}
s.log.Debug("chapter text fetch completed",
"chapter", ref.Number,
"url", ref.URL,
"response_bytes", len(raw),
)
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
}
container := htmlutil.FindFirst(root, s.ChapterTextSelector())
// <div id="content">…</div>
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
if container == nil {
// Fallback: try class-based selector.
container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"})
}
if container == nil {
return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL)
return scraper.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
}
text := htmlutil.NodeToMarkdown(container)
s.log.Debug("chapter text parsed",
"chapter", ref.Number,
"url", ref.URL,
"text_bytes", len(text),
)
return scraper.Chapter{
Ref: ref,
Text: text,