cfd893d24b
novelfire.net chapter-list pages (/chapters?page=N) are server-rendered — verified via curl. Switch urlClient to NewDirectHTTPClient alongside the existing chapterClient. Remove BROWSERLESS_URL_STRATEGY env var and clean up the now-irrelevant WaitFor/GotoOptions fields from both ScrapeChapterList and ScrapeChapterListPage ContentRequests.
765 lines
23 KiB
Go
765 lines
23 KiB
Go
// Package novelfire provides a NovelScraper implementation for novelfire.net.
|
|
//
|
|
// Site structure (as of 2025):
|
|
//
|
|
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
|
|
// Book page : https://novelfire.net/book/{slug}
|
|
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
|
|
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
|
|
package novelfire
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/url"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/browser"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
const (
|
|
baseURL = "https://novelfire.net"
|
|
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
|
|
rankingPath = "/genre-all/sort-popular/status-all/all-novel"
|
|
)
|
|
|
|
// rejectResourceTypes lists Browserless resource types to block on every request.
|
|
// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
|
|
// Everything else is safe to drop for HTML-only scraping.
|
|
var rejectResourceTypes = []string{
|
|
"cspviolationreport",
|
|
"eventsource",
|
|
"fedcm",
|
|
"font",
|
|
"image",
|
|
"manifest",
|
|
"media",
|
|
"other",
|
|
"ping",
|
|
"signedexchange",
|
|
"stylesheet",
|
|
"texttrack",
|
|
"websocket",
|
|
}
|
|
|
|
// RankingStore is the subset of storage.Store consumed by ScrapeRanking.
|
|
type RankingStore interface {
|
|
WriteRankingItem(ctx context.Context, item scraper.RankingItem) error
|
|
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
|
|
}
|
|
|
|
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
|
// It uses the /content strategy by default (rendered HTML via Browserless).
|
|
type Scraper struct {
|
|
client browser.BrowserClient
|
|
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
|
|
chapterClient browser.BrowserClient // direct HTTP client for chapter text (no JS rendering needed)
|
|
rankingStore RankingStore
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New returns a new novelfire Scraper.
|
|
// client is used for catalogue/metadata/ranking fetching (Browserless).
|
|
// urlClient is used for chapter list pagination; falls back to client if nil.
|
|
// chapterClient is used for chapter text fetching (plain HTTP); falls back to client if nil.
|
|
// rankingStore is optional; pass nil to disable freshness checks and per-item persistence.
|
|
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, chapterClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if urlClient == nil {
|
|
urlClient = client
|
|
}
|
|
if chapterClient == nil {
|
|
chapterClient = client
|
|
}
|
|
return &Scraper{client: client, urlClient: urlClient, chapterClient: chapterClient, rankingStore: rankingStore, log: log}
|
|
}
|
|
|
|
// SourceName implements NovelScraper.
|
|
func (s *Scraper) SourceName() string { return "novelfire.net" }
|
|
|
|
// ─── CatalogueProvider ───────────────────────────────────────────────────────
|
|
|
|
// ScrapeCatalogue streams all CatalogueEntry values across all pages.
|
|
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
|
entries := make(chan scraper.CatalogueEntry, 64)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
pageURL := baseURL + cataloguePath
|
|
page := 1
|
|
|
|
for pageURL != "" {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
|
s.log.Debug("catalogue page fetch starting",
|
|
"page", page,
|
|
"payload_url", pageURL,
|
|
"payload_wait_selector", ".novel-item",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
html, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("catalogue page fetch failed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"err", err,
|
|
)
|
|
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
|
return
|
|
}
|
|
s.log.Debug("catalogue page fetch completed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"response_bytes", len(html),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(html)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
// Extract novel cards: <div class="novel-item">
|
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
|
|
if len(cards) == 0 {
|
|
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
|
return
|
|
}
|
|
|
|
for _, card := range cards {
|
|
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
|
|
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
|
|
|
|
var title, href string
|
|
if titleNode != nil {
|
|
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
if linkNode != nil {
|
|
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
}
|
|
}
|
|
if href == "" || title == "" {
|
|
continue
|
|
}
|
|
|
|
bookURL := resolveURL(baseURL, href)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
|
|
}
|
|
}
|
|
|
|
// Find next page link: <a class="next" href="...">
|
|
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
|
if nextHref == "" {
|
|
break
|
|
}
|
|
pageURL = resolveURL(baseURL, nextHref)
|
|
page++
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── MetadataProvider ────────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
|
|
s.log.Debug("metadata fetch starting",
|
|
"payload_url", bookURL,
|
|
"payload_wait_selector", ".novel-title",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: bookURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
|
}
|
|
s.log.Debug("metadata fetch completed", "url", bookURL, "response_bytes", len(raw))
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
|
|
}
|
|
|
|
// <h1 class="novel-title">Title</h1>
|
|
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
|
// <span class="author"><a>Author Name</a></span>
|
|
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
|
// <figure class="cover"><img src="..."></figure>
|
|
var cover string
|
|
if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil {
|
|
cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"})
|
|
}
|
|
// <span class="status">Ongoing</span>
|
|
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
|
|
|
// Genres: all <a> tags inside <div class="genres">
|
|
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
|
var genres []string
|
|
if genresNode != nil {
|
|
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
|
}
|
|
|
|
// <div class="summary"><p>...</p></div>
|
|
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
|
|
// <span class="chapter-count">123 Chapters</span>
|
|
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
|
|
totalChapters := parseChapterCount(totalStr)
|
|
|
|
slug := slugFromURL(bookURL)
|
|
|
|
meta := scraper.BookMeta{
|
|
Slug: slug,
|
|
Title: title,
|
|
Author: author,
|
|
Cover: cover,
|
|
Status: status,
|
|
Genres: genres,
|
|
Summary: summary,
|
|
TotalChapters: totalChapters,
|
|
SourceURL: bookURL,
|
|
}
|
|
s.log.Debug("metadata parsed",
|
|
"url", bookURL,
|
|
"slug", meta.Slug,
|
|
"title", meta.Title,
|
|
"author", meta.Author,
|
|
"status", meta.Status,
|
|
"genres", meta.Genres,
|
|
"total_chapters", meta.TotalChapters,
|
|
)
|
|
return meta, nil
|
|
}
|
|
|
|
// ─── ChapterListProvider ─────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
|
|
var refs []scraper.ChapterRef
|
|
// Chapter list URL: {bookURL}/chapters?page=N
|
|
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
|
page := 1
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return refs, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
|
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
|
|
|
s.log.Debug("chapter list fetch starting",
|
|
"page", page,
|
|
"payload_url", pageURL,
|
|
"strategy", s.urlClient.Strategy(),
|
|
)
|
|
|
|
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("chapter list fetch failed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"err", err,
|
|
)
|
|
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
|
}
|
|
s.log.Debug("chapter list fetch completed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"response_bytes", len(raw),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
|
|
}
|
|
|
|
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
|
if chapterList == nil {
|
|
// No chapter list container on this page — we've gone past the last page.
|
|
s.log.Debug("chapter list container not found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
// Each chapter row: <li class="chapter-item"><a href="...">Title</a></li>
|
|
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
|
|
|
s.log.Debug("chapter list page parsed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"chapters_on_page", len(items),
|
|
"total_refs_so_far", len(refs),
|
|
)
|
|
|
|
// Zero items on this page means we've gone past the last page.
|
|
if len(items) == 0 {
|
|
s.log.Debug("no chapters on page, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
for _, item := range items {
|
|
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
|
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
if href == "" {
|
|
continue
|
|
}
|
|
chURL := resolveURL(baseURL, href)
|
|
num := chapterNumberFromURL(chURL)
|
|
if num <= 0 {
|
|
// Fall back to position if the URL has no parseable number.
|
|
num = len(refs) + 1
|
|
s.log.Warn("chapter number not parseable from URL, falling back to position",
|
|
"url", chURL,
|
|
"position", num,
|
|
)
|
|
}
|
|
refs = append(refs, scraper.ChapterRef{
|
|
Number: num,
|
|
Title: strings.TrimSpace(chTitle),
|
|
URL: chURL,
|
|
})
|
|
}
|
|
|
|
page++
|
|
}
|
|
|
|
return refs, nil
|
|
}
|
|
|
|
// ScrapeChapterListPage fetches and parses a single chapter-list page URL and
|
|
// returns all chapter refs found on that page without following pagination.
|
|
// pageURL should be the full URL including query params, e.g.:
|
|
//
|
|
// https://novelfire.net/book/shadow-slave/chapters?page=1
|
|
func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]scraper.ChapterRef, error) {
|
|
s.log.Info("scraping chapter list page (single)", "url", pageURL)
|
|
|
|
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chapter list page fetch: %w", err)
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chapter list page parse: %w", err)
|
|
}
|
|
|
|
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
|
if chapterList == nil {
|
|
return nil, fmt.Errorf("chapter list container not found in %s", pageURL)
|
|
}
|
|
|
|
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
|
var refs []scraper.ChapterRef
|
|
for _, item := range items {
|
|
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
|
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
if href == "" {
|
|
continue
|
|
}
|
|
chURL := resolveURL(baseURL, href)
|
|
num := chapterNumberFromURL(chURL)
|
|
if num <= 0 {
|
|
num = len(refs) + 1
|
|
}
|
|
refs = append(refs, scraper.ChapterRef{
|
|
Number: num,
|
|
Title: strings.TrimSpace(chTitle),
|
|
URL: chURL,
|
|
})
|
|
}
|
|
return refs, nil
|
|
}
|
|
|
|
// ─── RankingProvider ───────────────────────────────────────────────────────────
|
|
|
|
// hasNextPageLink returns true if the HTML document contains a pagination link
|
|
// with rel="next". novelfire.net uses:
|
|
//
|
|
// <a class="page-link" href="...?page=N" rel="next" ...>
|
|
func hasNextPageLink(root *html.Node) bool {
|
|
links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
|
|
for _, a := range links {
|
|
for _, attr := range a.Attr {
|
|
if attr.Key == "rel" && attr.Val == "next" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ScrapeRanking pages through up to maxPages pages of the popular-novels genre
|
|
// listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel).
|
|
// Pages are fetched one at a time, strictly sequentially.
|
|
// maxPages <= 0 means "fetch all pages until no more are found".
|
|
//
|
|
// If a RankingStore was provided and the stored ranking is fresh (< 24 hours old),
|
|
// both channels are closed immediately without any network traffic.
|
|
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
|
|
entries := make(chan scraper.BookMeta, 32)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
// Freshness check: skip scraping if data is recent enough.
|
|
if s.rankingStore != nil {
|
|
fresh, err := s.rankingStore.RankingFreshEnough(ctx, 24*time.Hour)
|
|
if err != nil {
|
|
s.log.Warn("ranking freshness check failed, proceeding with scrape", "err", err)
|
|
} else if fresh {
|
|
s.log.Info("ranking data is fresh, skipping scrape")
|
|
return
|
|
}
|
|
}
|
|
|
|
rank := 1
|
|
|
|
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
|
|
|
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
|
|
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
|
return
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
// Real novelfire.net popular listing structure:
|
|
// <li class="novel-item">
|
|
// <a href="/book/slug" title="Title">
|
|
// <figure class="novel-cover"><img data-src="..."></figure>
|
|
// <h4 class="novel-title text2row">Title</h4>
|
|
// </a>
|
|
// </li>
|
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
|
if len(cards) == 0 {
|
|
s.log.Debug("no novel cards found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
for _, card := range cards {
|
|
// The outer <a> carries the href and title attribute.
|
|
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
bookURL := resolveURL(baseURL, href)
|
|
if bookURL == "" {
|
|
continue
|
|
}
|
|
|
|
// Title: prefer <h4 class="novel-title"> text; fall back to <a title="...">
|
|
title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
|
|
if title == "" {
|
|
title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
|
|
}
|
|
if title == "" {
|
|
continue
|
|
}
|
|
|
|
// Cover: <figure class="novel-cover"><img data-src="...">
|
|
var cover string
|
|
if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
|
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
|
|
if cover == "" {
|
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
|
|
}
|
|
// Filter out base64 placeholder images.
|
|
if strings.HasPrefix(cover, "data:") {
|
|
cover = ""
|
|
}
|
|
if cover != "" && !strings.HasPrefix(cover, "http") {
|
|
cover = baseURL + cover
|
|
}
|
|
}
|
|
|
|
bookSlug := slugFromURL(bookURL)
|
|
|
|
meta := scraper.BookMeta{
|
|
Slug: bookSlug,
|
|
Title: title,
|
|
Cover: cover,
|
|
SourceURL: bookURL,
|
|
Ranking: rank,
|
|
}
|
|
rank++
|
|
|
|
// Persist item to store immediately.
|
|
if s.rankingStore != nil {
|
|
item := scraper.RankingItem{
|
|
Rank: meta.Ranking,
|
|
Slug: meta.Slug,
|
|
Title: meta.Title,
|
|
Cover: meta.Cover,
|
|
SourceURL: meta.SourceURL,
|
|
}
|
|
if werr := s.rankingStore.WriteRankingItem(ctx, item); werr != nil {
|
|
s.log.Warn("ranking item write failed", "slug", meta.Slug, "err", werr)
|
|
}
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- meta:
|
|
}
|
|
}
|
|
|
|
// Stop if no next-page link exists.
|
|
// The real pagination uses <a rel="next" ...> inside .pagination.
|
|
if !hasNextPageLink(root) {
|
|
s.log.Debug("no next-page link found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── ChapterTextProvider ─────────────────────────────────────────────────────
|
|
|
|
// retryGetContent calls client.GetContent up to maxAttempts times, backing off
|
|
// exponentially between retries. Only errors that look like transient Browserless
|
|
// failures (timeouts, 5xx responses) are retried; context cancellation and
|
|
// permanent errors are returned immediately.
|
|
func retryGetContent(
|
|
ctx context.Context,
|
|
log *slog.Logger,
|
|
client browser.BrowserClient,
|
|
req browser.ContentRequest,
|
|
maxAttempts int,
|
|
baseDelay time.Duration,
|
|
) (string, error) {
|
|
var lastErr error
|
|
delay := baseDelay
|
|
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
|
html, err := client.GetContent(ctx, req)
|
|
if err == nil {
|
|
return html, nil
|
|
}
|
|
lastErr = err
|
|
|
|
// Stop immediately on context cancellation.
|
|
if ctx.Err() != nil {
|
|
return "", err
|
|
}
|
|
|
|
if attempt < maxAttempts {
|
|
log.Warn("chapter fetch failed, retrying",
|
|
"url", req.URL,
|
|
"attempt", attempt,
|
|
"max_attempts", maxAttempts,
|
|
"retry_in", delay,
|
|
"err", err,
|
|
)
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
delay *= 2
|
|
}
|
|
}
|
|
return "", lastErr
|
|
}
|
|
|
|
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
|
s.log.Debug("chapter text fetch starting",
|
|
"chapter", ref.Number,
|
|
"title", ref.Title,
|
|
"payload_url", ref.URL,
|
|
"payload_wait_selector", "#content",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
raw, err := retryGetContent(ctx, s.log, s.chapterClient, browser.ContentRequest{
|
|
URL: ref.URL,
|
|
}, 9, 6*time.Second)
|
|
if err != nil {
|
|
s.log.Debug("chapter text fetch failed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"err", err,
|
|
)
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
|
|
}
|
|
if len(raw) > 0 {
|
|
preview := raw
|
|
if len(preview) > 500 {
|
|
preview = preview[:500]
|
|
}
|
|
s.log.Debug("chapter text fetch partial content",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"response_bytes", len(raw),
|
|
"preview", preview,
|
|
)
|
|
}
|
|
s.log.Debug("chapter text fetch completed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"response_bytes", len(raw),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
|
|
}
|
|
|
|
// <div id="content">…</div>
|
|
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
|
|
if container == nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
|
|
}
|
|
|
|
text := htmlutil.NodeToMarkdown(container)
|
|
|
|
s.log.Debug("chapter text parsed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"text_bytes", len(text),
|
|
)
|
|
|
|
return scraper.Chapter{
|
|
Ref: ref,
|
|
Text: text,
|
|
}, nil
|
|
}
|
|
|
|
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
|
|
func resolveURL(base, href string) string {
|
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
|
return href
|
|
}
|
|
b, err := url.Parse(base)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
ref, err := url.Parse(href)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
return b.ResolveReference(ref).String()
|
|
}
|
|
|
|
func slugFromURL(bookURL string) string {
|
|
u, err := url.Parse(bookURL)
|
|
if err != nil {
|
|
return bookURL
|
|
}
|
|
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
if len(parts) >= 2 && parts[0] == "book" {
|
|
return parts[1]
|
|
}
|
|
if len(parts) > 0 {
|
|
return parts[len(parts)-1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func parseChapterCount(s string) int {
|
|
// Formats: "123 Chapters", "1,234 Chapters", "123"
|
|
s = strings.ReplaceAll(s, ",", "")
|
|
fields := strings.Fields(s)
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(fields[0])
|
|
return n
|
|
}
|
|
|
|
// chapterNumberFromURL extracts the chapter number from a novelfire chapter URL.
|
|
//
|
|
// URL pattern: https://novelfire.net/book/{book-slug}/chapter-{N}
|
|
// The last path segment is expected to be "chapter-{N}" or "{N}".
|
|
// Returns 0 if no number can be parsed.
|
|
func chapterNumberFromURL(chapterURL string) int {
|
|
u, err := url.Parse(chapterURL)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
seg := path.Base(u.Path) // e.g. "chapter-42" or "42"
|
|
// Strip a "chapter-" prefix if present.
|
|
seg = strings.TrimPrefix(seg, "chapter-")
|
|
// Also handle "chap-", "ch-" variants used by some sites.
|
|
seg = strings.TrimPrefix(seg, "chap-")
|
|
seg = strings.TrimPrefix(seg, "ch-")
|
|
// Take only the leading digits (handles slugs like "42-title-text").
|
|
digits := strings.FieldsFunc(seg, func(r rune) bool {
|
|
return r < '0' || r > '9'
|
|
})
|
|
if len(digits) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(digits[0])
|
|
return n
|
|
}
|