1469e49190
- ScrapeRanking now accepts a maxPages int parameter (0 = all pages). Each page is fetched strictly sequentially; the next page is only requested after every entry from the current page has been sent, so there is no pre-fetching or look-ahead. Pagination stops automatically when no next-page link is present or when the rank-novels container is absent/empty. - The ranking URL pattern follows the existing catalogue convention: /ranking?page=N (next-page link detection as the stop condition). - Server: handleRankingRefresh reads an optional 'pages' form field and passes it to ScrapeRanking. Timeout scales at 90 s/page. - UI: Refresh Rankings button is now a small form with a numeric 'Pages' input (default 1), letting the user choose how many pages to pull in one refresh without touching the server config.
671 lines
21 KiB
Go
671 lines
21 KiB
Go
// Package novelfire provides a NovelScraper implementation for novelfire.net.
|
|
//
|
|
// Site structure (as of 2025):
|
|
//
|
|
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
|
|
// Book page : https://novelfire.net/book/{slug}
|
|
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
|
|
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
|
|
package novelfire
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/url"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/browser"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
|
)
|
|
|
|
const (
|
|
baseURL = "https://novelfire.net"
|
|
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
|
|
rankingPath = "/ranking"
|
|
)
|
|
|
|
// rejectResourceTypes lists Browserless resource types to block on every request.
|
|
// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
|
|
// Everything else is safe to drop for HTML-only scraping.
|
|
var rejectResourceTypes = []string{
|
|
"cspviolationreport",
|
|
"eventsource",
|
|
"fedcm",
|
|
"font",
|
|
"image",
|
|
"manifest",
|
|
"media",
|
|
"other",
|
|
"ping",
|
|
"signedexchange",
|
|
"stylesheet",
|
|
"texttrack",
|
|
"websocket",
|
|
}
|
|
|
|
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
|
// It uses the /content strategy by default (rendered HTML via Browserless).
|
|
type Scraper struct {
|
|
client browser.BrowserClient
|
|
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New returns a new novelfire Scraper.
|
|
// client is used for content fetching, urlClient is used for URL retrieval (chapter list).
|
|
// If urlClient is nil, client will be used for both.
|
|
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient) *Scraper {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if urlClient == nil {
|
|
urlClient = client
|
|
}
|
|
return &Scraper{client: client, urlClient: urlClient, log: log}
|
|
}
|
|
|
|
// SourceName implements NovelScraper.
|
|
func (s *Scraper) SourceName() string { return "novelfire.net" }
|
|
|
|
// ─── CatalogueProvider ───────────────────────────────────────────────────────
|
|
|
|
// ScrapeCatalogue streams all CatalogueEntry values across all pages.
|
|
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
|
entries := make(chan scraper.CatalogueEntry, 64)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
pageURL := baseURL + cataloguePath
|
|
page := 1
|
|
|
|
for pageURL != "" {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
|
s.log.Debug("catalogue page fetch starting",
|
|
"page", page,
|
|
"payload_url", pageURL,
|
|
"payload_wait_selector", ".novel-item",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
html, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("catalogue page fetch failed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"err", err,
|
|
)
|
|
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
|
return
|
|
}
|
|
s.log.Debug("catalogue page fetch completed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"response_bytes", len(html),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(html)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
// Extract novel cards: <div class="novel-item">
|
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
|
|
if len(cards) == 0 {
|
|
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
|
return
|
|
}
|
|
|
|
for _, card := range cards {
|
|
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
|
|
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
|
|
|
|
var title, href string
|
|
if titleNode != nil {
|
|
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
if linkNode != nil {
|
|
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
}
|
|
}
|
|
if href == "" || title == "" {
|
|
continue
|
|
}
|
|
|
|
bookURL := resolveURL(baseURL, href)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
|
|
}
|
|
}
|
|
|
|
// Find next page link: <a class="next" href="...">
|
|
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
|
if nextHref == "" {
|
|
break
|
|
}
|
|
pageURL = resolveURL(baseURL, nextHref)
|
|
page++
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── MetadataProvider ────────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
|
|
s.log.Debug("metadata fetch starting",
|
|
"payload_url", bookURL,
|
|
"payload_wait_selector", ".novel-title",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: bookURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
|
}
|
|
s.log.Debug("metadata fetch completed", "url", bookURL, "response_bytes", len(raw))
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
|
|
}
|
|
|
|
// <h1 class="novel-title">Title</h1>
|
|
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
|
// <span class="author"><a>Author Name</a></span>
|
|
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
|
// <figure class="cover"><img src="..."></figure>
|
|
var cover string
|
|
if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil {
|
|
cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"})
|
|
}
|
|
// <span class="status">Ongoing</span>
|
|
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
|
|
|
// Genres: all <a> tags inside <div class="genres">
|
|
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
|
var genres []string
|
|
if genresNode != nil {
|
|
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
|
}
|
|
|
|
// <div class="summary"><p>...</p></div>
|
|
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
|
|
// <span class="chapter-count">123 Chapters</span>
|
|
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
|
|
totalChapters := parseChapterCount(totalStr)
|
|
|
|
slug := slugFromURL(bookURL)
|
|
|
|
meta := scraper.BookMeta{
|
|
Slug: slug,
|
|
Title: title,
|
|
Author: author,
|
|
Cover: cover,
|
|
Status: status,
|
|
Genres: genres,
|
|
Summary: summary,
|
|
TotalChapters: totalChapters,
|
|
SourceURL: bookURL,
|
|
}
|
|
s.log.Debug("metadata parsed",
|
|
"url", bookURL,
|
|
"slug", meta.Slug,
|
|
"title", meta.Title,
|
|
"author", meta.Author,
|
|
"status", meta.Status,
|
|
"genres", meta.Genres,
|
|
"total_chapters", meta.TotalChapters,
|
|
)
|
|
return meta, nil
|
|
}
|
|
|
|
// ─── ChapterListProvider ─────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
|
|
var refs []scraper.ChapterRef
|
|
// Chapter list URL: {bookURL}/chapters?page=N
|
|
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
|
page := 1
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return refs, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
|
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
|
|
|
s.log.Debug("chapter list fetch starting",
|
|
"page", page,
|
|
"payload_url", pageURL,
|
|
"payload_wait_selector", ".chapter-list",
|
|
"payload_wait_selector_timeout_ms", 15000,
|
|
"payload_wait_timeout_ms", 2000,
|
|
"strategy", s.urlClient.Strategy(),
|
|
)
|
|
|
|
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
// Wait up to 15 s for the chapter list container to appear in the DOM.
|
|
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
|
|
// After the selector is found, wait an additional 2 s for any
|
|
// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
|
|
WaitForTimeout: 2000,
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
// Do NOT use BestAttempt — we want a complete page or a clear error,
|
|
// not silently partial HTML that looks like "no more chapters".
|
|
BestAttempt: false,
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("chapter list fetch failed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"err", err,
|
|
)
|
|
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
|
}
|
|
s.log.Debug("chapter list fetch completed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"response_bytes", len(raw),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
|
|
}
|
|
|
|
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
|
if chapterList == nil {
|
|
// No chapter list container on this page — we've gone past the last page.
|
|
s.log.Debug("chapter list container not found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
// Each chapter row: <li class="chapter-item"><a href="...">Title</a></li>
|
|
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
|
|
|
s.log.Debug("chapter list page parsed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"chapters_on_page", len(items),
|
|
"total_refs_so_far", len(refs),
|
|
)
|
|
|
|
// Zero items on this page means we've gone past the last page.
|
|
if len(items) == 0 {
|
|
s.log.Debug("no chapters on page, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
for _, item := range items {
|
|
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
|
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
if href == "" {
|
|
continue
|
|
}
|
|
chURL := resolveURL(baseURL, href)
|
|
num := chapterNumberFromURL(chURL)
|
|
if num <= 0 {
|
|
// Fall back to position if the URL has no parseable number.
|
|
num = len(refs) + 1
|
|
s.log.Warn("chapter number not parseable from URL, falling back to position",
|
|
"url", chURL,
|
|
"position", num,
|
|
)
|
|
}
|
|
refs = append(refs, scraper.ChapterRef{
|
|
Number: num,
|
|
Title: strings.TrimSpace(chTitle),
|
|
URL: chURL,
|
|
})
|
|
}
|
|
|
|
page++
|
|
}
|
|
|
|
return refs, nil
|
|
}
|
|
|
|
// ─── RankingProvider ───────────────────────────────────────────────────────────
|
|
|
|
// ScrapeRanking pages through up to maxPages ranking pages on novelfire.net/ranking.
|
|
// Pages are fetched one at a time, strictly sequentially: the next page is only
|
|
// requested after every entry from the current page has been sent to the channel.
|
|
// maxPages <= 0 means "fetch all pages until no more are found".
|
|
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
|
|
entries := make(chan scraper.BookMeta, 32)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
rank := 1
|
|
|
|
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
|
s.log.Info("scraping ranking page", "page", page, "url", pageURL)
|
|
|
|
// Always use the urlClient for the ranking page — it requires JS rendering.
|
|
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
BestAttempt: true,
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
|
|
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
|
return
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
rankList := htmlutil.FindFirst(root, scraper.Selector{Class: "rank-novels"})
|
|
if rankList == nil {
|
|
s.log.Debug("rank-novels container not found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
items := htmlutil.FindAll(rankList, scraper.Selector{Tag: "li", Class: "novel-item"})
|
|
if len(items) == 0 {
|
|
s.log.Debug("no ranking items on page, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
for _, item := range items {
|
|
// Cover: <figure class="cover"><a href="/book/slug"><img data-src="..."></a></figure>
|
|
var cover string
|
|
if fig := htmlutil.FindFirst(item, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
|
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
|
|
if cover != "" {
|
|
cover = baseURL + cover
|
|
}
|
|
}
|
|
|
|
// Title and URL: <h2 class="title"><a href="/book/slug">Title</a></h2>
|
|
titleNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "h2", Class: "title"})
|
|
var title, bookURL string
|
|
if titleNode != nil {
|
|
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a"})
|
|
if linkNode != nil {
|
|
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
|
bookURL = resolveURL(baseURL, href)
|
|
}
|
|
}
|
|
|
|
// Status: <span class="status"> Ongoing/Completed </span>
|
|
status := htmlutil.ExtractFirst(item, scraper.Selector{Tag: "span", Class: "status"})
|
|
|
|
// Genres: <div class="categories"><div class="scroll"><span>Genre1</span>...</div></div>
|
|
var genres []string
|
|
categoriesNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "div", Class: "categories"})
|
|
if categoriesNode != nil {
|
|
genres = htmlutil.ExtractAll(categoriesNode, scraper.Selector{Tag: "span", Multiple: true})
|
|
}
|
|
|
|
slug := slugFromURL(bookURL)
|
|
|
|
meta := scraper.BookMeta{
|
|
Slug: slug,
|
|
Title: title,
|
|
Cover: cover,
|
|
Status: strings.TrimSpace(status),
|
|
Genres: genres,
|
|
SourceURL: bookURL,
|
|
Ranking: rank,
|
|
}
|
|
rank++
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- meta:
|
|
}
|
|
}
|
|
|
|
// Stop if no next-page link exists (natural end of ranking list).
|
|
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
|
if nextHref == "" {
|
|
s.log.Debug("no next-page link found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── ChapterTextProvider ─────────────────────────────────────────────────────
|
|
|
|
// retryGetContent calls client.GetContent up to maxAttempts times, backing off
|
|
// exponentially between retries. Only errors that look like transient Browserless
|
|
// failures (timeouts, 5xx responses) are retried; context cancellation and
|
|
// permanent errors are returned immediately.
|
|
func retryGetContent(
|
|
ctx context.Context,
|
|
log *slog.Logger,
|
|
client browser.BrowserClient,
|
|
req browser.ContentRequest,
|
|
maxAttempts int,
|
|
baseDelay time.Duration,
|
|
) (string, error) {
|
|
var lastErr error
|
|
delay := baseDelay
|
|
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
|
html, err := client.GetContent(ctx, req)
|
|
if err == nil {
|
|
return html, nil
|
|
}
|
|
lastErr = err
|
|
|
|
// Stop immediately on context cancellation.
|
|
if ctx.Err() != nil {
|
|
return "", err
|
|
}
|
|
|
|
if attempt < maxAttempts {
|
|
log.Warn("chapter fetch failed, retrying",
|
|
"url", req.URL,
|
|
"attempt", attempt,
|
|
"max_attempts", maxAttempts,
|
|
"retry_in", delay,
|
|
"err", err,
|
|
)
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
delay *= 2
|
|
}
|
|
}
|
|
return "", lastErr
|
|
}
|
|
|
|
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
|
s.log.Debug("chapter text fetch starting",
|
|
"chapter", ref.Number,
|
|
"title", ref.Title,
|
|
"payload_url", ref.URL,
|
|
"payload_wait_selector", "#content",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{
|
|
URL: ref.URL,
|
|
WaitFor: &browser.WaitForSelector{Selector: "#content", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
BestAttempt: true,
|
|
}, 9, 6*time.Second)
|
|
if err != nil {
|
|
s.log.Debug("chapter text fetch failed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"err", err,
|
|
)
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
|
|
}
|
|
if len(raw) > 0 {
|
|
preview := raw
|
|
if len(preview) > 500 {
|
|
preview = preview[:500]
|
|
}
|
|
s.log.Debug("chapter text fetch partial content",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"response_bytes", len(raw),
|
|
"preview", preview,
|
|
)
|
|
}
|
|
s.log.Debug("chapter text fetch completed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"response_bytes", len(raw),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
|
|
}
|
|
|
|
// <div id="content">…</div>
|
|
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
|
|
if container == nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
|
|
}
|
|
|
|
text := htmlutil.NodeToMarkdown(container)
|
|
|
|
s.log.Debug("chapter text parsed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"text_bytes", len(text),
|
|
)
|
|
|
|
return scraper.Chapter{
|
|
Ref: ref,
|
|
Text: text,
|
|
}, nil
|
|
}
|
|
|
|
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
|
|
func resolveURL(base, href string) string {
|
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
|
return href
|
|
}
|
|
b, err := url.Parse(base)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
ref, err := url.Parse(href)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
return b.ResolveReference(ref).String()
|
|
}
|
|
|
|
func slugFromURL(bookURL string) string {
|
|
u, err := url.Parse(bookURL)
|
|
if err != nil {
|
|
return bookURL
|
|
}
|
|
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
if len(parts) >= 2 && parts[0] == "book" {
|
|
return parts[1]
|
|
}
|
|
if len(parts) > 0 {
|
|
return parts[len(parts)-1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func parseChapterCount(s string) int {
|
|
// Formats: "123 Chapters", "1,234 Chapters", "123"
|
|
s = strings.ReplaceAll(s, ",", "")
|
|
fields := strings.Fields(s)
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(fields[0])
|
|
return n
|
|
}
|
|
|
|
// chapterNumberFromURL extracts the chapter number from a novelfire chapter URL.
|
|
//
|
|
// URL pattern: https://novelfire.net/book/{book-slug}/chapter-{N}
|
|
// The last path segment is expected to be "chapter-{N}" or "{N}".
|
|
// Returns 0 if no number can be parsed.
|
|
func chapterNumberFromURL(chapterURL string) int {
|
|
u, err := url.Parse(chapterURL)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
seg := path.Base(u.Path) // e.g. "chapter-42" or "42"
|
|
// Strip a "chapter-" prefix if present.
|
|
seg = strings.TrimPrefix(seg, "chapter-")
|
|
// Also handle "chap-", "ch-" variants used by some sites.
|
|
seg = strings.TrimPrefix(seg, "chap-")
|
|
seg = strings.TrimPrefix(seg, "ch-")
|
|
// Take only the leading digits (handles slugs like "42-title-text").
|
|
digits := strings.FieldsFunc(seg, func(r rune) bool {
|
|
return r < '0' || r > '9'
|
|
})
|
|
if len(digits) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(digits[0])
|
|
return n
|
|
}
|