ca33f8c3cf
The previous selectors were based on a hypothetical structure that does not
match the actual site. The real novelfire.net popular listing uses:
<li class="novel-item"> (not <div>)
<a href="/book/slug" title="Title">
<figure class="novel-cover"><img data-src="/path.jpg"></figure>
<h4 class="novel-title text2row">Title</h4>
</a>
</li>
And pagination uses <a rel="next"> (not <a class="next">).
Changes:
- ScrapeRanking: use li.novel-item, h4.novel-title, figure.novel-cover,
img[data-src]; strip base64 placeholder covers
- hasNextPageLink(): new helper walking all <a> nodes for rel="next"
- Import golang.org/x/net/html for Node.Attr access
- Test fixtures rewritten to match real structure (li/h4/rel=next)
- Status and genres removed from ranking items (not present on listing page)
Verified end-to-end: ranking.json written with 24 items on first fetch
713 lines
22 KiB
Go
713 lines
22 KiB
Go
// Package novelfire provides a NovelScraper implementation for novelfire.net.
|
|
//
|
|
// Site structure (as of 2025):
|
|
//
|
|
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
|
|
// Book page : https://novelfire.net/book/{slug}
|
|
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
|
|
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
|
|
package novelfire
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/url"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/browser"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
const (
|
|
baseURL = "https://novelfire.net"
|
|
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
|
|
rankingPath = "/genre-all/sort-popular/status-all/all-novel"
|
|
)
|
|
|
|
// rejectResourceTypes lists Browserless resource types to block on every request.
|
|
// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls).
|
|
// Everything else is safe to drop for HTML-only scraping.
|
|
var rejectResourceTypes = []string{
|
|
"cspviolationreport",
|
|
"eventsource",
|
|
"fedcm",
|
|
"font",
|
|
"image",
|
|
"manifest",
|
|
"media",
|
|
"other",
|
|
"ping",
|
|
"signedexchange",
|
|
"stylesheet",
|
|
"texttrack",
|
|
"websocket",
|
|
}
|
|
|
|
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
|
// It uses the /content strategy by default (rendered HTML via Browserless).
|
|
type Scraper struct {
|
|
client browser.BrowserClient
|
|
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
|
|
pageCache scraper.RankingPageCacher
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New returns a new novelfire Scraper.
|
|
// client is used for content fetching, urlClient is used for URL retrieval (chapter list).
|
|
// If urlClient is nil, client will be used for both.
|
|
// pageCache is optional; pass nil to disable ranking page caching.
|
|
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, pageCache scraper.RankingPageCacher) *Scraper {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if urlClient == nil {
|
|
urlClient = client
|
|
}
|
|
return &Scraper{client: client, urlClient: urlClient, pageCache: pageCache, log: log}
|
|
}
|
|
|
|
// SourceName implements NovelScraper.
|
|
func (s *Scraper) SourceName() string { return "novelfire.net" }
|
|
|
|
// ─── CatalogueProvider ───────────────────────────────────────────────────────
|
|
|
|
// ScrapeCatalogue streams all CatalogueEntry values across all pages.
|
|
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
|
entries := make(chan scraper.CatalogueEntry, 64)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
pageURL := baseURL + cataloguePath
|
|
page := 1
|
|
|
|
for pageURL != "" {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
|
s.log.Debug("catalogue page fetch starting",
|
|
"page", page,
|
|
"payload_url", pageURL,
|
|
"payload_wait_selector", ".novel-item",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
html, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("catalogue page fetch failed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"err", err,
|
|
)
|
|
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
|
return
|
|
}
|
|
s.log.Debug("catalogue page fetch completed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"response_bytes", len(html),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(html)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
// Extract novel cards: <div class="novel-item">
|
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
|
|
if len(cards) == 0 {
|
|
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
|
return
|
|
}
|
|
|
|
for _, card := range cards {
|
|
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
|
|
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
|
|
|
|
var title, href string
|
|
if titleNode != nil {
|
|
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
if linkNode != nil {
|
|
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
}
|
|
}
|
|
if href == "" || title == "" {
|
|
continue
|
|
}
|
|
|
|
bookURL := resolveURL(baseURL, href)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
|
|
}
|
|
}
|
|
|
|
// Find next page link: <a class="next" href="...">
|
|
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
|
if nextHref == "" {
|
|
break
|
|
}
|
|
pageURL = resolveURL(baseURL, nextHref)
|
|
page++
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── MetadataProvider ────────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
|
|
s.log.Debug("metadata fetch starting",
|
|
"payload_url", bookURL,
|
|
"payload_wait_selector", ".novel-title",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: bookURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
|
}
|
|
s.log.Debug("metadata fetch completed", "url", bookURL, "response_bytes", len(raw))
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
|
|
}
|
|
|
|
// <h1 class="novel-title">Title</h1>
|
|
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
|
// <span class="author"><a>Author Name</a></span>
|
|
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
|
// <figure class="cover"><img src="..."></figure>
|
|
var cover string
|
|
if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil {
|
|
cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"})
|
|
}
|
|
// <span class="status">Ongoing</span>
|
|
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
|
|
|
// Genres: all <a> tags inside <div class="genres">
|
|
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
|
var genres []string
|
|
if genresNode != nil {
|
|
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
|
}
|
|
|
|
// <div class="summary"><p>...</p></div>
|
|
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
|
|
// <span class="chapter-count">123 Chapters</span>
|
|
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
|
|
totalChapters := parseChapterCount(totalStr)
|
|
|
|
slug := slugFromURL(bookURL)
|
|
|
|
meta := scraper.BookMeta{
|
|
Slug: slug,
|
|
Title: title,
|
|
Author: author,
|
|
Cover: cover,
|
|
Status: status,
|
|
Genres: genres,
|
|
Summary: summary,
|
|
TotalChapters: totalChapters,
|
|
SourceURL: bookURL,
|
|
}
|
|
s.log.Debug("metadata parsed",
|
|
"url", bookURL,
|
|
"slug", meta.Slug,
|
|
"title", meta.Title,
|
|
"author", meta.Author,
|
|
"status", meta.Status,
|
|
"genres", meta.Genres,
|
|
"total_chapters", meta.TotalChapters,
|
|
)
|
|
return meta, nil
|
|
}
|
|
|
|
// ─── ChapterListProvider ─────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
|
|
var refs []scraper.ChapterRef
|
|
// Chapter list URL: {bookURL}/chapters?page=N
|
|
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
|
page := 1
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return refs, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
|
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
|
|
|
s.log.Debug("chapter list fetch starting",
|
|
"page", page,
|
|
"payload_url", pageURL,
|
|
"payload_wait_selector", ".chapter-list",
|
|
"payload_wait_selector_timeout_ms", 15000,
|
|
"payload_wait_timeout_ms", 2000,
|
|
"strategy", s.urlClient.Strategy(),
|
|
)
|
|
|
|
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
// Wait up to 15 s for the chapter list container to appear in the DOM.
|
|
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
|
|
// After the selector is found, wait an additional 2 s for any
|
|
// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
|
|
WaitForTimeout: 2000,
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
// Do NOT use BestAttempt — we want a complete page or a clear error,
|
|
// not silently partial HTML that looks like "no more chapters".
|
|
BestAttempt: false,
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("chapter list fetch failed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"err", err,
|
|
)
|
|
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
|
}
|
|
s.log.Debug("chapter list fetch completed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"response_bytes", len(raw),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
|
|
}
|
|
|
|
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
|
if chapterList == nil {
|
|
// No chapter list container on this page — we've gone past the last page.
|
|
s.log.Debug("chapter list container not found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
// Each chapter row: <li class="chapter-item"><a href="...">Title</a></li>
|
|
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
|
|
|
s.log.Debug("chapter list page parsed",
|
|
"page", page,
|
|
"url", pageURL,
|
|
"chapters_on_page", len(items),
|
|
"total_refs_so_far", len(refs),
|
|
)
|
|
|
|
// Zero items on this page means we've gone past the last page.
|
|
if len(items) == 0 {
|
|
s.log.Debug("no chapters on page, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
for _, item := range items {
|
|
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
|
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
if href == "" {
|
|
continue
|
|
}
|
|
chURL := resolveURL(baseURL, href)
|
|
num := chapterNumberFromURL(chURL)
|
|
if num <= 0 {
|
|
// Fall back to position if the URL has no parseable number.
|
|
num = len(refs) + 1
|
|
s.log.Warn("chapter number not parseable from URL, falling back to position",
|
|
"url", chURL,
|
|
"position", num,
|
|
)
|
|
}
|
|
refs = append(refs, scraper.ChapterRef{
|
|
Number: num,
|
|
Title: strings.TrimSpace(chTitle),
|
|
URL: chURL,
|
|
})
|
|
}
|
|
|
|
page++
|
|
}
|
|
|
|
return refs, nil
|
|
}
|
|
|
|
// ─── RankingProvider ───────────────────────────────────────────────────────────
|
|
|
|
// hasNextPageLink returns true if the HTML document contains a pagination link
|
|
// with rel="next". novelfire.net uses:
|
|
//
|
|
// <a class="page-link" href="...?page=N" rel="next" ...>
|
|
func hasNextPageLink(root *html.Node) bool {
|
|
links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
|
|
for _, a := range links {
|
|
for _, attr := range a.Attr {
|
|
if attr.Key == "rel" && attr.Val == "next" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ScrapeRanking pages through up to maxPages pages of the popular-novels genre
|
|
// listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel).
|
|
// Pages are fetched one at a time, strictly sequentially.
|
|
// maxPages <= 0 means "fetch all pages until no more are found".
|
|
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
|
|
entries := make(chan scraper.BookMeta, 32)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
rank := 1
|
|
|
|
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
|
|
|
// Try to serve from disk cache before hitting the network.
|
|
var raw string
|
|
if s.pageCache != nil {
|
|
if cached, err := s.pageCache.ReadRankingPageCache(page); err != nil {
|
|
s.log.Warn("ranking page cache read error", "page", page, "err", err)
|
|
} else if cached != "" {
|
|
s.log.Info("serving ranking page from cache", "page", page)
|
|
raw = cached
|
|
}
|
|
}
|
|
|
|
if raw == "" {
|
|
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
|
|
fetched, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
})
|
|
if err != nil {
|
|
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
|
|
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
|
return
|
|
}
|
|
raw = fetched
|
|
|
|
// Persist to cache for future runs.
|
|
if s.pageCache != nil {
|
|
if werr := s.pageCache.WriteRankingPageCache(page, raw); werr != nil {
|
|
s.log.Warn("ranking page cache write error", "page", page, "err", werr)
|
|
}
|
|
}
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
// Real novelfire.net popular listing structure:
|
|
// <li class="novel-item">
|
|
// <a href="/book/slug" title="Title">
|
|
// <figure class="novel-cover"><img data-src="..."></figure>
|
|
// <h4 class="novel-title text2row">Title</h4>
|
|
// </a>
|
|
// </li>
|
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
|
if len(cards) == 0 {
|
|
s.log.Debug("no novel cards found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
|
|
for _, card := range cards {
|
|
// The outer <a> carries the href and title attribute.
|
|
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
bookURL := resolveURL(baseURL, href)
|
|
if bookURL == "" {
|
|
continue
|
|
}
|
|
|
|
// Title: prefer <h4 class="novel-title"> text; fall back to <a title="...">
|
|
title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
|
|
if title == "" {
|
|
title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
|
|
}
|
|
if title == "" {
|
|
continue
|
|
}
|
|
|
|
// Cover: <figure class="novel-cover"><img data-src="...">
|
|
var cover string
|
|
if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
|
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
|
|
if cover == "" {
|
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
|
|
}
|
|
// Filter out base64 placeholder images.
|
|
if strings.HasPrefix(cover, "data:") {
|
|
cover = ""
|
|
}
|
|
if cover != "" && !strings.HasPrefix(cover, "http") {
|
|
cover = baseURL + cover
|
|
}
|
|
}
|
|
|
|
slug := slugFromURL(bookURL)
|
|
|
|
meta := scraper.BookMeta{
|
|
Slug: slug,
|
|
Title: title,
|
|
Cover: cover,
|
|
SourceURL: bookURL,
|
|
Ranking: rank,
|
|
}
|
|
rank++
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- meta:
|
|
}
|
|
}
|
|
|
|
// Stop if no next-page link exists.
|
|
// The real pagination uses <a rel="next" ...> inside .pagination.
|
|
if !hasNextPageLink(root) {
|
|
s.log.Debug("no next-page link found, stopping pagination", "page", page)
|
|
break
|
|
}
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── ChapterTextProvider ─────────────────────────────────────────────────────
|
|
|
|
// retryGetContent calls client.GetContent up to maxAttempts times, backing off
|
|
// exponentially between retries. Only errors that look like transient Browserless
|
|
// failures (timeouts, 5xx responses) are retried; context cancellation and
|
|
// permanent errors are returned immediately.
|
|
func retryGetContent(
|
|
ctx context.Context,
|
|
log *slog.Logger,
|
|
client browser.BrowserClient,
|
|
req browser.ContentRequest,
|
|
maxAttempts int,
|
|
baseDelay time.Duration,
|
|
) (string, error) {
|
|
var lastErr error
|
|
delay := baseDelay
|
|
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
|
html, err := client.GetContent(ctx, req)
|
|
if err == nil {
|
|
return html, nil
|
|
}
|
|
lastErr = err
|
|
|
|
// Stop immediately on context cancellation.
|
|
if ctx.Err() != nil {
|
|
return "", err
|
|
}
|
|
|
|
if attempt < maxAttempts {
|
|
log.Warn("chapter fetch failed, retrying",
|
|
"url", req.URL,
|
|
"attempt", attempt,
|
|
"max_attempts", maxAttempts,
|
|
"retry_in", delay,
|
|
"err", err,
|
|
)
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
delay *= 2
|
|
}
|
|
}
|
|
return "", lastErr
|
|
}
|
|
|
|
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
|
s.log.Debug("chapter text fetch starting",
|
|
"chapter", ref.Number,
|
|
"title", ref.Title,
|
|
"payload_url", ref.URL,
|
|
"payload_wait_selector", "#content",
|
|
"payload_wait_selector_timeout_ms", 5000,
|
|
)
|
|
|
|
raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{
|
|
URL: ref.URL,
|
|
WaitFor: &browser.WaitForSelector{Selector: "#content", Timeout: 5000},
|
|
RejectResourceTypes: rejectResourceTypes,
|
|
GotoOptions: &browser.GotoOptions{Timeout: 60000},
|
|
BestAttempt: true,
|
|
}, 9, 6*time.Second)
|
|
if err != nil {
|
|
s.log.Debug("chapter text fetch failed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"err", err,
|
|
)
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
|
|
}
|
|
if len(raw) > 0 {
|
|
preview := raw
|
|
if len(preview) > 500 {
|
|
preview = preview[:500]
|
|
}
|
|
s.log.Debug("chapter text fetch partial content",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"response_bytes", len(raw),
|
|
"preview", preview,
|
|
)
|
|
}
|
|
s.log.Debug("chapter text fetch completed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"response_bytes", len(raw),
|
|
)
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
|
|
}
|
|
|
|
// <div id="content">…</div>
|
|
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
|
|
if container == nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
|
|
}
|
|
|
|
text := htmlutil.NodeToMarkdown(container)
|
|
|
|
s.log.Debug("chapter text parsed",
|
|
"chapter", ref.Number,
|
|
"url", ref.URL,
|
|
"text_bytes", len(text),
|
|
)
|
|
|
|
return scraper.Chapter{
|
|
Ref: ref,
|
|
Text: text,
|
|
}, nil
|
|
}
|
|
|
|
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
|
|
func resolveURL(base, href string) string {
|
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
|
return href
|
|
}
|
|
b, err := url.Parse(base)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
ref, err := url.Parse(href)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
return b.ResolveReference(ref).String()
|
|
}
|
|
|
|
func slugFromURL(bookURL string) string {
|
|
u, err := url.Parse(bookURL)
|
|
if err != nil {
|
|
return bookURL
|
|
}
|
|
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
if len(parts) >= 2 && parts[0] == "book" {
|
|
return parts[1]
|
|
}
|
|
if len(parts) > 0 {
|
|
return parts[len(parts)-1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func parseChapterCount(s string) int {
|
|
// Formats: "123 Chapters", "1,234 Chapters", "123"
|
|
s = strings.ReplaceAll(s, ",", "")
|
|
fields := strings.Fields(s)
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(fields[0])
|
|
return n
|
|
}
|
|
|
|
// chapterNumberFromURL extracts the chapter number from a novelfire chapter URL.
|
|
//
|
|
// URL pattern: https://novelfire.net/book/{book-slug}/chapter-{N}
|
|
// The last path segment is expected to be "chapter-{N}" or "{N}".
|
|
// Returns 0 if no number can be parsed.
|
|
func chapterNumberFromURL(chapterURL string) int {
|
|
u, err := url.Parse(chapterURL)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
seg := path.Base(u.Path) // e.g. "chapter-42" or "42"
|
|
// Strip a "chapter-" prefix if present.
|
|
seg = strings.TrimPrefix(seg, "chapter-")
|
|
// Also handle "chap-", "ch-" variants used by some sites.
|
|
seg = strings.TrimPrefix(seg, "chap-")
|
|
seg = strings.TrimPrefix(seg, "ch-")
|
|
// Take only the leading digits (handles slugs like "42-title-text").
|
|
digits := strings.FieldsFunc(seg, func(r rune) bool {
|
|
return r < '0' || r > '9'
|
|
})
|
|
if len(digits) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(digits[0])
|
|
return n
|
|
}
|