373 lines
11 KiB
Go
373 lines
11 KiB
Go
// Package novelfire provides a NovelScraper implementation for novelfire.net.
|
|
//
|
|
// Site structure (as of 2025):
|
|
//
|
|
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
|
|
// Book page : https://novelfire.net/book/{slug}
|
|
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
|
|
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
|
|
package novelfire
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/libnovel/scraper/internal/browser"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
|
)
|
|
|
|
const (
|
|
baseURL = "https://novelfire.net"
|
|
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
|
|
)
|
|
|
|
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
|
// It uses the /content strategy by default (rendered HTML via Browserless).
|
|
type Scraper struct {
|
|
client browser.BrowserClient
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New returns a new novelfire Scraper.
|
|
func New(client browser.BrowserClient, log *slog.Logger) *Scraper {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Scraper{client: client, log: log}
|
|
}
|
|
|
|
// SourceName implements NovelScraper.
|
|
func (s *Scraper) SourceName() string { return "novelfire.net" }
|
|
|
|
// ─── CatalogueProvider ───────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) CatalogueURL() string {
|
|
return baseURL + cataloguePath
|
|
}
|
|
|
|
func (s *Scraper) EntriesSelector() scraper.Selector {
|
|
// Each novel card: <div class="novel-item">
|
|
return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}
|
|
}
|
|
|
|
func (s *Scraper) NextPageSelector() scraper.Selector {
|
|
// <a class="next" href="...">
|
|
return scraper.Selector{Tag: "a", Class: "next", Attr: "href"}
|
|
}
|
|
|
|
// ScrapeCatalogue streams all CatalogueEntry values across all pages.
|
|
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
|
entries := make(chan scraper.CatalogueEntry, 64)
|
|
errs := make(chan error, 16)
|
|
|
|
go func() {
|
|
defer close(entries)
|
|
defer close(errs)
|
|
|
|
pageURL := s.CatalogueURL()
|
|
page := 1
|
|
|
|
for pageURL != "" {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
|
|
|
html, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: ".novel-item",
|
|
WaitForTimeout: 10000,
|
|
RejectResources: true,
|
|
})
|
|
if err != nil {
|
|
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
|
return
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(html)
|
|
if err != nil {
|
|
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
|
|
return
|
|
}
|
|
|
|
// Extract novel cards.
|
|
cards := htmlutil.FindAll(root, s.EntriesSelector())
|
|
if len(cards) == 0 {
|
|
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
|
return
|
|
}
|
|
|
|
for _, card := range cards {
|
|
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
|
|
titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"}
|
|
titleNode := htmlutil.FindFirst(card, titleSel)
|
|
|
|
var title, href string
|
|
if titleNode != nil {
|
|
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
if linkNode != nil {
|
|
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
}
|
|
}
|
|
if href == "" || title == "" {
|
|
continue
|
|
}
|
|
|
|
// Resolve relative URL.
|
|
bookURL := resolveURL(baseURL, href)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
|
|
}
|
|
}
|
|
|
|
// Find next page link.
|
|
nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
|
|
if nextHref == "" {
|
|
break
|
|
}
|
|
pageURL = resolveURL(baseURL, nextHref)
|
|
page++
|
|
}
|
|
}()
|
|
|
|
return entries, errs
|
|
}
|
|
|
|
// ─── MetadataProvider ────────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) MetadataSelectors() map[string]scraper.Selector {
|
|
return map[string]scraper.Selector{
|
|
// <h1 class="novel-title">Title</h1>
|
|
"title": {Tag: "h1", Class: "novel-title"},
|
|
// <span class="author"><a>Author Name</a></span>
|
|
"author": {Tag: "span", Class: "author"},
|
|
// <img class="cover" src="...">
|
|
"cover": {Tag: "img", Class: "cover", Attr: "src"},
|
|
// <span class="status">Ongoing</span>
|
|
"status": {Tag: "span", Class: "status"},
|
|
// <div class="genres"><a>Tag1</a><a>Tag2</a>…</div>
|
|
"genres": {Tag: "div", Class: "genres", Multiple: true},
|
|
// <div class="summary"><p>...</p></div>
|
|
"summary": {Tag: "div", Class: "summary"},
|
|
// <span class="chapter-count">123 Chapters</span>
|
|
"total_chapters": {Tag: "span", Class: "chapter-count"},
|
|
}
|
|
}
|
|
|
|
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: bookURL,
|
|
WaitFor: ".novel-title",
|
|
WaitForTimeout: 10000,
|
|
RejectResources: true,
|
|
})
|
|
if err != nil {
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
|
|
}
|
|
|
|
sels := s.MetadataSelectors()
|
|
|
|
title := htmlutil.ExtractFirst(root, sels["title"])
|
|
author := htmlutil.ExtractFirst(root, sels["author"])
|
|
cover := htmlutil.ExtractFirst(root, sels["cover"])
|
|
status := htmlutil.ExtractFirst(root, sels["status"])
|
|
|
|
// Genres: all <a> tags inside the genres div.
|
|
genresNode := htmlutil.FindFirst(root, sels["genres"])
|
|
var genres []string
|
|
if genresNode != nil {
|
|
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
|
}
|
|
|
|
summary := htmlutil.ExtractFirst(root, sels["summary"])
|
|
|
|
totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"])
|
|
totalChapters := parseChapterCount(totalStr)
|
|
|
|
// Derive slug from URL.
|
|
slug := slugFromURL(bookURL)
|
|
|
|
return scraper.BookMeta{
|
|
Slug: slug,
|
|
Title: title,
|
|
Author: author,
|
|
Cover: cover,
|
|
Status: status,
|
|
Genres: genres,
|
|
Summary: summary,
|
|
TotalChapters: totalChapters,
|
|
SourceURL: bookURL,
|
|
}, nil
|
|
}
|
|
|
|
// ─── ChapterListProvider ─────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ChaptersURL(bookURL string) string {
|
|
return strings.TrimRight(bookURL, "/") + "/chapters"
|
|
}
|
|
|
|
func (s *Scraper) ChapterEntrySelector() scraper.Selector {
|
|
// <li class="chapter-item"><a href="/book/slug/chapter-1">Chapter 1: Title</a></li>
|
|
return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true}
|
|
}
|
|
|
|
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
|
|
var refs []scraper.ChapterRef
|
|
pageURL := s.ChaptersURL(bookURL)
|
|
page := 1
|
|
|
|
for pageURL != "" {
|
|
select {
|
|
case <-ctx.Done():
|
|
return refs, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
|
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: pageURL,
|
|
WaitFor: ".chapter-item",
|
|
WaitForTimeout: 10000,
|
|
RejectResources: true,
|
|
})
|
|
if err != nil {
|
|
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
|
|
}
|
|
|
|
items := htmlutil.FindAll(root, s.ChapterEntrySelector())
|
|
for _, item := range items {
|
|
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"})
|
|
if linkNode == nil {
|
|
continue
|
|
}
|
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
|
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
|
if href == "" {
|
|
continue
|
|
}
|
|
chURL := resolveURL(baseURL, href)
|
|
num := len(refs) + 1
|
|
refs = append(refs, scraper.ChapterRef{
|
|
Number: num,
|
|
Title: strings.TrimSpace(chTitle),
|
|
URL: chURL,
|
|
})
|
|
}
|
|
|
|
// Next page.
|
|
nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
|
|
if nextHref == "" {
|
|
break
|
|
}
|
|
pageURL = resolveURL(baseURL, nextHref)
|
|
page++
|
|
}
|
|
|
|
return refs, nil
|
|
}
|
|
|
|
// ─── ChapterTextProvider ─────────────────────────────────────────────────────
|
|
|
|
func (s *Scraper) ChapterTextSelector() scraper.Selector {
|
|
// <div id="chapter-container"> or <div class="chapter-content">
|
|
return scraper.Selector{Tag: "div", ID: "chapter-container"}
|
|
}
|
|
|
|
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
|
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
|
|
URL: ref.URL,
|
|
WaitFor: "#chapter-container",
|
|
WaitForTimeout: 15000,
|
|
RejectResources: true,
|
|
})
|
|
if err != nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
|
|
}
|
|
|
|
root, err := htmlutil.ParseHTML(raw)
|
|
if err != nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
|
|
}
|
|
|
|
container := htmlutil.FindFirst(root, s.ChapterTextSelector())
|
|
if container == nil {
|
|
// Fallback: try class-based selector.
|
|
container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"})
|
|
}
|
|
if container == nil {
|
|
return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL)
|
|
}
|
|
|
|
text := htmlutil.NodeToMarkdown(container)
|
|
|
|
return scraper.Chapter{
|
|
Ref: ref,
|
|
Text: text,
|
|
}, nil
|
|
}
|
|
|
|
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
|
|
func resolveURL(base, href string) string {
|
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
|
return href
|
|
}
|
|
b, err := url.Parse(base)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
ref, err := url.Parse(href)
|
|
if err != nil {
|
|
return base + href
|
|
}
|
|
return b.ResolveReference(ref).String()
|
|
}
|
|
|
|
func slugFromURL(bookURL string) string {
|
|
u, err := url.Parse(bookURL)
|
|
if err != nil {
|
|
return bookURL
|
|
}
|
|
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
if len(parts) >= 2 && parts[0] == "book" {
|
|
return parts[1]
|
|
}
|
|
if len(parts) > 0 {
|
|
return parts[len(parts)-1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func parseChapterCount(s string) int {
|
|
// Formats: "123 Chapters", "1,234 Chapters", "123"
|
|
s = strings.ReplaceAll(s, ",", "")
|
|
fields := strings.Fields(s)
|
|
if len(fields) == 0 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(fields[0])
|
|
return n
|
|
}
|