feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
Admin
498
v3/backend/internal/novelfire/scraper.go
Normal file
498
v3/backend/internal/novelfire/scraper.go
Normal file
@@ -0,0 +1,498 @@
|
||||
// Package novelfire provides a NovelScraper implementation for novelfire.net.
|
||||
//
|
||||
// Site structure (as of 2025):
|
||||
//
|
||||
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
|
||||
// Book page : https://novelfire.net/book/{slug}
|
||||
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
|
||||
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
|
||||
package novelfire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/browser"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/novelfire/htmlutil"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
baseURL = "https://novelfire.net"
|
||||
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
|
||||
rankingPath = "/genre-all/sort-popular/status-all/all-novel"
|
||||
)
|
||||
|
||||
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
||||
type Scraper struct {
|
||||
client browser.Client
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// Compile-time interface check.
|
||||
var _ scraper.NovelScraper = (*Scraper)(nil)
|
||||
|
||||
// New returns a new novelfire Scraper backed by client.
|
||||
func New(client browser.Client, log *slog.Logger) *Scraper {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &Scraper{client: client, log: log}
|
||||
}
|
||||
|
||||
// SourceName implements NovelScraper.
|
||||
func (s *Scraper) SourceName() string { return "novelfire.net" }
|
||||
|
||||
// ── CatalogueProvider ─────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
|
||||
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||
entries := make(chan domain.CatalogueEntry, 64)
|
||||
errs := make(chan error, 16)
|
||||
|
||||
go func() {
|
||||
defer close(entries)
|
||||
defer close(errs)
|
||||
|
||||
pageURL := baseURL + cataloguePath
|
||||
page := 1
|
||||
|
||||
for pageURL != "" {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
||||
if len(cards) == 0 {
|
||||
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
||||
return
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
|
||||
|
||||
var title, href string
|
||||
if linkNode != nil {
|
||||
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
}
|
||||
if titleNode != nil {
|
||||
title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
|
||||
}
|
||||
if href == "" || title == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
bookURL := resolveURL(baseURL, href)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
|
||||
}
|
||||
}
|
||||
|
||||
if !hasNextPageLink(root) {
|
||||
break
|
||||
}
|
||||
nextHref := ""
|
||||
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
|
||||
if htmlutil.AttrVal(a, "rel") == "next" {
|
||||
nextHref = htmlutil.AttrVal(a, "href")
|
||||
break
|
||||
}
|
||||
}
|
||||
if nextHref == "" {
|
||||
break
|
||||
}
|
||||
pageURL = resolveURL(baseURL, nextHref)
|
||||
page++
|
||||
}
|
||||
}()
|
||||
|
||||
return entries, errs
|
||||
}
|
||||
|
||||
// ── MetadataProvider ──────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeMetadata fetches and parses book metadata from the book's landing page.
|
||||
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
|
||||
s.log.Debug("metadata fetch starting", "url", bookURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, bookURL)
|
||||
if err != nil {
|
||||
return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
return domain.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
|
||||
}
|
||||
|
||||
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
||||
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
||||
|
||||
var cover string
|
||||
if fig := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
|
||||
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
|
||||
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||
cover = baseURL + cover
|
||||
}
|
||||
}
|
||||
|
||||
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
||||
|
||||
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
||||
var genres []string
|
||||
if genresNode != nil {
|
||||
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
||||
}
|
||||
|
||||
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
|
||||
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
|
||||
totalChapters := parseChapterCount(totalStr)
|
||||
|
||||
slug := slugFromURL(bookURL)
|
||||
|
||||
meta := domain.BookMeta{
|
||||
Slug: slug,
|
||||
Title: title,
|
||||
Author: author,
|
||||
Cover: cover,
|
||||
Status: status,
|
||||
Genres: genres,
|
||||
Summary: summary,
|
||||
TotalChapters: totalChapters,
|
||||
SourceURL: bookURL,
|
||||
}
|
||||
s.log.Debug("metadata parsed", "slug", meta.Slug, "title", meta.Title)
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// ── ChapterListProvider ───────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
|
||||
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
|
||||
var refs []domain.ChapterRef
|
||||
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
||||
page := 1
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return refs, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
||||
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
if err != nil {
|
||||
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
|
||||
}
|
||||
|
||||
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
||||
if chapterList == nil {
|
||||
s.log.Debug("chapter list container not found, stopping pagination", "page", page)
|
||||
break
|
||||
}
|
||||
|
||||
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
||||
if len(items) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
||||
if linkNode == nil {
|
||||
continue
|
||||
}
|
||||
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
||||
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
||||
if href == "" {
|
||||
continue
|
||||
}
|
||||
chURL := resolveURL(baseURL, href)
|
||||
num := chapterNumberFromURL(chURL)
|
||||
if num <= 0 {
|
||||
num = len(refs) + 1
|
||||
s.log.Warn("chapter number not parseable from URL, falling back to position",
|
||||
"url", chURL, "position", num)
|
||||
}
|
||||
refs = append(refs, domain.ChapterRef{
|
||||
Number: num,
|
||||
Title: strings.TrimSpace(chTitle),
|
||||
URL: chURL,
|
||||
})
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
return refs, nil
|
||||
}
|
||||
|
||||
// ── ChapterTextProvider ───────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeChapterText fetches and parses a single chapter page.
|
||||
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
||||
s.log.Debug("chapter text fetch starting", "chapter", ref.Number, "url", ref.URL)
|
||||
|
||||
raw, err := retryGet(ctx, s.log, s.client, ref.URL, 9, 6*time.Second)
|
||||
if err != nil {
|
||||
return domain.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
return domain.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
|
||||
}
|
||||
|
||||
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if container == nil {
|
||||
return domain.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
|
||||
}
|
||||
|
||||
text := htmlutil.NodeToMarkdown(container)
|
||||
|
||||
s.log.Debug("chapter text parsed", "chapter", ref.Number, "text_bytes", len(text))
|
||||
|
||||
return domain.Chapter{Ref: ref, Text: text}, nil
|
||||
}
|
||||
|
||||
// ── RankingProvider ───────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeRanking pages through up to maxPages pages of the popular-novels listing.
|
||||
// maxPages <= 0 means all pages. The caller decides whether to persist items.
|
||||
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
|
||||
entries := make(chan domain.BookMeta, 32)
|
||||
errs := make(chan error, 16)
|
||||
|
||||
go func() {
|
||||
defer close(entries)
|
||||
defer close(errs)
|
||||
|
||||
rank := 1
|
||||
|
||||
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
||||
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
root, err := htmlutil.ParseHTML(raw)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
|
||||
return
|
||||
}
|
||||
|
||||
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
|
||||
if linkNode == nil {
|
||||
continue
|
||||
}
|
||||
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
bookURL := resolveURL(baseURL, href)
|
||||
if bookURL == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
|
||||
if title == "" {
|
||||
title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
|
||||
}
|
||||
if title == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var cover string
|
||||
if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
|
||||
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
|
||||
if cover == "" {
|
||||
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
|
||||
}
|
||||
if strings.HasPrefix(cover, "data:") {
|
||||
cover = ""
|
||||
}
|
||||
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||
cover = baseURL + cover
|
||||
}
|
||||
}
|
||||
|
||||
meta := domain.BookMeta{
|
||||
Slug: slugFromURL(bookURL),
|
||||
Title: title,
|
||||
Cover: cover,
|
||||
SourceURL: bookURL,
|
||||
Ranking: rank,
|
||||
}
|
||||
rank++
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case entries <- meta:
|
||||
}
|
||||
}
|
||||
|
||||
if !hasNextPageLink(root) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return entries, errs
|
||||
}
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) }
|
||||
|
||||
func hasNextPageLink(root *html.Node) bool {
|
||||
links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
|
||||
for _, a := range links {
|
||||
for _, attr := range a.Attr {
|
||||
if attr.Key == "rel" && attr.Val == "next" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func slugFromURL(bookURL string) string {
|
||||
u, err := url.Parse(bookURL)
|
||||
if err != nil {
|
||||
return bookURL
|
||||
}
|
||||
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if len(parts) >= 2 && parts[0] == "book" {
|
||||
return parts[1]
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseChapterCount(s string) int {
|
||||
s = strings.ReplaceAll(s, ",", "")
|
||||
fields := strings.Fields(s)
|
||||
if len(fields) == 0 {
|
||||
return 0
|
||||
}
|
||||
n, _ := strconv.Atoi(fields[0])
|
||||
return n
|
||||
}
|
||||
|
||||
func chapterNumberFromURL(chapterURL string) int {
|
||||
u, err := url.Parse(chapterURL)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
seg := path.Base(u.Path)
|
||||
seg = strings.TrimPrefix(seg, "chapter-")
|
||||
seg = strings.TrimPrefix(seg, "chap-")
|
||||
seg = strings.TrimPrefix(seg, "ch-")
|
||||
digits := strings.FieldsFunc(seg, func(r rune) bool {
|
||||
return r < '0' || r > '9'
|
||||
})
|
||||
if len(digits) == 0 {
|
||||
return 0
|
||||
}
|
||||
n, _ := strconv.Atoi(digits[0])
|
||||
return n
|
||||
}
|
||||
|
||||
// retryGet calls client.GetContent up to maxAttempts times with exponential backoff.
|
||||
// If the server returns 429 (ErrRateLimit), the suggested Retry-After delay is used
|
||||
// instead of the geometric backoff delay.
|
||||
func retryGet(
|
||||
ctx context.Context,
|
||||
log *slog.Logger,
|
||||
client browser.Client,
|
||||
pageURL string,
|
||||
maxAttempts int,
|
||||
baseDelay time.Duration,
|
||||
) (string, error) {
|
||||
var lastErr error
|
||||
delay := baseDelay
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
raw, err := client.GetContent(ctx, pageURL)
|
||||
if err == nil {
|
||||
return raw, nil
|
||||
}
|
||||
lastErr = err
|
||||
if ctx.Err() != nil {
|
||||
return "", err
|
||||
}
|
||||
if attempt < maxAttempts {
|
||||
// If the server is rate-limiting us, honour its Retry-After delay.
|
||||
waitFor := delay
|
||||
var rlErr *browser.RateLimitError
|
||||
if errors.As(err, &rlErr) {
|
||||
waitFor = rlErr.RetryAfter
|
||||
if log != nil {
|
||||
log.Warn("rate limited, backing off",
|
||||
"url", pageURL, "attempt", attempt, "retry_in", waitFor)
|
||||
}
|
||||
} else {
|
||||
if log != nil {
|
||||
log.Warn("fetch failed, retrying",
|
||||
"url", pageURL, "attempt", attempt, "retry_in", delay, "err", err)
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(waitFor):
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
Reference in New Issue
Block a user