kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat: initial commit

Browse Source
This commit is contained in:
Admin
2026-02-26 12:56:25 +05:00
commit d68ea71239
15 changed files with 1893 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
scraper/Dockerfile Normal file
View File

@@ -0,0 +1,44 @@
# ── Build stage ────────────────────────────────────────────────────────────────
FROM golang:1.25-alpine AS builder
WORKDIR /build
# Cache dependency downloads separately from source compilation.
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -ldflags="-s -w" -o /scraper ./cmd/scraper
# ── Runtime stage ──────────────────────────────────────────────────────────────
FROM alpine:3.20
# ca-certificates is required for HTTPS requests to novelfire.net.
RUN apk add --no-cache ca-certificates tzdata
WORKDIR /app
COPY --from=builder /scraper /app/scraper
# Create the default static output directory.
RUN mkdir -p /app/static/books
# Non-root user.
RUN addgroup -S scraper && adduser -S scraper -G scraper
RUN chown -R scraper:scraper /app
USER scraper
# ── Configuration ─────────────────────────────────────────────────────────────
ENV BROWSERLESS_URL=http://browserless:3000
ENV BROWSERLESS_STRATEGY=content
ENV SCRAPER_WORKERS=0
ENV SCRAPER_STATIC_ROOT=/app/static/books
ENV SCRAPER_HTTP_ADDR=:8080
EXPOSE 8080
# Default: run as an HTTP server. Override CMD to use "run" for one-shot.
ENTRYPOINT ["/app/scraper"]
CMD ["serve"]

150
scraper/cmd/scraper/main.go Normal file
View File

@@ -0,0 +1,150 @@
// Command scraper is the entrypoint for the libnovel scraper service.
//
// Usage (CLI one-shot):
//
// scraper run [--url <book-url>]
//
// Usage (HTTP server):
//
// scraper serve
//
// Environment variables:
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/signal"
"runtime"
"strconv"
"strings"
"syscall"
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/novelfire"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/server"
)
func main() {
log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
Level: slog.LevelInfo,
}))
if err := run(log); err != nil {
log.Error("fatal", "err", err)
os.Exit(1)
}
}
func run(log *slog.Logger) error {
args := os.Args[1:]
if len(args) == 0 {
printUsage()
return nil
}
cmd := strings.ToLower(args[0])
browserCfg := browser.Config{
BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3000"),
Token: envOr("BROWSERLESS_TOKEN", ""),
}
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyContent))))
bc := newBrowserClient(strategy, browserCfg)
nf := novelfire.New(bc, log)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
n, err := strconv.Atoi(s)
if err == nil && n > 0 {
workers = n
}
}
if workers == 0 {
workers = runtime.NumCPU()
}
oCfg := orchestrator.Config{
Workers: workers,
StaticRoot: envOr("SCRAPER_STATIC_ROOT", "./static/books"),
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
switch cmd {
case "run":
// Optional --url flag.
if len(args) >= 3 && args[1] == "--url" {
oCfg.SingleBookURL = args[2]
}
log.Info("starting one-shot scrape",
"strategy", strategy,
"workers", workers,
"static_root", oCfg.StaticRoot,
"single_book", oCfg.SingleBookURL,
)
o := orchestrator.New(oCfg, nf, log)
return o.Run(ctx)
case "serve":
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
log.Info("starting HTTP server",
"addr", addr,
"strategy", strategy,
"workers", workers,
)
srv := server.New(addr, oCfg, nf, log)
return srv.ListenAndServe(ctx)
default:
return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
}
}
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
switch strategy {
case browser.StrategyScrape:
return browser.NewScrapeClient(cfg)
case browser.StrategyCDP:
return browser.NewCDPClient(cfg)
default:
return browser.NewContentClient(cfg)
}
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func printUsage() {
fmt.Fprintf(os.Stderr, `libnovel scraper
Commands:
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
serve Start HTTP server (POST /scrape, POST /scrape/book)
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
`, runtime.NumCPU())
}

9
scraper/go.mod Normal file
View File

@@ -0,0 +1,9 @@
module github.com/libnovel/scraper
go 1.25.0
require (
github.com/gorilla/websocket v1.5.3 // indirect
golang.org/x/net v0.51.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

7
scraper/go.sum Normal file
View File

@@ -0,0 +1,7 @@
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

131
scraper/internal/browser/cdp.go Normal file
View File

@@ -0,0 +1,131 @@
package browser
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"sync/atomic"
"time"
"github.com/gorilla/websocket"
)
// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
type cdpClient struct {
cfg Config
}
// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
func NewCDPClient(cfg Config) BrowserClient {
if cfg.Timeout == 0 {
cfg.Timeout = 60 * time.Second
}
return &cdpClient{cfg: cfg}
}
func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
}
func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
}
// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
// navigates to pageURL, and invokes fn with a live CDPConn.
func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
wsURL += "/devtools/browser"
sep := "?"
if c.cfg.Token != "" {
wsURL += sep + "token=" + c.cfg.Token
sep = "&"
}
wsURL += sep + "url=" + pageURL
dialer := websocket.Dialer{
HandshakeTimeout: 15 * time.Second,
Proxy: http.ProxyFromEnvironment,
}
conn, _, err := dialer.DialContext(ctx, wsURL, nil)
if err != nil {
return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
}
cdp := &cdpConn{ws: conn}
defer cdp.Close()
return fn(ctx, cdp)
}
// ─── cdpConn ─────────────────────────────────────────────────────────────────
type cdpConn struct {
ws *websocket.Conn
counter atomic.Int64
}
type cdpRequest struct {
ID int64 `json:"id"`
Method string `json:"method"`
Params map[string]any `json:"params,omitempty"`
}
type cdpResponse struct {
ID int64 `json:"id"`
Result map[string]any `json:"result,omitempty"`
Error *struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error,omitempty"`
}
func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
id := c.counter.Add(1)
req := cdpRequest{ID: id, Method: method, Params: params}
data, err := json.Marshal(req)
if err != nil {
return nil, fmt.Errorf("cdp send: marshal: %w", err)
}
if dl, ok := ctx.Deadline(); ok {
_ = c.ws.SetWriteDeadline(dl)
}
if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
return nil, fmt.Errorf("cdp send: write: %w", err)
}
// Read messages until we find the response matching our id.
for {
if dl, ok := ctx.Deadline(); ok {
_ = c.ws.SetReadDeadline(dl)
}
_, msg, err := c.ws.ReadMessage()
if err != nil {
return nil, fmt.Errorf("cdp send: read: %w", err)
}
var resp cdpResponse
if err := json.Unmarshal(msg, &resp); err != nil {
continue // skip non-JSON frames (events etc.)
}
if resp.ID != id {
continue // event or different command reply
}
if resp.Error != nil {
return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
}
return resp.Result, nil
}
}
func (c *cdpConn) Close() error {
return c.ws.Close()
}

146
scraper/internal/browser/content_scrape.go Normal file
View File

@@ -0,0 +1,146 @@
package browser
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
// Config holds the connection parameters for a Browserless instance.
type Config struct {
// BaseURL is the HTTP base URL, e.g. "http://localhost:3000".
BaseURL string
// Token is the optional API token (BROWSERLESS_TOKEN env var).
Token string
// Timeout is the per-request HTTP timeout; defaults to 60 s.
Timeout time.Duration
}
// contentClient implements BrowserClient using the /content endpoint.
type contentClient struct {
cfg Config
http *http.Client
}
// NewContentClient returns a BrowserClient that uses POST /content.
func NewContentClient(cfg Config) BrowserClient {
if cfg.Timeout == 0 {
cfg.Timeout = 60 * time.Second
}
return &contentClient{
cfg: cfg,
http: &http.Client{Timeout: cfg.Timeout},
}
}
func (c *contentClient) Strategy() Strategy { return StrategyContent }
func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
body, err := json.Marshal(req)
if err != nil {
return "", fmt.Errorf("content: marshal request: %w", err)
}
url := c.cfg.BaseURL + "/content"
if c.cfg.Token != "" {
url += "?token=" + c.cfg.Token
}
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return "", fmt.Errorf("content: build request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(httpReq)
if err != nil {
return "", fmt.Errorf("content: do request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b)
}
raw, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("content: read body: %w", err)
}
return string(raw), nil
}
func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient")
}
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
return fmt.Errorf("content client does not support CDP; use NewCDPClient")
}
// ─── /scrape client ───────────────────────────────────────────────────────────
type scrapeClient struct {
cfg Config
http *http.Client
}
// NewScrapeClient returns a BrowserClient that uses POST /scrape.
func NewScrapeClient(cfg Config) BrowserClient {
if cfg.Timeout == 0 {
cfg.Timeout = 60 * time.Second
}
return &scrapeClient{
cfg: cfg,
http: &http.Client{Timeout: cfg.Timeout},
}
}
func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
}
func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
body, err := json.Marshal(req)
if err != nil {
return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
}
url := c.cfg.BaseURL + "/scrape"
if c.cfg.Token != "" {
url += "?token=" + c.cfg.Token
}
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(httpReq)
if err != nil {
return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
}
var result ScrapeResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
}
return result, nil
}
func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
}

101
scraper/internal/browser/interface.go Normal file
View File

@@ -0,0 +1,101 @@
// Package browser defines the BrowserClient interface and helper types for
// communicating with a Browserless instance.
package browser
import "context"
// Strategy selects which Browserless API endpoint / protocol to use.
type Strategy string
const (
// StrategyContent uses the POST /content endpoint, which returns the final
// rendered HTML of the page. Fastest; suitable for most JS-rendered sites.
StrategyContent Strategy = "content"
// StrategyScrape uses the POST /scrape endpoint, which accepts a list of
// CSS selectors and returns structured JSON. Good when you know exactly
// which elements you need.
StrategyScrape Strategy = "scrape"
// StrategyCDP uses the WebSocket /devtools/browser endpoint (Chrome
// DevTools Protocol). Most powerful; required for complex interactions
// (clicking, scrolling, waiting for network idle, etc.).
StrategyCDP Strategy = "cdp"
)
// ContentRequest is the body sent to POST /content.
type ContentRequest struct {
URL string `json:"url"`
WaitFor string `json:"waitForSelector,omitempty"`
WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms
RejectResources bool `json:"rejectResources,omitempty"`
}
// ScrapeElement is one element descriptor inside a ScrapeRequest.
type ScrapeElement struct {
Selector string `json:"selector"`
Timeout int `json:"timeout,omitempty"` // ms
}
// ScrapeRequest is the body sent to POST /scrape.
type ScrapeRequest struct {
URL string `json:"url"`
Elements []ScrapeElement `json:"elements"`
WaitFor string `json:"waitForSelector,omitempty"`
}
// ScrapeResult is one entry in the response from POST /scrape.
type ScrapeResult struct {
Selector string `json:"selector"`
Results []ScrapeElement `json:"results"`
}
// ScrapeAttribute holds a single attribute value from a scraped element.
type ScrapeAttribute struct {
Name string `json:"name"`
Value string `json:"value"`
}
// ScrapedElement is one item inside ScrapeResult.Results.
type ScrapedElement struct {
Text string `json:"text"`
Attributes []ScrapeAttribute `json:"attributes"`
}
// ScrapeResponse is the top-level response from POST /scrape.
type ScrapeResponse struct {
Data []ScrapeResult `json:"data"`
}
// BrowserClient is an abstraction over the three Browserless API strategies.
// Callers choose the strategy best suited to the target site; the interface
// signature is identical regardless of strategy.
type BrowserClient interface {
// Strategy returns the strategy this client uses.
Strategy() Strategy
// GetContent fetches the fully-rendered HTML of url using the /content
// endpoint. Only meaningful when Strategy() == StrategyContent.
GetContent(ctx context.Context, req ContentRequest) (string, error)
// ScrapePage calls the /scrape endpoint and returns structured data.
// Only meaningful when Strategy() == StrategyScrape.
ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error)
// CDPSession opens a CDP WebSocket session and calls fn with the raw
// WebSocket connection. Only meaningful when Strategy() == StrategyCDP.
// The session is closed when fn returns.
CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error
}
// CDPSessionFunc is the callback invoked inside a CDP session.
// conn is a live *websocket.Conn connected to a Browserless CDP endpoint.
type CDPSessionFunc func(ctx context.Context, conn CDPConn) error
// CDPConn is the minimal interface the orchestrator needs over a CDP WebSocket.
type CDPConn interface {
// Send sends a raw CDP command (JSON-encoded) and returns the response.
Send(ctx context.Context, method string, params map[string]any) (map[string]any, error)
// Close closes the underlying connection.
Close() error
}

372
scraper/internal/novelfire/scraper.go Normal file
View File

@@ -0,0 +1,372 @@
// Package novelfire provides a NovelScraper implementation for novelfire.net.
//
// Site structure (as of 2025):
//
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
// Book page : https://novelfire.net/book/{slug}
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
package novelfire
import (
"context"
"fmt"
"log/slog"
"net/url"
"strconv"
"strings"
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/scraper/htmlutil"
)
const (
baseURL = "https://novelfire.net"
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
)
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
// It uses the /content strategy by default (rendered HTML via Browserless).
type Scraper struct {
client browser.BrowserClient
log *slog.Logger
}
// New returns a new novelfire Scraper.
func New(client browser.BrowserClient, log *slog.Logger) *Scraper {
if log == nil {
log = slog.Default()
}
return &Scraper{client: client, log: log}
}
// SourceName implements NovelScraper.
func (s *Scraper) SourceName() string { return "novelfire.net" }
// ─── CatalogueProvider ───────────────────────────────────────────────────────
func (s *Scraper) CatalogueURL() string {
return baseURL + cataloguePath
}
func (s *Scraper) EntriesSelector() scraper.Selector {
// Each novel card: <div class="novel-item">
return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}
}
func (s *Scraper) NextPageSelector() scraper.Selector {
// <a class="next" href="...">
return scraper.Selector{Tag: "a", Class: "next", Attr: "href"}
}
// ScrapeCatalogue streams all CatalogueEntry values across all pages.
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
entries := make(chan scraper.CatalogueEntry, 64)
errs := make(chan error, 16)
go func() {
defer close(entries)
defer close(errs)
pageURL := s.CatalogueURL()
page := 1
for pageURL != "" {
select {
case <-ctx.Done():
return
default:
}
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
html, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: ".novel-item",
WaitForTimeout: 10000,
RejectResources: true,
})
if err != nil {
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
return
}
root, err := htmlutil.ParseHTML(html)
if err != nil {
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
return
}
// Extract novel cards.
cards := htmlutil.FindAll(root, s.EntriesSelector())
if len(cards) == 0 {
s.log.Warn("no novel cards found, stopping pagination", "page", page)
return
}
for _, card := range cards {
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"}
titleNode := htmlutil.FindFirst(card, titleSel)
var title, href string
if titleNode != nil {
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
if linkNode != nil {
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
}
}
if href == "" || title == "" {
continue
}
// Resolve relative URL.
bookURL := resolveURL(baseURL, href)
select {
case <-ctx.Done():
return
case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
}
}
// Find next page link.
nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
if nextHref == "" {
break
}
pageURL = resolveURL(baseURL, nextHref)
page++
}
}()
return entries, errs
}
// ─── MetadataProvider ────────────────────────────────────────────────────────
func (s *Scraper) MetadataSelectors() map[string]scraper.Selector {
return map[string]scraper.Selector{
// <h1 class="novel-title">Title</h1>
"title": {Tag: "h1", Class: "novel-title"},
// <span class="author"><a>Author Name</a></span>
"author": {Tag: "span", Class: "author"},
// <img class="cover" src="...">
"cover": {Tag: "img", Class: "cover", Attr: "src"},
// <span class="status">Ongoing</span>
"status": {Tag: "span", Class: "status"},
// <div class="genres"><a>Tag1</a><a>Tag2</a>…</div>
"genres": {Tag: "div", Class: "genres", Multiple: true},
// <div class="summary"><p>...</p></div>
"summary": {Tag: "div", Class: "summary"},
// <span class="chapter-count">123 Chapters</span>
"total_chapters": {Tag: "span", Class: "chapter-count"},
}
}
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: bookURL,
WaitFor: ".novel-title",
WaitForTimeout: 10000,
RejectResources: true,
})
if err != nil {
return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
}
sels := s.MetadataSelectors()
title := htmlutil.ExtractFirst(root, sels["title"])
author := htmlutil.ExtractFirst(root, sels["author"])
cover := htmlutil.ExtractFirst(root, sels["cover"])
status := htmlutil.ExtractFirst(root, sels["status"])
// Genres: all <a> tags inside the genres div.
genresNode := htmlutil.FindFirst(root, sels["genres"])
var genres []string
if genresNode != nil {
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
}
summary := htmlutil.ExtractFirst(root, sels["summary"])
totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"])
totalChapters := parseChapterCount(totalStr)
// Derive slug from URL.
slug := slugFromURL(bookURL)
return scraper.BookMeta{
Slug: slug,
Title: title,
Author: author,
Cover: cover,
Status: status,
Genres: genres,
Summary: summary,
TotalChapters: totalChapters,
SourceURL: bookURL,
}, nil
}
// ─── ChapterListProvider ─────────────────────────────────────────────────────
func (s *Scraper) ChaptersURL(bookURL string) string {
return strings.TrimRight(bookURL, "/") + "/chapters"
}
func (s *Scraper) ChapterEntrySelector() scraper.Selector {
// <li class="chapter-item"><a href="/book/slug/chapter-1">Chapter 1: Title</a></li>
return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true}
}
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
var refs []scraper.ChapterRef
pageURL := s.ChaptersURL(bookURL)
page := 1
for pageURL != "" {
select {
case <-ctx.Done():
return refs, ctx.Err()
default:
}
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: ".chapter-item",
WaitForTimeout: 10000,
RejectResources: true,
})
if err != nil {
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
}
items := htmlutil.FindAll(root, s.ChapterEntrySelector())
for _, item := range items {
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"})
if linkNode == nil {
continue
}
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
if href == "" {
continue
}
chURL := resolveURL(baseURL, href)
num := len(refs) + 1
refs = append(refs, scraper.ChapterRef{
Number: num,
Title: strings.TrimSpace(chTitle),
URL: chURL,
})
}
// Next page.
nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
if nextHref == "" {
break
}
pageURL = resolveURL(baseURL, nextHref)
page++
}
return refs, nil
}
// ─── ChapterTextProvider ─────────────────────────────────────────────────────
func (s *Scraper) ChapterTextSelector() scraper.Selector {
// <div id="chapter-container"> or <div class="chapter-content">
return scraper.Selector{Tag: "div", ID: "chapter-container"}
}
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: ref.URL,
WaitFor: "#chapter-container",
WaitForTimeout: 15000,
RejectResources: true,
})
if err != nil {
return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
}
container := htmlutil.FindFirst(root, s.ChapterTextSelector())
if container == nil {
// Fallback: try class-based selector.
container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"})
}
if container == nil {
return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL)
}
text := htmlutil.NodeToMarkdown(container)
return scraper.Chapter{
Ref: ref,
Text: text,
}, nil
}
// ─── helpers ─────────────────────────────────────────────────────────────────
func resolveURL(base, href string) string {
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
return href
}
b, err := url.Parse(base)
if err != nil {
return base + href
}
ref, err := url.Parse(href)
if err != nil {
return base + href
}
return b.ResolveReference(ref).String()
}
func slugFromURL(bookURL string) string {
u, err := url.Parse(bookURL)
if err != nil {
return bookURL
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
if len(parts) >= 2 && parts[0] == "book" {
return parts[1]
}
if len(parts) > 0 {
return parts[len(parts)-1]
}
return ""
}
func parseChapterCount(s string) int {
// Formats: "123 Chapters", "1,234 Chapters", "123"
s = strings.ReplaceAll(s, ",", "")
fields := strings.Fields(s)
if len(fields) == 0 {
return 0
}
n, _ := strconv.Atoi(fields[0])
return n
}

213
scraper/internal/orchestrator/orchestrator.go Normal file
View File

@@ -0,0 +1,213 @@
// Package orchestrator coordinates the catalogue walk, metadata extraction,
// chapter-list fetching, and parallel chapter scraping.
//
// Concurrency model
// - One goroutine runs ScrapeCatalogue and feeds book URLs into a channel.
// - For each book, a dedicated goroutine calls ScrapeMetadata (metadata goroutine).
// - ScrapeChapterList is called in the metadata goroutine once metadata is done.
// - N worker goroutines (default: runtime.NumCPU()) each pull ChapterRef values
// from a shared work queue and call ScrapeChapterText.
// - A sync.WaitGroup ensures all chapter workers finish before the orchestrator
// signals completion.
package orchestrator
import (
"context"
"fmt"
"log/slog"
"runtime"
"sync"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
)
// Config holds tunable parameters for the orchestrator.
type Config struct {
// Workers is the number of goroutines used to scrape chapters in parallel.
// Defaults to runtime.NumCPU() when 0.
Workers int
// StaticRoot is the path to the static/books output directory.
StaticRoot string
// SingleBookURL when non-empty causes the orchestrator to scrape only
// that one book instead of walking the full catalogue.
SingleBookURL string
}
// Orchestrator coordinates the full scrape pipeline.
type Orchestrator struct {
cfg Config
novel scraper.NovelScraper
writer *writer.Writer
log *slog.Logger
workers int
}
// New returns a new Orchestrator.
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
workers := cfg.Workers
if workers <= 0 {
workers = runtime.NumCPU()
}
return &Orchestrator{
cfg: cfg,
novel: novel,
writer: writer.New(cfg.StaticRoot),
log: log,
workers: workers,
}
}
// Run executes the full scrape pipeline and blocks until it is complete or ctx
// is cancelled.
func (o *Orchestrator) Run(ctx context.Context) error {
o.log.Info("orchestrator starting",
"source", o.novel.SourceName(),
"workers", o.workers,
"static_root", o.cfg.StaticRoot,
)
// chapterWork is the shared queue consumed by chapter worker goroutines.
type chapterJob struct {
slug string
ref scraper.ChapterRef
}
chapterWork := make(chan chapterJob, o.workers*4)
// Start chapter worker pool.
var chapterWG sync.WaitGroup
for i := 0; i < o.workers; i++ {
chapterWG.Add(1)
go func(workerID int) {
defer chapterWG.Done()
for job := range chapterWork {
select {
case <-ctx.Done():
return
default:
}
// Skip if already on disk.
if o.writer.ChapterExists(job.slug, job.ref) {
o.log.Debug("chapter already exists, skipping",
"book", job.slug, "chapter", job.ref.Number)
continue
}
chapter, err := o.novel.ScrapeChapterText(ctx, job.ref)
if err != nil {
o.log.Error("chapter scrape failed",
"book", job.slug,
"chapter", job.ref.Number,
"url", job.ref.URL,
"err", err,
)
continue
}
if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
o.log.Error("chapter write failed",
"book", job.slug,
"chapter", job.ref.Number,
"err", err,
)
continue
}
o.log.Info("chapter saved",
"book", job.slug,
"chapter", job.ref.Number,
"worker", workerID,
)
}
}(i)
}
// processBook scrapes metadata + chapter list for one book, then enqueues
// chapter jobs. It is called inside a goroutine per book.
processBook := func(bookURL string) {
// Metadata goroutine.
meta, err := o.novel.ScrapeMetadata(ctx, bookURL)
if err != nil {
o.log.Error("metadata scrape failed", "url", bookURL, "err", err)
return
}
// Persist / update metadata.yaml.
if err := o.writer.WriteMetadata(meta); err != nil {
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
// Continue — chapters can still be scraped.
}
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
// Fetch chapter list.
refs, err := o.novel.ScrapeChapterList(ctx, bookURL)
if err != nil {
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
return
}
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
// Enqueue chapter jobs.
for _, ref := range refs {
select {
case <-ctx.Done():
return
case chapterWork <- chapterJob{slug: meta.Slug, ref: ref}:
}
}
}
if o.cfg.SingleBookURL != "" {
// Single-book mode: skip catalogue entirely.
o.log.Info("single-book mode", "url", o.cfg.SingleBookURL)
processBook(o.cfg.SingleBookURL)
} else {
// Catalogue mode: stream every book.
entries, catErrs := o.novel.ScrapeCatalogue(ctx)
// Drain catalogue errors in a separate goroutine.
go func() {
for err := range catErrs {
o.log.Error("catalogue error", "err", err)
}
}()
var bookWG sync.WaitGroup
for entry := range entries {
select {
case <-ctx.Done():
break
default:
}
bookWG.Add(1)
bookURL := entry.URL
go func() {
defer bookWG.Done()
processBook(bookURL)
}()
}
// Wait for all book goroutines to enqueue their chapters before
// closing the chapter work queue.
bookWG.Wait()
}
// Signal chapter workers there is no more work.
close(chapterWork)
// Wait for all in-flight chapter scrapes to finish.
chapterWG.Wait()
if ctx.Err() != nil {
return fmt.Errorf("orchestrator: context cancelled: %w", ctx.Err())
}
o.log.Info("orchestrator finished")
return nil
}

216
scraper/internal/scraper/htmlutil/htmlutil.go Normal file
View File

@@ -0,0 +1,216 @@
// Package htmlutil provides helper functions for parsing HTML with
// golang.org/x/net/html and extracting values by Selector descriptors.
package htmlutil
import (
"strings"
"github.com/libnovel/scraper/internal/scraper"
"golang.org/x/net/html"
)
// ParseHTML parses raw HTML and returns the root node.
func ParseHTML(raw string) (*html.Node, error) {
return html.Parse(strings.NewReader(raw))
}
// selectorMatches reports whether node n matches sel.
func selectorMatches(n *html.Node, sel scraper.Selector) bool {
if n.Type != html.ElementNode {
return false
}
if sel.Tag != "" && n.Data != sel.Tag {
return false
}
if sel.ID != "" {
for _, a := range n.Attr {
if a.Key == "id" && a.Val == sel.ID {
goto checkClass
}
}
return false
}
checkClass:
if sel.Class != "" {
for _, a := range n.Attr {
if a.Key == "class" {
for _, cls := range strings.Fields(a.Val) {
if cls == sel.Class {
goto matched
}
}
}
}
return false
}
matched:
return true
}
// attrVal returns the value of attribute key from node n.
func attrVal(n *html.Node, key string) string {
for _, a := range n.Attr {
if a.Key == key {
return a.Val
}
}
return ""
}
// textContent returns the concatenated text content of all descendant text nodes.
func textContent(n *html.Node) string {
var sb strings.Builder
var walk func(*html.Node)
walk = func(cur *html.Node) {
if cur.Type == html.TextNode {
sb.WriteString(cur.Data)
}
for c := cur.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(n)
return strings.TrimSpace(sb.String())
}
// FindFirst returns the first node matching sel within root.
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
var found *html.Node
var walk func(*html.Node) bool
walk = func(n *html.Node) bool {
if selectorMatches(n, sel) {
found = n
return true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
if walk(c) {
return true
}
}
return false
}
walk(root)
return found
}
// FindAll returns all nodes matching sel within root.
func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
var results []*html.Node
var walk func(*html.Node)
walk = func(n *html.Node) {
if selectorMatches(n, sel) {
results = append(results, n)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(root)
return results
}
// ExtractText extracts a string value from node n using sel.
// If sel.Attr is set the attribute value is returned; otherwise the inner text.
func ExtractText(n *html.Node, sel scraper.Selector) string {
if sel.Attr != "" {
return attrVal(n, sel.Attr)
}
return textContent(n)
}
// ExtractFirst locates the first match in root and returns its text/attr value.
func ExtractFirst(root *html.Node, sel scraper.Selector) string {
n := FindFirst(root, sel)
if n == nil {
return ""
}
return ExtractText(n, sel)
}
// ExtractAll locates all matches in root and returns their text/attr values.
func ExtractAll(root *html.Node, sel scraper.Selector) []string {
nodes := FindAll(root, sel)
out := make([]string, 0, len(nodes))
for _, n := range nodes {
if v := ExtractText(n, sel); v != "" {
out = append(out, v)
}
}
return out
}
// InnerHTML returns the serialized inner HTML of node n.
func InnerHTML(n *html.Node) string {
var sb strings.Builder
for c := n.FirstChild; c != nil; c = c.NextSibling {
_ = html.Render(&sb, c)
}
return sb.String()
}
// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
// representation suitable for chapter storage. Block elements become newlines;
// inline elements are inlined.
func NodeToMarkdown(n *html.Node) string {
var sb strings.Builder
nodeToMD(n, &sb)
return strings.TrimSpace(sb.String())
}
var blockElements = map[string]bool{
"p": true, "div": true, "br": true, "h1": true, "h2": true,
"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
"blockquote": true, "pre": true, "hr": true,
}
func nodeToMD(n *html.Node, sb *strings.Builder) {
switch n.Type {
case html.TextNode:
sb.WriteString(n.Data)
case html.ElementNode:
tag := n.Data
switch tag {
case "br":
sb.WriteString("\n")
case "hr":
sb.WriteString("\n---\n")
case "h1", "h2", "h3", "h4", "h5", "h6":
level := int(tag[1] - '0')
sb.WriteString("\n" + strings.Repeat("#", level) + " ")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("\n\n")
return
case "p", "div", "blockquote":
sb.WriteString("\n")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("\n")
return
case "em", "i":
sb.WriteString("*")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("*")
return
case "strong", "b":
sb.WriteString("**")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("**")
return
case "script", "style", "noscript":
return // drop
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
if blockElements[tag] {
sb.WriteString("\n")
}
}
}

149
scraper/internal/scraper/interfaces.go Normal file
View File

@@ -0,0 +1,149 @@
// Package scraper defines the core interfaces and domain types for the libnovel
// scraping system. Each novel source implements these interfaces; the orchestrator
// wires them together without knowing anything about the concrete provider.
package scraper
import "context"
// ─── Domain types ────────────────────────────────────────────────────────────
// BookMeta carries all bibliographic information about a novel.
type BookMeta struct {
// Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world".
Slug string `yaml:"slug"`
// Title is the human-readable novel title.
Title string `yaml:"title"`
// Author of the novel.
Author string `yaml:"author"`
// Cover is an absolute URL to the cover image.
Cover string `yaml:"cover,omitempty"`
// Status is e.g. "Ongoing", "Completed".
Status string `yaml:"status,omitempty"`
// Genres is a list of genre tags.
Genres []string `yaml:"genres,omitempty"`
// Summary is the full description/synopsis text.
Summary string `yaml:"summary,omitempty"`
// TotalChapters is the total number of chapters known at scrape time.
TotalChapters int `yaml:"total_chapters,omitempty"`
// SourceURL is the canonical URL of the book's landing page.
SourceURL string `yaml:"source_url"`
}
// CatalogueEntry is a lightweight reference returned by CatalogueProvider.
type CatalogueEntry struct {
// Title is the novel title as shown in the catalogue listing.
Title string
// URL is the canonical landing-page URL of the novel.
URL string
}
// ChapterRef is a reference to a single chapter returned by ChapterListProvider.
type ChapterRef struct {
// Number is the 1-based chapter index within the book.
Number int
// Title is the chapter display title.
Title string
// URL is the full URL of the chapter page.
URL string
// Volume is an optional volume number (0 means no volume grouping).
Volume int
}
// Chapter contains the fully-extracted text of a single chapter.
type Chapter struct {
Ref ChapterRef
// Text is the plain / lightly-formatted chapter body (Markdown).
Text string
}
// ─── Scraping selector descriptors ───────────────────────────────────────────
// Selector describes how to locate an element in an HTML document.
// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set
// they are combined (AND semantics).
type Selector struct {
// Tag is the HTML element name, e.g. "div", "p", "h1".
Tag string
// Class is one CSS class name (without the leading dot).
Class string
// ID is the element id attribute (without the leading #).
ID string
// Attr is an optional attribute name whose value should be extracted
// instead of the text content (e.g. "href", "src").
Attr string
// Multiple indicates that all matching elements should be collected,
// not just the first one.
Multiple bool
}
// ─── Provider interfaces ──────────────────────────────────────────────────────
// CatalogueProvider can enumerate every novel available on a source site.
// It handles pagination transparently and streams CatalogueEntry values.
type CatalogueProvider interface {
// CatalogueURL returns the root URL of the catalogue listing.
CatalogueURL() string
// EntriesSelector returns the selector that matches each novel card / row
// in the catalogue listing page.
EntriesSelector() Selector
// NextPageSelector returns the selector for the "next page" link.
// If the current page has no next page the implementation must return
// ("", nil) from ScrapeNextPage.
NextPageSelector() Selector
// ScrapeCatalogue pages through the entire catalogue, sending
// CatalogueEntry values to the returned channel. The channel is closed
// when all pages have been scraped or ctx is cancelled.
// Errors are surfaced via the error channel; a non-nil error does not
// necessarily terminate scraping.
ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error)
}
// MetadataProvider can extract structured book metadata from a novel's landing page.
type MetadataProvider interface {
// MetadataSelectors returns a map of field name → Selector used to
// locate each metadata element on the book page.
// Required keys: "title", "author".
// Optional keys: "cover", "status", "genres", "summary", "total_chapters".
MetadataSelectors() map[string]Selector
// ScrapeMetadata fetches and parses the metadata for the book at bookURL.
ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error)
}
// ChapterListProvider can enumerate all chapters of a book from the chapter-list page.
type ChapterListProvider interface {
// ChaptersURL derives the chapter-list URL from a book landing-page URL.
ChaptersURL(bookURL string) string
// ChapterEntrySelector returns the selector that matches each chapter row
// in the chapter list page.
ChapterEntrySelector() Selector
// ScrapeChapterList returns all chapter references for a book, ordered
// by chapter number ascending.
ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error)
}
// ChapterTextProvider can extract the readable text from a single chapter page.
type ChapterTextProvider interface {
// ChapterTextSelector returns the selector that wraps the chapter body.
ChapterTextSelector() Selector
// ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown.
ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error)
}
// NovelScraper is the full interface that a concrete novel source must implement.
// It composes all four provider interfaces.
type NovelScraper interface {
CatalogueProvider
MetadataProvider
ChapterListProvider
ChapterTextProvider
// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
SourceName() string
}

132
scraper/internal/server/server.go Normal file
View File

@@ -0,0 +1,132 @@
// Package server exposes the scraper as an HTTP service.
//
// Endpoints:
//
// POST /scrape — enqueue a full catalogue scrape
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
// GET /health — liveness probe
package server
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"sync"
"time"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
)
// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
mu sync.Mutex
running bool
}
// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger) *Server {
return &Server{
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
}
}
// ListenAndServe starts the HTTP server and blocks until the provided context
// is cancelled.
func (s *Server) ListenAndServe(ctx context.Context) error {
mux := http.NewServeMux()
mux.HandleFunc("GET /health", s.handleHealth)
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
srv := &http.Server{
Addr: s.addr,
Handler: mux,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
IdleTimeout: 60 * time.Second,
}
errCh := make(chan error, 1)
go func() { errCh <- srv.ListenAndServe() }()
s.log.Info("HTTP server listening", "addr", s.addr)
select {
case <-ctx.Done():
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return srv.Shutdown(shutCtx)
case err := <-errCh:
return err
}
}
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
cfg := s.oCfg
cfg.SingleBookURL = "" // full catalogue
s.runAsync(w, cfg)
}
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
var body struct {
URL string `json:"url"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
return
}
cfg := s.oCfg
cfg.SingleBookURL = body.URL
s.runAsync(w, cfg)
}
// runAsync launches an orchestrator in the background and returns 202 Accepted.
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
s.mu.Lock()
if s.running {
s.mu.Unlock()
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
return
}
s.running = true
s.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
go func() {
defer func() {
s.mu.Lock()
s.running = false
s.mu.Unlock()
}()
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
o := orchestrator.New(cfg, s.novel, s.log)
if err := o.Run(ctx); err != nil {
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
}
}()
}

140
scraper/internal/writer/writer.go Normal file
View File

@@ -0,0 +1,140 @@
// Package writer handles persistence of scraped chapters and metadata.
//
// Directory layout:
//
// static/books/
// ├── {book-slug}/
// │ ├── metadata.yaml
// │ ├── vol-0/ (no volume grouping)
// │ │ ├── 1-50/
// │ │ │ ├── chapter-1.md
// │ │ │ └── …
// │ │ └── 51-100/
// │ │ └── …
// │ └── vol-1/
// │ └── …
package writer
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/libnovel/scraper/internal/scraper"
"gopkg.in/yaml.v3"
)
const chaptersPerFolder = 50
// Writer persists scraped content under a configurable root directory.
type Writer struct {
root string // e.g. "./static/books"
}
// New creates a Writer that stores files under root.
func New(root string) *Writer {
return &Writer{root: root}
}
// ─── Metadata ─────────────────────────────────────────────────────────────────
// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml.
// It creates the directory if it does not exist and overwrites any existing file.
func (w *Writer) WriteMetadata(meta scraper.BookMeta) error {
dir := w.bookDir(meta.Slug)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
}
path := filepath.Join(dir, "metadata.yaml")
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("writer: create metadata %s: %w", path, err)
}
defer f.Close()
enc := yaml.NewEncoder(f)
enc.SetIndent(2)
if err := enc.Encode(meta); err != nil {
return fmt.Errorf("writer: encode metadata: %w", err)
}
return enc.Close()
}
// ReadMetadata reads the metadata.yaml for slug if it exists.
// Returns (zero-value, false, nil) when the file does not exist.
func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) {
path := filepath.Join(w.bookDir(slug), "metadata.yaml")
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return scraper.BookMeta{}, false, nil
}
return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err)
}
var meta scraper.BookMeta
if err := yaml.Unmarshal(data, &meta); err != nil {
return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err)
}
return meta, true, nil
}
// ─── Chapters ─────────────────────────────────────────────────────────────────
// ChapterExists returns true if the markdown file for ref already exists on disk.
func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool {
_, err := os.Stat(w.chapterPath(slug, ref))
return err == nil
}
// WriteChapter writes chapter.Text to the appropriate markdown file.
// The parent directories are created on demand.
func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error {
path := w.chapterPath(slug, chapter.Ref)
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
}
// Build the markdown document.
var sb strings.Builder
sb.WriteString("# ")
sb.WriteString(chapter.Ref.Title)
sb.WriteString("\n\n")
sb.WriteString(chapter.Text)
sb.WriteString("\n")
if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
return fmt.Errorf("writer: write chapter %s: %w", path, err)
}
return nil
}
// ─── Path helpers ─────────────────────────────────────────────────────────────
// bookDir returns the root directory for a book slug.
func (w *Writer) bookDir(slug string) string {
return filepath.Join(w.root, slug)
}
// chapterPath computes the full file path for a chapter.
//
// vol-{volume}/{folderRange}/chapter-{number}.md
//
// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md
func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string {
vol := ref.Volume // 0 == no volume grouping
volDir := fmt.Sprintf("vol-%d", vol)
// Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", …
lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1
hi := lo + chaptersPerFolder - 1
rangeDir := fmt.Sprintf("%d-%d", lo, hi)
filename := fmt.Sprintf("chapter-%d.md", ref.Number)
return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename)
}