feat: initial commit

.env.example

@@ -0,0 +1,23 @@
+# libnovel scraper — environment overrides
+# Copy to .env and adjust values; do NOT commit this file with real secrets.
+
+# Browserless API token (leave empty to disable auth)
+BROWSERLESS_TOKEN=
+
+# Number of concurrent browser sessions in Browserless
+BROWSERLESS_CONCURRENT=10
+
+# Queue depth before Browserless returns 429
+BROWSERLESS_QUEUED=100
+
+# Per-session timeout in ms
+BROWSERLESS_TIMEOUT=60000
+
+# Which Browserless strategy the scraper uses: content | scrape | cdp
+BROWSERLESS_STRATEGY=content
+
+# Chapter worker goroutines (0 = NumCPU inside the container)
+SCRAPER_WORKERS=0
+
+# Host path to mount as the static output directory
+STATIC_ROOT=./static/books

docker-compose.yml

@@ -0,0 +1,60 @@
+version: "3.9"
+
+services:
+  # ─── Browserless ────────────────────────────────────────────────────────────
+  browserless:
+    image: ghcr.io/browserless/chromium:latest
+    container_name: libnovel-browserless
+    restart: unless-stopped
+    environment:
+      # Set a token to lock down the endpoint; the scraper reads it via
+      # BROWSERLESS_TOKEN below.
+      TOKEN: "${BROWSERLESS_TOKEN:-}"
+      # Allow up to 10 concurrent browser sessions.
+      CONCURRENT: "${BROWSERLESS_CONCURRENT:-10}"
+      # Queue up to 100 requests before returning 429.
+      QUEUED: "${BROWSERLESS_QUEUED:-100}"
+      # Per-session timeout in ms.
+      TIMEOUT: "${BROWSERLESS_TIMEOUT:-60000}"
+    ports:
+      - "3000:3000"
+    # Shared memory is required for Chrome.
+    shm_size: "2gb"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3000/json/version"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ─── Scraper ─────────────────────────────────────────────────────────────────
+  scraper:
+    build:
+      context: ./scraper
+      dockerfile: Dockerfile
+    container_name: libnovel-scraper
+    restart: unless-stopped
+    depends_on:
+      browserless:
+        condition: service_healthy
+    environment:
+      BROWSERLESS_URL: "http://browserless:3000"
+      BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}"
+      # content | scrape | cdp — swap to test different strategies.
+      BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-content}"
+      # 0 → defaults to NumCPU inside the container.
+      SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}"
+      SCRAPER_STATIC_ROOT: "/app/static/books"
+      SCRAPER_HTTP_ADDR: ":8080"
+    ports:
+      - "8080:8080"
+    volumes:
+      # Mount the host static directory so scraped content is available outside
+      # the container. Create ./static/books on the host first if needed.
+      - "${STATIC_ROOT:-./static/books}:/app/static/books"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:8080/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 3
+
+volumes: {}

scraper/Dockerfile

@@ -0,0 +1,44 @@
+# ── Build stage ────────────────────────────────────────────────────────────────
+FROM golang:1.25-alpine AS builder
+
+WORKDIR /build
+
+# Cache dependency downloads separately from source compilation.
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -ldflags="-s -w" -o /scraper ./cmd/scraper
+
+# ── Runtime stage ──────────────────────────────────────────────────────────────
+FROM alpine:3.20
+
+# ca-certificates is required for HTTPS requests to novelfire.net.
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /scraper /app/scraper
+
+# Create the default static output directory.
+RUN mkdir -p /app/static/books
+
+# Non-root user.
+RUN addgroup -S scraper && adduser -S scraper -G scraper
+RUN chown -R scraper:scraper /app
+USER scraper
+
+# ── Configuration ─────────────────────────────────────────────────────────────
+ENV BROWSERLESS_URL=http://browserless:3000
+ENV BROWSERLESS_STRATEGY=content
+ENV SCRAPER_WORKERS=0
+ENV SCRAPER_STATIC_ROOT=/app/static/books
+ENV SCRAPER_HTTP_ADDR=:8080
+
+EXPOSE 8080
+
+# Default: run as an HTTP server. Override CMD to use "run" for one-shot.
+ENTRYPOINT ["/app/scraper"]
+CMD ["serve"]

scraper/cmd/scraper/main.go

@@ -0,0 +1,150 @@
+// Command scraper is the entrypoint for the libnovel scraper service.
+//
+// Usage (CLI one-shot):
+//
+//	scraper run [--url <book-url>]
+//
+// Usage (HTTP server):
+//
+//	scraper serve
+//
+// Environment variables:
+//
+//	BROWSERLESS_URL      Browserless base URL           (default: http://localhost:3000)
+//	BROWSERLESS_TOKEN    Browserless API token          (default: "")
+//	BROWSERLESS_STRATEGY content | scrape | cdp         (default: content)
+//	SCRAPER_WORKERS      Chapter goroutine count        (default: NumCPU)
+//	SCRAPER_STATIC_ROOT  Output directory               (default: ./static/books)
+//	SCRAPER_HTTP_ADDR    HTTP listen address            (default: :8080)
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/libnovel/scraper/internal/browser"
+	"github.com/libnovel/scraper/internal/novelfire"
+	"github.com/libnovel/scraper/internal/orchestrator"
+	"github.com/libnovel/scraper/internal/server"
+)
+
+func main() {
+	log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	}))
+
+	if err := run(log); err != nil {
+		log.Error("fatal", "err", err)
+		os.Exit(1)
+	}
+}
+
+func run(log *slog.Logger) error {
+	args := os.Args[1:]
+	if len(args) == 0 {
+		printUsage()
+		return nil
+	}
+
+	cmd := strings.ToLower(args[0])
+
+	browserCfg := browser.Config{
+		BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3000"),
+		Token:   envOr("BROWSERLESS_TOKEN", ""),
+	}
+
+	strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyContent))))
+	bc := newBrowserClient(strategy, browserCfg)
+
+	nf := novelfire.New(bc, log)
+
+	workers := 0
+	if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
+		n, err := strconv.Atoi(s)
+		if err == nil && n > 0 {
+			workers = n
+		}
+	}
+	if workers == 0 {
+		workers = runtime.NumCPU()
+	}
+
+	oCfg := orchestrator.Config{
+		Workers:    workers,
+		StaticRoot: envOr("SCRAPER_STATIC_ROOT", "./static/books"),
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	switch cmd {
+	case "run":
+		// Optional --url flag.
+		if len(args) >= 3 && args[1] == "--url" {
+			oCfg.SingleBookURL = args[2]
+		}
+		log.Info("starting one-shot scrape",
+			"strategy", strategy,
+			"workers", workers,
+			"static_root", oCfg.StaticRoot,
+			"single_book", oCfg.SingleBookURL,
+		)
+		o := orchestrator.New(oCfg, nf, log)
+		return o.Run(ctx)
+
+	case "serve":
+		addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
+		log.Info("starting HTTP server",
+			"addr", addr,
+			"strategy", strategy,
+			"workers", workers,
+		)
+		srv := server.New(addr, oCfg, nf, log)
+		return srv.ListenAndServe(ctx)
+
+	default:
+		return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
+	}
+}
+
+func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
+	switch strategy {
+	case browser.StrategyScrape:
+		return browser.NewScrapeClient(cfg)
+	case browser.StrategyCDP:
+		return browser.NewCDPClient(cfg)
+	default:
+		return browser.NewContentClient(cfg)
+	}
+}
+
+func envOr(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
+
+func printUsage() {
+	fmt.Fprintf(os.Stderr, `libnovel scraper
+
+Commands:
+  run [--url <book-url>]   One-shot: scrape full catalogue, or a single book
+  serve                    Start HTTP server (POST /scrape, POST /scrape/book)
+
+Environment variables:
+  BROWSERLESS_URL          Browserless base URL      (default: http://localhost:3000)
+  BROWSERLESS_TOKEN        API token                 (default: "")
+  BROWSERLESS_STRATEGY     content | scrape | cdp    (default: content)
+  SCRAPER_WORKERS          Chapter goroutines        (default: NumCPU = %d)
+  SCRAPER_STATIC_ROOT      Output directory          (default: ./static/books)
+  SCRAPER_HTTP_ADDR        HTTP listen address       (default: :8080)
+`, runtime.NumCPU())
+}

scraper/go.mod

@@ -0,0 +1,9 @@
+module github.com/libnovel/scraper
+
+go 1.25.0
+
+require (
+	github.com/gorilla/websocket v1.5.3 // indirect
+	golang.org/x/net v0.51.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

scraper/go.sum

@@ -0,0 +1,7 @@
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scraper/internal/browser/cdp.go

@@ -0,0 +1,131 @@
+package browser
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
+type cdpClient struct {
+	cfg Config
+}
+
+// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
+func NewCDPClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 60 * time.Second
+	}
+	return &cdpClient{cfg: cfg}
+}
+
+func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
+
+func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
+	return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
+}
+
+func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
+	return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
+}
+
+// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
+// navigates to pageURL, and invokes fn with a live CDPConn.
+func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
+	// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
+	wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
+	wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
+	wsURL += "/devtools/browser"
+	sep := "?"
+	if c.cfg.Token != "" {
+		wsURL += sep + "token=" + c.cfg.Token
+		sep = "&"
+	}
+	wsURL += sep + "url=" + pageURL
+
+	dialer := websocket.Dialer{
+		HandshakeTimeout: 15 * time.Second,
+		Proxy:            http.ProxyFromEnvironment,
+	}
+
+	conn, _, err := dialer.DialContext(ctx, wsURL, nil)
+	if err != nil {
+		return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
+	}
+
+	cdp := &cdpConn{ws: conn}
+	defer cdp.Close()
+
+	return fn(ctx, cdp)
+}
+
+// ─── cdpConn ─────────────────────────────────────────────────────────────────
+
+type cdpConn struct {
+	ws      *websocket.Conn
+	counter atomic.Int64
+}
+
+type cdpRequest struct {
+	ID     int64          `json:"id"`
+	Method string         `json:"method"`
+	Params map[string]any `json:"params,omitempty"`
+}
+
+type cdpResponse struct {
+	ID     int64          `json:"id"`
+	Result map[string]any `json:"result,omitempty"`
+	Error  *struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	} `json:"error,omitempty"`
+}
+
+func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
+	id := c.counter.Add(1)
+
+	req := cdpRequest{ID: id, Method: method, Params: params}
+	data, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("cdp send: marshal: %w", err)
+	}
+
+	if dl, ok := ctx.Deadline(); ok {
+		_ = c.ws.SetWriteDeadline(dl)
+	}
+	if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
+		return nil, fmt.Errorf("cdp send: write: %w", err)
+	}
+
+	// Read messages until we find the response matching our id.
+	for {
+		if dl, ok := ctx.Deadline(); ok {
+			_ = c.ws.SetReadDeadline(dl)
+		}
+		_, msg, err := c.ws.ReadMessage()
+		if err != nil {
+			return nil, fmt.Errorf("cdp send: read: %w", err)
+		}
+		var resp cdpResponse
+		if err := json.Unmarshal(msg, &resp); err != nil {
+			continue // skip non-JSON frames (events etc.)
+		}
+		if resp.ID != id {
+			continue // event or different command reply
+		}
+		if resp.Error != nil {
+			return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
+		}
+		return resp.Result, nil
+	}
+}
+
+func (c *cdpConn) Close() error {
+	return c.ws.Close()
+}

scraper/internal/browser/content_scrape.go

@@ -0,0 +1,146 @@
+package browser
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// Config holds the connection parameters for a Browserless instance.
+type Config struct {
+	// BaseURL is the HTTP base URL, e.g. "http://localhost:3000".
+	BaseURL string
+	// Token is the optional API token (BROWSERLESS_TOKEN env var).
+	Token string
+	// Timeout is the per-request HTTP timeout; defaults to 60 s.
+	Timeout time.Duration
+}
+
+// contentClient implements BrowserClient using the /content endpoint.
+type contentClient struct {
+	cfg  Config
+	http *http.Client
+}
+
+// NewContentClient returns a BrowserClient that uses POST /content.
+func NewContentClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 60 * time.Second
+	}
+	return &contentClient{
+		cfg:  cfg,
+		http: &http.Client{Timeout: cfg.Timeout},
+	}
+}
+
+func (c *contentClient) Strategy() Strategy { return StrategyContent }
+
+func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return "", fmt.Errorf("content: marshal request: %w", err)
+	}
+
+	url := c.cfg.BaseURL + "/content"
+	if c.cfg.Token != "" {
+		url += "?token=" + c.cfg.Token
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return "", fmt.Errorf("content: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.http.Do(httpReq)
+	if err != nil {
+		return "", fmt.Errorf("content: do request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b)
+	}
+
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("content: read body: %w", err)
+	}
+	return string(raw), nil
+}
+
+func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
+	return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient")
+}
+
+func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
+	return fmt.Errorf("content client does not support CDP; use NewCDPClient")
+}
+
+// ─── /scrape client ───────────────────────────────────────────────────────────
+
+type scrapeClient struct {
+	cfg  Config
+	http *http.Client
+}
+
+// NewScrapeClient returns a BrowserClient that uses POST /scrape.
+func NewScrapeClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 60 * time.Second
+	}
+	return &scrapeClient{
+		cfg:  cfg,
+		http: &http.Client{Timeout: cfg.Timeout},
+	}
+}
+
+func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
+
+func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
+	return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
+}
+
+func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
+	}
+
+	url := c.cfg.BaseURL + "/scrape"
+	if c.cfg.Token != "" {
+		url += "?token=" + c.cfg.Token
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.http.Do(httpReq)
+	if err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
+	}
+
+	var result ScrapeResponse
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
+	}
+	return result, nil
+}
+
+func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
+	return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
+}

scraper/internal/browser/interface.go

@@ -0,0 +1,101 @@
+// Package browser defines the BrowserClient interface and helper types for
+// communicating with a Browserless instance.
+package browser
+
+import "context"
+
+// Strategy selects which Browserless API endpoint / protocol to use.
+type Strategy string
+
+const (
+	// StrategyContent uses the POST /content endpoint, which returns the final
+	// rendered HTML of the page. Fastest; suitable for most JS-rendered sites.
+	StrategyContent Strategy = "content"
+
+	// StrategyScrape uses the POST /scrape endpoint, which accepts a list of
+	// CSS selectors and returns structured JSON. Good when you know exactly
+	// which elements you need.
+	StrategyScrape Strategy = "scrape"
+
+	// StrategyCDP uses the WebSocket /devtools/browser endpoint (Chrome
+	// DevTools Protocol). Most powerful; required for complex interactions
+	// (clicking, scrolling, waiting for network idle, etc.).
+	StrategyCDP Strategy = "cdp"
+)
+
+// ContentRequest is the body sent to POST /content.
+type ContentRequest struct {
+	URL             string `json:"url"`
+	WaitFor         string `json:"waitForSelector,omitempty"`
+	WaitForTimeout  int    `json:"waitForTimeout,omitempty"` // ms
+	RejectResources bool   `json:"rejectResources,omitempty"`
+}
+
+// ScrapeElement is one element descriptor inside a ScrapeRequest.
+type ScrapeElement struct {
+	Selector string `json:"selector"`
+	Timeout  int    `json:"timeout,omitempty"` // ms
+}
+
+// ScrapeRequest is the body sent to POST /scrape.
+type ScrapeRequest struct {
+	URL      string          `json:"url"`
+	Elements []ScrapeElement `json:"elements"`
+	WaitFor  string          `json:"waitForSelector,omitempty"`
+}
+
+// ScrapeResult is one entry in the response from POST /scrape.
+type ScrapeResult struct {
+	Selector string          `json:"selector"`
+	Results  []ScrapeElement `json:"results"`
+}
+
+// ScrapeAttribute holds a single attribute value from a scraped element.
+type ScrapeAttribute struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+// ScrapedElement is one item inside ScrapeResult.Results.
+type ScrapedElement struct {
+	Text       string            `json:"text"`
+	Attributes []ScrapeAttribute `json:"attributes"`
+}
+
+// ScrapeResponse is the top-level response from POST /scrape.
+type ScrapeResponse struct {
+	Data []ScrapeResult `json:"data"`
+}
+
+// BrowserClient is an abstraction over the three Browserless API strategies.
+// Callers choose the strategy best suited to the target site; the interface
+// signature is identical regardless of strategy.
+type BrowserClient interface {
+	// Strategy returns the strategy this client uses.
+	Strategy() Strategy
+
+	// GetContent fetches the fully-rendered HTML of url using the /content
+	// endpoint. Only meaningful when Strategy() == StrategyContent.
+	GetContent(ctx context.Context, req ContentRequest) (string, error)
+
+	// ScrapePage calls the /scrape endpoint and returns structured data.
+	// Only meaningful when Strategy() == StrategyScrape.
+	ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error)
+
+	// CDPSession opens a CDP WebSocket session and calls fn with the raw
+	// WebSocket connection. Only meaningful when Strategy() == StrategyCDP.
+	// The session is closed when fn returns.
+	CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error
+}
+
+// CDPSessionFunc is the callback invoked inside a CDP session.
+// conn is a live *websocket.Conn connected to a Browserless CDP endpoint.
+type CDPSessionFunc func(ctx context.Context, conn CDPConn) error
+
+// CDPConn is the minimal interface the orchestrator needs over a CDP WebSocket.
+type CDPConn interface {
+	// Send sends a raw CDP command (JSON-encoded) and returns the response.
+	Send(ctx context.Context, method string, params map[string]any) (map[string]any, error)
+	// Close closes the underlying connection.
+	Close() error
+}

scraper/internal/novelfire/scraper.go

@@ -0,0 +1,372 @@
+// Package novelfire provides a NovelScraper implementation for novelfire.net.
+//
+// Site structure (as of 2025):
+//
+//	Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
+//	Book page : https://novelfire.net/book/{slug}
+//	Chapters  : https://novelfire.net/book/{slug}/chapters?page=N
+//	Chapter   : https://novelfire.net/book/{slug}/{chapter-slug}
+package novelfire
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/libnovel/scraper/internal/browser"
+	"github.com/libnovel/scraper/internal/scraper"
+	"github.com/libnovel/scraper/internal/scraper/htmlutil"
+)
+
+const (
+	baseURL       = "https://novelfire.net"
+	cataloguePath = "/genre-all/sort-new/status-all/all-novel"
+)
+
+// Scraper is the novelfire.net implementation of scraper.NovelScraper.
+// It uses the /content strategy by default (rendered HTML via Browserless).
+type Scraper struct {
+	client browser.BrowserClient
+	log    *slog.Logger
+}
+
+// New returns a new novelfire Scraper.
+func New(client browser.BrowserClient, log *slog.Logger) *Scraper {
+	if log == nil {
+		log = slog.Default()
+	}
+	return &Scraper{client: client, log: log}
+}
+
+// SourceName implements NovelScraper.
+func (s *Scraper) SourceName() string { return "novelfire.net" }
+
+// ─── CatalogueProvider ───────────────────────────────────────────────────────
+
+func (s *Scraper) CatalogueURL() string {
+	return baseURL + cataloguePath
+}
+
+func (s *Scraper) EntriesSelector() scraper.Selector {
+	// Each novel card: <div class="novel-item">
+	return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}
+}
+
+func (s *Scraper) NextPageSelector() scraper.Selector {
+	// <a class="next" href="...">
+	return scraper.Selector{Tag: "a", Class: "next", Attr: "href"}
+}
+
+// ScrapeCatalogue streams all CatalogueEntry values across all pages.
+func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
+	entries := make(chan scraper.CatalogueEntry, 64)
+	errs := make(chan error, 16)
+
+	go func() {
+		defer close(entries)
+		defer close(errs)
+
+		pageURL := s.CatalogueURL()
+		page := 1
+
+		for pageURL != "" {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
+			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
+
+			html, err := s.client.GetContent(ctx, browser.ContentRequest{
+				URL:             pageURL,
+				WaitFor:         ".novel-item",
+				WaitForTimeout:  10000,
+				RejectResources: true,
+			})
+			if err != nil {
+				errs <- fmt.Errorf("catalogue page %d: %w", page, err)
+				return
+			}
+
+			root, err := htmlutil.ParseHTML(html)
+			if err != nil {
+				errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
+				return
+			}
+
+			// Extract novel cards.
+			cards := htmlutil.FindAll(root, s.EntriesSelector())
+			if len(cards) == 0 {
+				s.log.Warn("no novel cards found, stopping pagination", "page", page)
+				return
+			}
+
+			for _, card := range cards {
+				// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
+				titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"}
+				titleNode := htmlutil.FindFirst(card, titleSel)
+
+				var title, href string
+				if titleNode != nil {
+					linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
+					if linkNode != nil {
+						title = htmlutil.ExtractText(linkNode, scraper.Selector{})
+						href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
+					}
+				}
+				if href == "" || title == "" {
+					continue
+				}
+
+				// Resolve relative URL.
+				bookURL := resolveURL(baseURL, href)
+				select {
+				case <-ctx.Done():
+					return
+				case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}:
+				}
+			}
+
+			// Find next page link.
+			nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
+			if nextHref == "" {
+				break
+			}
+			pageURL = resolveURL(baseURL, nextHref)
+			page++
+		}
+	}()
+
+	return entries, errs
+}
+
+// ─── MetadataProvider ────────────────────────────────────────────────────────
+
+func (s *Scraper) MetadataSelectors() map[string]scraper.Selector {
+	return map[string]scraper.Selector{
+		// <h1 class="novel-title">Title</h1>
+		"title": {Tag: "h1", Class: "novel-title"},
+		// <span class="author"><a>Author Name</a></span>
+		"author": {Tag: "span", Class: "author"},
+		// <img class="cover" src="...">
+		"cover": {Tag: "img", Class: "cover", Attr: "src"},
+		// <span class="status">Ongoing</span>
+		"status": {Tag: "span", Class: "status"},
+		// <div class="genres"><a>Tag1</a><a>Tag2</a>…</div>
+		"genres": {Tag: "div", Class: "genres", Multiple: true},
+		// <div class="summary"><p>...</p></div>
+		"summary": {Tag: "div", Class: "summary"},
+		// <span class="chapter-count">123 Chapters</span>
+		"total_chapters": {Tag: "span", Class: "chapter-count"},
+	}
+}
+
+func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) {
+	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
+		URL:             bookURL,
+		WaitFor:         ".novel-title",
+		WaitForTimeout:  10000,
+		RejectResources: true,
+	})
+	if err != nil {
+		return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
+	}
+
+	root, err := htmlutil.ParseHTML(raw)
+	if err != nil {
+		return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
+	}
+
+	sels := s.MetadataSelectors()
+
+	title := htmlutil.ExtractFirst(root, sels["title"])
+	author := htmlutil.ExtractFirst(root, sels["author"])
+	cover := htmlutil.ExtractFirst(root, sels["cover"])
+	status := htmlutil.ExtractFirst(root, sels["status"])
+
+	// Genres: all <a> tags inside the genres div.
+	genresNode := htmlutil.FindFirst(root, sels["genres"])
+	var genres []string
+	if genresNode != nil {
+		genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
+	}
+
+	summary := htmlutil.ExtractFirst(root, sels["summary"])
+
+	totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"])
+	totalChapters := parseChapterCount(totalStr)
+
+	// Derive slug from URL.
+	slug := slugFromURL(bookURL)
+
+	return scraper.BookMeta{
+		Slug:          slug,
+		Title:         title,
+		Author:        author,
+		Cover:         cover,
+		Status:        status,
+		Genres:        genres,
+		Summary:       summary,
+		TotalChapters: totalChapters,
+		SourceURL:     bookURL,
+	}, nil
+}
+
+// ─── ChapterListProvider ─────────────────────────────────────────────────────
+
+func (s *Scraper) ChaptersURL(bookURL string) string {
+	return strings.TrimRight(bookURL, "/") + "/chapters"
+}
+
+func (s *Scraper) ChapterEntrySelector() scraper.Selector {
+	// <li class="chapter-item"><a href="/book/slug/chapter-1">Chapter 1: Title</a></li>
+	return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true}
+}
+
+func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
+	var refs []scraper.ChapterRef
+	pageURL := s.ChaptersURL(bookURL)
+	page := 1
+
+	for pageURL != "" {
+		select {
+		case <-ctx.Done():
+			return refs, ctx.Err()
+		default:
+		}
+
+		s.log.Info("scraping chapter list", "page", page, "url", pageURL)
+
+		raw, err := s.client.GetContent(ctx, browser.ContentRequest{
+			URL:             pageURL,
+			WaitFor:         ".chapter-item",
+			WaitForTimeout:  10000,
+			RejectResources: true,
+		})
+		if err != nil {
+			return refs, fmt.Errorf("chapter list page %d: %w", page, err)
+		}
+
+		root, err := htmlutil.ParseHTML(raw)
+		if err != nil {
+			return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
+		}
+
+		items := htmlutil.FindAll(root, s.ChapterEntrySelector())
+		for _, item := range items {
+			linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"})
+			if linkNode == nil {
+				continue
+			}
+			href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
+			chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
+			if href == "" {
+				continue
+			}
+			chURL := resolveURL(baseURL, href)
+			num := len(refs) + 1
+			refs = append(refs, scraper.ChapterRef{
+				Number: num,
+				Title:  strings.TrimSpace(chTitle),
+				URL:    chURL,
+			})
+		}
+
+		// Next page.
+		nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector())
+		if nextHref == "" {
+			break
+		}
+		pageURL = resolveURL(baseURL, nextHref)
+		page++
+	}
+
+	return refs, nil
+}
+
+// ─── ChapterTextProvider ─────────────────────────────────────────────────────
+
+func (s *Scraper) ChapterTextSelector() scraper.Selector {
+	// <div id="chapter-container"> or <div class="chapter-content">
+	return scraper.Selector{Tag: "div", ID: "chapter-container"}
+}
+
+func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
+	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
+		URL:             ref.URL,
+		WaitFor:         "#chapter-container",
+		WaitForTimeout:  15000,
+		RejectResources: true,
+	})
+	if err != nil {
+		return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
+	}
+
+	root, err := htmlutil.ParseHTML(raw)
+	if err != nil {
+		return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
+	}
+
+	container := htmlutil.FindFirst(root, s.ChapterTextSelector())
+	if container == nil {
+		// Fallback: try class-based selector.
+		container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"})
+	}
+	if container == nil {
+		return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL)
+	}
+
+	text := htmlutil.NodeToMarkdown(container)
+
+	return scraper.Chapter{
+		Ref:  ref,
+		Text: text,
+	}, nil
+}
+
+// ─── helpers ─────────────────────────────────────────────────────────────────
+
+func resolveURL(base, href string) string {
+	if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
+		return href
+	}
+	b, err := url.Parse(base)
+	if err != nil {
+		return base + href
+	}
+	ref, err := url.Parse(href)
+	if err != nil {
+		return base + href
+	}
+	return b.ResolveReference(ref).String()
+}
+
+func slugFromURL(bookURL string) string {
+	u, err := url.Parse(bookURL)
+	if err != nil {
+		return bookURL
+	}
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(parts) >= 2 && parts[0] == "book" {
+		return parts[1]
+	}
+	if len(parts) > 0 {
+		return parts[len(parts)-1]
+	}
+	return ""
+}
+
+func parseChapterCount(s string) int {
+	// Formats: "123 Chapters", "1,234 Chapters", "123"
+	s = strings.ReplaceAll(s, ",", "")
+	fields := strings.Fields(s)
+	if len(fields) == 0 {
+		return 0
+	}
+	n, _ := strconv.Atoi(fields[0])
+	return n
+}

scraper/internal/orchestrator/orchestrator.go

@@ -0,0 +1,213 @@
+// Package orchestrator coordinates the catalogue walk, metadata extraction,
+// chapter-list fetching, and parallel chapter scraping.
+//
+// Concurrency model
+//   - One goroutine runs ScrapeCatalogue and feeds book URLs into a channel.
+//   - For each book, a dedicated goroutine calls ScrapeMetadata (metadata goroutine).
+//   - ScrapeChapterList is called in the metadata goroutine once metadata is done.
+//   - N worker goroutines (default: runtime.NumCPU()) each pull ChapterRef values
+//     from a shared work queue and call ScrapeChapterText.
+//   - A sync.WaitGroup ensures all chapter workers finish before the orchestrator
+//     signals completion.
+package orchestrator
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"runtime"
+	"sync"
+
+	"github.com/libnovel/scraper/internal/scraper"
+	"github.com/libnovel/scraper/internal/writer"
+)
+
+// Config holds tunable parameters for the orchestrator.
+type Config struct {
+	// Workers is the number of goroutines used to scrape chapters in parallel.
+	// Defaults to runtime.NumCPU() when 0.
+	Workers int
+
+	// StaticRoot is the path to the static/books output directory.
+	StaticRoot string
+
+	// SingleBookURL when non-empty causes the orchestrator to scrape only
+	// that one book instead of walking the full catalogue.
+	SingleBookURL string
+}
+
+// Orchestrator coordinates the full scrape pipeline.
+type Orchestrator struct {
+	cfg     Config
+	novel   scraper.NovelScraper
+	writer  *writer.Writer
+	log     *slog.Logger
+	workers int
+}
+
+// New returns a new Orchestrator.
+func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
+	workers := cfg.Workers
+	if workers <= 0 {
+		workers = runtime.NumCPU()
+	}
+	return &Orchestrator{
+		cfg:     cfg,
+		novel:   novel,
+		writer:  writer.New(cfg.StaticRoot),
+		log:     log,
+		workers: workers,
+	}
+}
+
+// Run executes the full scrape pipeline and blocks until it is complete or ctx
+// is cancelled.
+func (o *Orchestrator) Run(ctx context.Context) error {
+	o.log.Info("orchestrator starting",
+		"source", o.novel.SourceName(),
+		"workers", o.workers,
+		"static_root", o.cfg.StaticRoot,
+	)
+
+	// chapterWork is the shared queue consumed by chapter worker goroutines.
+	type chapterJob struct {
+		slug string
+		ref  scraper.ChapterRef
+	}
+	chapterWork := make(chan chapterJob, o.workers*4)
+
+	// Start chapter worker pool.
+	var chapterWG sync.WaitGroup
+	for i := 0; i < o.workers; i++ {
+		chapterWG.Add(1)
+		go func(workerID int) {
+			defer chapterWG.Done()
+			for job := range chapterWork {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+
+				// Skip if already on disk.
+				if o.writer.ChapterExists(job.slug, job.ref) {
+					o.log.Debug("chapter already exists, skipping",
+						"book", job.slug, "chapter", job.ref.Number)
+					continue
+				}
+
+				chapter, err := o.novel.ScrapeChapterText(ctx, job.ref)
+				if err != nil {
+					o.log.Error("chapter scrape failed",
+						"book", job.slug,
+						"chapter", job.ref.Number,
+						"url", job.ref.URL,
+						"err", err,
+					)
+					continue
+				}
+
+				if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
+					o.log.Error("chapter write failed",
+						"book", job.slug,
+						"chapter", job.ref.Number,
+						"err", err,
+					)
+					continue
+				}
+
+				o.log.Info("chapter saved",
+					"book", job.slug,
+					"chapter", job.ref.Number,
+					"worker", workerID,
+				)
+			}
+		}(i)
+	}
+
+	// processBook scrapes metadata + chapter list for one book, then enqueues
+	// chapter jobs. It is called inside a goroutine per book.
+	processBook := func(bookURL string) {
+		// Metadata goroutine.
+		meta, err := o.novel.ScrapeMetadata(ctx, bookURL)
+		if err != nil {
+			o.log.Error("metadata scrape failed", "url", bookURL, "err", err)
+			return
+		}
+
+		// Persist / update metadata.yaml.
+		if err := o.writer.WriteMetadata(meta); err != nil {
+			o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
+			// Continue — chapters can still be scraped.
+		}
+
+		o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
+
+		// Fetch chapter list.
+		refs, err := o.novel.ScrapeChapterList(ctx, bookURL)
+		if err != nil {
+			o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
+			return
+		}
+
+		o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
+
+		// Enqueue chapter jobs.
+		for _, ref := range refs {
+			select {
+			case <-ctx.Done():
+				return
+			case chapterWork <- chapterJob{slug: meta.Slug, ref: ref}:
+			}
+		}
+	}
+
+	if o.cfg.SingleBookURL != "" {
+		// Single-book mode: skip catalogue entirely.
+		o.log.Info("single-book mode", "url", o.cfg.SingleBookURL)
+		processBook(o.cfg.SingleBookURL)
+	} else {
+		// Catalogue mode: stream every book.
+		entries, catErrs := o.novel.ScrapeCatalogue(ctx)
+
+		// Drain catalogue errors in a separate goroutine.
+		go func() {
+			for err := range catErrs {
+				o.log.Error("catalogue error", "err", err)
+			}
+		}()
+
+		var bookWG sync.WaitGroup
+		for entry := range entries {
+			select {
+			case <-ctx.Done():
+				break
+			default:
+			}
+
+			bookWG.Add(1)
+			bookURL := entry.URL
+			go func() {
+				defer bookWG.Done()
+				processBook(bookURL)
+			}()
+		}
+
+		// Wait for all book goroutines to enqueue their chapters before
+		// closing the chapter work queue.
+		bookWG.Wait()
+	}
+
+	// Signal chapter workers there is no more work.
+	close(chapterWork)
+
+	// Wait for all in-flight chapter scrapes to finish.
+	chapterWG.Wait()
+
+	if ctx.Err() != nil {
+		return fmt.Errorf("orchestrator: context cancelled: %w", ctx.Err())
+	}
+
+	o.log.Info("orchestrator finished")
+	return nil
+}

scraper/internal/scraper/htmlutil/htmlutil.go

@@ -0,0 +1,216 @@
+// Package htmlutil provides helper functions for parsing HTML with
+// golang.org/x/net/html and extracting values by Selector descriptors.
+package htmlutil
+
+import (
+	"strings"
+
+	"github.com/libnovel/scraper/internal/scraper"
+	"golang.org/x/net/html"
+)
+
+// ParseHTML parses raw HTML and returns the root node.
+func ParseHTML(raw string) (*html.Node, error) {
+	return html.Parse(strings.NewReader(raw))
+}
+
+// selectorMatches reports whether node n matches sel.
+func selectorMatches(n *html.Node, sel scraper.Selector) bool {
+	if n.Type != html.ElementNode {
+		return false
+	}
+	if sel.Tag != "" && n.Data != sel.Tag {
+		return false
+	}
+	if sel.ID != "" {
+		for _, a := range n.Attr {
+			if a.Key == "id" && a.Val == sel.ID {
+				goto checkClass
+			}
+		}
+		return false
+	}
+checkClass:
+	if sel.Class != "" {
+		for _, a := range n.Attr {
+			if a.Key == "class" {
+				for _, cls := range strings.Fields(a.Val) {
+					if cls == sel.Class {
+						goto matched
+					}
+				}
+			}
+		}
+		return false
+	}
+matched:
+	return true
+}
+
+// attrVal returns the value of attribute key from node n.
+func attrVal(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+	return ""
+}
+
+// textContent returns the concatenated text content of all descendant text nodes.
+func textContent(n *html.Node) string {
+	var sb strings.Builder
+	var walk func(*html.Node)
+	walk = func(cur *html.Node) {
+		if cur.Type == html.TextNode {
+			sb.WriteString(cur.Data)
+		}
+		for c := cur.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(n)
+	return strings.TrimSpace(sb.String())
+}
+
+// FindFirst returns the first node matching sel within root.
+func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
+	var found *html.Node
+	var walk func(*html.Node) bool
+	walk = func(n *html.Node) bool {
+		if selectorMatches(n, sel) {
+			found = n
+			return true
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			if walk(c) {
+				return true
+			}
+		}
+		return false
+	}
+	walk(root)
+	return found
+}
+
+// FindAll returns all nodes matching sel within root.
+func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
+	var results []*html.Node
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if selectorMatches(n, sel) {
+			results = append(results, n)
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(root)
+	return results
+}
+
+// ExtractText extracts a string value from node n using sel.
+// If sel.Attr is set the attribute value is returned; otherwise the inner text.
+func ExtractText(n *html.Node, sel scraper.Selector) string {
+	if sel.Attr != "" {
+		return attrVal(n, sel.Attr)
+	}
+	return textContent(n)
+}
+
+// ExtractFirst locates the first match in root and returns its text/attr value.
+func ExtractFirst(root *html.Node, sel scraper.Selector) string {
+	n := FindFirst(root, sel)
+	if n == nil {
+		return ""
+	}
+	return ExtractText(n, sel)
+}
+
+// ExtractAll locates all matches in root and returns their text/attr values.
+func ExtractAll(root *html.Node, sel scraper.Selector) []string {
+	nodes := FindAll(root, sel)
+	out := make([]string, 0, len(nodes))
+	for _, n := range nodes {
+		if v := ExtractText(n, sel); v != "" {
+			out = append(out, v)
+		}
+	}
+	return out
+}
+
+// InnerHTML returns the serialized inner HTML of node n.
+func InnerHTML(n *html.Node) string {
+	var sb strings.Builder
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		_ = html.Render(&sb, c)
+	}
+	return sb.String()
+}
+
+// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
+// representation suitable for chapter storage. Block elements become newlines;
+// inline elements are inlined.
+func NodeToMarkdown(n *html.Node) string {
+	var sb strings.Builder
+	nodeToMD(n, &sb)
+	return strings.TrimSpace(sb.String())
+}
+
+var blockElements = map[string]bool{
+	"p": true, "div": true, "br": true, "h1": true, "h2": true,
+	"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
+	"blockquote": true, "pre": true, "hr": true,
+}
+
+func nodeToMD(n *html.Node, sb *strings.Builder) {
+	switch n.Type {
+	case html.TextNode:
+		sb.WriteString(n.Data)
+	case html.ElementNode:
+		tag := n.Data
+		switch tag {
+		case "br":
+			sb.WriteString("\n")
+		case "hr":
+			sb.WriteString("\n---\n")
+		case "h1", "h2", "h3", "h4", "h5", "h6":
+			level := int(tag[1] - '0')
+			sb.WriteString("\n" + strings.Repeat("#", level) + " ")
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				nodeToMD(c, sb)
+			}
+			sb.WriteString("\n\n")
+			return
+		case "p", "div", "blockquote":
+			sb.WriteString("\n")
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				nodeToMD(c, sb)
+			}
+			sb.WriteString("\n")
+			return
+		case "em", "i":
+			sb.WriteString("*")
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				nodeToMD(c, sb)
+			}
+			sb.WriteString("*")
+			return
+		case "strong", "b":
+			sb.WriteString("**")
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				nodeToMD(c, sb)
+			}
+			sb.WriteString("**")
+			return
+		case "script", "style", "noscript":
+			return // drop
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			nodeToMD(c, sb)
+		}
+		if blockElements[tag] {
+			sb.WriteString("\n")
+		}
+	}
+}

scraper/internal/scraper/interfaces.go

@@ -0,0 +1,149 @@
+// Package scraper defines the core interfaces and domain types for the libnovel
+// scraping system. Each novel source implements these interfaces; the orchestrator
+// wires them together without knowing anything about the concrete provider.
+package scraper
+
+import "context"
+
+// ─── Domain types ────────────────────────────────────────────────────────────
+
+// BookMeta carries all bibliographic information about a novel.
+type BookMeta struct {
+	// Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world".
+	Slug string `yaml:"slug"`
+	// Title is the human-readable novel title.
+	Title string `yaml:"title"`
+	// Author of the novel.
+	Author string `yaml:"author"`
+	// Cover is an absolute URL to the cover image.
+	Cover string `yaml:"cover,omitempty"`
+	// Status is e.g. "Ongoing", "Completed".
+	Status string `yaml:"status,omitempty"`
+	// Genres is a list of genre tags.
+	Genres []string `yaml:"genres,omitempty"`
+	// Summary is the full description/synopsis text.
+	Summary string `yaml:"summary,omitempty"`
+	// TotalChapters is the total number of chapters known at scrape time.
+	TotalChapters int `yaml:"total_chapters,omitempty"`
+	// SourceURL is the canonical URL of the book's landing page.
+	SourceURL string `yaml:"source_url"`
+}
+
+// CatalogueEntry is a lightweight reference returned by CatalogueProvider.
+type CatalogueEntry struct {
+	// Title is the novel title as shown in the catalogue listing.
+	Title string
+	// URL is the canonical landing-page URL of the novel.
+	URL string
+}
+
+// ChapterRef is a reference to a single chapter returned by ChapterListProvider.
+type ChapterRef struct {
+	// Number is the 1-based chapter index within the book.
+	Number int
+	// Title is the chapter display title.
+	Title string
+	// URL is the full URL of the chapter page.
+	URL string
+	// Volume is an optional volume number (0 means no volume grouping).
+	Volume int
+}
+
+// Chapter contains the fully-extracted text of a single chapter.
+type Chapter struct {
+	Ref ChapterRef
+	// Text is the plain / lightly-formatted chapter body (Markdown).
+	Text string
+}
+
+// ─── Scraping selector descriptors ───────────────────────────────────────────
+
+// Selector describes how to locate an element in an HTML document.
+// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set
+// they are combined (AND semantics).
+type Selector struct {
+	// Tag is the HTML element name, e.g. "div", "p", "h1".
+	Tag string
+	// Class is one CSS class name (without the leading dot).
+	Class string
+	// ID is the element id attribute (without the leading #).
+	ID string
+	// Attr is an optional attribute name whose value should be extracted
+	// instead of the text content (e.g. "href", "src").
+	Attr string
+	// Multiple indicates that all matching elements should be collected,
+	// not just the first one.
+	Multiple bool
+}
+
+// ─── Provider interfaces ──────────────────────────────────────────────────────
+
+// CatalogueProvider can enumerate every novel available on a source site.
+// It handles pagination transparently and streams CatalogueEntry values.
+type CatalogueProvider interface {
+	// CatalogueURL returns the root URL of the catalogue listing.
+	CatalogueURL() string
+
+	// EntriesSelector returns the selector that matches each novel card / row
+	// in the catalogue listing page.
+	EntriesSelector() Selector
+
+	// NextPageSelector returns the selector for the "next page" link.
+	// If the current page has no next page the implementation must return
+	// ("", nil) from ScrapeNextPage.
+	NextPageSelector() Selector
+
+	// ScrapeCatalogue pages through the entire catalogue, sending
+	// CatalogueEntry values to the returned channel. The channel is closed
+	// when all pages have been scraped or ctx is cancelled.
+	// Errors are surfaced via the error channel; a non-nil error does not
+	// necessarily terminate scraping.
+	ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error)
+}
+
+// MetadataProvider can extract structured book metadata from a novel's landing page.
+type MetadataProvider interface {
+	// MetadataSelectors returns a map of field name → Selector used to
+	// locate each metadata element on the book page.
+	// Required keys: "title", "author".
+	// Optional keys: "cover", "status", "genres", "summary", "total_chapters".
+	MetadataSelectors() map[string]Selector
+
+	// ScrapeMetadata fetches and parses the metadata for the book at bookURL.
+	ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error)
+}
+
+// ChapterListProvider can enumerate all chapters of a book from the chapter-list page.
+type ChapterListProvider interface {
+	// ChaptersURL derives the chapter-list URL from a book landing-page URL.
+	ChaptersURL(bookURL string) string
+
+	// ChapterEntrySelector returns the selector that matches each chapter row
+	// in the chapter list page.
+	ChapterEntrySelector() Selector
+
+	// ScrapeChapterList returns all chapter references for a book, ordered
+	// by chapter number ascending.
+	ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error)
+}
+
+// ChapterTextProvider can extract the readable text from a single chapter page.
+type ChapterTextProvider interface {
+	// ChapterTextSelector returns the selector that wraps the chapter body.
+	ChapterTextSelector() Selector
+
+	// ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown.
+	ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error)
+}
+
+// NovelScraper is the full interface that a concrete novel source must implement.
+// It composes all four provider interfaces.
+type NovelScraper interface {
+	CatalogueProvider
+	MetadataProvider
+	ChapterListProvider
+	ChapterTextProvider
+
+	// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
+	SourceName() string
+}

scraper/internal/server/server.go

@@ -0,0 +1,132 @@
+// Package server exposes the scraper as an HTTP service.
+//
+// Endpoints:
+//
+//	POST /scrape          — enqueue a full catalogue scrape
+//	POST /scrape/book     — enqueue a single-book scrape (JSON body: {"url":"..."})
+//	GET  /health          — liveness probe
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/libnovel/scraper/internal/orchestrator"
+	"github.com/libnovel/scraper/internal/scraper"
+)
+
+// Server wraps an HTTP mux with the scraping endpoints.
+type Server struct {
+	addr    string
+	oCfg    orchestrator.Config
+	novel   scraper.NovelScraper
+	log     *slog.Logger
+	mu      sync.Mutex
+	running bool
+}
+
+// New creates a new Server.
+func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger) *Server {
+	return &Server{
+		addr:  addr,
+		oCfg:  oCfg,
+		novel: novel,
+		log:   log,
+	}
+}
+
+// ListenAndServe starts the HTTP server and blocks until the provided context
+// is cancelled.
+func (s *Server) ListenAndServe(ctx context.Context) error {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /health", s.handleHealth)
+	mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
+	mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
+
+	srv := &http.Server{
+		Addr:         s.addr,
+		Handler:      mux,
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 15 * time.Second,
+		IdleTimeout:  60 * time.Second,
+	}
+
+	errCh := make(chan error, 1)
+	go func() { errCh <- srv.ListenAndServe() }()
+
+	s.log.Info("HTTP server listening", "addr", s.addr)
+
+	select {
+	case <-ctx.Done():
+		shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		return srv.Shutdown(shutCtx)
+	case err := <-errCh:
+		return err
+	}
+}
+
+func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
+}
+
+func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
+	cfg := s.oCfg
+	cfg.SingleBookURL = "" // full catalogue
+
+	s.runAsync(w, cfg)
+}
+
+func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
+	var body struct {
+		URL string `json:"url"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
+		http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
+		return
+	}
+
+	cfg := s.oCfg
+	cfg.SingleBookURL = body.URL
+
+	s.runAsync(w, cfg)
+}
+
+// runAsync launches an orchestrator in the background and returns 202 Accepted.
+// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
+func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
+	s.mu.Lock()
+	if s.running {
+		s.mu.Unlock()
+		http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
+		return
+	}
+	s.running = true
+	s.mu.Unlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
+
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			s.running = false
+			s.mu.Unlock()
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
+		defer cancel()
+
+		o := orchestrator.New(cfg, s.novel, s.log)
+		if err := o.Run(ctx); err != nil {
+			s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
+		}
+	}()
+}

scraper/internal/writer/writer.go

@@ -0,0 +1,140 @@
+// Package writer handles persistence of scraped chapters and metadata.
+//
+// Directory layout:
+//
+//	static/books/
+//	├── {book-slug}/
+//	│   ├── metadata.yaml
+//	│   ├── vol-0/          (no volume grouping)
+//	│   │   ├── 1-50/
+//	│   │   │   ├── chapter-1.md
+//	│   │   │   └── …
+//	│   │   └── 51-100/
+//	│   │       └── …
+//	│   └── vol-1/
+//	│       └── …
+package writer
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/libnovel/scraper/internal/scraper"
+	"gopkg.in/yaml.v3"
+)
+
+const chaptersPerFolder = 50
+
+// Writer persists scraped content under a configurable root directory.
+type Writer struct {
+	root string // e.g. "./static/books"
+}
+
+// New creates a Writer that stores files under root.
+func New(root string) *Writer {
+	return &Writer{root: root}
+}
+
+// ─── Metadata ─────────────────────────────────────────────────────────────────
+
+// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml.
+// It creates the directory if it does not exist and overwrites any existing file.
+func (w *Writer) WriteMetadata(meta scraper.BookMeta) error {
+	dir := w.bookDir(meta.Slug)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("writer: mkdir %s: %w", dir, err)
+	}
+
+	path := filepath.Join(dir, "metadata.yaml")
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("writer: create metadata %s: %w", path, err)
+	}
+	defer f.Close()
+
+	enc := yaml.NewEncoder(f)
+	enc.SetIndent(2)
+	if err := enc.Encode(meta); err != nil {
+		return fmt.Errorf("writer: encode metadata: %w", err)
+	}
+	return enc.Close()
+}
+
+// ReadMetadata reads the metadata.yaml for slug if it exists.
+// Returns (zero-value, false, nil) when the file does not exist.
+func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) {
+	path := filepath.Join(w.bookDir(slug), "metadata.yaml")
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return scraper.BookMeta{}, false, nil
+		}
+		return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err)
+	}
+
+	var meta scraper.BookMeta
+	if err := yaml.Unmarshal(data, &meta); err != nil {
+		return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err)
+	}
+	return meta, true, nil
+}
+
+// ─── Chapters ─────────────────────────────────────────────────────────────────
+
+// ChapterExists returns true if the markdown file for ref already exists on disk.
+func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool {
+	_, err := os.Stat(w.chapterPath(slug, ref))
+	return err == nil
+}
+
+// WriteChapter writes chapter.Text to the appropriate markdown file.
+// The parent directories are created on demand.
+func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error {
+	path := w.chapterPath(slug, chapter.Ref)
+	dir := filepath.Dir(path)
+
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("writer: mkdir %s: %w", dir, err)
+	}
+
+	// Build the markdown document.
+	var sb strings.Builder
+	sb.WriteString("# ")
+	sb.WriteString(chapter.Ref.Title)
+	sb.WriteString("\n\n")
+	sb.WriteString(chapter.Text)
+	sb.WriteString("\n")
+
+	if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
+		return fmt.Errorf("writer: write chapter %s: %w", path, err)
+	}
+	return nil
+}
+
+// ─── Path helpers ─────────────────────────────────────────────────────────────
+
+// bookDir returns the root directory for a book slug.
+func (w *Writer) bookDir(slug string) string {
+	return filepath.Join(w.root, slug)
+}
+
+// chapterPath computes the full file path for a chapter.
+//
+//	vol-{volume}/{folderRange}/chapter-{number}.md
+//
+// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md
+func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string {
+	vol := ref.Volume // 0 == no volume grouping
+	volDir := fmt.Sprintf("vol-%d", vol)
+
+	// Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", …
+	lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1
+	hi := lo + chaptersPerFolder - 1
+	rangeDir := fmt.Sprintf("%d-%d", lo, hi)
+
+	filename := fmt.Sprintf("chapter-%d.md", ref.Number)
+
+	return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename)
+}