commit d68ea712390ee858e329c2fc54720df5798f3dcf Author: Admin Date: Thu Feb 26 12:56:25 2026 +0500 feat: initial commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2120f62 --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# libnovel scraper — environment overrides +# Copy to .env and adjust values; do NOT commit this file with real secrets. + +# Browserless API token (leave empty to disable auth) +BROWSERLESS_TOKEN= + +# Number of concurrent browser sessions in Browserless +BROWSERLESS_CONCURRENT=10 + +# Queue depth before Browserless returns 429 +BROWSERLESS_QUEUED=100 + +# Per-session timeout in ms +BROWSERLESS_TIMEOUT=60000 + +# Which Browserless strategy the scraper uses: content | scrape | cdp +BROWSERLESS_STRATEGY=content + +# Chapter worker goroutines (0 = NumCPU inside the container) +SCRAPER_WORKERS=0 + +# Host path to mount as the static output directory +STATIC_ROOT=./static/books diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0f549db --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +version: "3.9" + +services: + # ─── Browserless ──────────────────────────────────────────────────────────── + browserless: + image: ghcr.io/browserless/chromium:latest + container_name: libnovel-browserless + restart: unless-stopped + environment: + # Set a token to lock down the endpoint; the scraper reads it via + # BROWSERLESS_TOKEN below. + TOKEN: "${BROWSERLESS_TOKEN:-}" + # Allow up to 10 concurrent browser sessions. + CONCURRENT: "${BROWSERLESS_CONCURRENT:-10}" + # Queue up to 100 requests before returning 429. + QUEUED: "${BROWSERLESS_QUEUED:-100}" + # Per-session timeout in ms. + TIMEOUT: "${BROWSERLESS_TIMEOUT:-60000}" + ports: + - "3000:3000" + # Shared memory is required for Chrome. + shm_size: "2gb" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/json/version"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── Scraper ───────────────────────────────────────────────────────────────── + scraper: + build: + context: ./scraper + dockerfile: Dockerfile + container_name: libnovel-scraper + restart: unless-stopped + depends_on: + browserless: + condition: service_healthy + environment: + BROWSERLESS_URL: "http://browserless:3000" + BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}" + # content | scrape | cdp — swap to test different strategies. + BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-content}" + # 0 → defaults to NumCPU inside the container. + SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}" + SCRAPER_STATIC_ROOT: "/app/static/books" + SCRAPER_HTTP_ADDR: ":8080" + ports: + - "8080:8080" + volumes: + # Mount the host static directory so scraped content is available outside + # the container. Create ./static/books on the host first if needed. + - "${STATIC_ROOT:-./static/books}:/app/static/books" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8080/health"] + interval: 15s + timeout: 5s + retries: 3 + +volumes: {} diff --git a/scraper/Dockerfile b/scraper/Dockerfile new file mode 100644 index 0000000..03aca88 --- /dev/null +++ b/scraper/Dockerfile @@ -0,0 +1,44 @@ +# ── Build stage ──────────────────────────────────────────────────────────────── +FROM golang:1.25-alpine AS builder + +WORKDIR /build + +# Cache dependency downloads separately from source compilation. +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -ldflags="-s -w" -o /scraper ./cmd/scraper + +# ── Runtime stage ────────────────────────────────────────────────────────────── +FROM alpine:3.20 + +# ca-certificates is required for HTTPS requests to novelfire.net. +RUN apk add --no-cache ca-certificates tzdata + +WORKDIR /app + +COPY --from=builder /scraper /app/scraper + +# Create the default static output directory. +RUN mkdir -p /app/static/books + +# Non-root user. +RUN addgroup -S scraper && adduser -S scraper -G scraper +RUN chown -R scraper:scraper /app +USER scraper + +# ── Configuration ───────────────────────────────────────────────────────────── +ENV BROWSERLESS_URL=http://browserless:3000 +ENV BROWSERLESS_STRATEGY=content +ENV SCRAPER_WORKERS=0 +ENV SCRAPER_STATIC_ROOT=/app/static/books +ENV SCRAPER_HTTP_ADDR=:8080 + +EXPOSE 8080 + +# Default: run as an HTTP server. Override CMD to use "run" for one-shot. +ENTRYPOINT ["/app/scraper"] +CMD ["serve"] diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go new file mode 100644 index 0000000..b9b41ce --- /dev/null +++ b/scraper/cmd/scraper/main.go @@ -0,0 +1,150 @@ +// Command scraper is the entrypoint for the libnovel scraper service. +// +// Usage (CLI one-shot): +// +// scraper run [--url ] +// +// Usage (HTTP server): +// +// scraper serve +// +// Environment variables: +// +// BROWSERLESS_URL Browserless base URL (default: http://localhost:3000) +// BROWSERLESS_TOKEN Browserless API token (default: "") +// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) +// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) +// SCRAPER_STATIC_ROOT Output directory (default: ./static/books) +// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "runtime" + "strconv" + "strings" + "syscall" + + "github.com/libnovel/scraper/internal/browser" + "github.com/libnovel/scraper/internal/novelfire" + "github.com/libnovel/scraper/internal/orchestrator" + "github.com/libnovel/scraper/internal/server" +) + +func main() { + log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + + if err := run(log); err != nil { + log.Error("fatal", "err", err) + os.Exit(1) + } +} + +func run(log *slog.Logger) error { + args := os.Args[1:] + if len(args) == 0 { + printUsage() + return nil + } + + cmd := strings.ToLower(args[0]) + + browserCfg := browser.Config{ + BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3000"), + Token: envOr("BROWSERLESS_TOKEN", ""), + } + + strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyContent)))) + bc := newBrowserClient(strategy, browserCfg) + + nf := novelfire.New(bc, log) + + workers := 0 + if s := os.Getenv("SCRAPER_WORKERS"); s != "" { + n, err := strconv.Atoi(s) + if err == nil && n > 0 { + workers = n + } + } + if workers == 0 { + workers = runtime.NumCPU() + } + + oCfg := orchestrator.Config{ + Workers: workers, + StaticRoot: envOr("SCRAPER_STATIC_ROOT", "./static/books"), + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + switch cmd { + case "run": + // Optional --url flag. + if len(args) >= 3 && args[1] == "--url" { + oCfg.SingleBookURL = args[2] + } + log.Info("starting one-shot scrape", + "strategy", strategy, + "workers", workers, + "static_root", oCfg.StaticRoot, + "single_book", oCfg.SingleBookURL, + ) + o := orchestrator.New(oCfg, nf, log) + return o.Run(ctx) + + case "serve": + addr := envOr("SCRAPER_HTTP_ADDR", ":8080") + log.Info("starting HTTP server", + "addr", addr, + "strategy", strategy, + "workers", workers, + ) + srv := server.New(addr, oCfg, nf, log) + return srv.ListenAndServe(ctx) + + default: + return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd) + } +} + +func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient { + switch strategy { + case browser.StrategyScrape: + return browser.NewScrapeClient(cfg) + case browser.StrategyCDP: + return browser.NewCDPClient(cfg) + default: + return browser.NewContentClient(cfg) + } +} + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func printUsage() { + fmt.Fprintf(os.Stderr, `libnovel scraper + +Commands: + run [--url ] One-shot: scrape full catalogue, or a single book + serve Start HTTP server (POST /scrape, POST /scrape/book) + +Environment variables: + BROWSERLESS_URL Browserless base URL (default: http://localhost:3000) + BROWSERLESS_TOKEN API token (default: "") + BROWSERLESS_STRATEGY content | scrape | cdp (default: content) + SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d) + SCRAPER_STATIC_ROOT Output directory (default: ./static/books) + SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) +`, runtime.NumCPU()) +} diff --git a/scraper/go.mod b/scraper/go.mod new file mode 100644 index 0000000..d181ba5 --- /dev/null +++ b/scraper/go.mod @@ -0,0 +1,9 @@ +module github.com/libnovel/scraper + +go 1.25.0 + +require ( + github.com/gorilla/websocket v1.5.3 // indirect + golang.org/x/net v0.51.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/scraper/go.sum b/scraper/go.sum new file mode 100644 index 0000000..f347a29 --- /dev/null +++ b/scraper/go.sum @@ -0,0 +1,7 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scraper/internal/browser/cdp.go b/scraper/internal/browser/cdp.go new file mode 100644 index 0000000..e9eb04f --- /dev/null +++ b/scraper/internal/browser/cdp.go @@ -0,0 +1,131 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" +) + +// cdpClient implements BrowserClient using the CDP WebSocket endpoint. +type cdpClient struct { + cfg Config +} + +// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions. +func NewCDPClient(cfg Config) BrowserClient { + if cfg.Timeout == 0 { + cfg.Timeout = 60 * time.Second + } + return &cdpClient{cfg: cfg} +} + +func (c *cdpClient) Strategy() Strategy { return StrategyCDP } + +func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) { + return "", fmt.Errorf("CDP client does not support /content; use NewContentClient") +} + +func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) { + return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient") +} + +// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint, +// navigates to pageURL, and invokes fn with a live CDPConn. +func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error { + // Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=... + wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1) + wsURL = strings.Replace(wsURL, "https://", "wss://", 1) + wsURL += "/devtools/browser" + sep := "?" + if c.cfg.Token != "" { + wsURL += sep + "token=" + c.cfg.Token + sep = "&" + } + wsURL += sep + "url=" + pageURL + + dialer := websocket.Dialer{ + HandshakeTimeout: 15 * time.Second, + Proxy: http.ProxyFromEnvironment, + } + + conn, _, err := dialer.DialContext(ctx, wsURL, nil) + if err != nil { + return fmt.Errorf("cdp: dial %s: %w", wsURL, err) + } + + cdp := &cdpConn{ws: conn} + defer cdp.Close() + + return fn(ctx, cdp) +} + +// ─── cdpConn ───────────────────────────────────────────────────────────────── + +type cdpConn struct { + ws *websocket.Conn + counter atomic.Int64 +} + +type cdpRequest struct { + ID int64 `json:"id"` + Method string `json:"method"` + Params map[string]any `json:"params,omitempty"` +} + +type cdpResponse struct { + ID int64 `json:"id"` + Result map[string]any `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) { + id := c.counter.Add(1) + + req := cdpRequest{ID: id, Method: method, Params: params} + data, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("cdp send: marshal: %w", err) + } + + if dl, ok := ctx.Deadline(); ok { + _ = c.ws.SetWriteDeadline(dl) + } + if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil { + return nil, fmt.Errorf("cdp send: write: %w", err) + } + + // Read messages until we find the response matching our id. + for { + if dl, ok := ctx.Deadline(); ok { + _ = c.ws.SetReadDeadline(dl) + } + _, msg, err := c.ws.ReadMessage() + if err != nil { + return nil, fmt.Errorf("cdp send: read: %w", err) + } + var resp cdpResponse + if err := json.Unmarshal(msg, &resp); err != nil { + continue // skip non-JSON frames (events etc.) + } + if resp.ID != id { + continue // event or different command reply + } + if resp.Error != nil { + return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message) + } + return resp.Result, nil + } +} + +func (c *cdpConn) Close() error { + return c.ws.Close() +} diff --git a/scraper/internal/browser/content_scrape.go b/scraper/internal/browser/content_scrape.go new file mode 100644 index 0000000..84d478a --- /dev/null +++ b/scraper/internal/browser/content_scrape.go @@ -0,0 +1,146 @@ +package browser + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// Config holds the connection parameters for a Browserless instance. +type Config struct { + // BaseURL is the HTTP base URL, e.g. "http://localhost:3000". + BaseURL string + // Token is the optional API token (BROWSERLESS_TOKEN env var). + Token string + // Timeout is the per-request HTTP timeout; defaults to 60 s. + Timeout time.Duration +} + +// contentClient implements BrowserClient using the /content endpoint. +type contentClient struct { + cfg Config + http *http.Client +} + +// NewContentClient returns a BrowserClient that uses POST /content. +func NewContentClient(cfg Config) BrowserClient { + if cfg.Timeout == 0 { + cfg.Timeout = 60 * time.Second + } + return &contentClient{ + cfg: cfg, + http: &http.Client{Timeout: cfg.Timeout}, + } +} + +func (c *contentClient) Strategy() Strategy { return StrategyContent } + +func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) { + body, err := json.Marshal(req) + if err != nil { + return "", fmt.Errorf("content: marshal request: %w", err) + } + + url := c.cfg.BaseURL + "/content" + if c.cfg.Token != "" { + url += "?token=" + c.cfg.Token + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("content: build request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(httpReq) + if err != nil { + return "", fmt.Errorf("content: do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b) + } + + raw, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("content: read body: %w", err) + } + return string(raw), nil +} + +func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) { + return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient") +} + +func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error { + return fmt.Errorf("content client does not support CDP; use NewCDPClient") +} + +// ─── /scrape client ─────────────────────────────────────────────────────────── + +type scrapeClient struct { + cfg Config + http *http.Client +} + +// NewScrapeClient returns a BrowserClient that uses POST /scrape. +func NewScrapeClient(cfg Config) BrowserClient { + if cfg.Timeout == 0 { + cfg.Timeout = 60 * time.Second + } + return &scrapeClient{ + cfg: cfg, + http: &http.Client{Timeout: cfg.Timeout}, + } +} + +func (c *scrapeClient) Strategy() Strategy { return StrategyScrape } + +func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) { + return "", fmt.Errorf("scrape client does not support /content; use NewContentClient") +} + +func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) { + body, err := json.Marshal(req) + if err != nil { + return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err) + } + + url := c.cfg.BaseURL + "/scrape" + if c.cfg.Token != "" { + url += "?token=" + c.cfg.Token + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(httpReq) + if err != nil { + return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b) + } + + var result ScrapeResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err) + } + return result, nil +} + +func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error { + return fmt.Errorf("scrape client does not support CDP; use NewCDPClient") +} diff --git a/scraper/internal/browser/interface.go b/scraper/internal/browser/interface.go new file mode 100644 index 0000000..6044e9f --- /dev/null +++ b/scraper/internal/browser/interface.go @@ -0,0 +1,101 @@ +// Package browser defines the BrowserClient interface and helper types for +// communicating with a Browserless instance. +package browser + +import "context" + +// Strategy selects which Browserless API endpoint / protocol to use. +type Strategy string + +const ( + // StrategyContent uses the POST /content endpoint, which returns the final + // rendered HTML of the page. Fastest; suitable for most JS-rendered sites. + StrategyContent Strategy = "content" + + // StrategyScrape uses the POST /scrape endpoint, which accepts a list of + // CSS selectors and returns structured JSON. Good when you know exactly + // which elements you need. + StrategyScrape Strategy = "scrape" + + // StrategyCDP uses the WebSocket /devtools/browser endpoint (Chrome + // DevTools Protocol). Most powerful; required for complex interactions + // (clicking, scrolling, waiting for network idle, etc.). + StrategyCDP Strategy = "cdp" +) + +// ContentRequest is the body sent to POST /content. +type ContentRequest struct { + URL string `json:"url"` + WaitFor string `json:"waitForSelector,omitempty"` + WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms + RejectResources bool `json:"rejectResources,omitempty"` +} + +// ScrapeElement is one element descriptor inside a ScrapeRequest. +type ScrapeElement struct { + Selector string `json:"selector"` + Timeout int `json:"timeout,omitempty"` // ms +} + +// ScrapeRequest is the body sent to POST /scrape. +type ScrapeRequest struct { + URL string `json:"url"` + Elements []ScrapeElement `json:"elements"` + WaitFor string `json:"waitForSelector,omitempty"` +} + +// ScrapeResult is one entry in the response from POST /scrape. +type ScrapeResult struct { + Selector string `json:"selector"` + Results []ScrapeElement `json:"results"` +} + +// ScrapeAttribute holds a single attribute value from a scraped element. +type ScrapeAttribute struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// ScrapedElement is one item inside ScrapeResult.Results. +type ScrapedElement struct { + Text string `json:"text"` + Attributes []ScrapeAttribute `json:"attributes"` +} + +// ScrapeResponse is the top-level response from POST /scrape. +type ScrapeResponse struct { + Data []ScrapeResult `json:"data"` +} + +// BrowserClient is an abstraction over the three Browserless API strategies. +// Callers choose the strategy best suited to the target site; the interface +// signature is identical regardless of strategy. +type BrowserClient interface { + // Strategy returns the strategy this client uses. + Strategy() Strategy + + // GetContent fetches the fully-rendered HTML of url using the /content + // endpoint. Only meaningful when Strategy() == StrategyContent. + GetContent(ctx context.Context, req ContentRequest) (string, error) + + // ScrapePage calls the /scrape endpoint and returns structured data. + // Only meaningful when Strategy() == StrategyScrape. + ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) + + // CDPSession opens a CDP WebSocket session and calls fn with the raw + // WebSocket connection. Only meaningful when Strategy() == StrategyCDP. + // The session is closed when fn returns. + CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error +} + +// CDPSessionFunc is the callback invoked inside a CDP session. +// conn is a live *websocket.Conn connected to a Browserless CDP endpoint. +type CDPSessionFunc func(ctx context.Context, conn CDPConn) error + +// CDPConn is the minimal interface the orchestrator needs over a CDP WebSocket. +type CDPConn interface { + // Send sends a raw CDP command (JSON-encoded) and returns the response. + Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) + // Close closes the underlying connection. + Close() error +} diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go new file mode 100644 index 0000000..09a1d95 --- /dev/null +++ b/scraper/internal/novelfire/scraper.go @@ -0,0 +1,372 @@ +// Package novelfire provides a NovelScraper implementation for novelfire.net. +// +// Site structure (as of 2025): +// +// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N +// Book page : https://novelfire.net/book/{slug} +// Chapters : https://novelfire.net/book/{slug}/chapters?page=N +// Chapter : https://novelfire.net/book/{slug}/{chapter-slug} +package novelfire + +import ( + "context" + "fmt" + "log/slog" + "net/url" + "strconv" + "strings" + + "github.com/libnovel/scraper/internal/browser" + "github.com/libnovel/scraper/internal/scraper" + "github.com/libnovel/scraper/internal/scraper/htmlutil" +) + +const ( + baseURL = "https://novelfire.net" + cataloguePath = "/genre-all/sort-new/status-all/all-novel" +) + +// Scraper is the novelfire.net implementation of scraper.NovelScraper. +// It uses the /content strategy by default (rendered HTML via Browserless). +type Scraper struct { + client browser.BrowserClient + log *slog.Logger +} + +// New returns a new novelfire Scraper. +func New(client browser.BrowserClient, log *slog.Logger) *Scraper { + if log == nil { + log = slog.Default() + } + return &Scraper{client: client, log: log} +} + +// SourceName implements NovelScraper. +func (s *Scraper) SourceName() string { return "novelfire.net" } + +// ─── CatalogueProvider ─────────────────────────────────────────────────────── + +func (s *Scraper) CatalogueURL() string { + return baseURL + cataloguePath +} + +func (s *Scraper) EntriesSelector() scraper.Selector { + // Each novel card:
+ return scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true} +} + +func (s *Scraper) NextPageSelector() scraper.Selector { + //

Title + titleSel := scraper.Selector{Tag: "h3", Class: "novel-title"} + titleNode := htmlutil.FindFirst(card, titleSel) + + var title, href string + if titleNode != nil { + linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"}) + if linkNode != nil { + title = htmlutil.ExtractText(linkNode, scraper.Selector{}) + href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + } + } + if href == "" || title == "" { + continue + } + + // Resolve relative URL. + bookURL := resolveURL(baseURL, href) + select { + case <-ctx.Done(): + return + case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}: + } + } + + // Find next page link. + nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector()) + if nextHref == "" { + break + } + pageURL = resolveURL(baseURL, nextHref) + page++ + } + }() + + return entries, errs +} + +// ─── MetadataProvider ──────────────────────────────────────────────────────── + +func (s *Scraper) MetadataSelectors() map[string]scraper.Selector { + return map[string]scraper.Selector{ + //

Title

+ "title": {Tag: "h1", Class: "novel-title"}, + // Author Name + "author": {Tag: "span", Class: "author"}, + // + "cover": {Tag: "img", Class: "cover", Attr: "src"}, + // Ongoing + "status": {Tag: "span", Class: "status"}, + // + "genres": {Tag: "div", Class: "genres", Multiple: true}, + //

...

+ "summary": {Tag: "div", Class: "summary"}, + // 123 Chapters + "total_chapters": {Tag: "span", Class: "chapter-count"}, + } +} + +func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.BookMeta, error) { + raw, err := s.client.GetContent(ctx, browser.ContentRequest{ + URL: bookURL, + WaitFor: ".novel-title", + WaitForTimeout: 10000, + RejectResources: true, + }) + if err != nil { + return scraper.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return scraper.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err) + } + + sels := s.MetadataSelectors() + + title := htmlutil.ExtractFirst(root, sels["title"]) + author := htmlutil.ExtractFirst(root, sels["author"]) + cover := htmlutil.ExtractFirst(root, sels["cover"]) + status := htmlutil.ExtractFirst(root, sels["status"]) + + // Genres: all tags inside the genres div. + genresNode := htmlutil.FindFirst(root, sels["genres"]) + var genres []string + if genresNode != nil { + genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true}) + } + + summary := htmlutil.ExtractFirst(root, sels["summary"]) + + totalStr := htmlutil.ExtractFirst(root, sels["total_chapters"]) + totalChapters := parseChapterCount(totalStr) + + // Derive slug from URL. + slug := slugFromURL(bookURL) + + return scraper.BookMeta{ + Slug: slug, + Title: title, + Author: author, + Cover: cover, + Status: status, + Genres: genres, + Summary: summary, + TotalChapters: totalChapters, + SourceURL: bookURL, + }, nil +} + +// ─── ChapterListProvider ───────────────────────────────────────────────────── + +func (s *Scraper) ChaptersURL(bookURL string) string { + return strings.TrimRight(bookURL, "/") + "/chapters" +} + +func (s *Scraper) ChapterEntrySelector() scraper.Selector { + //
  • Chapter 1: Title
  • + return scraper.Selector{Tag: "li", Class: "chapter-item", Multiple: true} +} + +func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) { + var refs []scraper.ChapterRef + pageURL := s.ChaptersURL(bookURL) + page := 1 + + for pageURL != "" { + select { + case <-ctx.Done(): + return refs, ctx.Err() + default: + } + + s.log.Info("scraping chapter list", "page", page, "url", pageURL) + + raw, err := s.client.GetContent(ctx, browser.ContentRequest{ + URL: pageURL, + WaitFor: ".chapter-item", + WaitForTimeout: 10000, + RejectResources: true, + }) + if err != nil { + return refs, fmt.Errorf("chapter list page %d: %w", page, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return refs, fmt.Errorf("chapter list page %d parse: %w", page, err) + } + + items := htmlutil.FindAll(root, s.ChapterEntrySelector()) + for _, item := range items { + linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a", Attr: "href"}) + if linkNode == nil { + continue + } + href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) + if href == "" { + continue + } + chURL := resolveURL(baseURL, href) + num := len(refs) + 1 + refs = append(refs, scraper.ChapterRef{ + Number: num, + Title: strings.TrimSpace(chTitle), + URL: chURL, + }) + } + + // Next page. + nextHref := htmlutil.ExtractFirst(root, s.NextPageSelector()) + if nextHref == "" { + break + } + pageURL = resolveURL(baseURL, nextHref) + page++ + } + + return refs, nil +} + +// ─── ChapterTextProvider ───────────────────────────────────────────────────── + +func (s *Scraper) ChapterTextSelector() scraper.Selector { + //
    or
    + return scraper.Selector{Tag: "div", ID: "chapter-container"} +} + +func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) { + raw, err := s.client.GetContent(ctx, browser.ContentRequest{ + URL: ref.URL, + WaitFor: "#chapter-container", + WaitForTimeout: 15000, + RejectResources: true, + }) + if err != nil { + return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return scraper.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err) + } + + container := htmlutil.FindFirst(root, s.ChapterTextSelector()) + if container == nil { + // Fallback: try class-based selector. + container = htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "chapter-content"}) + } + if container == nil { + return scraper.Chapter{}, fmt.Errorf("chapter %d: content container not found in %s", ref.Number, ref.URL) + } + + text := htmlutil.NodeToMarkdown(container) + + return scraper.Chapter{ + Ref: ref, + Text: text, + }, nil +} + +// ─── helpers ───────────────────────────────────────────────────────────────── + +func resolveURL(base, href string) string { + if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { + return href + } + b, err := url.Parse(base) + if err != nil { + return base + href + } + ref, err := url.Parse(href) + if err != nil { + return base + href + } + return b.ResolveReference(ref).String() +} + +func slugFromURL(bookURL string) string { + u, err := url.Parse(bookURL) + if err != nil { + return bookURL + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) >= 2 && parts[0] == "book" { + return parts[1] + } + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +func parseChapterCount(s string) int { + // Formats: "123 Chapters", "1,234 Chapters", "123" + s = strings.ReplaceAll(s, ",", "") + fields := strings.Fields(s) + if len(fields) == 0 { + return 0 + } + n, _ := strconv.Atoi(fields[0]) + return n +} diff --git a/scraper/internal/orchestrator/orchestrator.go b/scraper/internal/orchestrator/orchestrator.go new file mode 100644 index 0000000..d35a6c0 --- /dev/null +++ b/scraper/internal/orchestrator/orchestrator.go @@ -0,0 +1,213 @@ +// Package orchestrator coordinates the catalogue walk, metadata extraction, +// chapter-list fetching, and parallel chapter scraping. +// +// Concurrency model +// - One goroutine runs ScrapeCatalogue and feeds book URLs into a channel. +// - For each book, a dedicated goroutine calls ScrapeMetadata (metadata goroutine). +// - ScrapeChapterList is called in the metadata goroutine once metadata is done. +// - N worker goroutines (default: runtime.NumCPU()) each pull ChapterRef values +// from a shared work queue and call ScrapeChapterText. +// - A sync.WaitGroup ensures all chapter workers finish before the orchestrator +// signals completion. +package orchestrator + +import ( + "context" + "fmt" + "log/slog" + "runtime" + "sync" + + "github.com/libnovel/scraper/internal/scraper" + "github.com/libnovel/scraper/internal/writer" +) + +// Config holds tunable parameters for the orchestrator. +type Config struct { + // Workers is the number of goroutines used to scrape chapters in parallel. + // Defaults to runtime.NumCPU() when 0. + Workers int + + // StaticRoot is the path to the static/books output directory. + StaticRoot string + + // SingleBookURL when non-empty causes the orchestrator to scrape only + // that one book instead of walking the full catalogue. + SingleBookURL string +} + +// Orchestrator coordinates the full scrape pipeline. +type Orchestrator struct { + cfg Config + novel scraper.NovelScraper + writer *writer.Writer + log *slog.Logger + workers int +} + +// New returns a new Orchestrator. +func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator { + workers := cfg.Workers + if workers <= 0 { + workers = runtime.NumCPU() + } + return &Orchestrator{ + cfg: cfg, + novel: novel, + writer: writer.New(cfg.StaticRoot), + log: log, + workers: workers, + } +} + +// Run executes the full scrape pipeline and blocks until it is complete or ctx +// is cancelled. +func (o *Orchestrator) Run(ctx context.Context) error { + o.log.Info("orchestrator starting", + "source", o.novel.SourceName(), + "workers", o.workers, + "static_root", o.cfg.StaticRoot, + ) + + // chapterWork is the shared queue consumed by chapter worker goroutines. + type chapterJob struct { + slug string + ref scraper.ChapterRef + } + chapterWork := make(chan chapterJob, o.workers*4) + + // Start chapter worker pool. + var chapterWG sync.WaitGroup + for i := 0; i < o.workers; i++ { + chapterWG.Add(1) + go func(workerID int) { + defer chapterWG.Done() + for job := range chapterWork { + select { + case <-ctx.Done(): + return + default: + } + + // Skip if already on disk. + if o.writer.ChapterExists(job.slug, job.ref) { + o.log.Debug("chapter already exists, skipping", + "book", job.slug, "chapter", job.ref.Number) + continue + } + + chapter, err := o.novel.ScrapeChapterText(ctx, job.ref) + if err != nil { + o.log.Error("chapter scrape failed", + "book", job.slug, + "chapter", job.ref.Number, + "url", job.ref.URL, + "err", err, + ) + continue + } + + if err := o.writer.WriteChapter(job.slug, chapter); err != nil { + o.log.Error("chapter write failed", + "book", job.slug, + "chapter", job.ref.Number, + "err", err, + ) + continue + } + + o.log.Info("chapter saved", + "book", job.slug, + "chapter", job.ref.Number, + "worker", workerID, + ) + } + }(i) + } + + // processBook scrapes metadata + chapter list for one book, then enqueues + // chapter jobs. It is called inside a goroutine per book. + processBook := func(bookURL string) { + // Metadata goroutine. + meta, err := o.novel.ScrapeMetadata(ctx, bookURL) + if err != nil { + o.log.Error("metadata scrape failed", "url", bookURL, "err", err) + return + } + + // Persist / update metadata.yaml. + if err := o.writer.WriteMetadata(meta); err != nil { + o.log.Error("metadata write failed", "slug", meta.Slug, "err", err) + // Continue — chapters can still be scraped. + } + + o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title) + + // Fetch chapter list. + refs, err := o.novel.ScrapeChapterList(ctx, bookURL) + if err != nil { + o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err) + return + } + + o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs)) + + // Enqueue chapter jobs. + for _, ref := range refs { + select { + case <-ctx.Done(): + return + case chapterWork <- chapterJob{slug: meta.Slug, ref: ref}: + } + } + } + + if o.cfg.SingleBookURL != "" { + // Single-book mode: skip catalogue entirely. + o.log.Info("single-book mode", "url", o.cfg.SingleBookURL) + processBook(o.cfg.SingleBookURL) + } else { + // Catalogue mode: stream every book. + entries, catErrs := o.novel.ScrapeCatalogue(ctx) + + // Drain catalogue errors in a separate goroutine. + go func() { + for err := range catErrs { + o.log.Error("catalogue error", "err", err) + } + }() + + var bookWG sync.WaitGroup + for entry := range entries { + select { + case <-ctx.Done(): + break + default: + } + + bookWG.Add(1) + bookURL := entry.URL + go func() { + defer bookWG.Done() + processBook(bookURL) + }() + } + + // Wait for all book goroutines to enqueue their chapters before + // closing the chapter work queue. + bookWG.Wait() + } + + // Signal chapter workers there is no more work. + close(chapterWork) + + // Wait for all in-flight chapter scrapes to finish. + chapterWG.Wait() + + if ctx.Err() != nil { + return fmt.Errorf("orchestrator: context cancelled: %w", ctx.Err()) + } + + o.log.Info("orchestrator finished") + return nil +} diff --git a/scraper/internal/scraper/htmlutil/htmlutil.go b/scraper/internal/scraper/htmlutil/htmlutil.go new file mode 100644 index 0000000..d7cf1f5 --- /dev/null +++ b/scraper/internal/scraper/htmlutil/htmlutil.go @@ -0,0 +1,216 @@ +// Package htmlutil provides helper functions for parsing HTML with +// golang.org/x/net/html and extracting values by Selector descriptors. +package htmlutil + +import ( + "strings" + + "github.com/libnovel/scraper/internal/scraper" + "golang.org/x/net/html" +) + +// ParseHTML parses raw HTML and returns the root node. +func ParseHTML(raw string) (*html.Node, error) { + return html.Parse(strings.NewReader(raw)) +} + +// selectorMatches reports whether node n matches sel. +func selectorMatches(n *html.Node, sel scraper.Selector) bool { + if n.Type != html.ElementNode { + return false + } + if sel.Tag != "" && n.Data != sel.Tag { + return false + } + if sel.ID != "" { + for _, a := range n.Attr { + if a.Key == "id" && a.Val == sel.ID { + goto checkClass + } + } + return false + } +checkClass: + if sel.Class != "" { + for _, a := range n.Attr { + if a.Key == "class" { + for _, cls := range strings.Fields(a.Val) { + if cls == sel.Class { + goto matched + } + } + } + } + return false + } +matched: + return true +} + +// attrVal returns the value of attribute key from node n. +func attrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +// textContent returns the concatenated text content of all descendant text nodes. +func textContent(n *html.Node) string { + var sb strings.Builder + var walk func(*html.Node) + walk = func(cur *html.Node) { + if cur.Type == html.TextNode { + sb.WriteString(cur.Data) + } + for c := cur.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return strings.TrimSpace(sb.String()) +} + +// FindFirst returns the first node matching sel within root. +func FindFirst(root *html.Node, sel scraper.Selector) *html.Node { + var found *html.Node + var walk func(*html.Node) bool + walk = func(n *html.Node) bool { + if selectorMatches(n, sel) { + found = n + return true + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if walk(c) { + return true + } + } + return false + } + walk(root) + return found +} + +// FindAll returns all nodes matching sel within root. +func FindAll(root *html.Node, sel scraper.Selector) []*html.Node { + var results []*html.Node + var walk func(*html.Node) + walk = func(n *html.Node) { + if selectorMatches(n, sel) { + results = append(results, n) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(root) + return results +} + +// ExtractText extracts a string value from node n using sel. +// If sel.Attr is set the attribute value is returned; otherwise the inner text. +func ExtractText(n *html.Node, sel scraper.Selector) string { + if sel.Attr != "" { + return attrVal(n, sel.Attr) + } + return textContent(n) +} + +// ExtractFirst locates the first match in root and returns its text/attr value. +func ExtractFirst(root *html.Node, sel scraper.Selector) string { + n := FindFirst(root, sel) + if n == nil { + return "" + } + return ExtractText(n, sel) +} + +// ExtractAll locates all matches in root and returns their text/attr values. +func ExtractAll(root *html.Node, sel scraper.Selector) []string { + nodes := FindAll(root, sel) + out := make([]string, 0, len(nodes)) + for _, n := range nodes { + if v := ExtractText(n, sel); v != "" { + out = append(out, v) + } + } + return out +} + +// InnerHTML returns the serialized inner HTML of node n. +func InnerHTML(n *html.Node) string { + var sb strings.Builder + for c := n.FirstChild; c != nil; c = c.NextSibling { + _ = html.Render(&sb, c) + } + return sb.String() +} + +// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown +// representation suitable for chapter storage. Block elements become newlines; +// inline elements are inlined. +func NodeToMarkdown(n *html.Node) string { + var sb strings.Builder + nodeToMD(n, &sb) + return strings.TrimSpace(sb.String()) +} + +var blockElements = map[string]bool{ + "p": true, "div": true, "br": true, "h1": true, "h2": true, + "h3": true, "h4": true, "h5": true, "h6": true, "li": true, + "blockquote": true, "pre": true, "hr": true, +} + +func nodeToMD(n *html.Node, sb *strings.Builder) { + switch n.Type { + case html.TextNode: + sb.WriteString(n.Data) + case html.ElementNode: + tag := n.Data + switch tag { + case "br": + sb.WriteString("\n") + case "hr": + sb.WriteString("\n---\n") + case "h1", "h2", "h3", "h4", "h5", "h6": + level := int(tag[1] - '0') + sb.WriteString("\n" + strings.Repeat("#", level) + " ") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("\n\n") + return + case "p", "div", "blockquote": + sb.WriteString("\n") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("\n") + return + case "em", "i": + sb.WriteString("*") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("*") + return + case "strong", "b": + sb.WriteString("**") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("**") + return + case "script", "style", "noscript": + return // drop + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + if blockElements[tag] { + sb.WriteString("\n") + } + } +} diff --git a/scraper/internal/scraper/interfaces.go b/scraper/internal/scraper/interfaces.go new file mode 100644 index 0000000..7824a78 --- /dev/null +++ b/scraper/internal/scraper/interfaces.go @@ -0,0 +1,149 @@ +// Package scraper defines the core interfaces and domain types for the libnovel +// scraping system. Each novel source implements these interfaces; the orchestrator +// wires them together without knowing anything about the concrete provider. +package scraper + +import "context" + +// ─── Domain types ──────────────────────────────────────────────────────────── + +// BookMeta carries all bibliographic information about a novel. +type BookMeta struct { + // Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world". + Slug string `yaml:"slug"` + // Title is the human-readable novel title. + Title string `yaml:"title"` + // Author of the novel. + Author string `yaml:"author"` + // Cover is an absolute URL to the cover image. + Cover string `yaml:"cover,omitempty"` + // Status is e.g. "Ongoing", "Completed". + Status string `yaml:"status,omitempty"` + // Genres is a list of genre tags. + Genres []string `yaml:"genres,omitempty"` + // Summary is the full description/synopsis text. + Summary string `yaml:"summary,omitempty"` + // TotalChapters is the total number of chapters known at scrape time. + TotalChapters int `yaml:"total_chapters,omitempty"` + // SourceURL is the canonical URL of the book's landing page. + SourceURL string `yaml:"source_url"` +} + +// CatalogueEntry is a lightweight reference returned by CatalogueProvider. +type CatalogueEntry struct { + // Title is the novel title as shown in the catalogue listing. + Title string + // URL is the canonical landing-page URL of the novel. + URL string +} + +// ChapterRef is a reference to a single chapter returned by ChapterListProvider. +type ChapterRef struct { + // Number is the 1-based chapter index within the book. + Number int + // Title is the chapter display title. + Title string + // URL is the full URL of the chapter page. + URL string + // Volume is an optional volume number (0 means no volume grouping). + Volume int +} + +// Chapter contains the fully-extracted text of a single chapter. +type Chapter struct { + Ref ChapterRef + // Text is the plain / lightly-formatted chapter body (Markdown). + Text string +} + +// ─── Scraping selector descriptors ─────────────────────────────────────────── + +// Selector describes how to locate an element in an HTML document. +// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set +// they are combined (AND semantics). +type Selector struct { + // Tag is the HTML element name, e.g. "div", "p", "h1". + Tag string + // Class is one CSS class name (without the leading dot). + Class string + // ID is the element id attribute (without the leading #). + ID string + // Attr is an optional attribute name whose value should be extracted + // instead of the text content (e.g. "href", "src"). + Attr string + // Multiple indicates that all matching elements should be collected, + // not just the first one. + Multiple bool +} + +// ─── Provider interfaces ────────────────────────────────────────────────────── + +// CatalogueProvider can enumerate every novel available on a source site. +// It handles pagination transparently and streams CatalogueEntry values. +type CatalogueProvider interface { + // CatalogueURL returns the root URL of the catalogue listing. + CatalogueURL() string + + // EntriesSelector returns the selector that matches each novel card / row + // in the catalogue listing page. + EntriesSelector() Selector + + // NextPageSelector returns the selector for the "next page" link. + // If the current page has no next page the implementation must return + // ("", nil) from ScrapeNextPage. + NextPageSelector() Selector + + // ScrapeCatalogue pages through the entire catalogue, sending + // CatalogueEntry values to the returned channel. The channel is closed + // when all pages have been scraped or ctx is cancelled. + // Errors are surfaced via the error channel; a non-nil error does not + // necessarily terminate scraping. + ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error) +} + +// MetadataProvider can extract structured book metadata from a novel's landing page. +type MetadataProvider interface { + // MetadataSelectors returns a map of field name → Selector used to + // locate each metadata element on the book page. + // Required keys: "title", "author". + // Optional keys: "cover", "status", "genres", "summary", "total_chapters". + MetadataSelectors() map[string]Selector + + // ScrapeMetadata fetches and parses the metadata for the book at bookURL. + ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error) +} + +// ChapterListProvider can enumerate all chapters of a book from the chapter-list page. +type ChapterListProvider interface { + // ChaptersURL derives the chapter-list URL from a book landing-page URL. + ChaptersURL(bookURL string) string + + // ChapterEntrySelector returns the selector that matches each chapter row + // in the chapter list page. + ChapterEntrySelector() Selector + + // ScrapeChapterList returns all chapter references for a book, ordered + // by chapter number ascending. + ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error) +} + +// ChapterTextProvider can extract the readable text from a single chapter page. +type ChapterTextProvider interface { + // ChapterTextSelector returns the selector that wraps the chapter body. + ChapterTextSelector() Selector + + // ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown. + ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error) +} + +// NovelScraper is the full interface that a concrete novel source must implement. +// It composes all four provider interfaces. +type NovelScraper interface { + CatalogueProvider + MetadataProvider + ChapterListProvider + ChapterTextProvider + + // SourceName returns the human-readable name of this scraper, e.g. "novelfire.net". + SourceName() string +} diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go new file mode 100644 index 0000000..532fc49 --- /dev/null +++ b/scraper/internal/server/server.go @@ -0,0 +1,132 @@ +// Package server exposes the scraper as an HTTP service. +// +// Endpoints: +// +// POST /scrape — enqueue a full catalogue scrape +// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) +// GET /health — liveness probe +package server + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/libnovel/scraper/internal/orchestrator" + "github.com/libnovel/scraper/internal/scraper" +) + +// Server wraps an HTTP mux with the scraping endpoints. +type Server struct { + addr string + oCfg orchestrator.Config + novel scraper.NovelScraper + log *slog.Logger + mu sync.Mutex + running bool +} + +// New creates a new Server. +func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger) *Server { + return &Server{ + addr: addr, + oCfg: oCfg, + novel: novel, + log: log, + } +} + +// ListenAndServe starts the HTTP server and blocks until the provided context +// is cancelled. +func (s *Server) ListenAndServe(ctx context.Context) error { + mux := http.NewServeMux() + mux.HandleFunc("GET /health", s.handleHealth) + mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) + mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) + + srv := &http.Server{ + Addr: s.addr, + Handler: mux, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + } + + errCh := make(chan error, 1) + go func() { errCh <- srv.ListenAndServe() }() + + s.log.Info("HTTP server listening", "addr", s.addr) + + select { + case <-ctx.Done(): + shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return srv.Shutdown(shutCtx) + case err := <-errCh: + return err + } +} + +func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) +} + +func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { + cfg := s.oCfg + cfg.SingleBookURL = "" // full catalogue + + s.runAsync(w, cfg) +} + +func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { + http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) + return + } + + cfg := s.oCfg + cfg.SingleBookURL = body.URL + + s.runAsync(w, cfg) +} + +// runAsync launches an orchestrator in the background and returns 202 Accepted. +// Only one scrape job runs at a time; concurrent requests receive 409 Conflict. +func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { + s.mu.Lock() + if s.running { + s.mu.Unlock() + http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict) + return + } + s.running = true + s.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"}) + + go func() { + defer func() { + s.mu.Lock() + s.running = false + s.mu.Unlock() + }() + + ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) + defer cancel() + + o := orchestrator.New(cfg, s.novel, s.log) + if err := o.Run(ctx); err != nil { + s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) + } + }() +} diff --git a/scraper/internal/writer/writer.go b/scraper/internal/writer/writer.go new file mode 100644 index 0000000..9bc18dd --- /dev/null +++ b/scraper/internal/writer/writer.go @@ -0,0 +1,140 @@ +// Package writer handles persistence of scraped chapters and metadata. +// +// Directory layout: +// +// static/books/ +// ├── {book-slug}/ +// │ ├── metadata.yaml +// │ ├── vol-0/ (no volume grouping) +// │ │ ├── 1-50/ +// │ │ │ ├── chapter-1.md +// │ │ │ └── … +// │ │ └── 51-100/ +// │ │ └── … +// │ └── vol-1/ +// │ └── … +package writer + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/libnovel/scraper/internal/scraper" + "gopkg.in/yaml.v3" +) + +const chaptersPerFolder = 50 + +// Writer persists scraped content under a configurable root directory. +type Writer struct { + root string // e.g. "./static/books" +} + +// New creates a Writer that stores files under root. +func New(root string) *Writer { + return &Writer{root: root} +} + +// ─── Metadata ───────────────────────────────────────────────────────────────── + +// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml. +// It creates the directory if it does not exist and overwrites any existing file. +func (w *Writer) WriteMetadata(meta scraper.BookMeta) error { + dir := w.bookDir(meta.Slug) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("writer: mkdir %s: %w", dir, err) + } + + path := filepath.Join(dir, "metadata.yaml") + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("writer: create metadata %s: %w", path, err) + } + defer f.Close() + + enc := yaml.NewEncoder(f) + enc.SetIndent(2) + if err := enc.Encode(meta); err != nil { + return fmt.Errorf("writer: encode metadata: %w", err) + } + return enc.Close() +} + +// ReadMetadata reads the metadata.yaml for slug if it exists. +// Returns (zero-value, false, nil) when the file does not exist. +func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) { + path := filepath.Join(w.bookDir(slug), "metadata.yaml") + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return scraper.BookMeta{}, false, nil + } + return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err) + } + + var meta scraper.BookMeta + if err := yaml.Unmarshal(data, &meta); err != nil { + return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err) + } + return meta, true, nil +} + +// ─── Chapters ───────────────────────────────────────────────────────────────── + +// ChapterExists returns true if the markdown file for ref already exists on disk. +func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool { + _, err := os.Stat(w.chapterPath(slug, ref)) + return err == nil +} + +// WriteChapter writes chapter.Text to the appropriate markdown file. +// The parent directories are created on demand. +func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error { + path := w.chapterPath(slug, chapter.Ref) + dir := filepath.Dir(path) + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("writer: mkdir %s: %w", dir, err) + } + + // Build the markdown document. + var sb strings.Builder + sb.WriteString("# ") + sb.WriteString(chapter.Ref.Title) + sb.WriteString("\n\n") + sb.WriteString(chapter.Text) + sb.WriteString("\n") + + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + return fmt.Errorf("writer: write chapter %s: %w", path, err) + } + return nil +} + +// ─── Path helpers ───────────────────────────────────────────────────────────── + +// bookDir returns the root directory for a book slug. +func (w *Writer) bookDir(slug string) string { + return filepath.Join(w.root, slug) +} + +// chapterPath computes the full file path for a chapter. +// +// vol-{volume}/{folderRange}/chapter-{number}.md +// +// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md +func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string { + vol := ref.Volume // 0 == no volume grouping + volDir := fmt.Sprintf("vol-%d", vol) + + // Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", … + lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1 + hi := lo + chaptersPerFolder - 1 + rangeDir := fmt.Sprintf("%d-%d", lo, hi) + + filename := fmt.Sprintf("chapter-%d.md", ref.Number) + + return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename) +}