libnovel/scraper/cmd/scraper/main.go

// Command scraper is the entrypoint for the libnovel scraper service.
//
// Usage (CLI one-shot):
//
//	scraper run [--url <book-url>]
//
// Usage (HTTP server):
//
//	scraper serve
//
// Environment variables:
//
//	BROWSERLESS_URL          Browserless base URL           (default: http://localhost:3030)
//	BROWSERLESS_TOKEN        Browserless API token          (default: "")
//	BROWSERLESS_STRATEGY     content | scrape | cdp         (default: content)
//	BROWSERLESS_MAX_CONCURRENT  Max simultaneous browser sessions  (default: 5)
//	SCRAPER_WORKERS          Chapter goroutine count        (default: NumCPU)
//	SCRAPER_STATIC_ROOT      Output directory               (default: ./static/books)
//	SCRAPER_HTTP_ADDR        HTTP listen address            (default: :8080)
//	KOKORO_URL               Kokoro-FastAPI base URL        (default: "")
//	KOKORO_VOICE             Default TTS voice              (default: af_bella)
//	LOG_LEVEL                debug | info | warn | error    (default: info)
package main

import (
	"context"
	"fmt"
	"log/slog"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/libnovel/scraper/internal/browser"
	"github.com/libnovel/scraper/internal/novelfire"
	"github.com/libnovel/scraper/internal/orchestrator"
	"github.com/libnovel/scraper/internal/server"
	"github.com/libnovel/scraper/internal/writer"
)

func main() {
	logLevel := slog.LevelInfo
	if v := os.Getenv("LOG_LEVEL"); v != "" {
		if err := logLevel.UnmarshalText([]byte(v)); err != nil {
			fmt.Fprintf(os.Stderr, "invalid LOG_LEVEL %q, using info\n", v)
		}
	}
	log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
		Level: logLevel,
	}))

	if err := run(log); err != nil {
		log.Error("fatal", "err", err)
		os.Exit(1)
	}
}

func run(log *slog.Logger) error {
	args := os.Args[1:]
	if len(args) == 0 {
		printUsage()
		return nil
	}

	cmd := strings.ToLower(args[0])

	browserCfg := browser.Config{
		BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"),
		Token:   envOr("BROWSERLESS_TOKEN", ""),
	}
	browserCfg.MaxConcurrent = 5
	if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
		if n, err := strconv.Atoi(s); err == nil && n > 0 {
			browserCfg.MaxConcurrent = n
		}
	}
	if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
		if n, err := strconv.Atoi(s); err == nil && n > 0 {
			browserCfg.Timeout = time.Duration(n) * time.Second
		}
	}

	strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
	urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
	bc := newBrowserClient(strategy, browserCfg)
	urlClient := newBrowserClient(urlStrategy, browserCfg)

	staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
	w := writer.New(staticRoot)
	nf := novelfire.New(bc, log, urlClient, w)

	workers := 0
	if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
		n, err := strconv.Atoi(s)
		if err == nil && n > 0 {
			workers = n
		}
	}
	if workers == 0 {
		workers = runtime.NumCPU()
	}

	oCfg := orchestrator.Config{
		Workers:    workers,
		StaticRoot: staticRoot,
	}

	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()

	switch cmd {
	case "run":
		// Optional --url flag.
		if len(args) >= 3 && args[1] == "--url" {
			oCfg.SingleBookURL = args[2]
		}
		log.Info("starting one-shot scrape",
			"strategy", strategy,
			"workers", workers,
			"max_concurrent", browserCfg.MaxConcurrent,
			"static_root", oCfg.StaticRoot,
			"single_book", oCfg.SingleBookURL,
		)
		o := orchestrator.New(oCfg, nf, log)
		return o.Run(ctx)

	case "refresh":
		// refresh <slug> - re-scrape a book from its saved source_url
		if len(args) < 2 {
			return fmt.Errorf("refresh command requires a book slug argument")
		}
		slug := args[1]
		w := writer.New(oCfg.StaticRoot)
		meta, ok, err := w.ReadMetadata(slug)
		if err != nil {
			return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
		}
		if !ok {
			return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
		}
		if meta.SourceURL == "" {
			return fmt.Errorf("book %q has no source_url in metadata", slug)
		}
		oCfg.SingleBookURL = meta.SourceURL
		log.Info("refreshing book from source_url",
			"slug", slug,
			"source_url", meta.SourceURL,
		)
		o := orchestrator.New(oCfg, nf, log)
		return o.Run(ctx)

	case "serve":
		addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
		kokoroURL := envOr("KOKORO_URL", "")
		kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
		log.Info("starting HTTP server",
			"addr", addr,
			"strategy", strategy,
			"workers", workers,
			"max_concurrent", browserCfg.MaxConcurrent,
			"kokoro_url", kokoroURL,
			"kokoro_voice", kokoroVoice,
		)
		srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
		return srv.ListenAndServe(ctx)

	default:
		return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
	}
}

func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
	switch strategy {
	case browser.StrategyScrape:
		return browser.NewScrapeClient(cfg)
	case browser.StrategyCDP:
		return browser.NewCDPClient(cfg)
	case browser.StrategyDirect:
		return browser.NewDirectHTTPClient(cfg)
	default:
		return browser.NewContentClient(cfg)
	}
}

func envOr(key, fallback string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return fallback
}

func printUsage() {
	fmt.Fprintf(os.Stderr, `libnovel scraper

Commands:
  run [--url <book-url>]   One-shot: scrape full catalogue, or a single book
  refresh <slug>            Re-scrape a book from its saved source_url
  serve                    Start HTTP server (POST /scrape, POST /scrape/book)

Environment variables:
  BROWSERLESS_URL             Browserless base URL      (default: http://localhost:3030)
  BROWSERLESS_TOKEN           API token                 (default: "")
  BROWSERLESS_STRATEGY        content|scrape|cdp|direct (default: direct)
  BROWSERLESS_URL_STRATEGY    Strategy for URL retrieval (default: content)
  BROWSERLESS_MAX_CONCURRENT  Max simultaneous sessions (default: 5)
  BROWSERLESS_TIMEOUT         HTTP request timeout sec  (default: 90)
  SCRAPER_WORKERS             Chapter goroutines        (default: NumCPU = %d)
  SCRAPER_STATIC_ROOT         Output directory          (default: ./static/books)
  SCRAPER_HTTP_ADDR           HTTP listen address       (default: :8080)
  KOKORO_URL                  Kokoro-FastAPI base URL   (default: "", TTS disabled)
  KOKORO_VOICE                Default TTS voice         (default: af_bella)
  LOG_LEVEL                   debug|info|warn|error     (default: info)
`, runtime.NumCPU())
}