feat: initial commit

2026-02-26 12:56:25 +05:00
commit d68ea71239
15 changed files with 1893 additions and 0 deletions
--- a/scraper/cmd/scraper/main.go
+++ b/scraper/cmd/scraper/main.go
@@ -0,0 +1,150 @@
+// Command scraper is the entrypoint for the libnovel scraper service.
+//
+// Usage (CLI one-shot):
+//
+//	scraper run [--url <book-url>]
+//
+// Usage (HTTP server):
+//
+//	scraper serve
+//
+// Environment variables:
+//
+//	BROWSERLESS_URL      Browserless base URL           (default: http://localhost:3000)
+//	BROWSERLESS_TOKEN    Browserless API token          (default: "")
+//	BROWSERLESS_STRATEGY content | scrape | cdp         (default: content)
+//	SCRAPER_WORKERS      Chapter goroutine count        (default: NumCPU)
+//	SCRAPER_STATIC_ROOT  Output directory               (default: ./static/books)
+//	SCRAPER_HTTP_ADDR    HTTP listen address            (default: :8080)
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/libnovel/scraper/internal/browser"
+	"github.com/libnovel/scraper/internal/novelfire"
+	"github.com/libnovel/scraper/internal/orchestrator"
+	"github.com/libnovel/scraper/internal/server"
+)
+
+func main() {
+	log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	}))
+
+	if err := run(log); err != nil {
+		log.Error("fatal", "err", err)
+		os.Exit(1)
+	}
+}
+
+func run(log *slog.Logger) error {
+	args := os.Args[1:]
+	if len(args) == 0 {
+		printUsage()
+		return nil
+	}
+
+	cmd := strings.ToLower(args[0])
+
+	browserCfg := browser.Config{
+		BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3000"),
+		Token:   envOr("BROWSERLESS_TOKEN", ""),
+	}
+
+	strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyContent))))
+	bc := newBrowserClient(strategy, browserCfg)
+
+	nf := novelfire.New(bc, log)
+
+	workers := 0
+	if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
+		n, err := strconv.Atoi(s)
+		if err == nil && n > 0 {
+			workers = n
+		}
+	}
+	if workers == 0 {
+		workers = runtime.NumCPU()
+	}
+
+	oCfg := orchestrator.Config{
+		Workers:    workers,
+		StaticRoot: envOr("SCRAPER_STATIC_ROOT", "./static/books"),
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	switch cmd {
+	case "run":
+		// Optional --url flag.
+		if len(args) >= 3 && args[1] == "--url" {
+			oCfg.SingleBookURL = args[2]
+		}
+		log.Info("starting one-shot scrape",
+			"strategy", strategy,
+			"workers", workers,
+			"static_root", oCfg.StaticRoot,
+			"single_book", oCfg.SingleBookURL,
+		)
+		o := orchestrator.New(oCfg, nf, log)
+		return o.Run(ctx)
+
+	case "serve":
+		addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
+		log.Info("starting HTTP server",
+			"addr", addr,
+			"strategy", strategy,
+			"workers", workers,
+		)
+		srv := server.New(addr, oCfg, nf, log)
+		return srv.ListenAndServe(ctx)
+
+	default:
+		return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
+	}
+}
+
+func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
+	switch strategy {
+	case browser.StrategyScrape:
+		return browser.NewScrapeClient(cfg)
+	case browser.StrategyCDP:
+		return browser.NewCDPClient(cfg)
+	default:
+		return browser.NewContentClient(cfg)
+	}
+}
+
+func envOr(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
+
+func printUsage() {
+	fmt.Fprintf(os.Stderr, `libnovel scraper
+
+Commands:
+  run [--url <book-url>]   One-shot: scrape full catalogue, or a single book
+  serve                    Start HTTP server (POST /scrape, POST /scrape/book)
+
+Environment variables:
+  BROWSERLESS_URL          Browserless base URL      (default: http://localhost:3000)
+  BROWSERLESS_TOKEN        API token                 (default: "")
+  BROWSERLESS_STRATEGY     content | scrape | cdp    (default: content)
+  SCRAPER_WORKERS          Chapter goroutines        (default: NumCPU = %d)
+  SCRAPER_STATIC_ROOT      Output directory          (default: ./static/books)
+  SCRAPER_HTTP_ADDR        HTTP listen address       (default: :8080)
+`, runtime.NumCPU())
+}