246 lines
8.3 KiB
Go
246 lines
8.3 KiB
Go
// Command scraper is the entrypoint for the libnovel scraper service.
|
|
//
|
|
// Usage (CLI one-shot):
|
|
//
|
|
// scraper run [--url <book-url>]
|
|
//
|
|
// Usage (HTTP server):
|
|
//
|
|
// scraper serve
|
|
//
|
|
// Environment variables:
|
|
//
|
|
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
|
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
|
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP)
|
|
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
|
|
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
|
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
|
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
|
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
|
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
|
|
// POCKETBASE_ADMIN_EMAIL PocketBase admin email (default: admin@libnovel.local)
|
|
// POCKETBASE_ADMIN_PASSWORD PocketBase admin password (default: changeme123)
|
|
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
|
|
// MINIO_ACCESS_KEY MinIO access key (default: admin)
|
|
// MINIO_SECRET_KEY MinIO secret key (default: changeme123)
|
|
// MINIO_USE_SSL Use TLS for MinIO (default: false)
|
|
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
|
|
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
|
|
// LOG_LEVEL debug | info | warn | error (default: info)
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/browser"
|
|
"github.com/libnovel/scraper/internal/novelfire"
|
|
"github.com/libnovel/scraper/internal/orchestrator"
|
|
"github.com/libnovel/scraper/internal/server"
|
|
"github.com/libnovel/scraper/internal/storage"
|
|
)
|
|
|
|
func main() {
|
|
logLevel := slog.LevelInfo
|
|
if v := os.Getenv("LOG_LEVEL"); v != "" {
|
|
if err := logLevel.UnmarshalText([]byte(v)); err != nil {
|
|
fmt.Fprintf(os.Stderr, "invalid LOG_LEVEL %q, using info\n", v)
|
|
}
|
|
}
|
|
log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
|
|
Level: logLevel,
|
|
}))
|
|
|
|
if err := run(log); err != nil {
|
|
log.Error("fatal", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(log *slog.Logger) error {
|
|
args := os.Args[1:]
|
|
if len(args) == 0 {
|
|
printUsage()
|
|
return nil
|
|
}
|
|
|
|
cmd := strings.ToLower(args[0])
|
|
|
|
browserCfg := browser.Config{
|
|
BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"),
|
|
Token: envOr("BROWSERLESS_TOKEN", ""),
|
|
}
|
|
browserCfg.MaxConcurrent = 5
|
|
if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
|
|
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
|
browserCfg.MaxConcurrent = n
|
|
}
|
|
}
|
|
if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
|
|
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
|
browserCfg.Timeout = time.Duration(n) * time.Second
|
|
}
|
|
}
|
|
|
|
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
|
|
bc := newBrowserClient(strategy, browserCfg)
|
|
// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
|
|
directClient := browser.NewDirectHTTPClient(browserCfg)
|
|
|
|
// ── Storage backends ────────────────────────────────────────────────────
|
|
minioCfg := storage.MinioConfig{
|
|
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
|
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
|
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
|
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
|
|
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
|
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
|
}
|
|
pbCfg := storage.PocketBaseConfig{
|
|
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
|
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
|
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg, log)
|
|
if err != nil {
|
|
return fmt.Errorf("storage init failed: %w", err)
|
|
}
|
|
|
|
nf := novelfire.New(bc, log, directClient, directClient, store)
|
|
|
|
workers := 0
|
|
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
|
n, err := strconv.Atoi(s)
|
|
if err == nil && n > 0 {
|
|
workers = n
|
|
}
|
|
}
|
|
if workers == 0 {
|
|
workers = runtime.NumCPU()
|
|
}
|
|
|
|
oCfg := orchestrator.Config{
|
|
Workers: workers,
|
|
}
|
|
|
|
switch cmd {
|
|
case "run":
|
|
// Optional --url flag.
|
|
if len(args) >= 3 && args[1] == "--url" {
|
|
oCfg.SingleBookURL = args[2]
|
|
}
|
|
log.Info("starting one-shot scrape",
|
|
"strategy", strategy,
|
|
"workers", workers,
|
|
"max_concurrent", browserCfg.MaxConcurrent,
|
|
"single_book", oCfg.SingleBookURL,
|
|
"pocketbase_url", pbCfg.BaseURL,
|
|
"pocketbase_email", pbCfg.AdminEmail,
|
|
)
|
|
o := orchestrator.New(oCfg, nf, log, store)
|
|
return o.Run(ctx)
|
|
|
|
case "refresh":
|
|
// refresh <slug> - re-scrape a book from its saved source_url
|
|
if len(args) < 2 {
|
|
return fmt.Errorf("refresh command requires a book slug argument")
|
|
}
|
|
slug := args[1]
|
|
meta, ok, err := store.ReadMetadata(ctx, slug)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
|
|
}
|
|
if !ok {
|
|
return fmt.Errorf("book %q not found in store", slug)
|
|
}
|
|
if meta.SourceURL == "" {
|
|
return fmt.Errorf("book %q has no source_url in metadata", slug)
|
|
}
|
|
oCfg.SingleBookURL = meta.SourceURL
|
|
log.Info("refreshing book from source_url",
|
|
"slug", slug,
|
|
"source_url", meta.SourceURL,
|
|
"pocketbase_url", pbCfg.BaseURL,
|
|
"pocketbase_email", pbCfg.AdminEmail,
|
|
)
|
|
o := orchestrator.New(oCfg, nf, log, store)
|
|
return o.Run(ctx)
|
|
|
|
case "serve":
|
|
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
|
|
kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
|
|
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
|
|
log.Info("starting HTTP server",
|
|
"addr", addr,
|
|
"strategy", strategy,
|
|
"workers", workers,
|
|
"max_concurrent", browserCfg.MaxConcurrent,
|
|
"kokoro_url", kokoroURL,
|
|
"kokoro_voice", kokoroVoice,
|
|
"pocketbase_url", pbCfg.BaseURL,
|
|
"pocketbase_email", pbCfg.AdminEmail,
|
|
)
|
|
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
|
return srv.ListenAndServe(ctx)
|
|
|
|
default:
|
|
return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
|
|
}
|
|
}
|
|
|
|
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
|
switch strategy {
|
|
case browser.StrategyScrape:
|
|
return browser.NewScrapeClient(cfg)
|
|
case browser.StrategyCDP:
|
|
return browser.NewCDPClient(cfg)
|
|
case browser.StrategyDirect:
|
|
return browser.NewDirectHTTPClient(cfg)
|
|
default:
|
|
return browser.NewContentClient(cfg)
|
|
}
|
|
}
|
|
|
|
func envOr(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func printUsage() {
|
|
fmt.Fprintf(os.Stderr, `libnovel scraper
|
|
|
|
Commands:
|
|
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
|
|
refresh <slug> Re-scrape a book from its saved source_url
|
|
serve Start HTTP server (POST /scrape, POST /scrape/book)
|
|
|
|
Environment variables:
|
|
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
|
BROWSERLESS_TOKEN API token (default: "")
|
|
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP)
|
|
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
|
|
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
|
|
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
|
|
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
|
|
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
|
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
|
|
KOKORO_VOICE Default TTS voice (default: af_bella)
|
|
LOG_LEVEL debug|info|warn|error (default: info)
|
|
`, runtime.NumCPU())
|
|
}
|