feat(v3): add v3 stack — backend rewrite, renamed env vars, docs

- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
2026-03-22 17:27:32 +05:00
parent 29d0eeb7e8
commit a85636d5db
178 changed files with 25951 additions and 0 deletions
--- a/v3/backend/internal/config/config.go
+++ b/v3/backend/internal/config/config.go
@@ -0,0 +1,219 @@
+// Package config loads all service configuration from environment variables.
+// Both the runner and backend binaries call config.Load() at startup; each
+// uses only the sub-struct relevant to it.
+//
+// Every field has a documented default so the service starts sensibly without
+// any environment configuration (useful for local development).
+package config
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// PocketBase holds connection settings for the remote PocketBase instance.
+type PocketBase struct {
+	// URL is the base URL of the PocketBase instance, e.g. https://pb.libnovel.cc
+	URL string
+	// AdminEmail is the admin account email used for API authentication.
+	AdminEmail string
+	// AdminPassword is the admin account password.
+	AdminPassword string
+}
+
+// MinIO holds connection settings for the remote MinIO / S3-compatible store.
+type MinIO struct {
+	// Endpoint is the host:port of the MinIO S3 API, e.g. storage.libnovel.cc:443
+	Endpoint string
+	// PublicEndpoint is the browser-visible endpoint used for presigned URLs.
+	// Falls back to Endpoint when empty.
+	PublicEndpoint string
+	// AccessKey is the MinIO access key.
+	AccessKey string
+	// SecretKey is the MinIO secret key.
+	SecretKey string
+	// UseSSL enables TLS for the internal MinIO connection.
+	UseSSL bool
+	// PublicUseSSL enables TLS for presigned URL generation.
+	PublicUseSSL bool
+	// BucketChapters is the bucket that holds chapter markdown objects.
+	BucketChapters string
+	// BucketAudio is the bucket that holds generated audio MP3 objects.
+	BucketAudio string
+	// BucketAvatars is the bucket that holds user avatar images.
+	BucketAvatars string
+	// BucketBrowse is the bucket that holds cached browse page snapshots (JSON).
+	BucketBrowse string
+}
+
+// Kokoro holds connection settings for the Kokoro-FastAPI TTS service.
+type Kokoro struct {
+	// URL is the base URL of the Kokoro service, e.g. https://kokoro.libnovel.cc
+	// An empty string disables TTS generation.
+	URL string
+	// DefaultVoice is the voice used when none is specified.
+	DefaultVoice string
+}
+
+// HTTP holds settings for the HTTP server (backend only).
+type HTTP struct {
+	// Addr is the listen address, e.g. ":8080"
+	Addr string
+}
+
+// Meilisearch holds connection settings for the Meilisearch full-text search service.
+type Meilisearch struct {
+	// URL is the base URL of the Meilisearch instance, e.g. http://localhost:7700
+	// An empty string disables Meilisearch indexing and search.
+	URL string
+	// APIKey is the Meilisearch master/search API key.
+	APIKey string
+}
+
+// Valkey holds connection settings for the Valkey/Redis presign URL cache.
+type Valkey struct {
+	// Addr is the host:port of the Valkey instance, e.g. localhost:6379
+	// An empty string disables the Valkey cache (falls through to MinIO directly).
+	Addr string
+}
+
+// Runner holds settings specific to the runner/worker binary.
+type Runner struct {
+	// PollInterval is how often the runner checks PocketBase for pending tasks.
+	PollInterval time.Duration
+	// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
+	MaxConcurrentScrape int
+	// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
+	MaxConcurrentAudio int
+	// WorkerID is a unique identifier for this runner instance.
+	// Defaults to the system hostname.
+	WorkerID string
+	// Workers is the number of chapter-scraping goroutines per book.
+	Workers int
+	// Timeout is the per-request HTTP timeout for scraping.
+	Timeout time.Duration
+	// MetricsAddr is the listen address for the runner /metrics HTTP endpoint.
+	// Defaults to ":9091". Set to "" to disable.
+	MetricsAddr string
+	// CatalogueRefreshInterval is how often the runner walks the full catalogue,
+	// scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch.
+	// Defaults to 24h. Set to 0 to use the default.
+	CatalogueRefreshInterval time.Duration
+}
+
+// Config is the top-level configuration struct consumed by both binaries.
+type Config struct {
+	PocketBase  PocketBase
+	MinIO       MinIO
+	Kokoro      Kokoro
+	HTTP        HTTP
+	Runner      Runner
+	Meilisearch Meilisearch
+	Valkey      Valkey
+	// LogLevel is one of "debug", "info", "warn", "error".
+	LogLevel string
+}
+
+// Load reads all configuration from environment variables and returns a
+// populated Config. Missing variables fall back to documented defaults.
+func Load() Config {
+	workerID, _ := os.Hostname()
+	if workerID == "" {
+		workerID = "runner-default"
+	}
+
+	return Config{
+		LogLevel: envOr("LOG_LEVEL", "info"),
+
+		PocketBase: PocketBase{
+			URL:           envOr("POCKETBASE_URL", "http://localhost:8090"),
+			AdminEmail:    envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
+			AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
+		},
+
+		MinIO: MinIO{
+			Endpoint:       envOr("MINIO_ENDPOINT", "localhost:9000"),
+			PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
+			AccessKey:      envOr("MINIO_ACCESS_KEY", "admin"),
+			SecretKey:      envOr("MINIO_SECRET_KEY", "changeme123"),
+			UseSSL:         envBool("MINIO_USE_SSL", false),
+			PublicUseSSL:   envBool("MINIO_PUBLIC_USE_SSL", true),
+			BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
+			BucketAudio:    envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
+			BucketAvatars:  envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
+			BucketBrowse:   envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
+		},
+
+		Kokoro: Kokoro{
+			URL:          envOr("KOKORO_URL", ""),
+			DefaultVoice: envOr("KOKORO_VOICE", "af_bella"),
+		},
+
+		HTTP: HTTP{
+			Addr: envOr("BACKEND_HTTP_ADDR", ":8080"),
+		},
+
+		Runner: Runner{
+			PollInterval:             envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
+			MaxConcurrentScrape:      envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
+			MaxConcurrentAudio:       envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
+			WorkerID:                 envOr("RUNNER_WORKER_ID", workerID),
+			Workers:                  envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
+			Timeout:                  envDuration("RUNNER_TIMEOUT", 90*time.Second),
+			MetricsAddr:              envOr("RUNNER_METRICS_ADDR", ":9091"),
+			CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
+		},
+
+		Meilisearch: Meilisearch{
+			URL:    envOr("MEILI_URL", ""),
+			APIKey: envOr("MEILI_API_KEY", ""),
+		},
+
+		Valkey: Valkey{
+			Addr: envOr("VALKEY_ADDR", ""),
+		},
+	}
+}
+
+// ── helpers ───────────────────────────────────────────────────────────────────
+
+func envOr(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
+
+func envBool(key string, fallback bool) bool {
+	v := os.Getenv(key)
+	if v == "" {
+		return fallback
+	}
+	return strings.ToLower(v) == "true"
+}
+
+func envInt(key string, fallback int) int {
+	v := os.Getenv(key)
+	if v == "" {
+		return fallback
+	}
+	n, err := strconv.Atoi(v)
+	if err != nil || n < 0 {
+		return fallback
+	}
+	return n
+}
+
+func envDuration(key string, fallback time.Duration) time.Duration {
+	v := os.Getenv(key)
+	if v == "" {
+		return fallback
+	}
+	d, err := time.ParseDuration(v)
+	if err != nil {
+		return fallback
+	}
+	return d
+}
--- a/v3/backend/internal/config/config_test.go
+++ b/v3/backend/internal/config/config_test.go
@@ -0,0 +1,127 @@
+package config_test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/libnovel/backend/internal/config"
+)
+
+func TestLoad_Defaults(t *testing.T) {
+	// Unset all relevant vars so we test pure defaults.
+	unset := []string{
+		"LOG_LEVEL",
+		"POCKETBASE_URL", "POCKETBASE_ADMIN_EMAIL", "POCKETBASE_ADMIN_PASSWORD",
+		"MINIO_ENDPOINT", "MINIO_PUBLIC_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
+		"MINIO_USE_SSL", "MINIO_PUBLIC_USE_SSL",
+		"MINIO_BUCKET_CHAPTERS", "MINIO_BUCKET_AUDIO", "MINIO_BUCKET_AVATARS",
+		"KOKORO_URL", "KOKORO_VOICE",
+		"BACKEND_HTTP_ADDR",
+		"RUNNER_POLL_INTERVAL", "RUNNER_MAX_CONCURRENT_SCRAPE", "RUNNER_MAX_CONCURRENT_AUDIO",
+		"RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT",
+	}
+	for _, k := range unset {
+		t.Setenv(k, "")
+	}
+
+	cfg := config.Load()
+
+	if cfg.LogLevel != "info" {
+		t.Errorf("LogLevel: want info, got %q", cfg.LogLevel)
+	}
+	if cfg.PocketBase.URL != "http://localhost:8090" {
+		t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL)
+	}
+	if cfg.MinIO.BucketChapters != "libnovel-chapters" {
+		t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters)
+	}
+	if cfg.MinIO.UseSSL != false {
+		t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL)
+	}
+	if cfg.MinIO.PublicUseSSL != true {
+		t.Errorf("MinIO.PublicUseSSL: want true, got %v", cfg.MinIO.PublicUseSSL)
+	}
+	if cfg.Kokoro.DefaultVoice != "af_bella" {
+		t.Errorf("Kokoro.DefaultVoice: want af_bella, got %q", cfg.Kokoro.DefaultVoice)
+	}
+	if cfg.HTTP.Addr != ":8080" {
+		t.Errorf("HTTP.Addr: want :8080, got %q", cfg.HTTP.Addr)
+	}
+	if cfg.Runner.PollInterval != 30*time.Second {
+		t.Errorf("Runner.PollInterval: want 30s, got %v", cfg.Runner.PollInterval)
+	}
+	if cfg.Runner.MaxConcurrentScrape != 1 {
+		t.Errorf("Runner.MaxConcurrentScrape: want 1, got %d", cfg.Runner.MaxConcurrentScrape)
+	}
+	if cfg.Runner.MaxConcurrentAudio != 1 {
+		t.Errorf("Runner.MaxConcurrentAudio: want 1, got %d", cfg.Runner.MaxConcurrentAudio)
+	}
+}
+
+func TestLoad_EnvOverride(t *testing.T) {
+	t.Setenv("LOG_LEVEL", "debug")
+	t.Setenv("POCKETBASE_URL", "https://pb.libnovel.cc")
+	t.Setenv("MINIO_USE_SSL", "true")
+	t.Setenv("MINIO_PUBLIC_USE_SSL", "false")
+	t.Setenv("RUNNER_POLL_INTERVAL", "1m")
+	t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "5")
+	t.Setenv("RUNNER_WORKER_ID", "homelab-01")
+	t.Setenv("BACKEND_HTTP_ADDR", ":9090")
+	t.Setenv("KOKORO_URL", "https://kokoro.libnovel.cc")
+
+	cfg := config.Load()
+
+	if cfg.LogLevel != "debug" {
+		t.Errorf("LogLevel: want debug, got %q", cfg.LogLevel)
+	}
+	if cfg.PocketBase.URL != "https://pb.libnovel.cc" {
+		t.Errorf("PocketBase.URL: want https://pb.libnovel.cc, got %q", cfg.PocketBase.URL)
+	}
+	if !cfg.MinIO.UseSSL {
+		t.Error("MinIO.UseSSL: want true")
+	}
+	if cfg.MinIO.PublicUseSSL {
+		t.Error("MinIO.PublicUseSSL: want false")
+	}
+	if cfg.Runner.PollInterval != time.Minute {
+		t.Errorf("Runner.PollInterval: want 1m, got %v", cfg.Runner.PollInterval)
+	}
+	if cfg.Runner.MaxConcurrentScrape != 5 {
+		t.Errorf("Runner.MaxConcurrentScrape: want 5, got %d", cfg.Runner.MaxConcurrentScrape)
+	}
+	if cfg.Runner.WorkerID != "homelab-01" {
+		t.Errorf("Runner.WorkerID: want homelab-01, got %q", cfg.Runner.WorkerID)
+	}
+	if cfg.HTTP.Addr != ":9090" {
+		t.Errorf("HTTP.Addr: want :9090, got %q", cfg.HTTP.Addr)
+	}
+	if cfg.Kokoro.URL != "https://kokoro.libnovel.cc" {
+		t.Errorf("Kokoro.URL: want https://kokoro.libnovel.cc, got %q", cfg.Kokoro.URL)
+	}
+}
+
+func TestLoad_InvalidInt_FallsToDefault(t *testing.T) {
+	t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "notanumber")
+	cfg := config.Load()
+	if cfg.Runner.MaxConcurrentScrape != 1 {
+		t.Errorf("want default 1, got %d", cfg.Runner.MaxConcurrentScrape)
+	}
+}
+
+func TestLoad_InvalidDuration_FallsToDefault(t *testing.T) {
+	t.Setenv("RUNNER_POLL_INTERVAL", "notaduration")
+	cfg := config.Load()
+	if cfg.Runner.PollInterval != 30*time.Second {
+		t.Errorf("want default 30s, got %v", cfg.Runner.PollInterval)
+	}
+}
+
+func TestLoad_WorkerID_FallsToHostname(t *testing.T) {
+	t.Setenv("RUNNER_WORKER_ID", "")
+	cfg := config.Load()
+	host, _ := os.Hostname()
+	if host != "" && cfg.Runner.WorkerID != host {
+		t.Errorf("want hostname %q, got %q", host, cfg.Runner.WorkerID)
+	}
+}