feat(v3): add v3 stack — backend rewrite, renamed env vars, docs

- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
2026-03-22 17:27:32 +05:00
parent 29d0eeb7e8
commit a85636d5db
178 changed files with 25951 additions and 0 deletions
--- a/v3/backend/cmd/backend/main.go
+++ b/v3/backend/cmd/backend/main.go
@@ -0,0 +1,139 @@
+// Command backend is the LibNovel HTTP API server.
+//
+// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads,
+// scrape-task creation, presigned MinIO URLs, audio-task creation, reading
+// progress, live novelfire.net browse/search, and Kokoro voice list.
+//
+// All heavy lifting (scraping, TTS generation) is delegated to the runner
+// binary via PocketBase task records. The backend never scrapes directly.
+//
+// Usage:
+//
+//	backend          # start HTTP server (blocks until SIGINT/SIGTERM)
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/libnovel/backend/internal/backend"
+	"github.com/libnovel/backend/internal/config"
+	"github.com/libnovel/backend/internal/kokoro"
+	"github.com/libnovel/backend/internal/meili"
+	"github.com/libnovel/backend/internal/storage"
+)
+
+// version and commit are set at build time via -ldflags.
+var (
+	version = "dev"
+	commit  = "unknown"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "backend: fatal: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	cfg := config.Load()
+
+	// ── Logger ───────────────────────────────────────────────────────────────
+	log := buildLogger(cfg.LogLevel)
+	log.Info("backend starting",
+		"version", version,
+		"commit", commit,
+		"addr", cfg.HTTP.Addr,
+	)
+
+	// ── Context: cancel on SIGINT / SIGTERM ──────────────────────────────────
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// ── Storage ──────────────────────────────────────────────────────────────
+	store, err := storage.NewStore(ctx, cfg, log)
+	if err != nil {
+		return fmt.Errorf("init storage: %w", err)
+	}
+
+	// ── Kokoro (voice list only; audio generation is done by the runner) ─────
+	var kokoroClient kokoro.Client
+	if cfg.Kokoro.URL != "" {
+		kokoroClient = kokoro.New(cfg.Kokoro.URL)
+		log.Info("kokoro voices enabled", "url", cfg.Kokoro.URL)
+	} else {
+		log.Info("KOKORO_URL not set — voice list will use built-in fallback")
+		kokoroClient = &noopKokoro{}
+	}
+
+	// ── Meilisearch (search reads only; indexing is the runner's job) ────────
+	var searchIndex meili.Client
+	if cfg.Meilisearch.URL != "" {
+		searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey)
+		log.Info("meilisearch search enabled", "url", cfg.Meilisearch.URL)
+	} else {
+		log.Info("MEILI_URL not set — search will use PocketBase substring fallback")
+		searchIndex = meili.NoopClient{}
+	}
+
+	// ── Backend server ───────────────────────────────────────────────────────
+	srv := backend.New(
+		backend.Config{
+			Addr:         cfg.HTTP.Addr,
+			DefaultVoice: cfg.Kokoro.DefaultVoice,
+			Version:      version,
+			Commit:       commit,
+		},
+		backend.Dependencies{
+			BookReader:    store,
+			RankingStore:  store,
+			AudioStore:    store,
+			PresignStore:  store,
+			ProgressStore: store,
+			BrowseStore:   store,
+			CoverStore:    store,
+			Producer:      store,
+			TaskReader:    store,
+			SearchIndex:   searchIndex,
+			Kokoro:        kokoroClient,
+			Log:           log,
+		},
+	)
+
+	return srv.ListenAndServe(ctx)
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+func buildLogger(level string) *slog.Logger {
+	var lvl slog.Level
+	switch level {
+	case "debug":
+		lvl = slog.LevelDebug
+	case "warn":
+		lvl = slog.LevelWarn
+	case "error":
+		lvl = slog.LevelError
+	default:
+		lvl = slog.LevelInfo
+	}
+	return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}))
+}
+
+// noopKokoro is a no-op implementation used when KOKORO_URL is not set.
+// The backend only uses Kokoro for the voice list; audio generation is the
+// runner's responsibility. With no URL the built-in fallback list is served.
+type noopKokoro struct{}
+
+func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
+	return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)")
+}
+
+func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
+	return nil, nil
+}
--- a/v3/backend/cmd/backend/main_test.go
+++ b/v3/backend/cmd/backend/main_test.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+	"os"
+	"testing"
+)
+
+// TestBuildLogger verifies that buildLogger returns a non-nil logger for each
+// supported log level string and for unknown values.
+func TestBuildLogger(t *testing.T) {
+	for _, level := range []string{"debug", "info", "warn", "error", "unknown", ""} {
+		l := buildLogger(level)
+		if l == nil {
+			t.Errorf("buildLogger(%q) returned nil", level)
+		}
+	}
+}
+
+// TestNoopKokoro verifies that the no-op Kokoro stub returns the expected
+// sentinel error from GenerateAudio and nil, nil from ListVoices.
+func TestNoopKokoro(t *testing.T) {
+	noop := &noopKokoro{}
+
+	_, err := noop.GenerateAudio(t.Context(), "text", "af_bella")
+	if err == nil {
+		t.Fatal("noopKokoro.GenerateAudio: expected error, got nil")
+	}
+
+	voices, err := noop.ListVoices(t.Context())
+	if err != nil {
+		t.Fatalf("noopKokoro.ListVoices: unexpected error: %v", err)
+	}
+	if voices != nil {
+		t.Fatalf("noopKokoro.ListVoices: expected nil slice, got %v", voices)
+	}
+}
+
+// TestRunStorageUnreachable verifies that run() fails fast and returns a
+// descriptive error when PocketBase is unreachable.
+func TestRunStorageUnreachable(t *testing.T) {
+	// Point at an address nothing is listening on.
+	t.Setenv("POCKETBASE_URL", "http://127.0.0.1:19999")
+	// Use a fast listen address so we don't accidentally start a real server.
+	t.Setenv("BACKEND_HTTP_ADDR", "127.0.0.1:0")
+
+	err := run()
+	if err == nil {
+		t.Fatal("run() should have returned an error when storage is unreachable")
+	}
+
+	t.Logf("got expected error: %v", err)
+}
+
+// TestMain runs the test suite. No special setup required.
+func TestMain(m *testing.M) {
+	os.Exit(m.Run())
+}
--- a/v3/backend/cmd/healthcheck/main.go
+++ b/v3/backend/cmd/healthcheck/main.go
@@ -0,0 +1,89 @@
+// healthcheck is a static binary used by Docker HEALTHCHECK CMD in distroless
+// images (which have no shell, wget, or curl).
+//
+// Two modes:
+//
+//  1. HTTP mode (default):
+//     /healthcheck <url>
+//     Performs GET <url>; exits 0 if HTTP 2xx/3xx, 1 otherwise.
+//     Example: /healthcheck http://localhost:8080/health
+//
+//  2. File-liveness mode:
+//     /healthcheck file <path> <max_age_seconds>
+//     Reads <path>, parses its content as RFC3339 timestamp, and exits 1 if the
+//     timestamp is older than <max_age_seconds>. Used by the runner service which
+//     writes /tmp/runner.alive on every successful poll.
+//     Example: /healthcheck file /tmp/runner.alive 120
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+)
+
+func main() {
+	if len(os.Args) > 1 && os.Args[1] == "file" {
+		checkFile()
+		return
+	}
+	checkHTTP()
+}
+
+// checkHTTP performs a GET request and exits 0 on success, 1 on failure.
+func checkHTTP() {
+	url := "http://localhost:8080/health"
+	if len(os.Args) > 1 {
+		url = os.Args[1]
+	}
+	resp, err := http.Get(url) //nolint:gosec,noctx
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "healthcheck: %v\n", err)
+		os.Exit(1)
+	}
+	resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		fmt.Fprintf(os.Stderr, "healthcheck: status %d\n", resp.StatusCode)
+		os.Exit(1)
+	}
+}
+
+// checkFile reads a timestamp from a file and exits 1 if it is older than the
+// given max age. Usage: /healthcheck file <path> <max_age_seconds>
+func checkFile() {
+	if len(os.Args) < 4 {
+		fmt.Fprintln(os.Stderr, "healthcheck file: usage: /healthcheck file <path> <max_age_seconds>")
+		os.Exit(1)
+	}
+	path := os.Args[2]
+	maxAgeSec, err := strconv.ParseInt(os.Args[3], 10, 64)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "healthcheck file: invalid max_age_seconds %q: %v\n", os.Args[3], err)
+		os.Exit(1)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "healthcheck file: cannot read %s: %v\n", path, err)
+		os.Exit(1)
+	}
+
+	ts, err := time.Parse(time.RFC3339, string(data))
+	if err != nil {
+		// Fallback: use file mtime if content is not a valid timestamp.
+		info, statErr := os.Stat(path)
+		if statErr != nil {
+			fmt.Fprintf(os.Stderr, "healthcheck file: cannot stat %s: %v\n", path, statErr)
+			os.Exit(1)
+		}
+		ts = info.ModTime()
+	}
+
+	age := time.Since(ts)
+	if age > time.Duration(maxAgeSec)*time.Second {
+		fmt.Fprintf(os.Stderr, "healthcheck file: %s is %.0fs old (max %ds)\n", path, age.Seconds(), maxAgeSec)
+		os.Exit(1)
+	}
+}
--- a/v3/backend/cmd/runner/main.go
+++ b/v3/backend/cmd/runner/main.go
@@ -0,0 +1,159 @@
+// Command runner is the homelab worker binary.
+//
+// It polls PocketBase for pending scrape and audio tasks, executes them, and
+// writes results back. It connects directly to PocketBase and MinIO using
+// admin credentials loaded from environment variables.
+//
+// Usage:
+//
+//	runner          # start polling loop (blocks until SIGINT/SIGTERM)
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/libnovel/backend/internal/browser"
+	"github.com/libnovel/backend/internal/config"
+	"github.com/libnovel/backend/internal/kokoro"
+	"github.com/libnovel/backend/internal/meili"
+	"github.com/libnovel/backend/internal/novelfire"
+	"github.com/libnovel/backend/internal/runner"
+	"github.com/libnovel/backend/internal/storage"
+)
+
+// version and commit are set at build time via -ldflags.
+var (
+	version = "dev"
+	commit  = "unknown"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "runner: fatal: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	cfg := config.Load()
+
+	// ── Logger ──────────────────────────────────────────────────────────────
+	log := buildLogger(cfg.LogLevel)
+	log.Info("runner starting",
+		"version", version,
+		"commit", commit,
+		"worker_id", cfg.Runner.WorkerID,
+	)
+
+	// ── Context: cancel on SIGINT / SIGTERM ─────────────────────────────────
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// ── Storage ─────────────────────────────────────────────────────────────
+	store, err := storage.NewStore(ctx, cfg, log)
+	if err != nil {
+		return fmt.Errorf("init storage: %w", err)
+	}
+
+	// ── Browser / Scraper ───────────────────────────────────────────────────
+	workers := cfg.Runner.Workers
+	if workers <= 0 {
+		workers = runtime.NumCPU()
+	}
+	timeout := cfg.Runner.Timeout
+	if timeout <= 0 {
+		timeout = 90 * time.Second
+	}
+
+	browserClient := browser.NewDirectClient(browser.Config{
+		MaxConcurrent: workers,
+		Timeout:       timeout,
+	})
+	novel := novelfire.New(browserClient, log)
+
+	// ── Kokoro ──────────────────────────────────────────────────────────────
+	var kokoroClient kokoro.Client
+	if cfg.Kokoro.URL != "" {
+		kokoroClient = kokoro.New(cfg.Kokoro.URL)
+		log.Info("kokoro TTS enabled", "url", cfg.Kokoro.URL)
+	} else {
+		log.Warn("KOKORO_URL not set — audio tasks will fail")
+		kokoroClient = &noopKokoro{}
+	}
+
+	// ── Meilisearch ─────────────────────────────────────────────────────────
+	var searchIndex meili.Client
+	if cfg.Meilisearch.URL != "" {
+		if err := meili.Configure(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey); err != nil {
+			log.Warn("meilisearch configure failed — search indexing disabled", "err", err)
+			searchIndex = meili.NoopClient{}
+		} else {
+			searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey)
+			log.Info("meilisearch enabled", "url", cfg.Meilisearch.URL)
+		}
+	} else {
+		log.Info("MEILI_URL not set — search indexing disabled")
+		searchIndex = meili.NoopClient{}
+	}
+
+	// ── Runner ──────────────────────────────────────────────────────────────
+	rCfg := runner.Config{
+		WorkerID:                 cfg.Runner.WorkerID,
+		PollInterval:             cfg.Runner.PollInterval,
+		MaxConcurrentScrape:      cfg.Runner.MaxConcurrentScrape,
+		MaxConcurrentAudio:       cfg.Runner.MaxConcurrentAudio,
+		OrchestratorWorkers:      workers,
+		MetricsAddr:              cfg.Runner.MetricsAddr,
+		CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
+	}
+	deps := runner.Dependencies{
+		Consumer:    store,
+		BookWriter:  store,
+		BookReader:  store,
+		AudioStore:  store,
+		BrowseStore: store,
+		CoverStore:  store,
+		SearchIndex: searchIndex,
+		Novel:       novel,
+		Kokoro:      kokoroClient,
+		Log:         log,
+	}
+	r := runner.New(rCfg, deps)
+
+	return r.Run(ctx)
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+func buildLogger(level string) *slog.Logger {
+	var lvl slog.Level
+	switch level {
+	case "debug":
+		lvl = slog.LevelDebug
+	case "warn":
+		lvl = slog.LevelWarn
+	case "error":
+		lvl = slog.LevelError
+	default:
+		lvl = slog.LevelInfo
+	}
+	return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}))
+}
+
+// noopKokoro is a no-op implementation used when KOKORO_URL is not set.
+type noopKokoro struct{}
+
+func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
+	return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)")
+}
+
+func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
+	return nil, nil
+}