chore: migrate to v3, Doppler secrets, clean up legacy code

- Remove all pre-v3 code: scraper, ui-v2, backend v1, ios v1+v2, legacy CI workflows
- Flatten v3/ contents to repo root
- Add Doppler secrets management (project=libnovel, config=prd)
- Add justfile with doppler run wrappers for all docker compose commands
- Strip hardcoded env fallbacks from docker-compose.yml
- Add minimal README.md
- Clean up .gitignore

backend/internal/runner/runner.go

@@ -8,6 +8,9 @@
 //   - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
 //     the result back (up to MaxConcurrentAudio goroutines).
 //   - The runner is stateless between ticks; all state lives in PocketBase.
+//   - Atomic task counters are exposed via /metrics (see metrics.go).
+//   - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata
+//     hook injected at construction time.
 package runner
 
 import (
@@ -16,11 +19,13 @@ import (
 	"log/slog"
 	"os"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/libnovel/backend/internal/bookstore"
 	"github.com/libnovel/backend/internal/domain"
 	"github.com/libnovel/backend/internal/kokoro"
+	"github.com/libnovel/backend/internal/meili"
 	"github.com/libnovel/backend/internal/orchestrator"
 	"github.com/libnovel/backend/internal/scraper"
 	"github.com/libnovel/backend/internal/taskqueue"
@@ -44,9 +49,18 @@ type Config struct {
 	// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
 	// task is considered orphaned and reset to pending. Defaults to 2m when 0.
 	StaleTaskThreshold time.Duration
-	// BrowseRefreshInterval is how often the runner pre-fetches browse page
-	// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
-	BrowseRefreshInterval time.Duration
+	// CatalogueRefreshInterval is how often the runner walks the full catalogue,
+	// scrapes per-book metadata, downloads covers, and re-indexes everything in
+	// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
+	CatalogueRefreshInterval time.Duration
+	// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
+	// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
+	// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
+	// quick restarts where the catalogue is already up to date.
+	SkipInitialCatalogueRefresh bool
+	// MetricsAddr is the HTTP listen address for the /metrics endpoint.
+	// Defaults to ":9091". Set to "" to disable.
+	MetricsAddr string
 }
 
 // Dependencies are the external services the runner depends on.
@@ -59,8 +73,11 @@ type Dependencies struct {
 	BookReader bookstore.BookReader
 	// AudioStore persists generated audio and checks key existence.
 	AudioStore bookstore.AudioStore
-	// BrowseStore stores browse page snapshots in MinIO.
-	BrowseStore bookstore.BrowseStore
+	// CoverStore stores book cover images in MinIO.
+	CoverStore bookstore.CoverStore
+	// SearchIndex indexes books in Meilisearch after scraping.
+	// If nil a no-op is used.
+	SearchIndex meili.Client
 	// Novel is the scraper implementation.
 	Novel scraper.NovelScraper
 	// Kokoro is the TTS client.
@@ -73,10 +90,16 @@ type Dependencies struct {
 type Runner struct {
 	cfg  Config
 	deps Dependencies
+
+	// Atomic task counters — read by /metrics without locking.
+	tasksRunning   atomic.Int64
+	tasksCompleted atomic.Int64
+	tasksFailed    atomic.Int64
+
+	startedAt time.Time
 }
 
 // New creates a Runner from cfg and deps.
-// Any zero/nil field in deps will cause a panic at construction time to fail fast.
 func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.PollInterval <= 0 {
 		cfg.PollInterval = 30 * time.Second
@@ -96,63 +119,63 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.StaleTaskThreshold <= 0 {
 		cfg.StaleTaskThreshold = 2 * time.Minute
 	}
-	if cfg.BrowseRefreshInterval <= 0 {
-		cfg.BrowseRefreshInterval = 6 * time.Hour
+	if cfg.CatalogueRefreshInterval <= 0 {
+		cfg.CatalogueRefreshInterval = 24 * time.Hour
+	}
+	if cfg.MetricsAddr == "" {
+		cfg.MetricsAddr = ":9091"
 	}
 	if deps.Log == nil {
 		deps.Log = slog.Default()
 	}
-	return &Runner{cfg: cfg, deps: deps}
-}
-
-// livenessFile is the path written on every successful poll so that the Docker
-// healthcheck (CMD /healthcheck file /tmp/runner.alive <max_age>) can verify
-// the runner is still making progress.
-const livenessFile = "/tmp/runner.alive"
-
-// touchAlive writes the current UTC time to livenessFile. Errors are logged but
-// never fatal — liveness is best-effort and should not crash the runner.
-func (r *Runner) touchAlive() {
-	data := []byte(time.Now().UTC().Format(time.RFC3339))
-	if err := os.WriteFile(livenessFile, data, 0o644); err != nil {
-		r.deps.Log.Warn("runner: failed to write liveness file", "err", err)
+	if deps.SearchIndex == nil {
+		deps.SearchIndex = meili.NoopClient{}
 	}
+	return &Runner{cfg: cfg, deps: deps, startedAt: time.Now()}
 }
 
-// Run starts the poll loop, blocking until ctx is cancelled.
-// On each tick it claims and executes all available pending tasks.
-// Scrape and audio tasks run in separate goroutine pools bounded by
-// MaxConcurrentScrape and MaxConcurrentAudio respectively.
+// Run starts the poll loop and the metrics HTTP server, blocking until ctx is
+// cancelled.
 func (r *Runner) Run(ctx context.Context) error {
 	r.deps.Log.Info("runner: starting",
 		"worker_id", r.cfg.WorkerID,
 		"poll_interval", r.cfg.PollInterval,
 		"max_scrape", r.cfg.MaxConcurrentScrape,
 		"max_audio", r.cfg.MaxConcurrentAudio,
-		"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
+		"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
+		"metrics_addr", r.cfg.MetricsAddr,
 	)
 
+	// Start metrics HTTP server in background if configured.
+	if r.cfg.MetricsAddr != "" {
+		ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log)
+		go func() {
+			if err := ms.ListenAndServe(ctx); err != nil {
+				r.deps.Log.Error("runner: metrics server error", "err", err)
+			}
+		}()
+	}
+
 	scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
 	audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
 	var wg sync.WaitGroup
 
-	// Write liveness file immediately so the first healthcheck passes before
-	// the first poll completes.
-	r.touchAlive()
-
 	tick := time.NewTicker(r.cfg.PollInterval)
 	defer tick.Stop()
 
-	browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
-	defer browseTick.Stop()
+	catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
+	defer catalogueTick.Stop()
 
-	// Run one browse refresh and one poll immediately on startup.
-	go r.runBrowseRefresh(ctx)
+	// Run one catalogue refresh immediately on startup (unless skipped by flag).
+	if !r.cfg.SkipInitialCatalogueRefresh {
+		go r.runCatalogueRefresh(ctx)
+	} else {
+		r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
+	}
 
 	// Run one poll immediately on startup, then on each tick.
 	for {
 		r.poll(ctx, scrapeSem, audioSem, &wg)
-		r.touchAlive()
 
 		select {
 		case <-ctx.Done():
@@ -169,16 +192,24 @@ func (r *Runner) Run(ctx context.Context) error {
 				r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
 			}
 			return nil
-		case <-browseTick.C:
-			go r.runBrowseRefresh(ctx)
+		case <-catalogueTick.C:
+			go r.runCatalogueRefresh(ctx)
 		case <-tick.C:
 		}
 	}
 }
 
 // poll claims all available pending tasks and dispatches them to goroutines.
-// It claims tasks in a tight loop until no more are available.
 func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
+	// ── Heartbeat file ────────────────────────────────────────────────────
+	// Touch /tmp/runner.alive so the Docker health check can confirm the
+	// runner is actively polling. Failure is non-fatal — just log it.
+	if f, err := os.Create("/tmp/runner.alive"); err != nil {
+		r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
+	} else {
+		f.Close()
+	}
+
 	// ── Reap orphaned tasks ───────────────────────────────────────────────
 	if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
 		r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
@@ -197,23 +228,21 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
 			break
 		}
 		if !ok {
-			break // queue empty
+			break
 		}
-		// Acquire semaphore (non-blocking when full — leave task running).
 		select {
 		case scrapeSem <- struct{}{}:
 		default:
-			// Too many concurrent scrapes — the task stays claimed but we can't
-			// run it right now. Log and break; the next poll will pick it up if
-			// still running (it won't be re-claimed while status=running).
 			r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
 				"task_id", task.ID)
 			break
 		}
+		r.tasksRunning.Add(1)
 		wg.Add(1)
 		go func(t domain.ScrapeTask) {
 			defer wg.Done()
 			defer func() { <-scrapeSem }()
+			defer r.tasksRunning.Add(-1)
 			r.runScrapeTask(ctx, t)
 		}(task)
 	}
@@ -229,7 +258,7 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
 			break
 		}
 		if !ok {
-			break // queue empty
+			break
 		}
 		select {
 		case audioSem <- struct{}{}:
@@ -238,22 +267,36 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
 				"task_id", task.ID)
 			break
 		}
+		r.tasksRunning.Add(1)
 		wg.Add(1)
 		go func(t domain.AudioTask) {
 			defer wg.Done()
 			defer func() { <-audioSem }()
+			defer r.tasksRunning.Add(-1)
 			r.runAudioTask(ctx, t)
 		}(task)
 	}
 }
 
+// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in.
+func (r *Runner) newOrchestrator() *orchestrator.Orchestrator {
+	oCfg := orchestrator.Config{
+		Workers: r.cfg.OrchestratorWorkers,
+		PostMetadata: func(ctx context.Context, meta domain.BookMeta) {
+			if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
+				r.deps.Log.Warn("runner: meilisearch upsert failed",
+					"slug", meta.Slug, "err", err)
+			}
+		},
+	}
+	return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
+}
+
 // runScrapeTask executes one scrape task end-to-end and reports the result.
 func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 	log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
 	log.Info("runner: scrape task starting")
 
-	// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
-	// this task is still alive. Cancelled when the task finishes.
 	hbCtx, hbCancel := context.WithCancel(ctx)
 	defer hbCancel()
 	go func() {
@@ -271,9 +314,7 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 		}
 	}()
 
-	oCfg := orchestrator.Config{Workers: r.cfg.OrchestratorWorkers}
-	o := orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
-
+	o := r.newOrchestrator()
 	var result domain.ScrapeResult
 
 	switch task.Kind {
@@ -289,6 +330,13 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 	if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
 		log.Error("runner: FinishScrapeTask failed", "err", err)
 	}
+
+	if result.ErrorMessage != "" {
+		r.tasksFailed.Add(1)
+	} else {
+		r.tasksCompleted.Add(1)
+	}
+
 	log.Info("runner: scrape task finished",
 		"scraped", result.ChaptersScraped,
 		"skipped", result.ChaptersSkipped,
@@ -296,8 +344,7 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 	)
 }
 
-// runCatalogueTask runs a full catalogue scrape by iterating catalogue entries
-// and running a book task for each one.
+// runCatalogueTask runs a full catalogue scrape.
 func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
 	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
 	var result domain.ScrapeResult
@@ -328,17 +375,11 @@ func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o
 	return result
 }
 
-// runAudioTask executes one audio-generation task:
-//  1. Read chapter text from MinIO.
-//  2. Call Kokoro to generate audio.
-//  3. Upload MP3 to MinIO under the standard audio object key.
-//  4. Report result back to PocketBase.
+// runAudioTask executes one audio-generation task.
 func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 	log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
 	log.Info("runner: audio task starting")
 
-	// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
-	// this task is still alive. Cancelled when the task finishes.
 	hbCtx, hbCancel := context.WithCancel(ctx)
 	defer hbCancel()
 	go func() {
@@ -358,13 +399,13 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 
 	fail := func(msg string) {
 		log.Error("runner: audio task failed", "reason", msg)
+		r.tasksFailed.Add(1)
 		result := domain.AudioResult{ErrorMessage: msg}
 		if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
 			log.Error("runner: FinishAudioTask failed", "err", err)
 		}
 	}
 
-	// Step 1: read chapter text.
 	raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
 	if err != nil {
 		fail(fmt.Sprintf("read chapter: %v", err))
@@ -376,7 +417,6 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 		return
 	}
 
-	// Step 2: generate audio.
 	if r.deps.Kokoro == nil {
 		fail("kokoro client not configured")
 		return
@@ -387,14 +427,13 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 		return
 	}
 
-	// Step 3: upload to MinIO.
 	key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
 	if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
 		fail(fmt.Sprintf("put audio: %v", err))
 		return
 	}
 
-	// Step 4: report success.
+	r.tasksCompleted.Add(1)
 	result := domain.AudioResult{ObjectKey: key}
 	if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
 		log.Error("runner: FinishAudioTask failed", "err", err)