chore: migrate to v3, Doppler secrets, clean up legacy code

- Remove all pre-v3 code: scraper, ui-v2, backend v1, ios v1+v2, legacy CI workflows
- Flatten v3/ contents to repo root
- Add Doppler secrets management (project=libnovel, config=prd)
- Add justfile with doppler run wrappers for all docker compose commands
- Strip hardcoded env fallbacks from docker-compose.yml
- Add minimal README.md
- Clean up .gitignore

backend/internal/runner/browse_refresh.go

@@ -1,176 +0,0 @@
-package runner
-
-// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
-// browse page snapshots and stores them in MinIO.
-//
-// Design:
-//   - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
-//   - Fetches page 1 for each combination of the standard genre/sort/status
-//     filter values and stores the parsed JSON blob in MinIO via BrowseStore.
-//   - The backend's handleBrowse then serves from MinIO instead of calling
-//     novelfire.net live, which avoids IP-based rate-limiting on the server.
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"strings"
-	"time"
-)
-
-// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
-type browseNovelListing struct {
-	Slug  string `json:"slug"`
-	Title string `json:"title"`
-	Cover string `json:"cover"`
-	URL   string `json:"url"`
-}
-
-// browseSnapshot is the JSON structure stored in MinIO.
-type browseSnapshot struct {
-	Novels  []browseNovelListing `json:"novels"`
-	Page    int                  `json:"page"`
-	HasNext bool                 `json:"hasNext"`
-	// CachedAt is the UTC time the snapshot was written (ISO 8601).
-	CachedAt string `json:"cachedAt"`
-}
-
-// browseCombos lists the filter combinations to pre-fetch.
-// Each entry is (genre, sort, status, novelType).
-var browseCombos = []struct{ genre, sort, status, novelType string }{
-	{"all", "popular", "all", "all-novel"},
-	{"all", "popular", "ongoing", "all-novel"},
-	{"all", "popular", "completed", "all-novel"},
-	{"all", "new", "all", "all-novel"},
-	{"all", "new", "ongoing", "all-novel"},
-	{"all", "new", "completed", "all-novel"},
-	{"all", "top-rated", "all", "all-novel"},
-	{"all", "top-rated", "ongoing", "all-novel"},
-	{"all", "top-rated", "completed", "all-novel"},
-}
-
-const novelFireBrowseBase = "https://novelfire.net"
-
-// runBrowseRefresh fetches all browse combos from novelfire.net and stores
-// the results in MinIO. Errors per-combo are logged but do not abort the
-// whole refresh cycle.
-func (r *Runner) runBrowseRefresh(ctx context.Context) {
-	if r.deps.BrowseStore == nil {
-		r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
-		return
-	}
-
-	log := r.deps.Log.With("op", "browse_refresh")
-	log.Info("runner: browse refresh starting", "combos", len(browseCombos))
-
-	ok, fail := 0, 0
-	for _, c := range browseCombos {
-		if ctx.Err() != nil {
-			break
-		}
-		novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
-		if err != nil {
-			log.Warn("runner: browse fetch failed",
-				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
-			fail++
-			continue
-		}
-
-		snap := browseSnapshot{
-			Novels:   novels,
-			Page:     1,
-			HasNext:  hasNext,
-			CachedAt: time.Now().UTC().Format(time.RFC3339),
-		}
-		data, _ := json.Marshal(snap)
-		if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
-			log.Warn("runner: browse put failed",
-				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
-			fail++
-			continue
-		}
-		ok++
-	}
-
-	log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
-}
-
-// fetchBrowsePage calls novelfire.net and returns a list of novel listings
-// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
-func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
-	pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
-		novelFireBrowseBase, genre, sort, status, novelType)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
-	if err != nil {
-		return nil, false, fmt.Errorf("build request: %w", err)
-	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
-
-	httpClient := &http.Client{Timeout: 45 * time.Second}
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		_, _ = io.Copy(io.Discard, resp.Body)
-		return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
-	}
-
-	return parseBrowseHTML(resp.Body)
-}
-
-// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
-// in backend/handlers.go — kept separate to avoid coupling packages.
-func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
-	data, err := io.ReadAll(r)
-	if err != nil {
-		return nil, false, err
-	}
-	body := string(data)
-
-	hasNext := strings.Contains(body, `rel="next"`) ||
-		strings.Contains(body, `aria-label="Next"`) ||
-		strings.Contains(body, `class="next"`)
-
-	slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
-	titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
-	coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
-
-	slugMatches := slugRe.FindAllStringSubmatch(body, -1)
-	titleMatches := titleRe.FindAllStringSubmatch(body, -1)
-	coverMatches := coverRe.FindAllStringSubmatch(body, -1)
-
-	var novels []browseNovelListing
-	seen := make(map[string]bool)
-	for i, sm := range slugMatches {
-		slug := sm[1]
-		if seen[slug] {
-			continue
-		}
-		seen[slug] = true
-
-		item := browseNovelListing{
-			Slug: slug,
-			URL:  novelFireBrowseBase + "/book/" + slug,
-		}
-		if i < len(titleMatches) {
-			item.Title = strings.TrimSpace(titleMatches[i][1])
-		}
-		if i < len(coverMatches) {
-			item.Cover = coverMatches[i][1]
-		}
-		if item.Title != "" {
-			novels = append(novels, item)
-		}
-	}
-
-	return novels, hasNext, nil
-}

backend/internal/runner/catalogue_refresh.go

@@ -0,0 +1,185 @@
+package runner
+
+// catalogue_refresh.go — independent loop that walks the full novelfire.net
+// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and
+// indexes every book in Meilisearch.
+//
+// Design:
+//   - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
+//   - Also fires once on startup.
+//   - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate
+//     and call ScrapeMetadata for each entry.
+//   - Per-request random jitter (1–3s) prevents hammering novelfire.net.
+//   - Cover images are fetched from the URL embedded in BookMeta.Cover and
+//     stored in MinIO (browse bucket, key: covers/{slug}.jpg).
+//   - WriteMetadata + UpsertBook are called for every successfully scraped book.
+//   - Errors for individual books are logged and skipped; the loop continues.
+//   - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy
+//     path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the
+//     backend, which will serve from MinIO.
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"time"
+)
+
+// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
+// every book on novelfire.net, downloads covers to MinIO, and upserts to
+// Meilisearch. Errors for individual books are logged and skipped.
+func (r *Runner) runCatalogueRefresh(ctx context.Context) {
+	if r.deps.Novel == nil {
+		r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
+		return
+	}
+	if r.deps.BookWriter == nil {
+		r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured")
+		return
+	}
+
+	log := r.deps.Log.With("op", "catalogue_refresh")
+	log.Info("runner: catalogue refresh starting")
+
+	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
+
+	ok, skipped, errCount := 0, 0, 0
+	for entry := range entries {
+		if ctx.Err() != nil {
+			break
+		}
+
+		// Skip books already present in Meilisearch — they were indexed on a
+		// previous run. Re-indexing only happens when a scrape task is
+		// explicitly enqueued (e.g. via the admin UI or API).
+		if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
+			skipped++
+			continue
+		}
+
+		// Random jitter between books to avoid rate-limiting.
+		jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
+		select {
+		case <-ctx.Done():
+			break
+		case <-time.After(jitter):
+		}
+
+		meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
+		if err != nil {
+			log.Warn("runner: catalogue refresh: metadata scrape failed",
+				"url", entry.URL, "err", err)
+			errCount++
+			continue
+		}
+
+		// Rewrite cover URL to backend proxy path so UI never hits CDN directly.
+		originalCover := meta.Cover
+		meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug)
+
+		// Persist to PocketBase.
+		if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
+			log.Warn("runner: catalogue refresh: WriteMetadata failed",
+				"slug", meta.Slug, "err", err)
+			errCount++
+			continue
+		}
+
+		// Index in Meilisearch.
+		if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
+			log.Warn("runner: catalogue refresh: UpsertBook failed",
+				"slug", meta.Slug, "err", err)
+			// non-fatal — continue
+		}
+
+		// Download and store cover image in MinIO if we have a cover URL
+		// and a CoverStore is wired in.
+		if r.deps.CoverStore != nil && originalCover != "" {
+			if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
+				if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
+					log.Warn("runner: catalogue refresh: cover download failed",
+						"slug", meta.Slug, "url", originalCover, "err", err)
+					// non-fatal
+				}
+			}
+		}
+
+		ok++
+		if ok%100 == 0 {
+			log.Info("runner: catalogue refresh progress",
+				"scraped", ok, "errors", errCount)
+		}
+	}
+
+	if err := <-errCh; err != nil {
+		log.Warn("runner: catalogue refresh: catalogue stream error", "err", err)
+	}
+
+	log.Info("runner: catalogue refresh finished",
+		"ok", ok, "skipped", skipped, "errors", errCount)
+}
+
+// downloadCover fetches the cover image from coverURL and stores it in MinIO
+// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff
+// on transient errors (5xx, network failures).
+func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error {
+	const maxRetries = 3
+	delay := 2 * time.Second
+
+	var lastErr error
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if attempt > 0 {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(delay):
+			}
+			delay *= 2
+		}
+
+		data, err := fetchCoverBytes(ctx, coverURL)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+
+		if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil {
+			return fmt.Errorf("put cover: %w", err)
+		}
+		return nil
+	}
+	return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr)
+}
+
+// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body.
+func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("build request: %w", err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
+	req.Header.Set("Referer", "https://novelfire.net/")
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http get: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 500 {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL)
+	}
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL)
+	}
+
+	return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap
+}

backend/internal/runner/metrics.go

@@ -0,0 +1,92 @@
+package runner
+
+// metrics.go — lightweight HTTP metrics endpoint for the runner.
+//
+// GET /metrics returns a JSON document with live task counters and uptime.
+// No external dependency (no Prometheus); plain net/http only.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"time"
+)
+
+// metricsServer serves GET /metrics for the runner process.
+type metricsServer struct {
+	addr string
+	r    *Runner
+	log  *slog.Logger
+}
+
+func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer {
+	return &metricsServer{addr: addr, r: r, log: log}
+}
+
+// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or
+// a fatal listen error occurs.
+func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /metrics", ms.handleMetrics)
+	mux.HandleFunc("GET /health", ms.handleHealth)
+
+	srv := &http.Server{
+		Addr:         ms.addr,
+		Handler:      mux,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 5 * time.Second,
+		BaseContext:  func(_ net.Listener) context.Context { return ctx },
+	}
+
+	errCh := make(chan error, 1)
+	go func() {
+		ms.log.Info("runner: metrics server listening", "addr", ms.addr)
+		errCh <- srv.ListenAndServe()
+	}()
+
+	select {
+	case <-ctx.Done():
+		shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutCtx)
+		return nil
+	case err := <-errCh:
+		return fmt.Errorf("runner: metrics server: %w", err)
+	}
+}
+
+// handleMetrics handles GET /metrics.
+// Response shape (JSON):
+//
+//	{
+//	  "tasks_running":   N,
+//	  "tasks_completed": N,
+//	  "tasks_failed":    N,
+//	  "uptime_seconds":  N
+//	}
+func (ms *metricsServer) handleMetrics(w http.ResponseWriter, _ *http.Request) {
+	uptimeSec := int64(time.Since(ms.r.startedAt).Seconds())
+	metricsWriteJSON(w, 0, map[string]int64{
+		"tasks_running":   ms.r.tasksRunning.Load(),
+		"tasks_completed": ms.r.tasksCompleted.Load(),
+		"tasks_failed":    ms.r.tasksFailed.Load(),
+		"uptime_seconds":  uptimeSec,
+	})
+}
+
+// handleHealth handles GET /health — simple liveness probe for the metrics server.
+func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) {
+	metricsWriteJSON(w, 0, map[string]string{"status": "ok"})
+}
+
+// metricsWriteJSON writes v as a JSON response with the given status code.
+func metricsWriteJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	if status != 0 {
+		w.WriteHeader(status)
+	}
+	_ = json.NewEncoder(w).Encode(v)
+}

backend/internal/runner/runner.go

@@ -8,6 +8,9 @@
 //   - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
 //     the result back (up to MaxConcurrentAudio goroutines).
 //   - The runner is stateless between ticks; all state lives in PocketBase.
+//   - Atomic task counters are exposed via /metrics (see metrics.go).
+//   - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata
+//     hook injected at construction time.
 package runner
 
 import (
@@ -16,11 +19,13 @@ import (
 	"log/slog"
 	"os"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/libnovel/backend/internal/bookstore"
 	"github.com/libnovel/backend/internal/domain"
 	"github.com/libnovel/backend/internal/kokoro"
+	"github.com/libnovel/backend/internal/meili"
 	"github.com/libnovel/backend/internal/orchestrator"
 	"github.com/libnovel/backend/internal/scraper"
 	"github.com/libnovel/backend/internal/taskqueue"
@@ -44,9 +49,18 @@ type Config struct {
 	// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
 	// task is considered orphaned and reset to pending. Defaults to 2m when 0.
 	StaleTaskThreshold time.Duration
-	// BrowseRefreshInterval is how often the runner pre-fetches browse page
-	// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
-	BrowseRefreshInterval time.Duration
+	// CatalogueRefreshInterval is how often the runner walks the full catalogue,
+	// scrapes per-book metadata, downloads covers, and re-indexes everything in
+	// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
+	CatalogueRefreshInterval time.Duration
+	// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
+	// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
+	// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
+	// quick restarts where the catalogue is already up to date.
+	SkipInitialCatalogueRefresh bool
+	// MetricsAddr is the HTTP listen address for the /metrics endpoint.
+	// Defaults to ":9091". Set to "" to disable.
+	MetricsAddr string
 }
 
 // Dependencies are the external services the runner depends on.
@@ -59,8 +73,11 @@ type Dependencies struct {
 	BookReader bookstore.BookReader
 	// AudioStore persists generated audio and checks key existence.
 	AudioStore bookstore.AudioStore
-	// BrowseStore stores browse page snapshots in MinIO.
-	BrowseStore bookstore.BrowseStore
+	// CoverStore stores book cover images in MinIO.
+	CoverStore bookstore.CoverStore
+	// SearchIndex indexes books in Meilisearch after scraping.
+	// If nil a no-op is used.
+	SearchIndex meili.Client
 	// Novel is the scraper implementation.
 	Novel scraper.NovelScraper
 	// Kokoro is the TTS client.
@@ -73,10 +90,16 @@ type Dependencies struct {
 type Runner struct {
 	cfg  Config
 	deps Dependencies
+
+	// Atomic task counters — read by /metrics without locking.
+	tasksRunning   atomic.Int64
+	tasksCompleted atomic.Int64
+	tasksFailed    atomic.Int64
+
+	startedAt time.Time
 }
 
 // New creates a Runner from cfg and deps.
-// Any zero/nil field in deps will cause a panic at construction time to fail fast.
 func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.PollInterval <= 0 {
 		cfg.PollInterval = 30 * time.Second
@@ -96,63 +119,63 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.StaleTaskThreshold <= 0 {
 		cfg.StaleTaskThreshold = 2 * time.Minute
 	}
-	if cfg.BrowseRefreshInterval <= 0 {
-		cfg.BrowseRefreshInterval = 6 * time.Hour
+	if cfg.CatalogueRefreshInterval <= 0 {
+		cfg.CatalogueRefreshInterval = 24 * time.Hour
+	}
+	if cfg.MetricsAddr == "" {
+		cfg.MetricsAddr = ":9091"
 	}
 	if deps.Log == nil {
 		deps.Log = slog.Default()
 	}
-	return &Runner{cfg: cfg, deps: deps}
-}
-
-// livenessFile is the path written on every successful poll so that the Docker
-// healthcheck (CMD /healthcheck file /tmp/runner.alive <max_age>) can verify
-// the runner is still making progress.
-const livenessFile = "/tmp/runner.alive"
-
-// touchAlive writes the current UTC time to livenessFile. Errors are logged but
-// never fatal — liveness is best-effort and should not crash the runner.
-func (r *Runner) touchAlive() {
-	data := []byte(time.Now().UTC().Format(time.RFC3339))
-	if err := os.WriteFile(livenessFile, data, 0o644); err != nil {
-		r.deps.Log.Warn("runner: failed to write liveness file", "err", err)
+	if deps.SearchIndex == nil {
+		deps.SearchIndex = meili.NoopClient{}
 	}
+	return &Runner{cfg: cfg, deps: deps, startedAt: time.Now()}
 }
 
-// Run starts the poll loop, blocking until ctx is cancelled.
-// On each tick it claims and executes all available pending tasks.
-// Scrape and audio tasks run in separate goroutine pools bounded by
-// MaxConcurrentScrape and MaxConcurrentAudio respectively.
+// Run starts the poll loop and the metrics HTTP server, blocking until ctx is
+// cancelled.
 func (r *Runner) Run(ctx context.Context) error {
 	r.deps.Log.Info("runner: starting",
 		"worker_id", r.cfg.WorkerID,
 		"poll_interval", r.cfg.PollInterval,
 		"max_scrape", r.cfg.MaxConcurrentScrape,
 		"max_audio", r.cfg.MaxConcurrentAudio,
-		"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
+		"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
+		"metrics_addr", r.cfg.MetricsAddr,
 	)
 
+	// Start metrics HTTP server in background if configured.
+	if r.cfg.MetricsAddr != "" {
+		ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log)
+		go func() {
+			if err := ms.ListenAndServe(ctx); err != nil {
+				r.deps.Log.Error("runner: metrics server error", "err", err)
+			}
+		}()
+	}
+
 	scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
 	audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
 	var wg sync.WaitGroup
 
-	// Write liveness file immediately so the first healthcheck passes before
-	// the first poll completes.
-	r.touchAlive()
-
 	tick := time.NewTicker(r.cfg.PollInterval)
 	defer tick.Stop()
 
-	browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
-	defer browseTick.Stop()
+	catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
+	defer catalogueTick.Stop()
 
-	// Run one browse refresh and one poll immediately on startup.
-	go r.runBrowseRefresh(ctx)
+	// Run one catalogue refresh immediately on startup (unless skipped by flag).
+	if !r.cfg.SkipInitialCatalogueRefresh {
+		go r.runCatalogueRefresh(ctx)
+	} else {
+		r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
+	}
 
 	// Run one poll immediately on startup, then on each tick.
 	for {
 		r.poll(ctx, scrapeSem, audioSem, &wg)
-		r.touchAlive()
 
 		select {
 		case <-ctx.Done():
@@ -169,16 +192,24 @@ func (r *Runner) Run(ctx context.Context) error {
 				r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
 			}
 			return nil
-		case <-browseTick.C:
-			go r.runBrowseRefresh(ctx)
+		case <-catalogueTick.C:
+			go r.runCatalogueRefresh(ctx)
 		case <-tick.C:
 		}
 	}
 }
 
 // poll claims all available pending tasks and dispatches them to goroutines.
-// It claims tasks in a tight loop until no more are available.
 func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
+	// ── Heartbeat file ────────────────────────────────────────────────────
+	// Touch /tmp/runner.alive so the Docker health check can confirm the
+	// runner is actively polling. Failure is non-fatal — just log it.
+	if f, err := os.Create("/tmp/runner.alive"); err != nil {
+		r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
+	} else {
+		f.Close()
+	}
+
 	// ── Reap orphaned tasks ───────────────────────────────────────────────
 	if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
 		r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
@@ -197,23 +228,21 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
 			break
 		}
 		if !ok {
-			break // queue empty
+			break
 		}
-		// Acquire semaphore (non-blocking when full — leave task running).
 		select {
 		case scrapeSem <- struct{}{}:
 		default:
-			// Too many concurrent scrapes — the task stays claimed but we can't
-			// run it right now. Log and break; the next poll will pick it up if
-			// still running (it won't be re-claimed while status=running).
 			r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
 				"task_id", task.ID)
 			break
 		}
+		r.tasksRunning.Add(1)
 		wg.Add(1)
 		go func(t domain.ScrapeTask) {
 			defer wg.Done()
 			defer func() { <-scrapeSem }()
+			defer r.tasksRunning.Add(-1)
 			r.runScrapeTask(ctx, t)
 		}(task)
 	}
@@ -229,7 +258,7 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
 			break
 		}
 		if !ok {
-			break // queue empty
+			break
 		}
 		select {
 		case audioSem <- struct{}{}:
@@ -238,22 +267,36 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
 				"task_id", task.ID)
 			break
 		}
+		r.tasksRunning.Add(1)
 		wg.Add(1)
 		go func(t domain.AudioTask) {
 			defer wg.Done()
 			defer func() { <-audioSem }()
+			defer r.tasksRunning.Add(-1)
 			r.runAudioTask(ctx, t)
 		}(task)
 	}
 }
 
+// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in.
+func (r *Runner) newOrchestrator() *orchestrator.Orchestrator {
+	oCfg := orchestrator.Config{
+		Workers: r.cfg.OrchestratorWorkers,
+		PostMetadata: func(ctx context.Context, meta domain.BookMeta) {
+			if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
+				r.deps.Log.Warn("runner: meilisearch upsert failed",
+					"slug", meta.Slug, "err", err)
+			}
+		},
+	}
+	return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
+}
+
 // runScrapeTask executes one scrape task end-to-end and reports the result.
 func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 	log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
 	log.Info("runner: scrape task starting")
 
-	// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
-	// this task is still alive. Cancelled when the task finishes.
 	hbCtx, hbCancel := context.WithCancel(ctx)
 	defer hbCancel()
 	go func() {
@@ -271,9 +314,7 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 		}
 	}()
 
-	oCfg := orchestrator.Config{Workers: r.cfg.OrchestratorWorkers}
-	o := orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
-
+	o := r.newOrchestrator()
 	var result domain.ScrapeResult
 
 	switch task.Kind {
@@ -289,6 +330,13 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 	if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
 		log.Error("runner: FinishScrapeTask failed", "err", err)
 	}
+
+	if result.ErrorMessage != "" {
+		r.tasksFailed.Add(1)
+	} else {
+		r.tasksCompleted.Add(1)
+	}
+
 	log.Info("runner: scrape task finished",
 		"scraped", result.ChaptersScraped,
 		"skipped", result.ChaptersSkipped,
@@ -296,8 +344,7 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
 	)
 }
 
-// runCatalogueTask runs a full catalogue scrape by iterating catalogue entries
-// and running a book task for each one.
+// runCatalogueTask runs a full catalogue scrape.
 func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
 	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
 	var result domain.ScrapeResult
@@ -328,17 +375,11 @@ func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o
 	return result
 }
 
-// runAudioTask executes one audio-generation task:
-//  1. Read chapter text from MinIO.
-//  2. Call Kokoro to generate audio.
-//  3. Upload MP3 to MinIO under the standard audio object key.
-//  4. Report result back to PocketBase.
+// runAudioTask executes one audio-generation task.
 func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 	log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
 	log.Info("runner: audio task starting")
 
-	// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
-	// this task is still alive. Cancelled when the task finishes.
 	hbCtx, hbCancel := context.WithCancel(ctx)
 	defer hbCancel()
 	go func() {
@@ -358,13 +399,13 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 
 	fail := func(msg string) {
 		log.Error("runner: audio task failed", "reason", msg)
+		r.tasksFailed.Add(1)
 		result := domain.AudioResult{ErrorMessage: msg}
 		if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
 			log.Error("runner: FinishAudioTask failed", "err", err)
 		}
 	}
 
-	// Step 1: read chapter text.
 	raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
 	if err != nil {
 		fail(fmt.Sprintf("read chapter: %v", err))
@@ -376,7 +417,6 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 		return
 	}
 
-	// Step 2: generate audio.
 	if r.deps.Kokoro == nil {
 		fail("kokoro client not configured")
 		return
@@ -387,14 +427,13 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
 		return
 	}
 
-	// Step 3: upload to MinIO.
 	key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
 	if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
 		fail(fmt.Sprintf("put audio: %v", err))
 		return
 	}
 
-	// Step 4: report success.
+	r.tasksCompleted.Add(1)
 	result := domain.AudioResult{ObjectKey: key}
 	if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
 		log.Error("runner: FinishAudioTask failed", "err", err)

backend/internal/runner/runner_test.go

@@ -146,7 +146,7 @@ func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.B
 	return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
 }
 
-func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
+func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
 	return s.chapters, nil
 }