fix(runner): harden catalogue scrape against 429s; disable sourcemap upload

- scraper.go: ScrapeCatalogue now uses retryGet (9 attempts, 10s base) +
  500–1500ms inter-page jitter instead of bare GetContent. ScrapeMetadata
  also switched to retryGet so a single 429 on a book page is retried rather
  than aborting the whole refresh.
- catalogue_refresh.go: per-book delay is now configurable
  (RUNNER_CATALOGUE_REQUEST_DELAY, default 2s) + up to 50% random jitter
  applied before every metadata fetch. Only metadata is scraped here —
  chapters are fetched on-demand, not during catalogue refresh. Progress
  logged every 50 books instead of 100.
- config.go / runner.go / main.go: add CatalogueRequestDelay field wired
  from RUNNER_CATALOGUE_REQUEST_DELAY env var.
- release.yaml: comment out upload-sourcemaps job and remove it from the
  release needs; GlitchTip auth token needs refreshing after DB wipe.

backend/internal/runner/catalogue_refresh.go

@@ -6,17 +6,20 @@ package runner
 //
 // Design:
 //   - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
-//   - Also fires once on startup.
-//   - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate
-//     and call ScrapeMetadata for each entry.
-//   - Per-request random jitter (1–3s) prevents hammering novelfire.net.
-//   - Cover images are fetched from the URL embedded in BookMeta.Cover and
-//     stored in MinIO (browse bucket, key: covers/{slug}.jpg).
-//   - WriteMetadata + UpsertBook are called for every successfully scraped book.
-//   - Errors for individual books are logged and skipped; the loop continues.
-//   - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy
-//     path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the
-//     backend, which will serve from MinIO.
+//   - Also fires once on startup (unless SkipInitialCatalogueRefresh is set).
+//   - ScrapeCatalogue streams CatalogueEntry values over a channel — already has
+//     its own inter-page jitter + retryGet (see scraper.go).
+//   - Per-book: only metadata is scraped here (not chapters). Chapters are scraped
+//     on-demand when a user opens a book or via an explicit scrape task.
+//   - Between each metadata request a configurable base delay plus up to 50%
+//     random jitter is applied (CatalogueRequestDelay, default 2s). This keeps
+//     the request rate well below novelfire.net's rate limit even for ~15k books.
+//   - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff
+//     (up to 9 attempts), so transient rate limits are handled gracefully.
+//   - Cover images are fetched and stored in MinIO on first sight; subsequent
+//     refreshes skip covers that already exist (CoverExists check).
+//   - Books already present in Meilisearch are skipped entirely (fast path).
+//   - Errors for individual books are logged and skipped; the loop never aborts.
 
 import (
 	"context"
@@ -29,7 +32,7 @@ import (
 
 // runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
 // every book on novelfire.net, downloads covers to MinIO, and upserts to
-// Meilisearch. Errors for individual books are logged and skipped.
+// Meilisearch. Individual book failures are logged and skipped.
 func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 	if r.deps.Novel == nil {
 		r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
@@ -40,8 +43,9 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 		return
 	}
 
+	delay := r.cfg.CatalogueRequestDelay
 	log := r.deps.Log.With("op", "catalogue_refresh")
-	log.Info("runner: catalogue refresh starting")
+	log.Info("runner: catalogue refresh starting", "request_delay", delay)
 
 	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
 
@@ -51,26 +55,26 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 			break
 		}
 
-		// Skip books already present in Meilisearch — they were indexed on a
-		// previous run. Re-indexing only happens when a scrape task is
-		// explicitly enqueued (e.g. via the admin UI or API).
+		// Fast path: skip books already indexed in Meilisearch.
 		if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
 			skipped++
 			continue
 		}
 
-		// Random jitter between books to avoid rate-limiting.
-		jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
+		// Polite delay between metadata requests: base + up to 50% jitter.
+		// This applies before every fetch so we never fire bursts.
+		jitter := time.Duration(rand.Int63n(int64(delay / 2)))
 		select {
 		case <-ctx.Done():
 			break
-		case <-time.After(jitter):
+		case <-time.After(delay + jitter):
 		}
 
+		// ScrapeMetadata internally retries on 429 with exponential back-off.
 		meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
 		if err != nil {
-			log.Warn("runner: catalogue refresh: metadata scrape failed",
-				"url", entry.URL, "err", err)
+			log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book",
+				"slug", entry.Slug, "url", entry.URL, "err", err)
 			errCount++
 			continue
 		}
@@ -81,35 +85,32 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 
 		// Persist to PocketBase.
 		if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
-			log.Warn("runner: catalogue refresh: WriteMetadata failed",
+			log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book",
 				"slug", meta.Slug, "err", err)
 			errCount++
 			continue
 		}
 
-		// Index in Meilisearch.
+		// Index in Meilisearch (non-fatal).
 		if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
 			log.Warn("runner: catalogue refresh: UpsertBook failed",
 				"slug", meta.Slug, "err", err)
-			// non-fatal — continue
 		}
 
-		// Download and store cover image in MinIO if we have a cover URL
-		// and a CoverStore is wired in.
+		// Download cover to MinIO if not already cached (non-fatal).
 		if r.deps.CoverStore != nil && originalCover != "" {
 			if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
 				if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
 					log.Warn("runner: catalogue refresh: cover download failed",
 						"slug", meta.Slug, "url", originalCover, "err", err)
-					// non-fatal
 				}
 			}
 		}
 
 		ok++
-		if ok%100 == 0 {
+		if ok%50 == 0 {
 			log.Info("runner: catalogue refresh progress",
-				"scraped", ok, "errors", errCount)
+				"scraped", ok, "skipped", skipped, "errors", errCount)
 		}
 	}
 

backend/internal/runner/runner.go

@@ -62,6 +62,10 @@ type Config struct {
 	// scrapes per-book metadata, downloads covers, and re-indexes everything in
 	// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
 	CatalogueRefreshInterval time.Duration
+	// CatalogueRequestDelay is the base inter-request pause during a catalogue
+	// refresh metadata walk. Jitter of up to 50% is added on top.
+	// Defaults to 2s. Set via RUNNER_CATALOGUE_REQUEST_DELAY.
+	CatalogueRequestDelay time.Duration
 	// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
 	// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
 	// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
@@ -145,6 +149,9 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.CatalogueRefreshInterval <= 0 {
 		cfg.CatalogueRefreshInterval = 24 * time.Hour
 	}
+	if cfg.CatalogueRequestDelay <= 0 {
+		cfg.CatalogueRequestDelay = 2 * time.Second
+	}
 	if cfg.MetricsAddr == "" {
 		cfg.MetricsAddr = ":9091"
 	}