fix(runner): harden catalogue scrape against 429s; disable sourcemap upload

- scraper.go: ScrapeCatalogue now uses retryGet (9 attempts, 10s base) +
  500–1500ms inter-page jitter instead of bare GetContent. ScrapeMetadata
  also switched to retryGet so a single 429 on a book page is retried rather
  than aborting the whole refresh.
- catalogue_refresh.go: per-book delay is now configurable
  (RUNNER_CATALOGUE_REQUEST_DELAY, default 2s) + up to 50% random jitter
  applied before every metadata fetch. Only metadata is scraped here —
  chapters are fetched on-demand, not during catalogue refresh. Progress
  logged every 50 books instead of 100.
- config.go / runner.go / main.go: add CatalogueRequestDelay field wired
  from RUNNER_CATALOGUE_REQUEST_DELAY env var.
- release.yaml: comment out upload-sourcemaps job and remove it from the
  release needs; GlitchTip auth token needs refreshing after DB wipe.

backend/internal/config/config.go

@@ -126,6 +126,11 @@ type Runner struct {
 	// is already indexed and a 24h walk would be wasteful.
 	// Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true.
 	SkipInitialCatalogueRefresh bool
+	// CatalogueRequestDelay is the base delay inserted between per-book metadata
+	// requests during a catalogue refresh. A random jitter of up to 50% is added
+	// on top. Defaults to 2s. Increase to reduce 429 pressure on novelfire.net.
+	// Controlled by RUNNER_CATALOGUE_REQUEST_DELAY (e.g. "3s", "500ms").
+	CatalogueRequestDelay time.Duration
 }
 
 // Config is the top-level configuration struct consumed by both binaries.
@@ -196,6 +201,7 @@ func Load() Config {
 			MetricsAddr:                 envOr("RUNNER_METRICS_ADDR", ":9091"),
 			CatalogueRefreshInterval:    envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
 			SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false),
+			CatalogueRequestDelay:       envDuration("RUNNER_CATALOGUE_REQUEST_DELAY", 2*time.Second),
 		},
 
 		Meilisearch: Meilisearch{

backend/internal/novelfire/scraper.go

@@ -13,6 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"math/rand"
 	"net/url"
 	"path"
 	"strconv"
@@ -55,6 +56,9 @@ func (s *Scraper) SourceName() string { return "novelfire.net" }
 // ── CatalogueProvider ─────────────────────────────────────────────────────────
 
 // ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
+// Each page fetch uses retryGet with 429-aware exponential backoff.
+// A small inter-page delay (cataloguePageDelay) is inserted between requests to
+// avoid hammering the server when paging through hundreds of catalogue pages.
 func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
 	entries := make(chan domain.CatalogueEntry, 64)
 	errs := make(chan error, 16)
@@ -73,8 +77,18 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
 			default:
 			}
 
+			// Polite inter-page delay — skipped on the very first page.
+			if page > 1 {
+				jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond
+				select {
+				case <-ctx.Done():
+					return
+				case <-time.After(jitter):
+				}
+			}
+
 			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
-			raw, err := s.client.GetContent(ctx, pageURL)
+			raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second)
 			if err != nil {
 				errs <- fmt.Errorf("catalogue page %d: %w", page, err)
 				return
@@ -139,10 +153,11 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
 // ── MetadataProvider ──────────────────────────────────────────────────────────
 
 // ScrapeMetadata fetches and parses book metadata from the book's landing page.
+// Uses retryGet with 429-aware exponential backoff (up to 9 attempts).
 func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
 	s.log.Debug("metadata fetch starting", "url", bookURL)
 
-	raw, err := s.client.GetContent(ctx, bookURL)
+	raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second)
 	if err != nil {
 		return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
 	}

backend/internal/runner/catalogue_refresh.go

@@ -6,17 +6,20 @@ package runner
 //
 // Design:
 //   - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
-//   - Also fires once on startup.
-//   - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate
-//     and call ScrapeMetadata for each entry.
-//   - Per-request random jitter (1–3s) prevents hammering novelfire.net.
-//   - Cover images are fetched from the URL embedded in BookMeta.Cover and
-//     stored in MinIO (browse bucket, key: covers/{slug}.jpg).
-//   - WriteMetadata + UpsertBook are called for every successfully scraped book.
-//   - Errors for individual books are logged and skipped; the loop continues.
-//   - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy
-//     path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the
-//     backend, which will serve from MinIO.
+//   - Also fires once on startup (unless SkipInitialCatalogueRefresh is set).
+//   - ScrapeCatalogue streams CatalogueEntry values over a channel — already has
+//     its own inter-page jitter + retryGet (see scraper.go).
+//   - Per-book: only metadata is scraped here (not chapters). Chapters are scraped
+//     on-demand when a user opens a book or via an explicit scrape task.
+//   - Between each metadata request a configurable base delay plus up to 50%
+//     random jitter is applied (CatalogueRequestDelay, default 2s). This keeps
+//     the request rate well below novelfire.net's rate limit even for ~15k books.
+//   - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff
+//     (up to 9 attempts), so transient rate limits are handled gracefully.
+//   - Cover images are fetched and stored in MinIO on first sight; subsequent
+//     refreshes skip covers that already exist (CoverExists check).
+//   - Books already present in Meilisearch are skipped entirely (fast path).
+//   - Errors for individual books are logged and skipped; the loop never aborts.
 
 import (
 	"context"
@@ -29,7 +32,7 @@ import (
 
 // runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
 // every book on novelfire.net, downloads covers to MinIO, and upserts to
-// Meilisearch. Errors for individual books are logged and skipped.
+// Meilisearch. Individual book failures are logged and skipped.
 func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 	if r.deps.Novel == nil {
 		r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
@@ -40,8 +43,9 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 		return
 	}
 
+	delay := r.cfg.CatalogueRequestDelay
 	log := r.deps.Log.With("op", "catalogue_refresh")
-	log.Info("runner: catalogue refresh starting")
+	log.Info("runner: catalogue refresh starting", "request_delay", delay)
 
 	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
 
@@ -51,26 +55,26 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 			break
 		}
 
-		// Skip books already present in Meilisearch — they were indexed on a
-		// previous run. Re-indexing only happens when a scrape task is
-		// explicitly enqueued (e.g. via the admin UI or API).
+		// Fast path: skip books already indexed in Meilisearch.
 		if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
 			skipped++
 			continue
 		}
 
-		// Random jitter between books to avoid rate-limiting.
-		jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
+		// Polite delay between metadata requests: base + up to 50% jitter.
+		// This applies before every fetch so we never fire bursts.
+		jitter := time.Duration(rand.Int63n(int64(delay / 2)))
 		select {
 		case <-ctx.Done():
 			break
-		case <-time.After(jitter):
+		case <-time.After(delay + jitter):
 		}
 
+		// ScrapeMetadata internally retries on 429 with exponential back-off.
 		meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
 		if err != nil {
-			log.Warn("runner: catalogue refresh: metadata scrape failed",
-				"url", entry.URL, "err", err)
+			log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book",
+				"slug", entry.Slug, "url", entry.URL, "err", err)
 			errCount++
 			continue
 		}
@@ -81,35 +85,32 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 
 		// Persist to PocketBase.
 		if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
-			log.Warn("runner: catalogue refresh: WriteMetadata failed",
+			log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book",
 				"slug", meta.Slug, "err", err)
 			errCount++
 			continue
 		}
 
-		// Index in Meilisearch.
+		// Index in Meilisearch (non-fatal).
 		if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
 			log.Warn("runner: catalogue refresh: UpsertBook failed",
 				"slug", meta.Slug, "err", err)
-			// non-fatal — continue
 		}
 
-		// Download and store cover image in MinIO if we have a cover URL
-		// and a CoverStore is wired in.
+		// Download cover to MinIO if not already cached (non-fatal).
 		if r.deps.CoverStore != nil && originalCover != "" {
 			if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
 				if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
 					log.Warn("runner: catalogue refresh: cover download failed",
 						"slug", meta.Slug, "url", originalCover, "err", err)
-					// non-fatal
 				}
 			}
 		}
 
 		ok++
-		if ok%100 == 0 {
+		if ok%50 == 0 {
 			log.Info("runner: catalogue refresh progress",
-				"scraped", ok, "errors", errCount)
+				"scraped", ok, "skipped", skipped, "errors", errCount)
 		}
 	}
 

backend/internal/runner/runner.go

@@ -62,6 +62,10 @@ type Config struct {
 	// scrapes per-book metadata, downloads covers, and re-indexes everything in
 	// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
 	CatalogueRefreshInterval time.Duration
+	// CatalogueRequestDelay is the base inter-request pause during a catalogue
+	// refresh metadata walk. Jitter of up to 50% is added on top.
+	// Defaults to 2s. Set via RUNNER_CATALOGUE_REQUEST_DELAY.
+	CatalogueRequestDelay time.Duration
 	// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
 	// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
 	// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
@@ -145,6 +149,9 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.CatalogueRefreshInterval <= 0 {
 		cfg.CatalogueRefreshInterval = 24 * time.Hour
 	}
+	if cfg.CatalogueRequestDelay <= 0 {
+		cfg.CatalogueRequestDelay = 2 * time.Second
+	}
 	if cfg.MetricsAddr == "" {
 		cfg.MetricsAddr = ":9091"
 	}