fix(runner): harden catalogue scrape against 429s; disable sourcemap upload

- scraper.go: ScrapeCatalogue now uses retryGet (9 attempts, 10s base) +
  500–1500ms inter-page jitter instead of bare GetContent. ScrapeMetadata
  also switched to retryGet so a single 429 on a book page is retried rather
  than aborting the whole refresh.
- catalogue_refresh.go: per-book delay is now configurable
  (RUNNER_CATALOGUE_REQUEST_DELAY, default 2s) + up to 50% random jitter
  applied before every metadata fetch. Only metadata is scraped here —
  chapters are fetched on-demand, not during catalogue refresh. Progress
  logged every 50 books instead of 100.
- config.go / runner.go / main.go: add CatalogueRequestDelay field wired
  from RUNNER_CATALOGUE_REQUEST_DELAY env var.
- release.yaml: comment out upload-sourcemaps job and remove it from the
  release needs; GlitchTip auth token needs refreshing after DB wipe.

.gitea/workflows/release.yaml

@@ -136,52 +136,51 @@ jobs:
           cache-to: type=inline
 
   # ── ui: source map upload ─────────────────────────────────────────────────────
-  # Builds the UI with source maps and uploads them to GlitchTip so that error
+  # Commented out: GlitchTip project/auth token needs to be recreated after
-  # stack traces resolve to original .svelte/.ts file names and line numbers.
+  # the GlitchTip DB wipe. Re-enable once GLITCHTIP_AUTH_TOKEN is updated.
-  # Runs in parallel with docker-ui (both need check-ui to pass first).
+  # upload-sourcemaps:
-  upload-sourcemaps:
+  #   name: Upload source maps
-    name: Upload source maps
+  #   runs-on: ubuntu-latest
-    runs-on: ubuntu-latest
+  #   needs: [check-ui]
-    needs: [check-ui]
+  #   defaults:
-    defaults:
+  #     run:
-      run:
+  #       working-directory: ui
-        working-directory: ui
+  #   steps:
-    steps:
+  #     - uses: actions/checkout@v4
-      - uses: actions/checkout@v4
+  #
-
+  #     - uses: actions/setup-node@v4
-      - uses: actions/setup-node@v4
+  #       with:
-        with:
+  #         node-version: "22"
-          node-version: "22"
+  #         cache: npm
-          cache: npm
+  #         cache-dependency-path: ui/package-lock.json
-          cache-dependency-path: ui/package-lock.json
+  #
-
+  #     - name: Install dependencies
-      - name: Install dependencies
+  #       run: npm ci
-        run: npm ci
+  #
-
+  #     - name: Build with source maps
-      - name: Build with source maps
+  #       run: npm run build
-        run: npm run build
+  #
-
+  #     - name: Download glitchtip-cli
-      - name: Download glitchtip-cli
+  #       run: |
-        run: |
+  #         curl -L "https://gitlab.com/glitchtip/glitchtip-cli/-/jobs/artifacts/v0.1.0/raw/artifacts/glitchtip-cli-linux-x86_64?job=build-linux-x86_64" \
-          curl -L "https://gitlab.com/glitchtip/glitchtip-cli/-/jobs/artifacts/v0.1.0/raw/artifacts/glitchtip-cli-linux-x86_64?job=build-linux-x86_64" \
+  #           -o /usr/local/bin/glitchtip-cli
-            -o /usr/local/bin/glitchtip-cli
+  #         chmod +x /usr/local/bin/glitchtip-cli
-          chmod +x /usr/local/bin/glitchtip-cli
+  #
-
+  #     - name: Inject debug IDs into build artifacts
-      - name: Inject debug IDs into build artifacts
+  #       run: glitchtip-cli sourcemaps inject ./build
-        run: glitchtip-cli sourcemaps inject ./build
+  #       env:
-        env:
+  #         SENTRY_URL: https://errors.libnovel.cc/
-          SENTRY_URL: https://errors.libnovel.cc/
+  #         SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
+  #         SENTRY_ORG: libnovel
-          SENTRY_ORG: libnovel
+  #         SENTRY_PROJECT: libnovel-ui
-          SENTRY_PROJECT: libnovel-ui
+  #
-
+  #     - name: Upload source maps to GlitchTip
-      - name: Upload source maps to GlitchTip
+  #       run: glitchtip-cli sourcemaps upload ./build --release ${{ gitea.ref_name }}
-        run: glitchtip-cli sourcemaps upload ./build --release ${{ gitea.ref_name }}
+  #       env:
-        env:
+  #         SENTRY_URL: https://errors.libnovel.cc/
-          SENTRY_URL: https://errors.libnovel.cc/
+  #         SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
+  #         SENTRY_ORG: libnovel
-          SENTRY_ORG: libnovel
+  #         SENTRY_PROJECT: libnovel-ui
-          SENTRY_PROJECT: libnovel-ui
 
   # ── docker: ui ────────────────────────────────────────────────────────────────
   docker-ui:
@@ -261,7 +260,7 @@ jobs:
   release:
     name: Gitea Release
     runs-on: ubuntu-latest
-    needs: [docker-backend, docker-runner, docker-ui, docker-caddy, upload-sourcemaps]
+    needs: [docker-backend, docker-runner, docker-ui, docker-caddy]
     steps:
       - uses: actions/checkout@v4
         with:

backend/cmd/runner/main.go

@@ -152,6 +152,7 @@ func run() error {
 		OrchestratorWorkers:         workers,
 		MetricsAddr:                 cfg.Runner.MetricsAddr,
 		CatalogueRefreshInterval:    cfg.Runner.CatalogueRefreshInterval,
+		CatalogueRequestDelay:       cfg.Runner.CatalogueRequestDelay,
 		SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
 		RedisAddr:                   cfg.Redis.Addr,
 		RedisPassword:               cfg.Redis.Password,

backend/internal/config/config.go

@@ -126,6 +126,11 @@ type Runner struct {
 	// is already indexed and a 24h walk would be wasteful.
 	// Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true.
 	SkipInitialCatalogueRefresh bool
+	// CatalogueRequestDelay is the base delay inserted between per-book metadata
+	// requests during a catalogue refresh. A random jitter of up to 50% is added
+	// on top. Defaults to 2s. Increase to reduce 429 pressure on novelfire.net.
+	// Controlled by RUNNER_CATALOGUE_REQUEST_DELAY (e.g. "3s", "500ms").
+	CatalogueRequestDelay time.Duration
 }
 
 // Config is the top-level configuration struct consumed by both binaries.
@@ -196,6 +201,7 @@ func Load() Config {
 			MetricsAddr:                 envOr("RUNNER_METRICS_ADDR", ":9091"),
 			CatalogueRefreshInterval:    envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
 			SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false),
+			CatalogueRequestDelay:       envDuration("RUNNER_CATALOGUE_REQUEST_DELAY", 2*time.Second),
 		},
 
 		Meilisearch: Meilisearch{

backend/internal/novelfire/scraper.go

@@ -13,6 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"math/rand"
 	"net/url"
 	"path"
 	"strconv"
@@ -55,6 +56,9 @@ func (s *Scraper) SourceName() string { return "novelfire.net" }
 // ── CatalogueProvider ─────────────────────────────────────────────────────────
 
 // ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
+// Each page fetch uses retryGet with 429-aware exponential backoff.
+// A small inter-page delay (cataloguePageDelay) is inserted between requests to
+// avoid hammering the server when paging through hundreds of catalogue pages.
 func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
 	entries := make(chan domain.CatalogueEntry, 64)
 	errs := make(chan error, 16)
@@ -73,8 +77,18 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
 			default:
 			}
 
+			// Polite inter-page delay — skipped on the very first page.
+			if page > 1 {
+				jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond
+				select {
+				case <-ctx.Done():
+					return
+				case <-time.After(jitter):
+				}
+			}
+
 			s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
-			raw, err := s.client.GetContent(ctx, pageURL)
+			raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second)
 			if err != nil {
 				errs <- fmt.Errorf("catalogue page %d: %w", page, err)
 				return
@@ -139,10 +153,11 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
 // ── MetadataProvider ──────────────────────────────────────────────────────────
 
 // ScrapeMetadata fetches and parses book metadata from the book's landing page.
+// Uses retryGet with 429-aware exponential backoff (up to 9 attempts).
 func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
 	s.log.Debug("metadata fetch starting", "url", bookURL)
 
-	raw, err := s.client.GetContent(ctx, bookURL)
+	raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second)
 	if err != nil {
 		return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
 	}

backend/internal/runner/catalogue_refresh.go

@@ -6,17 +6,20 @@ package runner
 //
 // Design:
 //   - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
-//   - Also fires once on startup.
+//   - Also fires once on startup (unless SkipInitialCatalogueRefresh is set).
-//   - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate
+//   - ScrapeCatalogue streams CatalogueEntry values over a channel — already has
-//     and call ScrapeMetadata for each entry.
+//     its own inter-page jitter + retryGet (see scraper.go).
-//   - Per-request random jitter (1–3s) prevents hammering novelfire.net.
+//   - Per-book: only metadata is scraped here (not chapters). Chapters are scraped
-//   - Cover images are fetched from the URL embedded in BookMeta.Cover and
+//     on-demand when a user opens a book or via an explicit scrape task.
-//     stored in MinIO (browse bucket, key: covers/{slug}.jpg).
+//   - Between each metadata request a configurable base delay plus up to 50%
-//   - WriteMetadata + UpsertBook are called for every successfully scraped book.
+//     random jitter is applied (CatalogueRequestDelay, default 2s). This keeps
-//   - Errors for individual books are logged and skipped; the loop continues.
+//     the request rate well below novelfire.net's rate limit even for ~15k books.
-//   - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy
+//   - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff
-//     path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the
+//     (up to 9 attempts), so transient rate limits are handled gracefully.
-//     backend, which will serve from MinIO.
+//   - Cover images are fetched and stored in MinIO on first sight; subsequent
+//     refreshes skip covers that already exist (CoverExists check).
+//   - Books already present in Meilisearch are skipped entirely (fast path).
+//   - Errors for individual books are logged and skipped; the loop never aborts.
 
 import (
 	"context"
@@ -29,7 +32,7 @@ import (
 
 // runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
 // every book on novelfire.net, downloads covers to MinIO, and upserts to
-// Meilisearch. Errors for individual books are logged and skipped.
+// Meilisearch. Individual book failures are logged and skipped.
 func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 	if r.deps.Novel == nil {
 		r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
@@ -40,8 +43,9 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 		return
 	}
 
+	delay := r.cfg.CatalogueRequestDelay
 	log := r.deps.Log.With("op", "catalogue_refresh")
-	log.Info("runner: catalogue refresh starting")
+	log.Info("runner: catalogue refresh starting", "request_delay", delay)
 
 	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
 
@@ -51,26 +55,26 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 			break
 		}
 
-		// Skip books already present in Meilisearch — they were indexed on a
+		// Fast path: skip books already indexed in Meilisearch.
-		// previous run. Re-indexing only happens when a scrape task is
-		// explicitly enqueued (e.g. via the admin UI or API).
 		if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
 			skipped++
 			continue
 		}
 
-		// Random jitter between books to avoid rate-limiting.
+		// Polite delay between metadata requests: base + up to 50% jitter.
-		jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
+		// This applies before every fetch so we never fire bursts.
+		jitter := time.Duration(rand.Int63n(int64(delay / 2)))
 		select {
 		case <-ctx.Done():
 			break
-		case <-time.After(jitter):
+		case <-time.After(delay + jitter):
 		}
 
+		// ScrapeMetadata internally retries on 429 with exponential back-off.
 		meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
 		if err != nil {
-			log.Warn("runner: catalogue refresh: metadata scrape failed",
+			log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book",
-				"url", entry.URL, "err", err)
+				"slug", entry.Slug, "url", entry.URL, "err", err)
 			errCount++
 			continue
 		}
@@ -81,35 +85,32 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) {
 
 		// Persist to PocketBase.
 		if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
-			log.Warn("runner: catalogue refresh: WriteMetadata failed",
+			log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book",
 				"slug", meta.Slug, "err", err)
 			errCount++
 			continue
 		}
 
-		// Index in Meilisearch.
+		// Index in Meilisearch (non-fatal).
 		if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
 			log.Warn("runner: catalogue refresh: UpsertBook failed",
 				"slug", meta.Slug, "err", err)
-			// non-fatal — continue
 		}
 
-		// Download and store cover image in MinIO if we have a cover URL
+		// Download cover to MinIO if not already cached (non-fatal).
-		// and a CoverStore is wired in.
 		if r.deps.CoverStore != nil && originalCover != "" {
 			if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
 				if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
 					log.Warn("runner: catalogue refresh: cover download failed",
 						"slug", meta.Slug, "url", originalCover, "err", err)
-					// non-fatal
 				}
 			}
 		}
 
 		ok++
-		if ok%100 == 0 {
+		if ok%50 == 0 {
 			log.Info("runner: catalogue refresh progress",
-				"scraped", ok, "errors", errCount)
+				"scraped", ok, "skipped", skipped, "errors", errCount)
 		}
 	}
 

backend/internal/runner/runner.go

@@ -62,6 +62,10 @@ type Config struct {
 	// scrapes per-book metadata, downloads covers, and re-indexes everything in
 	// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
 	CatalogueRefreshInterval time.Duration
+	// CatalogueRequestDelay is the base inter-request pause during a catalogue
+	// refresh metadata walk. Jitter of up to 50% is added on top.
+	// Defaults to 2s. Set via RUNNER_CATALOGUE_REQUEST_DELAY.
+	CatalogueRequestDelay time.Duration
 	// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
 	// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
 	// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
@@ -145,6 +149,9 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.CatalogueRefreshInterval <= 0 {
 		cfg.CatalogueRefreshInterval = 24 * time.Hour
 	}
+	if cfg.CatalogueRequestDelay <= 0 {
+		cfg.CatalogueRequestDelay = 2 * time.Second
+	}
 	if cfg.MetricsAddr == "" {
 		cfg.MetricsAddr = ":9091"
 	}