From aef9e04419ab40cba02d1aa3febbe47a2499b8c4 Mon Sep 17 00:00:00 2001 From: Admin Date: Sat, 28 Mar 2026 16:28:36 +0500 Subject: [PATCH] fix(runner): harden catalogue scrape against 429s; disable sourcemap upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scraper.go: ScrapeCatalogue now uses retryGet (9 attempts, 10s base) + 500–1500ms inter-page jitter instead of bare GetContent. ScrapeMetadata also switched to retryGet so a single 429 on a book page is retried rather than aborting the whole refresh. - catalogue_refresh.go: per-book delay is now configurable (RUNNER_CATALOGUE_REQUEST_DELAY, default 2s) + up to 50% random jitter applied before every metadata fetch. Only metadata is scraped here — chapters are fetched on-demand, not during catalogue refresh. Progress logged every 50 books instead of 100. - config.go / runner.go / main.go: add CatalogueRequestDelay field wired from RUNNER_CATALOGUE_REQUEST_DELAY env var. - release.yaml: comment out upload-sourcemaps job and remove it from the release needs; GlitchTip auth token needs refreshing after DB wipe. --- .gitea/workflows/release.yaml | 93 ++++++++++---------- backend/cmd/runner/main.go | 1 + backend/internal/config/config.go | 6 ++ backend/internal/novelfire/scraper.go | 19 +++- backend/internal/runner/catalogue_refresh.go | 59 +++++++------ backend/internal/runner/runner.go | 7 ++ 6 files changed, 107 insertions(+), 78 deletions(-) diff --git a/.gitea/workflows/release.yaml b/.gitea/workflows/release.yaml index 00a56e5..5f1efd9 100644 --- a/.gitea/workflows/release.yaml +++ b/.gitea/workflows/release.yaml @@ -136,52 +136,51 @@ jobs: cache-to: type=inline # ── ui: source map upload ───────────────────────────────────────────────────── - # Builds the UI with source maps and uploads them to GlitchTip so that error - # stack traces resolve to original .svelte/.ts file names and line numbers. - # Runs in parallel with docker-ui (both need check-ui to pass first). - upload-sourcemaps: - name: Upload source maps - runs-on: ubuntu-latest - needs: [check-ui] - defaults: - run: - working-directory: ui - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: "22" - cache: npm - cache-dependency-path: ui/package-lock.json - - - name: Install dependencies - run: npm ci - - - name: Build with source maps - run: npm run build - - - name: Download glitchtip-cli - run: | - curl -L "https://gitlab.com/glitchtip/glitchtip-cli/-/jobs/artifacts/v0.1.0/raw/artifacts/glitchtip-cli-linux-x86_64?job=build-linux-x86_64" \ - -o /usr/local/bin/glitchtip-cli - chmod +x /usr/local/bin/glitchtip-cli - - - name: Inject debug IDs into build artifacts - run: glitchtip-cli sourcemaps inject ./build - env: - SENTRY_URL: https://errors.libnovel.cc/ - SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }} - SENTRY_ORG: libnovel - SENTRY_PROJECT: libnovel-ui - - - name: Upload source maps to GlitchTip - run: glitchtip-cli sourcemaps upload ./build --release ${{ gitea.ref_name }} - env: - SENTRY_URL: https://errors.libnovel.cc/ - SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }} - SENTRY_ORG: libnovel - SENTRY_PROJECT: libnovel-ui + # Commented out: GlitchTip project/auth token needs to be recreated after + # the GlitchTip DB wipe. Re-enable once GLITCHTIP_AUTH_TOKEN is updated. + # upload-sourcemaps: + # name: Upload source maps + # runs-on: ubuntu-latest + # needs: [check-ui] + # defaults: + # run: + # working-directory: ui + # steps: + # - uses: actions/checkout@v4 + # + # - uses: actions/setup-node@v4 + # with: + # node-version: "22" + # cache: npm + # cache-dependency-path: ui/package-lock.json + # + # - name: Install dependencies + # run: npm ci + # + # - name: Build with source maps + # run: npm run build + # + # - name: Download glitchtip-cli + # run: | + # curl -L "https://gitlab.com/glitchtip/glitchtip-cli/-/jobs/artifacts/v0.1.0/raw/artifacts/glitchtip-cli-linux-x86_64?job=build-linux-x86_64" \ + # -o /usr/local/bin/glitchtip-cli + # chmod +x /usr/local/bin/glitchtip-cli + # + # - name: Inject debug IDs into build artifacts + # run: glitchtip-cli sourcemaps inject ./build + # env: + # SENTRY_URL: https://errors.libnovel.cc/ + # SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }} + # SENTRY_ORG: libnovel + # SENTRY_PROJECT: libnovel-ui + # + # - name: Upload source maps to GlitchTip + # run: glitchtip-cli sourcemaps upload ./build --release ${{ gitea.ref_name }} + # env: + # SENTRY_URL: https://errors.libnovel.cc/ + # SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }} + # SENTRY_ORG: libnovel + # SENTRY_PROJECT: libnovel-ui # ── docker: ui ──────────────────────────────────────────────────────────────── docker-ui: @@ -261,7 +260,7 @@ jobs: release: name: Gitea Release runs-on: ubuntu-latest - needs: [docker-backend, docker-runner, docker-ui, docker-caddy, upload-sourcemaps] + needs: [docker-backend, docker-runner, docker-ui, docker-caddy] steps: - uses: actions/checkout@v4 with: diff --git a/backend/cmd/runner/main.go b/backend/cmd/runner/main.go index 8dbc561..4c68bad 100644 --- a/backend/cmd/runner/main.go +++ b/backend/cmd/runner/main.go @@ -152,6 +152,7 @@ func run() error { OrchestratorWorkers: workers, MetricsAddr: cfg.Runner.MetricsAddr, CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, + CatalogueRequestDelay: cfg.Runner.CatalogueRequestDelay, SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh, RedisAddr: cfg.Redis.Addr, RedisPassword: cfg.Redis.Password, diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 03457ec..998da9f 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -126,6 +126,11 @@ type Runner struct { // is already indexed and a 24h walk would be wasteful. // Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true. SkipInitialCatalogueRefresh bool + // CatalogueRequestDelay is the base delay inserted between per-book metadata + // requests during a catalogue refresh. A random jitter of up to 50% is added + // on top. Defaults to 2s. Increase to reduce 429 pressure on novelfire.net. + // Controlled by RUNNER_CATALOGUE_REQUEST_DELAY (e.g. "3s", "500ms"). + CatalogueRequestDelay time.Duration } // Config is the top-level configuration struct consumed by both binaries. @@ -196,6 +201,7 @@ func Load() Config { MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"), CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0), SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false), + CatalogueRequestDelay: envDuration("RUNNER_CATALOGUE_REQUEST_DELAY", 2*time.Second), }, Meilisearch: Meilisearch{ diff --git a/backend/internal/novelfire/scraper.go b/backend/internal/novelfire/scraper.go index b089677..8b53944 100644 --- a/backend/internal/novelfire/scraper.go +++ b/backend/internal/novelfire/scraper.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "log/slog" + "math/rand" "net/url" "path" "strconv" @@ -55,6 +56,9 @@ func (s *Scraper) SourceName() string { return "novelfire.net" } // ── CatalogueProvider ───────────────────────────────────────────────────────── // ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages. +// Each page fetch uses retryGet with 429-aware exponential backoff. +// A small inter-page delay (cataloguePageDelay) is inserted between requests to +// avoid hammering the server when paging through hundreds of catalogue pages. func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) { entries := make(chan domain.CatalogueEntry, 64) errs := make(chan error, 16) @@ -73,8 +77,18 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE default: } + // Polite inter-page delay — skipped on the very first page. + if page > 1 { + jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond + select { + case <-ctx.Done(): + return + case <-time.After(jitter): + } + } + s.log.Info("scraping catalogue page", "page", page, "url", pageURL) - raw, err := s.client.GetContent(ctx, pageURL) + raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second) if err != nil { errs <- fmt.Errorf("catalogue page %d: %w", page, err) return @@ -139,10 +153,11 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE // ── MetadataProvider ────────────────────────────────────────────────────────── // ScrapeMetadata fetches and parses book metadata from the book's landing page. +// Uses retryGet with 429-aware exponential backoff (up to 9 attempts). func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) { s.log.Debug("metadata fetch starting", "url", bookURL) - raw, err := s.client.GetContent(ctx, bookURL) + raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second) if err != nil { return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err) } diff --git a/backend/internal/runner/catalogue_refresh.go b/backend/internal/runner/catalogue_refresh.go index df388e1..6e2bea5 100644 --- a/backend/internal/runner/catalogue_refresh.go +++ b/backend/internal/runner/catalogue_refresh.go @@ -6,17 +6,20 @@ package runner // // Design: // - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run(). -// - Also fires once on startup. -// - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate -// and call ScrapeMetadata for each entry. -// - Per-request random jitter (1–3s) prevents hammering novelfire.net. -// - Cover images are fetched from the URL embedded in BookMeta.Cover and -// stored in MinIO (browse bucket, key: covers/{slug}.jpg). -// - WriteMetadata + UpsertBook are called for every successfully scraped book. -// - Errors for individual books are logged and skipped; the loop continues. -// - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy -// path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the -// backend, which will serve from MinIO. +// - Also fires once on startup (unless SkipInitialCatalogueRefresh is set). +// - ScrapeCatalogue streams CatalogueEntry values over a channel — already has +// its own inter-page jitter + retryGet (see scraper.go). +// - Per-book: only metadata is scraped here (not chapters). Chapters are scraped +// on-demand when a user opens a book or via an explicit scrape task. +// - Between each metadata request a configurable base delay plus up to 50% +// random jitter is applied (CatalogueRequestDelay, default 2s). This keeps +// the request rate well below novelfire.net's rate limit even for ~15k books. +// - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff +// (up to 9 attempts), so transient rate limits are handled gracefully. +// - Cover images are fetched and stored in MinIO on first sight; subsequent +// refreshes skip covers that already exist (CoverExists check). +// - Books already present in Meilisearch are skipped entirely (fast path). +// - Errors for individual books are logged and skipped; the loop never aborts. import ( "context" @@ -29,7 +32,7 @@ import ( // runCatalogueRefresh performs one full catalogue walk: scrapes metadata for // every book on novelfire.net, downloads covers to MinIO, and upserts to -// Meilisearch. Errors for individual books are logged and skipped. +// Meilisearch. Individual book failures are logged and skipped. func (r *Runner) runCatalogueRefresh(ctx context.Context) { if r.deps.Novel == nil { r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured") @@ -40,8 +43,9 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) { return } + delay := r.cfg.CatalogueRequestDelay log := r.deps.Log.With("op", "catalogue_refresh") - log.Info("runner: catalogue refresh starting") + log.Info("runner: catalogue refresh starting", "request_delay", delay) entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx) @@ -51,26 +55,26 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) { break } - // Skip books already present in Meilisearch — they were indexed on a - // previous run. Re-indexing only happens when a scrape task is - // explicitly enqueued (e.g. via the admin UI or API). + // Fast path: skip books already indexed in Meilisearch. if r.deps.SearchIndex.BookExists(ctx, entry.Slug) { skipped++ continue } - // Random jitter between books to avoid rate-limiting. - jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond + // Polite delay between metadata requests: base + up to 50% jitter. + // This applies before every fetch so we never fire bursts. + jitter := time.Duration(rand.Int63n(int64(delay / 2))) select { case <-ctx.Done(): break - case <-time.After(jitter): + case <-time.After(delay + jitter): } + // ScrapeMetadata internally retries on 429 with exponential back-off. meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL) if err != nil { - log.Warn("runner: catalogue refresh: metadata scrape failed", - "url", entry.URL, "err", err) + log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book", + "slug", entry.Slug, "url", entry.URL, "err", err) errCount++ continue } @@ -81,35 +85,32 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) { // Persist to PocketBase. if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil { - log.Warn("runner: catalogue refresh: WriteMetadata failed", + log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book", "slug", meta.Slug, "err", err) errCount++ continue } - // Index in Meilisearch. + // Index in Meilisearch (non-fatal). if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil { log.Warn("runner: catalogue refresh: UpsertBook failed", "slug", meta.Slug, "err", err) - // non-fatal — continue } - // Download and store cover image in MinIO if we have a cover URL - // and a CoverStore is wired in. + // Download cover to MinIO if not already cached (non-fatal). if r.deps.CoverStore != nil && originalCover != "" { if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) { if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil { log.Warn("runner: catalogue refresh: cover download failed", "slug", meta.Slug, "url", originalCover, "err", err) - // non-fatal } } } ok++ - if ok%100 == 0 { + if ok%50 == 0 { log.Info("runner: catalogue refresh progress", - "scraped", ok, "errors", errCount) + "scraped", ok, "skipped", skipped, "errors", errCount) } } diff --git a/backend/internal/runner/runner.go b/backend/internal/runner/runner.go index 181c536..9acfda3 100644 --- a/backend/internal/runner/runner.go +++ b/backend/internal/runner/runner.go @@ -62,6 +62,10 @@ type Config struct { // scrapes per-book metadata, downloads covers, and re-indexes everything in // Meilisearch. Defaults to 24h (expensive — full catalogue walk). CatalogueRefreshInterval time.Duration + // CatalogueRequestDelay is the base inter-request pause during a catalogue + // refresh metadata walk. Jitter of up to 50% is added on top. + // Defaults to 2s. Set via RUNNER_CATALOGUE_REQUEST_DELAY. + CatalogueRequestDelay time.Duration // SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that // otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval) // still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for @@ -145,6 +149,9 @@ func New(cfg Config, deps Dependencies) *Runner { if cfg.CatalogueRefreshInterval <= 0 { cfg.CatalogueRefreshInterval = 24 * time.Hour } + if cfg.CatalogueRequestDelay <= 0 { + cfg.CatalogueRequestDelay = 2 * time.Second + } if cfg.MetricsAddr == "" { cfg.MetricsAddr = ":9091" }