fix(runner): harden catalogue scrape against 429s; disable sourcemap upload
Some checks failed
Release / Test backend (push) Successful in 29s
Release / Check ui (push) Successful in 43s
CI / Test backend (pull_request) Successful in 28s
Release / Docker / caddy (push) Successful in 1m8s
CI / Docker / caddy (pull_request) Failing after 39s
CI / Check ui (pull_request) Successful in 55s
Release / Docker / runner (push) Successful in 2m22s
Release / Docker / ui (push) Successful in 2m20s
CI / Docker / backend (pull_request) Successful in 2m32s
CI / Docker / ui (pull_request) Successful in 1m28s
CI / Docker / runner (pull_request) Successful in 2m25s
Release / Docker / backend (push) Successful in 2m33s
Release / Gitea Release (push) Failing after 2s
Some checks failed
Release / Test backend (push) Successful in 29s
Release / Check ui (push) Successful in 43s
CI / Test backend (pull_request) Successful in 28s
Release / Docker / caddy (push) Successful in 1m8s
CI / Docker / caddy (pull_request) Failing after 39s
CI / Check ui (pull_request) Successful in 55s
Release / Docker / runner (push) Successful in 2m22s
Release / Docker / ui (push) Successful in 2m20s
CI / Docker / backend (pull_request) Successful in 2m32s
CI / Docker / ui (pull_request) Successful in 1m28s
CI / Docker / runner (pull_request) Successful in 2m25s
Release / Docker / backend (push) Successful in 2m33s
Release / Gitea Release (push) Failing after 2s
- scraper.go: ScrapeCatalogue now uses retryGet (9 attempts, 10s base) + 500–1500ms inter-page jitter instead of bare GetContent. ScrapeMetadata also switched to retryGet so a single 429 on a book page is retried rather than aborting the whole refresh. - catalogue_refresh.go: per-book delay is now configurable (RUNNER_CATALOGUE_REQUEST_DELAY, default 2s) + up to 50% random jitter applied before every metadata fetch. Only metadata is scraped here — chapters are fetched on-demand, not during catalogue refresh. Progress logged every 50 books instead of 100. - config.go / runner.go / main.go: add CatalogueRequestDelay field wired from RUNNER_CATALOGUE_REQUEST_DELAY env var. - release.yaml: comment out upload-sourcemaps job and remove it from the release needs; GlitchTip auth token needs refreshing after DB wipe.
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
@@ -55,6 +56,9 @@ func (s *Scraper) SourceName() string { return "novelfire.net" }
|
||||
// ── CatalogueProvider ─────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
|
||||
// Each page fetch uses retryGet with 429-aware exponential backoff.
|
||||
// A small inter-page delay (cataloguePageDelay) is inserted between requests to
|
||||
// avoid hammering the server when paging through hundreds of catalogue pages.
|
||||
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||
entries := make(chan domain.CatalogueEntry, 64)
|
||||
errs := make(chan error, 16)
|
||||
@@ -73,8 +77,18 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
default:
|
||||
}
|
||||
|
||||
// Polite inter-page delay — skipped on the very first page.
|
||||
if page > 1 {
|
||||
jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(jitter):
|
||||
}
|
||||
}
|
||||
|
||||
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
||||
return
|
||||
@@ -139,10 +153,11 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
// ── MetadataProvider ──────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeMetadata fetches and parses book metadata from the book's landing page.
|
||||
// Uses retryGet with 429-aware exponential backoff (up to 9 attempts).
|
||||
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
|
||||
s.log.Debug("metadata fetch starting", "url", bookURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, bookURL)
|
||||
raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second)
|
||||
if err != nil {
|
||||
return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user