libnovel/backend/internal/runner/catalogue_refresh.go

package runner

// catalogue_refresh.go — independent loop that walks the full novelfire.net
// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and
// indexes every book in Meilisearch.
//
// Design:
//   - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
//   - Also fires once on startup (unless SkipInitialCatalogueRefresh is set).
//   - ScrapeCatalogue streams CatalogueEntry values over a channel — already has
//     its own inter-page jitter + retryGet (see scraper.go).
//   - Per-book: only metadata is scraped here (not chapters). Chapters are scraped
//     on-demand when a user opens a book or via an explicit scrape task.
//   - Between each metadata request a configurable base delay plus up to 50%
//     random jitter is applied (CatalogueRequestDelay, default 2s). This keeps
//     the request rate well below novelfire.net's rate limit even for ~15k books.
//   - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff
//     (up to 9 attempts), so transient rate limits are handled gracefully.
//   - Cover images are fetched and stored in MinIO on first sight; subsequent
//     refreshes skip covers that already exist (CoverExists check).
//   - Books already present in Meilisearch are skipped entirely (fast path).
//   - Errors for individual books are logged and skipped; the loop never aborts.

import (
	"context"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"time"
)

// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
// every book on novelfire.net, downloads covers to MinIO, and upserts to
// Meilisearch. Individual book failures are logged and skipped.
func (r *Runner) runCatalogueRefresh(ctx context.Context) {
	if r.deps.Novel == nil {
		r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
		return
	}
	if r.deps.BookWriter == nil {
		r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured")
		return
	}

	delay := r.cfg.CatalogueRequestDelay
	log := r.deps.Log.With("op", "catalogue_refresh")
	log.Info("runner: catalogue refresh starting", "request_delay", delay)

	entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)

	ok, skipped, errCount := 0, 0, 0
	for entry := range entries {
		if ctx.Err() != nil {
			break
		}

		// Fast path: skip books already indexed in Meilisearch.
		if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
			skipped++
			continue
		}

		// Polite delay between metadata requests: base + up to 50% jitter.
		// This applies before every fetch so we never fire bursts.
		jitter := time.Duration(rand.Int63n(int64(delay / 2)))
		select {
		case <-ctx.Done():
			break
		case <-time.After(delay + jitter):
		}

		// ScrapeMetadata internally retries on 429 with exponential back-off.
		meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
		if err != nil {
			log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book",
				"slug", entry.Slug, "url", entry.URL, "err", err)
			errCount++
			continue
		}

		// Rewrite cover URL to backend proxy path so UI never hits CDN directly.
		originalCover := meta.Cover
		meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug)

		// Persist to PocketBase.
		if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
			log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book",
				"slug", meta.Slug, "err", err)
			errCount++
			continue
		}

		// Index in Meilisearch (non-fatal).
		if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
			log.Warn("runner: catalogue refresh: UpsertBook failed",
				"slug", meta.Slug, "err", err)
		}

		// Download cover to MinIO if not already cached (non-fatal).
		if r.deps.CoverStore != nil && originalCover != "" {
			if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
				if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
					log.Warn("runner: catalogue refresh: cover download failed",
						"slug", meta.Slug, "url", originalCover, "err", err)
				}
			}
		}

		ok++
		if ok%50 == 0 {
			log.Info("runner: catalogue refresh progress",
				"scraped", ok, "skipped", skipped, "errors", errCount)
		}
	}

	if err := <-errCh; err != nil {
		log.Warn("runner: catalogue refresh: catalogue stream error", "err", err)
	}

	log.Info("runner: catalogue refresh finished",
		"ok", ok, "skipped", skipped, "errors", errCount)
}

// downloadCover fetches the cover image from coverURL and stores it in MinIO
// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff
// on transient errors (5xx, network failures).
func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error {
	const maxRetries = 3
	delay := 2 * time.Second

	var lastErr error
	for attempt := 0; attempt < maxRetries; attempt++ {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		if attempt > 0 {
			select {
			case <-ctx.Done():
				return ctx.Err()
			case <-time.After(delay):
			}
			delay *= 2
		}

		data, err := fetchCoverBytes(ctx, coverURL)
		if err != nil {
			lastErr = err
			continue
		}

		if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil {
			return fmt.Errorf("put cover: %w", err)
		}
		return nil
	}
	return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr)
}

// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body.
func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil)
	if err != nil {
		return nil, fmt.Errorf("build request: %w", err)
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
	req.Header.Set("Referer", "https://novelfire.net/")

	client := &http.Client{Timeout: 30 * time.Second}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("http get: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 500 {
		_, _ = io.Copy(io.Discard, resp.Body)
		return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL)
	}
	if resp.StatusCode != http.StatusOK {
		_, _ = io.Copy(io.Discard, resp.Body)
		return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL)
	}

	return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap
}