refactor: audit, split server.go, add unit tests, and fix latent bugs

- Remove dead code: browser cdp/content_scrape strategies, writer package,
  printUsage, downloadAndStoreCoverCLI in main.go
- Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination
  hard cap removed, splitChapterTitle off-by-one in date extraction
- Split server.go (~1700 lines) into focused handler files:
  handlers_audio, handlers_browse, handlers_progress, handlers_ranking,
  handlers_scrape
- Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go
  to consolidate duplicate helpers
- Flatten deeply nested conditionals: voices() early-return guards,
  ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead
  strings.Cut line, splitChapterTitle double-nested unit/suffix loop
- Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases),
  orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle
  (22 cases); all pass with go build/vet/test clean

scraper/internal/server/handlers_audio.go

@@ -0,0 +1,541 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
+//
+// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
+//
+// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
+// Kokoro generates the audio, saves it to its own temp storage, and returns
+// the download filename in the X-Download-Path response header.
+// We cache that filename (in memory, keyed by slug/chapter/voice) and
+// return a proxy URL that the browser sets as audio.src.
+//
+// TTS is always generated at speed 1.0; playback speed is controlled
+// client-side via the <audio> element's playbackRate.
+//
+// On a cache hit the proxy URL is returned immediately without re-generating.
+// Concurrent requests for the same key are deduplicated.
+func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	n, err := strconv.Atoi(r.PathValue("n"))
+	if err != nil || n < 1 {
+		http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
+		return
+	}
+
+	// Parse optional voice from JSON body. Speed is intentionally ignored —
+	// TTS is always generated at 1.0; playback speed is applied client-side.
+	voice := s.kokoroVoice
+	var body struct {
+		Voice    string `json:"voice"`
+		MaxChars int    `json:"max_chars"`
+	}
+	if r.Body != nil {
+		_ = json.NewDecoder(r.Body).Decode(&body)
+	}
+	if body.Voice != "" {
+		voice = body.Voice
+	}
+
+	cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
+
+	// Fast path: already generated (check persistent store first).
+	if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
+		s.writeAudioResponse(w, slug, n, voice, filename)
+		return
+	}
+
+	// Deduplicate concurrent generation for the same key.
+	s.audioMu.Lock()
+	if ch, ok := s.audioInFlight[cacheKey]; ok {
+		s.audioMu.Unlock()
+		select {
+		case <-ch:
+		case <-r.Context().Done():
+			http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
+			return
+		}
+		// Check store again after waiting.
+		if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
+			s.writeAudioResponse(w, slug, n, voice, filename)
+		} else {
+			http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
+		}
+		return
+	}
+	ch := make(chan struct{})
+	s.audioInFlight[cacheKey] = ch
+	s.audioMu.Unlock()
+
+	defer func() {
+		s.audioMu.Lock()
+		delete(s.audioInFlight, cacheKey)
+		s.audioMu.Unlock()
+		close(ch)
+	}()
+
+	// Load and validate chapter text.
+	raw, err := s.store.ReadChapter(r.Context(), slug, n)
+	if err != nil {
+		http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
+		return
+	}
+	text := stripMarkdown(raw)
+	if text == "" {
+		http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
+		return
+	}
+	if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
+		text = string([]rune(text)[:body.MaxChars])
+	}
+	if s.kokoroURL == "" {
+		http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
+		return
+	}
+
+	// Call Kokoro POST /v1/audio/speech at speed 1.0.
+	// Kokoro saves the generated audio to its own temp storage and returns the
+	// download path in the X-Download-Path response header.
+	filename, err := s.generateSpeech(r.Context(), text, voice, 1.0)
+	if err != nil {
+		s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
+		http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
+		return
+	}
+
+	if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
+		s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
+	}
+
+	// Download generated audio from Kokoro and persist to MinIO synchronously
+	// so that the presigned URL returned to the client is immediately valid.
+	minioKey := s.store.AudioObjectKey(slug, n, voice)
+	audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
+	if dlErr != nil {
+		s.log.Warn("audio MinIO upload skipped: kokoro download failed",
+			"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
+	} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
+		s.log.Warn("audio MinIO upload failed",
+			"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
+		// upload failure is non-fatal; the client can still stream via Kokoro proxy
+	} else {
+		s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
+	}
+
+	s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
+	s.writeAudioResponse(w, slug, n, voice, filename)
+}
+
+// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
+// and returns the filename from the X-Download-Path response header.
+func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
+	reqBody, _ := json.Marshal(map[string]interface{}{
+		"model":                "kokoro",
+		"input":                text,
+		"voice":                voice,
+		"response_format":      "mp3",
+		"speed":                speed,
+		"stream":               false,
+		"return_download_link": true,
+	})
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
+	if err != nil {
+		return "", fmt.Errorf("build request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("kokoro request: %w", err)
+	}
+	defer resp.Body.Close()
+	// Drain body so the connection can be reused.
+	_, _ = io.Copy(io.Discard, resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
+	}
+
+	// X-Download-Path is e.g. "/download/speech_abc123.mp3"
+	dlPath := resp.Header.Get("X-Download-Path")
+	if dlPath == "" {
+		return "", fmt.Errorf("kokoro did not return X-Download-Path header")
+	}
+
+	// Extract just the filename from the path.
+	filename := dlPath
+	if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
+		filename = dlPath[idx+1:]
+	}
+	if filename == "" {
+		return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
+	}
+	return filename, nil
+}
+
+// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
+// using GET /v1/download/{filename} and returns the raw bytes.
+func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
+	url := s.kokoroURL + "/v1/download/" + filename
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("build download request: %w", err)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("kokoro download request: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read kokoro download body: %w", err)
+	}
+	return data, nil
+}
+
+// writeAudioResponse writes the JSON response for a generated audio chapter.
+// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
+func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, filename string) {
+	proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"url":      proxyURL,
+		"filename": filename,
+	})
+}
+
+// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
+// It looks up the Kokoro download filename for this chapter (voice) and
+// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
+func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	n, err := strconv.Atoi(r.PathValue("n"))
+	if err != nil || n < 1 {
+		http.NotFound(w, r)
+		return
+	}
+	voice := r.URL.Query().Get("voice")
+	if voice == "" {
+		voice = s.kokoroVoice
+	}
+
+	cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
+	filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
+	if !ok {
+		http.Error(w, "audio not generated yet", http.StatusNotFound)
+		return
+	}
+
+	kokoroURL := s.kokoroURL + "/v1/download/" + filename
+	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
+	if err != nil {
+		http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
+		return
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		http.Error(w, "kokoro download failed", http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
+		return
+	}
+
+	w.Header().Set("Content-Type", "audio/mpeg")
+	w.Header().Set("Cache-Control", "public, max-age=3600")
+	if cl := resp.Header.Get("Content-Length"); cl != "" {
+		w.Header().Set("Content-Length", cl)
+	}
+	_, _ = io.Copy(w, resp.Body)
+}
+
+// ─── Presigned URL handlers ───────────────────────────────────────────────────
+
+// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
+// Returns a short-lived presigned MinIO URL for the chapter markdown object.
+// The SvelteKit server uses this to fetch chapter content server-side.
+func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	n, err := strconv.Atoi(r.PathValue("n"))
+	if err != nil || n < 1 || slug == "" {
+		http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
+		return
+	}
+
+	url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
+	if err != nil {
+		s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
+		http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
+}
+
+// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
+// Returns a presigned MinIO URL for the audio object (if it has been generated).
+// Query params: voice (optional, defaults to server default).
+func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	n, err := strconv.Atoi(r.PathValue("n"))
+	if err != nil || n < 1 || slug == "" {
+		http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
+		return
+	}
+
+	voice := r.URL.Query().Get("voice")
+	if voice == "" {
+		voice = s.kokoroVoice
+	}
+
+	key := s.store.AudioObjectKey(slug, n, voice)
+
+	// Return 404 when the object hasn't been uploaded yet — the client treats
+	// this as "audio not ready" and will either poll or trigger generation.
+	if !s.store.AudioExists(r.Context(), key) {
+		http.NotFound(w, r)
+		return
+	}
+
+	url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
+	if err != nil {
+		s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
+		http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
+}
+
+// ─── Voices API ───────────────────────────────────────────────────────────────
+
+// handleVoices handles GET /api/voices.
+// Returns the list of available Kokoro voices as JSON: {"voices": [...]}
+func (s *Server) handleVoices(w http.ResponseWriter, _ *http.Request) {
+	voices := s.voices()
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{"voices": voices})
+}
+
+// ─── Voice sample generation ──────────────────────────────────────────────────
+
+// voiceSampleText is the short passage used for voice sample previews.
+const voiceSampleText = "The ancient library held secrets older than memory itself, its dust-laden shelves stretching upward into shadow. She reached for the worn leather spine, fingers trembling with anticipation."
+
+// voiceSampleKey returns the MinIO object key for a voice sample.
+// Key: _voice-samples/{voice}.mp3
+func voiceSampleKey(voice string) string {
+	safe := strings.Map(func(r rune) rune {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
+			(r >= '0' && r <= '9') || r == '_' || r == '-' {
+			return r
+		}
+		return '_'
+	}, voice)
+	return fmt.Sprintf("_voice-samples/%s.mp3", safe)
+}
+
+// warmVoiceSamples runs at startup in a background goroutine.
+// It generates a short audio sample for every available Kokoro voice that
+// doesn't already have one in MinIO, so the UI voice selector has playable
+// previews without requiring a manual trigger.
+// It respects ctx cancellation and waits up to 30 s for Kokoro to become
+// reachable before giving up.
+func (s *Server) warmVoiceSamples(ctx context.Context) {
+	if s.kokoroURL == "" {
+		return
+	}
+
+	// Wait for Kokoro to be reachable (it may still be starting up).
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		req, _ := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
+		resp, err := http.DefaultClient.Do(req)
+		if err == nil {
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				break
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(3 * time.Second):
+		}
+	}
+
+	voices := s.voices()
+	s.log.Info("warming voice samples", "voices", len(voices))
+
+	generated, skipped, failed := 0, 0, 0
+	for _, voice := range voices {
+		if ctx.Err() != nil {
+			return
+		}
+
+		key := voiceSampleKey(voice)
+		if s.store.AudioExists(ctx, key) {
+			skipped++
+			continue
+		}
+
+		filename, err := s.generateSpeech(ctx, voiceSampleText, voice, 1.0)
+		if err != nil {
+			s.log.Warn("voice sample warmup: generation failed", "voice", voice, "err", err)
+			failed++
+			continue
+		}
+
+		audioData, err := s.downloadFromKokoro(ctx, filename)
+		if err != nil {
+			s.log.Warn("voice sample warmup: download failed", "voice", voice, "err", err)
+			failed++
+			continue
+		}
+
+		if err := s.store.PutAudio(ctx, key, audioData); err != nil {
+			s.log.Warn("voice sample warmup: upload failed", "voice", voice, "key", key, "err", err)
+			failed++
+			continue
+		}
+
+		s.log.Debug("voice sample warmed", "voice", voice)
+		generated++
+	}
+
+	s.log.Info("voice sample warmup complete",
+		"generated", generated, "skipped", skipped, "failed", failed)
+}
+
+// handleGenerateVoiceSamples handles POST /api/audio/voice-samples.
+// It generates short audio samples for each available voice and stores them
+// in the audio MinIO bucket so the UI can play them during voice selection.
+// Already-generated samples are skipped (idempotent).
+// Optional JSON body: {"voices": ["af_bella", ...]} to generate a subset.
+// Returns: {"generated": [...], "skipped": [...], "failed": [...]}
+func (s *Server) handleGenerateVoiceSamples(w http.ResponseWriter, r *http.Request) {
+	if s.kokoroURL == "" {
+		http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
+		return
+	}
+
+	// Parse optional voice list from body.
+	var body struct {
+		Voices []string `json:"voices"`
+	}
+	if r.Body != nil {
+		_ = json.NewDecoder(r.Body).Decode(&body)
+	}
+
+	targetVoices := body.Voices
+	if len(targetVoices) == 0 {
+		targetVoices = s.voices()
+	}
+
+	type result struct {
+		Generated []string `json:"generated"`
+		Skipped   []string `json:"skipped"`
+		Failed    []string `json:"failed"`
+	}
+	var res result
+
+	for _, voice := range targetVoices {
+		key := voiceSampleKey(voice)
+
+		// Skip if already uploaded.
+		if s.store.AudioExists(r.Context(), key) {
+			res.Skipped = append(res.Skipped, voice)
+			s.log.Debug("voice sample already exists, skipping", "voice", voice)
+			continue
+		}
+
+		// Generate via Kokoro (speed 1.0 for samples).
+		filename, err := s.generateSpeech(r.Context(), voiceSampleText, voice, 1.0)
+		if err != nil {
+			s.log.Warn("voice sample generation failed", "voice", voice, "err", err)
+			res.Failed = append(res.Failed, voice)
+			continue
+		}
+
+		// Download from Kokoro and upload to MinIO.
+		audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
+		if dlErr != nil {
+			s.log.Warn("voice sample kokoro download failed", "voice", voice, "err", dlErr)
+			res.Failed = append(res.Failed, voice)
+			continue
+		}
+
+		if putErr := s.store.PutAudio(r.Context(), key, audioData); putErr != nil {
+			s.log.Warn("voice sample MinIO upload failed", "voice", voice, "key", key, "err", putErr)
+			res.Failed = append(res.Failed, voice)
+			continue
+		}
+
+		s.log.Info("voice sample generated", "voice", voice, "key", key)
+		res.Generated = append(res.Generated, voice)
+	}
+
+	if res.Generated == nil {
+		res.Generated = []string{}
+	}
+	if res.Skipped == nil {
+		res.Skipped = []string{}
+	}
+	if res.Failed == nil {
+		res.Failed = []string{}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(res)
+}
+
+// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
+// Returns a presigned URL for the voice sample audio file stored in MinIO.
+// Returns 404 if the sample has not been generated yet.
+func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
+	voice := r.PathValue("voice")
+	if voice == "" {
+		http.Error(w, `{"error":"missing voice"}`, http.StatusBadRequest)
+		return
+	}
+
+	key := voiceSampleKey(voice)
+
+	if !s.store.AudioExists(r.Context(), key) {
+		http.NotFound(w, r)
+		return
+	}
+
+	url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
+	if err != nil {
+		s.log.Error("presign voice sample failed", "voice", voice, "err", err)
+		http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
+}

scraper/internal/server/handlers_browse.go

@@ -0,0 +1,476 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/libnovel/scraper/internal/storage"
+	"golang.org/x/net/html"
+
+	"github.com/libnovel/scraper/internal/scraper/htmlutil"
+)
+
+// ─── Browse API ───────────────────────────────────────────────────────────────
+
+// NovelListing represents a single novel entry from the novelfire browse page.
+type NovelListing struct {
+	Slug     string `json:"slug"`
+	Title    string `json:"title"`
+	Cover    string `json:"cover"`
+	Rank     string `json:"rank"`
+	Rating   string `json:"rating"`
+	Chapters string `json:"chapters"`
+	URL      string `json:"url"`
+}
+
+const novelFireBase = "https://novelfire.net"
+const novelFireDomain = "novelfire.net"
+
+// handleBrowse handles GET /api/browse.
+// Query params:
+//
+//	page    (default 1)
+//	genre   (default "all")
+//	sort    (default "popular")
+//	status  (default "all")
+//	type    (default "all-novel")
+//
+// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
+//
+// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
+// if a snapshot exists, parse it and return structured JSON.
+// On a cache miss, fetch live from novelfire.net, return the result, and
+// trigger a background SingleFile snapshot + ranking population.
+func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	page := q.Get("page")
+	if page == "" {
+		page = "1"
+	}
+	genre := q.Get("genre")
+	if genre == "" {
+		genre = "all"
+	}
+	sortBy := q.Get("sort")
+	if sortBy == "" {
+		sortBy = "popular"
+	}
+	status := q.Get("status")
+	if status == "" {
+		status = "all"
+	}
+	novelType := q.Get("type")
+	if novelType == "" {
+		novelType = "all-novel"
+	}
+
+	pageNum, _ := strconv.Atoi(page)
+	if pageNum <= 0 {
+		pageNum = 1
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
+	defer cancel()
+
+	// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
+	cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
+	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
+		novels, hasNext := parseBrowsePage(strings.NewReader(html))
+		s.log.Debug("browse: served from cache", "key", cacheKey)
+		// Still fire background ranking population in case PocketBase ranking
+		// records are missing (e.g. after a schema reset / fresh deploy).
+		targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
+			novelFireBase, genre, sortBy, status, novelType, page)
+		s.triggerDirectScrape(cacheKey, targetURLForRanking)
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Cache-Control", "public, max-age=300")
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"novels":  novels,
+			"page":    pageNum,
+			"hasNext": hasNext,
+		})
+		return
+	}
+
+	// ── Live fallback: direct fetch from novelfire.net ───────────────────
+	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
+	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
+		novelFireBase, genre, sortBy, status, novelType, page)
+
+	var novels []NovelListing
+	var hasNext bool
+	var fetchErr error
+	for attempt := 1; attempt <= 3; attempt++ {
+		if attempt > 1 {
+			select {
+			case <-ctx.Done():
+				http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
+				return
+			case <-time.After(time.Duration(attempt) * time.Second):
+			}
+		}
+
+		var req *http.Request
+		req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
+		if fetchErr != nil {
+			http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
+			return
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+		// Do NOT set Accept-Encoding manually: Go's http.Transport handles
+		// transparent gzip decompression only when it adds the header itself.
+		// If we set it explicitly, Transport disables auto-decompression and
+		// parseBrowsePage receives raw gzip bytes instead of HTML.
+		req.Header.Set("Cache-Control", "no-cache")
+		req.Header.Set("Pragma", "no-cache")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			fetchErr = err
+			s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+			fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
+			s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
+			continue
+		}
+
+		novels, hasNext = parseBrowsePage(resp.Body)
+		resp.Body.Close()
+		fetchErr = nil
+		break
+	}
+	if fetchErr != nil {
+		s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
+		// ── In-memory fallback: use cached result from a prior successful fetch ──
+		s.browseMemCacheMu.RLock()
+		entry, memHit := s.browseMemCache[cacheKey]
+		s.browseMemCacheMu.RUnlock()
+		if memHit {
+			s.log.Warn("browse: upstream unavailable, serving stale in-memory cache",
+				"key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second))
+			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set("Cache-Control", "public, max-age=60")
+			_ = json.NewEncoder(w).Encode(map[string]interface{}{
+				"novels":  entry.novels,
+				"page":    pageNum,
+				"hasNext": entry.hasNext,
+			})
+			return
+		}
+		http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
+		return
+	}
+
+	// ── Populate in-memory cache with the fresh upstream result ──────────
+	if len(novels) > 0 {
+		s.browseMemCacheMu.Lock()
+		s.browseMemCache[cacheKey] = browseCacheEntry{
+			novels:   novels,
+			hasNext:  hasNext,
+			cachedAt: time.Now(),
+		}
+		s.browseMemCacheMu.Unlock()
+	}
+
+	// ── Background: fetch and cache page directly from novelfire.net ─────
+	// Fire-and-forget: stores raw HTML in MinIO and populates the ranking
+	// collection in PocketBase (no browser/SingleFile needed).
+	s.triggerDirectScrape(cacheKey, targetURL)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "public, max-age=300")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"novels":  novels,
+		"page":    pageNum,
+		"hasNext": hasNext,
+	})
+}
+
+// triggerDirectScrape fires a background goroutine that:
+//  1. Fetches pageURL directly from novelfire.net using Go's HTTP client
+//     (no browser/SingleFile needed — the page is server-rendered HTML).
+//  2. Stores the raw HTML in MinIO at cacheKey so future requests are served
+//     from cache without hitting the origin.
+//  3. Parses the HTML to extract novel listings.
+//  4. For each listing, upserts a ranking record in PocketBase (rank, slug,
+//     title, cover key, source_url).
+//  5. Fires a separate goroutine per cover image to download and store it at
+//     {domain}/assets/book-covers/{slug}.jpg in MinIO.
+//
+// It is a no-op when a refresh for this cache key is already in progress.
+// The goroutine uses a fresh context so it outlives the HTTP request.
+func (s *Server) triggerDirectScrape(cacheKey, pageURL string) {
+	s.browseMu.Lock()
+	if _, inflight := s.browseInFlight[cacheKey]; inflight {
+		s.browseMu.Unlock()
+		return
+	}
+	s.browseInFlight[cacheKey] = struct{}{}
+	s.browseMu.Unlock()
+
+	go func() {
+		defer func() {
+			s.browseMu.Lock()
+			delete(s.browseInFlight, cacheKey)
+			s.browseMu.Unlock()
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
+		if err != nil {
+			s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err)
+			return
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err)
+			return
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode)
+			return
+		}
+
+		htmlBytes, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr)
+			return
+		}
+		if len(htmlBytes) == 0 {
+			s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey)
+			return
+		}
+
+		// Store the HTML in MinIO so subsequent requests are cache-hits.
+		if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
+			s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr)
+			// Non-fatal: continue to populate PocketBase/covers even if MinIO write fails.
+		} else {
+			s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes))
+		}
+
+		// Parse to extract novel listings.
+		novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
+		if len(novels) == 0 {
+			s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey)
+			return
+		}
+
+		// Upsert each novel into PocketBase ranking and kick off cover downloads.
+		for i, novel := range novels {
+			rank := i + 1
+			coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
+
+			item := storage.RankingItem{
+				Rank:      rank,
+				Slug:      novel.Slug,
+				Title:     novel.Title,
+				Cover:     coverKey, // stored as MinIO key; UI fetches via /api/cover/...
+				SourceURL: novel.URL,
+			}
+			if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
+				s.log.Warn("triggerDirectScrape: WriteRankingItem failed",
+					"slug", novel.Slug, "err", werr)
+			}
+
+			if novel.Cover != "" {
+				go s.downloadAndStoreCover(coverKey, novel.Cover)
+			}
+		}
+
+		s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey)
+	}()
+}
+
+// warmBrowseCache checks whether the browse cache for page 1 is populated in
+// MinIO and, if not, triggers a background direct scrape.  This is called
+// once on server startup so the first user request is likely served from cache.
+func (s *Server) warmBrowseCache() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1)
+	if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
+		s.log.Debug("warmBrowseCache: page 1 already cached, skipping")
+		return
+	}
+
+	targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase)
+	s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape")
+	s.triggerDirectScrape(cacheKey, targetURL)
+}
+
+// downloadAndStoreCover delegates to storage.DownloadAndStoreCover.
+func (s *Server) downloadAndStoreCover(key, imageURL string) {
+	storage.DownloadAndStoreCover(s.store, s.log, key, imageURL)
+}
+
+// parseBrowsePage parses the novelfire HTML and extracts novel listings.
+// Returns novels and whether a "next page" link was found.
+func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
+	doc, err := html.Parse(r)
+	if err != nil {
+		return nil, false
+	}
+
+	var novels []NovelListing
+	hasNext := false
+
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "li":
+				if hasClass(n, "novel-item") {
+					if novel, ok := parseNovelItem(n); ok {
+						novels = append(novels, novel)
+					}
+				}
+				// pagination li with class "next"
+				if hasClass(n, "next") {
+					hasNext = true
+				}
+			case "a":
+				// Detect "next" pagination link
+				if hasClass(n, "next") || attrVal(n, "rel") == "next" {
+					hasNext = true
+				}
+				// Also check aria-label="Next"
+				if attrVal(n, "aria-label") == "Next" {
+					hasNext = true
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	return novels, hasNext
+}
+
+// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
+func parseNovelItem(li *html.Node) (NovelListing, bool) {
+	var novel NovelListing
+
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "a":
+				href := attrVal(n, "href")
+				if strings.HasPrefix(href, "/book/") {
+					slug := strings.TrimPrefix(href, "/book/")
+					slug = strings.TrimSuffix(slug, "/")
+					if novel.Slug == "" {
+						novel.Slug = slug
+						novel.URL = novelFireBase + href
+					}
+				}
+			case "img":
+				// lazy-loaded covers use data-src
+				src := attrVal(n, "data-src")
+				if src == "" {
+					src = attrVal(n, "src")
+				}
+				if src != "" && novel.Cover == "" {
+					if !strings.HasPrefix(src, "http") {
+						src = novelFireBase + src
+					}
+					novel.Cover = src
+				}
+			case "h4":
+				if hasClass(n, "novel-title") && novel.Title == "" {
+					novel.Title = strings.TrimSpace(textContent(n))
+				}
+			case "span":
+				cls := attrVal(n, "class")
+				if strings.Contains(cls, "_bl") && novel.Rank == "" {
+					novel.Rank = strings.TrimSpace(textContent(n))
+				}
+				if strings.Contains(cls, "_br") && novel.Rating == "" {
+					novel.Rating = strings.TrimSpace(textContent(n))
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(li)
+
+	// Extract chapter count from the novel stats text (contains "N Chapters")
+	novel.Chapters = extractChapters(li)
+
+	if novel.Slug == "" || novel.Title == "" {
+		return novel, false
+	}
+	return novel, true
+}
+
+// extractChapters finds the chapter count text within a novel-item node.
+func extractChapters(n *html.Node) string {
+	var result string
+	var walk func(*html.Node)
+	walk = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			cls := attrVal(node, "class")
+			if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
+				txt := strings.TrimSpace(textContent(node))
+				if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
+					// Extract just the numeric part if possible
+					result = txt
+					return
+				}
+			}
+		}
+		for c := node.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(n)
+	return result
+}
+
+// hasClass reports whether an HTML node has the given CSS class.
+func hasClass(n *html.Node, cls string) bool {
+	for _, a := range n.Attr {
+		if a.Key == "class" {
+			for _, c := range strings.Fields(a.Val) {
+				if c == cls {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// attrVal returns the value of an attribute on an HTML node, or "".
+// Delegates to htmlutil.AttrVal.
+func attrVal(n *html.Node, key string) string { return htmlutil.AttrVal(n, key) }
+
+// textContent returns the concatenated text content of a node and its descendants.
+// Delegates to htmlutil.TextContent.
+func textContent(n *html.Node) string { return htmlutil.TextContent(n) }

scraper/internal/server/handlers_progress.go

@@ -0,0 +1,103 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/libnovel/scraper/internal/storage"
+)
+
+// ─── Reading progress API ─────────────────────────────────────────────────────
+
+// handleGetProgress handles GET /api/progress.
+// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
+func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
+	sid := ensureSession(w, r)
+	entries, err := s.store.AllProgress(r.Context(), sid)
+	if err != nil {
+		s.log.Error("AllProgress failed", "err", err)
+		entries = nil
+	}
+
+	progress := make(map[string]interface{}, len(entries)*2)
+	for _, p := range entries {
+		progress[p.Slug] = p.Chapter
+		progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(progress)
+}
+
+// handleSetProgress handles POST /api/progress/{slug}.
+// Body: {"chapter": N}
+func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
+	sid := ensureSession(w, r)
+	slug := r.PathValue("slug")
+	if slug == "" {
+		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
+		return
+	}
+
+	var body struct {
+		Chapter int `json:"chapter"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
+		http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
+		return
+	}
+
+	p := storage.ReadingProgress{
+		Slug:      slug,
+		Chapter:   body.Chapter,
+		UpdatedAt: time.Now(),
+	}
+	if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
+		s.log.Error("SetProgress failed", "slug", slug, "err", err)
+		http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{})
+}
+
+// handleDeleteProgress handles DELETE /api/progress/{slug}.
+func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
+	sid := ensureSession(w, r)
+	slug := r.PathValue("slug")
+	if slug == "" {
+		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
+		return
+	}
+
+	if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
+		s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
+		// Non-fatal — treat as success.
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{})
+}
+
+// handleChapterText returns the plain text of a chapter (markdown stripped)
+// for server-side audio generation. Called by handleAudioGenerate internally.
+func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	n, err := strconv.Atoi(r.PathValue("n"))
+	if err != nil || n < 1 {
+		http.NotFound(w, r)
+		return
+	}
+	raw, err := s.store.ReadChapter(r.Context(), slug, n)
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-store")
+	fmt.Fprint(w, stripMarkdown(raw))
+}

scraper/internal/server/handlers_ranking.go

@@ -0,0 +1,86 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/libnovel/scraper/internal/storage"
+)
+
+// handleGetRanking returns all ranking items sorted by rank ascending.
+// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
+// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
+// without knowing about the internal MinIO topology.
+func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
+	items, err := s.store.ReadRankingItems(r.Context())
+	if err != nil {
+		s.log.Error("ranking read failed", "err", err)
+		http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
+		return
+	}
+	if items == nil {
+		items = []storage.RankingItem{}
+	}
+	// Rewrite cover keys to proxy URLs.
+	// Keys stored by triggerDirectScrape look like:
+	//   "novelfire.net/assets/book-covers/shadow-slave.jpg"
+	// We expose them as:
+	//   "/api/cover/novelfire.net/shadow-slave"
+	// (the handler strips the domain and slug from the path, reconstructs the key)
+	for i := range items {
+		cover := items[i].Cover
+		if cover != "" && !strings.HasPrefix(cover, "http") {
+			// cover is a MinIO key; extract domain + slug for the proxy path.
+			// Key format: {domain}/assets/book-covers/{slug}.jpg
+			parts := strings.SplitN(cover, "/assets/book-covers/", 2)
+			if len(parts) == 2 {
+				domain := parts[0]
+				slug := strings.TrimSuffix(parts[1], ".jpg")
+				items[i].Cover = "/api/cover/" + domain + "/" + slug
+			}
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(items)
+}
+
+// handleGetCover proxies a cover image stored in the MinIO browse bucket.
+// Route: GET /api/cover/{domain}/{slug}
+// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
+// fetches the object, and streams it to the client.
+// Returns 404 if not yet downloaded, allowing the UI to fall back to the
+// original source URL.
+func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
+	domain := r.PathValue("domain")
+	slug := r.PathValue("slug")
+	if domain == "" || slug == "" {
+		http.Error(w, "missing domain or slug", http.StatusBadRequest)
+		return
+	}
+
+	key := s.store.BrowseCoverKey(domain, slug)
+
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+	defer cancel()
+
+	data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
+	if err != nil {
+		s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
+		http.Error(w, "storage error", http.StatusInternalServerError)
+		return
+	}
+	if !ok {
+		http.NotFound(w, r)
+		return
+	}
+
+	if contentType == "" {
+		contentType = "image/jpeg"
+	}
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("Cache-Control", "public, max-age=86400")
+	_, _ = w.Write(data)
+}

scraper/internal/server/handlers_scrape.go

@@ -0,0 +1,247 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/libnovel/scraper/internal/orchestrator"
+	"github.com/libnovel/scraper/internal/storage"
+)
+
+func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
+	cfg := s.oCfg
+	cfg.SingleBookURL = "" // full catalogue
+
+	s.runAsync(w, cfg)
+}
+
+func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
+	var body struct {
+		URL string `json:"url"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
+		http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
+		return
+	}
+
+	cfg := s.oCfg
+	cfg.SingleBookURL = body.URL
+
+	s.runAsync(w, cfg)
+}
+
+// runAsync launches an orchestrator in the background and returns 202 Accepted.
+// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
+func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
+	s.mu.Lock()
+	if s.running {
+		s.mu.Unlock()
+		http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
+		return
+	}
+	s.running = true
+	s.mu.Unlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
+
+	go func() {
+		defer func() {
+			s.mu.Lock()
+			s.running = false
+			s.mu.Unlock()
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
+		defer cancel()
+
+		// Determine task kind and target.
+		kind := "catalogue"
+		targetURL := ""
+		if cfg.SingleBookURL != "" {
+			kind = "book"
+			targetURL = cfg.SingleBookURL
+		}
+
+		// Create the task record in PocketBase.
+		taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
+		if err != nil {
+			s.log.Warn("could not create scraping_tasks record", "err", err)
+			// Non-fatal: continue without task tracking.
+		}
+
+		// flush pushes the latest counters to PocketBase (best-effort).
+		flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
+			if taskID == "" {
+				return
+			}
+			u := storage.ScrapeTaskUpdate{
+				Status:          status,
+				BooksFound:      p.BooksFound,
+				ChaptersScraped: p.ChaptersScraped,
+				ChaptersSkipped: p.ChaptersSkipped,
+				Errors:          p.Errors,
+				ErrorMessage:    errMsg,
+			}
+			if finished {
+				u.Finished = time.Now().UTC()
+			}
+			if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
+				s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
+			}
+		}
+
+		cfg.OnProgress = func(p orchestrator.Progress) {
+			flush(p, "running", "", false)
+		}
+
+		o := orchestrator.New(cfg, s.novel, s.log, s.store)
+		runErr := o.Run(ctx)
+
+		// After a successful full-catalogue run, refresh the ranking list.
+		if runErr == nil && cfg.SingleBookURL == "" {
+			s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
+			rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
+			defer rankCancel()
+			rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
+			rank := 1
+			for meta := range rankEntries {
+				item := storage.RankingItem{
+					Rank:      rank,
+					Slug:      meta.Slug,
+					Title:     meta.Title,
+					Author:    meta.Author,
+					Cover:     meta.Cover,
+					Status:    meta.Status,
+					Genres:    meta.Genres,
+					SourceURL: meta.SourceURL,
+				}
+				if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
+					s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
+				}
+				rank++
+			}
+			if rerr := <-rankErrs; rerr != nil {
+				s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
+			} else {
+				s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
+			}
+		}
+
+		// Determine final status.
+		finalStatus := "done"
+		errMsg := ""
+		if runErr != nil {
+			s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
+			if ctx.Err() != nil {
+				finalStatus = "cancelled"
+			} else {
+				finalStatus = "failed"
+			}
+			errMsg = runErr.Error()
+		}
+
+		// Best-effort: read last known progress counters via a zero-value
+		// OnProgress — we don't have a snapshot here, so re-use whatever the
+		// last OnProgress call delivered (the orchestrator calls notify() at
+		// the very end, so this is always accurate after Run returns).
+		// We issue one final flush with the terminal status and finished time.
+		if taskID != "" {
+			// Re-fetch current counters by listing the task (cheapest path).
+			tasks, listErr := s.store.ListScrapeTasks(ctx)
+			var last storage.ScrapeTaskUpdate
+			if listErr == nil {
+				for _, t := range tasks {
+					if t.ID == taskID {
+						last = storage.ScrapeTaskUpdate{
+							BooksFound:      t.BooksFound,
+							ChaptersScraped: t.ChaptersScraped,
+							ChaptersSkipped: t.ChaptersSkipped,
+							Errors:          t.Errors,
+						}
+						break
+					}
+				}
+			}
+			last.Status = finalStatus
+			last.ErrorMessage = errMsg
+			last.Finished = time.Now().UTC()
+			if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
+				s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
+			}
+		}
+	}()
+}
+
+// ─── Scrape status API ────────────────────────────────────────────────────────
+
+// handleScrapeStatus handles GET /api/scrape/status.
+// Returns JSON: {"running": bool}
+func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
+	s.mu.Lock()
+	running := s.running
+	s.mu.Unlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
+}
+
+// handleScrapeTasks handles GET /api/scrape/tasks.
+// Returns JSON array of all scraping_tasks records, newest first.
+func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
+	tasks, err := s.store.ListScrapeTasks(r.Context())
+	if err != nil {
+		s.log.Error("handleScrapeTasks: list failed", "err", err)
+		http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
+		return
+	}
+	if tasks == nil {
+		tasks = []storage.ScrapeTask{}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(tasks)
+}
+
+// handleReindex handles POST /api/reindex/{slug}.
+// It rebuilds the chapters_idx PocketBase collection for the given book by
+// walking its MinIO objects. Use this when chapters were scraped but the index
+// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
+func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	if slug == "" {
+		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
+		return
+	}
+
+	type reindexer interface {
+		ReindexChapters(ctx context.Context, slug string) (int, error)
+	}
+	ri, ok := s.store.(reindexer)
+	if !ok {
+		http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
+		return
+	}
+
+	count, err := ri.ReindexChapters(r.Context(), slug)
+	if err != nil {
+		s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusInternalServerError)
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"error":   err.Error(),
+			"indexed": count,
+		})
+		return
+	}
+
+	s.log.Info("reindex complete", "slug", slug, "indexed", count)
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"slug":    slug,
+		"indexed": count,
+	})
+}

scraper/internal/server/integration_test.go

@@ -335,6 +335,9 @@ func TestServer_PresignChapter_NotFound(t *testing.T) {
 	// MinIO presign on a non-existent key returns an error; server returns 500.
 	// (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
 	t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
+	if resp.StatusCode != http.StatusInternalServerError && resp.StatusCode != http.StatusOK {
+		t.Errorf("status = %d, want 500 or 200", resp.StatusCode)
+	}
 }
 
 // TestServer_ChapterText writes a chapter and verifies
@@ -395,13 +398,6 @@ func mapKeys(m map[string]interface{}) []string {
 	return keys
 }
 
-func min(a, b int) int {
-	if a < b {
-		return a
-	}
-	return b
-}
-
 // cookieJar is a minimal http.CookieJar that stores cookies by host.
 type cookieJar struct {
 	cookies map[string][]*http.Cookie