libnovel/scraper/internal/server/server.go

// Package server exposes the scraper as an HTTP API service.
//
// Endpoints:
//
//	POST /scrape                          — enqueue a full catalogue scrape
//	POST /scrape/book                     — enqueue a single-book scrape (JSON body: {"url":"..."})
//	GET  /health                          — liveness probe
//	GET  /api/progress                    — get reading progress map (session-scoped)
//	POST /api/progress/{slug}             — set reading progress
//	DELETE /api/progress/{slug}           — delete reading progress
//	GET  /api/presign/chapter/{slug}/{n}  — presigned MinIO URL for chapter markdown
//	GET  /api/presign/audio/{slug}/{n}    — presigned MinIO URL for chapter audio
//	GET  /api/chapter-text/{slug}/{n}     — plain text of chapter (markdown stripped)
//	POST /api/audio/{slug}/{n}            — trigger Kokoro audio generation
//	GET  /api/audio-proxy/{slug}/{n}      — proxy generated audio from Kokoro
package server

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/libnovel/scraper/internal/orchestrator"
	"github.com/libnovel/scraper/internal/scraper"
	"github.com/libnovel/scraper/internal/storage"
	"golang.org/x/net/html"
)

// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
	addr        string
	oCfg        orchestrator.Config
	novel       scraper.NovelScraper
	log         *slog.Logger
	store       storage.Store
	mu          sync.Mutex
	running     bool
	kokoroURL   string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
	kokoroVoice string // default voice, e.g. af_bella

	// SingleFile CLI settings for browse-page snapshots.
	singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
	browserlessURL string // Browserless base URL, e.g. http://browserless:3000

	// voiceMu guards cachedVoices.
	voiceMu      sync.RWMutex
	cachedVoices []string // populated on first request from Kokoro /v1/audio/voices

	// audioMu guards audioInFlight only.
	// Completed audio filenames are persisted to the Store (PocketBase).
	// audioInFlight deduplicates concurrent generation requests for the same key.
	audioMu       sync.Mutex
	audioInFlight map[string]chan struct{} // cacheKey → closed when done

	// browseMu guards browseInFlight — keys of MinIO objects currently being
	// captured by a background SingleFile goroutine.
	browseMu       sync.Mutex
	browseInFlight map[string]struct{}
}

// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
	return &Server{
		addr:           addr,
		oCfg:           oCfg,
		novel:          novel,
		log:            log,
		store:          store,
		kokoroURL:      kokoroURL,
		kokoroVoice:    kokoroVoice,
		singleFilePath: os.Getenv("SINGLEFILE_PATH"),
		browserlessURL: os.Getenv("BROWSERLESS_URL"),
		audioInFlight:  make(map[string]chan struct{}),
		browseInFlight: make(map[string]struct{}),
	}
}

// voices returns the list of available Kokoro voices.  On the first call it
// fetches GET /v1/audio/voices from the Kokoro service and caches the result.
// If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to
// the hardcoded kokoroVoices list so the UI is never empty.
func (s *Server) voices() []string {
	s.voiceMu.RLock()
	cached := s.cachedVoices
	s.voiceMu.RUnlock()
	if len(cached) > 0 {
		return cached
	}

	if s.kokoroURL != "" {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
		if err == nil {
			req.Header.Set("Accept", "application/json")
			resp, err := http.DefaultClient.Do(req)
			if err == nil {
				defer resp.Body.Close()
				var payload struct {
					Voices []string `json:"voices"`
				}
				if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 {
					s.voiceMu.Lock()
					s.cachedVoices = payload.Voices
					s.voiceMu.Unlock()
					s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
					return payload.Voices
				}
			}
		}
		s.log.Warn("could not fetch kokoro voices, using built-in list")
	}

	return kokoroVoices
}

// ListenAndServe starts the HTTP server and blocks until the provided context
// is cancelled.
func (s *Server) ListenAndServe(ctx context.Context) error {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /health", s.handleHealth)
	mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
	mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
	// Browse API — fetches and parses novelfire catalogue page
	mux.HandleFunc("GET /api/browse", s.handleBrowse)
	// Ranking API
	mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
	// Scrape status
	mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
	mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
	// Re-index chapters for a book from MinIO into PocketBase chapters_idx
	mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
	// Progress API
	mux.HandleFunc("GET /api/progress", s.handleGetProgress)
	mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
	mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
	// Presigned URL API (for SvelteKit UI)
	mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
	mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
	// Plain-text chapter content (used server-side for audio generation)
	mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
	// Server-side audio generation via Kokoro /v1/audio/speech.
	// Generation can take several minutes, so wrap in its own timeout handler.
	audioGenHandler := http.TimeoutHandler(
		http.HandlerFunc(s.handleAudioGenerate),
		10*time.Minute,
		`{"error":"audio generation timed out"}`,
	)
	mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
	// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
	mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)

	srv := &http.Server{
		Addr:         s.addr,
		Handler:      mux,
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 60 * time.Second,
		IdleTimeout:  60 * time.Second,
	}

	errCh := make(chan error, 1)
	go func() { errCh <- srv.ListenAndServe() }()

	s.log.Info("HTTP server listening", "addr", s.addr)

	select {
	case <-ctx.Done():
		shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		return srv.Shutdown(shutCtx)
	case err := <-errCh:
		return err
	}
}

func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}

// handleGetRanking returns all ranking items sorted by rank ascending.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
	items, err := s.store.ReadRankingItems(r.Context())
	if err != nil {
		s.log.Error("ranking read failed", "err", err)
		http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	if items == nil {
		items = []storage.RankingItem{}
	}
	_ = json.NewEncoder(w).Encode(items)
}

// ─── Session cookie helpers ───────────────────────────────────────────────────

const sessionCookieName = "libnovel_session"

// sessionID returns the session ID from the request cookie, or "" if absent.
func sessionID(r *http.Request) string {
	c, err := r.Cookie(sessionCookieName)
	if err != nil {
		return ""
	}
	return c.Value
}

// newSessionID generates a random 16-byte hex session ID.
func newSessionID() (string, error) {
	b := make([]byte, 16)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

// ensureSession issues a new session cookie if the request does not already
// carry one, and returns the session ID (either existing or newly issued).
func ensureSession(w http.ResponseWriter, r *http.Request) string {
	if id := sessionID(r); id != "" {
		return id
	}
	id, err := newSessionID()
	if err != nil {
		// Very unlikely, but fall back to a timestamp-based ID.
		id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
	}
	http.SetCookie(w, &http.Cookie{
		Name:     sessionCookieName,
		Value:    id,
		Path:     "/",
		HttpOnly: true,
		SameSite: http.SameSiteLaxMode,
		MaxAge:   365 * 24 * 60 * 60, // 1 year
	})
	return id
}

// ─── Reading progress API ─────────────────────────────────────────────────────

// handleGetProgress handles GET /api/progress.
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
	sid := ensureSession(w, r)
	entries, err := s.store.AllProgress(r.Context(), sid)
	if err != nil {
		s.log.Error("AllProgress failed", "err", err)
		entries = nil
	}

	progress := make(map[string]interface{}, len(entries)*2)
	for _, p := range entries {
		progress[p.Slug] = p.Chapter
		progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
	}

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(progress)
}

// handleSetProgress handles POST /api/progress/{slug}.
// Body: {"chapter": N}
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
	sid := ensureSession(w, r)
	slug := r.PathValue("slug")
	if slug == "" {
		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
		return
	}

	var body struct {
		Chapter int `json:"chapter"`
	}
	if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
		http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
		return
	}

	p := storage.ReadingProgress{
		Slug:      slug,
		Chapter:   body.Chapter,
		UpdatedAt: time.Now(),
	}
	if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
		s.log.Error("SetProgress failed", "slug", slug, "err", err)
		http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]string{})
}

// handleDeleteProgress handles DELETE /api/progress/{slug}.
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
	sid := ensureSession(w, r)
	slug := r.PathValue("slug")
	if slug == "" {
		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
		return
	}

	if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
		s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
		// Non-fatal — treat as success.
	}

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]string{})
}

// handleChapterText returns the plain text of a chapter (markdown stripped)
// for server-side audio generation. Called by handleAudioGenerate internally.
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	n, err := strconv.Atoi(r.PathValue("n"))
	if err != nil || n < 1 {
		http.NotFound(w, r)
		return
	}
	raw, err := s.store.ReadChapter(r.Context(), slug, n)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Header().Set("Cache-Control", "no-store")
	fmt.Fprint(w, stripMarkdown(raw))
}

// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
//
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
//
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
// Kokoro generates the audio, saves it to its own temp storage, and returns
// the download filename in the X-Download-Path response header.
// We cache that filename (in memory, keyed by slug/chapter/voice/speed) and
// return a proxy URL that the browser sets as audio.src.
//
// On a cache hit the proxy URL is returned immediately without re-generating.
// Concurrent requests for the same key are deduplicated.
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	n, err := strconv.Atoi(r.PathValue("n"))
	if err != nil || n < 1 {
		http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
		return
	}

	// Parse optional voice/speed from JSON body.
	voice := s.kokoroVoice
	speed := 1.0
	var body struct {
		Voice    string  `json:"voice"`
		Speed    float64 `json:"speed"`
		MaxChars int     `json:"max_chars"`
	}
	if r.Body != nil {
		_ = json.NewDecoder(r.Body).Decode(&body)
	}
	if body.Voice != "" {
		voice = body.Voice
	}
	if body.Speed > 0 {
		speed = body.Speed
	}

	cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)

	// Fast path: already generated (check persistent store first).
	if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
		s.writeAudioResponse(w, slug, n, voice, speed, filename)
		return
	}

	// Deduplicate concurrent generation for the same key.
	s.audioMu.Lock()
	if ch, ok := s.audioInFlight[cacheKey]; ok {
		s.audioMu.Unlock()
		select {
		case <-ch:
		case <-r.Context().Done():
			http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
			return
		}
		// Check store again after waiting.
		if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
			s.writeAudioResponse(w, slug, n, voice, speed, filename)
		} else {
			http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
		}
		return
	}
	ch := make(chan struct{})
	s.audioInFlight[cacheKey] = ch
	s.audioMu.Unlock()

	defer func() {
		s.audioMu.Lock()
		delete(s.audioInFlight, cacheKey)
		s.audioMu.Unlock()
		close(ch)
	}()

	// Load and validate chapter text.
	raw, err := s.store.ReadChapter(r.Context(), slug, n)
	if err != nil {
		http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
		return
	}
	text := stripMarkdown(raw)
	if text == "" {
		http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
		return
	}
	if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
		text = string([]rune(text)[:body.MaxChars])
	}
	if s.kokoroURL == "" {
		http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
		return
	}

	// Call Kokoro POST /v1/audio/speech with return_download_link=true.
	// Kokoro saves the generated audio to its own temp storage and returns the
	// download path in the X-Download-Path response header.
	filename, err := s.generateSpeech(r.Context(), text, voice, speed)
	if err != nil {
		s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
		http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
		return
	}

	if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
		s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
	}

	// Download generated audio from Kokoro and persist to MinIO so that
	// presigned URLs for the audio object are accessible.
	go func() {
		audioData, dlErr := s.downloadFromKokoro(context.Background(), filename)
		if dlErr != nil {
			s.log.Warn("audio MinIO upload skipped: kokoro download failed",
				"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
			return
		}
		minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
		if putErr := s.store.PutAudio(context.Background(), minioKey, audioData); putErr != nil {
			s.log.Warn("audio MinIO upload failed",
				"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
		} else {
			s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
		}
	}()

	s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
	s.writeAudioResponse(w, slug, n, voice, speed, filename)
}

// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
// and returns the filename from the X-Download-Path response header.
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
	reqBody, _ := json.Marshal(map[string]interface{}{
		"model":                "kokoro",
		"input":                text,
		"voice":                voice,
		"response_format":      "mp3",
		"speed":                speed,
		"stream":               false,
		"return_download_link": true,
	})

	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
		s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
	if err != nil {
		return "", fmt.Errorf("build request: %w", err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("kokoro request: %w", err)
	}
	defer resp.Body.Close()
	// Drain body so the connection can be reused.
	_, _ = io.Copy(io.Discard, resp.Body)

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
	}

	// X-Download-Path is e.g. "/download/speech_abc123.mp3"
	dlPath := resp.Header.Get("X-Download-Path")
	if dlPath == "" {
		return "", fmt.Errorf("kokoro did not return X-Download-Path header")
	}

	// Extract just the filename from the path.
	filename := dlPath
	if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
		filename = dlPath[idx+1:]
	}
	if filename == "" {
		return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
	}
	return filename, nil
}

// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
// using GET /v1/download/{filename} and returns the raw bytes.
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
	url := s.kokoroURL + "/v1/download/" + filename
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, fmt.Errorf("build download request: %w", err)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("kokoro download request: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
	}
	data, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("read kokoro download body: %w", err)
	}
	return data, nil
}

// writeAudioResponse writes the JSON response for a generated audio chapter.
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
	proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]interface{}{
		"url":      proxyURL,
		"filename": filename,
	})
}

// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
// It looks up the Kokoro download filename for this chapter (voice/speed) and
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	n, err := strconv.Atoi(r.PathValue("n"))
	if err != nil || n < 1 {
		http.NotFound(w, r)
		return
	}
	voice := r.URL.Query().Get("voice")
	if voice == "" {
		voice = s.kokoroVoice
	}
	speedStr := r.URL.Query().Get("speed")
	speed := 1.0
	if speedStr != "" {
		if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
			speed = v
		}
	}

	cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
	filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
	if !ok {
		http.Error(w, "audio not generated yet", http.StatusNotFound)
		return
	}

	kokoroURL := s.kokoroURL + "/v1/download/" + filename
	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
	if err != nil {
		http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
		return
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		http.Error(w, "kokoro download failed", http.StatusBadGateway)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
		return
	}

	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Cache-Control", "public, max-age=3600")
	if cl := resp.Header.Get("Content-Length"); cl != "" {
		w.Header().Set("Content-Length", cl)
	}
	_, _ = io.Copy(w, resp.Body)
}

// ─── Presigned URL handlers ───────────────────────────────────────────────────

// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
// The SvelteKit server uses this to fetch chapter content server-side.
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	n, err := strconv.Atoi(r.PathValue("n"))
	if err != nil || n < 1 || slug == "" {
		http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
		return
	}

	url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
	if err != nil {
		s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
		http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}

// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
// Returns a presigned MinIO URL for the audio object (if it has been generated).
// Query params: voice, speed (optional, defaults to server defaults).
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	n, err := strconv.Atoi(r.PathValue("n"))
	if err != nil || n < 1 || slug == "" {
		http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
		return
	}

	voice := r.URL.Query().Get("voice")
	if voice == "" {
		voice = s.kokoroVoice
	}
	speed := 1.0
	if sv := r.URL.Query().Get("speed"); sv != "" {
		if v, err := strconv.ParseFloat(sv, 64); err == nil && v > 0 {
			speed = v
		}
	}

	key := s.store.AudioObjectKey(slug, n, voice, speed)
	url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
	if err != nil {
		s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
		http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}

func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
	cfg := s.oCfg
	cfg.SingleBookURL = "" // full catalogue

	s.runAsync(w, cfg)
}

func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
	var body struct {
		URL string `json:"url"`
	}
	if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
		http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
		return
	}

	cfg := s.oCfg
	cfg.SingleBookURL = body.URL

	s.runAsync(w, cfg)
}

// runAsync launches an orchestrator in the background and returns 202 Accepted.
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
	s.mu.Lock()
	if s.running {
		s.mu.Unlock()
		http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
		return
	}
	s.running = true
	s.mu.Unlock()

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusAccepted)
	_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})

	go func() {
		defer func() {
			s.mu.Lock()
			s.running = false
			s.mu.Unlock()
		}()

		ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
		defer cancel()

		// Determine task kind and target.
		kind := "catalogue"
		targetURL := ""
		if cfg.SingleBookURL != "" {
			kind = "book"
			targetURL = cfg.SingleBookURL
		}

		// Create the task record in PocketBase.
		taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
		if err != nil {
			s.log.Warn("could not create scraping_tasks record", "err", err)
			// Non-fatal: continue without task tracking.
		}

		// flush pushes the latest counters to PocketBase (best-effort).
		flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
			if taskID == "" {
				return
			}
			u := storage.ScrapeTaskUpdate{
				Status:          status,
				BooksFound:      p.BooksFound,
				ChaptersScraped: p.ChaptersScraped,
				ChaptersSkipped: p.ChaptersSkipped,
				Errors:          p.Errors,
				ErrorMessage:    errMsg,
			}
			if finished {
				u.Finished = time.Now().UTC()
			}
			if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
				s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
			}
		}

		cfg.OnProgress = func(p orchestrator.Progress) {
			flush(p, "running", "", false)
		}

		o := orchestrator.New(cfg, s.novel, s.log, s.store)
		runErr := o.Run(ctx)

		// Determine final status.
		finalStatus := "done"
		errMsg := ""
		if runErr != nil {
			s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
			if ctx.Err() != nil {
				finalStatus = "cancelled"
			} else {
				finalStatus = "failed"
			}
			errMsg = runErr.Error()
		}

		// Best-effort: read last known progress counters via a zero-value
		// OnProgress — we don't have a snapshot here, so re-use whatever the
		// last OnProgress call delivered (the orchestrator calls notify() at
		// the very end, so this is always accurate after Run returns).
		// We issue one final flush with the terminal status and finished time.
		if taskID != "" {
			// Re-fetch current counters by listing the task (cheapest path).
			tasks, listErr := s.store.ListScrapeTasks(ctx)
			var last storage.ScrapeTaskUpdate
			if listErr == nil {
				for _, t := range tasks {
					if t.ID == taskID {
						last = storage.ScrapeTaskUpdate{
							BooksFound:      t.BooksFound,
							ChaptersScraped: t.ChaptersScraped,
							ChaptersSkipped: t.ChaptersSkipped,
							Errors:          t.Errors,
						}
						break
					}
				}
			}
			last.Status = finalStatus
			last.ErrorMessage = errMsg
			last.Finished = time.Now().UTC()
			if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
				s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
			}
		}
	}()
}

// ─── Browse API ───────────────────────────────────────────────────────────────

// NovelListing represents a single novel entry from the novelfire browse page.
type NovelListing struct {
	Slug     string `json:"slug"`
	Title    string `json:"title"`
	Cover    string `json:"cover"`
	Rank     string `json:"rank"`
	Rating   string `json:"rating"`
	Chapters string `json:"chapters"`
	URL      string `json:"url"`
}

const novelFireBase = "https://novelfire.net"

// handleBrowse handles GET /api/browse.
// Query params:
//
//	page    (default 1)
//	genre   (default "all")
//	sort    (default "popular")
//	status  (default "all")
//	type    (default "all-novel")
//
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
//
// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
// parse and return it. Otherwise fetch live from novelfire.net.
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()
	page := q.Get("page")
	if page == "" {
		page = "1"
	}
	genre := q.Get("genre")
	if genre == "" {
		genre = "all"
	}
	sortBy := q.Get("sort")
	if sortBy == "" {
		sortBy = "popular"
	}
	status := q.Get("status")
	if status == "" {
		status = "all"
	}
	novelType := q.Get("type")
	if novelType == "" {
		novelType = "all-novel"
	}

	pageNum, _ := strconv.Atoi(page)
	if pageNum <= 0 {
		pageNum = 1
	}

	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
	defer cancel()

	// ── Cache-first: try MinIO snapshot ──────────────────────────────────
	cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
		novels, hasNext := parseBrowsePage(strings.NewReader(html))
		s.log.Debug("browse: served from cache", "key", cacheKey)
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Cache-Control", "public, max-age=300")
		_ = json.NewEncoder(w).Encode(map[string]interface{}{
			"novels":  novels,
			"page":    pageNum,
			"hasNext": hasNext,
		})
		return
	}

	// ── Live fallback: fetch from novelfire.net ───────────────────────────
	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
		novelFireBase, genre, sortBy, status, novelType, page)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
	if err != nil {
		http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
		return
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		s.log.Error("browse fetch failed", "url", targetURL, "err", err)
		http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway)
		return
	}

	novels, hasNext := parseBrowsePage(resp.Body)

	// ── Background: populate MinIO cache via SingleFile ───────────────────
	// Fire-and-forget: capture the JS-rendered page with SingleFile and store
	// it in MinIO so the next request is served from cache.
	s.triggerBrowseSnapshot(cacheKey, targetURL)

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Cache-Control", "public, max-age=300")
	_ = json.NewEncoder(w).Encode(map[string]interface{}{
		"novels":  novels,
		"page":    pageNum,
		"hasNext": hasNext,
	})
}

// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
// to capture the fully-rendered novelfire browse page and store it in MinIO.
// It is a no-op when:
//   - SINGLEFILE_PATH is not set (SingleFile not installed)
//   - a capture for this cache key is already in progress
//
// The goroutine uses a fresh context so it outlives the HTTP request.
func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
	if s.singleFilePath == "" {
		return
	}

	s.browseMu.Lock()
	if _, inflight := s.browseInFlight[cacheKey]; inflight {
		s.browseMu.Unlock()
		return
	}
	s.browseInFlight[cacheKey] = struct{}{}
	s.browseMu.Unlock()

	go func() {
		defer func() {
			s.browseMu.Lock()
			delete(s.browseInFlight, cacheKey)
			s.browseMu.Unlock()
		}()

		ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
		defer cancel()

		// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
		wsEndpoint := s.browserlessURL
		wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
		wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)

		tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
		if err != nil {
			s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
			return
		}
		tmpPath := tmpFile.Name()
		tmpFile.Close()
		defer os.Remove(tmpPath)

		//nolint:gosec
		cmd := exec.CommandContext(ctx, s.singleFilePath,
			pageURL,
			"--browser-server="+wsEndpoint,
			"--output="+tmpPath,
		)
		if out, runErr := cmd.CombinedOutput(); runErr != nil {
			s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
				"key", cacheKey, "err", runErr, "output", string(out))
			return
		}

		htmlBytes, readErr := os.ReadFile(tmpPath)
		if readErr != nil {
			s.log.Warn("triggerBrowseSnapshot: read output failed",
				"key", cacheKey, "err", readErr)
			return
		}

		if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
			s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
				"key", cacheKey, "err", putErr)
			return
		}

		s.log.Info("triggerBrowseSnapshot: cached browse page",
			"key", cacheKey, "bytes", len(htmlBytes))
	}()
}

// parseBrowsePage parses the novelfire HTML and extracts novel listings.
// Returns novels and whether a "next page" link was found.
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
	doc, err := html.Parse(r)
	if err != nil {
		return nil, false
	}

	var novels []NovelListing
	hasNext := false

	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.Data {
			case "li":
				if hasClass(n, "novel-item") {
					if novel, ok := parseNovelItem(n); ok {
						novels = append(novels, novel)
					}
				}
				// pagination li with class "next"
				if hasClass(n, "next") {
					hasNext = true
				}
			case "a":
				// Detect "next" pagination link
				if hasClass(n, "next") || attrVal(n, "rel") == "next" {
					hasNext = true
				}
				// Also check aria-label="Next"
				if attrVal(n, "aria-label") == "Next" {
					hasNext = true
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(doc)
	return novels, hasNext
}

// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
func parseNovelItem(li *html.Node) (NovelListing, bool) {
	var novel NovelListing

	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.Data {
			case "a":
				href := attrVal(n, "href")
				if strings.HasPrefix(href, "/book/") {
					slug := strings.TrimPrefix(href, "/book/")
					slug = strings.TrimSuffix(slug, "/")
					if novel.Slug == "" {
						novel.Slug = slug
						novel.URL = novelFireBase + href
					}
				}
			case "img":
				// lazy-loaded covers use data-src
				src := attrVal(n, "data-src")
				if src == "" {
					src = attrVal(n, "src")
				}
				if src != "" && novel.Cover == "" {
					if !strings.HasPrefix(src, "http") {
						src = novelFireBase + src
					}
					novel.Cover = src
				}
			case "h4":
				if hasClass(n, "novel-title") && novel.Title == "" {
					novel.Title = strings.TrimSpace(textContent(n))
				}
			case "span":
				cls := attrVal(n, "class")
				if strings.Contains(cls, "_bl") && novel.Rank == "" {
					novel.Rank = strings.TrimSpace(textContent(n))
				}
				if strings.Contains(cls, "_br") && novel.Rating == "" {
					novel.Rating = strings.TrimSpace(textContent(n))
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(li)

	// Extract chapter count from the novel stats text (contains "N Chapters")
	novel.Chapters = extractChapters(li)

	if novel.Slug == "" || novel.Title == "" {
		return novel, false
	}
	return novel, true
}

// extractChapters finds the chapter count text within a novel-item node.
func extractChapters(n *html.Node) string {
	var result string
	var walk func(*html.Node)
	walk = func(node *html.Node) {
		if node.Type == html.ElementNode {
			cls := attrVal(node, "class")
			if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
				txt := strings.TrimSpace(textContent(node))
				if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
					// Extract just the numeric part if possible
					result = txt
					return
				}
			}
		}
		for c := node.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(n)
	return result
}

// hasClass reports whether an HTML node has the given CSS class.
func hasClass(n *html.Node, cls string) bool {
	for _, a := range n.Attr {
		if a.Key == "class" {
			for _, c := range strings.Fields(a.Val) {
				if c == cls {
					return true
				}
			}
		}
	}
	return false
}

// attrVal returns the value of an attribute on an HTML node, or "".
func attrVal(n *html.Node, key string) string {
	for _, a := range n.Attr {
		if a.Key == key {
			return a.Val
		}
	}
	return ""
}

// textContent returns the concatenated text content of a node and its descendants.
func textContent(n *html.Node) string {
	if n.Type == html.TextNode {
		return n.Data
	}
	var sb strings.Builder
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		sb.WriteString(textContent(c))
	}
	return sb.String()
}

// ─── Scrape status API ────────────────────────────────────────────────────────

// handleScrapeStatus handles GET /api/scrape/status.
// Returns JSON: {"running": bool}
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
	s.mu.Lock()
	running := s.running
	s.mu.Unlock()

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
}

// handleScrapeTasks handles GET /api/scrape/tasks.
// Returns JSON array of all scraping_tasks records, newest first.
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
	tasks, err := s.store.ListScrapeTasks(r.Context())
	if err != nil {
		s.log.Error("handleScrapeTasks: list failed", "err", err)
		http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
		return
	}
	if tasks == nil {
		tasks = []storage.ScrapeTask{}
	}
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(tasks)
}

// handleReindex handles POST /api/reindex/{slug}.
// It rebuilds the chapters_idx PocketBase collection for the given book by
// walking its MinIO objects. Use this when chapters were scraped but the index
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	if slug == "" {
		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
		return
	}

	type reindexer interface {
		ReindexChapters(ctx context.Context, slug string) (int, error)
	}
	ri, ok := s.store.(reindexer)
	if !ok {
		http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
		return
	}

	count, err := ri.ReindexChapters(r.Context(), slug)
	if err != nil {
		s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusInternalServerError)
		_ = json.NewEncoder(w).Encode(map[string]interface{}{
			"error":   err.Error(),
			"indexed": count,
		})
		return
	}

	s.log.Info("reindex complete", "slug", slug, "indexed", count)
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]interface{}{
		"slug":    slug,
		"indexed": count,
	})
}