libnovel/scraper/internal/server/server.go

// Package server exposes the scraper as an HTTP API service.
//
// Endpoints:
//
//	POST /scrape                          — enqueue a full catalogue scrape
//	POST /scrape/book                     — enqueue a single-book scrape (JSON body: {"url":"..."})
//	GET  /health                          — liveness probe
//	GET  /api/progress                    — get reading progress map (session-scoped)
//	POST /api/progress/{slug}             — set reading progress
//	DELETE /api/progress/{slug}           — delete reading progress
//	GET  /api/presign/chapter/{slug}/{n}  — presigned MinIO URL for chapter markdown
//	GET  /api/presign/audio/{slug}/{n}    — presigned MinIO URL for chapter audio
//	GET  /api/chapter-text/{slug}/{n}     — plain text of chapter (markdown stripped)
//	POST /api/audio/{slug}/{n}            — trigger Kokoro audio generation (async, returns 202)
//	GET  /api/audio/status/{slug}/{n}     — poll audio generation job status
//	GET  /api/audio-proxy/{slug}/{n}      — proxy generated audio from Kokoro
package server

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log/slog"
	"net/http"
	"sync"
	"time"

	"github.com/libnovel/scraper/internal/orchestrator"
	"github.com/libnovel/scraper/internal/scraper"
	"github.com/libnovel/scraper/internal/storage"
)

// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
	addr        string
	oCfg        orchestrator.Config
	novel       scraper.NovelScraper
	log         *slog.Logger
	store       storage.Store
	mu          sync.Mutex
	running     bool
	kokoroURL   string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
	kokoroVoice string // default voice, e.g. af_bella

	// voiceMu guards cachedVoices.
	voiceMu      sync.RWMutex
	cachedVoices []string // populated on first request from Kokoro /v1/audio/voices

	// audioMu guards audioJobIDs only.
	// Completed audio filenames are persisted to the Store (PocketBase).
	// audioJobIDs deduplicates concurrent generation requests for the same key.
	audioMu     sync.Mutex
	audioJobIDs map[string]string // cacheKey → PocketBase job ID (empty string if record creation failed)

	// browseMu guards browseInFlight — keys currently being refreshed
	// in the background.
	browseMu       sync.Mutex
	browseInFlight map[string]struct{}

	// browseMemCache is a short-lived in-process cache for browse results.
	// It is populated whenever a live upstream fetch succeeds and used as a
	// last-resort fallback when both MinIO and the upstream are unavailable.
	// Key: the MinIO cache key (same as used for BrowseHTMLKey).
	browseMemCacheMu sync.RWMutex
	browseMemCache   map[string]browseCacheEntry
}

type browseCacheEntry struct {
	novels   []NovelListing
	hasNext  bool
	cachedAt time.Time
}

// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
	return &Server{
		addr:           addr,
		oCfg:           oCfg,
		novel:          novel,
		log:            log,
		store:          store,
		kokoroURL:      kokoroURL,
		kokoroVoice:    kokoroVoice,
		audioJobIDs:    make(map[string]string),
		browseInFlight: make(map[string]struct{}),
		browseMemCache: make(map[string]browseCacheEntry),
	}
}

// voices returns the list of available Kokoro voices.  On the first call it
// fetches GET /v1/audio/voices from the Kokoro service and caches the result.
// If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to
// the hardcoded kokoroVoices list so the UI is never empty.
func (s *Server) voices() []string {
	s.voiceMu.RLock()
	cached := s.cachedVoices
	s.voiceMu.RUnlock()
	if len(cached) > 0 {
		return cached
	}

	if s.kokoroURL == "" {
		return kokoroVoices
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
	if err != nil {
		s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err)
		return kokoroVoices
	}
	req.Header.Set("Accept", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err)
		return kokoroVoices
	}
	defer resp.Body.Close()
	var payload struct {
		Voices []string `json:"voices"`
	}
	if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&payload) != nil || len(payload.Voices) == 0 {
		s.log.Warn("could not fetch kokoro voices, using built-in list")
		return kokoroVoices
	}
	s.voiceMu.Lock()
	s.cachedVoices = payload.Voices
	s.voiceMu.Unlock()
	s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
	return payload.Voices
}

// ListenAndServe starts the HTTP server and blocks until the provided context
// is cancelled.
func (s *Server) ListenAndServe(ctx context.Context) error {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /health", s.handleHealth)
	mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
	mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
	mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
	// Browse API — fetches and parses novelfire catalogue page
	mux.HandleFunc("GET /api/browse", s.handleBrowse)
	// Ranking API
	mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
	// Cover image proxy (serves images stored in browse MinIO bucket)
	mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
	// Scrape status
	mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
	mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
	// Re-index chapters for a book from MinIO into PocketBase chapters_idx
	mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
	// On-demand preview (no store writes) — for books not yet in the library
	mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
	mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
	// Search: local PocketBase + remote novelfire.net
	mux.HandleFunc("GET /api/search", s.handleSearch)
	// Progress API
	mux.HandleFunc("GET /api/progress", s.handleGetProgress)
	mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
	mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
	// Presigned URL API (for SvelteKit UI)
	mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
	mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
	mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
	// Plain-text chapter content (used server-side for audio generation)
	mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
	// Voices list (proxied from Kokoro)
	mux.HandleFunc("GET /api/voices", s.handleVoices)
	// Voice sample generation — generates a short audio clip for each voice
	// and stores it in MinIO for UI preview playback.
	voiceSampleHandler := http.TimeoutHandler(
		http.HandlerFunc(s.handleGenerateVoiceSamples),
		15*time.Minute,
		`{"error":"voice sample generation timed out"}`,
	)
	mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler)
	// Server-side audio generation via Kokoro /v1/audio/speech.
	// POST returns 202 immediately and starts a background goroutine;
	// poll GET /api/audio/status/{slug}/{n} to track progress.
	mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
	// Audio job status polling endpoint.
	mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
	// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
	mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)

	srv := &http.Server{
		Addr:         s.addr,
		Handler:      mux,
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 60 * time.Second,
		IdleTimeout:  60 * time.Second,
	}

	errCh := make(chan error, 1)
	go func() { errCh <- srv.ListenAndServe() }()

	s.log.Info("HTTP server listening", "addr", s.addr)

	// Pre-populate voice samples in the background so the UI voice selector
	// has playable previews without requiring a manual trigger.
	go s.warmVoiceSamples(ctx)

	// Warm the browse cache on startup: if page 1 is not cached in MinIO yet,
	// trigger a background SingleFile snapshot immediately so the first user
	// request is served from cache rather than hitting novelfire.net live.
	go s.warmBrowseCache()

	select {
	case <-ctx.Done():
		shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		return srv.Shutdown(shutCtx)
	case err := <-errCh:
		return err
	}
}

func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}

// ─── Session cookie helpers ───────────────────────────────────────────────────

const sessionCookieName = "libnovel_session"

// sessionID returns the session ID from the request cookie, or "" if absent.
func sessionID(r *http.Request) string {
	c, err := r.Cookie(sessionCookieName)
	if err != nil {
		return ""
	}
	return c.Value
}

// newSessionID generates a random 16-byte hex session ID.
func newSessionID() (string, error) {
	b := make([]byte, 16)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

// ensureSession issues a new session cookie if the request does not already
// carry one, and returns the session ID (either existing or newly issued).
func ensureSession(w http.ResponseWriter, r *http.Request) string {
	if id := sessionID(r); id != "" {
		return id
	}
	id, err := newSessionID()
	if err != nil {
		// Very unlikely, but fall back to a timestamp-based ID.
		id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
	}
	http.SetCookie(w, &http.Cookie{
		Name:     sessionCookieName,
		Value:    id,
		Path:     "/",
		HttpOnly: true,
		SameSite: http.SameSiteLaxMode,
		MaxAge:   365 * 24 * 60 * 60, // 1 year
	})
	return id
}