// Package server exposes the scraper as an HTTP API service. // // Endpoints: // // POST /scrape — enqueue a full catalogue scrape // POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) // GET /health — liveness probe // GET /api/progress — get reading progress map (session-scoped) // POST /api/progress/{slug} — set reading progress // DELETE /api/progress/{slug} — delete reading progress // GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown // GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio // GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped) // POST /api/audio/{slug}/{n} — trigger Kokoro audio generation (async, returns 202) // GET /api/audio/status/{slug}/{n} — poll audio generation job status // GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro package server import ( "context" "crypto/rand" "encoding/hex" "encoding/json" "fmt" "log/slog" "net/http" "sync" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/storage" ) // Server wraps an HTTP mux with the scraping endpoints. type Server struct { addr string oCfg orchestrator.Config novel scraper.NovelScraper log *slog.Logger store storage.Store mu sync.Mutex running bool kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella // voiceMu guards cachedVoices. voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices // audioMu guards audioJobIDs only. // Completed audio filenames are persisted to the Store (PocketBase). // audioJobIDs deduplicates concurrent generation requests for the same key. audioMu sync.Mutex audioJobIDs map[string]string // cacheKey → PocketBase job ID (empty string if record creation failed) // browseMu guards browseInFlight — keys currently being refreshed // in the background. browseMu sync.Mutex browseInFlight map[string]struct{} // browseMemCache is a short-lived in-process cache for browse results. // It is populated whenever a live upstream fetch succeeds and used as a // last-resort fallback when both MinIO and the upstream are unavailable. // Key: the MinIO cache key (same as used for BrowseHTMLKey). browseMemCacheMu sync.RWMutex browseMemCache map[string]browseCacheEntry } type browseCacheEntry struct { novels []NovelListing hasNext bool cachedAt time.Time } // New creates a new Server. func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server { return &Server{ addr: addr, oCfg: oCfg, novel: novel, log: log, store: store, kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, audioJobIDs: make(map[string]string), browseInFlight: make(map[string]struct{}), browseMemCache: make(map[string]browseCacheEntry), } } // voices returns the list of available Kokoro voices. On the first call it // fetches GET /v1/audio/voices from the Kokoro service and caches the result. // If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to // the hardcoded kokoroVoices list so the UI is never empty. func (s *Server) voices() []string { s.voiceMu.RLock() cached := s.cachedVoices s.voiceMu.RUnlock() if len(cached) > 0 { return cached } if s.kokoroURL == "" { return kokoroVoices } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil) if err != nil { s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err) return kokoroVoices } req.Header.Set("Accept", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err) return kokoroVoices } defer resp.Body.Close() var payload struct { Voices []string `json:"voices"` } if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&payload) != nil || len(payload.Voices) == 0 { s.log.Warn("could not fetch kokoro voices, using built-in list") return kokoroVoices } s.voiceMu.Lock() s.cachedVoices = payload.Voices s.voiceMu.Unlock() s.log.Info("fetched kokoro voices", "count", len(payload.Voices)) return payload.Voices } // ListenAndServe starts the HTTP server and blocks until the provided context // is cancelled. func (s *Server) ListenAndServe(ctx context.Context) error { mux := http.NewServeMux() mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange) // Browse API — fetches and parses novelfire catalogue page mux.HandleFunc("GET /api/browse", s.handleBrowse) // Ranking API mux.HandleFunc("GET /api/ranking", s.handleGetRanking) // Cover image proxy (serves images stored in browse MinIO bucket) mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover) // Scrape status mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus) mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks) // Re-index chapters for a book from MinIO into PocketBase chapters_idx mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex) // On-demand preview (no store writes) — for books not yet in the library mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview) mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview) // Search: local PocketBase + remote novelfire.net mux.HandleFunc("GET /api/search", s.handleSearch) // Progress API mux.HandleFunc("GET /api/progress", s.handleGetProgress) mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress) // Presigned URL API (for SvelteKit UI) mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter) mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio) mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample) // Plain-text chapter content (used server-side for audio generation) mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText) // Voices list (proxied from Kokoro) mux.HandleFunc("GET /api/voices", s.handleVoices) // Voice sample generation — generates a short audio clip for each voice // and stores it in MinIO for UI preview playback. voiceSampleHandler := http.TimeoutHandler( http.HandlerFunc(s.handleGenerateVoiceSamples), 15*time.Minute, `{"error":"voice sample generation timed out"}`, ) mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler) // Server-side audio generation via Kokoro /v1/audio/speech. // POST returns 202 immediately and starts a background goroutine; // poll GET /api/audio/status/{slug}/{n} to track progress. mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate) // Audio job status polling endpoint. mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus) // Proxy route: fetches the generated file from Kokoro /v1/download/{filename}. mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy) srv := &http.Server{ Addr: s.addr, Handler: mux, ReadTimeout: 15 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, } errCh := make(chan error, 1) go func() { errCh <- srv.ListenAndServe() }() s.log.Info("HTTP server listening", "addr", s.addr) // Pre-populate voice samples in the background so the UI voice selector // has playable previews without requiring a manual trigger. go s.warmVoiceSamples(ctx) // Warm the browse cache on startup: if page 1 is not cached in MinIO yet, // trigger a background SingleFile snapshot immediately so the first user // request is served from cache rather than hitting novelfire.net live. go s.warmBrowseCache() select { case <-ctx.Done(): shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return srv.Shutdown(shutCtx) case err := <-errCh: return err } } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } // ─── Session cookie helpers ─────────────────────────────────────────────────── const sessionCookieName = "libnovel_session" // sessionID returns the session ID from the request cookie, or "" if absent. func sessionID(r *http.Request) string { c, err := r.Cookie(sessionCookieName) if err != nil { return "" } return c.Value } // newSessionID generates a random 16-byte hex session ID. func newSessionID() (string, error) { b := make([]byte, 16) if _, err := rand.Read(b); err != nil { return "", err } return hex.EncodeToString(b), nil } // ensureSession issues a new session cookie if the request does not already // carry one, and returns the session ID (either existing or newly issued). func ensureSession(w http.ResponseWriter, r *http.Request) string { if id := sessionID(r); id != "" { return id } id, err := newSessionID() if err != nil { // Very unlikely, but fall back to a timestamp-based ID. id = fmt.Sprintf("fallback-%d", time.Now().UnixNano()) } http.SetCookie(w, &http.Cookie{ Name: sessionCookieName, Value: id, Path: "/", HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 365 * 24 * 60 * 60, // 1 year }) return id }