// Package server exposes the scraper as an HTTP API service. // // Endpoints: // // POST /scrape — enqueue a full catalogue scrape // POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) // GET /health — liveness probe // GET /api/progress — get reading progress map (session-scoped) // POST /api/progress/{slug} — set reading progress // DELETE /api/progress/{slug} — delete reading progress // GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown // GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio // GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped) // POST /api/audio/{slug}/{n} — trigger Kokoro audio generation // GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro package server import ( "bytes" "context" "crypto/rand" "encoding/hex" "encoding/json" "fmt" "io" "log/slog" "net/http" "strconv" "strings" "sync" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/storage" ) // Server wraps an HTTP mux with the scraping endpoints. type Server struct { addr string oCfg orchestrator.Config novel scraper.NovelScraper log *slog.Logger store storage.Store mu sync.Mutex running bool rankingRunning bool kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella // voiceMu guards cachedVoices. voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices // audioMu guards audioInFlight only. // Completed audio filenames are persisted to the Store (PocketBase). // audioInFlight deduplicates concurrent generation requests for the same key. audioMu sync.Mutex audioInFlight map[string]chan struct{} // cacheKey → closed when done } // New creates a new Server. func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server { return &Server{ addr: addr, oCfg: oCfg, novel: novel, log: log, store: store, kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, audioInFlight: make(map[string]chan struct{}), } } // voices returns the list of available Kokoro voices. On the first call it // fetches GET /v1/audio/voices from the Kokoro service and caches the result. // If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to // the hardcoded kokoroVoices list so the UI is never empty. func (s *Server) voices() []string { s.voiceMu.RLock() cached := s.cachedVoices s.voiceMu.RUnlock() if len(cached) > 0 { return cached } if s.kokoroURL != "" { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil) if err == nil { req.Header.Set("Accept", "application/json") resp, err := http.DefaultClient.Do(req) if err == nil { defer resp.Body.Close() var payload struct { Voices []string `json:"voices"` } if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 { s.voiceMu.Lock() s.cachedVoices = payload.Voices s.voiceMu.Unlock() s.log.Info("fetched kokoro voices", "count", len(payload.Voices)) return payload.Voices } } } s.log.Warn("could not fetch kokoro voices, using built-in list") } return kokoroVoices } // ListenAndServe starts the HTTP server and blocks until the provided context // is cancelled. func (s *Server) ListenAndServe(ctx context.Context) error { mux := http.NewServeMux() mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) // Progress API mux.HandleFunc("GET /api/progress", s.handleGetProgress) mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress) // Presigned URL API (for SvelteKit UI) mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter) mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio) // Plain-text chapter content (used server-side for audio generation) mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText) // Server-side audio generation via Kokoro /v1/audio/speech. // Generation can take several minutes, so wrap in its own timeout handler. audioGenHandler := http.TimeoutHandler( http.HandlerFunc(s.handleAudioGenerate), 10*time.Minute, `{"error":"audio generation timed out"}`, ) mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler) // Proxy route: fetches the generated file from Kokoro /v1/download/{filename}. mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy) srv := &http.Server{ Addr: s.addr, Handler: mux, ReadTimeout: 15 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, } errCh := make(chan error, 1) go func() { errCh <- srv.ListenAndServe() }() s.log.Info("HTTP server listening", "addr", s.addr) select { case <-ctx.Done(): shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return srv.Shutdown(shutCtx) case err := <-errCh: return err } } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } // ─── Session cookie helpers ─────────────────────────────────────────────────── const sessionCookieName = "libnovel_session" // sessionID returns the session ID from the request cookie, or "" if absent. func sessionID(r *http.Request) string { c, err := r.Cookie(sessionCookieName) if err != nil { return "" } return c.Value } // newSessionID generates a random 16-byte hex session ID. func newSessionID() (string, error) { b := make([]byte, 16) if _, err := rand.Read(b); err != nil { return "", err } return hex.EncodeToString(b), nil } // ensureSession issues a new session cookie if the request does not already // carry one, and returns the session ID (either existing or newly issued). func ensureSession(w http.ResponseWriter, r *http.Request) string { if id := sessionID(r); id != "" { return id } id, err := newSessionID() if err != nil { // Very unlikely, but fall back to a timestamp-based ID. id = fmt.Sprintf("fallback-%d", time.Now().UnixNano()) } http.SetCookie(w, &http.Cookie{ Name: sessionCookieName, Value: id, Path: "/", HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 365 * 24 * 60 * 60, // 1 year }) return id } // ─── Reading progress API ───────────────────────────────────────────────────── // handleGetProgress handles GET /api/progress. // Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...} func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) { sid := ensureSession(w, r) entries, err := s.store.AllProgress(r.Context(), sid) if err != nil { s.log.Error("AllProgress failed", "err", err) entries = nil } progress := make(map[string]interface{}, len(entries)*2) for _, p := range entries { progress[p.Slug] = p.Chapter progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli() } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(progress) } // handleSetProgress handles POST /api/progress/{slug}. // Body: {"chapter": N} func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) { sid := ensureSession(w, r) slug := r.PathValue("slug") if slug == "" { http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest) return } var body struct { Chapter int `json:"chapter"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 { http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest) return } p := storage.ReadingProgress{ Slug: slug, Chapter: body.Chapter, UpdatedAt: time.Now(), } if err := s.store.SetProgress(r.Context(), sid, p); err != nil { s.log.Error("SetProgress failed", "slug", slug, "err", err) http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{}) } // handleDeleteProgress handles DELETE /api/progress/{slug}. func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) { sid := ensureSession(w, r) slug := r.PathValue("slug") if slug == "" { http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest) return } if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil { s.log.Error("DeleteProgress failed", "slug", slug, "err", err) // Non-fatal — treat as success. } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{}) } // handleChapterText returns the plain text of a chapter (markdown stripped) // for server-side audio generation. Called by handleAudioGenerate internally. func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.NotFound(w, r) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Cache-Control", "no-store") fmt.Fprint(w, stripMarkdown(raw)) } // ─── Audio generation via Kokoro /v1/audio/speech ──────────────────────────── // // handleAudioGenerate handles POST /api/audio/{slug}/{n}. // // It calls Kokoro's POST /v1/audio/speech with return_download_link=true. // Kokoro generates the audio, saves it to its own temp storage, and returns // the download filename in the X-Download-Path response header. // We cache that filename (in memory, keyed by slug/chapter/voice/speed) and // return a proxy URL that the browser sets as audio.src. // // On a cache hit the proxy URL is returned immediately without re-generating. // Concurrent requests for the same key are deduplicated. func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest) return } // Parse optional voice/speed from JSON body. voice := s.kokoroVoice speed := 1.0 var body struct { Voice string `json:"voice"` Speed float64 `json:"speed"` } if r.Body != nil { _ = json.NewDecoder(r.Body).Decode(&body) } if body.Voice != "" { voice = body.Voice } if body.Speed > 0 { speed = body.Speed } cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) // Fast path: already generated (check persistent store first). if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) return } // Deduplicate concurrent generation for the same key. s.audioMu.Lock() if ch, ok := s.audioInFlight[cacheKey]; ok { s.audioMu.Unlock() select { case <-ch: case <-r.Context().Done(): http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) return } // Check store again after waiting. if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) } else { http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) } return } ch := make(chan struct{}) s.audioInFlight[cacheKey] = ch s.audioMu.Unlock() defer func() { s.audioMu.Lock() delete(s.audioInFlight, cacheKey) s.audioMu.Unlock() close(ch) }() // Load and validate chapter text. raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) return } text := stripMarkdown(raw) if text == "" { http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity) return } if s.kokoroURL == "" { http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable) return } // Call Kokoro POST /v1/audio/speech with return_download_link=true. // Kokoro saves the generated audio to its own temp storage and returns the // download path in the X-Download-Path response header. filename, err := s.generateSpeech(r.Context(), text, voice, speed) if err != nil { s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err) http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway) return } _ = s.store.SetAudioCache(r.Context(), cacheKey, filename) s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) s.writeAudioResponse(w, slug, n, voice, speed, filename) } // generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true // and returns the filename from the X-Download-Path response header. func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) { reqBody, _ := json.Marshal(map[string]interface{}{ "model": "kokoro", "input": text, "voice": voice, "response_format": "mp3", "speed": speed, "stream": false, "return_download_link": true, }) req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody)) if err != nil { return "", fmt.Errorf("build request: %w", err) } req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { return "", fmt.Errorf("kokoro request: %w", err) } defer resp.Body.Close() // Drain body so the connection can be reused. _, _ = io.Copy(io.Discard, resp.Body) if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("kokoro status %d", resp.StatusCode) } // X-Download-Path is e.g. "/download/speech_abc123.mp3" dlPath := resp.Header.Get("X-Download-Path") if dlPath == "" { return "", fmt.Errorf("kokoro did not return X-Download-Path header") } // Extract just the filename from the path. filename := dlPath if idx := strings.LastIndex(dlPath, "/"); idx >= 0 { filename = dlPath[idx+1:] } if filename == "" { return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath) } return filename, nil } // writeAudioResponse writes the JSON response for a generated audio chapter. // The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}. func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) { proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{ "url": proxyURL, "filename": filename, }) } // handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}. // It looks up the Kokoro download filename for this chapter (voice/speed) and // proxies GET /v1/download/{filename} from the Kokoro server back to the browser. func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } voice := r.URL.Query().Get("voice") if voice == "" { voice = s.kokoroVoice } speedStr := r.URL.Query().Get("speed") speed := 1.0 if speedStr != "" { if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { speed = v } } cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) filename, ok := s.store.GetAudioCache(r.Context(), cacheKey) if !ok { http.Error(w, "audio not generated yet", http.StatusNotFound) return } kokoroURL := s.kokoroURL + "/v1/download/" + filename req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil) if err != nil { http.Error(w, "failed to build proxy request", http.StatusInternalServerError) return } resp, err := http.DefaultClient.Do(req) if err != nil { http.Error(w, "kokoro download failed", http.StatusBadGateway) return } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway) return } w.Header().Set("Content-Type", "audio/mpeg") w.Header().Set("Cache-Control", "public, max-age=3600") if cl := resp.Header.Get("Content-Length"); cl != "" { w.Header().Set("Content-Length", cl) } _, _ = io.Copy(w, resp.Body) } // ─── Presigned URL handlers ─────────────────────────────────────────────────── // handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}. // Returns a short-lived presigned MinIO URL for the chapter markdown object. // The SvelteKit server uses this to fetch chapter content server-side. func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 || slug == "" { http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest) return } url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute) if err != nil { s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err) http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"url": url}) } // handlePresignAudio handles GET /api/presign/audio/{slug}/{n}. // Returns a presigned MinIO URL for the audio object (if it has been generated). // Query params: voice, speed (optional, defaults to server defaults). func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 || slug == "" { http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest) return } voice := r.URL.Query().Get("voice") if voice == "" { voice = s.kokoroVoice } speed := 1.0 if sv := r.URL.Query().Get("speed"); sv != "" { if v, err := strconv.ParseFloat(sv, 64); err == nil && v > 0 { speed = v } } key := s.store.AudioObjectKey(slug, n, voice, speed) url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour) if err != nil { s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err) http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"url": url}) } func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { cfg := s.oCfg cfg.SingleBookURL = "" // full catalogue s.runAsync(w, cfg) } func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { var body struct { URL string `json:"url"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) return } cfg := s.oCfg cfg.SingleBookURL = body.URL s.runAsync(w, cfg) } // runAsync launches an orchestrator in the background and returns 202 Accepted. // Only one scrape job runs at a time; concurrent requests receive 409 Conflict. func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { s.mu.Lock() if s.running { s.mu.Unlock() http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict) return } s.running = true s.mu.Unlock() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"}) go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() o := orchestrator.New(cfg, s.novel, s.log, s.store) if err := o.Run(ctx); err != nil { s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) } }() }