// Package server exposes the scraper as an HTTP service. // // Endpoints: // // POST /scrape — enqueue a full catalogue scrape // POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) // GET /health — liveness probe package server import ( "bytes" "context" "encoding/json" "fmt" "io" "log/slog" "net/http" "strconv" "strings" "sync" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/writer" ) // Server wraps an HTTP mux with the scraping endpoints. type Server struct { addr string oCfg orchestrator.Config novel scraper.NovelScraper log *slog.Logger writer *writer.Writer mu sync.Mutex running bool rankingRunning bool kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella // voiceMu guards cachedVoices. voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices // audioMu guards audioCache and audioInFlight. // audioCache maps a cache key to the Kokoro download filename returned by // POST /v1/audio/speech with return_download_link=true. // audioInFlight deduplicates concurrent generation requests for the same key. audioMu sync.Mutex audioCache map[string]string // cacheKey → kokoro download filename audioInFlight map[string]chan struct{} // cacheKey → closed when done } // New creates a new Server. func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server { return &Server{ addr: addr, oCfg: oCfg, novel: novel, log: log, writer: writer.New(oCfg.StaticRoot), kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, audioCache: make(map[string]string), audioInFlight: make(map[string]chan struct{}), } } // voices returns the list of available Kokoro voices. On the first call it // fetches GET /v1/audio/voices from the Kokoro service and caches the result. // If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to // the hardcoded kokoroVoices list so the UI is never empty. func (s *Server) voices() []string { s.voiceMu.RLock() cached := s.cachedVoices s.voiceMu.RUnlock() if len(cached) > 0 { return cached } if s.kokoroURL != "" { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil) if err == nil { req.Header.Set("Accept", "application/json") resp, err := http.DefaultClient.Do(req) if err == nil { defer resp.Body.Close() var payload struct { Voices []string `json:"voices"` } if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 { s.voiceMu.Lock() s.cachedVoices = payload.Voices s.voiceMu.Unlock() s.log.Info("fetched kokoro voices", "count", len(payload.Voices)) return payload.Voices } } } s.log.Warn("could not fetch kokoro voices, using built-in list") } return kokoroVoices } // ListenAndServe starts the HTTP server and blocks until the provided context // is cancelled. func (s *Server) ListenAndServe(ctx context.Context) error { mux := http.NewServeMux() mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) // UI routes mux.HandleFunc("GET /", s.handleHome) mux.HandleFunc("GET /scrape", s.handleScrape) mux.HandleFunc("GET /ranking", s.handleRanking) mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh) mux.HandleFunc("GET /ranking/view", s.handleRankingView) mux.HandleFunc("GET /books/{slug}", s.handleBook) mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter) mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage) mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook) mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus) mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) // Plain-text chapter content for browser-side TTS mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText) // Server-side audio generation via Kokoro /v1/audio/speech. // Generation can take several minutes, so wrap in its own timeout handler. audioGenHandler := http.TimeoutHandler( http.HandlerFunc(s.handleAudioGenerate), 10*time.Minute, `{"error":"audio generation timed out"}`, ) mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler) // Proxy route: fetches the generated file from Kokoro /v1/download/{filename}. mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy) srv := &http.Server{ Addr: s.addr, Handler: mux, ReadTimeout: 15 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, } errCh := make(chan error, 1) go func() { errCh <- srv.ListenAndServe() }() s.log.Info("HTTP server listening", "addr", s.addr) select { case <-ctx.Done(): shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return srv.Shutdown(shutCtx) case err := <-errCh: return err } } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } // handleChapterText returns the plain text of a chapter (markdown stripped) // for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI. func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.NotFound(w, r) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Cache-Control", "no-store") fmt.Fprint(w, stripMarkdown(raw)) } // ─── Audio generation via Kokoro /v1/audio/speech ──────────────────────────── // // handleAudioGenerate handles POST /ui/audio/{slug}/{n}. // // It calls Kokoro's POST /v1/audio/speech with return_download_link=true. // Kokoro generates the audio, saves it to its own temp storage, and returns // the download filename in the X-Download-Path response header. // We cache that filename (in memory, keyed by slug/chapter/voice/speed) and // return a proxy URL that the browser sets as audio.src. // // On a cache hit the proxy URL is returned immediately without re-generating. // Concurrent requests for the same key are deduplicated. func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest) return } // Parse optional voice/speed from JSON body. voice := s.kokoroVoice speed := 1.0 var body struct { Voice string `json:"voice"` Speed float64 `json:"speed"` } if r.Body != nil { _ = json.NewDecoder(r.Body).Decode(&body) } if body.Voice != "" { voice = body.Voice } if body.Speed > 0 { speed = body.Speed } cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) // Fast path: already generated this session. s.audioMu.Lock() if filename, ok := s.audioCache[cacheKey]; ok { s.audioMu.Unlock() s.writeAudioResponse(w, slug, n, voice, speed, filename) return } // Deduplicate concurrent generation for the same key. if ch, ok := s.audioInFlight[cacheKey]; ok { s.audioMu.Unlock() select { case <-ch: case <-r.Context().Done(): http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) return } s.audioMu.Lock() filename, ok := s.audioCache[cacheKey] s.audioMu.Unlock() if ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) } else { http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) } return } ch := make(chan struct{}) s.audioInFlight[cacheKey] = ch s.audioMu.Unlock() defer func() { s.audioMu.Lock() delete(s.audioInFlight, cacheKey) s.audioMu.Unlock() close(ch) }() // Load and validate chapter text. raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) return } text := stripMarkdown(raw) if text == "" { http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity) return } if s.kokoroURL == "" { http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable) return } // Call Kokoro POST /v1/audio/speech with return_download_link=true. // Kokoro saves the generated audio to its own temp storage and returns the // download path in the X-Download-Path response header. filename, err := s.generateSpeech(r.Context(), text, voice, speed) if err != nil { s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err) http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway) return } s.audioMu.Lock() s.audioCache[cacheKey] = filename s.audioMu.Unlock() s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) s.writeAudioResponse(w, slug, n, voice, speed, filename) } // generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true // and returns the filename from the X-Download-Path response header. func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) { reqBody, _ := json.Marshal(map[string]interface{}{ "model": "kokoro", "input": text, "voice": voice, "response_format": "mp3", "speed": speed, "stream": false, "return_download_link": true, }) req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody)) if err != nil { return "", fmt.Errorf("build request: %w", err) } req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { return "", fmt.Errorf("kokoro request: %w", err) } defer resp.Body.Close() // Drain body so the connection can be reused. _, _ = io.Copy(io.Discard, resp.Body) if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("kokoro status %d", resp.StatusCode) } // X-Download-Path is e.g. "/download/speech_abc123.mp3" dlPath := resp.Header.Get("X-Download-Path") if dlPath == "" { return "", fmt.Errorf("kokoro did not return X-Download-Path header") } // Extract just the filename from the path. filename := dlPath if idx := strings.LastIndex(dlPath, "/"); idx >= 0 { filename = dlPath[idx+1:] } if filename == "" { return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath) } return filename, nil } // writeAudioResponse writes the JSON response for a generated audio chapter. // The URL points to our proxy handler which fetches from Kokoro on demand. func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) { proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{ "url": proxyURL, "filename": filename, }) } // handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}. // It looks up the Kokoro download filename for this chapter (voice/speed) and // proxies GET /v1/download/{filename} from the Kokoro server back to the browser. func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } voice := r.URL.Query().Get("voice") if voice == "" { voice = s.kokoroVoice } speedStr := r.URL.Query().Get("speed") speed := 1.0 if speedStr != "" { if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { speed = v } } cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) s.audioMu.Lock() filename, ok := s.audioCache[cacheKey] s.audioMu.Unlock() if !ok { http.Error(w, "audio not generated yet", http.StatusNotFound) return } kokoroURL := s.kokoroURL + "/v1/download/" + filename req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil) if err != nil { http.Error(w, "failed to build proxy request", http.StatusInternalServerError) return } resp, err := http.DefaultClient.Do(req) if err != nil { http.Error(w, "kokoro download failed", http.StatusBadGateway) return } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway) return } w.Header().Set("Content-Type", "audio/mpeg") w.Header().Set("Cache-Control", "public, max-age=3600") if cl := resp.Header.Get("Content-Length"); cl != "" { w.Header().Set("Content-Length", cl) } _, _ = io.Copy(w, resp.Body) } func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { cfg := s.oCfg cfg.SingleBookURL = "" // full catalogue s.runAsync(w, cfg) } func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { var body struct { URL string `json:"url"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) return } cfg := s.oCfg cfg.SingleBookURL = body.URL s.runAsync(w, cfg) } // runAsync launches an orchestrator in the background and returns 202 Accepted. // Only one scrape job runs at a time; concurrent requests receive 409 Conflict. func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { s.mu.Lock() if s.running { s.mu.Unlock() http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict) return } s.running = true s.mu.Unlock() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"}) go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() o := orchestrator.New(cfg, s.novel, s.log) if err := o.Run(ctx); err != nil { s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) } }() }