// Package server exposes the scraper as an HTTP service. // // Endpoints: // // POST /scrape — enqueue a full catalogue scrape // POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) // GET /health — liveness probe package server import ( "bytes" "context" "encoding/json" "fmt" "io" "log/slog" "net/http" "os" "strconv" "strings" "sync" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/writer" ) // Server wraps an HTTP mux with the scraping endpoints. type Server struct { addr string oCfg orchestrator.Config novel scraper.NovelScraper log *slog.Logger writer *writer.Writer mu sync.Mutex running bool rankingRunning bool kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella // audioMu guards audioInFlight. // audioInFlight maps an audio cache key to a channel that is closed when // the in-flight Kokoro request for that key finishes (successfully or not). // This prevents duplicate concurrent TTS generation for the same file. audioMu sync.Mutex audioInFlight map[string]chan struct{} } // New creates a new Server. func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server { return &Server{ addr: addr, oCfg: oCfg, novel: novel, log: log, writer: writer.New(oCfg.StaticRoot), kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, audioInFlight: make(map[string]chan struct{}), } } // ListenAndServe starts the HTTP server and blocks until the provided context // is cancelled. func (s *Server) ListenAndServe(ctx context.Context) error { mux := http.NewServeMux() mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) // UI routes mux.HandleFunc("GET /", s.handleHome) mux.HandleFunc("GET /scrape", s.handleScrape) mux.HandleFunc("GET /ranking", s.handleRanking) mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh) mux.HandleFunc("GET /ranking/view", s.handleRankingView) mux.HandleFunc("GET /books/{slug}", s.handleBook) mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter) mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage) mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook) mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus) mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) // Plain-text chapter content for browser-side TTS mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText) // Server-side audio generation and serving. // Audio generation can take several minutes for long chapters, so wrap it // in its own timeout handler instead of relying on the server WriteTimeout. audioGenHandler := http.TimeoutHandler( http.HandlerFunc(s.handleAudioGenerate), 10*time.Minute, `{"error":"audio generation timed out"}`, ) mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler) mux.HandleFunc("GET /ui/audio/{slug}/{n}/status", s.handleAudioStatus) mux.HandleFunc("GET /ui/audio-file/{slug}/{n}", s.handleAudioFile) mux.HandleFunc("GET /ui/audio-file/{slug}/{n}/part/{p}", s.handleAudioFilePart) srv := &http.Server{ Addr: s.addr, Handler: mux, ReadTimeout: 15 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, } errCh := make(chan error, 1) go func() { errCh <- srv.ListenAndServe() }() s.log.Info("HTTP server listening", "addr", s.addr) select { case <-ctx.Done(): shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return srv.Shutdown(shutCtx) case err := <-errCh: return err } } func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } // handleChapterText returns the plain text of a chapter (markdown stripped) // for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI. func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.NotFound(w, r) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Cache-Control", "no-store") fmt.Fprint(w, stripMarkdown(raw)) } // ─── Chunked audio generation ──────────────────────────────────────────────── // // handleAudioGenerate handles POST /ui/audio/{slug}/{n}. // // Flow: // 1. If the merged MP3 already exists on disk → return it immediately. // 2. Otherwise split the chapter text into up to audioParts equal-ish chunks // (by paragraph), generate part 0 synchronously (so the browser can start // playing right away), then launch a background goroutine that generates // parts 1…N and merges them into the final file. // 3. Response: {"url":"","parts":N,"merged":false} // (or {"url":"","parts":1,"merged":true} on a cache hit). // // Deduplication: if another request is already generating the *merged* file // for the same (slug,n,voice,speed) key, the new request blocks until it // finishes and then serves the cached result. const audioParts = 10 // handleAudioGenerate handles POST /ui/audio/{slug}/{n}. func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest) return } // Parse optional voice/speed from JSON body. voice := s.kokoroVoice speed := 1.0 var body struct { Voice string `json:"voice"` Speed float64 `json:"speed"` } if r.Body != nil { _ = json.NewDecoder(r.Body).Decode(&body) } if body.Voice != "" { voice = body.Voice } if body.Speed > 0 { speed = body.Speed } audioPath := s.writer.AudioPath(slug, n, voice, speed) // Fast path: merged file already on disk. if _, err := os.Stat(audioPath); err == nil { s.writeAudioResponse(w, slug, n, voice, speed, 1, true) return } // Deduplicate concurrent generation requests for the same merged file. cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) s.audioMu.Lock() if ch, ok := s.audioInFlight[cacheKey]; ok { s.audioMu.Unlock() select { case <-ch: case <-r.Context().Done(): http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) return } if _, err := os.Stat(audioPath); err == nil { s.writeAudioResponse(w, slug, n, voice, speed, 1, true) } else { http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) } return } ch := make(chan struct{}) s.audioInFlight[cacheKey] = ch s.audioMu.Unlock() defer func() { s.audioMu.Lock() delete(s.audioInFlight, cacheKey) s.audioMu.Unlock() close(ch) }() // Load and validate chapter text. raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) return } text := stripMarkdown(raw) if text == "" { http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity) return } if s.kokoroURL == "" { http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable) return } // Ensure audio dir exists. if err := os.MkdirAll(s.writer.AudioDir(slug), 0o755); err != nil { http.Error(w, `{"error":"failed to create audio dir"}`, http.StatusInternalServerError) return } parts := splitTextIntoParts(text, audioParts) totalParts := len(parts) // Generate part 0 synchronously so the browser can start playing immediately. if err := s.generateAudioPart(r.Context(), slug, n, voice, speed, 0, parts[0]); err != nil { s.log.Error("part 0 generation failed", "err", err) http.Error(w, `{"error":"part 0 generation failed"}`, http.StatusBadGateway) return } if totalParts == 1 { // Only one part — rename it to the final path directly. partPath := s.writer.AudioPartPath(slug, n, voice, speed, 0) if err := os.Rename(partPath, audioPath); err != nil { http.Error(w, `{"error":"failed to save audio"}`, http.StatusInternalServerError) return } s.log.Info("audio generated (single part)", "slug", slug, "chapter", n) s.writeAudioResponse(w, slug, n, voice, speed, 1, true) return } // Return part-0 URL immediately; generate the rest in the background. s.writeAudioResponse(w, slug, n, voice, speed, totalParts, false) // Background: generate parts 1…N then merge. go func() { bgCtx := context.Background() for p := 1; p < totalParts; p++ { if err := s.generateAudioPart(bgCtx, slug, n, voice, speed, p, parts[p]); err != nil { s.log.Error("background part generation failed", "part", p, "err", err) return } } if err := s.mergeAudioParts(slug, n, voice, speed, totalParts); err != nil { s.log.Error("audio merge failed", "slug", slug, "chapter", n, "err", err) return } s.log.Info("audio merged", "slug", slug, "chapter", n, "parts", totalParts) }() } // splitTextIntoParts divides text (paragraphs separated by blank lines) into // at most n equal-ish chunks. Returns at least 1 element. func splitTextIntoParts(text string, n int) []string { // Split into paragraphs on blank lines. raw := strings.Split(text, "\n\n") var paras []string for _, p := range raw { p = strings.TrimSpace(p) if p != "" { paras = append(paras, p) } } if len(paras) == 0 { return []string{text} } if n > len(paras) { n = len(paras) } if n < 1 { n = 1 } chunks := make([]string, n) chunkSize := (len(paras) + n - 1) / n // ceiling division for i := 0; i < n; i++ { start := i * chunkSize end := start + chunkSize if start >= len(paras) { // Fewer paragraphs than requested parts: return what we have. chunks = chunks[:i] break } if end > len(paras) { end = len(paras) } chunks[i] = strings.Join(paras[start:end], "\n\n") } if len(chunks) == 0 { return []string{text} } return chunks } // generateAudioPart calls Kokoro for a single text chunk and writes the result // atomically to AudioPartPath(…, part). func (s *Server) generateAudioPart(ctx context.Context, slug string, n int, voice string, speed float64, part int, text string) error { reqBody, _ := json.Marshal(map[string]interface{}{ "model": "kokoro", "input": text, "voice": voice, "response_format": "mp3", "speed": speed, "stream": false, }) req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody)) if err != nil { return fmt.Errorf("build request: %w", err) } req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { return fmt.Errorf("kokoro request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) return fmt.Errorf("kokoro status %d: %s", resp.StatusCode, string(body)) } partPath := s.writer.AudioPartPath(slug, n, voice, speed, part) tmpPath := partPath + ".tmp" f, err := os.Create(tmpPath) if err != nil { return fmt.Errorf("create temp file: %w", err) } if _, err := io.Copy(f, resp.Body); err != nil { f.Close() os.Remove(tmpPath) return fmt.Errorf("write audio: %w", err) } f.Close() if err := os.Rename(tmpPath, partPath); err != nil { os.Remove(tmpPath) return fmt.Errorf("rename temp file: %w", err) } return nil } // mergeAudioParts concatenates totalParts part files in order into AudioPath, // then removes the individual part files. func (s *Server) mergeAudioParts(slug string, n int, voice string, speed float64, totalParts int) error { audioPath := s.writer.AudioPath(slug, n, voice, speed) tmpPath := audioPath + ".tmp" out, err := os.Create(tmpPath) if err != nil { return fmt.Errorf("create merged temp: %w", err) } for p := 0; p < totalParts; p++ { partPath := s.writer.AudioPartPath(slug, n, voice, speed, p) data, err := os.ReadFile(partPath) if err != nil { out.Close() os.Remove(tmpPath) return fmt.Errorf("read part %d: %w", p, err) } if _, err := out.Write(data); err != nil { out.Close() os.Remove(tmpPath) return fmt.Errorf("write merged part %d: %w", p, err) } } out.Close() if err := os.Rename(tmpPath, audioPath); err != nil { os.Remove(tmpPath) return fmt.Errorf("rename merged: %w", err) } // Clean up part files (best-effort). for p := 0; p < totalParts; p++ { os.Remove(s.writer.AudioPartPath(slug, n, voice, speed, p)) } return nil } func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, parts int, merged bool) { var url string if merged { url = fmt.Sprintf("/ui/audio-file/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) } else { url = fmt.Sprintf("/ui/audio-file/%s/%d/part/0?voice=%s&speed=%.1f", slug, n, voice, speed) } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{ "url": url, "parts": parts, "merged": merged, }) } // writeAudioURL is kept for backward compatibility (used by dedup waiters). func (s *Server) writeAudioURL(w http.ResponseWriter, slug string, n int, voice string, speed float64) { s.writeAudioResponse(w, slug, n, voice, speed, 1, true) } // handleAudioStatus handles GET /ui/audio/{slug}/{n}/status. // Returns {"merged":true/false,"url":"..."} so the browser can poll for the // merged file after receiving a parts response from handleAudioGenerate. func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest) return } voice := r.URL.Query().Get("voice") if voice == "" { voice = s.kokoroVoice } speedStr := r.URL.Query().Get("speed") speed := 1.0 if speedStr != "" { if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { speed = v } } audioPath := s.writer.AudioPath(slug, n, voice, speed) merged := false if _, err := os.Stat(audioPath); err == nil { merged = true } mergedURL := fmt.Sprintf("/ui/audio-file/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{ "merged": merged, "url": mergedURL, }) } // handleAudioFile handles GET /ui/audio-file/{slug}/{n}. // Serves the cached merged MP3 file identified by voice and speed query params. func (s *Server) handleAudioFile(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } voice := r.URL.Query().Get("voice") if voice == "" { voice = s.kokoroVoice } speedStr := r.URL.Query().Get("speed") speed := 1.0 if speedStr != "" { if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { speed = v } } audioPath := s.writer.AudioPath(slug, n, voice, speed) if _, err := os.Stat(audioPath); err != nil { http.NotFound(w, r) return } w.Header().Set("Content-Type", "audio/mpeg") w.Header().Set("Cache-Control", "public, max-age=86400") http.ServeFile(w, r, audioPath) } // handleAudioFilePart handles GET /ui/audio-file/{slug}/{n}/part/{p}. // Serves a specific MP3 part file. func (s *Server) handleAudioFilePart(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } p, err := strconv.Atoi(r.PathValue("p")) if err != nil || p < 0 { http.NotFound(w, r) return } voice := r.URL.Query().Get("voice") if voice == "" { voice = s.kokoroVoice } speedStr := r.URL.Query().Get("speed") speed := 1.0 if speedStr != "" { if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { speed = v } } partPath := s.writer.AudioPartPath(slug, n, voice, speed, p) if _, err := os.Stat(partPath); err != nil { http.NotFound(w, r) return } w.Header().Set("Content-Type", "audio/mpeg") w.Header().Set("Cache-Control", "no-store") http.ServeFile(w, r, partPath) } func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { cfg := s.oCfg cfg.SingleBookURL = "" // full catalogue s.runAsync(w, cfg) } func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { var body struct { URL string `json:"url"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) return } cfg := s.oCfg cfg.SingleBookURL = body.URL s.runAsync(w, cfg) } // runAsync launches an orchestrator in the background and returns 202 Accepted. // Only one scrape job runs at a time; concurrent requests receive 409 Conflict. func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { s.mu.Lock() if s.running { s.mu.Unlock() http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict) return } s.running = true s.mu.Unlock() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"}) go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() o := orchestrator.New(cfg, s.novel, s.log) if err := o.Run(ctx); err != nil { s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) } }() }