From db59d55f42f03b5e882a3b475cdc49840cf8006c Mon Sep 17 00:00:00 2001 From: Admin Date: Sun, 1 Mar 2026 23:37:05 +0500 Subject: [PATCH] Fix summary overlay scrolling and paragraph formatting on mobile - Make overlay scroll within the viewport instead of being clipped (overflow-y-auto on backdrop, items-start alignment) - Split summary text into paragraphs in the zoom overlay (newline-split with sentence-boundary fallback) - Add Summary heading in overlay for context - Remove old merge-poller JS; simplify generateAudio to use data.url directly (Kokoro backend rewrite cleanup) --- scraper/internal/server/server.go | 454 ++++++++----------------- scraper/internal/server/ui.go | 537 ++++++++++++++---------------- 2 files changed, 390 insertions(+), 601 deletions(-) diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 2b1097b..567b34e 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -15,7 +15,6 @@ import ( "io" "log/slog" "net/http" - "os" "strconv" "strings" "sync" @@ -39,12 +38,13 @@ type Server struct { kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella - // audioMu guards audioInFlight. - // audioInFlight maps an audio cache key to a channel that is closed when - // the in-flight Kokoro request for that key finishes (successfully or not). - // This prevents duplicate concurrent TTS generation for the same file. + // audioMu guards audioCache and audioInFlight. + // audioCache maps a cache key to the Kokoro download filename returned by + // POST /v1/audio/speech with return_download_link=true. + // audioInFlight deduplicates concurrent generation requests for the same key. audioMu sync.Mutex - audioInFlight map[string]chan struct{} + audioCache map[string]string // cacheKey → kokoro download filename + audioInFlight map[string]chan struct{} // cacheKey → closed when done } // New creates a new Server. @@ -57,6 +57,7 @@ func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log writer: writer.New(oCfg.StaticRoot), kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, + audioCache: make(map[string]string), audioInFlight: make(map[string]chan struct{}), } } @@ -82,18 +83,16 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) // Plain-text chapter content for browser-side TTS mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText) - // Server-side audio generation and serving. - // Audio generation can take several minutes for long chapters, so wrap it - // in its own timeout handler instead of relying on the server WriteTimeout. + // Server-side audio generation via Kokoro /v1/audio/speech. + // Generation can take several minutes, so wrap in its own timeout handler. audioGenHandler := http.TimeoutHandler( http.HandlerFunc(s.handleAudioGenerate), 10*time.Minute, `{"error":"audio generation timed out"}`, ) mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler) - mux.HandleFunc("GET /ui/audio/{slug}/{n}/status", s.handleAudioStatus) - mux.HandleFunc("GET /ui/audio-file/{slug}/{n}", s.handleAudioFile) - mux.HandleFunc("GET /ui/audio-file/{slug}/{n}/part/{p}", s.handleAudioFilePart) + // Proxy route: fetches the generated file from Kokoro /v1/download/{filename}. + mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy) srv := &http.Server{ Addr: s.addr, @@ -142,25 +141,18 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, stripMarkdown(raw)) } -// ─── Chunked audio generation ──────────────────────────────────────────────── +// ─── Audio generation via Kokoro /v1/audio/speech ──────────────────────────── // // handleAudioGenerate handles POST /ui/audio/{slug}/{n}. // -// Flow: -// 1. If the merged MP3 already exists on disk → return it immediately. -// 2. Otherwise split the chapter text into up to audioParts equal-ish chunks -// (by paragraph), generate part 0 synchronously (so the browser can start -// playing right away), then launch a background goroutine that generates -// parts 1…N and merges them into the final file. -// 3. Response: {"url":"","parts":N,"merged":false} -// (or {"url":"","parts":1,"merged":true} on a cache hit). +// It calls Kokoro's POST /v1/audio/speech with return_download_link=true. +// Kokoro generates the audio, saves it to its own temp storage, and returns +// the download filename in the X-Download-Path response header. +// We cache that filename (in memory, keyed by slug/chapter/voice/speed) and +// return a proxy URL that the browser sets as audio.src. // -// Deduplication: if another request is already generating the *merged* file -// for the same (slug,n,voice,speed) key, the new request blocks until it -// finishes and then serves the cached result. -const audioParts = 10 - -// handleAudioGenerate handles POST /ui/audio/{slug}/{n}. +// On a cache hit the proxy URL is returned immediately without re-generating. +// Concurrent requests for the same key are deduplicated. func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) @@ -186,18 +178,17 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { speed = body.Speed } - audioPath := s.writer.AudioPath(slug, n, voice, speed) + cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - // Fast path: merged file already on disk. - if _, err := os.Stat(audioPath); err == nil { - s.writeAudioResponse(w, slug, n, voice, speed, 1, true) + // Fast path: already generated this session. + s.audioMu.Lock() + if filename, ok := s.audioCache[cacheKey]; ok { + s.audioMu.Unlock() + s.writeAudioResponse(w, slug, n, voice, speed, filename) return } - // Deduplicate concurrent generation requests for the same merged file. - cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - - s.audioMu.Lock() + // Deduplicate concurrent generation for the same key. if ch, ok := s.audioInFlight[cacheKey]; ok { s.audioMu.Unlock() select { @@ -206,8 +197,11 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) return } - if _, err := os.Stat(audioPath); err == nil { - s.writeAudioResponse(w, slug, n, voice, speed, 1, true) + s.audioMu.Lock() + filename, ok := s.audioCache[cacheKey] + s.audioMu.Unlock() + if ok { + s.writeAudioResponse(w, slug, n, voice, speed, filename) } else { http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) } @@ -240,307 +234,141 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { return } - // Ensure audio dir exists. - if err := os.MkdirAll(s.writer.AudioDir(slug), 0o755); err != nil { - http.Error(w, `{"error":"failed to create audio dir"}`, http.StatusInternalServerError) + // Call Kokoro POST /v1/audio/speech with return_download_link=true. + // Kokoro saves the generated audio to its own temp storage and returns the + // download path in the X-Download-Path response header. + filename, err := s.generateSpeech(r.Context(), text, voice, speed) + if err != nil { + s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err) + http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway) return } - parts := splitTextIntoParts(text, audioParts) - totalParts := len(parts) + s.audioMu.Lock() + s.audioCache[cacheKey] = filename + s.audioMu.Unlock() - // Generate part 0 synchronously so the browser can start playing immediately. - if err := s.generateAudioPart(r.Context(), slug, n, voice, speed, 0, parts[0]); err != nil { - s.log.Error("part 0 generation failed", "err", err) - http.Error(w, `{"error":"part 0 generation failed"}`, http.StatusBadGateway) - return - } - - if totalParts == 1 { - // Only one part — rename it to the final path directly. - partPath := s.writer.AudioPartPath(slug, n, voice, speed, 0) - if err := os.Rename(partPath, audioPath); err != nil { - http.Error(w, `{"error":"failed to save audio"}`, http.StatusInternalServerError) - return - } - s.log.Info("audio generated (single part)", "slug", slug, "chapter", n) - s.writeAudioResponse(w, slug, n, voice, speed, 1, true) - return - } - - // Return part-0 URL immediately; generate the rest in the background. - s.writeAudioResponse(w, slug, n, voice, speed, totalParts, false) - - // Background: generate parts 1…N then merge. - go func() { - bgCtx := context.Background() - for p := 1; p < totalParts; p++ { - if err := s.generateAudioPart(bgCtx, slug, n, voice, speed, p, parts[p]); err != nil { - s.log.Error("background part generation failed", "part", p, "err", err) - return - } - } - if err := s.mergeAudioParts(slug, n, voice, speed, totalParts); err != nil { - s.log.Error("audio merge failed", "slug", slug, "chapter", n, "err", err) - return - } - s.log.Info("audio merged", "slug", slug, "chapter", n, "parts", totalParts) - }() + s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) + s.writeAudioResponse(w, slug, n, voice, speed, filename) } -// splitTextIntoParts divides text (paragraphs separated by blank lines) into -// at most n equal-ish chunks. Returns at least 1 element. -func splitTextIntoParts(text string, n int) []string { - // Split into paragraphs on blank lines. - raw := strings.Split(text, "\n\n") - var paras []string - for _, p := range raw { - p = strings.TrimSpace(p) - if p != "" { - paras = append(paras, p) - } - } - if len(paras) == 0 { - return []string{text} - } - if n > len(paras) { - n = len(paras) - } - if n < 1 { - n = 1 - } - - chunks := make([]string, n) - chunkSize := (len(paras) + n - 1) / n // ceiling division - for i := 0; i < n; i++ { - start := i * chunkSize - end := start + chunkSize - if start >= len(paras) { - // Fewer paragraphs than requested parts: return what we have. - chunks = chunks[:i] - break - } - if end > len(paras) { - end = len(paras) - } - chunks[i] = strings.Join(paras[start:end], "\n\n") - } - if len(chunks) == 0 { - return []string{text} - } - return chunks -} - -// generateAudioPart calls Kokoro for a single text chunk and writes the result -// atomically to AudioPartPath(…, part). -func (s *Server) generateAudioPart(ctx context.Context, slug string, n int, voice string, speed float64, part int, text string) error { +// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true +// and returns the filename from the X-Download-Path response header. +func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) { reqBody, _ := json.Marshal(map[string]interface{}{ - "model": "kokoro", - "input": text, - "voice": voice, - "response_format": "mp3", - "speed": speed, - "stream": false, + "model": "kokoro", + "input": text, + "voice": voice, + "response_format": "mp3", + "speed": speed, + "stream": false, + "return_download_link": true, }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody)) if err != nil { - return fmt.Errorf("build request: %w", err) + return "", fmt.Errorf("build request: %w", err) } req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { - return fmt.Errorf("kokoro request: %w", err) + return "", fmt.Errorf("kokoro request: %w", err) + } + defer resp.Body.Close() + // Drain body so the connection can be reused. + _, _ = io.Copy(io.Discard, resp.Body) + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("kokoro status %d", resp.StatusCode) + } + + // X-Download-Path is e.g. "/download/speech_abc123.mp3" + dlPath := resp.Header.Get("X-Download-Path") + if dlPath == "" { + return "", fmt.Errorf("kokoro did not return X-Download-Path header") + } + + // Extract just the filename from the path. + filename := dlPath + if idx := strings.LastIndex(dlPath, "/"); idx >= 0 { + filename = dlPath[idx+1:] + } + if filename == "" { + return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath) + } + return filename, nil +} + +// writeAudioResponse writes the JSON response for a generated audio chapter. +// The URL points to our proxy handler which fetches from Kokoro on demand. +func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) { + proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "url": proxyURL, + "filename": filename, + }) +} + +// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}. +// It looks up the Kokoro download filename for this chapter (voice/speed) and +// proxies GET /v1/download/{filename} from the Kokoro server back to the browser. +func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + http.NotFound(w, r) + return + } + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.kokoroVoice + } + speedStr := r.URL.Query().Get("speed") + speed := 1.0 + if speedStr != "" { + if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { + speed = v + } + } + + cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) + s.audioMu.Lock() + filename, ok := s.audioCache[cacheKey] + s.audioMu.Unlock() + + if !ok { + http.Error(w, "audio not generated yet", http.StatusNotFound) + return + } + + kokoroURL := s.kokoroURL + "/v1/download/" + filename + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil) + if err != nil { + http.Error(w, "failed to build proxy request", http.StatusInternalServerError) + return + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + http.Error(w, "kokoro download failed", http.StatusBadGateway) + return } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("kokoro status %d: %s", resp.StatusCode, string(body)) - } - - partPath := s.writer.AudioPartPath(slug, n, voice, speed, part) - tmpPath := partPath + ".tmp" - f, err := os.Create(tmpPath) - if err != nil { - return fmt.Errorf("create temp file: %w", err) - } - if _, err := io.Copy(f, resp.Body); err != nil { - f.Close() - os.Remove(tmpPath) - return fmt.Errorf("write audio: %w", err) - } - f.Close() - if err := os.Rename(tmpPath, partPath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("rename temp file: %w", err) - } - return nil -} - -// mergeAudioParts concatenates totalParts part files in order into AudioPath, -// then removes the individual part files. -func (s *Server) mergeAudioParts(slug string, n int, voice string, speed float64, totalParts int) error { - audioPath := s.writer.AudioPath(slug, n, voice, speed) - tmpPath := audioPath + ".tmp" - - out, err := os.Create(tmpPath) - if err != nil { - return fmt.Errorf("create merged temp: %w", err) - } - - for p := 0; p < totalParts; p++ { - partPath := s.writer.AudioPartPath(slug, n, voice, speed, p) - data, err := os.ReadFile(partPath) - if err != nil { - out.Close() - os.Remove(tmpPath) - return fmt.Errorf("read part %d: %w", p, err) - } - if _, err := out.Write(data); err != nil { - out.Close() - os.Remove(tmpPath) - return fmt.Errorf("write merged part %d: %w", p, err) - } - } - out.Close() - - if err := os.Rename(tmpPath, audioPath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("rename merged: %w", err) - } - - // Clean up part files (best-effort). - for p := 0; p < totalParts; p++ { - os.Remove(s.writer.AudioPartPath(slug, n, voice, speed, p)) - } - return nil -} - -func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, parts int, merged bool) { - var url string - if merged { - url = fmt.Sprintf("/ui/audio-file/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) - } else { - url = fmt.Sprintf("/ui/audio-file/%s/%d/part/0?voice=%s&speed=%.1f", slug, n, voice, speed) - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{ - "url": url, - "parts": parts, - "merged": merged, - }) -} - -// writeAudioURL is kept for backward compatibility (used by dedup waiters). -func (s *Server) writeAudioURL(w http.ResponseWriter, slug string, n int, voice string, speed float64) { - s.writeAudioResponse(w, slug, n, voice, speed, 1, true) -} - -// handleAudioStatus handles GET /ui/audio/{slug}/{n}/status. -// Returns {"merged":true/false,"url":"..."} so the browser can poll for the -// merged file after receiving a parts response from handleAudioGenerate. -func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest) - return - } - voice := r.URL.Query().Get("voice") - if voice == "" { - voice = s.kokoroVoice - } - speedStr := r.URL.Query().Get("speed") - speed := 1.0 - if speedStr != "" { - if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { - speed = v - } - } - - audioPath := s.writer.AudioPath(slug, n, voice, speed) - merged := false - if _, err := os.Stat(audioPath); err == nil { - merged = true - } - - mergedURL := fmt.Sprintf("/ui/audio-file/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{ - "merged": merged, - "url": mergedURL, - }) -} - -// handleAudioFile handles GET /ui/audio-file/{slug}/{n}. -// Serves the cached merged MP3 file identified by voice and speed query params. -func (s *Server) handleAudioFile(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.NotFound(w, r) - return - } - voice := r.URL.Query().Get("voice") - if voice == "" { - voice = s.kokoroVoice - } - speedStr := r.URL.Query().Get("speed") - speed := 1.0 - if speedStr != "" { - if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { - speed = v - } - } - - audioPath := s.writer.AudioPath(slug, n, voice, speed) - if _, err := os.Stat(audioPath); err != nil { - http.NotFound(w, r) + http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway) return } w.Header().Set("Content-Type", "audio/mpeg") - w.Header().Set("Cache-Control", "public, max-age=86400") - http.ServeFile(w, r, audioPath) -} - -// handleAudioFilePart handles GET /ui/audio-file/{slug}/{n}/part/{p}. -// Serves a specific MP3 part file. -func (s *Server) handleAudioFilePart(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.NotFound(w, r) - return + w.Header().Set("Cache-Control", "public, max-age=3600") + if cl := resp.Header.Get("Content-Length"); cl != "" { + w.Header().Set("Content-Length", cl) } - p, err := strconv.Atoi(r.PathValue("p")) - if err != nil || p < 0 { - http.NotFound(w, r) - return - } - voice := r.URL.Query().Get("voice") - if voice == "" { - voice = s.kokoroVoice - } - speedStr := r.URL.Query().Get("speed") - speed := 1.0 - if speedStr != "" { - if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { - speed = v - } - } - - partPath := s.writer.AudioPartPath(slug, n, voice, speed, p) - if _, err := os.Stat(partPath); err != nil { - http.NotFound(w, r) - return - } - - w.Header().Set("Content-Type", "audio/mpeg") - w.Header().Set("Cache-Control", "no-store") - http.ServeFile(w, r, partPath) + _, _ = io.Copy(w, resp.Body) } func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index 9862b91..bfac0fa 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -1429,12 +1429,13 @@ const bookTmpl = ` @@ -1475,7 +1476,7 @@ const bookTmpl = ` {{if .Meta.Summary}}

0;});}paras.forEach(function(p){var el=document.createElement('p');el.textContent=p.trim();c.appendChild(el);});o.classList.remove('hidden');o.classList.add('flex');})(this.dataset.full)" data-full="{{.Meta.Summary}}">{{.Meta.Summary}}

{{end}} @@ -1862,14 +1863,13 @@ const chapterTmpl = ` {{end}} - + @@ -1982,55 +1982,89 @@ const chapterTmpl = ` - -