refactor: audit, split server.go, add unit tests, and fix latent bugs

- Remove dead code: browser cdp/content_scrape strategies, writer package,
  printUsage, downloadAndStoreCoverCLI in main.go
- Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination
  hard cap removed, splitChapterTitle off-by-one in date extraction
- Split server.go (~1700 lines) into focused handler files:
  handlers_audio, handlers_browse, handlers_progress, handlers_ranking,
  handlers_scrape
- Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go
  to consolidate duplicate helpers
- Flatten deeply nested conditionals: voices() early-return guards,
  ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead
  strings.Cut line, splitChapterTitle double-nested unit/suffix loop
- Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases),
  orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle
  (22 cases); all pass with go build/vet/test clean
This commit is contained in:
Admin
2026-03-04 22:14:23 +05:00
parent 7b48707cd9
commit fb6b364382
26 changed files with 2359 additions and 2392 deletions

View File

@@ -0,0 +1,541 @@
package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
)
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
//
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
//
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
// Kokoro generates the audio, saves it to its own temp storage, and returns
// the download filename in the X-Download-Path response header.
// We cache that filename (in memory, keyed by slug/chapter/voice) and
// return a proxy URL that the browser sets as audio.src.
//
// TTS is always generated at speed 1.0; playback speed is controlled
// client-side via the <audio> element's playbackRate.
//
// On a cache hit the proxy URL is returned immediately without re-generating.
// Concurrent requests for the same key are deduplicated.
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
return
}
// Parse optional voice from JSON body. Speed is intentionally ignored —
// TTS is always generated at 1.0; playback speed is applied client-side.
voice := s.kokoroVoice
var body struct {
Voice string `json:"voice"`
MaxChars int `json:"max_chars"`
}
if r.Body != nil {
_ = json.NewDecoder(r.Body).Decode(&body)
}
if body.Voice != "" {
voice = body.Voice
}
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
// Fast path: already generated (check persistent store first).
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, filename)
return
}
// Deduplicate concurrent generation for the same key.
s.audioMu.Lock()
if ch, ok := s.audioInFlight[cacheKey]; ok {
s.audioMu.Unlock()
select {
case <-ch:
case <-r.Context().Done():
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
return
}
// Check store again after waiting.
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, filename)
} else {
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
}
return
}
ch := make(chan struct{})
s.audioInFlight[cacheKey] = ch
s.audioMu.Unlock()
defer func() {
s.audioMu.Lock()
delete(s.audioInFlight, cacheKey)
s.audioMu.Unlock()
close(ch)
}()
// Load and validate chapter text.
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
return
}
text := stripMarkdown(raw)
if text == "" {
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
return
}
if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
text = string([]rune(text)[:body.MaxChars])
}
if s.kokoroURL == "" {
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
return
}
// Call Kokoro POST /v1/audio/speech at speed 1.0.
// Kokoro saves the generated audio to its own temp storage and returns the
// download path in the X-Download-Path response header.
filename, err := s.generateSpeech(r.Context(), text, voice, 1.0)
if err != nil {
s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
return
}
if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
}
// Download generated audio from Kokoro and persist to MinIO synchronously
// so that the presigned URL returned to the client is immediately valid.
minioKey := s.store.AudioObjectKey(slug, n, voice)
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
if dlErr != nil {
s.log.Warn("audio MinIO upload skipped: kokoro download failed",
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
s.log.Warn("audio MinIO upload failed",
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
// upload failure is non-fatal; the client can still stream via Kokoro proxy
} else {
s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
}
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
s.writeAudioResponse(w, slug, n, voice, filename)
}
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
// and returns the filename from the X-Download-Path response header.
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
reqBody, _ := json.Marshal(map[string]interface{}{
"model": "kokoro",
"input": text,
"voice": voice,
"response_format": "mp3",
"speed": speed,
"stream": false,
"return_download_link": true,
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
if err != nil {
return "", fmt.Errorf("build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", fmt.Errorf("kokoro request: %w", err)
}
defer resp.Body.Close()
// Drain body so the connection can be reused.
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
}
// X-Download-Path is e.g. "/download/speech_abc123.mp3"
dlPath := resp.Header.Get("X-Download-Path")
if dlPath == "" {
return "", fmt.Errorf("kokoro did not return X-Download-Path header")
}
// Extract just the filename from the path.
filename := dlPath
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
filename = dlPath[idx+1:]
}
if filename == "" {
return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
}
return filename, nil
}
// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
// using GET /v1/download/{filename} and returns the raw bytes.
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
url := s.kokoroURL + "/v1/download/" + filename
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("build download request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("kokoro download request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read kokoro download body: %w", err)
}
return data, nil
}
// writeAudioResponse writes the JSON response for a generated audio chapter.
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, filename string) {
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"url": proxyURL,
"filename": filename,
})
}
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
// It looks up the Kokoro download filename for this chapter (voice) and
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
http.NotFound(w, r)
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.kokoroVoice
}
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
if !ok {
http.Error(w, "audio not generated yet", http.StatusNotFound)
return
}
kokoroURL := s.kokoroURL + "/v1/download/" + filename
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
if err != nil {
http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
http.Error(w, "kokoro download failed", http.StatusBadGateway)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "audio/mpeg")
w.Header().Set("Cache-Control", "public, max-age=3600")
if cl := resp.Header.Get("Content-Length"); cl != "" {
w.Header().Set("Content-Length", cl)
}
_, _ = io.Copy(w, resp.Body)
}
// ─── Presigned URL handlers ───────────────────────────────────────────────────
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
// The SvelteKit server uses this to fetch chapter content server-side.
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
return
}
url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
if err != nil {
s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
// Returns a presigned MinIO URL for the audio object (if it has been generated).
// Query params: voice (optional, defaults to server default).
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.kokoroVoice
}
key := s.store.AudioObjectKey(slug, n, voice)
// Return 404 when the object hasn't been uploaded yet — the client treats
// this as "audio not ready" and will either poll or trigger generation.
if !s.store.AudioExists(r.Context(), key) {
http.NotFound(w, r)
return
}
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
if err != nil {
s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}
// ─── Voices API ───────────────────────────────────────────────────────────────
// handleVoices handles GET /api/voices.
// Returns the list of available Kokoro voices as JSON: {"voices": [...]}
func (s *Server) handleVoices(w http.ResponseWriter, _ *http.Request) {
voices := s.voices()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]interface{}{"voices": voices})
}
// ─── Voice sample generation ──────────────────────────────────────────────────
// voiceSampleText is the short passage used for voice sample previews.
const voiceSampleText = "The ancient library held secrets older than memory itself, its dust-laden shelves stretching upward into shadow. She reached for the worn leather spine, fingers trembling with anticipation."
// voiceSampleKey returns the MinIO object key for a voice sample.
// Key: _voice-samples/{voice}.mp3
func voiceSampleKey(voice string) string {
safe := strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
return r
}
return '_'
}, voice)
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
}
// warmVoiceSamples runs at startup in a background goroutine.
// It generates a short audio sample for every available Kokoro voice that
// doesn't already have one in MinIO, so the UI voice selector has playable
// previews without requiring a manual trigger.
// It respects ctx cancellation and waits up to 30 s for Kokoro to become
// reachable before giving up.
func (s *Server) warmVoiceSamples(ctx context.Context) {
if s.kokoroURL == "" {
return
}
// Wait for Kokoro to be reachable (it may still be starting up).
deadline := time.Now().Add(30 * time.Second)
for time.Now().Before(deadline) {
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
resp, err := http.DefaultClient.Do(req)
if err == nil {
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
break
}
}
select {
case <-ctx.Done():
return
case <-time.After(3 * time.Second):
}
}
voices := s.voices()
s.log.Info("warming voice samples", "voices", len(voices))
generated, skipped, failed := 0, 0, 0
for _, voice := range voices {
if ctx.Err() != nil {
return
}
key := voiceSampleKey(voice)
if s.store.AudioExists(ctx, key) {
skipped++
continue
}
filename, err := s.generateSpeech(ctx, voiceSampleText, voice, 1.0)
if err != nil {
s.log.Warn("voice sample warmup: generation failed", "voice", voice, "err", err)
failed++
continue
}
audioData, err := s.downloadFromKokoro(ctx, filename)
if err != nil {
s.log.Warn("voice sample warmup: download failed", "voice", voice, "err", err)
failed++
continue
}
if err := s.store.PutAudio(ctx, key, audioData); err != nil {
s.log.Warn("voice sample warmup: upload failed", "voice", voice, "key", key, "err", err)
failed++
continue
}
s.log.Debug("voice sample warmed", "voice", voice)
generated++
}
s.log.Info("voice sample warmup complete",
"generated", generated, "skipped", skipped, "failed", failed)
}
// handleGenerateVoiceSamples handles POST /api/audio/voice-samples.
// It generates short audio samples for each available voice and stores them
// in the audio MinIO bucket so the UI can play them during voice selection.
// Already-generated samples are skipped (idempotent).
// Optional JSON body: {"voices": ["af_bella", ...]} to generate a subset.
// Returns: {"generated": [...], "skipped": [...], "failed": [...]}
func (s *Server) handleGenerateVoiceSamples(w http.ResponseWriter, r *http.Request) {
if s.kokoroURL == "" {
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
return
}
// Parse optional voice list from body.
var body struct {
Voices []string `json:"voices"`
}
if r.Body != nil {
_ = json.NewDecoder(r.Body).Decode(&body)
}
targetVoices := body.Voices
if len(targetVoices) == 0 {
targetVoices = s.voices()
}
type result struct {
Generated []string `json:"generated"`
Skipped []string `json:"skipped"`
Failed []string `json:"failed"`
}
var res result
for _, voice := range targetVoices {
key := voiceSampleKey(voice)
// Skip if already uploaded.
if s.store.AudioExists(r.Context(), key) {
res.Skipped = append(res.Skipped, voice)
s.log.Debug("voice sample already exists, skipping", "voice", voice)
continue
}
// Generate via Kokoro (speed 1.0 for samples).
filename, err := s.generateSpeech(r.Context(), voiceSampleText, voice, 1.0)
if err != nil {
s.log.Warn("voice sample generation failed", "voice", voice, "err", err)
res.Failed = append(res.Failed, voice)
continue
}
// Download from Kokoro and upload to MinIO.
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
if dlErr != nil {
s.log.Warn("voice sample kokoro download failed", "voice", voice, "err", dlErr)
res.Failed = append(res.Failed, voice)
continue
}
if putErr := s.store.PutAudio(r.Context(), key, audioData); putErr != nil {
s.log.Warn("voice sample MinIO upload failed", "voice", voice, "key", key, "err", putErr)
res.Failed = append(res.Failed, voice)
continue
}
s.log.Info("voice sample generated", "voice", voice, "key", key)
res.Generated = append(res.Generated, voice)
}
if res.Generated == nil {
res.Generated = []string{}
}
if res.Skipped == nil {
res.Skipped = []string{}
}
if res.Failed == nil {
res.Failed = []string{}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(res)
}
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
// Returns a presigned URL for the voice sample audio file stored in MinIO.
// Returns 404 if the sample has not been generated yet.
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
voice := r.PathValue("voice")
if voice == "" {
http.Error(w, `{"error":"missing voice"}`, http.StatusBadRequest)
return
}
key := voiceSampleKey(voice)
if !s.store.AudioExists(r.Context(), key) {
http.NotFound(w, r)
return
}
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
if err != nil {
s.log.Error("presign voice sample failed", "voice", voice, "err", err)
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}