feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
Admin
160
v3/backend/internal/kokoro/client.go
Normal file
160
v3/backend/internal/kokoro/client.go
Normal file
@@ -0,0 +1,160 @@
|
||||
// Package kokoro provides a client for the Kokoro-FastAPI TTS service.
|
||||
//
|
||||
// The Kokoro API is an OpenAI-compatible audio speech API that returns a
|
||||
// download link (X-Download-Path header) instead of streaming audio directly.
|
||||
// GenerateAudio handles the two-step flow: POST /v1/audio/speech → GET /v1/download/{file}.
|
||||
package kokoro
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client is the interface for interacting with the Kokoro TTS service.
|
||||
type Client interface {
|
||||
// GenerateAudio synthesises text using voice and returns raw MP3 bytes.
|
||||
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
|
||||
|
||||
// ListVoices returns the available voice IDs. Falls back to an empty slice
|
||||
// on error — callers should treat an empty list as "service unavailable".
|
||||
ListVoices(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// httpClient is the concrete Kokoro HTTP client.
|
||||
type httpClient struct {
|
||||
baseURL string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// New returns a Kokoro Client targeting baseURL (e.g. "https://kokoro.example.com").
|
||||
func New(baseURL string) Client {
|
||||
return &httpClient{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
http: &http.Client{Timeout: 10 * time.Minute},
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateAudio calls POST /v1/audio/speech (return_download_link=true) and then
|
||||
// downloads the resulting MP3 from GET /v1/download/{filename}.
|
||||
func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) {
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("kokoro: empty text")
|
||||
}
|
||||
if voice == "" {
|
||||
voice = "af_bella"
|
||||
}
|
||||
|
||||
// ── Step 1: request generation ────────────────────────────────────────────
|
||||
reqBody, err := json.Marshal(map[string]any{
|
||||
"model": "kokoro",
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": "mp3",
|
||||
"speed": 1.0,
|
||||
"stream": false,
|
||||
"return_download_link": true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: build speech request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: speech request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("kokoro: speech returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
dlPath := resp.Header.Get("X-Download-Path")
|
||||
if dlPath == "" {
|
||||
return nil, fmt.Errorf("kokoro: no X-Download-Path header in response")
|
||||
}
|
||||
filename := dlPath
|
||||
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
||||
filename = dlPath[idx+1:]
|
||||
}
|
||||
if filename == "" {
|
||||
return nil, fmt.Errorf("kokoro: empty filename in X-Download-Path: %q", dlPath)
|
||||
}
|
||||
|
||||
// ── Step 2: download the generated file ───────────────────────────────────
|
||||
dlURL := c.baseURL + "/v1/download/" + filename
|
||||
dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: build download request: %w", err)
|
||||
}
|
||||
|
||||
dlResp, err := c.http.Do(dlReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: download request: %w", err)
|
||||
}
|
||||
defer dlResp.Body.Close()
|
||||
|
||||
if dlResp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("kokoro: download returned %d", dlResp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(dlResp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: read download body: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ListVoices calls GET /v1/audio/voices and returns the list of voice IDs.
|
||||
func (c *httpClient) ListVoices(ctx context.Context) ([]string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
c.baseURL+"/v1/audio/voices", nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: build voices request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro: voices request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("kokoro: voices returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Voices []string `json:"voices"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("kokoro: decode voices response: %w", err)
|
||||
}
|
||||
return result.Voices, nil
|
||||
}
|
||||
|
||||
// VoiceSampleKey returns the MinIO object key for a voice sample MP3.
|
||||
// Key: _voice-samples/{voice}.mp3 (sanitised).
|
||||
func VoiceSampleKey(voice string) string {
|
||||
safe := strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
|
||||
}
|
||||
291
v3/backend/internal/kokoro/client_test.go
Normal file
291
v3/backend/internal/kokoro/client_test.go
Normal file
@@ -0,0 +1,291 @@
|
||||
package kokoro_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
)
|
||||
|
||||
// ── VoiceSampleKey ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestVoiceSampleKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
voice string
|
||||
want string
|
||||
}{
|
||||
{"af_bella", "_voice-samples/af_bella.mp3"},
|
||||
{"am_echo", "_voice-samples/am_echo.mp3"},
|
||||
{"voice with spaces", "_voice-samples/voice_with_spaces.mp3"},
|
||||
{"special!@#chars", "_voice-samples/special___chars.mp3"},
|
||||
{"", "_voice-samples/.mp3"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.voice, func(t *testing.T) {
|
||||
got := kokoro.VoiceSampleKey(tt.voice)
|
||||
if got != tt.want {
|
||||
t.Errorf("VoiceSampleKey(%q) = %q, want %q", tt.voice, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── GenerateAudio ─────────────────────────────────────────────────────────────
|
||||
|
||||
func TestGenerateAudio_EmptyText(t *testing.T) {
|
||||
srv := httptest.NewServer(http.NotFoundHandler())
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.GenerateAudio(context.Background(), "", "af_bella")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty text, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "empty text") {
|
||||
t.Errorf("expected 'empty text' in error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAudio_DefaultVoice(t *testing.T) {
|
||||
// Tracks that the voice defaults to af_bella when empty.
|
||||
var capturedBody string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/speech" {
|
||||
buf := make([]byte, 512)
|
||||
n, _ := r.Body.Read(buf)
|
||||
capturedBody = string(buf[:n])
|
||||
w.Header().Set("X-Download-Path", "/download/test_file.mp3")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(r.URL.Path, "/v1/download/") {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("fake-mp3-data"))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
data, err := c.GenerateAudio(context.Background(), "hello world", "")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if string(data) != "fake-mp3-data" {
|
||||
t.Errorf("unexpected data: %q", string(data))
|
||||
}
|
||||
if !strings.Contains(capturedBody, `"af_bella"`) {
|
||||
t.Errorf("expected default voice af_bella in request body, got: %s", capturedBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAudio_SpeechNon200(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/speech" {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for non-200 speech response")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "500") {
|
||||
t.Errorf("expected 500 in error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAudio_NoDownloadPathHeader(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/speech" {
|
||||
// No X-Download-Path header
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing X-Download-Path")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "X-Download-Path") {
|
||||
t.Errorf("expected X-Download-Path in error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAudio_DownloadFails(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/speech" {
|
||||
w.Header().Set("X-Download-Path", "/v1/download/speech.mp3")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(r.URL.Path, "/v1/download/") {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for failed download")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "404") {
|
||||
t.Errorf("expected 404 in error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAudio_FullPath(t *testing.T) {
|
||||
// X-Download-Path with a full path: extract just filename.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/speech" {
|
||||
w.Header().Set("X-Download-Path", "/some/nested/path/audio_abc123.mp3")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/download/audio_abc123.mp3" {
|
||||
_, _ = w.Write([]byte("audio-bytes"))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
data, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if string(data) != "audio-bytes" {
|
||||
t.Errorf("unexpected data: %q", string(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAudio_ContextCancelled(t *testing.T) {
|
||||
// Server that hangs — context should cancel before we get a response.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Never respond.
|
||||
select {}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.GenerateAudio(ctx, "text", "af_bella")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for cancelled context")
|
||||
}
|
||||
}
|
||||
|
||||
// ── ListVoices ────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestListVoices_Success(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/voices" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"voices":["af_bella","am_adam","bf_emma"]}`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
voices, err := c.ListVoices(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(voices) != 3 {
|
||||
t.Errorf("expected 3 voices, got %d: %v", len(voices), voices)
|
||||
}
|
||||
if voices[0] != "af_bella" {
|
||||
t.Errorf("expected first voice to be af_bella, got %q", voices[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVoices_Non200(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.ListVoices(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for non-200 response")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "503") {
|
||||
t.Errorf("expected 503 in error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVoices_MalformedJSON(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(`not-json`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
_, err := c.ListVoices(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for malformed JSON")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVoices_EmptyVoices(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"voices":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL)
|
||||
voices, err := c.ListVoices(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(voices) != 0 {
|
||||
t.Errorf("expected 0 voices, got %d", len(voices))
|
||||
}
|
||||
}
|
||||
|
||||
// ── New ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestNew_TrailingSlashStripped(t *testing.T) {
|
||||
// Verify that a trailing slash on baseURL doesn't produce double-slash paths.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/audio/voices" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"voices":["af_bella"]}`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := kokoro.New(srv.URL + "/") // trailing slash
|
||||
voices, err := c.ListVoices(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(voices) == 0 {
|
||||
t.Error("expected at least one voice")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user