feat: add Cloudflare AI TTS engine (aura-2-en) with voice grouping in UI
All checks were successful
Release / Test backend (push) Successful in 43s
Release / Check ui (push) Successful in 43s
Release / Docker / caddy (push) Successful in 46s
Release / Docker / backend (push) Successful in 2m45s
Release / Docker / runner (push) Successful in 2m53s
Release / Docker / ui (push) Successful in 2m5s
Release / Gitea Release (push) Successful in 41s

This commit is contained in:
Admin
2026-04-04 11:12:55 +05:00
parent 9806f0d894
commit aaa008ac99
12 changed files with 402 additions and 12 deletions

View File

@@ -44,6 +44,7 @@ import (
"strings"
"time"
"github.com/libnovel/backend/internal/cfai"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/meili"
@@ -774,7 +775,13 @@ func (s *Server) handleAudioStream(w http.ResponseWriter, r *http.Request) {
// Open the TTS stream (WAV or MP3 depending on format param).
var audioStream io.ReadCloser
if format == "wav" {
if pockettts.IsPocketTTSVoice(voice) {
if cfai.IsCFAIVoice(voice) {
if s.deps.CFAI == nil {
jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured")
return
}
audioStream, err = s.deps.CFAI.StreamAudioWAV(r.Context(), text, voice)
} else if pockettts.IsPocketTTSVoice(voice) {
if s.deps.PocketTTS == nil {
jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured")
return
@@ -788,7 +795,13 @@ func (s *Server) handleAudioStream(w http.ResponseWriter, r *http.Request) {
audioStream, err = s.deps.Kokoro.StreamAudioWAV(r.Context(), text, voice)
}
} else {
if pockettts.IsPocketTTSVoice(voice) {
if cfai.IsCFAIVoice(voice) {
if s.deps.CFAI == nil {
jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured")
return
}
audioStream, err = s.deps.CFAI.StreamAudioMP3(r.Context(), text, voice)
} else if pockettts.IsPocketTTSVoice(voice) {
if s.deps.PocketTTS == nil {
jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured")
return
@@ -1343,6 +1356,9 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
}
key := kokoro.VoiceSampleKey(voice)
if cfai.IsCFAIVoice(voice) {
key = cfai.VoiceSampleKey(voice)
}
// Generate sample on demand when it is not in MinIO yet.
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
@@ -1352,7 +1368,13 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
mp3 []byte
err error
)
if pockettts.IsPocketTTSVoice(voice) {
if cfai.IsCFAIVoice(voice) {
if s.deps.CFAI == nil {
jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured")
return
}
mp3, err = s.deps.CFAI.GenerateAudio(r.Context(), voiceSampleText, voice)
} else if pockettts.IsPocketTTSVoice(voice) {
if s.deps.PocketTTS == nil {
jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured")
return

View File

@@ -30,6 +30,7 @@ import (
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/cfai"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/meili"
@@ -69,6 +70,9 @@ type Dependencies struct {
// PocketTTS is the pocket-tts client (used for voice list only in the backend;
// audio generation is done by the runner).
PocketTTS pockettts.Client
// CFAI is the Cloudflare Workers AI TTS client (used for voice sample
// generation and audio-stream live TTS; audio task generation is done by the runner).
CFAI cfai.Client
// Log is the structured logger.
Log *slog.Logger
}
@@ -338,6 +342,23 @@ func (s *Server) voices(ctx context.Context) []domain.Voice {
}
}
// ── Cloudflare AI voices ──────────────────────────────────────────────────
if s.deps.CFAI != nil {
for _, speaker := range cfai.Speakers() {
gender := "m"
if cfai.IsFemale(speaker) {
gender = "f"
}
result = append(result, domain.Voice{
ID: cfai.VoiceID(speaker),
Engine: "cfai",
Lang: "en",
Gender: gender,
})
}
s.deps.Log.Info("backend: loaded CF AI voices", "count", len(cfai.Speakers()))
}
s.voiceMu.Lock()
s.cachedVoices = result
s.voiceMu.Unlock()

View File

@@ -0,0 +1,214 @@
// Package cfai provides a client for Cloudflare Workers AI Text-to-Speech models.
//
// The Cloudflare Workers AI REST API is used to run TTS models:
//
// POST https://api.cloudflare.com/client/v4/accounts/{accountID}/ai/run/{model}
// Authorization: Bearer {apiToken}
// Content-Type: application/json
// { "text": "...", "speaker": "luna" }
//
// → 200 audio/mpeg — raw MP3 bytes
//
// Currently supported model: @cf/deepgram/aura-2-en (40 English speakers).
// Voice IDs are prefixed with "cfai:" to distinguish them from Kokoro/pocket-tts
// voices (e.g. "cfai:luna", "cfai:orion").
//
// The API is batch-only (no streaming), so GenerateAudio waits for the full
// response. There is no 100-second Cloudflare proxy timeout because we are
// calling the Cloudflare API directly, not routing through a Cloudflare-proxied
// homelab tunnel.
package cfai
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
const (
// DefaultModel is the Cloudflare Workers AI TTS model used by default.
DefaultModel = "@cf/deepgram/aura-2-en"
// voicePrefix is the prefix used to namespace CF AI voice IDs.
voicePrefix = "cfai:"
)
// aura2Speakers is the exhaustive list of speakers supported by aura-2-en.
var aura2Speakers = []string{
"amalthea", "andromeda", "apollo", "arcas", "aries", "asteria",
"athena", "atlas", "aurora", "callista", "cora", "cordelia",
"delia", "draco", "electra", "harmonia", "helena", "hera",
"hermes", "hyperion", "iris", "janus", "juno", "jupiter",
"luna", "mars", "minerva", "neptune", "odysseus", "ophelia",
"orion", "orpheus", "pandora", "phoebe", "pluto", "saturn",
"thalia", "theia", "vesta", "zeus",
}
// femaleSpeakers is the set of aura-2-en speaker names that are female voices.
var femaleSpeakers = map[string]struct{}{
"amalthea": {}, "andromeda": {}, "aries": {}, "asteria": {},
"athena": {}, "aurora": {}, "callista": {}, "cora": {},
"cordelia": {}, "delia": {}, "electra": {}, "harmonia": {},
"helena": {}, "hera": {}, "iris": {}, "juno": {},
"luna": {}, "minerva": {}, "ophelia": {}, "pandora": {},
"phoebe": {}, "thalia": {}, "theia": {}, "vesta": {},
}
// IsCFAIVoice reports whether voice is served by the Cloudflare AI client.
// CF AI voices use the "cfai:" prefix, e.g. "cfai:luna".
func IsCFAIVoice(voice string) bool {
return strings.HasPrefix(voice, voicePrefix)
}
// SpeakerName strips the "cfai:" prefix and returns the bare speaker name.
// If voice is not a CF AI voice the original string is returned unchanged.
func SpeakerName(voice string) string {
return strings.TrimPrefix(voice, voicePrefix)
}
// VoiceID returns the full voice ID (with prefix) for a bare speaker name.
func VoiceID(speaker string) string {
return voicePrefix + speaker
}
// VoiceSampleKey returns the MinIO object key for a CF AI voice sample MP3.
func VoiceSampleKey(voice string) string {
safe := strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
return r
}
return '_'
}, voice)
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
}
// IsFemale reports whether the given CF AI voice ID (with or without prefix)
// is a female speaker.
func IsFemale(voice string) bool {
speaker := SpeakerName(voice)
_, ok := femaleSpeakers[speaker]
return ok
}
// Speakers returns all available bare speaker names for aura-2-en.
func Speakers() []string {
out := make([]string, len(aura2Speakers))
copy(out, aura2Speakers)
return out
}
// Client is the interface for interacting with Cloudflare Workers AI TTS.
type Client interface {
// GenerateAudio synthesises text using the given voice (e.g. "cfai:luna")
// and returns raw MP3 bytes.
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
// StreamAudioMP3 is not natively supported by the CF AI batch API.
// It buffers the full response and returns an io.ReadCloser over the bytes,
// so callers can use it like a stream without special-casing.
StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error)
// StreamAudioWAV is not natively supported; the CF AI model returns MP3.
// This method returns the same MP3 bytes wrapped as an io.ReadCloser.
StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error)
// ListVoices returns all available voice IDs (with the "cfai:" prefix).
ListVoices(ctx context.Context) ([]string, error)
}
// httpClient is the concrete CF AI HTTP client.
type httpClient struct {
accountID string
apiToken string
model string
http *http.Client
}
// New returns a Client for the given Cloudflare account and API token.
// model defaults to DefaultModel when empty.
func New(accountID, apiToken, model string) Client {
if model == "" {
model = DefaultModel
}
return &httpClient{
accountID: accountID,
apiToken: apiToken,
model: model,
http: &http.Client{Timeout: 5 * time.Minute},
}
}
// GenerateAudio calls the Cloudflare Workers AI TTS endpoint and returns MP3 bytes.
func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) {
if text == "" {
return nil, fmt.Errorf("cfai: empty text")
}
speaker := SpeakerName(voice)
if speaker == "" {
speaker = "luna"
}
body, err := json.Marshal(map[string]any{
"text": text,
"speaker": speaker,
})
if err != nil {
return nil, fmt.Errorf("cfai: marshal request: %w", err)
}
url := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/ai/run/%s",
c.accountID, c.model)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("cfai: build request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiToken)
req.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("cfai: request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("cfai: server returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
mp3, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("cfai: read response: %w", err)
}
return mp3, nil
}
// StreamAudioMP3 generates audio and wraps the MP3 bytes as an io.ReadCloser.
func (c *httpClient) StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) {
mp3, err := c.GenerateAudio(ctx, text, voice)
if err != nil {
return nil, err
}
return io.NopCloser(bytes.NewReader(mp3)), nil
}
// StreamAudioWAV generates audio (MP3) and wraps it as an io.ReadCloser.
// Note: the CF AI aura-2-en model returns MP3 regardless of the method name.
func (c *httpClient) StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) {
return c.StreamAudioMP3(ctx, text, voice)
}
// ListVoices returns all available CF AI voice IDs (with the "cfai:" prefix).
func (c *httpClient) ListVoices(_ context.Context) ([]string, error) {
ids := make([]string, len(aura2Speakers))
for i, s := range aura2Speakers {
ids[i] = VoiceID(s)
}
return ids, nil
}

View File

@@ -66,6 +66,18 @@ type PocketTTS struct {
URL string
}
// CFAI holds credentials for Cloudflare Workers AI TTS.
type CFAI struct {
// AccountID is the Cloudflare account ID.
// An empty string disables CF AI generation.
AccountID string
// APIToken is a Workers AI API token with Workers AI Read+Edit permissions.
APIToken string
// Model is the Workers AI TTS model ID.
// Defaults to "@cf/deepgram/aura-2-en" when empty.
Model string
}
// LibreTranslate holds connection settings for a self-hosted LibreTranslate instance.
type LibreTranslate struct {
// URL is the base URL of the LibreTranslate instance, e.g. https://translate.libnovel.cc
@@ -153,6 +165,7 @@ type Config struct {
MinIO MinIO
Kokoro Kokoro
PocketTTS PocketTTS
CFAI CFAI
LibreTranslate LibreTranslate
HTTP HTTP
Runner Runner
@@ -203,6 +216,12 @@ func Load() Config {
URL: envOr("POCKET_TTS_URL", ""),
},
CFAI: CFAI{
AccountID: envOr("CFAI_ACCOUNT_ID", ""),
APIToken: envOr("CFAI_API_TOKEN", ""),
Model: envOr("CFAI_TTS_MODEL", ""),
},
LibreTranslate: LibreTranslate{
URL: envOr("LIBRETRANSLATE_URL", ""),
APIKey: envOr("LIBRETRANSLATE_API_KEY", ""),

View File

@@ -27,6 +27,7 @@ import (
"go.opentelemetry.io/otel/codes"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/cfai"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/libretranslate"
@@ -112,6 +113,9 @@ type Dependencies struct {
// PocketTTS is the pocket-tts client (CPU, kyutai voices: alba, marius, etc.).
// If nil, pocket-tts voice tasks will fail with a clear error.
PocketTTS pockettts.Client
// CFAI is the Cloudflare Workers AI TTS client (cfai:* prefixed voices).
// If nil, CF AI voice tasks will fail with a clear error.
CFAI cfai.Client
// LibreTranslate is the machine translation client.
// If nil, translation tasks will fail with a clear error.
LibreTranslate libretranslate.Client
@@ -555,6 +559,18 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
return
}
log.Info("runner: audio generated via pocket-tts", "voice", task.Voice)
} else if cfai.IsCFAIVoice(task.Voice) {
if r.deps.CFAI == nil {
fail("cloudflare AI client not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN empty)")
return
}
var genErr error
audioData, genErr = r.deps.CFAI.GenerateAudio(ctx, text, task.Voice)
if genErr != nil {
fail(fmt.Sprintf("cfai generate: %v", genErr))
return
}
log.Info("runner: audio generated via cloudflare AI", "voice", task.Voice)
} else {
if r.deps.Kokoro == nil {
fail("kokoro client not configured (KOKORO_URL is empty)")