Add async audio generation: job tracking in PocketBase + UI polling
Some checks failed
Deploy / Deploy Production (push) Has been skipped
Deploy / Cleanup Preview (push) Has been skipped
Deploy / Deploy Preview (push) Failing after 1s
CI / UI / Build (pull_request) Failing after 7s
CI / Scraper / Lint (pull_request) Failing after 8s
CI / Scraper / Test (pull_request) Failing after 9s
CI / Scraper / Build (pull_request) Has been skipped
iOS CI / Build (pull_request) Failing after 2s
iOS CI / Test (pull_request) Has been skipped
Some checks failed
Deploy / Deploy Production (push) Has been skipped
Deploy / Cleanup Preview (push) Has been skipped
Deploy / Deploy Preview (push) Failing after 1s
CI / UI / Build (pull_request) Failing after 7s
CI / Scraper / Lint (pull_request) Failing after 8s
CI / Scraper / Test (pull_request) Failing after 9s
CI / Scraper / Build (pull_request) Has been skipped
iOS CI / Build (pull_request) Failing after 2s
iOS CI / Test (pull_request) Has been skipped
Replace blocking POST /api/audio with a non-blocking 202 flow: the Go
handler immediately enqueues a job in a new `audio_jobs` PocketBase
collection and returns {job_id, status}. A background goroutine runs
the actual Kokoro TTS work and updates job status (pending → generating
→ done/failed). A new GET /api/audio/status/{slug}/{n} endpoint lets
clients poll progress. The SvelteKit proxy and AudioPlayer.svelte are
updated to POST, then poll the status route every 2s until done.
This commit is contained in:
@@ -173,6 +173,18 @@ func (s *mockStore) UpdateScrapeTask(_ context.Context, _ string, _ storage.Scra
|
||||
func (s *mockStore) ListScrapeTasks(_ context.Context) ([]storage.ScrapeTask, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *mockStore) CreateAudioJob(_ context.Context, _ string, _ int, _ string) (string, error) {
|
||||
return "audio-job-id", nil
|
||||
}
|
||||
func (s *mockStore) UpdateAudioJob(_ context.Context, _, _, _ string, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) GetAudioJob(_ context.Context, _ string) (storage.AudioJob, bool, error) {
|
||||
return storage.AudioJob{}, false, nil
|
||||
}
|
||||
func (s *mockStore) ListAudioJobs(_ context.Context) ([]storage.AudioJob, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -16,17 +16,15 @@ import (
|
||||
//
|
||||
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
|
||||
//
|
||||
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
|
||||
// Kokoro generates the audio, saves it to its own temp storage, and returns
|
||||
// the download filename in the X-Download-Path response header.
|
||||
// We cache that filename (in memory, keyed by slug/chapter/voice) and
|
||||
// return a proxy URL that the browser sets as audio.src.
|
||||
// The handler is non-blocking: it creates an audio_jobs record in PocketBase
|
||||
// with status="pending", then fires a background goroutine to call Kokoro.
|
||||
// The caller should poll GET /api/audio/status/{slug}/{n} to track progress.
|
||||
//
|
||||
// TTS is always generated at speed 1.0; playback speed is controlled
|
||||
// client-side via the <audio> element's playbackRate.
|
||||
// If audio is already cached (audio_cache hit) the handler returns
|
||||
// status=200 with the proxy URL immediately — no job is created.
|
||||
//
|
||||
// On a cache hit the proxy URL is returned immediately without re-generating.
|
||||
// Concurrent requests for the same key are deduplicated.
|
||||
// Concurrent requests for the same key are deduplicated via audioJobIDs:
|
||||
// the second caller gets a 202 with the existing job_id immediately.
|
||||
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
@@ -35,8 +33,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Parse optional voice from JSON body. Speed is intentionally ignored —
|
||||
// TTS is always generated at 1.0; playback speed is applied client-side.
|
||||
// Parse optional voice from JSON body.
|
||||
voice := s.kokoroVoice
|
||||
var body struct {
|
||||
Voice string `json:"voice"`
|
||||
@@ -58,84 +55,197 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Deduplicate concurrent generation for the same key.
|
||||
// If a goroutine is already running for this key, return the existing job_id.
|
||||
s.audioMu.Lock()
|
||||
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
||||
if jobID, ok := s.audioJobIDs[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-r.Context().Done():
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
// Check store again after waiting.
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, filename)
|
||||
} else {
|
||||
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"job_id": jobID, "status": "generating"})
|
||||
return
|
||||
}
|
||||
ch := make(chan struct{})
|
||||
s.audioInFlight[cacheKey] = ch
|
||||
|
||||
// Create the PocketBase job record.
|
||||
jobID, createErr := s.store.CreateAudioJob(r.Context(), slug, n, voice)
|
||||
if createErr != nil {
|
||||
s.audioMu.Unlock()
|
||||
s.log.Warn("audio: failed to create job record", "slug", slug, "chapter", n, "err", createErr)
|
||||
// Non-fatal: still proceed, just won't have a persistent job record.
|
||||
jobID = ""
|
||||
}
|
||||
|
||||
s.audioJobIDs[cacheKey] = jobID
|
||||
s.audioMu.Unlock()
|
||||
|
||||
defer func() {
|
||||
s.audioMu.Lock()
|
||||
delete(s.audioInFlight, cacheKey)
|
||||
s.audioMu.Unlock()
|
||||
close(ch)
|
||||
// Fire background goroutine — request context must NOT be used here since
|
||||
// the handler returns immediately.
|
||||
maxChars := body.MaxChars
|
||||
go func() {
|
||||
defer func() {
|
||||
s.audioMu.Lock()
|
||||
delete(s.audioJobIDs, cacheKey)
|
||||
s.audioMu.Unlock()
|
||||
}()
|
||||
|
||||
bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
s.runAudioGeneration(bgCtx, jobID, slug, n, voice, maxChars, cacheKey)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"job_id": jobID, "status": "pending"})
|
||||
}
|
||||
|
||||
// runAudioGeneration performs the actual Kokoro TTS work in a goroutine.
|
||||
// It updates the audio_jobs record as it progresses and writes to audio_cache
|
||||
// and MinIO on success.
|
||||
func (s *Server) runAudioGeneration(ctx context.Context, jobID, slug string, n int, voice string, maxChars int, cacheKey string) {
|
||||
markFailed := func(msg string) {
|
||||
if jobID == "" {
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateAudioJob(ctx, jobID, "failed", msg, time.Now()); err != nil {
|
||||
s.log.Warn("audio: failed to update job to failed", "job_id", jobID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Transition to "generating".
|
||||
if jobID != "" {
|
||||
if err := s.store.UpdateAudioJob(ctx, jobID, "generating", "", time.Time{}); err != nil {
|
||||
s.log.Warn("audio: failed to mark job generating", "job_id", jobID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Load and validate chapter text.
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
raw, err := s.store.ReadChapter(ctx, slug, n)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
||||
s.log.Error("audio: chapter not found", "slug", slug, "chapter", n, "err", err)
|
||||
markFailed("chapter not found")
|
||||
return
|
||||
}
|
||||
text := stripMarkdown(raw)
|
||||
if text == "" {
|
||||
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
|
||||
markFailed("chapter text is empty")
|
||||
return
|
||||
}
|
||||
if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
|
||||
text = string([]rune(text)[:body.MaxChars])
|
||||
if maxChars > 0 && len([]rune(text)) > maxChars {
|
||||
text = string([]rune(text)[:maxChars])
|
||||
}
|
||||
if s.kokoroURL == "" {
|
||||
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
||||
markFailed("kokoro not configured")
|
||||
return
|
||||
}
|
||||
|
||||
// Call Kokoro POST /v1/audio/speech at speed 1.0.
|
||||
// Kokoro saves the generated audio to its own temp storage and returns the
|
||||
// download path in the X-Download-Path response header.
|
||||
filename, err := s.generateSpeech(r.Context(), text, voice, 1.0)
|
||||
// Call Kokoro.
|
||||
filename, err := s.generateSpeech(ctx, text, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
||||
http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
|
||||
s.log.Error("audio: kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
||||
markFailed(err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
|
||||
s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
|
||||
if err := s.store.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
s.log.Warn("audio: cache write failed", "slug", slug, "chapter", n, "err", err)
|
||||
}
|
||||
|
||||
// Download generated audio from Kokoro and persist to MinIO synchronously
|
||||
// so that the presigned URL returned to the client is immediately valid.
|
||||
// Download from Kokoro and persist to MinIO.
|
||||
minioKey := s.store.AudioObjectKey(slug, n, voice)
|
||||
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
|
||||
audioData, dlErr := s.downloadFromKokoro(ctx, filename)
|
||||
if dlErr != nil {
|
||||
s.log.Warn("audio MinIO upload skipped: kokoro download failed",
|
||||
s.log.Warn("audio: MinIO upload skipped: kokoro download failed",
|
||||
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
|
||||
} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
|
||||
s.log.Warn("audio MinIO upload failed",
|
||||
} else if putErr := s.store.PutAudio(ctx, minioKey, audioData); putErr != nil {
|
||||
s.log.Warn("audio: MinIO upload failed",
|
||||
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
|
||||
// upload failure is non-fatal; the client can still stream via Kokoro proxy
|
||||
} else {
|
||||
s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
|
||||
s.log.Info("audio: uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
|
||||
}
|
||||
|
||||
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
||||
s.writeAudioResponse(w, slug, n, voice, filename)
|
||||
// Mark job done.
|
||||
if jobID != "" {
|
||||
if err := s.store.UpdateAudioJob(ctx, jobID, "done", "", time.Now()); err != nil {
|
||||
s.log.Warn("audio: failed to mark job done", "job_id", jobID, "err", err)
|
||||
}
|
||||
}
|
||||
s.log.Info("audio: generation complete", "slug", slug, "chapter", n, "filename", filename)
|
||||
}
|
||||
|
||||
// handleAudioStatus handles GET /api/audio/status/{slug}/{n}.
|
||||
// Returns the current generation status for the given chapter + voice.
|
||||
//
|
||||
// Query params: voice (optional, defaults to server default).
|
||||
//
|
||||
// Possible responses:
|
||||
// - 200 {"status":"done","url":"/api/audio-proxy/..."} — audio ready
|
||||
// - 200 {"status":"pending"|"generating","job_id":"..."} — in progress
|
||||
// - 200 {"status":"idle"} — no job yet
|
||||
// - 200 {"status":"failed","error":"..."} — last job failed
|
||||
func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
// Fast path: audio already in audio_cache → done.
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"status": "done",
|
||||
"url": proxyURL,
|
||||
"filename": filename,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Check in-flight map for live job ID.
|
||||
s.audioMu.Lock()
|
||||
liveJobID, inFlight := s.audioJobIDs[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
|
||||
if inFlight {
|
||||
// Look up persistent record for richer status.
|
||||
if job, ok, _ := s.store.GetAudioJob(r.Context(), cacheKey); ok {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"status": job.Status,
|
||||
"job_id": liveJobID,
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"status": "generating",
|
||||
"job_id": liveJobID,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Not in-flight: check persistent record for last known result.
|
||||
job, ok, _ := s.store.GetAudioJob(r.Context(), cacheKey)
|
||||
if !ok {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "idle"})
|
||||
return
|
||||
}
|
||||
|
||||
resp := map[string]string{
|
||||
"status": job.Status,
|
||||
"job_id": job.ID,
|
||||
}
|
||||
if job.Status == "failed" && job.ErrorMessage != "" {
|
||||
resp["error"] = job.ErrorMessage
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
|
||||
@@ -210,7 +320,7 @@ func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byt
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
||||
// writeAudioResponse writes the JSON response for an already-cached audio chapter.
|
||||
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
|
||||
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, filename string) {
|
||||
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown
|
||||
// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio
|
||||
// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped)
|
||||
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation
|
||||
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation (async, returns 202)
|
||||
// GET /api/audio/status/{slug}/{n} — poll audio generation job status
|
||||
// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro
|
||||
package server
|
||||
|
||||
@@ -47,11 +48,11 @@ type Server struct {
|
||||
voiceMu sync.RWMutex
|
||||
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
||||
|
||||
// audioMu guards audioInFlight only.
|
||||
// audioMu guards audioJobIDs only.
|
||||
// Completed audio filenames are persisted to the Store (PocketBase).
|
||||
// audioInFlight deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
||||
// audioJobIDs deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioJobIDs map[string]string // cacheKey → PocketBase job ID (empty string if record creation failed)
|
||||
|
||||
// browseMu guards browseInFlight — keys currently being refreshed
|
||||
// in the background.
|
||||
@@ -82,7 +83,7 @@ func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log
|
||||
store: store,
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
audioInFlight: make(map[string]chan struct{}),
|
||||
audioJobIDs: make(map[string]string),
|
||||
browseInFlight: make(map[string]struct{}),
|
||||
browseMemCache: make(map[string]browseCacheEntry),
|
||||
}
|
||||
@@ -177,13 +178,11 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
)
|
||||
mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler)
|
||||
// Server-side audio generation via Kokoro /v1/audio/speech.
|
||||
// Generation can take several minutes, so wrap in its own timeout handler.
|
||||
audioGenHandler := http.TimeoutHandler(
|
||||
http.HandlerFunc(s.handleAudioGenerate),
|
||||
10*time.Minute,
|
||||
`{"error":"audio generation timed out"}`,
|
||||
)
|
||||
mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
|
||||
// POST returns 202 immediately and starts a background goroutine;
|
||||
// poll GET /api/audio/status/{slug}/{n} to track progress.
|
||||
mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
|
||||
// Audio job status polling endpoint.
|
||||
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
|
||||
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
|
||||
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
|
||||
|
||||
@@ -396,6 +396,66 @@ func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error)
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// ─── Audio jobs ───────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) CreateAudioJob(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
return h.pb.CreateAudioJob(ctx, slug, chapter, voice)
|
||||
}
|
||||
|
||||
func (h *HybridStore) UpdateAudioJob(ctx context.Context, id, status, errMsg string, finished time.Time) error {
|
||||
return h.pb.UpdateAudioJob(ctx, id, status, errMsg, finished)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetAudioJob(ctx context.Context, cacheKey string) (AudioJob, bool, error) {
|
||||
rec, ok, err := h.pb.GetAudioJob(ctx, cacheKey)
|
||||
if err != nil || !ok {
|
||||
return AudioJob{}, ok, err
|
||||
}
|
||||
job := AudioJob{
|
||||
ID: strVal(rec, "id"),
|
||||
CacheKey: strVal(rec, "cache_key"),
|
||||
Slug: strVal(rec, "slug"),
|
||||
Chapter: int(floatVal(rec, "chapter")),
|
||||
Voice: strVal(rec, "voice"),
|
||||
Status: strVal(rec, "status"),
|
||||
ErrorMessage: strVal(rec, "error_message"),
|
||||
}
|
||||
if ts, ok := rec["started"].(string); ok {
|
||||
job.Started, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
if ts, ok := rec["finished"].(string); ok && ts != "" {
|
||||
job.Finished, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return job, true, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListAudioJobs(ctx context.Context) ([]AudioJob, error) {
|
||||
rows, err := h.pb.ListAudioJobs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs := make([]AudioJob, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
job := AudioJob{
|
||||
ID: strVal(r, "id"),
|
||||
CacheKey: strVal(r, "cache_key"),
|
||||
Slug: strVal(r, "slug"),
|
||||
Chapter: int(floatVal(r, "chapter")),
|
||||
Voice: strVal(r, "voice"),
|
||||
Status: strVal(r, "status"),
|
||||
ErrorMessage: strVal(r, "error_message"),
|
||||
}
|
||||
if ts, ok := r["started"].(string); ok {
|
||||
job.Started, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
if ts, ok := r["finished"].(string); ok && ts != "" {
|
||||
job.Finished, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
|
||||
|
||||
@@ -396,6 +396,20 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
|
||||
{"name": "error_message", "type": "text"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "audio_jobs",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "cache_key", "type": "text", "required": true}, // "slug/chapter/voice"
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "chapter", "type": "number"},
|
||||
{"name": "voice", "type": "text"},
|
||||
{"name": "status", "type": "text", "required": true}, // "pending" | "generating" | "done" | "failed"
|
||||
{"name": "error_message", "type": "text"},
|
||||
{"name": "started", "type": "date"},
|
||||
{"name": "finished", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "user_sessions",
|
||||
"type": "base",
|
||||
@@ -798,6 +812,76 @@ func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]i
|
||||
return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
||||
}
|
||||
|
||||
// ─── Audio jobs ───────────────────────────────────────────────────────────────
|
||||
|
||||
// CreateAudioJob inserts a new audio_jobs record with status="pending".
|
||||
func (s *PocketBaseStore) CreateAudioJob(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
|
||||
data := map[string]interface{}{
|
||||
"cache_key": cacheKey,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"voice": voice,
|
||||
"status": "pending",
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/audio_jobs/records", data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("pocketbase: CreateAudioJob: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var rec map[string]interface{}
|
||||
if err := json.Unmarshal(b, &rec); err != nil {
|
||||
return "", fmt.Errorf("pocketbase: CreateAudioJob: decode: %w", err)
|
||||
}
|
||||
id, _ := rec["id"].(string)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateAudioJob patches status, error_message, and optionally finished on an audio_jobs record.
|
||||
func (s *PocketBaseStore) UpdateAudioJob(ctx context.Context, id, status, errMsg string, finished time.Time) error {
|
||||
data := map[string]interface{}{
|
||||
"status": status,
|
||||
"error_message": errMsg,
|
||||
}
|
||||
if !finished.IsZero() {
|
||||
data["finished"] = finished.UTC().Format(time.RFC3339)
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: UpdateAudioJob id=%s: status %d: %s", id, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAudioJob returns the most recent audio_jobs record for the given cache key.
|
||||
func (s *PocketBaseStore) GetAudioJob(ctx context.Context, cacheKey string) (map[string]interface{}, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "audio_jobs",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return rec, true, nil
|
||||
}
|
||||
|
||||
// ListAudioJobs returns all audio_jobs sorted by started descending.
|
||||
func (s *PocketBaseStore) ListAudioJobs(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "audio_jobs", "", "-started")
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// pbEsc escapes a string for use in a PocketBase filter expression.
|
||||
|
||||
@@ -30,6 +30,20 @@ type ReadingProgress struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// AudioJob represents a single audio-generation job record from the
|
||||
// audio_jobs collection.
|
||||
type AudioJob struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"` // "slug/chapter/voice"
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Voice string `json:"voice"`
|
||||
Status string `json:"status"` // "pending" | "generating" | "done" | "failed"
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeTask represents a single scraping job record from the scraping_tasks
|
||||
// collection.
|
||||
type ScrapeTask struct {
|
||||
@@ -177,4 +191,17 @@ type Store interface {
|
||||
UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error
|
||||
// ListScrapeTasks returns all tasks sorted by started descending.
|
||||
ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error)
|
||||
|
||||
// ── Audio jobs ─────────────────────────────────────────────────────────
|
||||
|
||||
// CreateAudioJob inserts a new audio_jobs record with status="pending"
|
||||
// and returns the assigned ID.
|
||||
CreateAudioJob(ctx context.Context, slug string, chapter int, voice string) (string, error)
|
||||
// UpdateAudioJob patches an existing audio job record (status, error, finished).
|
||||
UpdateAudioJob(ctx context.Context, id, status, errMsg string, finished time.Time) error
|
||||
// GetAudioJob returns the most recent audio job for the given cache key,
|
||||
// or (zero, false, nil) if none exists.
|
||||
GetAudioJob(ctx context.Context, cacheKey string) (AudioJob, bool, error)
|
||||
// ListAudioJobs returns all audio jobs sorted by started descending.
|
||||
ListAudioJobs(ctx context.Context) ([]AudioJob, error)
|
||||
}
|
||||
|
||||
@@ -331,6 +331,51 @@
|
||||
return data.url;
|
||||
}
|
||||
|
||||
type AudioStatusResponse =
|
||||
| { status: 'done'; url: string; filename: string }
|
||||
| { status: 'pending' | 'generating'; job_id: string }
|
||||
| { status: 'idle' }
|
||||
| { status: 'failed'; error?: string };
|
||||
|
||||
/**
|
||||
* Poll GET /api/audio/status/[slug]/[n]?voice=... every `intervalMs` ms
|
||||
* until status is "done" or "failed" (or the caller cancels via signal).
|
||||
*
|
||||
* Returns the final status response, or throws on network error / cancellation.
|
||||
*/
|
||||
async function pollAudioStatus(
|
||||
targetSlug: string,
|
||||
targetChapter: number,
|
||||
targetVoice: string,
|
||||
intervalMs = 2000,
|
||||
signal?: AbortSignal
|
||||
): Promise<AudioStatusResponse> {
|
||||
const qs = new URLSearchParams();
|
||||
if (targetVoice) qs.set('voice', targetVoice);
|
||||
const url = `/api/audio/status/${targetSlug}/${targetChapter}?${qs.toString()}`;
|
||||
|
||||
while (true) {
|
||||
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError');
|
||||
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`Status poll HTTP ${res.status}`);
|
||||
const data = (await res.json()) as AudioStatusResponse;
|
||||
|
||||
if (data.status === 'done' || data.status === 'failed') {
|
||||
return data;
|
||||
}
|
||||
|
||||
// Still pending/generating — wait then retry.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, intervalMs);
|
||||
signal?.addEventListener('abort', () => {
|
||||
clearTimeout(timer);
|
||||
reject(new DOMException('Aborted', 'AbortError'));
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pre-fetch next chapter ─────────────────────────────────────────────────
|
||||
|
||||
async function prefetchNext() {
|
||||
@@ -354,7 +399,7 @@
|
||||
return;
|
||||
}
|
||||
|
||||
// Slow path: trigger Kokoro generation in background
|
||||
// Slow path: trigger Kokoro generation (non-blocking POST), then poll.
|
||||
const res = await fetch(`/api/audio/${slug}/${nextChapter}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -362,13 +407,31 @@
|
||||
});
|
||||
if (!res.ok) throw new Error(`Prefetch generation failed: HTTP ${res.status}`);
|
||||
|
||||
// If the scraper returned cached audio immediately (200), use the url.
|
||||
if (res.status === 200) {
|
||||
const cached = (await res.json()) as { url: string };
|
||||
stopNextProgress();
|
||||
audioStore.nextProgress = 100;
|
||||
audioStore.nextAudioUrl = cached.url;
|
||||
audioStore.nextStatus = 'prefetched';
|
||||
return;
|
||||
}
|
||||
|
||||
// 202: poll until done.
|
||||
const final = await pollAudioStatus(slug, nextChapter, voice);
|
||||
stopNextProgress();
|
||||
audioStore.nextProgress = 100;
|
||||
|
||||
const url2 = await tryPresign(slug, nextChapter, voice);
|
||||
if (!url2) throw new Error('Prefetch: audio generated but presign returned 404');
|
||||
if (final.status === 'failed') {
|
||||
throw new Error(`Prefetch failed: ${(final as { error?: string }).error ?? 'unknown'}`);
|
||||
}
|
||||
|
||||
audioStore.nextAudioUrl = url2;
|
||||
// Use the URL from the status response, or fall back to presign.
|
||||
const doneUrl =
|
||||
(final as { url?: string }).url ?? (await tryPresign(slug, nextChapter, voice));
|
||||
if (!doneUrl) throw new Error('Prefetch: audio done but no URL available');
|
||||
|
||||
audioStore.nextAudioUrl = doneUrl;
|
||||
audioStore.nextStatus = 'prefetched';
|
||||
} catch {
|
||||
stopNextProgress();
|
||||
@@ -447,7 +510,7 @@
|
||||
return;
|
||||
}
|
||||
|
||||
// Slow path: trigger Kokoro generation.
|
||||
// Slow path: trigger Kokoro generation (non-blocking POST), then poll.
|
||||
audioStore.status = 'generating';
|
||||
startProgress();
|
||||
|
||||
@@ -458,11 +521,32 @@
|
||||
});
|
||||
if (!res.ok) throw new Error(`Generation failed: HTTP ${res.status}`);
|
||||
|
||||
// If the scraper returned cached audio immediately (200), use the url.
|
||||
if (res.status === 200) {
|
||||
const cached = (await res.json()) as { url: string };
|
||||
await finishProgress();
|
||||
audioStore.audioUrl = cached.url;
|
||||
audioStore.status = 'ready';
|
||||
maybeStartPrefetch();
|
||||
return;
|
||||
}
|
||||
|
||||
// 202: poll until done.
|
||||
const final = await pollAudioStatus(slug, chapter, voice);
|
||||
|
||||
if (final.status === 'failed') {
|
||||
throw new Error(
|
||||
`Generation failed: ${(final as { error?: string }).error ?? 'unknown error'}`
|
||||
);
|
||||
}
|
||||
|
||||
await finishProgress();
|
||||
|
||||
const url2 = await tryPresign(slug, chapter, voice);
|
||||
if (!url2) throw new Error('Audio generated but presign returned 404');
|
||||
audioStore.audioUrl = url2;
|
||||
// Use the URL from the status response, or fall back to presign.
|
||||
const doneUrl =
|
||||
(final as { url?: string }).url ?? (await tryPresign(slug, chapter, voice));
|
||||
if (!doneUrl) throw new Error('Audio generated but no URL available');
|
||||
audioStore.audioUrl = doneUrl;
|
||||
audioStore.status = 'ready';
|
||||
// Don't restore time for freshly generated audio — position is 0
|
||||
// Immediately start pre-generating the next chapter in background.
|
||||
|
||||
@@ -11,8 +11,12 @@ const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
* Keeps the scraper URL server-side — the browser never needs to know it.
|
||||
*
|
||||
* Body: { voice?: string }
|
||||
* Response: { url: string, filename: string }
|
||||
* where `url` is a relative path to GET /api/audio/[slug]/[n]?voice=...
|
||||
*
|
||||
* Responses:
|
||||
* 200 { url: string, filename: string } — audio already cached; url is a
|
||||
* relative path to GET /api/audio/[slug]/[n]?voice=...
|
||||
* 202 { job_id: string, status: "pending"|"generating" } — generation
|
||||
* enqueued; poll GET /api/audio/status/[slug]/[n]?voice=... until done.
|
||||
*/
|
||||
export const POST: RequestHandler = async ({ params, request }) => {
|
||||
const { slug, n } = params;
|
||||
@@ -40,18 +44,28 @@ export const POST: RequestHandler = async ({ params, request }) => {
|
||||
error(scraperRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
|
||||
}
|
||||
|
||||
const data = (await scraperRes.json()) as { url: string; filename: string };
|
||||
const data = (await scraperRes.json()) as
|
||||
| { url: string; filename: string }
|
||||
| { job_id: string; status: string };
|
||||
|
||||
// The scraper returns a proxy URL pointing to /api/audio-proxy/... — we rewrite
|
||||
// it to our own /api/audio/[slug]/[n]?... so the browser never calls the scraper directly.
|
||||
const voice = body.voice ?? '';
|
||||
const qs = new URLSearchParams();
|
||||
if (voice) qs.set('voice', voice);
|
||||
|
||||
// 202 Accepted: generation enqueued — return job_id + status for polling.
|
||||
if (scraperRes.status === 202 || 'job_id' in data) {
|
||||
return new Response(JSON.stringify(data), {
|
||||
status: 202,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
// 200: audio was already cached — rewrite the proxy URL through our own handler.
|
||||
const cached = data as { url: string; filename: string };
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
url: `/api/audio/${slug}/${chapter}?${qs.toString()}`,
|
||||
filename: data.filename
|
||||
filename: cached.filename
|
||||
}),
|
||||
{ headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
|
||||
67
ui/src/routes/api/audio/status/[slug]/[n]/+server.ts
Normal file
67
ui/src/routes/api/audio/status/[slug]/[n]/+server.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { log } from '$lib/server/logger';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
|
||||
/**
|
||||
* GET /api/audio/status/[slug]/[n]?voice=...
|
||||
* Proxies the audio generation status check to the scraper's
|
||||
* GET /api/audio/status/{slug}/{n} endpoint.
|
||||
*
|
||||
* Possible responses from scraper (passed through as-is):
|
||||
* {"status":"done","url":"/api/audio-proxy/...","filename":"..."}
|
||||
* {"status":"pending"|"generating","job_id":"..."}
|
||||
* {"status":"idle"}
|
||||
* {"status":"failed","error":"..."}
|
||||
*
|
||||
* When status is "done" the scraper returns a proxy URL pointing to its own
|
||||
* /api/audio-proxy/... — we rewrite this to our own
|
||||
* /api/audio/[slug]/[n]?voice=... so the browser never calls the scraper.
|
||||
*/
|
||||
export const GET: RequestHandler = async ({ params, url }) => {
|
||||
const { slug, n } = params;
|
||||
const chapter = parseInt(n, 10);
|
||||
if (!slug || !chapter || chapter < 1) {
|
||||
error(400, 'Invalid slug or chapter number');
|
||||
}
|
||||
|
||||
const voice = url.searchParams.get('voice') ?? '';
|
||||
const qs = new URLSearchParams();
|
||||
if (voice) qs.set('voice', voice);
|
||||
|
||||
const scraperRes = await fetch(
|
||||
`${SCRAPER_URL}/api/audio/status/${slug}/${chapter}?${qs.toString()}`
|
||||
);
|
||||
|
||||
if (!scraperRes.ok) {
|
||||
const text = await scraperRes.text().catch(() => '');
|
||||
log.error('audio', 'scraper audio status check failed', {
|
||||
slug,
|
||||
chapter,
|
||||
status: scraperRes.status,
|
||||
body: text
|
||||
});
|
||||
error(scraperRes.status as Parameters<typeof error>[0], text || 'Status check failed');
|
||||
}
|
||||
|
||||
const data = (await scraperRes.json()) as {
|
||||
status: string;
|
||||
job_id?: string;
|
||||
url?: string;
|
||||
filename?: string;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
// Rewrite the proxy URL if the audio is done so it routes through us.
|
||||
if (data.status === 'done' && data.url) {
|
||||
const rewrittenQs = new URLSearchParams();
|
||||
if (voice) rewrittenQs.set('voice', voice);
|
||||
data.url = `/api/audio/${slug}/${chapter}?${rewrittenQs.toString()}`;
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify(data), {
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user