Add async audio generation: job tracking in PocketBase + UI polling

Replace blocking POST /api/audio with a non-blocking 202 flow: the Go
handler immediately enqueues a job in a new `audio_jobs` PocketBase
collection and returns {job_id, status}. A background goroutine runs
the actual Kokoro TTS work and updates job status (pending → generating
→ done/failed). A new GET /api/audio/status/{slug}/{n} endpoint lets
clients poll progress. The SvelteKit proxy and AudioPlayer.svelte are
updated to POST, then poll the status route every 2s until done.

scraper/internal/server/handlers_audio.go

@@ -16,17 +16,15 @@ import (
 //
 // handleAudioGenerate handles POST /api/audio/{slug}/{n}.
 //
-// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
-// Kokoro generates the audio, saves it to its own temp storage, and returns
-// the download filename in the X-Download-Path response header.
-// We cache that filename (in memory, keyed by slug/chapter/voice) and
-// return a proxy URL that the browser sets as audio.src.
+// The handler is non-blocking: it creates an audio_jobs record in PocketBase
+// with status="pending", then fires a background goroutine to call Kokoro.
+// The caller should poll GET /api/audio/status/{slug}/{n} to track progress.
 //
-// TTS is always generated at speed 1.0; playback speed is controlled
-// client-side via the <audio> element's playbackRate.
+// If audio is already cached (audio_cache hit) the handler returns
+// status=200 with the proxy URL immediately — no job is created.
 //
-// On a cache hit the proxy URL is returned immediately without re-generating.
-// Concurrent requests for the same key are deduplicated.
+// Concurrent requests for the same key are deduplicated via audioJobIDs:
+// the second caller gets a 202 with the existing job_id immediately.
 func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
 	slug := r.PathValue("slug")
 	n, err := strconv.Atoi(r.PathValue("n"))
@@ -35,8 +33,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Parse optional voice from JSON body. Speed is intentionally ignored —
-	// TTS is always generated at 1.0; playback speed is applied client-side.
+	// Parse optional voice from JSON body.
 	voice := s.kokoroVoice
 	var body struct {
 		Voice    string `json:"voice"`
@@ -58,84 +55,197 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Deduplicate concurrent generation for the same key.
+	// If a goroutine is already running for this key, return the existing job_id.
 	s.audioMu.Lock()
-	if ch, ok := s.audioInFlight[cacheKey]; ok {
+	if jobID, ok := s.audioJobIDs[cacheKey]; ok {
 		s.audioMu.Unlock()
-		select {
-		case <-ch:
-		case <-r.Context().Done():
-			http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
-			return
-		}
-		// Check store again after waiting.
-		if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
-			s.writeAudioResponse(w, slug, n, voice, filename)
-		} else {
-			http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
-		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusAccepted)
+		_ = json.NewEncoder(w).Encode(map[string]string{"job_id": jobID, "status": "generating"})
 		return
 	}
-	ch := make(chan struct{})
-	s.audioInFlight[cacheKey] = ch
+
+	// Create the PocketBase job record.
+	jobID, createErr := s.store.CreateAudioJob(r.Context(), slug, n, voice)
+	if createErr != nil {
+		s.audioMu.Unlock()
+		s.log.Warn("audio: failed to create job record", "slug", slug, "chapter", n, "err", createErr)
+		// Non-fatal: still proceed, just won't have a persistent job record.
+		jobID = ""
+	}
+
+	s.audioJobIDs[cacheKey] = jobID
 	s.audioMu.Unlock()
 
-	defer func() {
-		s.audioMu.Lock()
-		delete(s.audioInFlight, cacheKey)
-		s.audioMu.Unlock()
-		close(ch)
+	// Fire background goroutine — request context must NOT be used here since
+	// the handler returns immediately.
+	maxChars := body.MaxChars
+	go func() {
+		defer func() {
+			s.audioMu.Lock()
+			delete(s.audioJobIDs, cacheKey)
+			s.audioMu.Unlock()
+		}()
+
+		bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+
+		s.runAudioGeneration(bgCtx, jobID, slug, n, voice, maxChars, cacheKey)
 	}()
 
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	_ = json.NewEncoder(w).Encode(map[string]string{"job_id": jobID, "status": "pending"})
+}
+
+// runAudioGeneration performs the actual Kokoro TTS work in a goroutine.
+// It updates the audio_jobs record as it progresses and writes to audio_cache
+// and MinIO on success.
+func (s *Server) runAudioGeneration(ctx context.Context, jobID, slug string, n int, voice string, maxChars int, cacheKey string) {
+	markFailed := func(msg string) {
+		if jobID == "" {
+			return
+		}
+		if err := s.store.UpdateAudioJob(ctx, jobID, "failed", msg, time.Now()); err != nil {
+			s.log.Warn("audio: failed to update job to failed", "job_id", jobID, "err", err)
+		}
+	}
+
+	// Transition to "generating".
+	if jobID != "" {
+		if err := s.store.UpdateAudioJob(ctx, jobID, "generating", "", time.Time{}); err != nil {
+			s.log.Warn("audio: failed to mark job generating", "job_id", jobID, "err", err)
+		}
+	}
+
 	// Load and validate chapter text.
-	raw, err := s.store.ReadChapter(r.Context(), slug, n)
+	raw, err := s.store.ReadChapter(ctx, slug, n)
 	if err != nil {
-		http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
+		s.log.Error("audio: chapter not found", "slug", slug, "chapter", n, "err", err)
+		markFailed("chapter not found")
 		return
 	}
 	text := stripMarkdown(raw)
 	if text == "" {
-		http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
+		markFailed("chapter text is empty")
 		return
 	}
-	if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
-		text = string([]rune(text)[:body.MaxChars])
+	if maxChars > 0 && len([]rune(text)) > maxChars {
+		text = string([]rune(text)[:maxChars])
 	}
 	if s.kokoroURL == "" {
-		http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
+		markFailed("kokoro not configured")
 		return
 	}
 
-	// Call Kokoro POST /v1/audio/speech at speed 1.0.
-	// Kokoro saves the generated audio to its own temp storage and returns the
-	// download path in the X-Download-Path response header.
-	filename, err := s.generateSpeech(r.Context(), text, voice, 1.0)
+	// Call Kokoro.
+	filename, err := s.generateSpeech(ctx, text, voice, 1.0)
 	if err != nil {
-		s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
-		http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
+		s.log.Error("audio: kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
+		markFailed(err.Error())
 		return
 	}
 
-	if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
-		s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
+	if err := s.store.SetAudioCache(ctx, cacheKey, filename); err != nil {
+		s.log.Warn("audio: cache write failed", "slug", slug, "chapter", n, "err", err)
 	}
 
-	// Download generated audio from Kokoro and persist to MinIO synchronously
-	// so that the presigned URL returned to the client is immediately valid.
+	// Download from Kokoro and persist to MinIO.
 	minioKey := s.store.AudioObjectKey(slug, n, voice)
-	audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
+	audioData, dlErr := s.downloadFromKokoro(ctx, filename)
 	if dlErr != nil {
-		s.log.Warn("audio MinIO upload skipped: kokoro download failed",
+		s.log.Warn("audio: MinIO upload skipped: kokoro download failed",
 			"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
-	} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
-		s.log.Warn("audio MinIO upload failed",
+	} else if putErr := s.store.PutAudio(ctx, minioKey, audioData); putErr != nil {
+		s.log.Warn("audio: MinIO upload failed",
 			"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
-		// upload failure is non-fatal; the client can still stream via Kokoro proxy
 	} else {
-		s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
+		s.log.Info("audio: uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
 	}
 
-	s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
-	s.writeAudioResponse(w, slug, n, voice, filename)
+	// Mark job done.
+	if jobID != "" {
+		if err := s.store.UpdateAudioJob(ctx, jobID, "done", "", time.Now()); err != nil {
+			s.log.Warn("audio: failed to mark job done", "job_id", jobID, "err", err)
+		}
+	}
+	s.log.Info("audio: generation complete", "slug", slug, "chapter", n, "filename", filename)
+}
+
+// handleAudioStatus handles GET /api/audio/status/{slug}/{n}.
+// Returns the current generation status for the given chapter + voice.
+//
+// Query params: voice (optional, defaults to server default).
+//
+// Possible responses:
+//   - 200 {"status":"done","url":"/api/audio-proxy/..."}    — audio ready
+//   - 200 {"status":"pending"|"generating","job_id":"..."}  — in progress
+//   - 200 {"status":"idle"}                                 — no job yet
+//   - 200 {"status":"failed","error":"..."}                 — last job failed
+func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
+	slug := r.PathValue("slug")
+	n, err := strconv.Atoi(r.PathValue("n"))
+	if err != nil || n < 1 || slug == "" {
+		http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
+		return
+	}
+
+	voice := r.URL.Query().Get("voice")
+	if voice == "" {
+		voice = s.kokoroVoice
+	}
+
+	cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
+
+	w.Header().Set("Content-Type", "application/json")
+
+	// Fast path: audio already in audio_cache → done.
+	if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
+		proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"status":   "done",
+			"url":      proxyURL,
+			"filename": filename,
+		})
+		return
+	}
+
+	// Check in-flight map for live job ID.
+	s.audioMu.Lock()
+	liveJobID, inFlight := s.audioJobIDs[cacheKey]
+	s.audioMu.Unlock()
+
+	if inFlight {
+		// Look up persistent record for richer status.
+		if job, ok, _ := s.store.GetAudioJob(r.Context(), cacheKey); ok {
+			_ = json.NewEncoder(w).Encode(map[string]string{
+				"status": job.Status,
+				"job_id": liveJobID,
+			})
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"status": "generating",
+			"job_id": liveJobID,
+		})
+		return
+	}
+
+	// Not in-flight: check persistent record for last known result.
+	job, ok, _ := s.store.GetAudioJob(r.Context(), cacheKey)
+	if !ok {
+		_ = json.NewEncoder(w).Encode(map[string]string{"status": "idle"})
+		return
+	}
+
+	resp := map[string]string{
+		"status": job.Status,
+		"job_id": job.ID,
+	}
+	if job.Status == "failed" && job.ErrorMessage != "" {
+		resp["error"] = job.ErrorMessage
+	}
+	_ = json.NewEncoder(w).Encode(resp)
 }
 
 // generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
@@ -210,7 +320,7 @@ func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byt
 	return data, nil
 }
 
-// writeAudioResponse writes the JSON response for a generated audio chapter.
+// writeAudioResponse writes the JSON response for an already-cached audio chapter.
 // The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
 func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, filename string) {
 	proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)

scraper/internal/server/server.go

@@ -11,7 +11,8 @@
 //	GET  /api/presign/chapter/{slug}/{n}  — presigned MinIO URL for chapter markdown
 //	GET  /api/presign/audio/{slug}/{n}    — presigned MinIO URL for chapter audio
 //	GET  /api/chapter-text/{slug}/{n}     — plain text of chapter (markdown stripped)
-//	POST /api/audio/{slug}/{n}            — trigger Kokoro audio generation
+//	POST /api/audio/{slug}/{n}            — trigger Kokoro audio generation (async, returns 202)
+//	GET  /api/audio/status/{slug}/{n}     — poll audio generation job status
 //	GET  /api/audio-proxy/{slug}/{n}      — proxy generated audio from Kokoro
 package server
 
@@ -47,11 +48,11 @@ type Server struct {
 	voiceMu      sync.RWMutex
 	cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
 
-	// audioMu guards audioInFlight only.
+	// audioMu guards audioJobIDs only.
 	// Completed audio filenames are persisted to the Store (PocketBase).
-	// audioInFlight deduplicates concurrent generation requests for the same key.
-	audioMu       sync.Mutex
-	audioInFlight map[string]chan struct{} // cacheKey → closed when done
+	// audioJobIDs deduplicates concurrent generation requests for the same key.
+	audioMu     sync.Mutex
+	audioJobIDs map[string]string // cacheKey → PocketBase job ID (empty string if record creation failed)
 
 	// browseMu guards browseInFlight — keys currently being refreshed
 	// in the background.
@@ -82,7 +83,7 @@ func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log
 		store:          store,
 		kokoroURL:      kokoroURL,
 		kokoroVoice:    kokoroVoice,
-		audioInFlight:  make(map[string]chan struct{}),
+		audioJobIDs:    make(map[string]string),
 		browseInFlight: make(map[string]struct{}),
 		browseMemCache: make(map[string]browseCacheEntry),
 	}
@@ -177,13 +178,11 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
 	)
 	mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler)
 	// Server-side audio generation via Kokoro /v1/audio/speech.
-	// Generation can take several minutes, so wrap in its own timeout handler.
-	audioGenHandler := http.TimeoutHandler(
-		http.HandlerFunc(s.handleAudioGenerate),
-		10*time.Minute,
-		`{"error":"audio generation timed out"}`,
-	)
-	mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
+	// POST returns 202 immediately and starts a background goroutine;
+	// poll GET /api/audio/status/{slug}/{n} to track progress.
+	mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
+	// Audio job status polling endpoint.
+	mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
 	// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
 	mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)