Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 55s
Release / Test backend (push) Successful in 56s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 1m6s
Release / Docker / caddy (push) Successful in 41s
CI / Backend (pull_request) Successful in 58s
CI / UI (pull_request) Successful in 55s
Release / Docker / runner (push) Failing after 55s
Release / Docker / backend (push) Failing after 1m23s
Release / Gitea Release (push) Has been skipped
Add GET /api/audio-stream/{slug}/{n}?voice= that streams MP3 audio to the
client as TTS generates it, while simultaneously uploading to MinIO. On
subsequent requests the endpoint redirects to the presigned MinIO URL,
skipping generation entirely.
- PocketTTS: StreamAudioMP3 pipes live WAV response body through ffmpeg
(streaming transcode — no full-buffer wait)
- Kokoro: StreamAudioMP3 uses stream:true mode, returning MP3 frames
directly without the two-step download-link flow
- AudioStore: PutAudioStream added for multipart MinIO upload from reader
- WriteTimeout bumped 60s → 15min to accommodate full-chapter streams
- X-Accel-Buffering: no header disables Caddy/nginx response buffering
389 lines
12 KiB
Go
389 lines
12 KiB
Go
package runner_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/libnovel/backend/internal/domain"
|
|
"github.com/libnovel/backend/internal/runner"
|
|
)
|
|
|
|
// ── Stub types ────────────────────────────────────────────────────────────────
|
|
|
|
// stubConsumer is a test double for taskqueue.Consumer.
|
|
type stubConsumer struct {
|
|
scrapeQueue []domain.ScrapeTask
|
|
audioQueue []domain.AudioTask
|
|
scrapeIdx int
|
|
audioIdx int
|
|
finished []string
|
|
failCalled []string
|
|
claimErr error
|
|
}
|
|
|
|
func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
|
if s.claimErr != nil {
|
|
return domain.ScrapeTask{}, false, s.claimErr
|
|
}
|
|
if s.scrapeIdx >= len(s.scrapeQueue) {
|
|
return domain.ScrapeTask{}, false, nil
|
|
}
|
|
t := s.scrapeQueue[s.scrapeIdx]
|
|
s.scrapeIdx++
|
|
return t, true, nil
|
|
}
|
|
|
|
func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
|
if s.claimErr != nil {
|
|
return domain.AudioTask{}, false, s.claimErr
|
|
}
|
|
if s.audioIdx >= len(s.audioQueue) {
|
|
return domain.AudioTask{}, false, nil
|
|
}
|
|
t := s.audioQueue[s.audioIdx]
|
|
s.audioIdx++
|
|
return t, true, nil
|
|
}
|
|
|
|
func (s *stubConsumer) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
|
|
return domain.TranslationTask{}, false, nil
|
|
}
|
|
|
|
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
|
|
s.finished = append(s.finished, id)
|
|
return nil
|
|
}
|
|
|
|
func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error {
|
|
s.finished = append(s.finished, id)
|
|
return nil
|
|
}
|
|
|
|
func (s *stubConsumer) FinishTranslationTask(_ context.Context, id string, _ domain.TranslationResult) error {
|
|
s.finished = append(s.finished, id)
|
|
return nil
|
|
}
|
|
|
|
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
|
|
s.failCalled = append(s.failCalled, id)
|
|
return nil
|
|
}
|
|
|
|
func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
|
|
|
func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
// stubBookWriter satisfies bookstore.BookWriter (no-op).
|
|
type stubBookWriter struct{}
|
|
|
|
func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
|
|
func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error {
|
|
return nil
|
|
}
|
|
func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
|
|
return nil
|
|
}
|
|
func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
|
|
return false
|
|
}
|
|
|
|
// stubBookReader satisfies bookstore.BookReader — returns a single chapter.
|
|
type stubBookReader struct {
|
|
text string
|
|
readErr error
|
|
}
|
|
|
|
func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
|
|
return s.text, s.readErr
|
|
}
|
|
func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
|
|
return domain.BookMeta{}, false, nil
|
|
}
|
|
func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
|
|
func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil }
|
|
func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
|
|
func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
|
|
return nil, nil
|
|
}
|
|
func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 }
|
|
func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
// stubAudioStore satisfies bookstore.AudioStore.
|
|
type stubAudioStore struct {
|
|
putCalled atomic.Int32
|
|
putErr error
|
|
}
|
|
|
|
func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string {
|
|
return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3"
|
|
}
|
|
func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false }
|
|
func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error {
|
|
s.putCalled.Add(1)
|
|
return s.putErr
|
|
}
|
|
func (s *stubAudioStore) PutAudioStream(_ context.Context, _ string, _ io.Reader, _ int64, _ string) error {
|
|
s.putCalled.Add(1)
|
|
return s.putErr
|
|
}
|
|
|
|
// stubNovelScraper satisfies scraper.NovelScraper minimally.
|
|
type stubNovelScraper struct {
|
|
entries []domain.CatalogueEntry
|
|
metaErr error
|
|
chapters []domain.ChapterRef
|
|
}
|
|
|
|
func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
|
ch := make(chan domain.CatalogueEntry, len(s.entries))
|
|
errCh := make(chan error, 1)
|
|
for _, e := range s.entries {
|
|
ch <- e
|
|
}
|
|
close(ch)
|
|
close(errCh)
|
|
return ch, errCh
|
|
}
|
|
|
|
func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
|
|
if s.metaErr != nil {
|
|
return domain.BookMeta{}, s.metaErr
|
|
}
|
|
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
|
|
}
|
|
|
|
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
|
|
return s.chapters, nil
|
|
}
|
|
|
|
func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
|
return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil
|
|
}
|
|
|
|
func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) {
|
|
ch := make(chan domain.BookMeta)
|
|
errCh := make(chan error, 1)
|
|
close(ch)
|
|
close(errCh)
|
|
return ch, errCh
|
|
}
|
|
|
|
func (s *stubNovelScraper) SourceName() string { return "stub" }
|
|
|
|
// stubKokoro satisfies kokoro.Client.
|
|
type stubKokoro struct {
|
|
data []byte
|
|
genErr error
|
|
called atomic.Int32
|
|
}
|
|
|
|
func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
|
|
s.called.Add(1)
|
|
return s.data, s.genErr
|
|
}
|
|
|
|
func (s *stubKokoro) StreamAudioMP3(_ context.Context, _, _ string) (io.ReadCloser, error) {
|
|
s.called.Add(1)
|
|
if s.genErr != nil {
|
|
return nil, s.genErr
|
|
}
|
|
return io.NopCloser(bytes.NewReader(s.data)), nil
|
|
}
|
|
|
|
func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) {
|
|
return []string{"af_bella"}, nil
|
|
}
|
|
|
|
// ── stripMarkdown helper ──────────────────────────────────────────────────────
|
|
|
|
func TestStripMarkdownViaAudioTask(t *testing.T) {
|
|
// Verify markdown is stripped before sending to Kokoro.
|
|
// We inject chapter text with markdown; the kokoro stub verifies data flows.
|
|
consumer := &stubConsumer{
|
|
audioQueue: []domain.AudioTask{
|
|
{ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
|
},
|
|
}
|
|
bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."}
|
|
audioStore := &stubAudioStore{}
|
|
kokoroStub := &stubKokoro{data: []byte("mp3")}
|
|
|
|
cfg := runner.Config{
|
|
WorkerID: "test",
|
|
PollInterval: time.Hour, // long poll — we'll cancel manually
|
|
}
|
|
deps := runner.Dependencies{
|
|
Consumer: consumer,
|
|
BookWriter: &stubBookWriter{},
|
|
BookReader: bookReader,
|
|
AudioStore: audioStore,
|
|
Novel: &stubNovelScraper{},
|
|
Kokoro: kokoroStub,
|
|
}
|
|
|
|
r := runner.New(cfg, deps)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = r.Run(ctx)
|
|
|
|
if kokoroStub.called.Load() != 1 {
|
|
t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load())
|
|
}
|
|
if audioStore.putCalled.Load() != 1 {
|
|
t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load())
|
|
}
|
|
}
|
|
|
|
func TestAudioTask_ReadChapterError(t *testing.T) {
|
|
consumer := &stubConsumer{
|
|
audioQueue: []domain.AudioTask{
|
|
{ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
|
},
|
|
}
|
|
bookReader := &stubBookReader{readErr: errors.New("chapter not found")}
|
|
audioStore := &stubAudioStore{}
|
|
kokoroStub := &stubKokoro{data: []byte("mp3")}
|
|
|
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
|
deps := runner.Dependencies{
|
|
Consumer: consumer,
|
|
BookWriter: &stubBookWriter{},
|
|
BookReader: bookReader,
|
|
AudioStore: audioStore,
|
|
Novel: &stubNovelScraper{},
|
|
Kokoro: kokoroStub,
|
|
}
|
|
|
|
r := runner.New(cfg, deps)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = r.Run(ctx)
|
|
|
|
// Kokoro should not be called; FinishAudioTask should be called with error.
|
|
if kokoroStub.called.Load() != 0 {
|
|
t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load())
|
|
}
|
|
if len(consumer.finished) != 1 {
|
|
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
|
|
}
|
|
}
|
|
|
|
func TestAudioTask_KokoroError(t *testing.T) {
|
|
consumer := &stubConsumer{
|
|
audioQueue: []domain.AudioTask{
|
|
{ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
|
},
|
|
}
|
|
bookReader := &stubBookReader{text: "Chapter text."}
|
|
audioStore := &stubAudioStore{}
|
|
kokoroStub := &stubKokoro{genErr: errors.New("tts failed")}
|
|
|
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
|
deps := runner.Dependencies{
|
|
Consumer: consumer,
|
|
BookWriter: &stubBookWriter{},
|
|
BookReader: bookReader,
|
|
AudioStore: audioStore,
|
|
Novel: &stubNovelScraper{},
|
|
Kokoro: kokoroStub,
|
|
}
|
|
|
|
r := runner.New(cfg, deps)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = r.Run(ctx)
|
|
|
|
if audioStore.putCalled.Load() != 0 {
|
|
t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load())
|
|
}
|
|
if len(consumer.finished) != 1 {
|
|
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
|
|
}
|
|
}
|
|
|
|
func TestScrapeTask_BookKind(t *testing.T) {
|
|
consumer := &stubConsumer{
|
|
scrapeQueue: []domain.ScrapeTask{
|
|
{ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning},
|
|
},
|
|
}
|
|
|
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
|
deps := runner.Dependencies{
|
|
Consumer: consumer,
|
|
BookWriter: &stubBookWriter{},
|
|
BookReader: &stubBookReader{},
|
|
AudioStore: &stubAudioStore{},
|
|
Novel: &stubNovelScraper{},
|
|
Kokoro: &stubKokoro{},
|
|
}
|
|
|
|
r := runner.New(cfg, deps)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = r.Run(ctx)
|
|
|
|
if len(consumer.finished) != 1 || consumer.finished[0] != "s1" {
|
|
t.Errorf("expected task s1 finished, got %v", consumer.finished)
|
|
}
|
|
}
|
|
|
|
func TestScrapeTask_UnknownKind(t *testing.T) {
|
|
consumer := &stubConsumer{
|
|
scrapeQueue: []domain.ScrapeTask{
|
|
{ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning},
|
|
},
|
|
}
|
|
|
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
|
deps := runner.Dependencies{
|
|
Consumer: consumer,
|
|
BookWriter: &stubBookWriter{},
|
|
BookReader: &stubBookReader{},
|
|
AudioStore: &stubAudioStore{},
|
|
Novel: &stubNovelScraper{},
|
|
Kokoro: &stubKokoro{},
|
|
}
|
|
|
|
r := runner.New(cfg, deps)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = r.Run(ctx)
|
|
|
|
// Unknown kind still finishes the task (with error message in result).
|
|
if len(consumer.finished) != 1 || consumer.finished[0] != "s2" {
|
|
t.Errorf("expected task s2 finished, got %v", consumer.finished)
|
|
}
|
|
}
|
|
|
|
func TestRun_CancelImmediately(t *testing.T) {
|
|
consumer := &stubConsumer{}
|
|
cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond}
|
|
deps := runner.Dependencies{
|
|
Consumer: consumer,
|
|
BookWriter: &stubBookWriter{},
|
|
BookReader: &stubBookReader{},
|
|
AudioStore: &stubAudioStore{},
|
|
Novel: &stubNovelScraper{},
|
|
Kokoro: &stubKokoro{},
|
|
}
|
|
|
|
r := runner.New(cfg, deps)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // cancel before Run
|
|
|
|
err := r.Run(ctx)
|
|
if err != nil {
|
|
t.Errorf("expected nil on graceful shutdown, got %v", err)
|
|
}
|
|
}
|