//go:build integration // Integration tests for HybridStore (PocketBase + MinIO) end-to-end. // // Run with: // // MINIO_ENDPOINT=localhost:9000 \ // POCKETBASE_URL=http://localhost:8090 \ // go test -v -tags integration -timeout 120s \ // github.com/libnovel/scraper/internal/storage package storage import ( "context" "fmt" "strings" "testing" "time" "github.com/libnovel/scraper/internal/scraper" ) // newTestHybridStore constructs a HybridStore from environment variables. // Skips the test if either MINIO_ENDPOINT or POCKETBASE_URL is unset. func newTestHybridStore(t *testing.T) *HybridStore { t.Helper() if ep := envOr("MINIO_ENDPOINT", ""); ep == "" { t.Skip("MINIO_ENDPOINT not set — skipping HybridStore integration test") } if u := envOr("POCKETBASE_URL", ""); u == "" { t.Skip("POCKETBASE_URL not set — skipping HybridStore integration test") } pbCfg := PocketBaseConfig{ BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), } minioCfg := MinioConfig{ Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), UseSSL: envOr("MINIO_USE_SSL", "false") == "true", BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() hs, err := NewHybridStore(ctx, pbCfg, minioCfg) if err != nil { t.Fatalf("NewHybridStore: %v", err) } return hs } // ─── Tests ──────────────────────────────────────────────────────────────────── // TestHybridStore_WriteReadMetadata exercises WriteMetadata → ReadMetadata round-trip. func TestHybridStore_WriteReadMetadata(t *testing.T) { hs := newTestHybridStore(t) slug := testSlug(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug)) }) meta := scraper.BookMeta{ Slug: slug, Title: "Hybrid Store Test Novel", Author: "Test Author", Cover: "https://example.com/cover.jpg", Status: "Ongoing", Genres: []string{"Fantasy", "Action"}, Summary: "A novel for integration testing.", TotalChapters: 99, SourceURL: fmt.Sprintf("https://example.com/book/%s", slug), Ranking: 5, } t.Run("WriteMetadata", func(t *testing.T) { if err := hs.WriteMetadata(ctx, meta); err != nil { t.Fatalf("WriteMetadata: %v", err) } t.Logf("wrote metadata for slug=%q", slug) }) t.Run("ReadMetadata", func(t *testing.T) { got, found, err := hs.ReadMetadata(ctx, slug) if err != nil { t.Fatalf("ReadMetadata: %v", err) } if !found { t.Fatal("ReadMetadata: not found after WriteMetadata") } t.Logf("read: %+v", got) if got.Title != meta.Title { t.Errorf("Title = %q, want %q", got.Title, meta.Title) } if got.Author != meta.Author { t.Errorf("Author = %q, want %q", got.Author, meta.Author) } if got.TotalChapters != meta.TotalChapters { t.Errorf("TotalChapters = %d, want %d", got.TotalChapters, meta.TotalChapters) } if got.Ranking != meta.Ranking { t.Errorf("Ranking = %d, want %d", got.Ranking, meta.Ranking) } }) t.Run("MetadataMtime", func(t *testing.T) { mtime := hs.MetadataMtime(ctx, slug) if mtime == 0 { t.Error("MetadataMtime returned 0") } t.Logf("mtime: %d (%s)", mtime, time.Unix(mtime, 0)) }) t.Run("ReadMetadata_NotFound", func(t *testing.T) { _, found, err := hs.ReadMetadata(ctx, "this-slug-does-not-exist-xyz") if err != nil { t.Fatalf("ReadMetadata (miss): %v", err) } if found { t.Error("ReadMetadata returned found=true for a non-existent slug") } }) } // TestHybridStore_WriteReadChapter exercises WriteChapter (MinIO blob + PocketBase // index), ReadChapter, CountChapters, and ListChapters. func TestHybridStore_WriteReadChapter(t *testing.T) { hs := newTestHybridStore(t) slug := testSlug(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug)) // MinIO objects are not cleaned up — they use the test slug as prefix // and are effectively isolated. }) chapters := []scraper.Chapter{ { Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: The Beginning", Volume: 0}, Text: "The first chapter text with enough content to be meaningful for a real novel chapter.", }, { Ref: scraper.ChapterRef{Number: 2, Title: "Chapter 2: Rising Action", Volume: 0}, Text: "The second chapter text continues the story from where the first left off.", }, { Ref: scraper.ChapterRef{Number: 3, Title: "Chapter 3: Climax", Volume: 0}, Text: "The third chapter text reaches the peak of tension and conflict.", }, } t.Run("WriteChapter", func(t *testing.T) { for _, ch := range chapters { if err := hs.WriteChapter(ctx, slug, ch); err != nil { t.Fatalf("WriteChapter(%d): %v", ch.Ref.Number, err) } t.Logf("wrote chapter %d", ch.Ref.Number) } }) t.Run("ChapterExists", func(t *testing.T) { for _, ch := range chapters { if !hs.ChapterExists(ctx, slug, ch.Ref) { t.Errorf("ChapterExists(chapter %d) = false after WriteChapter", ch.Ref.Number) } } missing := scraper.ChapterRef{Number: 999, Volume: 0} if hs.ChapterExists(ctx, slug, missing) { t.Error("ChapterExists(999) = true for a chapter that was never written") } }) t.Run("ReadChapter", func(t *testing.T) { for _, ch := range chapters { got, err := hs.ReadChapter(ctx, slug, ch.Ref.Number) if err != nil { t.Fatalf("ReadChapter(%d): %v", ch.Ref.Number, err) } // WriteChapter prepends "# \n\n" and appends "\n". expectedPrefix := "# " + ch.Ref.Title if !strings.HasPrefix(got, expectedPrefix) { t.Errorf("chapter %d: content doesn't start with expected header\ngot: %q\nwant prefix: %q", ch.Ref.Number, got[:min(len(got), 80)], expectedPrefix) } if !strings.Contains(got, ch.Text) { t.Errorf("chapter %d: content doesn't contain original text", ch.Ref.Number) } t.Logf("chapter %d: %d bytes", ch.Ref.Number, len(got)) } }) t.Run("CountChapters", func(t *testing.T) { count := hs.CountChapters(ctx, slug) if count != len(chapters) { t.Errorf("CountChapters = %d, want %d", count, len(chapters)) } }) t.Run("ListChapters", func(t *testing.T) { infos, err := hs.ListChapters(ctx, slug) if err != nil { t.Fatalf("ListChapters: %v", err) } if len(infos) != len(chapters) { t.Errorf("ListChapters returned %d entries, want %d", len(infos), len(chapters)) } for i, info := range infos { t.Logf("infos[%d]: number=%d title=%q date=%q", i, info.Number, info.Title, info.Date) } // Verify sorted order. for i := 1; i < len(infos); i++ { if infos[i].Number <= infos[i-1].Number { t.Errorf("ListChapters not sorted: infos[%d].Number=%d <= infos[%d].Number=%d", i, infos[i].Number, i-1, infos[i-1].Number) } } }) } // TestHybridStore_WriteReadRanking exercises WriteRankingItem → ReadRankingItems // round-trip and RankingFreshEnough. func TestHybridStore_WriteReadRanking(t *testing.T) { hs := newTestHybridStore(t) slug1 := "integ-rank-1-" + fmt.Sprintf("%d", time.Now().UnixMilli()) slug2 := "integ-rank-2-" + fmt.Sprintf("%d", time.Now().UnixMilli()) slug3 := "integ-rank-3-" + fmt.Sprintf("%d", time.Now().UnixMilli()) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() for _, sl := range []string{slug1, slug2, slug3} { _ = hs.pb.pb.deleteWhere(cleanCtx, "ranking", fmt.Sprintf(`slug="%s"`, sl)) } }) items := []RankingItem{ {Rank: 1, Slug: slug1, Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top"}, {Rank: 2, Slug: slug2, Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"}, {Rank: 3, Slug: slug3, Title: "Third Novel"}, } t.Run("WriteRankingItem", func(t *testing.T) { for _, item := range items { if err := hs.WriteRankingItem(ctx, item); err != nil { t.Fatalf("WriteRankingItem(%s): %v", item.Slug, err) } } t.Logf("wrote %d ranking items", len(items)) }) t.Run("ReadRankingItems", func(t *testing.T) { got, err := hs.ReadRankingItems(ctx) if err != nil { t.Fatalf("ReadRankingItems: %v", err) } // Filter to just our test slugs (other tests may leave rows). var ours []RankingItem slugSet := map[string]bool{slug1: true, slug2: true, slug3: true} for _, g := range got { if slugSet[g.Slug] { ours = append(ours, g) } } if len(ours) != 3 { t.Fatalf("ReadRankingItems returned %d test items, want 3", len(ours)) } // Verify order by rank. for i := 1; i < len(ours); i++ { if ours[i].Rank <= ours[i-1].Rank { t.Errorf("items not sorted by rank: ours[%d].Rank=%d, ours[%d].Rank=%d", i, ours[i].Rank, i-1, ours[i-1].Rank) } } // Verify fields. if ours[0].Title != "Top Novel" { t.Errorf("ours[0].Title = %q, want %q", ours[0].Title, "Top Novel") } if ours[0].Author != "Author A" { t.Errorf("ours[0].Author = %q, want %q", ours[0].Author, "Author A") } t.Logf("ranking items: %+v", ours) }) t.Run("RankingFreshEnough", func(t *testing.T) { fresh, err := hs.RankingFreshEnough(ctx, 24*time.Hour) if err != nil { t.Fatalf("RankingFreshEnough: %v", err) } if !fresh { t.Error("RankingFreshEnough(24h) returned false immediately after writing items") } t.Logf("ranking fresh=true") }) } // TestHybridStore_Progress exercises SetProgress → GetProgress → AllProgress → // DeleteProgress via the HybridStore. func TestHybridStore_Progress(t *testing.T) { hs := newTestHybridStore(t) slug := testSlug(t) const sessionID = "hybrid-test-session-abc" ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = hs.pb.pb.deleteWhere(cleanCtx, "progress", fmt.Sprintf(`session_id="%s"`, sessionID)) }) p := ReadingProgress{Slug: slug, Chapter: 7, UpdatedAt: time.Now()} t.Run("SetProgress", func(t *testing.T) { if err := hs.SetProgress(ctx, sessionID, p); err != nil { t.Fatalf("SetProgress: %v", err) } }) t.Run("GetProgress", func(t *testing.T) { got, ok := hs.GetProgress(ctx, sessionID, slug) if !ok { t.Fatal("GetProgress: not found after SetProgress") } if got.Chapter != 7 { t.Errorf("Chapter = %d, want 7", got.Chapter) } if got.Slug != slug { t.Errorf("Slug = %q, want %q", got.Slug, slug) } t.Logf("progress: chapter=%d slug=%q updated=%s", got.Chapter, got.Slug, got.UpdatedAt) }) t.Run("AllProgress", func(t *testing.T) { all, err := hs.AllProgress(ctx, sessionID) if err != nil { t.Fatalf("AllProgress: %v", err) } found := false for _, item := range all { if item.Slug == slug { found = true } } if !found { t.Errorf("AllProgress did not contain slug %q (total=%d)", slug, len(all)) } }) t.Run("DeleteProgress", func(t *testing.T) { if err := hs.DeleteProgress(ctx, sessionID, slug); err != nil { t.Fatalf("DeleteProgress: %v", err) } _, ok := hs.GetProgress(ctx, sessionID, slug) if ok { t.Error("GetProgress returned ok=true after DeleteProgress") } }) } // TestHybridStore_PresignChapter writes a chapter to MinIO via HybridStore, // then calls PresignChapter and verifies a non-empty URL is returned. func TestHybridStore_PresignChapter(t *testing.T) { hs := newTestHybridStore(t) slug := testSlug(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() ch := scraper.Chapter{ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Presign Test", Volume: 0}, Text: "Text for the presign chapter test.", } if err := hs.WriteChapter(ctx, slug, ch); err != nil { t.Fatalf("WriteChapter: %v", err) } url, err := hs.PresignChapter(ctx, slug, 1, 10*time.Minute) if err != nil { t.Fatalf("PresignChapter: %v", err) } if url == "" { t.Fatal("PresignChapter returned empty URL") } if !strings.HasPrefix(url, "http") { t.Errorf("PresignChapter URL does not start with http: %q", url) } t.Logf("presigned chapter URL: %s", url) } // TestHybridStore_PresignAudio puts a fake audio blob into MinIO via the // underlying MinioClient and verifies PresignAudio returns a valid URL. func TestHybridStore_PresignAudio(t *testing.T) { hs := newTestHybridStore(t) slug := testSlug(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() key := hs.AudioObjectKey(slug, 1, "af_bella", 1.0) fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test") if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil { t.Fatalf("PutAudio: %v", err) } url, err := hs.PresignAudio(ctx, key, 10*time.Minute) if err != nil { t.Fatalf("PresignAudio: %v", err) } if url == "" { t.Fatal("PresignAudio returned empty URL") } if !strings.HasPrefix(url, "http") { t.Errorf("PresignAudio URL does not start with http: %q", url) } t.Logf("presigned audio URL: %s", url) } // TestHybridStore_AudioCache exercises SetAudioCache → GetAudioCache via HybridStore. func TestHybridStore_AudioCache(t *testing.T) { hs := newTestHybridStore(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() cacheKey := fmt.Sprintf("hybrid-audio-test-%d", time.Now().UnixMilli()) const filename = "speech_hybrid123.mp3" t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = hs.pb.pb.deleteWhere(cleanCtx, "audio_cache", fmt.Sprintf(`cache_key="%s"`, cacheKey)) }) if err := hs.SetAudioCache(ctx, cacheKey, filename); err != nil { t.Fatalf("SetAudioCache: %v", err) } got, ok := hs.GetAudioCache(ctx, cacheKey) if !ok { t.Fatal("GetAudioCache returned ok=false after SetAudioCache") } if got != filename { t.Errorf("filename = %q, want %q", got, filename) } t.Logf("audio cache: cacheKey=%q filename=%q", cacheKey, got) } // ─── helpers ────────────────────────────────────────────────────────────────── func min(a, b int) int { if a < b { return a } return b }