test: add integration tests for storage, server, and scrape+store flows
- scraper/internal/storage/integration_test.go: MinioClient and PocketBaseStore integration tests covering chapter/audio round-trips, presign URLs, book metadata, chapter index, ranking, progress, and audio cache CRUD - scraper/internal/storage/hybrid_integration_test.go: HybridStore end-to-end tests for metadata, chapters, ranking, progress, presign, and audio cache - scraper/internal/storage/scrape_integration_test.go: live Browserless + storage tests that scrape book metadata and first 3 chapters, store them, and verify the round-trip via HybridStore - scraper/internal/server/integration_test.go: HTTP server functional tests for health, scrape status, presign chapter, reading progress, and chapter-text endpoints against real MinIO + PocketBase backends - justfile: task runner at repo root with recipes for build, test, lint, UI, docker-compose, and individual service management
This commit is contained in:
Admin
457
scraper/internal/storage/hybrid_integration_test.go
Normal file
457
scraper/internal/storage/hybrid_integration_test.go
Normal file
@@ -0,0 +1,457 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for HybridStore (PocketBase + MinIO) end-to-end.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// newTestHybridStore constructs a HybridStore from environment variables.
|
||||
// Skips the test if either MINIO_ENDPOINT or POCKETBASE_URL is unset.
|
||||
func newTestHybridStore(t *testing.T) *HybridStore {
|
||||
t.Helper()
|
||||
if ep := envOr("MINIO_ENDPOINT", ""); ep == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping HybridStore integration test")
|
||||
}
|
||||
if u := envOr("POCKETBASE_URL", ""); u == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping HybridStore integration test")
|
||||
}
|
||||
|
||||
pbCfg := PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
minioCfg := MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestHybridStore_WriteReadMetadata exercises WriteMetadata → ReadMetadata round-trip.
|
||||
func TestHybridStore_WriteReadMetadata(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
meta := scraper.BookMeta{
|
||||
Slug: slug,
|
||||
Title: "Hybrid Store Test Novel",
|
||||
Author: "Test Author",
|
||||
Cover: "https://example.com/cover.jpg",
|
||||
Status: "Ongoing",
|
||||
Genres: []string{"Fantasy", "Action"},
|
||||
Summary: "A novel for integration testing.",
|
||||
TotalChapters: 99,
|
||||
SourceURL: fmt.Sprintf("https://example.com/book/%s", slug),
|
||||
Ranking: 5,
|
||||
}
|
||||
|
||||
t.Run("WriteMetadata", func(t *testing.T) {
|
||||
if err := hs.WriteMetadata(ctx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
t.Logf("wrote metadata for slug=%q", slug)
|
||||
})
|
||||
|
||||
t.Run("ReadMetadata", func(t *testing.T) {
|
||||
got, found, err := hs.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
t.Logf("read: %+v", got)
|
||||
if got.Title != meta.Title {
|
||||
t.Errorf("Title = %q, want %q", got.Title, meta.Title)
|
||||
}
|
||||
if got.Author != meta.Author {
|
||||
t.Errorf("Author = %q, want %q", got.Author, meta.Author)
|
||||
}
|
||||
if got.TotalChapters != meta.TotalChapters {
|
||||
t.Errorf("TotalChapters = %d, want %d", got.TotalChapters, meta.TotalChapters)
|
||||
}
|
||||
if got.Ranking != meta.Ranking {
|
||||
t.Errorf("Ranking = %d, want %d", got.Ranking, meta.Ranking)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("MetadataMtime", func(t *testing.T) {
|
||||
mtime := hs.MetadataMtime(ctx, slug)
|
||||
if mtime == 0 {
|
||||
t.Error("MetadataMtime returned 0")
|
||||
}
|
||||
t.Logf("mtime: %d (%s)", mtime, time.Unix(mtime, 0))
|
||||
})
|
||||
|
||||
t.Run("ReadMetadata_NotFound", func(t *testing.T) {
|
||||
_, found, err := hs.ReadMetadata(ctx, "this-slug-does-not-exist-xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata (miss): %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("ReadMetadata returned found=true for a non-existent slug")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_WriteReadChapter exercises WriteChapter (MinIO blob + PocketBase
|
||||
// index), ReadChapter, CountChapters, and ListChapters.
|
||||
func TestHybridStore_WriteReadChapter(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
// MinIO objects are not cleaned up — they use the test slug as prefix
|
||||
// and are effectively isolated.
|
||||
})
|
||||
|
||||
chapters := []scraper.Chapter{
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: The Beginning", Volume: 0},
|
||||
Text: "The first chapter text with enough content to be meaningful for a real novel chapter.",
|
||||
},
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 2, Title: "Chapter 2: Rising Action", Volume: 0},
|
||||
Text: "The second chapter text continues the story from where the first left off.",
|
||||
},
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 3, Title: "Chapter 3: Climax", Volume: 0},
|
||||
Text: "The third chapter text reaches the peak of tension and conflict.",
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("WriteChapter", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
if err := hs.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ch.Ref.Number, err)
|
||||
}
|
||||
t.Logf("wrote chapter %d", ch.Ref.Number)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ChapterExists", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
if !hs.ChapterExists(ctx, slug, ch.Ref) {
|
||||
t.Errorf("ChapterExists(chapter %d) = false after WriteChapter", ch.Ref.Number)
|
||||
}
|
||||
}
|
||||
missing := scraper.ChapterRef{Number: 999, Volume: 0}
|
||||
if hs.ChapterExists(ctx, slug, missing) {
|
||||
t.Error("ChapterExists(999) = true for a chapter that was never written")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ReadChapter", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
got, err := hs.ReadChapter(ctx, slug, ch.Ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ch.Ref.Number, err)
|
||||
}
|
||||
// WriteChapter prepends "# <title>\n\n" and appends "\n".
|
||||
expectedPrefix := "# " + ch.Ref.Title
|
||||
if !strings.HasPrefix(got, expectedPrefix) {
|
||||
t.Errorf("chapter %d: content doesn't start with expected header\ngot: %q\nwant prefix: %q",
|
||||
ch.Ref.Number, got[:min(len(got), 80)], expectedPrefix)
|
||||
}
|
||||
if !strings.Contains(got, ch.Text) {
|
||||
t.Errorf("chapter %d: content doesn't contain original text", ch.Ref.Number)
|
||||
}
|
||||
t.Logf("chapter %d: %d bytes", ch.Ref.Number, len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CountChapters", func(t *testing.T) {
|
||||
count := hs.CountChapters(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(chapters))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListChapters", func(t *testing.T) {
|
||||
infos, err := hs.ListChapters(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapters: %v", err)
|
||||
}
|
||||
if len(infos) != len(chapters) {
|
||||
t.Errorf("ListChapters returned %d entries, want %d", len(infos), len(chapters))
|
||||
}
|
||||
for i, info := range infos {
|
||||
t.Logf("infos[%d]: number=%d title=%q date=%q", i, info.Number, info.Title, info.Date)
|
||||
}
|
||||
// Verify sorted order.
|
||||
for i := 1; i < len(infos); i++ {
|
||||
if infos[i].Number <= infos[i-1].Number {
|
||||
t.Errorf("ListChapters not sorted: infos[%d].Number=%d <= infos[%d].Number=%d",
|
||||
i, infos[i].Number, i-1, infos[i-1].Number)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_WriteReadRanking exercises WriteRanking → ReadRankingItems
|
||||
// round-trip.
|
||||
func TestHybridStore_WriteReadRanking(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
items := []RankingItem{
|
||||
{Rank: 1, Slug: "top-novel", Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top-novel"},
|
||||
{Rank: 2, Slug: "second-novel", Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"},
|
||||
{Rank: 3, Slug: "third-novel", Title: "Third Novel"},
|
||||
}
|
||||
|
||||
t.Run("WriteRanking", func(t *testing.T) {
|
||||
if err := hs.WriteRanking(ctx, items); err != nil {
|
||||
t.Fatalf("WriteRanking: %v", err)
|
||||
}
|
||||
t.Logf("wrote %d ranking items", len(items))
|
||||
})
|
||||
|
||||
t.Run("ReadRankingItems", func(t *testing.T) {
|
||||
got, err := hs.ReadRankingItems(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadRankingItems: %v", err)
|
||||
}
|
||||
if len(got) != len(items) {
|
||||
t.Errorf("ReadRankingItems returned %d items, want %d", len(got), len(items))
|
||||
}
|
||||
for i, item := range got {
|
||||
t.Logf("items[%d]: rank=%d slug=%q title=%q", i, item.Rank, item.Slug, item.Title)
|
||||
}
|
||||
if len(got) > 0 {
|
||||
if got[0].Rank != 1 {
|
||||
t.Errorf("items[0].Rank = %d, want 1", got[0].Rank)
|
||||
}
|
||||
if got[0].Title != "Top Novel" {
|
||||
t.Errorf("items[0].Title = %q, want %q", got[0].Title, "Top Novel")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("RankingFileInfo", func(t *testing.T) {
|
||||
fi, err := hs.RankingFileInfo(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RankingFileInfo: %v", err)
|
||||
}
|
||||
if fi == nil {
|
||||
t.Fatal("RankingFileInfo returned nil")
|
||||
}
|
||||
if fi.ModTime().IsZero() {
|
||||
t.Error("RankingFileInfo.ModTime() is zero")
|
||||
}
|
||||
t.Logf("ranking file info: modtime=%s", fi.ModTime())
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_Progress exercises SetProgress → GetProgress → AllProgress →
|
||||
// DeleteProgress via the HybridStore.
|
||||
func TestHybridStore_Progress(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
const sessionID = "hybrid-test-session-abc"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, sessionID))
|
||||
})
|
||||
|
||||
p := ReadingProgress{Slug: slug, Chapter: 7, UpdatedAt: time.Now()}
|
||||
|
||||
t.Run("SetProgress", func(t *testing.T) {
|
||||
if err := hs.SetProgress(ctx, sessionID, p); err != nil {
|
||||
t.Fatalf("SetProgress: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetProgress", func(t *testing.T) {
|
||||
got, ok := hs.GetProgress(ctx, sessionID, slug)
|
||||
if !ok {
|
||||
t.Fatal("GetProgress: not found after SetProgress")
|
||||
}
|
||||
if got.Chapter != 7 {
|
||||
t.Errorf("Chapter = %d, want 7", got.Chapter)
|
||||
}
|
||||
if got.Slug != slug {
|
||||
t.Errorf("Slug = %q, want %q", got.Slug, slug)
|
||||
}
|
||||
t.Logf("progress: chapter=%d slug=%q updated=%s", got.Chapter, got.Slug, got.UpdatedAt)
|
||||
})
|
||||
|
||||
t.Run("AllProgress", func(t *testing.T) {
|
||||
all, err := hs.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("AllProgress: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range all {
|
||||
if item.Slug == slug {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("AllProgress did not contain slug %q (total=%d)", slug, len(all))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteProgress", func(t *testing.T) {
|
||||
if err := hs.DeleteProgress(ctx, sessionID, slug); err != nil {
|
||||
t.Fatalf("DeleteProgress: %v", err)
|
||||
}
|
||||
_, ok := hs.GetProgress(ctx, sessionID, slug)
|
||||
if ok {
|
||||
t.Error("GetProgress returned ok=true after DeleteProgress")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_PresignChapter writes a chapter to MinIO via HybridStore,
|
||||
// then calls PresignChapter and verifies a non-empty URL is returned.
|
||||
func TestHybridStore_PresignChapter(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Presign Test", Volume: 0},
|
||||
Text: "Text for the presign chapter test.",
|
||||
}
|
||||
|
||||
if err := hs.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
|
||||
url, err := hs.PresignChapter(ctx, slug, 1, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignChapter: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignChapter returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("PresignChapter URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned chapter URL: %s", url)
|
||||
}
|
||||
|
||||
// TestHybridStore_PresignAudio puts a fake audio blob into MinIO via the
|
||||
// underlying MinioClient and verifies PresignAudio returns a valid URL.
|
||||
func TestHybridStore_PresignAudio(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
key := hs.AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test")
|
||||
|
||||
if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
|
||||
url, err := hs.PresignAudio(ctx, key, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignAudio: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignAudio returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("PresignAudio URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned audio URL: %s", url)
|
||||
}
|
||||
|
||||
// TestHybridStore_AudioCache exercises SetAudioCache → GetAudioCache via HybridStore.
|
||||
func TestHybridStore_AudioCache(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("hybrid-audio-test-%d", time.Now().UnixMilli())
|
||||
const filename = "speech_hybrid123.mp3"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, cacheKey))
|
||||
})
|
||||
|
||||
if err := hs.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
t.Fatalf("SetAudioCache: %v", err)
|
||||
}
|
||||
|
||||
got, ok := hs.GetAudioCache(ctx, cacheKey)
|
||||
if !ok {
|
||||
t.Fatal("GetAudioCache returned ok=false after SetAudioCache")
|
||||
}
|
||||
if got != filename {
|
||||
t.Errorf("filename = %q, want %q", got, filename)
|
||||
}
|
||||
t.Logf("audio cache: cacheKey=%q filename=%q", cacheKey, got)
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
672
scraper/internal/storage/integration_test.go
Normal file
672
scraper/internal/storage/integration_test.go
Normal file
@@ -0,0 +1,672 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for MinioClient and PocketBaseStore against live instances.
|
||||
//
|
||||
// These tests require running MinIO and PocketBase services. They are gated
|
||||
// behind the "integration" build tag and are never run in a normal `go test ./...`.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func newTestMinioClient(t *testing.T) *MinioClient {
|
||||
t.Helper()
|
||||
endpoint := os.Getenv("MINIO_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping MinIO integration test")
|
||||
}
|
||||
useSSL := os.Getenv("MINIO_USE_SSL") == "true"
|
||||
cfg := MinioConfig{
|
||||
Endpoint: endpoint,
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: useSSL,
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
mc, err := NewMinioClient(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMinioClient: %v", err)
|
||||
}
|
||||
return mc
|
||||
}
|
||||
|
||||
func newTestPocketBaseStore(t *testing.T) *PocketBaseStore {
|
||||
t.Helper()
|
||||
pbURL := os.Getenv("POCKETBASE_URL")
|
||||
if pbURL == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping PocketBase integration test")
|
||||
}
|
||||
cfg := PocketBaseConfig{
|
||||
BaseURL: pbURL,
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
store := NewPocketBaseStore(cfg)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := store.EnsureCollections(ctx); err != nil {
|
||||
t.Logf("EnsureCollections (may be harmless): %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// testSlug generates a unique test slug to avoid collisions between parallel runs.
|
||||
func testSlug(t *testing.T) string {
|
||||
t.Helper()
|
||||
safe := strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, strings.ToLower(t.Name()))
|
||||
// Truncate and append a timestamp to keep it unique.
|
||||
if len(safe) > 30 {
|
||||
safe = safe[:30]
|
||||
}
|
||||
return fmt.Sprintf("test-%s-%d", safe, time.Now().UnixMilli()%100000)
|
||||
}
|
||||
|
||||
// ─── MinioClient tests ────────────────────────────────────────────────────────
|
||||
|
||||
// TestMinioClient_ChapterRoundTrip verifies PutChapter → GetChapter →
|
||||
// ChapterExists → ListChapterKeys for a single chapter.
|
||||
func TestMinioClient_ChapterRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
const n = 1
|
||||
content := "# Chapter 1\n\nHello integration world.\n"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Run("PutChapter", func(t *testing.T) {
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter: %v", err)
|
||||
}
|
||||
t.Logf("stored chapter at key: %s", chapterKey(slug, vol, n))
|
||||
})
|
||||
|
||||
t.Run("GetChapter", func(t *testing.T) {
|
||||
got, err := mc.GetChapter(ctx, slug, vol, n)
|
||||
if err != nil {
|
||||
t.Fatalf("GetChapter: %v", err)
|
||||
}
|
||||
if got != content {
|
||||
t.Errorf("GetChapter round-trip mismatch:\ngot: %q\nwant: %q", got, content)
|
||||
}
|
||||
t.Logf("retrieved %d bytes", len(got))
|
||||
})
|
||||
|
||||
t.Run("ChapterExists", func(t *testing.T) {
|
||||
if !mc.ChapterExists(ctx, slug, vol, n) {
|
||||
t.Error("ChapterExists returned false for a just-stored chapter")
|
||||
}
|
||||
if mc.ChapterExists(ctx, slug, vol, 999) {
|
||||
t.Error("ChapterExists returned true for a chapter that was never stored")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListChapterKeys", func(t *testing.T) {
|
||||
keys, err := mc.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterKeys: %v", err)
|
||||
}
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("ListChapterKeys returned %d keys, want 1: %v", len(keys), keys)
|
||||
}
|
||||
expectedKey := chapterKey(slug, vol, n)
|
||||
if keys[0] != expectedKey {
|
||||
t.Errorf("key = %q, want %q", keys[0], expectedKey)
|
||||
}
|
||||
t.Logf("keys: %v", keys)
|
||||
})
|
||||
}
|
||||
|
||||
// TestMinioClient_MultiChapterList stores several chapters and verifies
|
||||
// ListChapterKeys returns them all.
|
||||
func TestMinioClient_MultiChapterList(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Store chapters 1, 2, 51 (crosses the 1-50 folder boundary).
|
||||
chapters := []int{1, 2, 51}
|
||||
for _, n := range chapters {
|
||||
content := fmt.Sprintf("# Chapter %d\n\nContent for chapter %d.\n", n, n)
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter(%d): %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
keys, err := mc.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterKeys: %v", err)
|
||||
}
|
||||
t.Logf("keys: %v", keys)
|
||||
if len(keys) != len(chapters) {
|
||||
t.Errorf("ListChapterKeys returned %d keys, want %d", len(keys), len(chapters))
|
||||
}
|
||||
|
||||
count := mc.CountChapters(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(chapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMinioClient_PresignChapter verifies PresignChapter returns a non-empty URL.
|
||||
func TestMinioClient_PresignChapter(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
const n = 1
|
||||
content := "# Presign test\n\nSome content.\n"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter: %v", err)
|
||||
}
|
||||
|
||||
url, err := mc.PresignChapter(ctx, slug, vol, n, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignChapter: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignChapter returned empty URL")
|
||||
}
|
||||
t.Logf("presigned URL: %s", url)
|
||||
|
||||
// URL must be an http(s) URL and contain the slug somewhere.
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("URL does not start with http: %q", url)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMinioClient_AudioRoundTrip verifies PutAudio → GetAudio → AudioExists.
|
||||
func TestMinioClient_AudioRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
|
||||
// Use minimal fake MP3 bytes (just a recognisable prefix).
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00integration-test-audio")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Run("PutAudio", func(t *testing.T) {
|
||||
if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
t.Logf("stored audio at key: %s", key)
|
||||
})
|
||||
|
||||
t.Run("GetAudio", func(t *testing.T) {
|
||||
got, err := mc.GetAudio(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudio: %v", err)
|
||||
}
|
||||
if string(got) != string(fakeAudio) {
|
||||
t.Errorf("GetAudio round-trip mismatch: got %d bytes, want %d", len(got), len(fakeAudio))
|
||||
}
|
||||
t.Logf("retrieved %d bytes", len(got))
|
||||
})
|
||||
|
||||
t.Run("AudioExists", func(t *testing.T) {
|
||||
if !mc.AudioExists(ctx, key) {
|
||||
t.Error("AudioExists returned false for a just-stored audio object")
|
||||
}
|
||||
if mc.AudioExists(ctx, "nonexistent/key.mp3") {
|
||||
t.Error("AudioExists returned true for a key that was never stored")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMinioClient_PresignAudio verifies PresignAudio returns a non-empty URL.
|
||||
func TestMinioClient_PresignAudio(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00presign-audio-test")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
|
||||
url, err := mc.PresignAudio(ctx, key, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignAudio: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignAudio returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned audio URL: %s", url)
|
||||
}
|
||||
|
||||
// ─── PocketBaseStore tests ────────────────────────────────────────────────────
|
||||
|
||||
// TestPocketBaseStore_Ping verifies that admin auth works.
|
||||
func TestPocketBaseStore_Ping(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := store.Ping(ctx); err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
t.Log("Ping succeeded")
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_BookRoundTrip tests UpsertBook → GetBook → ListBooks →
|
||||
// BookMetaUpdated.
|
||||
func TestPocketBaseStore_BookRoundTrip(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Clean up after test.
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
t.Run("UpsertBook_Create", func(t *testing.T) {
|
||||
err := store.UpsertBook(ctx, slug,
|
||||
"Integration Test Novel", "Test Author",
|
||||
"https://example.com/cover.jpg", "Ongoing",
|
||||
"A test summary.", "https://example.com/book/test",
|
||||
[]string{"Action", "Fantasy"}, 42, 7,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBook (create): %v", err)
|
||||
}
|
||||
t.Logf("created book %q", slug)
|
||||
})
|
||||
|
||||
t.Run("GetBook", func(t *testing.T) {
|
||||
rec, found, err := store.GetBook(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBook: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetBook: book not found after UpsertBook")
|
||||
}
|
||||
t.Logf("GetBook record: %v", rec)
|
||||
if rec["title"] != "Integration Test Novel" {
|
||||
t.Errorf("title = %v, want %q", rec["title"], "Integration Test Novel")
|
||||
}
|
||||
if rec["author"] != "Test Author" {
|
||||
t.Errorf("author = %v, want %q", rec["author"], "Test Author")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListBooks", func(t *testing.T) {
|
||||
books, err := store.ListBooks(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBooks: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, b := range books {
|
||||
if s, _ := b["slug"].(string); s == slug {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("ListBooks did not return book with slug %q (total=%d)", slug, len(books))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UpsertBook_Update", func(t *testing.T) {
|
||||
err := store.UpsertBook(ctx, slug,
|
||||
"Integration Test Novel", "Test Author Updated",
|
||||
"", "Completed", "", "https://example.com/book/test",
|
||||
nil, 100, 3,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBook (update): %v", err)
|
||||
}
|
||||
rec, found, err := store.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetBook after update: found=%v err=%v", found, err)
|
||||
}
|
||||
if rec["author"] != "Test Author Updated" {
|
||||
t.Errorf("author after update = %v, want %q", rec["author"], "Test Author Updated")
|
||||
}
|
||||
if rec["status"] != "Completed" {
|
||||
t.Errorf("status after update = %v, want %q", rec["status"], "Completed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("BookMetaUpdated", func(t *testing.T) {
|
||||
ts, err := store.BookMetaUpdated(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("BookMetaUpdated: %v", err)
|
||||
}
|
||||
if ts.IsZero() {
|
||||
t.Error("BookMetaUpdated returned zero time")
|
||||
}
|
||||
t.Logf("meta_updated: %s", ts)
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_ChapterIdx tests UpsertChapterIdx → ListChapterIdx →
|
||||
// CountChapterIdx.
|
||||
func TestPocketBaseStore_ChapterIdx(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
chapters := []struct {
|
||||
n int
|
||||
title string
|
||||
date string
|
||||
}{
|
||||
{1, "Chapter 1: The Beginning", "2 days ago"},
|
||||
{2, "Chapter 2: Rising Action", "1 day ago"},
|
||||
{3, "Chapter 3: Climax", "3 hours ago"},
|
||||
}
|
||||
|
||||
for _, ch := range chapters {
|
||||
if err := store.UpsertChapterIdx(ctx, slug, ch.n, ch.title, ch.date); err != nil {
|
||||
t.Fatalf("UpsertChapterIdx(%d): %v", ch.n, err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("ListChapterIdx", func(t *testing.T) {
|
||||
rows, err := store.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterIdx: %v", err)
|
||||
}
|
||||
if len(rows) != len(chapters) {
|
||||
t.Errorf("ListChapterIdx returned %d rows, want %d", len(rows), len(chapters))
|
||||
}
|
||||
for i, row := range rows {
|
||||
t.Logf("row[%d]: number=%v title=%v date_label=%v", i, row["number"], row["title"], row["date_label"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CountChapterIdx", func(t *testing.T) {
|
||||
count := store.CountChapterIdx(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapterIdx = %d, want %d", count, len(chapters))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UpsertChapterIdx_Update", func(t *testing.T) {
|
||||
// Re-upsert chapter 2 with an updated title.
|
||||
if err := store.UpsertChapterIdx(ctx, slug, 2, "Chapter 2: Revised Title", "1 day ago"); err != nil {
|
||||
t.Fatalf("UpsertChapterIdx (update): %v", err)
|
||||
}
|
||||
rows, err := store.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterIdx after update: %v", err)
|
||||
}
|
||||
if store.CountChapterIdx(ctx, slug) != len(chapters) {
|
||||
t.Errorf("count changed after update: got %d, want %d", len(rows), len(chapters))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_Ranking tests SetRanking → GetRanking → RankingModTime.
|
||||
func TestPocketBaseStore_Ranking(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const testData = `[{"rank":1,"slug":"test-book","title":"Test Book"}]`
|
||||
|
||||
t.Run("SetRanking", func(t *testing.T) {
|
||||
if err := store.SetRanking(ctx, testData); err != nil {
|
||||
t.Fatalf("SetRanking: %v", err)
|
||||
}
|
||||
t.Log("SetRanking succeeded")
|
||||
})
|
||||
|
||||
t.Run("GetRanking", func(t *testing.T) {
|
||||
data, updated, err := store.GetRanking(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRanking: %v", err)
|
||||
}
|
||||
if data == "" {
|
||||
t.Error("GetRanking returned empty data")
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("GetRanking returned zero updated time")
|
||||
}
|
||||
t.Logf("data: %s", data)
|
||||
t.Logf("updated: %s", updated)
|
||||
})
|
||||
|
||||
t.Run("RankingModTime", func(t *testing.T) {
|
||||
fi, err := store.RankingModTime(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RankingModTime: %v", err)
|
||||
}
|
||||
if fi == nil {
|
||||
t.Fatal("RankingModTime returned nil FileInfo")
|
||||
}
|
||||
if fi.ModTime().IsZero() {
|
||||
t.Error("RankingModTime.ModTime() is zero")
|
||||
}
|
||||
t.Logf("ranking modtime: %s", fi.ModTime())
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_RankingPageHTML tests SetRankingPageHTML → GetRankingPageHTML.
|
||||
func TestPocketBaseStore_RankingPageHTML(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const page = 9999 // unlikely to collide with real data
|
||||
const html = `<html><body>integration test page 9999</body></html>`
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "ranking_html", fmt.Sprintf(`page=%d`, page))
|
||||
})
|
||||
|
||||
t.Run("SetRankingPageHTML", func(t *testing.T) {
|
||||
if err := store.SetRankingPageHTML(ctx, page, html); err != nil {
|
||||
t.Fatalf("SetRankingPageHTML: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetRankingPageHTML", func(t *testing.T) {
|
||||
got, updated, err := store.GetRankingPageHTML(ctx, page)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRankingPageHTML: %v", err)
|
||||
}
|
||||
if got != html {
|
||||
t.Errorf("html mismatch:\ngot: %q\nwant: %q", got, html)
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("updated time is zero")
|
||||
}
|
||||
t.Logf("retrieved HTML (%d bytes), updated=%s", len(got), updated)
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_Progress tests SetProgress → GetProgress → AllProgress →
|
||||
// DeleteProgress.
|
||||
func TestPocketBaseStore_Progress(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
const sessionID = "integration-test-session-xyz"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, sessionID))
|
||||
})
|
||||
|
||||
t.Run("SetProgress", func(t *testing.T) {
|
||||
if err := store.SetProgress(ctx, sessionID, slug, 5); err != nil {
|
||||
t.Fatalf("SetProgress: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetProgress", func(t *testing.T) {
|
||||
ch, updated, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProgress: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetProgress: not found after SetProgress")
|
||||
}
|
||||
if ch != 5 {
|
||||
t.Errorf("chapter = %d, want 5", ch)
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("updated time is zero")
|
||||
}
|
||||
t.Logf("chapter=%d updated=%s", ch, updated)
|
||||
})
|
||||
|
||||
t.Run("AllProgress", func(t *testing.T) {
|
||||
rows, err := store.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("AllProgress: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range rows {
|
||||
if s, _ := r["slug"].(string); s == slug {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("AllProgress did not include slug %q (total=%d)", slug, len(rows))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SetProgress_Update", func(t *testing.T) {
|
||||
if err := store.SetProgress(ctx, sessionID, slug, 12); err != nil {
|
||||
t.Fatalf("SetProgress (update): %v", err)
|
||||
}
|
||||
ch, _, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetProgress after update: found=%v err=%v", found, err)
|
||||
}
|
||||
if ch != 12 {
|
||||
t.Errorf("chapter after update = %d, want 12", ch)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteProgress", func(t *testing.T) {
|
||||
if err := store.DeleteProgress(ctx, sessionID, slug); err != nil {
|
||||
t.Fatalf("DeleteProgress: %v", err)
|
||||
}
|
||||
_, _, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProgress after delete: %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("GetProgress returned found=true after DeleteProgress")
|
||||
}
|
||||
t.Log("DeleteProgress confirmed")
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_AudioCache tests SetAudioCache → GetAudioCache.
|
||||
func TestPocketBaseStore_AudioCache(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("integration-audio-cache-test-%d", time.Now().UnixMilli())
|
||||
const filename = "speech_abc123.mp3"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, cacheKey))
|
||||
})
|
||||
|
||||
t.Run("SetAudioCache", func(t *testing.T) {
|
||||
if err := store.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
t.Fatalf("SetAudioCache: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetAudioCache", func(t *testing.T) {
|
||||
got, found, err := store.GetAudioCache(ctx, cacheKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioCache: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetAudioCache: not found after SetAudioCache")
|
||||
}
|
||||
if got != filename {
|
||||
t.Errorf("filename = %q, want %q", got, filename)
|
||||
}
|
||||
t.Logf("filename: %s", got)
|
||||
})
|
||||
|
||||
t.Run("GetAudioCache_Miss", func(t *testing.T) {
|
||||
got, found, err := store.GetAudioCache(ctx, "does-not-exist-ever")
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioCache (miss): %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Errorf("GetAudioCache returned found=true for missing key, filename=%q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
203
scraper/internal/storage/scrape_integration_test.go
Normal file
203
scraper/internal/storage/scrape_integration_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests that combine live scraping (Browserless) with real storage
|
||||
// (MinIO + PocketBase) via HybridStore.
|
||||
//
|
||||
// These tests require ALL THREE services to be running. They are gated behind
|
||||
// the "integration" build tag and skipped when any service URL is missing.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// BROWSERLESS_URL=http://localhost:3030 \
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 600s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
const (
|
||||
scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world"
|
||||
scrapeTestBookSlug = "a-dragon-against-the-whole-world"
|
||||
)
|
||||
|
||||
// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore,
|
||||
// skipping the test if any required env var is absent.
|
||||
func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) {
|
||||
t.Helper()
|
||||
|
||||
browserlessURL := os.Getenv("BROWSERLESS_URL")
|
||||
if browserlessURL == "" {
|
||||
t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
|
||||
client := browser.NewContentClient(browser.Config{
|
||||
BaseURL: browserlessURL,
|
||||
Token: os.Getenv("BROWSERLESS_TOKEN"),
|
||||
Timeout: 120 * time.Second,
|
||||
MaxConcurrent: 1,
|
||||
})
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
sc := novelfire.New(client, log, client, nil)
|
||||
hs := newTestHybridStore(t)
|
||||
return sc, hs
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores
|
||||
// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip.
|
||||
func TestScrapeAndStore_BookMetadata(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
slug := scrapeTestBookSlug + "-scrapetest"
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// 1. Scrape metadata from the live site.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeMetadata: %v", err)
|
||||
}
|
||||
t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d",
|
||||
meta.Slug, meta.Title, meta.Author, meta.TotalChapters)
|
||||
|
||||
// Override slug with our test-specific value to avoid polluting real data.
|
||||
meta.Slug = slug
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteMetadata(storeCtx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, found, err := hs.ReadMetadata(storeCtx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
|
||||
t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
|
||||
|
||||
if got.Title == "" {
|
||||
t.Error("Title is empty after round-trip")
|
||||
}
|
||||
if got.Author == "" {
|
||||
t.Error("Author is empty after round-trip")
|
||||
}
|
||||
if got.TotalChapters < 1 {
|
||||
t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the
|
||||
// live site and stores each via HybridStore.WriteChapter, then verifies
|
||||
// ReadChapter returns non-empty markdown with the expected header.
|
||||
func TestScrapeAndStore_First3Chapters(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
// Use a unique test slug so we don't pollute the real book.
|
||||
slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000)
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// Pre-build chapter refs (known URLs for this test book).
|
||||
refs := []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"},
|
||||
{Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"},
|
||||
{Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"},
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
ref := ref // capture loop variable
|
||||
t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
|
||||
// 1. Scrape chapter text.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
ch, err := sc.ScrapeChapterText(scrapeCtx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
|
||||
}
|
||||
t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text))
|
||||
|
||||
if len(ch.Text) < 100 {
|
||||
t.Errorf("scraped text too short (%d bytes)", len(ch.Text))
|
||||
}
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteChapter(storeCtx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, err := hs.ReadChapter(storeCtx, slug, ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatalf("ReadChapter(%d): returned empty string", ref.Number)
|
||||
}
|
||||
if len(got) < 100 {
|
||||
t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got))
|
||||
}
|
||||
|
||||
// WriteChapter prepends "# <title>\n\n".
|
||||
if !strings.HasPrefix(got, "# ") {
|
||||
t.Errorf("chapter %d: stored content does not start with markdown header: %q",
|
||||
ref.Number, got[:min(len(got), 60)])
|
||||
}
|
||||
|
||||
// Verify the original scraped text body is present.
|
||||
if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) {
|
||||
t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number)
|
||||
}
|
||||
|
||||
t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got))
|
||||
})
|
||||
}
|
||||
|
||||
// After all chapters written, verify count.
|
||||
countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer countCancel()
|
||||
|
||||
count := hs.CountChapters(countCtx, slug)
|
||||
if count != len(refs) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(refs))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user