From 2f0857be45499eab3d46331a01fa28cd02354e08 Mon Sep 17 00:00:00 2001 From: Admin Date: Tue, 3 Mar 2026 14:54:43 +0500 Subject: [PATCH] test: add integration tests for storage, server, and scrape+store flows - scraper/internal/storage/integration_test.go: MinioClient and PocketBaseStore integration tests covering chapter/audio round-trips, presign URLs, book metadata, chapter index, ranking, progress, and audio cache CRUD - scraper/internal/storage/hybrid_integration_test.go: HybridStore end-to-end tests for metadata, chapters, ranking, progress, presign, and audio cache - scraper/internal/storage/scrape_integration_test.go: live Browserless + storage tests that scrape book metadata and first 3 chapters, store them, and verify the round-trip via HybridStore - scraper/internal/server/integration_test.go: HTTP server functional tests for health, scrape status, presign chapter, reading progress, and chapter-text endpoints against real MinIO + PocketBase backends - justfile: task runner at repo root with recipes for build, test, lint, UI, docker-compose, and individual service management --- justfile | 124 ++++ scraper/internal/server/integration_test.go | 416 +++++++++++ .../storage/hybrid_integration_test.go | 457 ++++++++++++ scraper/internal/storage/integration_test.go | 672 ++++++++++++++++++ .../storage/scrape_integration_test.go | 203 ++++++ 5 files changed, 1872 insertions(+) create mode 100644 justfile create mode 100644 scraper/internal/server/integration_test.go create mode 100644 scraper/internal/storage/hybrid_integration_test.go create mode 100644 scraper/internal/storage/integration_test.go create mode 100644 scraper/internal/storage/scrape_integration_test.go diff --git a/justfile b/justfile new file mode 100644 index 0000000..a70ae5e --- /dev/null +++ b/justfile @@ -0,0 +1,124 @@ +# justfile — libnovel-v2 task runner +# Install just: https://just.systems + +scraper_dir := "scraper" +ui_dir := "ui" + +# ─── Build ──────────────────────────────────────────────────────────────────── + +# Build the scraper binary +build: + cd {{scraper_dir}} && go build -o bin/scraper ./cmd/scraper + +# Build and verify all Go packages compile cleanly +build-all: + cd {{scraper_dir}} && go build ./... + +# ─── Tests ──────────────────────────────────────────────────────────────────── + +# Run unit tests only (no integration services required) +test: + cd {{scraper_dir}} && go test ./... + +# Run integration tests (requires MinIO, PocketBase, optional Browserless) +# Override env vars as needed, e.g.: +# just test-integration MINIO_ENDPOINT=localhost:9000 +test-integration: + cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./... + +# Run unit + integration tests +test-all: test test-integration + +# Run a specific package's integration tests, e.g.: +# just test-pkg internal/storage +test-pkg pkg: + cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./{{pkg}}/... + +# ─── Code quality ───────────────────────────────────────────────────────────── + +# Run go vet on all packages (including integration build tag) +lint: + cd {{scraper_dir}} && go vet ./... + cd {{scraper_dir}} && go vet -tags integration ./... + +# ─── UI ─────────────────────────────────────────────────────────────────────── + +# Type-check the SvelteKit UI +ui-check: + cd {{ui_dir}} && npx svelte-check + +# Start the SvelteKit dev server +ui-dev: + cd {{ui_dir}} && npm run dev + +# Install UI dependencies +ui-install: + cd {{ui_dir}} && npm install + +# Build the UI for production +ui-build: + cd {{ui_dir}} && npm run build + +# ─── Docker Compose ─────────────────────────────────────────────────────────── + +# Start all services (browserless, kokoro, scraper, minio, pocketbase) +up: + docker compose up -d + +# Stop all services +down: + docker compose down + +# Tail logs for all services +logs: + docker compose logs -f + +# Tail logs for a specific service, e.g.: just logs-service scraper +logs-service service: + docker compose logs -f {{service}} + +# Rebuild and restart a specific service +restart service: + docker compose up -d --build {{service}} + +# ─── Local dev: individual services ────────────────────────────────────────── + +# Start only PocketBase (for local storage testing) +pb-up: + docker compose up -d pocketbase + +# Start only MinIO (for local storage testing) +minio-up: + docker compose up -d minio + +# Start only Browserless (for local scraping tests) +browserless-up: + docker compose up -d browserless + +# Start storage backends only (MinIO + PocketBase) +storage-up: + docker compose up -d minio pocketbase + +# ─── Convenience ───────────────────────────────────────────────────────────── + +# Show status of all docker compose services +status: + docker compose ps + +# Remove all stopped containers and unused images +prune: + docker compose down --remove-orphans + docker image prune -f + +# One-shot scrape of the full catalogue (requires services to be running) +scrape-run: build + cd {{scraper_dir}} && ./bin/scraper run + +# One-shot scrape of a single book URL, e.g.: +# just scrape-book https://novelfire.net/book/my-novel +scrape-book url: build + cd {{scraper_dir}} && ./bin/scraper run --url {{url}} + +# Start the HTTP server +serve: build + cd {{scraper_dir}} && ./bin/scraper serve diff --git a/scraper/internal/server/integration_test.go b/scraper/internal/server/integration_test.go new file mode 100644 index 0000000..3967c4e --- /dev/null +++ b/scraper/internal/server/integration_test.go @@ -0,0 +1,416 @@ +//go:build integration + +// Integration tests for the HTTP server against live MinIO + PocketBase backends. +// +// The server is started on a random port for each test; real HybridStore +// backends are used. Browserless-dependent tests are skipped unless +// BROWSERLESS_URL is set. +// +// Run with: +// +// MINIO_ENDPOINT=localhost:9000 \ +// POCKETBASE_URL=http://localhost:8090 \ +// go test -v -tags integration -timeout 120s \ +// github.com/libnovel/scraper/internal/server +package server + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + neturl "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/libnovel/scraper/internal/orchestrator" + "github.com/libnovel/scraper/internal/scraper" + "github.com/libnovel/scraper/internal/storage" +) + +// ─── fixture helpers ────────────────────────────────────────────────────────── + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +// newTestStore creates a HybridStore from env vars, skipping if not configured. +func newTestStore(t *testing.T) *storage.HybridStore { + t.Helper() + if os.Getenv("MINIO_ENDPOINT") == "" { + t.Skip("MINIO_ENDPOINT not set — skipping server integration test") + } + if os.Getenv("POCKETBASE_URL") == "" { + t.Skip("POCKETBASE_URL not set — skipping server integration test") + } + + pbCfg := storage.PocketBaseConfig{ + BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), + AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), + } + minioCfg := storage.MinioConfig{ + Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), + AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), + SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), + UseSSL: envOr("MINIO_USE_SSL", "false") == "true", + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg) + if err != nil { + t.Fatalf("NewHybridStore: %v", err) + } + return hs +} + +// startTestServer starts a real Server on a random free port and returns the +// base URL. The server is shut down when the test finishes. +func startTestServer(t *testing.T, store storage.Store) string { + t.Helper() + + // Find a free port. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen: %v", err) + } + addr := ln.Addr().String() + ln.Close() + + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + + // nopScraper satisfies scraper.NovelScraper without hitting the network. + srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella") + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + ready := make(chan struct{}) + go func() { + // Signal readiness after a short delay to let the listener bind. + go func() { + time.Sleep(50 * time.Millisecond) + close(ready) + }() + _ = srv.ListenAndServe(ctx) + }() + + <-ready + + // Wait until the server actually accepts connections. + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + resp, err := http.Get("http://" + addr + "/health") + if err == nil { + resp.Body.Close() + break + } + time.Sleep(20 * time.Millisecond) + } + + return "http://" + addr +} + +// nopScraper is a no-op NovelScraper implementation for tests that don't +// exercise scraping functionality. +type nopScraper struct{} + +func (nopScraper) SourceName() string { return "nop" } +func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) { + ch := make(chan scraper.CatalogueEntry) + errs := make(chan error) + close(ch) + close(errs) + return ch, errs +} +func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) { + return scraper.BookMeta{}, nil +} +func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) { + return nil, nil +} +func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) { + return scraper.Chapter{Ref: ref}, nil +} +func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) { + ch := make(chan scraper.BookMeta) + errs := make(chan error) + close(ch) + close(errs) + return ch, errs +} + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +// TestServer_Health verifies GET /health returns 200 with status:ok. +func TestServer_Health(t *testing.T) { + store := newTestStore(t) + base := startTestServer(t, store) + + resp, err := http.Get(base + "/health") + if err != nil { + t.Fatalf("GET /health: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } + + var body map[string]string + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode health body: %v", err) + } + if body["status"] != "ok" { + t.Errorf("status field = %q, want %q", body["status"], "ok") + } + t.Logf("health response: %v", body) +} + +// TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false +// when no scrape is running. +func TestServer_ScrapeStatus(t *testing.T) { + store := newTestStore(t) + base := startTestServer(t, store) + + resp, err := http.Get(base + "/api/scrape/status") + if err != nil { + t.Fatalf("GET /api/scrape/status: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } + + var body map[string]bool + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body["running"] { + t.Error("scrape/status.running = true, want false") + } + t.Logf("scrape status: %v", body) +} + +// TestServer_PresignChapter writes a chapter to MinIO, then calls +// GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned. +func TestServer_PresignChapter(t *testing.T) { + store := newTestStore(t) + base := startTestServer(t, store) + + // Write a chapter directly via the store so we have something to presign. + slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + ch := scraper.Chapter{ + Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0}, + Text: "Content for the server presign integration test.", + } + if err := store.WriteChapter(ctx, slug, ch); err != nil { + t.Fatalf("WriteChapter: %v", err) + } + t.Logf("stored chapter for slug=%q", slug) + + // Call the presign endpoint. + url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug) + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } + + var body map[string]string + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode presign response: %v", err) + } + presignedURL := body["url"] + if presignedURL == "" { + t.Fatal("presign response has empty url field") + } + if !strings.HasPrefix(presignedURL, "http") { + t.Errorf("presigned URL does not start with http: %q", presignedURL) + } + t.Logf("presigned URL: %s", presignedURL) +} + +// TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress. +func TestServer_Progress(t *testing.T) { + store := newTestStore(t) + base := startTestServer(t, store) + + slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000) + + // Use a persistent http.Client to carry the session cookie. + jar := &cookieJar{cookies: make(map[string][]*http.Cookie)} + client := &http.Client{Jar: jar} + + // POST /api/progress/{slug} + setURL := fmt.Sprintf("%s/api/progress/%s", base, slug) + body, _ := json.Marshal(map[string]int{"chapter": 5}) + resp, err := client.Post(setURL, "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST %s: %v", setURL, err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("POST progress status = %d, want 200", resp.StatusCode) + } + t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode) + + // GET /api/progress + getURL := fmt.Sprintf("%s/api/progress", base) + resp2, err := client.Get(getURL) + if err != nil { + t.Fatalf("GET %s: %v", getURL, err) + } + defer resp2.Body.Close() + + if resp2.StatusCode != http.StatusOK { + t.Errorf("GET progress status = %d, want 200", resp2.StatusCode) + } + + var progress map[string]interface{} + if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil { + t.Fatalf("decode progress response: %v", err) + } + t.Logf("progress: %v", progress) + + // The slug should appear with chapter value 5. + if ch, ok := progress[slug]; !ok { + t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress)) + } else { + // JSON numbers decode as float64. + chNum, _ := ch.(float64) + if int(chNum) != 5 { + t.Errorf("progress[%q] = %v, want 5", slug, ch) + } + } + + // DELETE /api/progress/{slug} + delURL := fmt.Sprintf("%s/api/progress/%s", base, slug) + delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil) + delResp, err := client.Do(delReq) + if err != nil { + t.Fatalf("DELETE %s: %v", delURL, err) + } + delResp.Body.Close() + if delResp.StatusCode != http.StatusOK { + t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode) + } + t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode) +} + +// TestServer_PresignChapter_NotFound verifies that presigning a non-existent +// chapter returns 500 (presign fails on missing object). +func TestServer_PresignChapter_NotFound(t *testing.T) { + store := newTestStore(t) + base := startTestServer(t, store) + + url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base) + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + defer resp.Body.Close() + + // MinIO presign on a non-existent key returns an error; server returns 500. + // (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.) + t.Logf("presign non-existent chapter status: %d", resp.StatusCode) +} + +// TestServer_ChapterText writes a chapter and verifies +// GET /api/chapter-text/{slug}/{n} returns the stripped plain text. +func TestServer_ChapterText(t *testing.T) { + store := newTestStore(t) + base := startTestServer(t, store) + + slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + const chapterText = "The quick brown fox jumps over the lazy dog near the river." + ch := scraper.Chapter{ + Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0}, + Text: chapterText, + } + if err := store.WriteChapter(ctx, slug, ch); err != nil { + t.Fatalf("WriteChapter: %v", err) + } + + url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug) + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } + + var buf strings.Builder + rawBytes, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + buf.Write(rawBytes) + text := buf.String() + t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)]) + + if text == "" { + t.Error("chapter-text returned empty body") + } + // The stripped text should contain our chapter text (markdown heading stripped). + if !strings.Contains(text, chapterText) { + t.Errorf("chapter text does not contain expected content %q", chapterText) + } +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +func mapKeys(m map[string]interface{}) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// cookieJar is a minimal http.CookieJar that stores cookies by host. +type cookieJar struct { + cookies map[string][]*http.Cookie +} + +func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) { + j.cookies[u.Host] = append(j.cookies[u.Host], cookies...) +} + +func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie { + return j.cookies[u.Host] +} diff --git a/scraper/internal/storage/hybrid_integration_test.go b/scraper/internal/storage/hybrid_integration_test.go new file mode 100644 index 0000000..06d7621 --- /dev/null +++ b/scraper/internal/storage/hybrid_integration_test.go @@ -0,0 +1,457 @@ +//go:build integration + +// Integration tests for HybridStore (PocketBase + MinIO) end-to-end. +// +// Run with: +// +// MINIO_ENDPOINT=localhost:9000 \ +// POCKETBASE_URL=http://localhost:8090 \ +// go test -v -tags integration -timeout 120s \ +// github.com/libnovel/scraper/internal/storage +package storage + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/libnovel/scraper/internal/scraper" +) + +// newTestHybridStore constructs a HybridStore from environment variables. +// Skips the test if either MINIO_ENDPOINT or POCKETBASE_URL is unset. +func newTestHybridStore(t *testing.T) *HybridStore { + t.Helper() + if ep := envOr("MINIO_ENDPOINT", ""); ep == "" { + t.Skip("MINIO_ENDPOINT not set — skipping HybridStore integration test") + } + if u := envOr("POCKETBASE_URL", ""); u == "" { + t.Skip("POCKETBASE_URL not set — skipping HybridStore integration test") + } + + pbCfg := PocketBaseConfig{ + BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), + AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), + } + minioCfg := MinioConfig{ + Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), + AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), + SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), + UseSSL: envOr("MINIO_USE_SSL", "false") == "true", + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + hs, err := NewHybridStore(ctx, pbCfg, minioCfg) + if err != nil { + t.Fatalf("NewHybridStore: %v", err) + } + return hs +} + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +// TestHybridStore_WriteReadMetadata exercises WriteMetadata → ReadMetadata round-trip. +func TestHybridStore_WriteReadMetadata(t *testing.T) { + hs := newTestHybridStore(t) + slug := testSlug(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug)) + }) + + meta := scraper.BookMeta{ + Slug: slug, + Title: "Hybrid Store Test Novel", + Author: "Test Author", + Cover: "https://example.com/cover.jpg", + Status: "Ongoing", + Genres: []string{"Fantasy", "Action"}, + Summary: "A novel for integration testing.", + TotalChapters: 99, + SourceURL: fmt.Sprintf("https://example.com/book/%s", slug), + Ranking: 5, + } + + t.Run("WriteMetadata", func(t *testing.T) { + if err := hs.WriteMetadata(ctx, meta); err != nil { + t.Fatalf("WriteMetadata: %v", err) + } + t.Logf("wrote metadata for slug=%q", slug) + }) + + t.Run("ReadMetadata", func(t *testing.T) { + got, found, err := hs.ReadMetadata(ctx, slug) + if err != nil { + t.Fatalf("ReadMetadata: %v", err) + } + if !found { + t.Fatal("ReadMetadata: not found after WriteMetadata") + } + t.Logf("read: %+v", got) + if got.Title != meta.Title { + t.Errorf("Title = %q, want %q", got.Title, meta.Title) + } + if got.Author != meta.Author { + t.Errorf("Author = %q, want %q", got.Author, meta.Author) + } + if got.TotalChapters != meta.TotalChapters { + t.Errorf("TotalChapters = %d, want %d", got.TotalChapters, meta.TotalChapters) + } + if got.Ranking != meta.Ranking { + t.Errorf("Ranking = %d, want %d", got.Ranking, meta.Ranking) + } + }) + + t.Run("MetadataMtime", func(t *testing.T) { + mtime := hs.MetadataMtime(ctx, slug) + if mtime == 0 { + t.Error("MetadataMtime returned 0") + } + t.Logf("mtime: %d (%s)", mtime, time.Unix(mtime, 0)) + }) + + t.Run("ReadMetadata_NotFound", func(t *testing.T) { + _, found, err := hs.ReadMetadata(ctx, "this-slug-does-not-exist-xyz") + if err != nil { + t.Fatalf("ReadMetadata (miss): %v", err) + } + if found { + t.Error("ReadMetadata returned found=true for a non-existent slug") + } + }) +} + +// TestHybridStore_WriteReadChapter exercises WriteChapter (MinIO blob + PocketBase +// index), ReadChapter, CountChapters, and ListChapters. +func TestHybridStore_WriteReadChapter(t *testing.T) { + hs := newTestHybridStore(t) + slug := testSlug(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug)) + // MinIO objects are not cleaned up — they use the test slug as prefix + // and are effectively isolated. + }) + + chapters := []scraper.Chapter{ + { + Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: The Beginning", Volume: 0}, + Text: "The first chapter text with enough content to be meaningful for a real novel chapter.", + }, + { + Ref: scraper.ChapterRef{Number: 2, Title: "Chapter 2: Rising Action", Volume: 0}, + Text: "The second chapter text continues the story from where the first left off.", + }, + { + Ref: scraper.ChapterRef{Number: 3, Title: "Chapter 3: Climax", Volume: 0}, + Text: "The third chapter text reaches the peak of tension and conflict.", + }, + } + + t.Run("WriteChapter", func(t *testing.T) { + for _, ch := range chapters { + if err := hs.WriteChapter(ctx, slug, ch); err != nil { + t.Fatalf("WriteChapter(%d): %v", ch.Ref.Number, err) + } + t.Logf("wrote chapter %d", ch.Ref.Number) + } + }) + + t.Run("ChapterExists", func(t *testing.T) { + for _, ch := range chapters { + if !hs.ChapterExists(ctx, slug, ch.Ref) { + t.Errorf("ChapterExists(chapter %d) = false after WriteChapter", ch.Ref.Number) + } + } + missing := scraper.ChapterRef{Number: 999, Volume: 0} + if hs.ChapterExists(ctx, slug, missing) { + t.Error("ChapterExists(999) = true for a chapter that was never written") + } + }) + + t.Run("ReadChapter", func(t *testing.T) { + for _, ch := range chapters { + got, err := hs.ReadChapter(ctx, slug, ch.Ref.Number) + if err != nil { + t.Fatalf("ReadChapter(%d): %v", ch.Ref.Number, err) + } + // WriteChapter prepends "# \n\n" and appends "\n". + expectedPrefix := "# " + ch.Ref.Title + if !strings.HasPrefix(got, expectedPrefix) { + t.Errorf("chapter %d: content doesn't start with expected header\ngot: %q\nwant prefix: %q", + ch.Ref.Number, got[:min(len(got), 80)], expectedPrefix) + } + if !strings.Contains(got, ch.Text) { + t.Errorf("chapter %d: content doesn't contain original text", ch.Ref.Number) + } + t.Logf("chapter %d: %d bytes", ch.Ref.Number, len(got)) + } + }) + + t.Run("CountChapters", func(t *testing.T) { + count := hs.CountChapters(ctx, slug) + if count != len(chapters) { + t.Errorf("CountChapters = %d, want %d", count, len(chapters)) + } + }) + + t.Run("ListChapters", func(t *testing.T) { + infos, err := hs.ListChapters(ctx, slug) + if err != nil { + t.Fatalf("ListChapters: %v", err) + } + if len(infos) != len(chapters) { + t.Errorf("ListChapters returned %d entries, want %d", len(infos), len(chapters)) + } + for i, info := range infos { + t.Logf("infos[%d]: number=%d title=%q date=%q", i, info.Number, info.Title, info.Date) + } + // Verify sorted order. + for i := 1; i < len(infos); i++ { + if infos[i].Number <= infos[i-1].Number { + t.Errorf("ListChapters not sorted: infos[%d].Number=%d <= infos[%d].Number=%d", + i, infos[i].Number, i-1, infos[i-1].Number) + } + } + }) +} + +// TestHybridStore_WriteReadRanking exercises WriteRanking → ReadRankingItems +// round-trip. +func TestHybridStore_WriteReadRanking(t *testing.T) { + hs := newTestHybridStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + items := []RankingItem{ + {Rank: 1, Slug: "top-novel", Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top-novel"}, + {Rank: 2, Slug: "second-novel", Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"}, + {Rank: 3, Slug: "third-novel", Title: "Third Novel"}, + } + + t.Run("WriteRanking", func(t *testing.T) { + if err := hs.WriteRanking(ctx, items); err != nil { + t.Fatalf("WriteRanking: %v", err) + } + t.Logf("wrote %d ranking items", len(items)) + }) + + t.Run("ReadRankingItems", func(t *testing.T) { + got, err := hs.ReadRankingItems(ctx) + if err != nil { + t.Fatalf("ReadRankingItems: %v", err) + } + if len(got) != len(items) { + t.Errorf("ReadRankingItems returned %d items, want %d", len(got), len(items)) + } + for i, item := range got { + t.Logf("items[%d]: rank=%d slug=%q title=%q", i, item.Rank, item.Slug, item.Title) + } + if len(got) > 0 { + if got[0].Rank != 1 { + t.Errorf("items[0].Rank = %d, want 1", got[0].Rank) + } + if got[0].Title != "Top Novel" { + t.Errorf("items[0].Title = %q, want %q", got[0].Title, "Top Novel") + } + } + }) + + t.Run("RankingFileInfo", func(t *testing.T) { + fi, err := hs.RankingFileInfo(ctx) + if err != nil { + t.Fatalf("RankingFileInfo: %v", err) + } + if fi == nil { + t.Fatal("RankingFileInfo returned nil") + } + if fi.ModTime().IsZero() { + t.Error("RankingFileInfo.ModTime() is zero") + } + t.Logf("ranking file info: modtime=%s", fi.ModTime()) + }) +} + +// TestHybridStore_Progress exercises SetProgress → GetProgress → AllProgress → +// DeleteProgress via the HybridStore. +func TestHybridStore_Progress(t *testing.T) { + hs := newTestHybridStore(t) + slug := testSlug(t) + const sessionID = "hybrid-test-session-abc" + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = hs.pb.pb.deleteWhere(cleanCtx, "progress", + fmt.Sprintf(`session_id="%s"`, sessionID)) + }) + + p := ReadingProgress{Slug: slug, Chapter: 7, UpdatedAt: time.Now()} + + t.Run("SetProgress", func(t *testing.T) { + if err := hs.SetProgress(ctx, sessionID, p); err != nil { + t.Fatalf("SetProgress: %v", err) + } + }) + + t.Run("GetProgress", func(t *testing.T) { + got, ok := hs.GetProgress(ctx, sessionID, slug) + if !ok { + t.Fatal("GetProgress: not found after SetProgress") + } + if got.Chapter != 7 { + t.Errorf("Chapter = %d, want 7", got.Chapter) + } + if got.Slug != slug { + t.Errorf("Slug = %q, want %q", got.Slug, slug) + } + t.Logf("progress: chapter=%d slug=%q updated=%s", got.Chapter, got.Slug, got.UpdatedAt) + }) + + t.Run("AllProgress", func(t *testing.T) { + all, err := hs.AllProgress(ctx, sessionID) + if err != nil { + t.Fatalf("AllProgress: %v", err) + } + found := false + for _, item := range all { + if item.Slug == slug { + found = true + } + } + if !found { + t.Errorf("AllProgress did not contain slug %q (total=%d)", slug, len(all)) + } + }) + + t.Run("DeleteProgress", func(t *testing.T) { + if err := hs.DeleteProgress(ctx, sessionID, slug); err != nil { + t.Fatalf("DeleteProgress: %v", err) + } + _, ok := hs.GetProgress(ctx, sessionID, slug) + if ok { + t.Error("GetProgress returned ok=true after DeleteProgress") + } + }) +} + +// TestHybridStore_PresignChapter writes a chapter to MinIO via HybridStore, +// then calls PresignChapter and verifies a non-empty URL is returned. +func TestHybridStore_PresignChapter(t *testing.T) { + hs := newTestHybridStore(t) + slug := testSlug(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + ch := scraper.Chapter{ + Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Presign Test", Volume: 0}, + Text: "Text for the presign chapter test.", + } + + if err := hs.WriteChapter(ctx, slug, ch); err != nil { + t.Fatalf("WriteChapter: %v", err) + } + + url, err := hs.PresignChapter(ctx, slug, 1, 10*time.Minute) + if err != nil { + t.Fatalf("PresignChapter: %v", err) + } + if url == "" { + t.Fatal("PresignChapter returned empty URL") + } + if !strings.HasPrefix(url, "http") { + t.Errorf("PresignChapter URL does not start with http: %q", url) + } + t.Logf("presigned chapter URL: %s", url) +} + +// TestHybridStore_PresignAudio puts a fake audio blob into MinIO via the +// underlying MinioClient and verifies PresignAudio returns a valid URL. +func TestHybridStore_PresignAudio(t *testing.T) { + hs := newTestHybridStore(t) + slug := testSlug(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + key := hs.AudioObjectKey(slug, 1, "af_bella", 1.0) + fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test") + + if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil { + t.Fatalf("PutAudio: %v", err) + } + + url, err := hs.PresignAudio(ctx, key, 10*time.Minute) + if err != nil { + t.Fatalf("PresignAudio: %v", err) + } + if url == "" { + t.Fatal("PresignAudio returned empty URL") + } + if !strings.HasPrefix(url, "http") { + t.Errorf("PresignAudio URL does not start with http: %q", url) + } + t.Logf("presigned audio URL: %s", url) +} + +// TestHybridStore_AudioCache exercises SetAudioCache → GetAudioCache via HybridStore. +func TestHybridStore_AudioCache(t *testing.T) { + hs := newTestHybridStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cacheKey := fmt.Sprintf("hybrid-audio-test-%d", time.Now().UnixMilli()) + const filename = "speech_hybrid123.mp3" + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = hs.pb.pb.deleteWhere(cleanCtx, "audio_cache", + fmt.Sprintf(`cache_key="%s"`, cacheKey)) + }) + + if err := hs.SetAudioCache(ctx, cacheKey, filename); err != nil { + t.Fatalf("SetAudioCache: %v", err) + } + + got, ok := hs.GetAudioCache(ctx, cacheKey) + if !ok { + t.Fatal("GetAudioCache returned ok=false after SetAudioCache") + } + if got != filename { + t.Errorf("filename = %q, want %q", got, filename) + } + t.Logf("audio cache: cacheKey=%q filename=%q", cacheKey, got) +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/scraper/internal/storage/integration_test.go b/scraper/internal/storage/integration_test.go new file mode 100644 index 0000000..7da8c9c --- /dev/null +++ b/scraper/internal/storage/integration_test.go @@ -0,0 +1,672 @@ +//go:build integration + +// Integration tests for MinioClient and PocketBaseStore against live instances. +// +// These tests require running MinIO and PocketBase services. They are gated +// behind the "integration" build tag and are never run in a normal `go test ./...`. +// +// Run with: +// +// MINIO_ENDPOINT=localhost:9000 \ +// POCKETBASE_URL=http://localhost:8090 \ +// go test -v -tags integration -timeout 120s \ +// github.com/libnovel/scraper/internal/storage +package storage + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" +) + +// ─── helpers ────────────────────────────────────────────────────────────────── + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func newTestMinioClient(t *testing.T) *MinioClient { + t.Helper() + endpoint := os.Getenv("MINIO_ENDPOINT") + if endpoint == "" { + t.Skip("MINIO_ENDPOINT not set — skipping MinIO integration test") + } + useSSL := os.Getenv("MINIO_USE_SSL") == "true" + cfg := MinioConfig{ + Endpoint: endpoint, + AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), + SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), + UseSSL: useSSL, + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + mc, err := NewMinioClient(ctx, cfg) + if err != nil { + t.Fatalf("NewMinioClient: %v", err) + } + return mc +} + +func newTestPocketBaseStore(t *testing.T) *PocketBaseStore { + t.Helper() + pbURL := os.Getenv("POCKETBASE_URL") + if pbURL == "" { + t.Skip("POCKETBASE_URL not set — skipping PocketBase integration test") + } + cfg := PocketBaseConfig{ + BaseURL: pbURL, + AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), + } + store := NewPocketBaseStore(cfg) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := store.EnsureCollections(ctx); err != nil { + t.Logf("EnsureCollections (may be harmless): %v", err) + } + return store +} + +// testSlug generates a unique test slug to avoid collisions between parallel runs. +func testSlug(t *testing.T) string { + t.Helper() + safe := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + return r + } + return '-' + }, strings.ToLower(t.Name())) + // Truncate and append a timestamp to keep it unique. + if len(safe) > 30 { + safe = safe[:30] + } + return fmt.Sprintf("test-%s-%d", safe, time.Now().UnixMilli()%100000) +} + +// ─── MinioClient tests ──────────────────────────────────────────────────────── + +// TestMinioClient_ChapterRoundTrip verifies PutChapter → GetChapter → +// ChapterExists → ListChapterKeys for a single chapter. +func TestMinioClient_ChapterRoundTrip(t *testing.T) { + mc := newTestMinioClient(t) + slug := testSlug(t) + const vol = 0 + const n = 1 + content := "# Chapter 1\n\nHello integration world.\n" + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Run("PutChapter", func(t *testing.T) { + if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil { + t.Fatalf("PutChapter: %v", err) + } + t.Logf("stored chapter at key: %s", chapterKey(slug, vol, n)) + }) + + t.Run("GetChapter", func(t *testing.T) { + got, err := mc.GetChapter(ctx, slug, vol, n) + if err != nil { + t.Fatalf("GetChapter: %v", err) + } + if got != content { + t.Errorf("GetChapter round-trip mismatch:\ngot: %q\nwant: %q", got, content) + } + t.Logf("retrieved %d bytes", len(got)) + }) + + t.Run("ChapterExists", func(t *testing.T) { + if !mc.ChapterExists(ctx, slug, vol, n) { + t.Error("ChapterExists returned false for a just-stored chapter") + } + if mc.ChapterExists(ctx, slug, vol, 999) { + t.Error("ChapterExists returned true for a chapter that was never stored") + } + }) + + t.Run("ListChapterKeys", func(t *testing.T) { + keys, err := mc.ListChapterKeys(ctx, slug) + if err != nil { + t.Fatalf("ListChapterKeys: %v", err) + } + if len(keys) != 1 { + t.Fatalf("ListChapterKeys returned %d keys, want 1: %v", len(keys), keys) + } + expectedKey := chapterKey(slug, vol, n) + if keys[0] != expectedKey { + t.Errorf("key = %q, want %q", keys[0], expectedKey) + } + t.Logf("keys: %v", keys) + }) +} + +// TestMinioClient_MultiChapterList stores several chapters and verifies +// ListChapterKeys returns them all. +func TestMinioClient_MultiChapterList(t *testing.T) { + mc := newTestMinioClient(t) + slug := testSlug(t) + const vol = 0 + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Store chapters 1, 2, 51 (crosses the 1-50 folder boundary). + chapters := []int{1, 2, 51} + for _, n := range chapters { + content := fmt.Sprintf("# Chapter %d\n\nContent for chapter %d.\n", n, n) + if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil { + t.Fatalf("PutChapter(%d): %v", n, err) + } + } + + keys, err := mc.ListChapterKeys(ctx, slug) + if err != nil { + t.Fatalf("ListChapterKeys: %v", err) + } + t.Logf("keys: %v", keys) + if len(keys) != len(chapters) { + t.Errorf("ListChapterKeys returned %d keys, want %d", len(keys), len(chapters)) + } + + count := mc.CountChapters(ctx, slug) + if count != len(chapters) { + t.Errorf("CountChapters = %d, want %d", count, len(chapters)) + } +} + +// TestMinioClient_PresignChapter verifies PresignChapter returns a non-empty URL. +func TestMinioClient_PresignChapter(t *testing.T) { + mc := newTestMinioClient(t) + slug := testSlug(t) + const vol = 0 + const n = 1 + content := "# Presign test\n\nSome content.\n" + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil { + t.Fatalf("PutChapter: %v", err) + } + + url, err := mc.PresignChapter(ctx, slug, vol, n, 10*time.Minute) + if err != nil { + t.Fatalf("PresignChapter: %v", err) + } + if url == "" { + t.Fatal("PresignChapter returned empty URL") + } + t.Logf("presigned URL: %s", url) + + // URL must be an http(s) URL and contain the slug somewhere. + if !strings.HasPrefix(url, "http") { + t.Errorf("URL does not start with http: %q", url) + } +} + +// TestMinioClient_AudioRoundTrip verifies PutAudio → GetAudio → AudioExists. +func TestMinioClient_AudioRoundTrip(t *testing.T) { + mc := newTestMinioClient(t) + slug := testSlug(t) + key := AudioObjectKey(slug, 1, "af_bella", 1.0) + + // Use minimal fake MP3 bytes (just a recognisable prefix). + fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00integration-test-audio") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Run("PutAudio", func(t *testing.T) { + if err := mc.PutAudio(ctx, key, fakeAudio); err != nil { + t.Fatalf("PutAudio: %v", err) + } + t.Logf("stored audio at key: %s", key) + }) + + t.Run("GetAudio", func(t *testing.T) { + got, err := mc.GetAudio(ctx, key) + if err != nil { + t.Fatalf("GetAudio: %v", err) + } + if string(got) != string(fakeAudio) { + t.Errorf("GetAudio round-trip mismatch: got %d bytes, want %d", len(got), len(fakeAudio)) + } + t.Logf("retrieved %d bytes", len(got)) + }) + + t.Run("AudioExists", func(t *testing.T) { + if !mc.AudioExists(ctx, key) { + t.Error("AudioExists returned false for a just-stored audio object") + } + if mc.AudioExists(ctx, "nonexistent/key.mp3") { + t.Error("AudioExists returned true for a key that was never stored") + } + }) +} + +// TestMinioClient_PresignAudio verifies PresignAudio returns a non-empty URL. +func TestMinioClient_PresignAudio(t *testing.T) { + mc := newTestMinioClient(t) + slug := testSlug(t) + key := AudioObjectKey(slug, 1, "af_bella", 1.0) + fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00presign-audio-test") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := mc.PutAudio(ctx, key, fakeAudio); err != nil { + t.Fatalf("PutAudio: %v", err) + } + + url, err := mc.PresignAudio(ctx, key, 10*time.Minute) + if err != nil { + t.Fatalf("PresignAudio: %v", err) + } + if url == "" { + t.Fatal("PresignAudio returned empty URL") + } + if !strings.HasPrefix(url, "http") { + t.Errorf("URL does not start with http: %q", url) + } + t.Logf("presigned audio URL: %s", url) +} + +// ─── PocketBaseStore tests ──────────────────────────────────────────────────── + +// TestPocketBaseStore_Ping verifies that admin auth works. +func TestPocketBaseStore_Ping(t *testing.T) { + store := newTestPocketBaseStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := store.Ping(ctx); err != nil { + t.Fatalf("Ping: %v", err) + } + t.Log("Ping succeeded") +} + +// TestPocketBaseStore_BookRoundTrip tests UpsertBook → GetBook → ListBooks → +// BookMetaUpdated. +func TestPocketBaseStore_BookRoundTrip(t *testing.T) { + store := newTestPocketBaseStore(t) + slug := testSlug(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Clean up after test. + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = store.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug)) + }) + + t.Run("UpsertBook_Create", func(t *testing.T) { + err := store.UpsertBook(ctx, slug, + "Integration Test Novel", "Test Author", + "https://example.com/cover.jpg", "Ongoing", + "A test summary.", "https://example.com/book/test", + []string{"Action", "Fantasy"}, 42, 7, + ) + if err != nil { + t.Fatalf("UpsertBook (create): %v", err) + } + t.Logf("created book %q", slug) + }) + + t.Run("GetBook", func(t *testing.T) { + rec, found, err := store.GetBook(ctx, slug) + if err != nil { + t.Fatalf("GetBook: %v", err) + } + if !found { + t.Fatal("GetBook: book not found after UpsertBook") + } + t.Logf("GetBook record: %v", rec) + if rec["title"] != "Integration Test Novel" { + t.Errorf("title = %v, want %q", rec["title"], "Integration Test Novel") + } + if rec["author"] != "Test Author" { + t.Errorf("author = %v, want %q", rec["author"], "Test Author") + } + }) + + t.Run("ListBooks", func(t *testing.T) { + books, err := store.ListBooks(ctx) + if err != nil { + t.Fatalf("ListBooks: %v", err) + } + found := false + for _, b := range books { + if s, _ := b["slug"].(string); s == slug { + found = true + break + } + } + if !found { + t.Errorf("ListBooks did not return book with slug %q (total=%d)", slug, len(books)) + } + }) + + t.Run("UpsertBook_Update", func(t *testing.T) { + err := store.UpsertBook(ctx, slug, + "Integration Test Novel", "Test Author Updated", + "", "Completed", "", "https://example.com/book/test", + nil, 100, 3, + ) + if err != nil { + t.Fatalf("UpsertBook (update): %v", err) + } + rec, found, err := store.GetBook(ctx, slug) + if err != nil || !found { + t.Fatalf("GetBook after update: found=%v err=%v", found, err) + } + if rec["author"] != "Test Author Updated" { + t.Errorf("author after update = %v, want %q", rec["author"], "Test Author Updated") + } + if rec["status"] != "Completed" { + t.Errorf("status after update = %v, want %q", rec["status"], "Completed") + } + }) + + t.Run("BookMetaUpdated", func(t *testing.T) { + ts, err := store.BookMetaUpdated(ctx, slug) + if err != nil { + t.Fatalf("BookMetaUpdated: %v", err) + } + if ts.IsZero() { + t.Error("BookMetaUpdated returned zero time") + } + t.Logf("meta_updated: %s", ts) + }) +} + +// TestPocketBaseStore_ChapterIdx tests UpsertChapterIdx → ListChapterIdx → +// CountChapterIdx. +func TestPocketBaseStore_ChapterIdx(t *testing.T) { + store := newTestPocketBaseStore(t) + slug := testSlug(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = store.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug)) + }) + + chapters := []struct { + n int + title string + date string + }{ + {1, "Chapter 1: The Beginning", "2 days ago"}, + {2, "Chapter 2: Rising Action", "1 day ago"}, + {3, "Chapter 3: Climax", "3 hours ago"}, + } + + for _, ch := range chapters { + if err := store.UpsertChapterIdx(ctx, slug, ch.n, ch.title, ch.date); err != nil { + t.Fatalf("UpsertChapterIdx(%d): %v", ch.n, err) + } + } + + t.Run("ListChapterIdx", func(t *testing.T) { + rows, err := store.ListChapterIdx(ctx, slug) + if err != nil { + t.Fatalf("ListChapterIdx: %v", err) + } + if len(rows) != len(chapters) { + t.Errorf("ListChapterIdx returned %d rows, want %d", len(rows), len(chapters)) + } + for i, row := range rows { + t.Logf("row[%d]: number=%v title=%v date_label=%v", i, row["number"], row["title"], row["date_label"]) + } + }) + + t.Run("CountChapterIdx", func(t *testing.T) { + count := store.CountChapterIdx(ctx, slug) + if count != len(chapters) { + t.Errorf("CountChapterIdx = %d, want %d", count, len(chapters)) + } + }) + + t.Run("UpsertChapterIdx_Update", func(t *testing.T) { + // Re-upsert chapter 2 with an updated title. + if err := store.UpsertChapterIdx(ctx, slug, 2, "Chapter 2: Revised Title", "1 day ago"); err != nil { + t.Fatalf("UpsertChapterIdx (update): %v", err) + } + rows, err := store.ListChapterIdx(ctx, slug) + if err != nil { + t.Fatalf("ListChapterIdx after update: %v", err) + } + if store.CountChapterIdx(ctx, slug) != len(chapters) { + t.Errorf("count changed after update: got %d, want %d", len(rows), len(chapters)) + } + }) +} + +// TestPocketBaseStore_Ranking tests SetRanking → GetRanking → RankingModTime. +func TestPocketBaseStore_Ranking(t *testing.T) { + store := newTestPocketBaseStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + const testData = `[{"rank":1,"slug":"test-book","title":"Test Book"}]` + + t.Run("SetRanking", func(t *testing.T) { + if err := store.SetRanking(ctx, testData); err != nil { + t.Fatalf("SetRanking: %v", err) + } + t.Log("SetRanking succeeded") + }) + + t.Run("GetRanking", func(t *testing.T) { + data, updated, err := store.GetRanking(ctx) + if err != nil { + t.Fatalf("GetRanking: %v", err) + } + if data == "" { + t.Error("GetRanking returned empty data") + } + if updated.IsZero() { + t.Error("GetRanking returned zero updated time") + } + t.Logf("data: %s", data) + t.Logf("updated: %s", updated) + }) + + t.Run("RankingModTime", func(t *testing.T) { + fi, err := store.RankingModTime(ctx) + if err != nil { + t.Fatalf("RankingModTime: %v", err) + } + if fi == nil { + t.Fatal("RankingModTime returned nil FileInfo") + } + if fi.ModTime().IsZero() { + t.Error("RankingModTime.ModTime() is zero") + } + t.Logf("ranking modtime: %s", fi.ModTime()) + }) +} + +// TestPocketBaseStore_RankingPageHTML tests SetRankingPageHTML → GetRankingPageHTML. +func TestPocketBaseStore_RankingPageHTML(t *testing.T) { + store := newTestPocketBaseStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + const page = 9999 // unlikely to collide with real data + const html = `<html><body>integration test page 9999</body></html>` + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = store.pb.deleteWhere(cleanCtx, "ranking_html", fmt.Sprintf(`page=%d`, page)) + }) + + t.Run("SetRankingPageHTML", func(t *testing.T) { + if err := store.SetRankingPageHTML(ctx, page, html); err != nil { + t.Fatalf("SetRankingPageHTML: %v", err) + } + }) + + t.Run("GetRankingPageHTML", func(t *testing.T) { + got, updated, err := store.GetRankingPageHTML(ctx, page) + if err != nil { + t.Fatalf("GetRankingPageHTML: %v", err) + } + if got != html { + t.Errorf("html mismatch:\ngot: %q\nwant: %q", got, html) + } + if updated.IsZero() { + t.Error("updated time is zero") + } + t.Logf("retrieved HTML (%d bytes), updated=%s", len(got), updated) + }) +} + +// TestPocketBaseStore_Progress tests SetProgress → GetProgress → AllProgress → +// DeleteProgress. +func TestPocketBaseStore_Progress(t *testing.T) { + store := newTestPocketBaseStore(t) + slug := testSlug(t) + const sessionID = "integration-test-session-xyz" + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = store.pb.deleteWhere(cleanCtx, "progress", + fmt.Sprintf(`session_id="%s"`, sessionID)) + }) + + t.Run("SetProgress", func(t *testing.T) { + if err := store.SetProgress(ctx, sessionID, slug, 5); err != nil { + t.Fatalf("SetProgress: %v", err) + } + }) + + t.Run("GetProgress", func(t *testing.T) { + ch, updated, found, err := store.GetProgress(ctx, sessionID, slug) + if err != nil { + t.Fatalf("GetProgress: %v", err) + } + if !found { + t.Fatal("GetProgress: not found after SetProgress") + } + if ch != 5 { + t.Errorf("chapter = %d, want 5", ch) + } + if updated.IsZero() { + t.Error("updated time is zero") + } + t.Logf("chapter=%d updated=%s", ch, updated) + }) + + t.Run("AllProgress", func(t *testing.T) { + rows, err := store.AllProgress(ctx, sessionID) + if err != nil { + t.Fatalf("AllProgress: %v", err) + } + found := false + for _, r := range rows { + if s, _ := r["slug"].(string); s == slug { + found = true + } + } + if !found { + t.Errorf("AllProgress did not include slug %q (total=%d)", slug, len(rows)) + } + }) + + t.Run("SetProgress_Update", func(t *testing.T) { + if err := store.SetProgress(ctx, sessionID, slug, 12); err != nil { + t.Fatalf("SetProgress (update): %v", err) + } + ch, _, found, err := store.GetProgress(ctx, sessionID, slug) + if err != nil || !found { + t.Fatalf("GetProgress after update: found=%v err=%v", found, err) + } + if ch != 12 { + t.Errorf("chapter after update = %d, want 12", ch) + } + }) + + t.Run("DeleteProgress", func(t *testing.T) { + if err := store.DeleteProgress(ctx, sessionID, slug); err != nil { + t.Fatalf("DeleteProgress: %v", err) + } + _, _, found, err := store.GetProgress(ctx, sessionID, slug) + if err != nil { + t.Fatalf("GetProgress after delete: %v", err) + } + if found { + t.Error("GetProgress returned found=true after DeleteProgress") + } + t.Log("DeleteProgress confirmed") + }) +} + +// TestPocketBaseStore_AudioCache tests SetAudioCache → GetAudioCache. +func TestPocketBaseStore_AudioCache(t *testing.T) { + store := newTestPocketBaseStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cacheKey := fmt.Sprintf("integration-audio-cache-test-%d", time.Now().UnixMilli()) + const filename = "speech_abc123.mp3" + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = store.pb.deleteWhere(cleanCtx, "audio_cache", + fmt.Sprintf(`cache_key="%s"`, cacheKey)) + }) + + t.Run("SetAudioCache", func(t *testing.T) { + if err := store.SetAudioCache(ctx, cacheKey, filename); err != nil { + t.Fatalf("SetAudioCache: %v", err) + } + }) + + t.Run("GetAudioCache", func(t *testing.T) { + got, found, err := store.GetAudioCache(ctx, cacheKey) + if err != nil { + t.Fatalf("GetAudioCache: %v", err) + } + if !found { + t.Fatal("GetAudioCache: not found after SetAudioCache") + } + if got != filename { + t.Errorf("filename = %q, want %q", got, filename) + } + t.Logf("filename: %s", got) + }) + + t.Run("GetAudioCache_Miss", func(t *testing.T) { + got, found, err := store.GetAudioCache(ctx, "does-not-exist-ever") + if err != nil { + t.Fatalf("GetAudioCache (miss): %v", err) + } + if found { + t.Errorf("GetAudioCache returned found=true for missing key, filename=%q", got) + } + }) +} diff --git a/scraper/internal/storage/scrape_integration_test.go b/scraper/internal/storage/scrape_integration_test.go new file mode 100644 index 0000000..95db0c6 --- /dev/null +++ b/scraper/internal/storage/scrape_integration_test.go @@ -0,0 +1,203 @@ +//go:build integration + +// Integration tests that combine live scraping (Browserless) with real storage +// (MinIO + PocketBase) via HybridStore. +// +// These tests require ALL THREE services to be running. They are gated behind +// the "integration" build tag and skipped when any service URL is missing. +// +// Run with: +// +// BROWSERLESS_URL=http://localhost:3030 \ +// MINIO_ENDPOINT=localhost:9000 \ +// POCKETBASE_URL=http://localhost:8090 \ +// go test -v -tags integration -timeout 600s \ +// github.com/libnovel/scraper/internal/storage +package storage + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + "testing" + "time" + + "github.com/libnovel/scraper/internal/browser" + "github.com/libnovel/scraper/internal/novelfire" + "github.com/libnovel/scraper/internal/scraper" +) + +const ( + scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world" + scrapeTestBookSlug = "a-dragon-against-the-whole-world" +) + +// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore, +// skipping the test if any required env var is absent. +func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) { + t.Helper() + + browserlessURL := os.Getenv("BROWSERLESS_URL") + if browserlessURL == "" { + t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test") + } + if os.Getenv("MINIO_ENDPOINT") == "" { + t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test") + } + if os.Getenv("POCKETBASE_URL") == "" { + t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test") + } + + client := browser.NewContentClient(browser.Config{ + BaseURL: browserlessURL, + Token: os.Getenv("BROWSERLESS_TOKEN"), + Timeout: 120 * time.Second, + MaxConcurrent: 1, + }) + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + sc := novelfire.New(client, log, client, nil) + hs := newTestHybridStore(t) + return sc, hs +} + +// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores +// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip. +func TestScrapeAndStore_BookMetadata(t *testing.T) { + sc, hs := newScrapeAndStoreFixture(t) + + slug := scrapeTestBookSlug + "-scrapetest" + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug)) + }) + + // 1. Scrape metadata from the live site. + scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second) + defer scrapeCancel() + + meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL) + if err != nil { + t.Fatalf("ScrapeMetadata: %v", err) + } + t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d", + meta.Slug, meta.Title, meta.Author, meta.TotalChapters) + + // Override slug with our test-specific value to avoid polluting real data. + meta.Slug = slug + + // 2. Write to HybridStore. + storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second) + defer storeCancel() + + if err := hs.WriteMetadata(storeCtx, meta); err != nil { + t.Fatalf("WriteMetadata: %v", err) + } + + // 3. Read back and verify. + got, found, err := hs.ReadMetadata(storeCtx, slug) + if err != nil { + t.Fatalf("ReadMetadata: %v", err) + } + if !found { + t.Fatal("ReadMetadata: not found after WriteMetadata") + } + + t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters) + + if got.Title == "" { + t.Error("Title is empty after round-trip") + } + if got.Author == "" { + t.Error("Author is empty after round-trip") + } + if got.TotalChapters < 1 { + t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters) + } +} + +// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the +// live site and stores each via HybridStore.WriteChapter, then verifies +// ReadChapter returns non-empty markdown with the expected header. +func TestScrapeAndStore_First3Chapters(t *testing.T) { + sc, hs := newScrapeAndStoreFixture(t) + + // Use a unique test slug so we don't pollute the real book. + slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000) + + t.Cleanup(func() { + cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug)) + }) + + // Pre-build chapter refs (known URLs for this test book). + refs := []scraper.ChapterRef{ + {Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"}, + {Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"}, + {Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"}, + } + + for _, ref := range refs { + ref := ref // capture loop variable + t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) { + // 1. Scrape chapter text. + scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second) + defer scrapeCancel() + + ch, err := sc.ScrapeChapterText(scrapeCtx, ref) + if err != nil { + t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err) + } + t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text)) + + if len(ch.Text) < 100 { + t.Errorf("scraped text too short (%d bytes)", len(ch.Text)) + } + + // 2. Write to HybridStore. + storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second) + defer storeCancel() + + if err := hs.WriteChapter(storeCtx, slug, ch); err != nil { + t.Fatalf("WriteChapter(%d): %v", ref.Number, err) + } + + // 3. Read back and verify. + got, err := hs.ReadChapter(storeCtx, slug, ref.Number) + if err != nil { + t.Fatalf("ReadChapter(%d): %v", ref.Number, err) + } + if got == "" { + t.Fatalf("ReadChapter(%d): returned empty string", ref.Number) + } + if len(got) < 100 { + t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got)) + } + + // WriteChapter prepends "# <title>\n\n". + if !strings.HasPrefix(got, "# ") { + t.Errorf("chapter %d: stored content does not start with markdown header: %q", + ref.Number, got[:min(len(got), 60)]) + } + + // Verify the original scraped text body is present. + if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) { + t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number) + } + + t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got)) + }) + } + + // After all chapters written, verify count. + countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer countCancel() + + count := hs.CountChapters(countCtx, slug) + if count != len(refs) { + t.Errorf("CountChapters = %d, want %d", count, len(refs)) + } +}