diff --git a/justfile b/justfile
new file mode 100644
index 0000000..a70ae5e
--- /dev/null
+++ b/justfile
@@ -0,0 +1,124 @@
+# justfile — libnovel-v2 task runner
+# Install just: https://just.systems
+
+scraper_dir := "scraper"
+ui_dir := "ui"
+
+# ─── Build ────────────────────────────────────────────────────────────────────
+
+# Build the scraper binary
+build:
+ cd {{scraper_dir}} && go build -o bin/scraper ./cmd/scraper
+
+# Build and verify all Go packages compile cleanly
+build-all:
+ cd {{scraper_dir}} && go build ./...
+
+# ─── Tests ────────────────────────────────────────────────────────────────────
+
+# Run unit tests only (no integration services required)
+test:
+ cd {{scraper_dir}} && go test ./...
+
+# Run integration tests (requires MinIO, PocketBase, optional Browserless)
+# Override env vars as needed, e.g.:
+# just test-integration MINIO_ENDPOINT=localhost:9000
+test-integration:
+ cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./...
+
+# Run unit + integration tests
+test-all: test test-integration
+
+# Run a specific package's integration tests, e.g.:
+# just test-pkg internal/storage
+test-pkg pkg:
+ cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./{{pkg}}/...
+
+# ─── Code quality ─────────────────────────────────────────────────────────────
+
+# Run go vet on all packages (including integration build tag)
+lint:
+ cd {{scraper_dir}} && go vet ./...
+ cd {{scraper_dir}} && go vet -tags integration ./...
+
+# ─── UI ───────────────────────────────────────────────────────────────────────
+
+# Type-check the SvelteKit UI
+ui-check:
+ cd {{ui_dir}} && npx svelte-check
+
+# Start the SvelteKit dev server
+ui-dev:
+ cd {{ui_dir}} && npm run dev
+
+# Install UI dependencies
+ui-install:
+ cd {{ui_dir}} && npm install
+
+# Build the UI for production
+ui-build:
+ cd {{ui_dir}} && npm run build
+
+# ─── Docker Compose ───────────────────────────────────────────────────────────
+
+# Start all services (browserless, kokoro, scraper, minio, pocketbase)
+up:
+ docker compose up -d
+
+# Stop all services
+down:
+ docker compose down
+
+# Tail logs for all services
+logs:
+ docker compose logs -f
+
+# Tail logs for a specific service, e.g.: just logs-service scraper
+logs-service service:
+ docker compose logs -f {{service}}
+
+# Rebuild and restart a specific service
+restart service:
+ docker compose up -d --build {{service}}
+
+# ─── Local dev: individual services ──────────────────────────────────────────
+
+# Start only PocketBase (for local storage testing)
+pb-up:
+ docker compose up -d pocketbase
+
+# Start only MinIO (for local storage testing)
+minio-up:
+ docker compose up -d minio
+
+# Start only Browserless (for local scraping tests)
+browserless-up:
+ docker compose up -d browserless
+
+# Start storage backends only (MinIO + PocketBase)
+storage-up:
+ docker compose up -d minio pocketbase
+
+# ─── Convenience ─────────────────────────────────────────────────────────────
+
+# Show status of all docker compose services
+status:
+ docker compose ps
+
+# Remove all stopped containers and unused images
+prune:
+ docker compose down --remove-orphans
+ docker image prune -f
+
+# One-shot scrape of the full catalogue (requires services to be running)
+scrape-run: build
+ cd {{scraper_dir}} && ./bin/scraper run
+
+# One-shot scrape of a single book URL, e.g.:
+# just scrape-book https://novelfire.net/book/my-novel
+scrape-book url: build
+ cd {{scraper_dir}} && ./bin/scraper run --url {{url}}
+
+# Start the HTTP server
+serve: build
+ cd {{scraper_dir}} && ./bin/scraper serve
diff --git a/scraper/internal/server/integration_test.go b/scraper/internal/server/integration_test.go
new file mode 100644
index 0000000..3967c4e
--- /dev/null
+++ b/scraper/internal/server/integration_test.go
@@ -0,0 +1,416 @@
+//go:build integration
+
+// Integration tests for the HTTP server against live MinIO + PocketBase backends.
+//
+// The server is started on a random port for each test; real HybridStore
+// backends are used. Browserless-dependent tests are skipped unless
+// BROWSERLESS_URL is set.
+//
+// Run with:
+//
+// MINIO_ENDPOINT=localhost:9000 \
+// POCKETBASE_URL=http://localhost:8090 \
+// go test -v -tags integration -timeout 120s \
+// github.com/libnovel/scraper/internal/server
+package server
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "net"
+ "net/http"
+ neturl "net/url"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/libnovel/scraper/internal/orchestrator"
+ "github.com/libnovel/scraper/internal/scraper"
+ "github.com/libnovel/scraper/internal/storage"
+)
+
+// ─── fixture helpers ──────────────────────────────────────────────────────────
+
+func envOr(key, def string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return def
+}
+
+// newTestStore creates a HybridStore from env vars, skipping if not configured.
+func newTestStore(t *testing.T) *storage.HybridStore {
+ t.Helper()
+ if os.Getenv("MINIO_ENDPOINT") == "" {
+ t.Skip("MINIO_ENDPOINT not set — skipping server integration test")
+ }
+ if os.Getenv("POCKETBASE_URL") == "" {
+ t.Skip("POCKETBASE_URL not set — skipping server integration test")
+ }
+
+ pbCfg := storage.PocketBaseConfig{
+ BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
+ AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
+ AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
+ }
+ minioCfg := storage.MinioConfig{
+ Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
+ AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
+ SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
+ UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
+ BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
+ BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
+ if err != nil {
+ t.Fatalf("NewHybridStore: %v", err)
+ }
+ return hs
+}
+
+// startTestServer starts a real Server on a random free port and returns the
+// base URL. The server is shut down when the test finishes.
+func startTestServer(t *testing.T, store storage.Store) string {
+ t.Helper()
+
+ // Find a free port.
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("net.Listen: %v", err)
+ }
+ addr := ln.Addr().String()
+ ln.Close()
+
+ log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
+
+ // nopScraper satisfies scraper.NovelScraper without hitting the network.
+ srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella")
+
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+
+ ready := make(chan struct{})
+ go func() {
+ // Signal readiness after a short delay to let the listener bind.
+ go func() {
+ time.Sleep(50 * time.Millisecond)
+ close(ready)
+ }()
+ _ = srv.ListenAndServe(ctx)
+ }()
+
+ <-ready
+
+ // Wait until the server actually accepts connections.
+ deadline := time.Now().Add(3 * time.Second)
+ for time.Now().Before(deadline) {
+ resp, err := http.Get("http://" + addr + "/health")
+ if err == nil {
+ resp.Body.Close()
+ break
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+
+ return "http://" + addr
+}
+
+// nopScraper is a no-op NovelScraper implementation for tests that don't
+// exercise scraping functionality.
+type nopScraper struct{}
+
+func (nopScraper) SourceName() string { return "nop" }
+func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
+ ch := make(chan scraper.CatalogueEntry)
+ errs := make(chan error)
+ close(ch)
+ close(errs)
+ return ch, errs
+}
+func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
+ return scraper.BookMeta{}, nil
+}
+func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
+ return nil, nil
+}
+func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
+ return scraper.Chapter{Ref: ref}, nil
+}
+func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
+ ch := make(chan scraper.BookMeta)
+ errs := make(chan error)
+ close(ch)
+ close(errs)
+ return ch, errs
+}
+
+// ─── Tests ────────────────────────────────────────────────────────────────────
+
+// TestServer_Health verifies GET /health returns 200 with status:ok.
+func TestServer_Health(t *testing.T) {
+ store := newTestStore(t)
+ base := startTestServer(t, store)
+
+ resp, err := http.Get(base + "/health")
+ if err != nil {
+ t.Fatalf("GET /health: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Errorf("status = %d, want 200", resp.StatusCode)
+ }
+
+ var body map[string]string
+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+ t.Fatalf("decode health body: %v", err)
+ }
+ if body["status"] != "ok" {
+ t.Errorf("status field = %q, want %q", body["status"], "ok")
+ }
+ t.Logf("health response: %v", body)
+}
+
+// TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false
+// when no scrape is running.
+func TestServer_ScrapeStatus(t *testing.T) {
+ store := newTestStore(t)
+ base := startTestServer(t, store)
+
+ resp, err := http.Get(base + "/api/scrape/status")
+ if err != nil {
+ t.Fatalf("GET /api/scrape/status: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Errorf("status = %d, want 200", resp.StatusCode)
+ }
+
+ var body map[string]bool
+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+ t.Fatalf("decode body: %v", err)
+ }
+ if body["running"] {
+ t.Error("scrape/status.running = true, want false")
+ }
+ t.Logf("scrape status: %v", body)
+}
+
+// TestServer_PresignChapter writes a chapter to MinIO, then calls
+// GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned.
+func TestServer_PresignChapter(t *testing.T) {
+ store := newTestStore(t)
+ base := startTestServer(t, store)
+
+ // Write a chapter directly via the store so we have something to presign.
+ slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000)
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+
+ ch := scraper.Chapter{
+ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0},
+ Text: "Content for the server presign integration test.",
+ }
+ if err := store.WriteChapter(ctx, slug, ch); err != nil {
+ t.Fatalf("WriteChapter: %v", err)
+ }
+ t.Logf("stored chapter for slug=%q", slug)
+
+ // Call the presign endpoint.
+ url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug)
+ resp, err := http.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Errorf("status = %d, want 200", resp.StatusCode)
+ }
+
+ var body map[string]string
+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+ t.Fatalf("decode presign response: %v", err)
+ }
+ presignedURL := body["url"]
+ if presignedURL == "" {
+ t.Fatal("presign response has empty url field")
+ }
+ if !strings.HasPrefix(presignedURL, "http") {
+ t.Errorf("presigned URL does not start with http: %q", presignedURL)
+ }
+ t.Logf("presigned URL: %s", presignedURL)
+}
+
+// TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress.
+func TestServer_Progress(t *testing.T) {
+ store := newTestStore(t)
+ base := startTestServer(t, store)
+
+ slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000)
+
+ // Use a persistent http.Client to carry the session cookie.
+ jar := &cookieJar{cookies: make(map[string][]*http.Cookie)}
+ client := &http.Client{Jar: jar}
+
+ // POST /api/progress/{slug}
+ setURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
+ body, _ := json.Marshal(map[string]int{"chapter": 5})
+ resp, err := client.Post(setURL, "application/json", bytes.NewReader(body))
+ if err != nil {
+ t.Fatalf("POST %s: %v", setURL, err)
+ }
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ t.Errorf("POST progress status = %d, want 200", resp.StatusCode)
+ }
+ t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode)
+
+ // GET /api/progress
+ getURL := fmt.Sprintf("%s/api/progress", base)
+ resp2, err := client.Get(getURL)
+ if err != nil {
+ t.Fatalf("GET %s: %v", getURL, err)
+ }
+ defer resp2.Body.Close()
+
+ if resp2.StatusCode != http.StatusOK {
+ t.Errorf("GET progress status = %d, want 200", resp2.StatusCode)
+ }
+
+ var progress map[string]interface{}
+ if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil {
+ t.Fatalf("decode progress response: %v", err)
+ }
+ t.Logf("progress: %v", progress)
+
+ // The slug should appear with chapter value 5.
+ if ch, ok := progress[slug]; !ok {
+ t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress))
+ } else {
+ // JSON numbers decode as float64.
+ chNum, _ := ch.(float64)
+ if int(chNum) != 5 {
+ t.Errorf("progress[%q] = %v, want 5", slug, ch)
+ }
+ }
+
+ // DELETE /api/progress/{slug}
+ delURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
+ delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil)
+ delResp, err := client.Do(delReq)
+ if err != nil {
+ t.Fatalf("DELETE %s: %v", delURL, err)
+ }
+ delResp.Body.Close()
+ if delResp.StatusCode != http.StatusOK {
+ t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode)
+ }
+ t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode)
+}
+
+// TestServer_PresignChapter_NotFound verifies that presigning a non-existent
+// chapter returns 500 (presign fails on missing object).
+func TestServer_PresignChapter_NotFound(t *testing.T) {
+ store := newTestStore(t)
+ base := startTestServer(t, store)
+
+ url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base)
+ resp, err := http.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer resp.Body.Close()
+
+ // MinIO presign on a non-existent key returns an error; server returns 500.
+ // (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
+ t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
+}
+
+// TestServer_ChapterText writes a chapter and verifies
+// GET /api/chapter-text/{slug}/{n} returns the stripped plain text.
+func TestServer_ChapterText(t *testing.T) {
+ store := newTestStore(t)
+ base := startTestServer(t, store)
+
+ slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000)
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+
+ const chapterText = "The quick brown fox jumps over the lazy dog near the river."
+ ch := scraper.Chapter{
+ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0},
+ Text: chapterText,
+ }
+ if err := store.WriteChapter(ctx, slug, ch); err != nil {
+ t.Fatalf("WriteChapter: %v", err)
+ }
+
+ url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug)
+ resp, err := http.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Errorf("status = %d, want 200", resp.StatusCode)
+ }
+
+ var buf strings.Builder
+ rawBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("read body: %v", err)
+ }
+ buf.Write(rawBytes)
+ text := buf.String()
+ t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)])
+
+ if text == "" {
+ t.Error("chapter-text returned empty body")
+ }
+ // The stripped text should contain our chapter text (markdown heading stripped).
+ if !strings.Contains(text, chapterText) {
+ t.Errorf("chapter text does not contain expected content %q", chapterText)
+ }
+}
+
+// ─── helpers ──────────────────────────────────────────────────────────────────
+
+func mapKeys(m map[string]interface{}) []string {
+ keys := make([]string, 0, len(m))
+ for k := range m {
+ keys = append(keys, k)
+ }
+ return keys
+}
+
+func min(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+}
+
+// cookieJar is a minimal http.CookieJar that stores cookies by host.
+type cookieJar struct {
+ cookies map[string][]*http.Cookie
+}
+
+func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) {
+ j.cookies[u.Host] = append(j.cookies[u.Host], cookies...)
+}
+
+func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie {
+ return j.cookies[u.Host]
+}
diff --git a/scraper/internal/storage/hybrid_integration_test.go b/scraper/internal/storage/hybrid_integration_test.go
new file mode 100644
index 0000000..06d7621
--- /dev/null
+++ b/scraper/internal/storage/hybrid_integration_test.go
@@ -0,0 +1,457 @@
+//go:build integration
+
+// Integration tests for HybridStore (PocketBase + MinIO) end-to-end.
+//
+// Run with:
+//
+// MINIO_ENDPOINT=localhost:9000 \
+// POCKETBASE_URL=http://localhost:8090 \
+// go test -v -tags integration -timeout 120s \
+// github.com/libnovel/scraper/internal/storage
+package storage
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/libnovel/scraper/internal/scraper"
+)
+
+// newTestHybridStore constructs a HybridStore from environment variables.
+// Skips the test if either MINIO_ENDPOINT or POCKETBASE_URL is unset.
+func newTestHybridStore(t *testing.T) *HybridStore {
+ t.Helper()
+ if ep := envOr("MINIO_ENDPOINT", ""); ep == "" {
+ t.Skip("MINIO_ENDPOINT not set — skipping HybridStore integration test")
+ }
+ if u := envOr("POCKETBASE_URL", ""); u == "" {
+ t.Skip("POCKETBASE_URL not set — skipping HybridStore integration test")
+ }
+
+ pbCfg := PocketBaseConfig{
+ BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
+ AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
+ AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
+ }
+ minioCfg := MinioConfig{
+ Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
+ AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
+ SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
+ UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
+ BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
+ BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ hs, err := NewHybridStore(ctx, pbCfg, minioCfg)
+ if err != nil {
+ t.Fatalf("NewHybridStore: %v", err)
+ }
+ return hs
+}
+
+// ─── Tests ────────────────────────────────────────────────────────────────────
+
+// TestHybridStore_WriteReadMetadata exercises WriteMetadata → ReadMetadata round-trip.
+func TestHybridStore_WriteReadMetadata(t *testing.T) {
+ hs := newTestHybridStore(t)
+ slug := testSlug(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
+ })
+
+ meta := scraper.BookMeta{
+ Slug: slug,
+ Title: "Hybrid Store Test Novel",
+ Author: "Test Author",
+ Cover: "https://example.com/cover.jpg",
+ Status: "Ongoing",
+ Genres: []string{"Fantasy", "Action"},
+ Summary: "A novel for integration testing.",
+ TotalChapters: 99,
+ SourceURL: fmt.Sprintf("https://example.com/book/%s", slug),
+ Ranking: 5,
+ }
+
+ t.Run("WriteMetadata", func(t *testing.T) {
+ if err := hs.WriteMetadata(ctx, meta); err != nil {
+ t.Fatalf("WriteMetadata: %v", err)
+ }
+ t.Logf("wrote metadata for slug=%q", slug)
+ })
+
+ t.Run("ReadMetadata", func(t *testing.T) {
+ got, found, err := hs.ReadMetadata(ctx, slug)
+ if err != nil {
+ t.Fatalf("ReadMetadata: %v", err)
+ }
+ if !found {
+ t.Fatal("ReadMetadata: not found after WriteMetadata")
+ }
+ t.Logf("read: %+v", got)
+ if got.Title != meta.Title {
+ t.Errorf("Title = %q, want %q", got.Title, meta.Title)
+ }
+ if got.Author != meta.Author {
+ t.Errorf("Author = %q, want %q", got.Author, meta.Author)
+ }
+ if got.TotalChapters != meta.TotalChapters {
+ t.Errorf("TotalChapters = %d, want %d", got.TotalChapters, meta.TotalChapters)
+ }
+ if got.Ranking != meta.Ranking {
+ t.Errorf("Ranking = %d, want %d", got.Ranking, meta.Ranking)
+ }
+ })
+
+ t.Run("MetadataMtime", func(t *testing.T) {
+ mtime := hs.MetadataMtime(ctx, slug)
+ if mtime == 0 {
+ t.Error("MetadataMtime returned 0")
+ }
+ t.Logf("mtime: %d (%s)", mtime, time.Unix(mtime, 0))
+ })
+
+ t.Run("ReadMetadata_NotFound", func(t *testing.T) {
+ _, found, err := hs.ReadMetadata(ctx, "this-slug-does-not-exist-xyz")
+ if err != nil {
+ t.Fatalf("ReadMetadata (miss): %v", err)
+ }
+ if found {
+ t.Error("ReadMetadata returned found=true for a non-existent slug")
+ }
+ })
+}
+
+// TestHybridStore_WriteReadChapter exercises WriteChapter (MinIO blob + PocketBase
+// index), ReadChapter, CountChapters, and ListChapters.
+func TestHybridStore_WriteReadChapter(t *testing.T) {
+ hs := newTestHybridStore(t)
+ slug := testSlug(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
+ // MinIO objects are not cleaned up — they use the test slug as prefix
+ // and are effectively isolated.
+ })
+
+ chapters := []scraper.Chapter{
+ {
+ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: The Beginning", Volume: 0},
+ Text: "The first chapter text with enough content to be meaningful for a real novel chapter.",
+ },
+ {
+ Ref: scraper.ChapterRef{Number: 2, Title: "Chapter 2: Rising Action", Volume: 0},
+ Text: "The second chapter text continues the story from where the first left off.",
+ },
+ {
+ Ref: scraper.ChapterRef{Number: 3, Title: "Chapter 3: Climax", Volume: 0},
+ Text: "The third chapter text reaches the peak of tension and conflict.",
+ },
+ }
+
+ t.Run("WriteChapter", func(t *testing.T) {
+ for _, ch := range chapters {
+ if err := hs.WriteChapter(ctx, slug, ch); err != nil {
+ t.Fatalf("WriteChapter(%d): %v", ch.Ref.Number, err)
+ }
+ t.Logf("wrote chapter %d", ch.Ref.Number)
+ }
+ })
+
+ t.Run("ChapterExists", func(t *testing.T) {
+ for _, ch := range chapters {
+ if !hs.ChapterExists(ctx, slug, ch.Ref) {
+ t.Errorf("ChapterExists(chapter %d) = false after WriteChapter", ch.Ref.Number)
+ }
+ }
+ missing := scraper.ChapterRef{Number: 999, Volume: 0}
+ if hs.ChapterExists(ctx, slug, missing) {
+ t.Error("ChapterExists(999) = true for a chapter that was never written")
+ }
+ })
+
+ t.Run("ReadChapter", func(t *testing.T) {
+ for _, ch := range chapters {
+ got, err := hs.ReadChapter(ctx, slug, ch.Ref.Number)
+ if err != nil {
+ t.Fatalf("ReadChapter(%d): %v", ch.Ref.Number, err)
+ }
+ // WriteChapter prepends "#
\n\n" and appends "\n".
+ expectedPrefix := "# " + ch.Ref.Title
+ if !strings.HasPrefix(got, expectedPrefix) {
+ t.Errorf("chapter %d: content doesn't start with expected header\ngot: %q\nwant prefix: %q",
+ ch.Ref.Number, got[:min(len(got), 80)], expectedPrefix)
+ }
+ if !strings.Contains(got, ch.Text) {
+ t.Errorf("chapter %d: content doesn't contain original text", ch.Ref.Number)
+ }
+ t.Logf("chapter %d: %d bytes", ch.Ref.Number, len(got))
+ }
+ })
+
+ t.Run("CountChapters", func(t *testing.T) {
+ count := hs.CountChapters(ctx, slug)
+ if count != len(chapters) {
+ t.Errorf("CountChapters = %d, want %d", count, len(chapters))
+ }
+ })
+
+ t.Run("ListChapters", func(t *testing.T) {
+ infos, err := hs.ListChapters(ctx, slug)
+ if err != nil {
+ t.Fatalf("ListChapters: %v", err)
+ }
+ if len(infos) != len(chapters) {
+ t.Errorf("ListChapters returned %d entries, want %d", len(infos), len(chapters))
+ }
+ for i, info := range infos {
+ t.Logf("infos[%d]: number=%d title=%q date=%q", i, info.Number, info.Title, info.Date)
+ }
+ // Verify sorted order.
+ for i := 1; i < len(infos); i++ {
+ if infos[i].Number <= infos[i-1].Number {
+ t.Errorf("ListChapters not sorted: infos[%d].Number=%d <= infos[%d].Number=%d",
+ i, infos[i].Number, i-1, infos[i-1].Number)
+ }
+ }
+ })
+}
+
+// TestHybridStore_WriteReadRanking exercises WriteRanking → ReadRankingItems
+// round-trip.
+func TestHybridStore_WriteReadRanking(t *testing.T) {
+ hs := newTestHybridStore(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ items := []RankingItem{
+ {Rank: 1, Slug: "top-novel", Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top-novel"},
+ {Rank: 2, Slug: "second-novel", Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"},
+ {Rank: 3, Slug: "third-novel", Title: "Third Novel"},
+ }
+
+ t.Run("WriteRanking", func(t *testing.T) {
+ if err := hs.WriteRanking(ctx, items); err != nil {
+ t.Fatalf("WriteRanking: %v", err)
+ }
+ t.Logf("wrote %d ranking items", len(items))
+ })
+
+ t.Run("ReadRankingItems", func(t *testing.T) {
+ got, err := hs.ReadRankingItems(ctx)
+ if err != nil {
+ t.Fatalf("ReadRankingItems: %v", err)
+ }
+ if len(got) != len(items) {
+ t.Errorf("ReadRankingItems returned %d items, want %d", len(got), len(items))
+ }
+ for i, item := range got {
+ t.Logf("items[%d]: rank=%d slug=%q title=%q", i, item.Rank, item.Slug, item.Title)
+ }
+ if len(got) > 0 {
+ if got[0].Rank != 1 {
+ t.Errorf("items[0].Rank = %d, want 1", got[0].Rank)
+ }
+ if got[0].Title != "Top Novel" {
+ t.Errorf("items[0].Title = %q, want %q", got[0].Title, "Top Novel")
+ }
+ }
+ })
+
+ t.Run("RankingFileInfo", func(t *testing.T) {
+ fi, err := hs.RankingFileInfo(ctx)
+ if err != nil {
+ t.Fatalf("RankingFileInfo: %v", err)
+ }
+ if fi == nil {
+ t.Fatal("RankingFileInfo returned nil")
+ }
+ if fi.ModTime().IsZero() {
+ t.Error("RankingFileInfo.ModTime() is zero")
+ }
+ t.Logf("ranking file info: modtime=%s", fi.ModTime())
+ })
+}
+
+// TestHybridStore_Progress exercises SetProgress → GetProgress → AllProgress →
+// DeleteProgress via the HybridStore.
+func TestHybridStore_Progress(t *testing.T) {
+ hs := newTestHybridStore(t)
+ slug := testSlug(t)
+ const sessionID = "hybrid-test-session-abc"
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = hs.pb.pb.deleteWhere(cleanCtx, "progress",
+ fmt.Sprintf(`session_id="%s"`, sessionID))
+ })
+
+ p := ReadingProgress{Slug: slug, Chapter: 7, UpdatedAt: time.Now()}
+
+ t.Run("SetProgress", func(t *testing.T) {
+ if err := hs.SetProgress(ctx, sessionID, p); err != nil {
+ t.Fatalf("SetProgress: %v", err)
+ }
+ })
+
+ t.Run("GetProgress", func(t *testing.T) {
+ got, ok := hs.GetProgress(ctx, sessionID, slug)
+ if !ok {
+ t.Fatal("GetProgress: not found after SetProgress")
+ }
+ if got.Chapter != 7 {
+ t.Errorf("Chapter = %d, want 7", got.Chapter)
+ }
+ if got.Slug != slug {
+ t.Errorf("Slug = %q, want %q", got.Slug, slug)
+ }
+ t.Logf("progress: chapter=%d slug=%q updated=%s", got.Chapter, got.Slug, got.UpdatedAt)
+ })
+
+ t.Run("AllProgress", func(t *testing.T) {
+ all, err := hs.AllProgress(ctx, sessionID)
+ if err != nil {
+ t.Fatalf("AllProgress: %v", err)
+ }
+ found := false
+ for _, item := range all {
+ if item.Slug == slug {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("AllProgress did not contain slug %q (total=%d)", slug, len(all))
+ }
+ })
+
+ t.Run("DeleteProgress", func(t *testing.T) {
+ if err := hs.DeleteProgress(ctx, sessionID, slug); err != nil {
+ t.Fatalf("DeleteProgress: %v", err)
+ }
+ _, ok := hs.GetProgress(ctx, sessionID, slug)
+ if ok {
+ t.Error("GetProgress returned ok=true after DeleteProgress")
+ }
+ })
+}
+
+// TestHybridStore_PresignChapter writes a chapter to MinIO via HybridStore,
+// then calls PresignChapter and verifies a non-empty URL is returned.
+func TestHybridStore_PresignChapter(t *testing.T) {
+ hs := newTestHybridStore(t)
+ slug := testSlug(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ ch := scraper.Chapter{
+ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Presign Test", Volume: 0},
+ Text: "Text for the presign chapter test.",
+ }
+
+ if err := hs.WriteChapter(ctx, slug, ch); err != nil {
+ t.Fatalf("WriteChapter: %v", err)
+ }
+
+ url, err := hs.PresignChapter(ctx, slug, 1, 10*time.Minute)
+ if err != nil {
+ t.Fatalf("PresignChapter: %v", err)
+ }
+ if url == "" {
+ t.Fatal("PresignChapter returned empty URL")
+ }
+ if !strings.HasPrefix(url, "http") {
+ t.Errorf("PresignChapter URL does not start with http: %q", url)
+ }
+ t.Logf("presigned chapter URL: %s", url)
+}
+
+// TestHybridStore_PresignAudio puts a fake audio blob into MinIO via the
+// underlying MinioClient and verifies PresignAudio returns a valid URL.
+func TestHybridStore_PresignAudio(t *testing.T) {
+ hs := newTestHybridStore(t)
+ slug := testSlug(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ key := hs.AudioObjectKey(slug, 1, "af_bella", 1.0)
+ fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test")
+
+ if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil {
+ t.Fatalf("PutAudio: %v", err)
+ }
+
+ url, err := hs.PresignAudio(ctx, key, 10*time.Minute)
+ if err != nil {
+ t.Fatalf("PresignAudio: %v", err)
+ }
+ if url == "" {
+ t.Fatal("PresignAudio returned empty URL")
+ }
+ if !strings.HasPrefix(url, "http") {
+ t.Errorf("PresignAudio URL does not start with http: %q", url)
+ }
+ t.Logf("presigned audio URL: %s", url)
+}
+
+// TestHybridStore_AudioCache exercises SetAudioCache → GetAudioCache via HybridStore.
+func TestHybridStore_AudioCache(t *testing.T) {
+ hs := newTestHybridStore(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ cacheKey := fmt.Sprintf("hybrid-audio-test-%d", time.Now().UnixMilli())
+ const filename = "speech_hybrid123.mp3"
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = hs.pb.pb.deleteWhere(cleanCtx, "audio_cache",
+ fmt.Sprintf(`cache_key="%s"`, cacheKey))
+ })
+
+ if err := hs.SetAudioCache(ctx, cacheKey, filename); err != nil {
+ t.Fatalf("SetAudioCache: %v", err)
+ }
+
+ got, ok := hs.GetAudioCache(ctx, cacheKey)
+ if !ok {
+ t.Fatal("GetAudioCache returned ok=false after SetAudioCache")
+ }
+ if got != filename {
+ t.Errorf("filename = %q, want %q", got, filename)
+ }
+ t.Logf("audio cache: cacheKey=%q filename=%q", cacheKey, got)
+}
+
+// ─── helpers ──────────────────────────────────────────────────────────────────
+
+func min(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+}
diff --git a/scraper/internal/storage/integration_test.go b/scraper/internal/storage/integration_test.go
new file mode 100644
index 0000000..7da8c9c
--- /dev/null
+++ b/scraper/internal/storage/integration_test.go
@@ -0,0 +1,672 @@
+//go:build integration
+
+// Integration tests for MinioClient and PocketBaseStore against live instances.
+//
+// These tests require running MinIO and PocketBase services. They are gated
+// behind the "integration" build tag and are never run in a normal `go test ./...`.
+//
+// Run with:
+//
+// MINIO_ENDPOINT=localhost:9000 \
+// POCKETBASE_URL=http://localhost:8090 \
+// go test -v -tags integration -timeout 120s \
+// github.com/libnovel/scraper/internal/storage
+package storage
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+ "testing"
+ "time"
+)
+
+// ─── helpers ──────────────────────────────────────────────────────────────────
+
+func envOr(key, def string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return def
+}
+
+func newTestMinioClient(t *testing.T) *MinioClient {
+ t.Helper()
+ endpoint := os.Getenv("MINIO_ENDPOINT")
+ if endpoint == "" {
+ t.Skip("MINIO_ENDPOINT not set — skipping MinIO integration test")
+ }
+ useSSL := os.Getenv("MINIO_USE_SSL") == "true"
+ cfg := MinioConfig{
+ Endpoint: endpoint,
+ AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
+ SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
+ UseSSL: useSSL,
+ BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
+ BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+ mc, err := NewMinioClient(ctx, cfg)
+ if err != nil {
+ t.Fatalf("NewMinioClient: %v", err)
+ }
+ return mc
+}
+
+func newTestPocketBaseStore(t *testing.T) *PocketBaseStore {
+ t.Helper()
+ pbURL := os.Getenv("POCKETBASE_URL")
+ if pbURL == "" {
+ t.Skip("POCKETBASE_URL not set — skipping PocketBase integration test")
+ }
+ cfg := PocketBaseConfig{
+ BaseURL: pbURL,
+ AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
+ AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
+ }
+ store := NewPocketBaseStore(cfg)
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+ if err := store.EnsureCollections(ctx); err != nil {
+ t.Logf("EnsureCollections (may be harmless): %v", err)
+ }
+ return store
+}
+
+// testSlug generates a unique test slug to avoid collisions between parallel runs.
+func testSlug(t *testing.T) string {
+ t.Helper()
+ safe := strings.Map(func(r rune) rune {
+ if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' {
+ return r
+ }
+ return '-'
+ }, strings.ToLower(t.Name()))
+ // Truncate and append a timestamp to keep it unique.
+ if len(safe) > 30 {
+ safe = safe[:30]
+ }
+ return fmt.Sprintf("test-%s-%d", safe, time.Now().UnixMilli()%100000)
+}
+
+// ─── MinioClient tests ────────────────────────────────────────────────────────
+
+// TestMinioClient_ChapterRoundTrip verifies PutChapter → GetChapter →
+// ChapterExists → ListChapterKeys for a single chapter.
+func TestMinioClient_ChapterRoundTrip(t *testing.T) {
+ mc := newTestMinioClient(t)
+ slug := testSlug(t)
+ const vol = 0
+ const n = 1
+ content := "# Chapter 1\n\nHello integration world.\n"
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Run("PutChapter", func(t *testing.T) {
+ if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
+ t.Fatalf("PutChapter: %v", err)
+ }
+ t.Logf("stored chapter at key: %s", chapterKey(slug, vol, n))
+ })
+
+ t.Run("GetChapter", func(t *testing.T) {
+ got, err := mc.GetChapter(ctx, slug, vol, n)
+ if err != nil {
+ t.Fatalf("GetChapter: %v", err)
+ }
+ if got != content {
+ t.Errorf("GetChapter round-trip mismatch:\ngot: %q\nwant: %q", got, content)
+ }
+ t.Logf("retrieved %d bytes", len(got))
+ })
+
+ t.Run("ChapterExists", func(t *testing.T) {
+ if !mc.ChapterExists(ctx, slug, vol, n) {
+ t.Error("ChapterExists returned false for a just-stored chapter")
+ }
+ if mc.ChapterExists(ctx, slug, vol, 999) {
+ t.Error("ChapterExists returned true for a chapter that was never stored")
+ }
+ })
+
+ t.Run("ListChapterKeys", func(t *testing.T) {
+ keys, err := mc.ListChapterKeys(ctx, slug)
+ if err != nil {
+ t.Fatalf("ListChapterKeys: %v", err)
+ }
+ if len(keys) != 1 {
+ t.Fatalf("ListChapterKeys returned %d keys, want 1: %v", len(keys), keys)
+ }
+ expectedKey := chapterKey(slug, vol, n)
+ if keys[0] != expectedKey {
+ t.Errorf("key = %q, want %q", keys[0], expectedKey)
+ }
+ t.Logf("keys: %v", keys)
+ })
+}
+
+// TestMinioClient_MultiChapterList stores several chapters and verifies
+// ListChapterKeys returns them all.
+func TestMinioClient_MultiChapterList(t *testing.T) {
+ mc := newTestMinioClient(t)
+ slug := testSlug(t)
+ const vol = 0
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ // Store chapters 1, 2, 51 (crosses the 1-50 folder boundary).
+ chapters := []int{1, 2, 51}
+ for _, n := range chapters {
+ content := fmt.Sprintf("# Chapter %d\n\nContent for chapter %d.\n", n, n)
+ if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
+ t.Fatalf("PutChapter(%d): %v", n, err)
+ }
+ }
+
+ keys, err := mc.ListChapterKeys(ctx, slug)
+ if err != nil {
+ t.Fatalf("ListChapterKeys: %v", err)
+ }
+ t.Logf("keys: %v", keys)
+ if len(keys) != len(chapters) {
+ t.Errorf("ListChapterKeys returned %d keys, want %d", len(keys), len(chapters))
+ }
+
+ count := mc.CountChapters(ctx, slug)
+ if count != len(chapters) {
+ t.Errorf("CountChapters = %d, want %d", count, len(chapters))
+ }
+}
+
+// TestMinioClient_PresignChapter verifies PresignChapter returns a non-empty URL.
+func TestMinioClient_PresignChapter(t *testing.T) {
+ mc := newTestMinioClient(t)
+ slug := testSlug(t)
+ const vol = 0
+ const n = 1
+ content := "# Presign test\n\nSome content.\n"
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
+ t.Fatalf("PutChapter: %v", err)
+ }
+
+ url, err := mc.PresignChapter(ctx, slug, vol, n, 10*time.Minute)
+ if err != nil {
+ t.Fatalf("PresignChapter: %v", err)
+ }
+ if url == "" {
+ t.Fatal("PresignChapter returned empty URL")
+ }
+ t.Logf("presigned URL: %s", url)
+
+ // URL must be an http(s) URL and contain the slug somewhere.
+ if !strings.HasPrefix(url, "http") {
+ t.Errorf("URL does not start with http: %q", url)
+ }
+}
+
+// TestMinioClient_AudioRoundTrip verifies PutAudio → GetAudio → AudioExists.
+func TestMinioClient_AudioRoundTrip(t *testing.T) {
+ mc := newTestMinioClient(t)
+ slug := testSlug(t)
+ key := AudioObjectKey(slug, 1, "af_bella", 1.0)
+
+ // Use minimal fake MP3 bytes (just a recognisable prefix).
+ fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00integration-test-audio")
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Run("PutAudio", func(t *testing.T) {
+ if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
+ t.Fatalf("PutAudio: %v", err)
+ }
+ t.Logf("stored audio at key: %s", key)
+ })
+
+ t.Run("GetAudio", func(t *testing.T) {
+ got, err := mc.GetAudio(ctx, key)
+ if err != nil {
+ t.Fatalf("GetAudio: %v", err)
+ }
+ if string(got) != string(fakeAudio) {
+ t.Errorf("GetAudio round-trip mismatch: got %d bytes, want %d", len(got), len(fakeAudio))
+ }
+ t.Logf("retrieved %d bytes", len(got))
+ })
+
+ t.Run("AudioExists", func(t *testing.T) {
+ if !mc.AudioExists(ctx, key) {
+ t.Error("AudioExists returned false for a just-stored audio object")
+ }
+ if mc.AudioExists(ctx, "nonexistent/key.mp3") {
+ t.Error("AudioExists returned true for a key that was never stored")
+ }
+ })
+}
+
+// TestMinioClient_PresignAudio verifies PresignAudio returns a non-empty URL.
+func TestMinioClient_PresignAudio(t *testing.T) {
+ mc := newTestMinioClient(t)
+ slug := testSlug(t)
+ key := AudioObjectKey(slug, 1, "af_bella", 1.0)
+ fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00presign-audio-test")
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
+ t.Fatalf("PutAudio: %v", err)
+ }
+
+ url, err := mc.PresignAudio(ctx, key, 10*time.Minute)
+ if err != nil {
+ t.Fatalf("PresignAudio: %v", err)
+ }
+ if url == "" {
+ t.Fatal("PresignAudio returned empty URL")
+ }
+ if !strings.HasPrefix(url, "http") {
+ t.Errorf("URL does not start with http: %q", url)
+ }
+ t.Logf("presigned audio URL: %s", url)
+}
+
+// ─── PocketBaseStore tests ────────────────────────────────────────────────────
+
+// TestPocketBaseStore_Ping verifies that admin auth works.
+func TestPocketBaseStore_Ping(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ if err := store.Ping(ctx); err != nil {
+ t.Fatalf("Ping: %v", err)
+ }
+ t.Log("Ping succeeded")
+}
+
+// TestPocketBaseStore_BookRoundTrip tests UpsertBook → GetBook → ListBooks →
+// BookMetaUpdated.
+func TestPocketBaseStore_BookRoundTrip(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+ slug := testSlug(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ // Clean up after test.
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = store.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
+ })
+
+ t.Run("UpsertBook_Create", func(t *testing.T) {
+ err := store.UpsertBook(ctx, slug,
+ "Integration Test Novel", "Test Author",
+ "https://example.com/cover.jpg", "Ongoing",
+ "A test summary.", "https://example.com/book/test",
+ []string{"Action", "Fantasy"}, 42, 7,
+ )
+ if err != nil {
+ t.Fatalf("UpsertBook (create): %v", err)
+ }
+ t.Logf("created book %q", slug)
+ })
+
+ t.Run("GetBook", func(t *testing.T) {
+ rec, found, err := store.GetBook(ctx, slug)
+ if err != nil {
+ t.Fatalf("GetBook: %v", err)
+ }
+ if !found {
+ t.Fatal("GetBook: book not found after UpsertBook")
+ }
+ t.Logf("GetBook record: %v", rec)
+ if rec["title"] != "Integration Test Novel" {
+ t.Errorf("title = %v, want %q", rec["title"], "Integration Test Novel")
+ }
+ if rec["author"] != "Test Author" {
+ t.Errorf("author = %v, want %q", rec["author"], "Test Author")
+ }
+ })
+
+ t.Run("ListBooks", func(t *testing.T) {
+ books, err := store.ListBooks(ctx)
+ if err != nil {
+ t.Fatalf("ListBooks: %v", err)
+ }
+ found := false
+ for _, b := range books {
+ if s, _ := b["slug"].(string); s == slug {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("ListBooks did not return book with slug %q (total=%d)", slug, len(books))
+ }
+ })
+
+ t.Run("UpsertBook_Update", func(t *testing.T) {
+ err := store.UpsertBook(ctx, slug,
+ "Integration Test Novel", "Test Author Updated",
+ "", "Completed", "", "https://example.com/book/test",
+ nil, 100, 3,
+ )
+ if err != nil {
+ t.Fatalf("UpsertBook (update): %v", err)
+ }
+ rec, found, err := store.GetBook(ctx, slug)
+ if err != nil || !found {
+ t.Fatalf("GetBook after update: found=%v err=%v", found, err)
+ }
+ if rec["author"] != "Test Author Updated" {
+ t.Errorf("author after update = %v, want %q", rec["author"], "Test Author Updated")
+ }
+ if rec["status"] != "Completed" {
+ t.Errorf("status after update = %v, want %q", rec["status"], "Completed")
+ }
+ })
+
+ t.Run("BookMetaUpdated", func(t *testing.T) {
+ ts, err := store.BookMetaUpdated(ctx, slug)
+ if err != nil {
+ t.Fatalf("BookMetaUpdated: %v", err)
+ }
+ if ts.IsZero() {
+ t.Error("BookMetaUpdated returned zero time")
+ }
+ t.Logf("meta_updated: %s", ts)
+ })
+}
+
+// TestPocketBaseStore_ChapterIdx tests UpsertChapterIdx → ListChapterIdx →
+// CountChapterIdx.
+func TestPocketBaseStore_ChapterIdx(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+ slug := testSlug(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = store.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
+ })
+
+ chapters := []struct {
+ n int
+ title string
+ date string
+ }{
+ {1, "Chapter 1: The Beginning", "2 days ago"},
+ {2, "Chapter 2: Rising Action", "1 day ago"},
+ {3, "Chapter 3: Climax", "3 hours ago"},
+ }
+
+ for _, ch := range chapters {
+ if err := store.UpsertChapterIdx(ctx, slug, ch.n, ch.title, ch.date); err != nil {
+ t.Fatalf("UpsertChapterIdx(%d): %v", ch.n, err)
+ }
+ }
+
+ t.Run("ListChapterIdx", func(t *testing.T) {
+ rows, err := store.ListChapterIdx(ctx, slug)
+ if err != nil {
+ t.Fatalf("ListChapterIdx: %v", err)
+ }
+ if len(rows) != len(chapters) {
+ t.Errorf("ListChapterIdx returned %d rows, want %d", len(rows), len(chapters))
+ }
+ for i, row := range rows {
+ t.Logf("row[%d]: number=%v title=%v date_label=%v", i, row["number"], row["title"], row["date_label"])
+ }
+ })
+
+ t.Run("CountChapterIdx", func(t *testing.T) {
+ count := store.CountChapterIdx(ctx, slug)
+ if count != len(chapters) {
+ t.Errorf("CountChapterIdx = %d, want %d", count, len(chapters))
+ }
+ })
+
+ t.Run("UpsertChapterIdx_Update", func(t *testing.T) {
+ // Re-upsert chapter 2 with an updated title.
+ if err := store.UpsertChapterIdx(ctx, slug, 2, "Chapter 2: Revised Title", "1 day ago"); err != nil {
+ t.Fatalf("UpsertChapterIdx (update): %v", err)
+ }
+ rows, err := store.ListChapterIdx(ctx, slug)
+ if err != nil {
+ t.Fatalf("ListChapterIdx after update: %v", err)
+ }
+ if store.CountChapterIdx(ctx, slug) != len(chapters) {
+ t.Errorf("count changed after update: got %d, want %d", len(rows), len(chapters))
+ }
+ })
+}
+
+// TestPocketBaseStore_Ranking tests SetRanking → GetRanking → RankingModTime.
+func TestPocketBaseStore_Ranking(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ const testData = `[{"rank":1,"slug":"test-book","title":"Test Book"}]`
+
+ t.Run("SetRanking", func(t *testing.T) {
+ if err := store.SetRanking(ctx, testData); err != nil {
+ t.Fatalf("SetRanking: %v", err)
+ }
+ t.Log("SetRanking succeeded")
+ })
+
+ t.Run("GetRanking", func(t *testing.T) {
+ data, updated, err := store.GetRanking(ctx)
+ if err != nil {
+ t.Fatalf("GetRanking: %v", err)
+ }
+ if data == "" {
+ t.Error("GetRanking returned empty data")
+ }
+ if updated.IsZero() {
+ t.Error("GetRanking returned zero updated time")
+ }
+ t.Logf("data: %s", data)
+ t.Logf("updated: %s", updated)
+ })
+
+ t.Run("RankingModTime", func(t *testing.T) {
+ fi, err := store.RankingModTime(ctx)
+ if err != nil {
+ t.Fatalf("RankingModTime: %v", err)
+ }
+ if fi == nil {
+ t.Fatal("RankingModTime returned nil FileInfo")
+ }
+ if fi.ModTime().IsZero() {
+ t.Error("RankingModTime.ModTime() is zero")
+ }
+ t.Logf("ranking modtime: %s", fi.ModTime())
+ })
+}
+
+// TestPocketBaseStore_RankingPageHTML tests SetRankingPageHTML → GetRankingPageHTML.
+func TestPocketBaseStore_RankingPageHTML(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ const page = 9999 // unlikely to collide with real data
+ const html = `integration test page 9999`
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = store.pb.deleteWhere(cleanCtx, "ranking_html", fmt.Sprintf(`page=%d`, page))
+ })
+
+ t.Run("SetRankingPageHTML", func(t *testing.T) {
+ if err := store.SetRankingPageHTML(ctx, page, html); err != nil {
+ t.Fatalf("SetRankingPageHTML: %v", err)
+ }
+ })
+
+ t.Run("GetRankingPageHTML", func(t *testing.T) {
+ got, updated, err := store.GetRankingPageHTML(ctx, page)
+ if err != nil {
+ t.Fatalf("GetRankingPageHTML: %v", err)
+ }
+ if got != html {
+ t.Errorf("html mismatch:\ngot: %q\nwant: %q", got, html)
+ }
+ if updated.IsZero() {
+ t.Error("updated time is zero")
+ }
+ t.Logf("retrieved HTML (%d bytes), updated=%s", len(got), updated)
+ })
+}
+
+// TestPocketBaseStore_Progress tests SetProgress → GetProgress → AllProgress →
+// DeleteProgress.
+func TestPocketBaseStore_Progress(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+ slug := testSlug(t)
+ const sessionID = "integration-test-session-xyz"
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = store.pb.deleteWhere(cleanCtx, "progress",
+ fmt.Sprintf(`session_id="%s"`, sessionID))
+ })
+
+ t.Run("SetProgress", func(t *testing.T) {
+ if err := store.SetProgress(ctx, sessionID, slug, 5); err != nil {
+ t.Fatalf("SetProgress: %v", err)
+ }
+ })
+
+ t.Run("GetProgress", func(t *testing.T) {
+ ch, updated, found, err := store.GetProgress(ctx, sessionID, slug)
+ if err != nil {
+ t.Fatalf("GetProgress: %v", err)
+ }
+ if !found {
+ t.Fatal("GetProgress: not found after SetProgress")
+ }
+ if ch != 5 {
+ t.Errorf("chapter = %d, want 5", ch)
+ }
+ if updated.IsZero() {
+ t.Error("updated time is zero")
+ }
+ t.Logf("chapter=%d updated=%s", ch, updated)
+ })
+
+ t.Run("AllProgress", func(t *testing.T) {
+ rows, err := store.AllProgress(ctx, sessionID)
+ if err != nil {
+ t.Fatalf("AllProgress: %v", err)
+ }
+ found := false
+ for _, r := range rows {
+ if s, _ := r["slug"].(string); s == slug {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("AllProgress did not include slug %q (total=%d)", slug, len(rows))
+ }
+ })
+
+ t.Run("SetProgress_Update", func(t *testing.T) {
+ if err := store.SetProgress(ctx, sessionID, slug, 12); err != nil {
+ t.Fatalf("SetProgress (update): %v", err)
+ }
+ ch, _, found, err := store.GetProgress(ctx, sessionID, slug)
+ if err != nil || !found {
+ t.Fatalf("GetProgress after update: found=%v err=%v", found, err)
+ }
+ if ch != 12 {
+ t.Errorf("chapter after update = %d, want 12", ch)
+ }
+ })
+
+ t.Run("DeleteProgress", func(t *testing.T) {
+ if err := store.DeleteProgress(ctx, sessionID, slug); err != nil {
+ t.Fatalf("DeleteProgress: %v", err)
+ }
+ _, _, found, err := store.GetProgress(ctx, sessionID, slug)
+ if err != nil {
+ t.Fatalf("GetProgress after delete: %v", err)
+ }
+ if found {
+ t.Error("GetProgress returned found=true after DeleteProgress")
+ }
+ t.Log("DeleteProgress confirmed")
+ })
+}
+
+// TestPocketBaseStore_AudioCache tests SetAudioCache → GetAudioCache.
+func TestPocketBaseStore_AudioCache(t *testing.T) {
+ store := newTestPocketBaseStore(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ cacheKey := fmt.Sprintf("integration-audio-cache-test-%d", time.Now().UnixMilli())
+ const filename = "speech_abc123.mp3"
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = store.pb.deleteWhere(cleanCtx, "audio_cache",
+ fmt.Sprintf(`cache_key="%s"`, cacheKey))
+ })
+
+ t.Run("SetAudioCache", func(t *testing.T) {
+ if err := store.SetAudioCache(ctx, cacheKey, filename); err != nil {
+ t.Fatalf("SetAudioCache: %v", err)
+ }
+ })
+
+ t.Run("GetAudioCache", func(t *testing.T) {
+ got, found, err := store.GetAudioCache(ctx, cacheKey)
+ if err != nil {
+ t.Fatalf("GetAudioCache: %v", err)
+ }
+ if !found {
+ t.Fatal("GetAudioCache: not found after SetAudioCache")
+ }
+ if got != filename {
+ t.Errorf("filename = %q, want %q", got, filename)
+ }
+ t.Logf("filename: %s", got)
+ })
+
+ t.Run("GetAudioCache_Miss", func(t *testing.T) {
+ got, found, err := store.GetAudioCache(ctx, "does-not-exist-ever")
+ if err != nil {
+ t.Fatalf("GetAudioCache (miss): %v", err)
+ }
+ if found {
+ t.Errorf("GetAudioCache returned found=true for missing key, filename=%q", got)
+ }
+ })
+}
diff --git a/scraper/internal/storage/scrape_integration_test.go b/scraper/internal/storage/scrape_integration_test.go
new file mode 100644
index 0000000..95db0c6
--- /dev/null
+++ b/scraper/internal/storage/scrape_integration_test.go
@@ -0,0 +1,203 @@
+//go:build integration
+
+// Integration tests that combine live scraping (Browserless) with real storage
+// (MinIO + PocketBase) via HybridStore.
+//
+// These tests require ALL THREE services to be running. They are gated behind
+// the "integration" build tag and skipped when any service URL is missing.
+//
+// Run with:
+//
+// BROWSERLESS_URL=http://localhost:3030 \
+// MINIO_ENDPOINT=localhost:9000 \
+// POCKETBASE_URL=http://localhost:8090 \
+// go test -v -tags integration -timeout 600s \
+// github.com/libnovel/scraper/internal/storage
+package storage
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/libnovel/scraper/internal/browser"
+ "github.com/libnovel/scraper/internal/novelfire"
+ "github.com/libnovel/scraper/internal/scraper"
+)
+
+const (
+ scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world"
+ scrapeTestBookSlug = "a-dragon-against-the-whole-world"
+)
+
+// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore,
+// skipping the test if any required env var is absent.
+func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) {
+ t.Helper()
+
+ browserlessURL := os.Getenv("BROWSERLESS_URL")
+ if browserlessURL == "" {
+ t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test")
+ }
+ if os.Getenv("MINIO_ENDPOINT") == "" {
+ t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test")
+ }
+ if os.Getenv("POCKETBASE_URL") == "" {
+ t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test")
+ }
+
+ client := browser.NewContentClient(browser.Config{
+ BaseURL: browserlessURL,
+ Token: os.Getenv("BROWSERLESS_TOKEN"),
+ Timeout: 120 * time.Second,
+ MaxConcurrent: 1,
+ })
+ log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
+ sc := novelfire.New(client, log, client, nil)
+ hs := newTestHybridStore(t)
+ return sc, hs
+}
+
+// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores
+// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip.
+func TestScrapeAndStore_BookMetadata(t *testing.T) {
+ sc, hs := newScrapeAndStoreFixture(t)
+
+ slug := scrapeTestBookSlug + "-scrapetest"
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
+ })
+
+ // 1. Scrape metadata from the live site.
+ scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer scrapeCancel()
+
+ meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL)
+ if err != nil {
+ t.Fatalf("ScrapeMetadata: %v", err)
+ }
+ t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d",
+ meta.Slug, meta.Title, meta.Author, meta.TotalChapters)
+
+ // Override slug with our test-specific value to avoid polluting real data.
+ meta.Slug = slug
+
+ // 2. Write to HybridStore.
+ storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer storeCancel()
+
+ if err := hs.WriteMetadata(storeCtx, meta); err != nil {
+ t.Fatalf("WriteMetadata: %v", err)
+ }
+
+ // 3. Read back and verify.
+ got, found, err := hs.ReadMetadata(storeCtx, slug)
+ if err != nil {
+ t.Fatalf("ReadMetadata: %v", err)
+ }
+ if !found {
+ t.Fatal("ReadMetadata: not found after WriteMetadata")
+ }
+
+ t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
+
+ if got.Title == "" {
+ t.Error("Title is empty after round-trip")
+ }
+ if got.Author == "" {
+ t.Error("Author is empty after round-trip")
+ }
+ if got.TotalChapters < 1 {
+ t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters)
+ }
+}
+
+// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the
+// live site and stores each via HybridStore.WriteChapter, then verifies
+// ReadChapter returns non-empty markdown with the expected header.
+func TestScrapeAndStore_First3Chapters(t *testing.T) {
+ sc, hs := newScrapeAndStoreFixture(t)
+
+ // Use a unique test slug so we don't pollute the real book.
+ slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000)
+
+ t.Cleanup(func() {
+ cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
+ })
+
+ // Pre-build chapter refs (known URLs for this test book).
+ refs := []scraper.ChapterRef{
+ {Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"},
+ {Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"},
+ {Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"},
+ }
+
+ for _, ref := range refs {
+ ref := ref // capture loop variable
+ t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
+ // 1. Scrape chapter text.
+ scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second)
+ defer scrapeCancel()
+
+ ch, err := sc.ScrapeChapterText(scrapeCtx, ref)
+ if err != nil {
+ t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
+ }
+ t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text))
+
+ if len(ch.Text) < 100 {
+ t.Errorf("scraped text too short (%d bytes)", len(ch.Text))
+ }
+
+ // 2. Write to HybridStore.
+ storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer storeCancel()
+
+ if err := hs.WriteChapter(storeCtx, slug, ch); err != nil {
+ t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
+ }
+
+ // 3. Read back and verify.
+ got, err := hs.ReadChapter(storeCtx, slug, ref.Number)
+ if err != nil {
+ t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
+ }
+ if got == "" {
+ t.Fatalf("ReadChapter(%d): returned empty string", ref.Number)
+ }
+ if len(got) < 100 {
+ t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got))
+ }
+
+ // WriteChapter prepends "# \n\n".
+ if !strings.HasPrefix(got, "# ") {
+ t.Errorf("chapter %d: stored content does not start with markdown header: %q",
+ ref.Number, got[:min(len(got), 60)])
+ }
+
+ // Verify the original scraped text body is present.
+ if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) {
+ t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number)
+ }
+
+ t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got))
+ })
+ }
+
+ // After all chapters written, verify count.
+ countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer countCancel()
+
+ count := hs.CountChapters(countCtx, slug)
+ if count != len(refs) {
+ t.Errorf("CountChapters = %d, want %d", count, len(refs))
+ }
+}