test: add integration tests for storage, server, and scrape+store flows
- scraper/internal/storage/integration_test.go: MinioClient and PocketBaseStore integration tests covering chapter/audio round-trips, presign URLs, book metadata, chapter index, ranking, progress, and audio cache CRUD - scraper/internal/storage/hybrid_integration_test.go: HybridStore end-to-end tests for metadata, chapters, ranking, progress, presign, and audio cache - scraper/internal/storage/scrape_integration_test.go: live Browserless + storage tests that scrape book metadata and first 3 chapters, store them, and verify the round-trip via HybridStore - scraper/internal/server/integration_test.go: HTTP server functional tests for health, scrape status, presign chapter, reading progress, and chapter-text endpoints against real MinIO + PocketBase backends - justfile: task runner at repo root with recipes for build, test, lint, UI, docker-compose, and individual service management
This commit is contained in:
Admin
416
scraper/internal/server/integration_test.go
Normal file
416
scraper/internal/server/integration_test.go
Normal file
@@ -0,0 +1,416 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for the HTTP server against live MinIO + PocketBase backends.
|
||||
//
|
||||
// The server is started on a random port for each test; real HybridStore
|
||||
// backends are used. Browserless-dependent tests are skipped unless
|
||||
// BROWSERLESS_URL is set.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/server
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
neturl "net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// ─── fixture helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// newTestStore creates a HybridStore from env vars, skipping if not configured.
|
||||
func newTestStore(t *testing.T) *storage.HybridStore {
|
||||
t.Helper()
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping server integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping server integration test")
|
||||
}
|
||||
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// startTestServer starts a real Server on a random free port and returns the
|
||||
// base URL. The server is shut down when the test finishes.
|
||||
func startTestServer(t *testing.T, store storage.Store) string {
|
||||
t.Helper()
|
||||
|
||||
// Find a free port.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
|
||||
// nopScraper satisfies scraper.NovelScraper without hitting the network.
|
||||
srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
ready := make(chan struct{})
|
||||
go func() {
|
||||
// Signal readiness after a short delay to let the listener bind.
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(ready)
|
||||
}()
|
||||
_ = srv.ListenAndServe(ctx)
|
||||
}()
|
||||
|
||||
<-ready
|
||||
|
||||
// Wait until the server actually accepts connections.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
return "http://" + addr
|
||||
}
|
||||
|
||||
// nopScraper is a no-op NovelScraper implementation for tests that don't
|
||||
// exercise scraping functionality.
|
||||
type nopScraper struct{}
|
||||
|
||||
func (nopScraper) SourceName() string { return "nop" }
|
||||
func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
||||
ch := make(chan scraper.CatalogueEntry)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
|
||||
return scraper.BookMeta{}, nil
|
||||
}
|
||||
func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
||||
return scraper.Chapter{Ref: ref}, nil
|
||||
}
|
||||
func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
ch := make(chan scraper.BookMeta)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestServer_Health verifies GET /health returns 200 with status:ok.
|
||||
func TestServer_Health(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
resp, err := http.Get(base + "/health")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /health: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode health body: %v", err)
|
||||
}
|
||||
if body["status"] != "ok" {
|
||||
t.Errorf("status field = %q, want %q", body["status"], "ok")
|
||||
}
|
||||
t.Logf("health response: %v", body)
|
||||
}
|
||||
|
||||
// TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false
|
||||
// when no scrape is running.
|
||||
func TestServer_ScrapeStatus(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
resp, err := http.Get(base + "/api/scrape/status")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /api/scrape/status: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]bool
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
if body["running"] {
|
||||
t.Error("scrape/status.running = true, want false")
|
||||
}
|
||||
t.Logf("scrape status: %v", body)
|
||||
}
|
||||
|
||||
// TestServer_PresignChapter writes a chapter to MinIO, then calls
|
||||
// GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned.
|
||||
func TestServer_PresignChapter(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
// Write a chapter directly via the store so we have something to presign.
|
||||
slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0},
|
||||
Text: "Content for the server presign integration test.",
|
||||
}
|
||||
if err := store.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
t.Logf("stored chapter for slug=%q", slug)
|
||||
|
||||
// Call the presign endpoint.
|
||||
url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode presign response: %v", err)
|
||||
}
|
||||
presignedURL := body["url"]
|
||||
if presignedURL == "" {
|
||||
t.Fatal("presign response has empty url field")
|
||||
}
|
||||
if !strings.HasPrefix(presignedURL, "http") {
|
||||
t.Errorf("presigned URL does not start with http: %q", presignedURL)
|
||||
}
|
||||
t.Logf("presigned URL: %s", presignedURL)
|
||||
}
|
||||
|
||||
// TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress.
|
||||
func TestServer_Progress(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000)
|
||||
|
||||
// Use a persistent http.Client to carry the session cookie.
|
||||
jar := &cookieJar{cookies: make(map[string][]*http.Cookie)}
|
||||
client := &http.Client{Jar: jar}
|
||||
|
||||
// POST /api/progress/{slug}
|
||||
setURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
|
||||
body, _ := json.Marshal(map[string]int{"chapter": 5})
|
||||
resp, err := client.Post(setURL, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("POST %s: %v", setURL, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("POST progress status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode)
|
||||
|
||||
// GET /api/progress
|
||||
getURL := fmt.Sprintf("%s/api/progress", base)
|
||||
resp2, err := client.Get(getURL)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", getURL, err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
t.Errorf("GET progress status = %d, want 200", resp2.StatusCode)
|
||||
}
|
||||
|
||||
var progress map[string]interface{}
|
||||
if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil {
|
||||
t.Fatalf("decode progress response: %v", err)
|
||||
}
|
||||
t.Logf("progress: %v", progress)
|
||||
|
||||
// The slug should appear with chapter value 5.
|
||||
if ch, ok := progress[slug]; !ok {
|
||||
t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress))
|
||||
} else {
|
||||
// JSON numbers decode as float64.
|
||||
chNum, _ := ch.(float64)
|
||||
if int(chNum) != 5 {
|
||||
t.Errorf("progress[%q] = %v, want 5", slug, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// DELETE /api/progress/{slug}
|
||||
delURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
|
||||
delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil)
|
||||
delResp, err := client.Do(delReq)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", delURL, err)
|
||||
}
|
||||
delResp.Body.Close()
|
||||
if delResp.StatusCode != http.StatusOK {
|
||||
t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode)
|
||||
}
|
||||
t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode)
|
||||
}
|
||||
|
||||
// TestServer_PresignChapter_NotFound verifies that presigning a non-existent
|
||||
// chapter returns 500 (presign fails on missing object).
|
||||
func TestServer_PresignChapter_NotFound(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// MinIO presign on a non-existent key returns an error; server returns 500.
|
||||
// (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
|
||||
t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// TestServer_ChapterText writes a chapter and verifies
|
||||
// GET /api/chapter-text/{slug}/{n} returns the stripped plain text.
|
||||
func TestServer_ChapterText(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const chapterText = "The quick brown fox jumps over the lazy dog near the river."
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0},
|
||||
Text: chapterText,
|
||||
}
|
||||
if err := store.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
rawBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read body: %v", err)
|
||||
}
|
||||
buf.Write(rawBytes)
|
||||
text := buf.String()
|
||||
t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)])
|
||||
|
||||
if text == "" {
|
||||
t.Error("chapter-text returned empty body")
|
||||
}
|
||||
// The stripped text should contain our chapter text (markdown heading stripped).
|
||||
if !strings.Contains(text, chapterText) {
|
||||
t.Errorf("chapter text does not contain expected content %q", chapterText)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func mapKeys(m map[string]interface{}) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// cookieJar is a minimal http.CookieJar that stores cookies by host.
|
||||
type cookieJar struct {
|
||||
cookies map[string][]*http.Cookie
|
||||
}
|
||||
|
||||
func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) {
|
||||
j.cookies[u.Host] = append(j.cookies[u.Host], cookies...)
|
||||
}
|
||||
|
||||
func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie {
|
||||
return j.cookies[u.Host]
|
||||
}
|
||||
457
scraper/internal/storage/hybrid_integration_test.go
Normal file
457
scraper/internal/storage/hybrid_integration_test.go
Normal file
@@ -0,0 +1,457 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for HybridStore (PocketBase + MinIO) end-to-end.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// newTestHybridStore constructs a HybridStore from environment variables.
|
||||
// Skips the test if either MINIO_ENDPOINT or POCKETBASE_URL is unset.
|
||||
func newTestHybridStore(t *testing.T) *HybridStore {
|
||||
t.Helper()
|
||||
if ep := envOr("MINIO_ENDPOINT", ""); ep == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping HybridStore integration test")
|
||||
}
|
||||
if u := envOr("POCKETBASE_URL", ""); u == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping HybridStore integration test")
|
||||
}
|
||||
|
||||
pbCfg := PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
minioCfg := MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestHybridStore_WriteReadMetadata exercises WriteMetadata → ReadMetadata round-trip.
|
||||
func TestHybridStore_WriteReadMetadata(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
meta := scraper.BookMeta{
|
||||
Slug: slug,
|
||||
Title: "Hybrid Store Test Novel",
|
||||
Author: "Test Author",
|
||||
Cover: "https://example.com/cover.jpg",
|
||||
Status: "Ongoing",
|
||||
Genres: []string{"Fantasy", "Action"},
|
||||
Summary: "A novel for integration testing.",
|
||||
TotalChapters: 99,
|
||||
SourceURL: fmt.Sprintf("https://example.com/book/%s", slug),
|
||||
Ranking: 5,
|
||||
}
|
||||
|
||||
t.Run("WriteMetadata", func(t *testing.T) {
|
||||
if err := hs.WriteMetadata(ctx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
t.Logf("wrote metadata for slug=%q", slug)
|
||||
})
|
||||
|
||||
t.Run("ReadMetadata", func(t *testing.T) {
|
||||
got, found, err := hs.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
t.Logf("read: %+v", got)
|
||||
if got.Title != meta.Title {
|
||||
t.Errorf("Title = %q, want %q", got.Title, meta.Title)
|
||||
}
|
||||
if got.Author != meta.Author {
|
||||
t.Errorf("Author = %q, want %q", got.Author, meta.Author)
|
||||
}
|
||||
if got.TotalChapters != meta.TotalChapters {
|
||||
t.Errorf("TotalChapters = %d, want %d", got.TotalChapters, meta.TotalChapters)
|
||||
}
|
||||
if got.Ranking != meta.Ranking {
|
||||
t.Errorf("Ranking = %d, want %d", got.Ranking, meta.Ranking)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("MetadataMtime", func(t *testing.T) {
|
||||
mtime := hs.MetadataMtime(ctx, slug)
|
||||
if mtime == 0 {
|
||||
t.Error("MetadataMtime returned 0")
|
||||
}
|
||||
t.Logf("mtime: %d (%s)", mtime, time.Unix(mtime, 0))
|
||||
})
|
||||
|
||||
t.Run("ReadMetadata_NotFound", func(t *testing.T) {
|
||||
_, found, err := hs.ReadMetadata(ctx, "this-slug-does-not-exist-xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata (miss): %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("ReadMetadata returned found=true for a non-existent slug")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_WriteReadChapter exercises WriteChapter (MinIO blob + PocketBase
|
||||
// index), ReadChapter, CountChapters, and ListChapters.
|
||||
func TestHybridStore_WriteReadChapter(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
// MinIO objects are not cleaned up — they use the test slug as prefix
|
||||
// and are effectively isolated.
|
||||
})
|
||||
|
||||
chapters := []scraper.Chapter{
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: The Beginning", Volume: 0},
|
||||
Text: "The first chapter text with enough content to be meaningful for a real novel chapter.",
|
||||
},
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 2, Title: "Chapter 2: Rising Action", Volume: 0},
|
||||
Text: "The second chapter text continues the story from where the first left off.",
|
||||
},
|
||||
{
|
||||
Ref: scraper.ChapterRef{Number: 3, Title: "Chapter 3: Climax", Volume: 0},
|
||||
Text: "The third chapter text reaches the peak of tension and conflict.",
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("WriteChapter", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
if err := hs.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ch.Ref.Number, err)
|
||||
}
|
||||
t.Logf("wrote chapter %d", ch.Ref.Number)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ChapterExists", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
if !hs.ChapterExists(ctx, slug, ch.Ref) {
|
||||
t.Errorf("ChapterExists(chapter %d) = false after WriteChapter", ch.Ref.Number)
|
||||
}
|
||||
}
|
||||
missing := scraper.ChapterRef{Number: 999, Volume: 0}
|
||||
if hs.ChapterExists(ctx, slug, missing) {
|
||||
t.Error("ChapterExists(999) = true for a chapter that was never written")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ReadChapter", func(t *testing.T) {
|
||||
for _, ch := range chapters {
|
||||
got, err := hs.ReadChapter(ctx, slug, ch.Ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ch.Ref.Number, err)
|
||||
}
|
||||
// WriteChapter prepends "# <title>\n\n" and appends "\n".
|
||||
expectedPrefix := "# " + ch.Ref.Title
|
||||
if !strings.HasPrefix(got, expectedPrefix) {
|
||||
t.Errorf("chapter %d: content doesn't start with expected header\ngot: %q\nwant prefix: %q",
|
||||
ch.Ref.Number, got[:min(len(got), 80)], expectedPrefix)
|
||||
}
|
||||
if !strings.Contains(got, ch.Text) {
|
||||
t.Errorf("chapter %d: content doesn't contain original text", ch.Ref.Number)
|
||||
}
|
||||
t.Logf("chapter %d: %d bytes", ch.Ref.Number, len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CountChapters", func(t *testing.T) {
|
||||
count := hs.CountChapters(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(chapters))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListChapters", func(t *testing.T) {
|
||||
infos, err := hs.ListChapters(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapters: %v", err)
|
||||
}
|
||||
if len(infos) != len(chapters) {
|
||||
t.Errorf("ListChapters returned %d entries, want %d", len(infos), len(chapters))
|
||||
}
|
||||
for i, info := range infos {
|
||||
t.Logf("infos[%d]: number=%d title=%q date=%q", i, info.Number, info.Title, info.Date)
|
||||
}
|
||||
// Verify sorted order.
|
||||
for i := 1; i < len(infos); i++ {
|
||||
if infos[i].Number <= infos[i-1].Number {
|
||||
t.Errorf("ListChapters not sorted: infos[%d].Number=%d <= infos[%d].Number=%d",
|
||||
i, infos[i].Number, i-1, infos[i-1].Number)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_WriteReadRanking exercises WriteRanking → ReadRankingItems
|
||||
// round-trip.
|
||||
func TestHybridStore_WriteReadRanking(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
items := []RankingItem{
|
||||
{Rank: 1, Slug: "top-novel", Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top-novel"},
|
||||
{Rank: 2, Slug: "second-novel", Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"},
|
||||
{Rank: 3, Slug: "third-novel", Title: "Third Novel"},
|
||||
}
|
||||
|
||||
t.Run("WriteRanking", func(t *testing.T) {
|
||||
if err := hs.WriteRanking(ctx, items); err != nil {
|
||||
t.Fatalf("WriteRanking: %v", err)
|
||||
}
|
||||
t.Logf("wrote %d ranking items", len(items))
|
||||
})
|
||||
|
||||
t.Run("ReadRankingItems", func(t *testing.T) {
|
||||
got, err := hs.ReadRankingItems(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadRankingItems: %v", err)
|
||||
}
|
||||
if len(got) != len(items) {
|
||||
t.Errorf("ReadRankingItems returned %d items, want %d", len(got), len(items))
|
||||
}
|
||||
for i, item := range got {
|
||||
t.Logf("items[%d]: rank=%d slug=%q title=%q", i, item.Rank, item.Slug, item.Title)
|
||||
}
|
||||
if len(got) > 0 {
|
||||
if got[0].Rank != 1 {
|
||||
t.Errorf("items[0].Rank = %d, want 1", got[0].Rank)
|
||||
}
|
||||
if got[0].Title != "Top Novel" {
|
||||
t.Errorf("items[0].Title = %q, want %q", got[0].Title, "Top Novel")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("RankingFileInfo", func(t *testing.T) {
|
||||
fi, err := hs.RankingFileInfo(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RankingFileInfo: %v", err)
|
||||
}
|
||||
if fi == nil {
|
||||
t.Fatal("RankingFileInfo returned nil")
|
||||
}
|
||||
if fi.ModTime().IsZero() {
|
||||
t.Error("RankingFileInfo.ModTime() is zero")
|
||||
}
|
||||
t.Logf("ranking file info: modtime=%s", fi.ModTime())
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_Progress exercises SetProgress → GetProgress → AllProgress →
|
||||
// DeleteProgress via the HybridStore.
|
||||
func TestHybridStore_Progress(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
const sessionID = "hybrid-test-session-abc"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, sessionID))
|
||||
})
|
||||
|
||||
p := ReadingProgress{Slug: slug, Chapter: 7, UpdatedAt: time.Now()}
|
||||
|
||||
t.Run("SetProgress", func(t *testing.T) {
|
||||
if err := hs.SetProgress(ctx, sessionID, p); err != nil {
|
||||
t.Fatalf("SetProgress: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetProgress", func(t *testing.T) {
|
||||
got, ok := hs.GetProgress(ctx, sessionID, slug)
|
||||
if !ok {
|
||||
t.Fatal("GetProgress: not found after SetProgress")
|
||||
}
|
||||
if got.Chapter != 7 {
|
||||
t.Errorf("Chapter = %d, want 7", got.Chapter)
|
||||
}
|
||||
if got.Slug != slug {
|
||||
t.Errorf("Slug = %q, want %q", got.Slug, slug)
|
||||
}
|
||||
t.Logf("progress: chapter=%d slug=%q updated=%s", got.Chapter, got.Slug, got.UpdatedAt)
|
||||
})
|
||||
|
||||
t.Run("AllProgress", func(t *testing.T) {
|
||||
all, err := hs.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("AllProgress: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range all {
|
||||
if item.Slug == slug {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("AllProgress did not contain slug %q (total=%d)", slug, len(all))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteProgress", func(t *testing.T) {
|
||||
if err := hs.DeleteProgress(ctx, sessionID, slug); err != nil {
|
||||
t.Fatalf("DeleteProgress: %v", err)
|
||||
}
|
||||
_, ok := hs.GetProgress(ctx, sessionID, slug)
|
||||
if ok {
|
||||
t.Error("GetProgress returned ok=true after DeleteProgress")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestHybridStore_PresignChapter writes a chapter to MinIO via HybridStore,
|
||||
// then calls PresignChapter and verifies a non-empty URL is returned.
|
||||
func TestHybridStore_PresignChapter(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Presign Test", Volume: 0},
|
||||
Text: "Text for the presign chapter test.",
|
||||
}
|
||||
|
||||
if err := hs.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
|
||||
url, err := hs.PresignChapter(ctx, slug, 1, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignChapter: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignChapter returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("PresignChapter URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned chapter URL: %s", url)
|
||||
}
|
||||
|
||||
// TestHybridStore_PresignAudio puts a fake audio blob into MinIO via the
|
||||
// underlying MinioClient and verifies PresignAudio returns a valid URL.
|
||||
func TestHybridStore_PresignAudio(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
key := hs.AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test")
|
||||
|
||||
if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
|
||||
url, err := hs.PresignAudio(ctx, key, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignAudio: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignAudio returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("PresignAudio URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned audio URL: %s", url)
|
||||
}
|
||||
|
||||
// TestHybridStore_AudioCache exercises SetAudioCache → GetAudioCache via HybridStore.
|
||||
func TestHybridStore_AudioCache(t *testing.T) {
|
||||
hs := newTestHybridStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("hybrid-audio-test-%d", time.Now().UnixMilli())
|
||||
const filename = "speech_hybrid123.mp3"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, cacheKey))
|
||||
})
|
||||
|
||||
if err := hs.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
t.Fatalf("SetAudioCache: %v", err)
|
||||
}
|
||||
|
||||
got, ok := hs.GetAudioCache(ctx, cacheKey)
|
||||
if !ok {
|
||||
t.Fatal("GetAudioCache returned ok=false after SetAudioCache")
|
||||
}
|
||||
if got != filename {
|
||||
t.Errorf("filename = %q, want %q", got, filename)
|
||||
}
|
||||
t.Logf("audio cache: cacheKey=%q filename=%q", cacheKey, got)
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
672
scraper/internal/storage/integration_test.go
Normal file
672
scraper/internal/storage/integration_test.go
Normal file
@@ -0,0 +1,672 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for MinioClient and PocketBaseStore against live instances.
|
||||
//
|
||||
// These tests require running MinIO and PocketBase services. They are gated
|
||||
// behind the "integration" build tag and are never run in a normal `go test ./...`.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func newTestMinioClient(t *testing.T) *MinioClient {
|
||||
t.Helper()
|
||||
endpoint := os.Getenv("MINIO_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping MinIO integration test")
|
||||
}
|
||||
useSSL := os.Getenv("MINIO_USE_SSL") == "true"
|
||||
cfg := MinioConfig{
|
||||
Endpoint: endpoint,
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: useSSL,
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
mc, err := NewMinioClient(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMinioClient: %v", err)
|
||||
}
|
||||
return mc
|
||||
}
|
||||
|
||||
func newTestPocketBaseStore(t *testing.T) *PocketBaseStore {
|
||||
t.Helper()
|
||||
pbURL := os.Getenv("POCKETBASE_URL")
|
||||
if pbURL == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping PocketBase integration test")
|
||||
}
|
||||
cfg := PocketBaseConfig{
|
||||
BaseURL: pbURL,
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
store := NewPocketBaseStore(cfg)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := store.EnsureCollections(ctx); err != nil {
|
||||
t.Logf("EnsureCollections (may be harmless): %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// testSlug generates a unique test slug to avoid collisions between parallel runs.
|
||||
func testSlug(t *testing.T) string {
|
||||
t.Helper()
|
||||
safe := strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, strings.ToLower(t.Name()))
|
||||
// Truncate and append a timestamp to keep it unique.
|
||||
if len(safe) > 30 {
|
||||
safe = safe[:30]
|
||||
}
|
||||
return fmt.Sprintf("test-%s-%d", safe, time.Now().UnixMilli()%100000)
|
||||
}
|
||||
|
||||
// ─── MinioClient tests ────────────────────────────────────────────────────────
|
||||
|
||||
// TestMinioClient_ChapterRoundTrip verifies PutChapter → GetChapter →
|
||||
// ChapterExists → ListChapterKeys for a single chapter.
|
||||
func TestMinioClient_ChapterRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
const n = 1
|
||||
content := "# Chapter 1\n\nHello integration world.\n"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Run("PutChapter", func(t *testing.T) {
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter: %v", err)
|
||||
}
|
||||
t.Logf("stored chapter at key: %s", chapterKey(slug, vol, n))
|
||||
})
|
||||
|
||||
t.Run("GetChapter", func(t *testing.T) {
|
||||
got, err := mc.GetChapter(ctx, slug, vol, n)
|
||||
if err != nil {
|
||||
t.Fatalf("GetChapter: %v", err)
|
||||
}
|
||||
if got != content {
|
||||
t.Errorf("GetChapter round-trip mismatch:\ngot: %q\nwant: %q", got, content)
|
||||
}
|
||||
t.Logf("retrieved %d bytes", len(got))
|
||||
})
|
||||
|
||||
t.Run("ChapterExists", func(t *testing.T) {
|
||||
if !mc.ChapterExists(ctx, slug, vol, n) {
|
||||
t.Error("ChapterExists returned false for a just-stored chapter")
|
||||
}
|
||||
if mc.ChapterExists(ctx, slug, vol, 999) {
|
||||
t.Error("ChapterExists returned true for a chapter that was never stored")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListChapterKeys", func(t *testing.T) {
|
||||
keys, err := mc.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterKeys: %v", err)
|
||||
}
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("ListChapterKeys returned %d keys, want 1: %v", len(keys), keys)
|
||||
}
|
||||
expectedKey := chapterKey(slug, vol, n)
|
||||
if keys[0] != expectedKey {
|
||||
t.Errorf("key = %q, want %q", keys[0], expectedKey)
|
||||
}
|
||||
t.Logf("keys: %v", keys)
|
||||
})
|
||||
}
|
||||
|
||||
// TestMinioClient_MultiChapterList stores several chapters and verifies
|
||||
// ListChapterKeys returns them all.
|
||||
func TestMinioClient_MultiChapterList(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Store chapters 1, 2, 51 (crosses the 1-50 folder boundary).
|
||||
chapters := []int{1, 2, 51}
|
||||
for _, n := range chapters {
|
||||
content := fmt.Sprintf("# Chapter %d\n\nContent for chapter %d.\n", n, n)
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter(%d): %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
keys, err := mc.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterKeys: %v", err)
|
||||
}
|
||||
t.Logf("keys: %v", keys)
|
||||
if len(keys) != len(chapters) {
|
||||
t.Errorf("ListChapterKeys returned %d keys, want %d", len(keys), len(chapters))
|
||||
}
|
||||
|
||||
count := mc.CountChapters(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(chapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMinioClient_PresignChapter verifies PresignChapter returns a non-empty URL.
|
||||
func TestMinioClient_PresignChapter(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
const vol = 0
|
||||
const n = 1
|
||||
content := "# Presign test\n\nSome content.\n"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := mc.PutChapter(ctx, slug, vol, n, content); err != nil {
|
||||
t.Fatalf("PutChapter: %v", err)
|
||||
}
|
||||
|
||||
url, err := mc.PresignChapter(ctx, slug, vol, n, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignChapter: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignChapter returned empty URL")
|
||||
}
|
||||
t.Logf("presigned URL: %s", url)
|
||||
|
||||
// URL must be an http(s) URL and contain the slug somewhere.
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("URL does not start with http: %q", url)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMinioClient_AudioRoundTrip verifies PutAudio → GetAudio → AudioExists.
|
||||
func TestMinioClient_AudioRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
|
||||
// Use minimal fake MP3 bytes (just a recognisable prefix).
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00integration-test-audio")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Run("PutAudio", func(t *testing.T) {
|
||||
if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
t.Logf("stored audio at key: %s", key)
|
||||
})
|
||||
|
||||
t.Run("GetAudio", func(t *testing.T) {
|
||||
got, err := mc.GetAudio(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudio: %v", err)
|
||||
}
|
||||
if string(got) != string(fakeAudio) {
|
||||
t.Errorf("GetAudio round-trip mismatch: got %d bytes, want %d", len(got), len(fakeAudio))
|
||||
}
|
||||
t.Logf("retrieved %d bytes", len(got))
|
||||
})
|
||||
|
||||
t.Run("AudioExists", func(t *testing.T) {
|
||||
if !mc.AudioExists(ctx, key) {
|
||||
t.Error("AudioExists returned false for a just-stored audio object")
|
||||
}
|
||||
if mc.AudioExists(ctx, "nonexistent/key.mp3") {
|
||||
t.Error("AudioExists returned true for a key that was never stored")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMinioClient_PresignAudio verifies PresignAudio returns a non-empty URL.
|
||||
func TestMinioClient_PresignAudio(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00presign-audio-test")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := mc.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
t.Fatalf("PutAudio: %v", err)
|
||||
}
|
||||
|
||||
url, err := mc.PresignAudio(ctx, key, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("PresignAudio: %v", err)
|
||||
}
|
||||
if url == "" {
|
||||
t.Fatal("PresignAudio returned empty URL")
|
||||
}
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
t.Errorf("URL does not start with http: %q", url)
|
||||
}
|
||||
t.Logf("presigned audio URL: %s", url)
|
||||
}
|
||||
|
||||
// ─── PocketBaseStore tests ────────────────────────────────────────────────────
|
||||
|
||||
// TestPocketBaseStore_Ping verifies that admin auth works.
|
||||
func TestPocketBaseStore_Ping(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := store.Ping(ctx); err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
t.Log("Ping succeeded")
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_BookRoundTrip tests UpsertBook → GetBook → ListBooks →
|
||||
// BookMetaUpdated.
|
||||
func TestPocketBaseStore_BookRoundTrip(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Clean up after test.
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
t.Run("UpsertBook_Create", func(t *testing.T) {
|
||||
err := store.UpsertBook(ctx, slug,
|
||||
"Integration Test Novel", "Test Author",
|
||||
"https://example.com/cover.jpg", "Ongoing",
|
||||
"A test summary.", "https://example.com/book/test",
|
||||
[]string{"Action", "Fantasy"}, 42, 7,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBook (create): %v", err)
|
||||
}
|
||||
t.Logf("created book %q", slug)
|
||||
})
|
||||
|
||||
t.Run("GetBook", func(t *testing.T) {
|
||||
rec, found, err := store.GetBook(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBook: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetBook: book not found after UpsertBook")
|
||||
}
|
||||
t.Logf("GetBook record: %v", rec)
|
||||
if rec["title"] != "Integration Test Novel" {
|
||||
t.Errorf("title = %v, want %q", rec["title"], "Integration Test Novel")
|
||||
}
|
||||
if rec["author"] != "Test Author" {
|
||||
t.Errorf("author = %v, want %q", rec["author"], "Test Author")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListBooks", func(t *testing.T) {
|
||||
books, err := store.ListBooks(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBooks: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, b := range books {
|
||||
if s, _ := b["slug"].(string); s == slug {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("ListBooks did not return book with slug %q (total=%d)", slug, len(books))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UpsertBook_Update", func(t *testing.T) {
|
||||
err := store.UpsertBook(ctx, slug,
|
||||
"Integration Test Novel", "Test Author Updated",
|
||||
"", "Completed", "", "https://example.com/book/test",
|
||||
nil, 100, 3,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBook (update): %v", err)
|
||||
}
|
||||
rec, found, err := store.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetBook after update: found=%v err=%v", found, err)
|
||||
}
|
||||
if rec["author"] != "Test Author Updated" {
|
||||
t.Errorf("author after update = %v, want %q", rec["author"], "Test Author Updated")
|
||||
}
|
||||
if rec["status"] != "Completed" {
|
||||
t.Errorf("status after update = %v, want %q", rec["status"], "Completed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("BookMetaUpdated", func(t *testing.T) {
|
||||
ts, err := store.BookMetaUpdated(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("BookMetaUpdated: %v", err)
|
||||
}
|
||||
if ts.IsZero() {
|
||||
t.Error("BookMetaUpdated returned zero time")
|
||||
}
|
||||
t.Logf("meta_updated: %s", ts)
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_ChapterIdx tests UpsertChapterIdx → ListChapterIdx →
|
||||
// CountChapterIdx.
|
||||
func TestPocketBaseStore_ChapterIdx(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
chapters := []struct {
|
||||
n int
|
||||
title string
|
||||
date string
|
||||
}{
|
||||
{1, "Chapter 1: The Beginning", "2 days ago"},
|
||||
{2, "Chapter 2: Rising Action", "1 day ago"},
|
||||
{3, "Chapter 3: Climax", "3 hours ago"},
|
||||
}
|
||||
|
||||
for _, ch := range chapters {
|
||||
if err := store.UpsertChapterIdx(ctx, slug, ch.n, ch.title, ch.date); err != nil {
|
||||
t.Fatalf("UpsertChapterIdx(%d): %v", ch.n, err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("ListChapterIdx", func(t *testing.T) {
|
||||
rows, err := store.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterIdx: %v", err)
|
||||
}
|
||||
if len(rows) != len(chapters) {
|
||||
t.Errorf("ListChapterIdx returned %d rows, want %d", len(rows), len(chapters))
|
||||
}
|
||||
for i, row := range rows {
|
||||
t.Logf("row[%d]: number=%v title=%v date_label=%v", i, row["number"], row["title"], row["date_label"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CountChapterIdx", func(t *testing.T) {
|
||||
count := store.CountChapterIdx(ctx, slug)
|
||||
if count != len(chapters) {
|
||||
t.Errorf("CountChapterIdx = %d, want %d", count, len(chapters))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("UpsertChapterIdx_Update", func(t *testing.T) {
|
||||
// Re-upsert chapter 2 with an updated title.
|
||||
if err := store.UpsertChapterIdx(ctx, slug, 2, "Chapter 2: Revised Title", "1 day ago"); err != nil {
|
||||
t.Fatalf("UpsertChapterIdx (update): %v", err)
|
||||
}
|
||||
rows, err := store.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChapterIdx after update: %v", err)
|
||||
}
|
||||
if store.CountChapterIdx(ctx, slug) != len(chapters) {
|
||||
t.Errorf("count changed after update: got %d, want %d", len(rows), len(chapters))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_Ranking tests SetRanking → GetRanking → RankingModTime.
|
||||
func TestPocketBaseStore_Ranking(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const testData = `[{"rank":1,"slug":"test-book","title":"Test Book"}]`
|
||||
|
||||
t.Run("SetRanking", func(t *testing.T) {
|
||||
if err := store.SetRanking(ctx, testData); err != nil {
|
||||
t.Fatalf("SetRanking: %v", err)
|
||||
}
|
||||
t.Log("SetRanking succeeded")
|
||||
})
|
||||
|
||||
t.Run("GetRanking", func(t *testing.T) {
|
||||
data, updated, err := store.GetRanking(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRanking: %v", err)
|
||||
}
|
||||
if data == "" {
|
||||
t.Error("GetRanking returned empty data")
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("GetRanking returned zero updated time")
|
||||
}
|
||||
t.Logf("data: %s", data)
|
||||
t.Logf("updated: %s", updated)
|
||||
})
|
||||
|
||||
t.Run("RankingModTime", func(t *testing.T) {
|
||||
fi, err := store.RankingModTime(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RankingModTime: %v", err)
|
||||
}
|
||||
if fi == nil {
|
||||
t.Fatal("RankingModTime returned nil FileInfo")
|
||||
}
|
||||
if fi.ModTime().IsZero() {
|
||||
t.Error("RankingModTime.ModTime() is zero")
|
||||
}
|
||||
t.Logf("ranking modtime: %s", fi.ModTime())
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_RankingPageHTML tests SetRankingPageHTML → GetRankingPageHTML.
|
||||
func TestPocketBaseStore_RankingPageHTML(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const page = 9999 // unlikely to collide with real data
|
||||
const html = `<html><body>integration test page 9999</body></html>`
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "ranking_html", fmt.Sprintf(`page=%d`, page))
|
||||
})
|
||||
|
||||
t.Run("SetRankingPageHTML", func(t *testing.T) {
|
||||
if err := store.SetRankingPageHTML(ctx, page, html); err != nil {
|
||||
t.Fatalf("SetRankingPageHTML: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetRankingPageHTML", func(t *testing.T) {
|
||||
got, updated, err := store.GetRankingPageHTML(ctx, page)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRankingPageHTML: %v", err)
|
||||
}
|
||||
if got != html {
|
||||
t.Errorf("html mismatch:\ngot: %q\nwant: %q", got, html)
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("updated time is zero")
|
||||
}
|
||||
t.Logf("retrieved HTML (%d bytes), updated=%s", len(got), updated)
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_Progress tests SetProgress → GetProgress → AllProgress →
|
||||
// DeleteProgress.
|
||||
func TestPocketBaseStore_Progress(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
slug := testSlug(t)
|
||||
const sessionID = "integration-test-session-xyz"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, sessionID))
|
||||
})
|
||||
|
||||
t.Run("SetProgress", func(t *testing.T) {
|
||||
if err := store.SetProgress(ctx, sessionID, slug, 5); err != nil {
|
||||
t.Fatalf("SetProgress: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetProgress", func(t *testing.T) {
|
||||
ch, updated, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProgress: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetProgress: not found after SetProgress")
|
||||
}
|
||||
if ch != 5 {
|
||||
t.Errorf("chapter = %d, want 5", ch)
|
||||
}
|
||||
if updated.IsZero() {
|
||||
t.Error("updated time is zero")
|
||||
}
|
||||
t.Logf("chapter=%d updated=%s", ch, updated)
|
||||
})
|
||||
|
||||
t.Run("AllProgress", func(t *testing.T) {
|
||||
rows, err := store.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("AllProgress: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range rows {
|
||||
if s, _ := r["slug"].(string); s == slug {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("AllProgress did not include slug %q (total=%d)", slug, len(rows))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SetProgress_Update", func(t *testing.T) {
|
||||
if err := store.SetProgress(ctx, sessionID, slug, 12); err != nil {
|
||||
t.Fatalf("SetProgress (update): %v", err)
|
||||
}
|
||||
ch, _, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetProgress after update: found=%v err=%v", found, err)
|
||||
}
|
||||
if ch != 12 {
|
||||
t.Errorf("chapter after update = %d, want 12", ch)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteProgress", func(t *testing.T) {
|
||||
if err := store.DeleteProgress(ctx, sessionID, slug); err != nil {
|
||||
t.Fatalf("DeleteProgress: %v", err)
|
||||
}
|
||||
_, _, found, err := store.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProgress after delete: %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("GetProgress returned found=true after DeleteProgress")
|
||||
}
|
||||
t.Log("DeleteProgress confirmed")
|
||||
})
|
||||
}
|
||||
|
||||
// TestPocketBaseStore_AudioCache tests SetAudioCache → GetAudioCache.
|
||||
func TestPocketBaseStore_AudioCache(t *testing.T) {
|
||||
store := newTestPocketBaseStore(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := fmt.Sprintf("integration-audio-cache-test-%d", time.Now().UnixMilli())
|
||||
const filename = "speech_abc123.mp3"
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = store.pb.deleteWhere(cleanCtx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, cacheKey))
|
||||
})
|
||||
|
||||
t.Run("SetAudioCache", func(t *testing.T) {
|
||||
if err := store.SetAudioCache(ctx, cacheKey, filename); err != nil {
|
||||
t.Fatalf("SetAudioCache: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("GetAudioCache", func(t *testing.T) {
|
||||
got, found, err := store.GetAudioCache(ctx, cacheKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioCache: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("GetAudioCache: not found after SetAudioCache")
|
||||
}
|
||||
if got != filename {
|
||||
t.Errorf("filename = %q, want %q", got, filename)
|
||||
}
|
||||
t.Logf("filename: %s", got)
|
||||
})
|
||||
|
||||
t.Run("GetAudioCache_Miss", func(t *testing.T) {
|
||||
got, found, err := store.GetAudioCache(ctx, "does-not-exist-ever")
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioCache (miss): %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Errorf("GetAudioCache returned found=true for missing key, filename=%q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
203
scraper/internal/storage/scrape_integration_test.go
Normal file
203
scraper/internal/storage/scrape_integration_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests that combine live scraping (Browserless) with real storage
|
||||
// (MinIO + PocketBase) via HybridStore.
|
||||
//
|
||||
// These tests require ALL THREE services to be running. They are gated behind
|
||||
// the "integration" build tag and skipped when any service URL is missing.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// BROWSERLESS_URL=http://localhost:3030 \
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 600s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
const (
|
||||
scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world"
|
||||
scrapeTestBookSlug = "a-dragon-against-the-whole-world"
|
||||
)
|
||||
|
||||
// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore,
|
||||
// skipping the test if any required env var is absent.
|
||||
func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) {
|
||||
t.Helper()
|
||||
|
||||
browserlessURL := os.Getenv("BROWSERLESS_URL")
|
||||
if browserlessURL == "" {
|
||||
t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
|
||||
client := browser.NewContentClient(browser.Config{
|
||||
BaseURL: browserlessURL,
|
||||
Token: os.Getenv("BROWSERLESS_TOKEN"),
|
||||
Timeout: 120 * time.Second,
|
||||
MaxConcurrent: 1,
|
||||
})
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
sc := novelfire.New(client, log, client, nil)
|
||||
hs := newTestHybridStore(t)
|
||||
return sc, hs
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores
|
||||
// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip.
|
||||
func TestScrapeAndStore_BookMetadata(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
slug := scrapeTestBookSlug + "-scrapetest"
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// 1. Scrape metadata from the live site.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeMetadata: %v", err)
|
||||
}
|
||||
t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d",
|
||||
meta.Slug, meta.Title, meta.Author, meta.TotalChapters)
|
||||
|
||||
// Override slug with our test-specific value to avoid polluting real data.
|
||||
meta.Slug = slug
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteMetadata(storeCtx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, found, err := hs.ReadMetadata(storeCtx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
|
||||
t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
|
||||
|
||||
if got.Title == "" {
|
||||
t.Error("Title is empty after round-trip")
|
||||
}
|
||||
if got.Author == "" {
|
||||
t.Error("Author is empty after round-trip")
|
||||
}
|
||||
if got.TotalChapters < 1 {
|
||||
t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the
|
||||
// live site and stores each via HybridStore.WriteChapter, then verifies
|
||||
// ReadChapter returns non-empty markdown with the expected header.
|
||||
func TestScrapeAndStore_First3Chapters(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
// Use a unique test slug so we don't pollute the real book.
|
||||
slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000)
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// Pre-build chapter refs (known URLs for this test book).
|
||||
refs := []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"},
|
||||
{Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"},
|
||||
{Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"},
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
ref := ref // capture loop variable
|
||||
t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
|
||||
// 1. Scrape chapter text.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
ch, err := sc.ScrapeChapterText(scrapeCtx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
|
||||
}
|
||||
t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text))
|
||||
|
||||
if len(ch.Text) < 100 {
|
||||
t.Errorf("scraped text too short (%d bytes)", len(ch.Text))
|
||||
}
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteChapter(storeCtx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, err := hs.ReadChapter(storeCtx, slug, ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatalf("ReadChapter(%d): returned empty string", ref.Number)
|
||||
}
|
||||
if len(got) < 100 {
|
||||
t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got))
|
||||
}
|
||||
|
||||
// WriteChapter prepends "# <title>\n\n".
|
||||
if !strings.HasPrefix(got, "# ") {
|
||||
t.Errorf("chapter %d: stored content does not start with markdown header: %q",
|
||||
ref.Number, got[:min(len(got), 60)])
|
||||
}
|
||||
|
||||
// Verify the original scraped text body is present.
|
||||
if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) {
|
||||
t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number)
|
||||
}
|
||||
|
||||
t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got))
|
||||
})
|
||||
}
|
||||
|
||||
// After all chapters written, verify count.
|
||||
countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer countCancel()
|
||||
|
||||
count := hs.CountChapters(countCtx, slug)
|
||||
if count != len(refs) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(refs))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user