test: add integration tests for storage, server, and scrape+store flows
- scraper/internal/storage/integration_test.go: MinioClient and PocketBaseStore integration tests covering chapter/audio round-trips, presign URLs, book metadata, chapter index, ranking, progress, and audio cache CRUD - scraper/internal/storage/hybrid_integration_test.go: HybridStore end-to-end tests for metadata, chapters, ranking, progress, presign, and audio cache - scraper/internal/storage/scrape_integration_test.go: live Browserless + storage tests that scrape book metadata and first 3 chapters, store them, and verify the round-trip via HybridStore - scraper/internal/server/integration_test.go: HTTP server functional tests for health, scrape status, presign chapter, reading progress, and chapter-text endpoints against real MinIO + PocketBase backends - justfile: task runner at repo root with recipes for build, test, lint, UI, docker-compose, and individual service management
This commit is contained in:
Admin
203
scraper/internal/storage/scrape_integration_test.go
Normal file
203
scraper/internal/storage/scrape_integration_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests that combine live scraping (Browserless) with real storage
|
||||
// (MinIO + PocketBase) via HybridStore.
|
||||
//
|
||||
// These tests require ALL THREE services to be running. They are gated behind
|
||||
// the "integration" build tag and skipped when any service URL is missing.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// BROWSERLESS_URL=http://localhost:3030 \
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 600s \
|
||||
// github.com/libnovel/scraper/internal/storage
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
const (
|
||||
scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world"
|
||||
scrapeTestBookSlug = "a-dragon-against-the-whole-world"
|
||||
)
|
||||
|
||||
// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore,
|
||||
// skipping the test if any required env var is absent.
|
||||
func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) {
|
||||
t.Helper()
|
||||
|
||||
browserlessURL := os.Getenv("BROWSERLESS_URL")
|
||||
if browserlessURL == "" {
|
||||
t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test")
|
||||
}
|
||||
|
||||
client := browser.NewContentClient(browser.Config{
|
||||
BaseURL: browserlessURL,
|
||||
Token: os.Getenv("BROWSERLESS_TOKEN"),
|
||||
Timeout: 120 * time.Second,
|
||||
MaxConcurrent: 1,
|
||||
})
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
sc := novelfire.New(client, log, client, nil)
|
||||
hs := newTestHybridStore(t)
|
||||
return sc, hs
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores
|
||||
// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip.
|
||||
func TestScrapeAndStore_BookMetadata(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
slug := scrapeTestBookSlug + "-scrapetest"
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// 1. Scrape metadata from the live site.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeMetadata: %v", err)
|
||||
}
|
||||
t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d",
|
||||
meta.Slug, meta.Title, meta.Author, meta.TotalChapters)
|
||||
|
||||
// Override slug with our test-specific value to avoid polluting real data.
|
||||
meta.Slug = slug
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteMetadata(storeCtx, meta); err != nil {
|
||||
t.Fatalf("WriteMetadata: %v", err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, found, err := hs.ReadMetadata(storeCtx, slug)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMetadata: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("ReadMetadata: not found after WriteMetadata")
|
||||
}
|
||||
|
||||
t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
|
||||
|
||||
if got.Title == "" {
|
||||
t.Error("Title is empty after round-trip")
|
||||
}
|
||||
if got.Author == "" {
|
||||
t.Error("Author is empty after round-trip")
|
||||
}
|
||||
if got.TotalChapters < 1 {
|
||||
t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the
|
||||
// live site and stores each via HybridStore.WriteChapter, then verifies
|
||||
// ReadChapter returns non-empty markdown with the expected header.
|
||||
func TestScrapeAndStore_First3Chapters(t *testing.T) {
|
||||
sc, hs := newScrapeAndStoreFixture(t)
|
||||
|
||||
// Use a unique test slug so we don't pollute the real book.
|
||||
slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000)
|
||||
|
||||
t.Cleanup(func() {
|
||||
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
|
||||
})
|
||||
|
||||
// Pre-build chapter refs (known URLs for this test book).
|
||||
refs := []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"},
|
||||
{Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"},
|
||||
{Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"},
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
ref := ref // capture loop variable
|
||||
t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
|
||||
// 1. Scrape chapter text.
|
||||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer scrapeCancel()
|
||||
|
||||
ch, err := sc.ScrapeChapterText(scrapeCtx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
|
||||
}
|
||||
t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text))
|
||||
|
||||
if len(ch.Text) < 100 {
|
||||
t.Errorf("scraped text too short (%d bytes)", len(ch.Text))
|
||||
}
|
||||
|
||||
// 2. Write to HybridStore.
|
||||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer storeCancel()
|
||||
|
||||
if err := hs.WriteChapter(storeCtx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
|
||||
// 3. Read back and verify.
|
||||
got, err := hs.ReadChapter(storeCtx, slug, ref.Number)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatalf("ReadChapter(%d): returned empty string", ref.Number)
|
||||
}
|
||||
if len(got) < 100 {
|
||||
t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got))
|
||||
}
|
||||
|
||||
// WriteChapter prepends "# <title>\n\n".
|
||||
if !strings.HasPrefix(got, "# ") {
|
||||
t.Errorf("chapter %d: stored content does not start with markdown header: %q",
|
||||
ref.Number, got[:min(len(got), 60)])
|
||||
}
|
||||
|
||||
// Verify the original scraped text body is present.
|
||||
if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) {
|
||||
t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number)
|
||||
}
|
||||
|
||||
t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got))
|
||||
})
|
||||
}
|
||||
|
||||
// After all chapters written, verify count.
|
||||
countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer countCancel()
|
||||
|
||||
count := hs.CountChapters(countCtx, slug)
|
||||
if count != len(refs) {
|
||||
t.Errorf("CountChapters = %d, want %d", count, len(refs))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user