//go:build integration // Integration tests that combine live scraping (Browserless) with real storage // (MinIO + PocketBase) via HybridStore. // // These tests require ALL THREE services to be running. They are gated behind // the "integration" build tag and skipped when any service URL is missing. // // Run with: // // BROWSERLESS_URL=http://localhost:3030 \ // MINIO_ENDPOINT=localhost:9000 \ // POCKETBASE_URL=http://localhost:8090 \ // go test -v -tags integration -timeout 600s \ // github.com/libnovel/scraper/internal/storage package storage import ( "context" "fmt" "log/slog" "os" "strings" "testing" "time" "github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/novelfire" "github.com/libnovel/scraper/internal/scraper" ) const ( scrapeTestBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world" scrapeTestBookSlug = "a-dragon-against-the-whole-world" ) // newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore, // skipping the test if any required env var is absent. func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) { t.Helper() browserlessURL := os.Getenv("BROWSERLESS_URL") if browserlessURL == "" { t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test") } if os.Getenv("MINIO_ENDPOINT") == "" { t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test") } if os.Getenv("POCKETBASE_URL") == "" { t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test") } client := browser.NewContentClient(browser.Config{ BaseURL: browserlessURL, Token: os.Getenv("BROWSERLESS_TOKEN"), Timeout: 120 * time.Second, MaxConcurrent: 1, }) log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) sc := novelfire.New(client, log, client, nil) hs := newTestHybridStore(t) return sc, hs } // TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores // it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip. func TestScrapeAndStore_BookMetadata(t *testing.T) { sc, hs := newScrapeAndStoreFixture(t) slug := scrapeTestBookSlug + "-scrapetest" t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug)) }) // 1. Scrape metadata from the live site. scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second) defer scrapeCancel() meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL) if err != nil { t.Fatalf("ScrapeMetadata: %v", err) } t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d", meta.Slug, meta.Title, meta.Author, meta.TotalChapters) // Override slug with our test-specific value to avoid polluting real data. meta.Slug = slug // 2. Write to HybridStore. storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second) defer storeCancel() if err := hs.WriteMetadata(storeCtx, meta); err != nil { t.Fatalf("WriteMetadata: %v", err) } // 3. Read back and verify. got, found, err := hs.ReadMetadata(storeCtx, slug) if err != nil { t.Fatalf("ReadMetadata: %v", err) } if !found { t.Fatal("ReadMetadata: not found after WriteMetadata") } t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters) if got.Title == "" { t.Error("Title is empty after round-trip") } if got.Author == "" { t.Error("Author is empty after round-trip") } if got.TotalChapters < 1 { t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters) } } // TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the // live site and stores each via HybridStore.WriteChapter, then verifies // ReadChapter returns non-empty markdown with the expected header. func TestScrapeAndStore_First3Chapters(t *testing.T) { sc, hs := newScrapeAndStoreFixture(t) // Use a unique test slug so we don't pollute the real book. slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000) t.Cleanup(func() { cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug)) }) // Pre-build chapter refs (known URLs for this test book). refs := []scraper.ChapterRef{ {Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"}, {Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"}, {Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"}, } for _, ref := range refs { ref := ref // capture loop variable t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) { // 1. Scrape chapter text. scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second) defer scrapeCancel() ch, err := sc.ScrapeChapterText(scrapeCtx, ref) if err != nil { t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err) } t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text)) if len(ch.Text) < 100 { t.Errorf("scraped text too short (%d bytes)", len(ch.Text)) } // 2. Write to HybridStore. storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second) defer storeCancel() if err := hs.WriteChapter(storeCtx, slug, ch); err != nil { t.Fatalf("WriteChapter(%d): %v", ref.Number, err) } // 3. Read back and verify. got, err := hs.ReadChapter(storeCtx, slug, ref.Number) if err != nil { t.Fatalf("ReadChapter(%d): %v", ref.Number, err) } if got == "" { t.Fatalf("ReadChapter(%d): returned empty string", ref.Number) } if len(got) < 100 { t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got)) } // WriteChapter prepends "#