libnovel/scraper/internal/storage/scrape_integration_test.go

//go:build integration

// Integration tests that combine live scraping (Browserless) with real storage
// (MinIO + PocketBase) via HybridStore.
//
// These tests require ALL THREE services to be running. They are gated behind
// the "integration" build tag and skipped when any service URL is missing.
//
// Run with:
//
//	BROWSERLESS_URL=http://localhost:3030 \
//	MINIO_ENDPOINT=localhost:9000 \
//	POCKETBASE_URL=http://localhost:8090 \
//	go test -v -tags integration -timeout 600s \
//	    github.com/libnovel/scraper/internal/storage
package storage

import (
	"context"
	"fmt"
	"log/slog"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/libnovel/scraper/internal/browser"
	"github.com/libnovel/scraper/internal/novelfire"
	"github.com/libnovel/scraper/internal/scraper"
)

const (
	scrapeTestBookURL  = "https://novelfire.net/book/a-dragon-against-the-whole-world"
	scrapeTestBookSlug = "a-dragon-against-the-whole-world"
)

// newScrapeAndStoreFixture builds a novelfire Scraper and a HybridStore,
// skipping the test if any required env var is absent.
func newScrapeAndStoreFixture(t *testing.T) (*novelfire.Scraper, *HybridStore) {
	t.Helper()

	browserlessURL := os.Getenv("BROWSERLESS_URL")
	if browserlessURL == "" {
		t.Skip("BROWSERLESS_URL not set — skipping scrape+store integration test")
	}
	if os.Getenv("MINIO_ENDPOINT") == "" {
		t.Skip("MINIO_ENDPOINT not set — skipping scrape+store integration test")
	}
	if os.Getenv("POCKETBASE_URL") == "" {
		t.Skip("POCKETBASE_URL not set — skipping scrape+store integration test")
	}

	client := browser.NewContentClient(browser.Config{
		BaseURL:       browserlessURL,
		Token:         os.Getenv("BROWSERLESS_TOKEN"),
		Timeout:       120 * time.Second,
		MaxConcurrent: 1,
	})
	log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
	sc := novelfire.New(client, log, client, nil, nil)
	hs := newTestHybridStore(t)
	return sc, hs
}

// TestScrapeAndStore_BookMetadata scrapes the test book's metadata and stores
// it via HybridStore.WriteMetadata, then verifies a ReadMetadata round-trip.
func TestScrapeAndStore_BookMetadata(t *testing.T) {
	sc, hs := newScrapeAndStoreFixture(t)

	slug := scrapeTestBookSlug + "-scrapetest"
	t.Cleanup(func() {
		cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_ = hs.pb.pb.deleteWhere(cleanCtx, "books", fmt.Sprintf(`slug="%s"`, slug))
	})

	// 1. Scrape metadata from the live site.
	scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer scrapeCancel()

	meta, err := sc.ScrapeMetadata(scrapeCtx, scrapeTestBookURL)
	if err != nil {
		t.Fatalf("ScrapeMetadata: %v", err)
	}
	t.Logf("scraped: slug=%q title=%q author=%q totalChapters=%d",
		meta.Slug, meta.Title, meta.Author, meta.TotalChapters)

	// Override slug with our test-specific value to avoid polluting real data.
	meta.Slug = slug

	// 2. Write to HybridStore.
	storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer storeCancel()

	if err := hs.WriteMetadata(storeCtx, meta); err != nil {
		t.Fatalf("WriteMetadata: %v", err)
	}

	// 3. Read back and verify.
	got, found, err := hs.ReadMetadata(storeCtx, slug)
	if err != nil {
		t.Fatalf("ReadMetadata: %v", err)
	}
	if !found {
		t.Fatal("ReadMetadata: not found after WriteMetadata")
	}

	t.Logf("read back: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)

	if got.Title == "" {
		t.Error("Title is empty after round-trip")
	}
	if got.Author == "" {
		t.Error("Author is empty after round-trip")
	}
	if got.TotalChapters < 1 {
		t.Errorf("TotalChapters = %d, want >= 1", got.TotalChapters)
	}
}

// TestScrapeAndStore_First3Chapters scrapes chapters 1, 2, and 3 from the
// live site and stores each via HybridStore.WriteChapter, then verifies
// ReadChapter returns non-empty markdown with the expected header.
func TestScrapeAndStore_First3Chapters(t *testing.T) {
	sc, hs := newScrapeAndStoreFixture(t)

	// Use a unique test slug so we don't pollute the real book.
	slug := fmt.Sprintf("%s-chtest-%d", scrapeTestBookSlug, time.Now().UnixMilli()%100000)

	t.Cleanup(func() {
		cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_ = hs.pb.pb.deleteWhere(cleanCtx, "chapters_idx", fmt.Sprintf(`slug="%s"`, slug))
	})

	// Pre-build chapter refs (known URLs for this test book).
	refs := []scraper.ChapterRef{
		{Number: 1, Title: "Chapter 1", Volume: 0, URL: scrapeTestBookURL + "/chapter-1"},
		{Number: 2, Title: "Chapter 2", Volume: 0, URL: scrapeTestBookURL + "/chapter-2"},
		{Number: 3, Title: "Chapter 3", Volume: 0, URL: scrapeTestBookURL + "/chapter-3"},
	}

	for _, ref := range refs {
		ref := ref // capture loop variable
		t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
			// 1. Scrape chapter text.
			scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 120*time.Second)
			defer scrapeCancel()

			ch, err := sc.ScrapeChapterText(scrapeCtx, ref)
			if err != nil {
				t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
			}
			t.Logf("scraped chapter %d: %d bytes of markdown", ref.Number, len(ch.Text))

			if len(ch.Text) < 100 {
				t.Errorf("scraped text too short (%d bytes)", len(ch.Text))
			}

			// 2. Write to HybridStore.
			storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
			defer storeCancel()

			if err := hs.WriteChapter(storeCtx, slug, ch); err != nil {
				t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
			}

			// 3. Read back and verify.
			got, err := hs.ReadChapter(storeCtx, slug, ref.Number)
			if err != nil {
				t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
			}
			if got == "" {
				t.Fatalf("ReadChapter(%d): returned empty string", ref.Number)
			}
			if len(got) < 100 {
				t.Errorf("ReadChapter(%d): content too short (%d bytes)", ref.Number, len(got))
			}

			// WriteChapter prepends "# <title>\n\n".
			if !strings.HasPrefix(got, "# ") {
				t.Errorf("chapter %d: stored content does not start with markdown header: %q",
					ref.Number, got[:min(len(got), 60)])
			}

			// Verify the original scraped text body is present.
			if !strings.Contains(got, ch.Text[:min(len(ch.Text), 50)]) {
				t.Errorf("chapter %d: stored content does not contain scraped text excerpt", ref.Number)
			}

			t.Logf("chapter %d stored and verified: %d bytes", ref.Number, len(got))
		})
	}

	// After all chapters written, verify count.
	countCtx, countCancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer countCancel()

	count := hs.CountChapters(countCtx, slug)
	if count != len(refs) {
		t.Errorf("CountChapters = %d, want %d", count, len(refs))
	}
}