libnovel/scraper/internal/server/integration_test.go

//go:build integration

// Integration tests for the HTTP server against live MinIO + PocketBase backends.
//
// The server is started on a random port for each test; real HybridStore
// backends are used. Browserless-dependent tests are skipped unless
// BROWSERLESS_URL is set.
//
// Run with:
//
//	MINIO_ENDPOINT=localhost:9000 \
//	POCKETBASE_URL=http://localhost:8090 \
//	go test -v -tags integration -timeout 120s \
//	    github.com/libnovel/scraper/internal/server
package server

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	neturl "net/url"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/libnovel/scraper/internal/orchestrator"
	"github.com/libnovel/scraper/internal/scraper"
	"github.com/libnovel/scraper/internal/storage"
)

// ─── fixture helpers ──────────────────────────────────────────────────────────

func envOr(key, def string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return def
}

// newTestStore creates a HybridStore from env vars, skipping if not configured.
func newTestStore(t *testing.T) *storage.HybridStore {
	t.Helper()
	if os.Getenv("MINIO_ENDPOINT") == "" {
		t.Skip("MINIO_ENDPOINT not set — skipping server integration test")
	}
	if os.Getenv("POCKETBASE_URL") == "" {
		t.Skip("POCKETBASE_URL not set — skipping server integration test")
	}

	pbCfg := storage.PocketBaseConfig{
		BaseURL:       envOr("POCKETBASE_URL", "http://localhost:8090"),
		AdminEmail:    envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
		AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
	}
	minioCfg := storage.MinioConfig{
		Endpoint:       envOr("MINIO_ENDPOINT", "localhost:9000"),
		AccessKey:      envOr("MINIO_ACCESS_KEY", "admin"),
		SecretKey:      envOr("MINIO_SECRET_KEY", "changeme123"),
		UseSSL:         envOr("MINIO_USE_SSL", "false") == "true",
		BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
		BucketAudio:    envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
	if err != nil {
		t.Fatalf("NewHybridStore: %v", err)
	}
	return hs
}

// startTestServer starts a real Server on a random free port and returns the
// base URL. The server is shut down when the test finishes.
func startTestServer(t *testing.T, store storage.Store) string {
	t.Helper()

	// Find a free port.
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("net.Listen: %v", err)
	}
	addr := ln.Addr().String()
	ln.Close()

	log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))

	// nopScraper satisfies scraper.NovelScraper without hitting the network.
	srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella")

	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)

	ready := make(chan struct{})
	go func() {
		// Signal readiness after a short delay to let the listener bind.
		go func() {
			time.Sleep(50 * time.Millisecond)
			close(ready)
		}()
		_ = srv.ListenAndServe(ctx)
	}()

	<-ready

	// Wait until the server actually accepts connections.
	deadline := time.Now().Add(3 * time.Second)
	for time.Now().Before(deadline) {
		resp, err := http.Get("http://" + addr + "/health")
		if err == nil {
			resp.Body.Close()
			break
		}
		time.Sleep(20 * time.Millisecond)
	}

	return "http://" + addr
}

// nopScraper is a no-op NovelScraper implementation for tests that don't
// exercise scraping functionality.
type nopScraper struct{}

func (nopScraper) SourceName() string { return "nop" }
func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
	ch := make(chan scraper.CatalogueEntry)
	errs := make(chan error)
	close(ch)
	close(errs)
	return ch, errs
}
func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
	return scraper.BookMeta{}, nil
}
func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
	return nil, nil
}
func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
	return scraper.Chapter{Ref: ref}, nil
}
func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
	ch := make(chan scraper.BookMeta)
	errs := make(chan error)
	close(ch)
	close(errs)
	return ch, errs
}

// ─── Tests ────────────────────────────────────────────────────────────────────

// TestServer_Health verifies GET /health returns 200 with status:ok.
func TestServer_Health(t *testing.T) {
	store := newTestStore(t)
	base := startTestServer(t, store)

	resp, err := http.Get(base + "/health")
	if err != nil {
		t.Fatalf("GET /health: %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		t.Errorf("status = %d, want 200", resp.StatusCode)
	}

	var body map[string]string
	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
		t.Fatalf("decode health body: %v", err)
	}
	if body["status"] != "ok" {
		t.Errorf("status field = %q, want %q", body["status"], "ok")
	}
	t.Logf("health response: %v", body)
}

// TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false
// when no scrape is running.
func TestServer_ScrapeStatus(t *testing.T) {
	store := newTestStore(t)
	base := startTestServer(t, store)

	resp, err := http.Get(base + "/api/scrape/status")
	if err != nil {
		t.Fatalf("GET /api/scrape/status: %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		t.Errorf("status = %d, want 200", resp.StatusCode)
	}

	var body map[string]bool
	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
		t.Fatalf("decode body: %v", err)
	}
	if body["running"] {
		t.Error("scrape/status.running = true, want false")
	}
	t.Logf("scrape status: %v", body)
}

// TestServer_PresignChapter writes a chapter to MinIO, then calls
// GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned.
func TestServer_PresignChapter(t *testing.T) {
	store := newTestStore(t)
	base := startTestServer(t, store)

	// Write a chapter directly via the store so we have something to presign.
	slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000)
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()

	ch := scraper.Chapter{
		Ref:  scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0},
		Text: "Content for the server presign integration test.",
	}
	if err := store.WriteChapter(ctx, slug, ch); err != nil {
		t.Fatalf("WriteChapter: %v", err)
	}
	t.Logf("stored chapter for slug=%q", slug)

	// Call the presign endpoint.
	url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug)
	resp, err := http.Get(url)
	if err != nil {
		t.Fatalf("GET %s: %v", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		t.Errorf("status = %d, want 200", resp.StatusCode)
	}

	var body map[string]string
	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
		t.Fatalf("decode presign response: %v", err)
	}
	presignedURL := body["url"]
	if presignedURL == "" {
		t.Fatal("presign response has empty url field")
	}
	if !strings.HasPrefix(presignedURL, "http") {
		t.Errorf("presigned URL does not start with http: %q", presignedURL)
	}
	t.Logf("presigned URL: %s", presignedURL)
}

// TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress.
func TestServer_Progress(t *testing.T) {
	store := newTestStore(t)
	base := startTestServer(t, store)

	slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000)

	// Use a persistent http.Client to carry the session cookie.
	jar := &cookieJar{cookies: make(map[string][]*http.Cookie)}
	client := &http.Client{Jar: jar}

	// POST /api/progress/{slug}
	setURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
	body, _ := json.Marshal(map[string]int{"chapter": 5})
	resp, err := client.Post(setURL, "application/json", bytes.NewReader(body))
	if err != nil {
		t.Fatalf("POST %s: %v", setURL, err)
	}
	resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Errorf("POST progress status = %d, want 200", resp.StatusCode)
	}
	t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode)

	// GET /api/progress
	getURL := fmt.Sprintf("%s/api/progress", base)
	resp2, err := client.Get(getURL)
	if err != nil {
		t.Fatalf("GET %s: %v", getURL, err)
	}
	defer resp2.Body.Close()

	if resp2.StatusCode != http.StatusOK {
		t.Errorf("GET progress status = %d, want 200", resp2.StatusCode)
	}

	var progress map[string]interface{}
	if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil {
		t.Fatalf("decode progress response: %v", err)
	}
	t.Logf("progress: %v", progress)

	// The slug should appear with chapter value 5.
	if ch, ok := progress[slug]; !ok {
		t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress))
	} else {
		// JSON numbers decode as float64.
		chNum, _ := ch.(float64)
		if int(chNum) != 5 {
			t.Errorf("progress[%q] = %v, want 5", slug, ch)
		}
	}

	// DELETE /api/progress/{slug}
	delURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
	delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil)
	delResp, err := client.Do(delReq)
	if err != nil {
		t.Fatalf("DELETE %s: %v", delURL, err)
	}
	delResp.Body.Close()
	if delResp.StatusCode != http.StatusOK {
		t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode)
	}
	t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode)
}

// TestServer_PresignChapter_NotFound verifies that presigning a non-existent
// chapter returns 500 (presign fails on missing object).
func TestServer_PresignChapter_NotFound(t *testing.T) {
	store := newTestStore(t)
	base := startTestServer(t, store)

	url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base)
	resp, err := http.Get(url)
	if err != nil {
		t.Fatalf("GET %s: %v", url, err)
	}
	defer resp.Body.Close()

	// MinIO presign on a non-existent key returns an error; server returns 500.
	// (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
	t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
	if resp.StatusCode != http.StatusInternalServerError && resp.StatusCode != http.StatusOK {
		t.Errorf("status = %d, want 500 or 200", resp.StatusCode)
	}
}

// TestServer_ChapterText writes a chapter and verifies
// GET /api/chapter-text/{slug}/{n} returns the stripped plain text.
func TestServer_ChapterText(t *testing.T) {
	store := newTestStore(t)
	base := startTestServer(t, store)

	slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000)
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()

	const chapterText = "The quick brown fox jumps over the lazy dog near the river."
	ch := scraper.Chapter{
		Ref:  scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0},
		Text: chapterText,
	}
	if err := store.WriteChapter(ctx, slug, ch); err != nil {
		t.Fatalf("WriteChapter: %v", err)
	}

	url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug)
	resp, err := http.Get(url)
	if err != nil {
		t.Fatalf("GET %s: %v", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		t.Errorf("status = %d, want 200", resp.StatusCode)
	}

	var buf strings.Builder
	rawBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("read body: %v", err)
	}
	buf.Write(rawBytes)
	text := buf.String()
	t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)])

	if text == "" {
		t.Error("chapter-text returned empty body")
	}
	// The stripped text should contain our chapter text (markdown heading stripped).
	if !strings.Contains(text, chapterText) {
		t.Errorf("chapter text does not contain expected content %q", chapterText)
	}
}

// ─── helpers ──────────────────────────────────────────────────────────────────

func mapKeys(m map[string]interface{}) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

// cookieJar is a minimal http.CookieJar that stores cookies by host.
type cookieJar struct {
	cookies map[string][]*http.Cookie
}

func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) {
	j.cookies[u.Host] = append(j.cookies[u.Host], cookies...)
}

func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie {
	return j.cookies[u.Host]
}