//go:build integration // Integration tests for the HTTP server against live MinIO + PocketBase backends. // // The server is started on a random port for each test; real HybridStore // backends are used. Browserless-dependent tests are skipped unless // BROWSERLESS_URL is set. // // Run with: // // MINIO_ENDPOINT=localhost:9000 \ // POCKETBASE_URL=http://localhost:8090 \ // go test -v -tags integration -timeout 120s \ // github.com/libnovel/scraper/internal/server package server import ( "bytes" "context" "encoding/json" "fmt" "io" "log/slog" "net" "net/http" neturl "net/url" "os" "strings" "testing" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/storage" ) // ─── fixture helpers ────────────────────────────────────────────────────────── func envOr(key, def string) string { if v := os.Getenv(key); v != "" { return v } return def } // newTestStore creates a HybridStore from env vars, skipping if not configured. func newTestStore(t *testing.T) *storage.HybridStore { t.Helper() if os.Getenv("MINIO_ENDPOINT") == "" { t.Skip("MINIO_ENDPOINT not set — skipping server integration test") } if os.Getenv("POCKETBASE_URL") == "" { t.Skip("POCKETBASE_URL not set — skipping server integration test") } pbCfg := storage.PocketBaseConfig{ BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), } minioCfg := storage.MinioConfig{ Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), UseSSL: envOr("MINIO_USE_SSL", "false") == "true", BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg) if err != nil { t.Fatalf("NewHybridStore: %v", err) } return hs } // startTestServer starts a real Server on a random free port and returns the // base URL. The server is shut down when the test finishes. func startTestServer(t *testing.T, store storage.Store) string { t.Helper() // Find a free port. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("net.Listen: %v", err) } addr := ln.Addr().String() ln.Close() log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) // nopScraper satisfies scraper.NovelScraper without hitting the network. srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella") ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) ready := make(chan struct{}) go func() { // Signal readiness after a short delay to let the listener bind. go func() { time.Sleep(50 * time.Millisecond) close(ready) }() _ = srv.ListenAndServe(ctx) }() <-ready // Wait until the server actually accepts connections. deadline := time.Now().Add(3 * time.Second) for time.Now().Before(deadline) { resp, err := http.Get("http://" + addr + "/health") if err == nil { resp.Body.Close() break } time.Sleep(20 * time.Millisecond) } return "http://" + addr } // nopScraper is a no-op NovelScraper implementation for tests that don't // exercise scraping functionality. type nopScraper struct{} func (nopScraper) SourceName() string { return "nop" } func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) { ch := make(chan scraper.CatalogueEntry) errs := make(chan error) close(ch) close(errs) return ch, errs } func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) { return scraper.BookMeta{}, nil } func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) { return nil, nil } func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) { return scraper.Chapter{Ref: ref}, nil } func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) { ch := make(chan scraper.BookMeta) errs := make(chan error) close(ch) close(errs) return ch, errs } // ─── Tests ──────────────────────────────────────────────────────────────────── // TestServer_Health verifies GET /health returns 200 with status:ok. func TestServer_Health(t *testing.T) { store := newTestStore(t) base := startTestServer(t, store) resp, err := http.Get(base + "/health") if err != nil { t.Fatalf("GET /health: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("status = %d, want 200", resp.StatusCode) } var body map[string]string if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("decode health body: %v", err) } if body["status"] != "ok" { t.Errorf("status field = %q, want %q", body["status"], "ok") } t.Logf("health response: %v", body) } // TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false // when no scrape is running. func TestServer_ScrapeStatus(t *testing.T) { store := newTestStore(t) base := startTestServer(t, store) resp, err := http.Get(base + "/api/scrape/status") if err != nil { t.Fatalf("GET /api/scrape/status: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("status = %d, want 200", resp.StatusCode) } var body map[string]bool if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("decode body: %v", err) } if body["running"] { t.Error("scrape/status.running = true, want false") } t.Logf("scrape status: %v", body) } // TestServer_PresignChapter writes a chapter to MinIO, then calls // GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned. func TestServer_PresignChapter(t *testing.T) { store := newTestStore(t) base := startTestServer(t, store) // Write a chapter directly via the store so we have something to presign. slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000) ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() ch := scraper.Chapter{ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0}, Text: "Content for the server presign integration test.", } if err := store.WriteChapter(ctx, slug, ch); err != nil { t.Fatalf("WriteChapter: %v", err) } t.Logf("stored chapter for slug=%q", slug) // Call the presign endpoint. url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug) resp, err := http.Get(url) if err != nil { t.Fatalf("GET %s: %v", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("status = %d, want 200", resp.StatusCode) } var body map[string]string if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("decode presign response: %v", err) } presignedURL := body["url"] if presignedURL == "" { t.Fatal("presign response has empty url field") } if !strings.HasPrefix(presignedURL, "http") { t.Errorf("presigned URL does not start with http: %q", presignedURL) } t.Logf("presigned URL: %s", presignedURL) } // TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress. func TestServer_Progress(t *testing.T) { store := newTestStore(t) base := startTestServer(t, store) slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000) // Use a persistent http.Client to carry the session cookie. jar := &cookieJar{cookies: make(map[string][]*http.Cookie)} client := &http.Client{Jar: jar} // POST /api/progress/{slug} setURL := fmt.Sprintf("%s/api/progress/%s", base, slug) body, _ := json.Marshal(map[string]int{"chapter": 5}) resp, err := client.Post(setURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST %s: %v", setURL, err) } resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("POST progress status = %d, want 200", resp.StatusCode) } t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode) // GET /api/progress getURL := fmt.Sprintf("%s/api/progress", base) resp2, err := client.Get(getURL) if err != nil { t.Fatalf("GET %s: %v", getURL, err) } defer resp2.Body.Close() if resp2.StatusCode != http.StatusOK { t.Errorf("GET progress status = %d, want 200", resp2.StatusCode) } var progress map[string]interface{} if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil { t.Fatalf("decode progress response: %v", err) } t.Logf("progress: %v", progress) // The slug should appear with chapter value 5. if ch, ok := progress[slug]; !ok { t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress)) } else { // JSON numbers decode as float64. chNum, _ := ch.(float64) if int(chNum) != 5 { t.Errorf("progress[%q] = %v, want 5", slug, ch) } } // DELETE /api/progress/{slug} delURL := fmt.Sprintf("%s/api/progress/%s", base, slug) delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil) delResp, err := client.Do(delReq) if err != nil { t.Fatalf("DELETE %s: %v", delURL, err) } delResp.Body.Close() if delResp.StatusCode != http.StatusOK { t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode) } t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode) } // TestServer_PresignChapter_NotFound verifies that presigning a non-existent // chapter returns 500 (presign fails on missing object). func TestServer_PresignChapter_NotFound(t *testing.T) { store := newTestStore(t) base := startTestServer(t, store) url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base) resp, err := http.Get(url) if err != nil { t.Fatalf("GET %s: %v", url, err) } defer resp.Body.Close() // MinIO presign on a non-existent key returns an error; server returns 500. // (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.) t.Logf("presign non-existent chapter status: %d", resp.StatusCode) } // TestServer_ChapterText writes a chapter and verifies // GET /api/chapter-text/{slug}/{n} returns the stripped plain text. func TestServer_ChapterText(t *testing.T) { store := newTestStore(t) base := startTestServer(t, store) slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000) ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() const chapterText = "The quick brown fox jumps over the lazy dog near the river." ch := scraper.Chapter{ Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0}, Text: chapterText, } if err := store.WriteChapter(ctx, slug, ch); err != nil { t.Fatalf("WriteChapter: %v", err) } url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug) resp, err := http.Get(url) if err != nil { t.Fatalf("GET %s: %v", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("status = %d, want 200", resp.StatusCode) } var buf strings.Builder rawBytes, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("read body: %v", err) } buf.Write(rawBytes) text := buf.String() t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)]) if text == "" { t.Error("chapter-text returned empty body") } // The stripped text should contain our chapter text (markdown heading stripped). if !strings.Contains(text, chapterText) { t.Errorf("chapter text does not contain expected content %q", chapterText) } } // ─── helpers ────────────────────────────────────────────────────────────────── func mapKeys(m map[string]interface{}) []string { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } return keys } func min(a, b int) int { if a < b { return a } return b } // cookieJar is a minimal http.CookieJar that stores cookies by host. type cookieJar struct { cookies map[string][]*http.Cookie } func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) { j.cookies[u.Host] = append(j.cookies[u.Host], cookies...) } func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie { return j.cookies[u.Host] }