test: add integration tests for storage, server, and scrape+store flows
- scraper/internal/storage/integration_test.go: MinioClient and PocketBaseStore integration tests covering chapter/audio round-trips, presign URLs, book metadata, chapter index, ranking, progress, and audio cache CRUD - scraper/internal/storage/hybrid_integration_test.go: HybridStore end-to-end tests for metadata, chapters, ranking, progress, presign, and audio cache - scraper/internal/storage/scrape_integration_test.go: live Browserless + storage tests that scrape book metadata and first 3 chapters, store them, and verify the round-trip via HybridStore - scraper/internal/server/integration_test.go: HTTP server functional tests for health, scrape status, presign chapter, reading progress, and chapter-text endpoints against real MinIO + PocketBase backends - justfile: task runner at repo root with recipes for build, test, lint, UI, docker-compose, and individual service management
This commit is contained in:
Admin
416
scraper/internal/server/integration_test.go
Normal file
416
scraper/internal/server/integration_test.go
Normal file
@@ -0,0 +1,416 @@
|
||||
//go:build integration
|
||||
|
||||
// Integration tests for the HTTP server against live MinIO + PocketBase backends.
|
||||
//
|
||||
// The server is started on a random port for each test; real HybridStore
|
||||
// backends are used. Browserless-dependent tests are skipped unless
|
||||
// BROWSERLESS_URL is set.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// MINIO_ENDPOINT=localhost:9000 \
|
||||
// POCKETBASE_URL=http://localhost:8090 \
|
||||
// go test -v -tags integration -timeout 120s \
|
||||
// github.com/libnovel/scraper/internal/server
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
neturl "net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// ─── fixture helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// newTestStore creates a HybridStore from env vars, skipping if not configured.
|
||||
func newTestStore(t *testing.T) *storage.HybridStore {
|
||||
t.Helper()
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||||
t.Skip("MINIO_ENDPOINT not set — skipping server integration test")
|
||||
}
|
||||
if os.Getenv("POCKETBASE_URL") == "" {
|
||||
t.Skip("POCKETBASE_URL not set — skipping server integration test")
|
||||
}
|
||||
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// startTestServer starts a real Server on a random free port and returns the
|
||||
// base URL. The server is shut down when the test finishes.
|
||||
func startTestServer(t *testing.T, store storage.Store) string {
|
||||
t.Helper()
|
||||
|
||||
// Find a free port.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
|
||||
// nopScraper satisfies scraper.NovelScraper without hitting the network.
|
||||
srv := New(addr, orchestrator.Config{}, nopScraper{}, log, store, "", "af_bella")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
ready := make(chan struct{})
|
||||
go func() {
|
||||
// Signal readiness after a short delay to let the listener bind.
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(ready)
|
||||
}()
|
||||
_ = srv.ListenAndServe(ctx)
|
||||
}()
|
||||
|
||||
<-ready
|
||||
|
||||
// Wait until the server actually accepts connections.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get("http://" + addr + "/health")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
return "http://" + addr
|
||||
}
|
||||
|
||||
// nopScraper is a no-op NovelScraper implementation for tests that don't
|
||||
// exercise scraping functionality.
|
||||
type nopScraper struct{}
|
||||
|
||||
func (nopScraper) SourceName() string { return "nop" }
|
||||
func (nopScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
||||
ch := make(chan scraper.CatalogueEntry)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
func (nopScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
|
||||
return scraper.BookMeta{}, nil
|
||||
}
|
||||
func (nopScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (nopScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
||||
return scraper.Chapter{Ref: ref}, nil
|
||||
}
|
||||
func (nopScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
ch := make(chan scraper.BookMeta)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestServer_Health verifies GET /health returns 200 with status:ok.
|
||||
func TestServer_Health(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
resp, err := http.Get(base + "/health")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /health: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode health body: %v", err)
|
||||
}
|
||||
if body["status"] != "ok" {
|
||||
t.Errorf("status field = %q, want %q", body["status"], "ok")
|
||||
}
|
||||
t.Logf("health response: %v", body)
|
||||
}
|
||||
|
||||
// TestServer_ScrapeStatus verifies GET /api/scrape/status returns running:false
|
||||
// when no scrape is running.
|
||||
func TestServer_ScrapeStatus(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
resp, err := http.Get(base + "/api/scrape/status")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /api/scrape/status: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]bool
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
if body["running"] {
|
||||
t.Error("scrape/status.running = true, want false")
|
||||
}
|
||||
t.Logf("scrape status: %v", body)
|
||||
}
|
||||
|
||||
// TestServer_PresignChapter writes a chapter to MinIO, then calls
|
||||
// GET /api/presign/chapter/{slug}/{n} and verifies a URL is returned.
|
||||
func TestServer_PresignChapter(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
// Write a chapter directly via the store so we have something to presign.
|
||||
slug := fmt.Sprintf("server-presign-test-%d", time.Now().UnixMilli()%100000)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Server Presign Test", Volume: 0},
|
||||
Text: "Content for the server presign integration test.",
|
||||
}
|
||||
if err := store.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
t.Logf("stored chapter for slug=%q", slug)
|
||||
|
||||
// Call the presign endpoint.
|
||||
url := fmt.Sprintf("%s/api/presign/chapter/%s/1", base, slug)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode presign response: %v", err)
|
||||
}
|
||||
presignedURL := body["url"]
|
||||
if presignedURL == "" {
|
||||
t.Fatal("presign response has empty url field")
|
||||
}
|
||||
if !strings.HasPrefix(presignedURL, "http") {
|
||||
t.Errorf("presigned URL does not start with http: %q", presignedURL)
|
||||
}
|
||||
t.Logf("presigned URL: %s", presignedURL)
|
||||
}
|
||||
|
||||
// TestServer_Progress exercises POST /api/progress/{slug} and GET /api/progress.
|
||||
func TestServer_Progress(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
slug := fmt.Sprintf("server-progress-test-%d", time.Now().UnixMilli()%100000)
|
||||
|
||||
// Use a persistent http.Client to carry the session cookie.
|
||||
jar := &cookieJar{cookies: make(map[string][]*http.Cookie)}
|
||||
client := &http.Client{Jar: jar}
|
||||
|
||||
// POST /api/progress/{slug}
|
||||
setURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
|
||||
body, _ := json.Marshal(map[string]int{"chapter": 5})
|
||||
resp, err := client.Post(setURL, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("POST %s: %v", setURL, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("POST progress status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
t.Logf("POST /api/progress/%s → %d", slug, resp.StatusCode)
|
||||
|
||||
// GET /api/progress
|
||||
getURL := fmt.Sprintf("%s/api/progress", base)
|
||||
resp2, err := client.Get(getURL)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", getURL, err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
t.Errorf("GET progress status = %d, want 200", resp2.StatusCode)
|
||||
}
|
||||
|
||||
var progress map[string]interface{}
|
||||
if err := json.NewDecoder(resp2.Body).Decode(&progress); err != nil {
|
||||
t.Fatalf("decode progress response: %v", err)
|
||||
}
|
||||
t.Logf("progress: %v", progress)
|
||||
|
||||
// The slug should appear with chapter value 5.
|
||||
if ch, ok := progress[slug]; !ok {
|
||||
t.Errorf("slug %q not found in progress map; keys: %v", slug, mapKeys(progress))
|
||||
} else {
|
||||
// JSON numbers decode as float64.
|
||||
chNum, _ := ch.(float64)
|
||||
if int(chNum) != 5 {
|
||||
t.Errorf("progress[%q] = %v, want 5", slug, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// DELETE /api/progress/{slug}
|
||||
delURL := fmt.Sprintf("%s/api/progress/%s", base, slug)
|
||||
delReq, _ := http.NewRequest(http.MethodDelete, delURL, nil)
|
||||
delResp, err := client.Do(delReq)
|
||||
if err != nil {
|
||||
t.Fatalf("DELETE %s: %v", delURL, err)
|
||||
}
|
||||
delResp.Body.Close()
|
||||
if delResp.StatusCode != http.StatusOK {
|
||||
t.Errorf("DELETE progress status = %d, want 200", delResp.StatusCode)
|
||||
}
|
||||
t.Logf("DELETE /api/progress/%s → %d", slug, delResp.StatusCode)
|
||||
}
|
||||
|
||||
// TestServer_PresignChapter_NotFound verifies that presigning a non-existent
|
||||
// chapter returns 500 (presign fails on missing object).
|
||||
func TestServer_PresignChapter_NotFound(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
url := fmt.Sprintf("%s/api/presign/chapter/does-not-exist-slug/999", base)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// MinIO presign on a non-existent key returns an error; server returns 500.
|
||||
// (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
|
||||
t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// TestServer_ChapterText writes a chapter and verifies
|
||||
// GET /api/chapter-text/{slug}/{n} returns the stripped plain text.
|
||||
func TestServer_ChapterText(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
base := startTestServer(t, store)
|
||||
|
||||
slug := fmt.Sprintf("server-chtext-test-%d", time.Now().UnixMilli()%100000)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const chapterText = "The quick brown fox jumps over the lazy dog near the river."
|
||||
ch := scraper.Chapter{
|
||||
Ref: scraper.ChapterRef{Number: 1, Title: "Chapter 1: Text Test", Volume: 0},
|
||||
Text: chapterText,
|
||||
}
|
||||
if err := store.WriteChapter(ctx, slug, ch); err != nil {
|
||||
t.Fatalf("WriteChapter: %v", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/api/chapter-text/%s/1", base, slug)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
rawBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read body: %v", err)
|
||||
}
|
||||
buf.Write(rawBytes)
|
||||
text := buf.String()
|
||||
t.Logf("chapter text (%d bytes): %q", len(text), text[:min(len(text), 120)])
|
||||
|
||||
if text == "" {
|
||||
t.Error("chapter-text returned empty body")
|
||||
}
|
||||
// The stripped text should contain our chapter text (markdown heading stripped).
|
||||
if !strings.Contains(text, chapterText) {
|
||||
t.Errorf("chapter text does not contain expected content %q", chapterText)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func mapKeys(m map[string]interface{}) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// cookieJar is a minimal http.CookieJar that stores cookies by host.
|
||||
type cookieJar struct {
|
||||
cookies map[string][]*http.Cookie
|
||||
}
|
||||
|
||||
func (j *cookieJar) SetCookies(u *neturl.URL, cookies []*http.Cookie) {
|
||||
j.cookies[u.Host] = append(j.cookies[u.Host], cookies...)
|
||||
}
|
||||
|
||||
func (j *cookieJar) Cookies(u *neturl.URL) []*http.Cookie {
|
||||
return j.cookies[u.Host]
|
||||
}
|
||||
Reference in New Issue
Block a user